This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 058c65565c9 HDDS-13706. Limit max-uploads at 1000 (#9060)
058c65565c9 is described below
commit 058c65565c97f5bec72b30fa8e011cf86a5df483
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Sep 24 22:48:48 2025 +0200
HDDS-13706. Limit max-uploads at 1000 (#9060)
---
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 52 +++++++++++++---------
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 1 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 6 ++-
3 files changed, 36 insertions(+), 23 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index 3fe4f6ab5a1..00b48262993 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -97,6 +97,7 @@
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import javax.xml.bind.DatatypeConverter;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
@@ -128,6 +129,8 @@
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
/**
* This is an abstract class to test the AWS Java S3 SDK operations.
@@ -141,6 +144,9 @@
@TestMethodOrder(MethodOrderer.MethodName.class)
public abstract class AbstractS3SDKV1Tests extends OzoneTestBase {
+ // server-side limitation
+ private static final int MAX_UPLOADS_LIMIT = 1000;
+
/**
* There are still some unsupported S3 operations.
* Current unsupported S3 operations (non-exhaustive):
@@ -758,6 +764,7 @@ public void testListMultipartUploads() {
uploadIds.add(uploadId3);
ListMultipartUploadsRequest listMultipartUploadsRequest = new
ListMultipartUploadsRequest(bucketName);
+ listMultipartUploadsRequest.setMaxUploads(5000);
MultipartUploadListing result =
s3Client.listMultipartUploads(listMultipartUploadsRequest);
@@ -768,26 +775,31 @@ public void testListMultipartUploads() {
assertEquals(uploadIds, listUploadIds);
}
- @Test
- public void testListMultipartUploadsPagination() {
- final String bucketName = getBucketName();
+ @ParameterizedTest
+ @ValueSource(ints = {10, 5000})
+ public void testListMultipartUploadsPagination(int requestedMaxUploads) {
+ final String bucketName = getBucketName() + "-" + requestedMaxUploads;
final String multipartKeyPrefix = getKeyName("multipart");
s3Client.createBucket(bucketName);
- // Create 25 multipart uploads to test pagination
+ // Create multipart uploads to test pagination
List<String> allKeys = new ArrayList<>();
Map<String, String> keyToUploadId = new HashMap<>();
- for (int i = 0; i < 25; i++) {
- String key = String.format("%s-%03d", multipartKeyPrefix, i);
+ final int effectiveMaxUploads = Math.min(requestedMaxUploads,
MAX_UPLOADS_LIMIT);
+ final int uploadsCreated = 2 * effectiveMaxUploads + 5;
+ final int expectedPages = uploadsCreated / effectiveMaxUploads + 1;
+
+ for (int i = 0; i < uploadsCreated; i++) {
+ String key = String.format("%s-%04d", multipartKeyPrefix, i);
allKeys.add(key);
String uploadId = initiateMultipartUpload(bucketName, key, null, null,
null);
keyToUploadId.put(key, uploadId);
}
Collections.sort(allKeys);
- // Test pagination with maxUploads=10
+ // Test pagination
Set<String> retrievedKeys = new HashSet<>();
String keyMarker = null;
String uploadIdMarker = null;
@@ -796,18 +808,19 @@ public void testListMultipartUploadsPagination() {
do {
ListMultipartUploadsRequest request = new
ListMultipartUploadsRequest(bucketName)
- .withMaxUploads(10)
+ .withMaxUploads(requestedMaxUploads)
.withKeyMarker(keyMarker)
.withUploadIdMarker(uploadIdMarker);
MultipartUploadListing result = s3Client.listMultipartUploads(request);
+ pageCount++;
// Verify page size
- if (pageCount < 2) {
- assertEquals(10, result.getMultipartUploads().size());
+ if (pageCount < expectedPages) {
+ assertEquals(effectiveMaxUploads, result.getMultipartUploads().size());
assertTrue(result.isTruncated());
} else {
- assertEquals(5, result.getMultipartUploads().size());
+ assertEquals(uploadsCreated % effectiveMaxUploads,
result.getMultipartUploads().size());
assertFalse(result.isTruncated());
}
@@ -822,7 +835,7 @@ public void testListMultipartUploadsPagination() {
assertNull(result.getPrefix());
assertEquals(result.getUploadIdMarker(), uploadIdMarker);
assertEquals(result.getKeyMarker(), keyMarker);
- assertEquals(result.getMaxUploads(), 10);
+ assertEquals(effectiveMaxUploads, result.getMaxUploads());
// Verify next markers content
if (result.isTruncated()) {
@@ -840,20 +853,19 @@ public void testListMultipartUploadsPagination() {
uploadIdMarker = result.getNextUploadIdMarker();
truncated = result.isTruncated();
- pageCount++;
} while (truncated);
// Verify pagination results
- assertEquals(3, pageCount, "Should have exactly 3 pages");
- assertEquals(25, retrievedKeys.size(), "Should retrieve all uploads");
+ assertEquals(expectedPages, pageCount);
+ assertEquals(uploadsCreated, retrievedKeys.size(), "Should retrieve all
uploads");
assertEquals(
allKeys,
retrievedKeys.stream().sorted().collect(Collectors.toList()),
"Retrieved keys should match expected keys in order");
// Test with prefix
- String prefix = multipartKeyPrefix + "-01";
+ String prefix = multipartKeyPrefix + "-001";
ListMultipartUploadsRequest prefixRequest = new
ListMultipartUploadsRequest(bucketName)
.withPrefix(prefix);
@@ -861,11 +873,9 @@ public void testListMultipartUploadsPagination() {
assertEquals(prefix, prefixResult.getPrefix());
assertEquals(
- Arrays.asList(multipartKeyPrefix + "-010", multipartKeyPrefix + "-011",
- multipartKeyPrefix + "-012", multipartKeyPrefix + "-013",
- multipartKeyPrefix + "-014", multipartKeyPrefix + "-015",
- multipartKeyPrefix + "-016", multipartKeyPrefix + "-017",
- multipartKeyPrefix + "-018", multipartKeyPrefix + "-019"),
+ IntStream.rangeClosed(0, 9)
+ .mapToObj(i -> prefix + i)
+ .collect(Collectors.toList()),
prefixResult.getMultipartUploads().stream()
.map(MultipartUpload::getKey)
.collect(Collectors.toList()));
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 79928c3264f..925e2e75df5 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -1228,6 +1228,7 @@ public void testListMultipartUploads() {
ListMultipartUploadsRequest correctRequest =
ListMultipartUploadsRequest.builder()
.bucket(DEFAULT_BUCKET_NAME)
.expectedBucketOwner(correctOwner)
+ .maxUploads(5000)
.build();
verifyPassBucketOwnershipVerification(() ->
s3Client.listMultipartUploads(correctRequest));
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 6ba83605392..066b31fb7d1 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -355,9 +355,11 @@ public Response listMultipartUploads(
int maxUploads)
throws OS3Exception, IOException {
- if (maxUploads < 1 || maxUploads > 1000) {
+ if (maxUploads < 1) {
throw newError(S3ErrorTable.INVALID_ARGUMENT, "max-uploads",
- new Exception("max-uploads must be between 1 and 1000"));
+ new Exception("max-uploads must be positive"));
+ } else {
+ maxUploads = Math.min(maxUploads, 1000);
}
long startNanos = Time.monotonicNowNanos();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
