This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b968353f645 HDDS-13447. [S3G] ListObjectsV2 should accept maxKeys=0
(#8833)
b968353f645 is described below
commit b968353f6451f52da09c8a8ced703275f970517e
Author: Jimmy_kiet <[email protected]>
AuthorDate: Mon Aug 4 07:42:41 2025 +0800
HDDS-13447. [S3G] ListObjectsV2 should accept maxKeys=0 (#8833)
Co-authored-by: Wei-Chiu Chuang <[email protected]>
Co-authored-by: peterxcli <[email protected]>
---
.../dist/src/main/smoketest/s3/objectlist.robot | 6 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 88 +++++++++++-----------
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 49 +++++++++---
3 files changed, 88 insertions(+), 55 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot
index 677d7b44d50..1793a4b6081 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot
@@ -41,9 +41,9 @@ List objects with negative max-keys should fail
${result} = Execute AWSS3APICli and checkrc list-objects-v2 --bucket
${BUCKET} --max-keys -1 255
Should Contain ${result} InvalidArgument
-List objects with zero max-keys should fail
- ${result} = Execute AWSS3APICli and checkrc list-objects-v2 --bucket
${BUCKET} --max-keys 0 255
- Should Contain ${result} InvalidArgument
+List objects with zero max-keys should not fail
+ ${result} = Execute AWSS3APICli and checkrc list-objects-v2 --bucket
${BUCKET} --max-keys 0 0
+ Should not Contain ${result} InvalidArgument
List objects with max-keys exceeding config limit should not return more than
limit
Prepare Many Objects In Bucket 1100
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index ed75c68cb26..6ba83605392 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -225,53 +225,55 @@ public Response get(
}
String lastKey = null;
int count = 0;
- while (ozoneKeyIterator != null && ozoneKeyIterator.hasNext()) {
- OzoneKey next = ozoneKeyIterator.next();
- if (bucket != null && bucket.getBucketLayout().isFileSystemOptimized() &&
- StringUtils.isNotEmpty(prefix) &&
- !next.getName().startsWith(prefix)) {
- // prefix has delimiter but key don't have
- // example prefix: dir1/ key: dir123
- continue;
- }
- if (startAfter != null && count == 0 && Objects.equals(startAfter,
next.getName())) {
- continue;
- }
- String relativeKeyName = next.getName().substring(prefix.length());
-
- int depth = StringUtils.countMatches(relativeKeyName, delimiter);
- if (!StringUtils.isEmpty(delimiter)) {
- if (depth > 0) {
- // means key has multiple delimiters in its value.
- // ex: dir/dir1/dir2, where delimiter is "/" and prefix is dir/
- String dirName = relativeKeyName.substring(0, relativeKeyName
- .indexOf(delimiter));
- if (!dirName.equals(prevDir)) {
- response.addPrefix(EncodingTypeObject.createNullable(
- prefix + dirName + delimiter, encodingType));
- prevDir = dirName;
+ if (maxKeys > 0) {
+ while (ozoneKeyIterator != null && ozoneKeyIterator.hasNext()) {
+ OzoneKey next = ozoneKeyIterator.next();
+ if (bucket != null && bucket.getBucketLayout().isFileSystemOptimized()
&&
+ StringUtils.isNotEmpty(prefix) &&
+ !next.getName().startsWith(prefix)) {
+ // prefix has delimiter but key don't have
+ // example prefix: dir1/ key: dir123
+ continue;
+ }
+ if (startAfter != null && count == 0 && Objects.equals(startAfter,
next.getName())) {
+ continue;
+ }
+ String relativeKeyName = next.getName().substring(prefix.length());
+
+ int depth = StringUtils.countMatches(relativeKeyName, delimiter);
+ if (!StringUtils.isEmpty(delimiter)) {
+ if (depth > 0) {
+ // means key has multiple delimiters in its value.
+ // ex: dir/dir1/dir2, where delimiter is "/" and prefix is dir/
+ String dirName = relativeKeyName.substring(0, relativeKeyName
+ .indexOf(delimiter));
+ if (!dirName.equals(prevDir)) {
+ response.addPrefix(EncodingTypeObject.createNullable(
+ prefix + dirName + delimiter, encodingType));
+ prevDir = dirName;
+ count++;
+ }
+ } else if (relativeKeyName.endsWith(delimiter)) {
+ // means or key is same as prefix with delimiter at end and ends
with
+ // delimiter. ex: dir/, where prefix is dir and delimiter is /
+ response.addPrefix(
+ EncodingTypeObject.createNullable(relativeKeyName,
encodingType));
+ count++;
+ } else {
+ // means our key is matched with prefix if prefix is given and it
+ // does not have any common prefix.
+ addKey(response, next);
count++;
}
- } else if (relativeKeyName.endsWith(delimiter)) {
- // means or key is same as prefix with delimiter at end and ends with
- // delimiter. ex: dir/, where prefix is dir and delimiter is /
- response.addPrefix(
- EncodingTypeObject.createNullable(relativeKeyName,
encodingType));
- count++;
} else {
- // means our key is matched with prefix if prefix is given and it
- // does not have any common prefix.
addKey(response, next);
count++;
}
- } else {
- addKey(response, next);
- count++;
- }
- if (count == maxKeys) {
- lastKey = next.getName();
- break;
+ if (count == maxKeys) {
+ lastKey = next.getName();
+ break;
+ }
}
}
@@ -279,7 +281,7 @@ public Response get(
if (count < maxKeys) {
response.setTruncated(false);
- } else if (ozoneKeyIterator.hasNext()) {
+ } else if (ozoneKeyIterator.hasNext() && lastKey != null) {
response.setTruncated(true);
ContinueToken nextToken = new ContinueToken(lastKey, prevDir);
response.setNextToken(nextToken.encodeToString());
@@ -303,8 +305,8 @@ public Response get(
}
private int validateMaxKeys(int maxKeys) throws OS3Exception {
- if (maxKeys <= 0) {
- throw newError(S3ErrorTable.INVALID_ARGUMENT, "maxKeys must be > 0");
+ if (maxKeys < 0) {
+ throw newError(S3ErrorTable.INVALID_ARGUMENT, "maxKeys must be >= 0");
}
return Math.min(maxKeys, maxKeysLimit);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
index d55846c10af..f23f8f81b60 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java
@@ -523,26 +523,57 @@ public void testEncodingTypeException() throws
IOException {
}
@Test
- public void testListObjectsWithInvalidMaxKeys() throws Exception {
- OzoneClient client = createClientWithKeys("file1");
+ public void testListObjectsWithNegativeMaxKeys() throws Exception {
+ OzoneClient client = new OzoneClientStub();
client.getObjectStore().createS3Bucket("bucket");
BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder()
.setClient(client)
.build();
- // maxKeys < 0
+ // maxKeys < 0 should throw InvalidArgument
OS3Exception e1 = assertThrows(OS3Exception.class, () ->
bucketEndpoint.get("bucket", null, null, null, -1, null,
null, null, null, null, null, null, 1000)
);
assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e1.getCode());
+ }
- // maxKeys == 0
- OS3Exception e2 = assertThrows(OS3Exception.class, () ->
- bucketEndpoint.get("bucket", null, null, null, 0, null,
- null, null, null, null, null, null, 1000)
- );
- assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e2.getCode());
+ @Test
+ public void testListObjectsWithZeroMaxKeys() throws Exception {
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket("bucket");
+ BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder()
+ .setClient(client)
+ .build();
+
+ // maxKeys = 0, should return empty list and not throw.
+ ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get(
+ "bucket", null, null, null, 0, null,
+ null, null, null, null, null, null, 1000).getEntity();
+
+ assertEquals(0, response.getContents().size());
+ assertFalse(response.isTruncated());
+ }
+
+ @Test
+ public void testListObjectsWithZeroMaxKeysInNonEmptyBucket() throws
Exception {
+ OzoneClient client = createClientWithKeys("file1", "file2", "file3",
"file4", "file5");
+ BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder()
+ .setClient(client)
+ .build();
+
+ ListObjectResponse response = (ListObjectResponse) bucketEndpoint.get(
+ "b1", null, null, null, 0, null,
+ null, null, null, null, null, null, 1000).getEntity();
+
+ // Should return empty list and not throw.
+ assertEquals(0, response.getContents().size());
+ assertFalse(response.isTruncated());
+
+ ListObjectResponse fullResponse = (ListObjectResponse) bucketEndpoint.get(
+ "b1", null, null, null, 1000, null,
+ null, null, null, null, null, null, 1000).getEntity();
+ assertEquals(5, fullResponse.getContents().size());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
