This is an automated email from the ASF dual-hosted git repository.
ivandika pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d89c6ad8244 HDDS-5195. Support uploading objects with presigned URL
(#9000)
d89c6ad8244 is described below
commit d89c6ad82445e64b413aade98f06d9e8e514b100
Author: Hsu Han Wen <[email protected]>
AuthorDate: Thu Sep 11 11:49:14 2025 +0800
HDDS-5195. Support uploading objects with presigned URL (#9000)
---
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 18 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 236 +++++++++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 494 ++++++++++++++++++++-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 8 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 2 +-
.../ozone/s3/signature/StringToSignProducer.java | 6 +
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 6 +-
.../hadoop/ozone/s3/endpoint/EndpointBuilder.java | 17 +
.../s3/endpoint/TestMultipartUploadWithCopy.java | 4 +
9 files changed, 775 insertions(+), 16 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
index 33b7788e696..bb5ba27a28e 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
@@ -22,6 +22,8 @@
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.security.MessageDigest;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.lang3.RandomUtils;
import org.apache.ozone.test.InputSubstream;
@@ -30,6 +32,8 @@
*/
public final class S3SDKTestUtils {
+ public static final Pattern UPLOAD_ID_PATTERN =
Pattern.compile("<UploadId>(.+?)</UploadId>");
+
private S3SDKTestUtils() {
}
@@ -76,4 +80,18 @@ public static void createFile(File newFile, int size) throws
IOException {
file.getFD().sync();
file.close();
}
+
+ /**
+ * Extract the UploadId from XML string.
+ *
+ * @param xml The XML string.
+ * @return The UploadId, or null if not found.
+ */
+ public static String extractUploadId(String xml) {
+ Matcher matcher = UPLOAD_ID_PATTERN.matcher(xml);
+ if (matcher.find()) {
+ return matcher.group(1);
+ }
+ return null;
+ }
}
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index 133bc031ab3..eed72db3c13 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.MB;
import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest;
import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -71,14 +72,17 @@
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.services.s3.transfer.model.UploadResult;
-import com.amazonaws.util.IOUtils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -95,6 +99,7 @@
import java.util.Set;
import java.util.stream.Collectors;
import javax.xml.bind.DatatypeConverter;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.hdds.client.ReplicationConfig;
@@ -112,7 +117,9 @@
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.s3.MultiS3GatewayService;
import org.apache.hadoop.ozone.s3.S3ClientFactory;
+import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils;
import org.apache.hadoop.ozone.s3.endpoint.S3Owner;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ozone.test.OzoneTestBase;
import org.junit.jupiter.api.MethodOrderer;
@@ -1144,6 +1151,233 @@ public void testPresignedUrlHead() throws IOException {
}
}
+ @Test
+ public void testPresignedUrlPutObject() throws Exception {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String expectedContent = "bar";
+ s3Client.createBucket(bucketName);
+
+ // Set the presigned URL to expire after one hour.
+ Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60));
+
+ // Test PutObjectRequest presigned URL
+ GeneratePresignedUrlRequest generatePresignedUrlRequest =
+ new GeneratePresignedUrlRequest(bucketName, keyName)
+ .withMethod(HttpMethod.PUT)
+ .withExpiration(expiration);
+ URL presignedUrl =
s3Client.generatePresignedUrl(generatePresignedUrlRequest);
+
+ HttpURLConnection connection = null;
+ try {
+ connection = (HttpURLConnection) presignedUrl.openConnection();
+ connection.setRequestMethod("PUT");
+ connection.setDoOutput(true);
+ try (OutputStream os = connection.getOutputStream()) {
+ os.write(expectedContent.getBytes(StandardCharsets.UTF_8));
+ os.flush();
+ }
+
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode, "PutObject presigned URL should return
200 OK");
+ String actualContent;
+ S3Object s3Object = s3Client.getObject(bucketName, keyName);
+ try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) {
+ actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+ }
+ assertEquals(expectedContent, actualContent, "Downloaded content should
match uploaded content");
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+ }
+
+ @Test
+ public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws
Exception {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final Map<String, String> userMetadata = new HashMap<>();
+ userMetadata.put("key1", "value1");
+ userMetadata.put("key2", "value2");
+ final Map<String, String> tags = new HashMap<>();
+ tags.put("tag1", "value1");
+ tags.put("tag2", "value2");
+
+ s3Client.createBucket(bucketName);
+
+ File multipartUploadFile =
Files.createFile(tempDir.resolve("multipartupload.txt")).toFile();
+ createFile(multipartUploadFile, (int) (10 * MB));
+
+ // create MPU using presigned URL
+ GeneratePresignedUrlRequest initMPUPresignedUrlRequest =
+ createInitMPUPresignedUrlRequest(bucketName, keyName, userMetadata,
tags);
+
+
+ String uploadId = initMultipartUpload(initMPUPresignedUrlRequest);
+
+ // upload parts using presigned URL
+ List<PartETag> completedParts = uploadParts(multipartUploadFile,
bucketName, keyName, uploadId);
+
+ // Complete multipart upload using presigned URL
+ completeMPU(bucketName, keyName, uploadId, completedParts);
+
+ // Verify upload result
+ ObjectMetadata objectMeta = s3Client.getObjectMetadata(bucketName,
keyName);
+ assertEquals(userMetadata, objectMeta.getUserMetadata());
+
+ // Verify content
+ S3Object s3Object = s3Client.getObject(bucketName, keyName);
+ assertEquals(tags.size(), s3Object.getTaggingCount());
+ String actualContent;
+ try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) {
+ actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+ }
+ String expectedContent = new
String(Files.readAllBytes(multipartUploadFile.toPath()),
StandardCharsets.UTF_8);
+ assertEquals(expectedContent, actualContent, "Downloaded content should
match uploaded content");
+ }
+
+ private static void completeMPU(String bucketName, String keyName, String
uploadId, List<PartETag> completedParts)
+ throws IOException {
+ GeneratePresignedUrlRequest request = new
GeneratePresignedUrlRequest(bucketName, keyName)
+ .withMethod(HttpMethod.POST)
+ .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 * 60));
+ request.addRequestParameter("uploadId", uploadId);
+
+ HttpURLConnection httpConnection = null;
+ try {
+ httpConnection = (HttpURLConnection)
s3Client.generatePresignedUrl(request).openConnection();
+ httpConnection.setRequestMethod("POST");
+ httpConnection.setDoOutput(true);
+
+ // Generate completion XML payload
+ StringBuilder completionXml = new StringBuilder();
+ completionXml.append("<CompleteMultipartUpload>\n");
+ for (PartETag part : completedParts) {
+ completionXml.append(" <Part>\n");
+ completionXml.append("
<PartNumber>").append(part.getPartNumber()).append("</PartNumber>\n");
+ completionXml.append("
<ETag>").append(part.getETag()).append("</ETag>\n");
+ completionXml.append(" </Part>\n");
+ }
+ completionXml.append("</CompleteMultipartUpload>");
+
+ byte[] completionPayloadBytes =
completionXml.toString().getBytes(StandardCharsets.UTF_8);
+ try (OutputStream os = httpConnection.getOutputStream()) {
+ IOUtils.write(completionPayloadBytes, os);
+ }
+
+ int responseCode = httpConnection.getResponseCode();
+ assertEquals(200, responseCode, "Complete multipart upload should return
200 OK");
+ } finally {
+ if (httpConnection != null) {
+ httpConnection.disconnect();
+ }
+ }
+ }
+
+ private static List<PartETag> uploadParts(File multipartUploadFile, String
bucketName, String keyName,
+ String uploadId) throws
IOException {
+ List<PartETag> completedParts = new ArrayList<>();
+ ByteBuffer byteBuffer = ByteBuffer.allocate((int) (5 * MB));
+ long filePosition = 0;
+ long fileLength = multipartUploadFile.length();
+ int partNumber = 1;
+
+ try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
+ while (filePosition < fileLength) {
+ file.seek(filePosition);
+ long bytesRead = file.getChannel().read(byteBuffer);
+ byteBuffer.flip();
+
+ // generate presigned URL for each part
+ GeneratePresignedUrlRequest request = new
GeneratePresignedUrlRequest(bucketName, keyName)
+ .withMethod(HttpMethod.PUT)
+ .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 *
60));
+ request.addRequestParameter("partNumber", String.valueOf(partNumber));
+ request.addRequestParameter("uploadId", uploadId);
+
+ URL presignedUrl = s3Client.generatePresignedUrl(request);
+
+ // upload each part using presigned URL
+ HttpURLConnection connection = null;
+ try {
+ connection = (HttpURLConnection) presignedUrl.openConnection();
+ connection.setDoOutput(true);
+ connection.setRequestMethod("PUT");
+ connection.setRequestProperty("Content-Length",
String.valueOf(byteBuffer.remaining()));
+
+ try (OutputStream os = connection.getOutputStream()) {
+ os.write(byteBuffer.array(), 0, byteBuffer.remaining());
+ os.flush();
+ }
+
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode, String.format("Upload part %d should
return 200 OK", partNumber));
+
+ String etag = connection.getHeaderField("ETag");
+ PartETag partETag = new PartETag(partNumber, etag);
+ completedParts.add(partETag);
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+
+ byteBuffer.clear();
+ filePosition += bytesRead;
+ partNumber++;
+ }
+ }
+ return completedParts;
+ }
+
+ private String initMultipartUpload(GeneratePresignedUrlRequest request)
throws IOException {
+ URL presignedUrl = s3Client.generatePresignedUrl(request);
+ String uploadId;
+ HttpURLConnection httpConnection = null;
+ try {
+ httpConnection = (HttpURLConnection) presignedUrl.openConnection();
+ httpConnection.setRequestMethod("POST");
+ httpConnection.setDoOutput(true);
+
request.getCustomRequestHeaders().forEach(httpConnection::setRequestProperty);
+
+ httpConnection.connect();
+ int initMPUConnectionResponseCode = httpConnection.getResponseCode();
+ assertEquals(200, initMPUConnectionResponseCode);
+ try (InputStream is = httpConnection.getInputStream()) {
+ String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
+ uploadId = S3SDKTestUtils.extractUploadId(responseXml);
+ }
+ } finally {
+ if (httpConnection != null) {
+ httpConnection.disconnect();
+ }
+ }
+ return uploadId;
+ }
+
+ private GeneratePresignedUrlRequest createInitMPUPresignedUrlRequest(String
bucketName, String keyName,
+
Map<String, String> userMetadata,
+
Map<String, String> tags) throws Exception {
+ GeneratePresignedUrlRequest initMPUPresignUrlRequest = new
GeneratePresignedUrlRequest(bucketName, keyName)
+ .withMethod(HttpMethod.POST)
+ .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 * 60));
+
+ userMetadata.forEach((k, v) -> {
+
initMPUPresignUrlRequest.putCustomRequestHeader(CUSTOM_METADATA_HEADER_PREFIX +
k, v);
+ });
+
+ StringBuilder tagValueBuilder = new StringBuilder();
+ for (Map.Entry<String, String> entry : tags.entrySet()) {
+ if (tagValueBuilder.length() > 0) {
+ tagValueBuilder.append('&');
+ }
+
tagValueBuilder.append(entry.getKey()).append('=').append(URLEncoder.encode(entry.getValue(),
"UTF-8"));
+ }
+ initMPUPresignUrlRequest.putCustomRequestHeader(S3Consts.TAG_HEADER,
tagValueBuilder.toString());
+ return initMPUPresignUrlRequest;
+ }
+
/**
* Tests the functionality to create a snapshot of an Ozone bucket and then
read files
* from the snapshot directory using the S3 SDK.
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index c4d6ce761a2..ff54190d3f3 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -28,10 +28,12 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static software.amazon.awssdk.core.sync.RequestBody.fromString;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -49,6 +51,7 @@
import java.util.Map;
import java.util.stream.Collectors;
import javax.xml.bind.DatatypeConverter;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationFactor;
@@ -65,6 +68,7 @@
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.s3.MultiS3GatewayService;
import org.apache.hadoop.ozone.s3.S3ClientFactory;
+import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils;
import org.apache.hadoop.ozone.s3.endpoint.S3Owner;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ozone.test.OzoneTestBase;
@@ -77,6 +81,7 @@
import org.junit.jupiter.api.function.Executable;
import org.junit.jupiter.api.io.TempDir;
import software.amazon.awssdk.core.ResponseBytes;
+import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.http.HttpExecuteRequest;
import software.amazon.awssdk.http.HttpExecuteResponse;
@@ -127,12 +132,20 @@
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import
software.amazon.awssdk.services.s3.presigner.model.CompleteMultipartUploadPresignRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.CreateMultipartUploadPresignRequest;
import
software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import
software.amazon.awssdk.services.s3.presigner.model.HeadBucketPresignRequest;
import
software.amazon.awssdk.services.s3.presigner.model.HeadObjectPresignRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.PresignedCompleteMultipartUploadRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.PresignedCreateMultipartUploadRequest;
import
software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
import
software.amazon.awssdk.services.s3.presigner.model.PresignedHeadBucketRequest;
import
software.amazon.awssdk.services.s3.presigner.model.PresignedHeadObjectRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
+import
software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest;
import software.amazon.awssdk.transfer.s3.model.FileDownload;
@@ -451,12 +464,7 @@ public void testPresignedUrlGet() throws Exception {
.key(keyName),
RequestBody.fromString(content));
- try (S3Presigner presigner = S3Presigner.builder()
- // TODO: Find a way to retrieve the path style configuration from
S3Client instead
-
.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
-
.endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get())
- .region(s3Client.serviceClientConfiguration().region())
-
.credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build())
{
+ try (S3Presigner presigner = createS3Presigner()) {
GetObjectRequest objectRequest = GetObjectRequest.builder()
.bucket(bucketName)
.key(keyName)
@@ -523,12 +531,7 @@ public void testPresignedUrlHead() throws Exception {
.key(keyName),
RequestBody.fromString(content));
- try (S3Presigner presigner = S3Presigner.builder()
- // TODO: Find a way to retrieve the path style configuration from
S3Client instead
-
.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
-
.endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get())
- .region(s3Client.serviceClientConfiguration().region())
-
.credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build())
{
+ try (S3Presigner presigner = createS3Presigner()) {
HeadObjectRequest objectRequest = HeadObjectRequest.builder()
.bucket(bucketName)
@@ -616,6 +619,473 @@ public void testPresignedUrlHead() throws Exception {
}
}
+ @Test
+ public void testPresignedUrlPut() throws Exception {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ try (S3Presigner presigner = createS3Presigner()) {
+
+ PutObjectRequest objectRequest = PutObjectRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .build();
+
+ PutObjectPresignRequest presignRequest =
PutObjectPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(10))
+ .putObjectRequest(objectRequest)
+ .build();
+
+ PresignedPutObjectRequest presignedRequest =
presigner.presignPutObject(presignRequest);
+
+ // use http url connection
+ HttpURLConnection connection = null;
+ String expectedContent;
+ String actualContent;
+ try {
+ expectedContent = "This is a test content for presigned PUT URL.";
+ connection = (HttpURLConnection)
presignedRequest.url().openConnection();
+ connection.setRequestMethod("PUT");
+ connection.setDoOutput(true);
+ try (OutputStream os = connection.getOutputStream()) {
+ os.write(expectedContent.getBytes(StandardCharsets.UTF_8));
+ os.flush();
+ }
+
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode, "PutObject presigned URL should return
200 OK");
+
+ //verify the object was uploaded
+ ResponseInputStream<GetObjectResponse> object1 = s3Client.getObject(b1
-> b1.bucket(bucketName).key(keyName));
+ actualContent = IoUtils.toUtf8String(object1);
+ assertEquals(expectedContent, actualContent);
+
+ // Use the AWS SDK for Java SdkHttpClient class to test the PUT request
+ expectedContent = "This content is for testing the SdkHttpClient PUT
request.";
+ SdkHttpRequest request = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.PUT)
+ .uri(presignedRequest.url().toURI())
+ .build();
+
+ byte[] bytes = expectedContent.getBytes(StandardCharsets.UTF_8);
+ HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
+ .request(request)
+ .contentStreamProvider(() -> new ByteArrayInputStream(bytes))
+ .build();
+
+ try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) {
+ HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
+ assertEquals(200, response.httpResponse().statusCode(),
+ "PutObject presigned URL should return 200 OK via
SdkHttpClient");
+ }
+
+ //verify the object was uploaded
+ ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b
-> b.bucket(bucketName).key(keyName));
+ actualContent = IoUtils.toUtf8String(object);
+ assertEquals(expectedContent, actualContent);
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws
Exception {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final Map<String, String> userMetadata = new HashMap<>();
+ userMetadata.put("key1", "value1");
+ userMetadata.put("key2", "value2");
+
+ List<Tag> tags = Arrays.asList(
+ Tag.builder().key("tag1").value("value1").build(),
+ Tag.builder().key("tag2").value("value2").build()
+ );
+
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ File multipartUploadFile =
Files.createFile(tempDir.resolve("multipartupload.txt")).toFile();
+ createFile(multipartUploadFile, (int) (10 * MB));
+
+ try (S3Presigner presigner = createS3Presigner()) {
+
+ // generate create MPU presigned URL
+ CreateMultipartUploadRequest createRequest =
CreateMultipartUploadRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .metadata(userMetadata)
+ .tagging(Tagging.builder().tagSet(tags).build())
+ .build();
+
+ CreateMultipartUploadPresignRequest createMPUPresignRequest =
CreateMultipartUploadPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(10))
+ .createMultipartUploadRequest(createRequest)
+ .build();
+
+ PresignedCreateMultipartUploadRequest presignCreateMultipartUpload =
+ presigner.presignCreateMultipartUpload(createMPUPresignRequest);
+
+ mpuWithHttpURLConnection(presignCreateMultipartUpload,
multipartUploadFile, bucketName, keyName, presigner,
+ userMetadata, tags);
+ mpuWithSdkHttpClient(presignCreateMultipartUpload, multipartUploadFile,
bucketName, keyName, presigner,
+ userMetadata, tags);
+ }
+ }
+
+ private void mpuWithHttpURLConnection(PresignedCreateMultipartUploadRequest
presignCreateMultipartUpload,
+ File multipartUploadFile, String
bucketName, String keyName,
+ S3Presigner presigner, Map<String,
String> userMetadata,
+ List<Tag> tags) throws IOException {
+ // create MPU using presigned URL
+ String uploadId;
+ HttpURLConnection createMultiPartUploadConnection = null;
+ try {
+ createMultiPartUploadConnection =
openHttpUrlConnection(presignCreateMultipartUpload);
+ int createMultiPartUploadConnectionResponseCode =
createMultiPartUploadConnection.getResponseCode();
+ assertEquals(200, createMultiPartUploadConnectionResponseCode,
+ "CreateMultipartUploadPresignRequest should return 200 OK");
+
+ try (InputStream is = createMultiPartUploadConnection.getInputStream()) {
+ String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
+ uploadId = S3SDKTestUtils.extractUploadId(responseXml);
+ }
+ } finally {
+ if (createMultiPartUploadConnection != null) {
+ createMultiPartUploadConnection.disconnect();
+ }
+ }
+
+ // Upload parts using presigned URL
+ List<CompletedPart> completedParts =
+ uploadPartWithHttpURLConnection(multipartUploadFile, bucketName,
keyName, presigner, uploadId);
+
+ // complete MPU using presigned URL
+ CompleteMultipartUploadRequest completeRequest =
CompleteMultipartUploadRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .uploadId(uploadId)
+
.multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
+ .build();
+ CompleteMultipartUploadPresignRequest
completeMultipartUploadPresignRequest =
+ CompleteMultipartUploadPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(10))
+ .completeMultipartUploadRequest(completeRequest)
+ .build();
+
+ PresignedCompleteMultipartUploadRequest
presignedCompleteMultipartUploadRequest =
+
presigner.presignCompleteMultipartUpload(completeMultipartUploadPresignRequest);
+
+ completeMPUWithHttpUrlConnection(presignedCompleteMultipartUploadRequest,
completedParts);
+
+ // verify upload result
+ HeadObjectResponse headObjectResponse = s3Client.headObject(b ->
b.bucket(bucketName).key(keyName));
+ assertTrue(headObjectResponse.hasMetadata());
+ assertEquals(userMetadata, headObjectResponse.metadata());
+
+ ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b ->
b.bucket(bucketName).key(keyName));
+ assertEquals(tags.size(), object.response().tagCount());
+ String actualContent = IoUtils.toUtf8String(object);
+ String originalContent = new
String(Files.readAllBytes(multipartUploadFile.toPath()),
StandardCharsets.UTF_8);
+ assertEquals(originalContent, actualContent, "Uploaded file content should
match original file content");
+ }
+
+ private List<CompletedPart> uploadPartWithHttpURLConnection(File
multipartUploadFile, String bucketName,
+ String keyName,
S3Presigner presigner, String uploadId)
+ throws IOException {
+ List<CompletedPart> completedParts = new ArrayList<>();
+ int partNumber = 1;
+ ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB));
+
+ try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
+ long fileSize = file.length();
+ long position = 0;
+
+ while (position < fileSize) {
+ file.seek(position);
+ long read = file.getChannel().read(bb);
+
+ bb.flip();
+
+ // First create an UploadPartRequest
+ UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .uploadId(uploadId)
+ .partNumber(partNumber)
+ .contentLength((long) bb.remaining())
+ .build();
+
+ // Generate presigned URL for each part
+ UploadPartPresignRequest presignRequest =
UploadPartPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(10))
+ .uploadPartRequest(uploadPartRequest)
+ .build();
+
+ PresignedUploadPartRequest presignedRequest =
presigner.presignUploadPart(presignRequest);
+
+ // use presigned URL to upload the part
+ HttpURLConnection connection = null;
+ try {
+ connection = (HttpURLConnection)
presignedRequest.url().openConnection();
+ connection.setDoOutput(true);
+ connection.setRequestMethod("PUT");
+
+ try (OutputStream os = connection.getOutputStream()) {
+ os.write(bb.array(), 0, bb.remaining());
+ os.flush();
+ }
+
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode,
+ String.format("Upload part %d should return 200 OK",
partNumber));
+
+ String etag = connection.getHeaderField("ETag");
+ CompletedPart part = CompletedPart.builder()
+ .partNumber(partNumber)
+ .eTag(etag)
+ .build();
+ completedParts.add(part);
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
+
+ bb.clear();
+ position += read;
+ partNumber++;
+ }
+ }
+ return completedParts;
+ }
+
+ private HttpURLConnection
openHttpUrlConnection(PresignedCreateMultipartUploadRequest request) throws
IOException {
+ HttpURLConnection connection = (HttpURLConnection)
request.url().openConnection();
+ connection.setDoOutput(true);
+ connection.setRequestMethod("POST");
+ for (Map.Entry<String, List<String>> entry :
request.signedHeaders().entrySet()) {
+ String key = entry.getKey();
+ for (String value : entry.getValue()) {
+ connection.setRequestProperty(key, value);
+ }
+ }
+ return connection;
+ }
+
+ private void
completeMPUWithHttpUrlConnection(PresignedCompleteMultipartUploadRequest
request,
+ List<CompletedPart>
completedParts) throws IOException {
+ HttpURLConnection completeMultipartUploadConnection = null;
+ try {
+ completeMultipartUploadConnection = (HttpURLConnection)
request.url().openConnection();
+ completeMultipartUploadConnection.setDoOutput(true);
+ completeMultipartUploadConnection.setRequestMethod("POST");
+
+ // copy headers
+ for (Map.Entry<String, List<String>> entry :
request.signedHeaders().entrySet()) {
+ String key = entry.getKey();
+ for (String value : entry.getValue()) {
+ completeMultipartUploadConnection.setRequestProperty(key, value);
+ }
+ }
+
+ String xmlPayload = buildCompleteMultipartUploadXml(completedParts);
+ byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8);
+ try (OutputStream os =
completeMultipartUploadConnection.getOutputStream()) {
+ IOUtils.write(payloadBytes, os);
+ os.flush();
+ }
+
+ int completeMultipartUploadConnectionResponseCode =
completeMultipartUploadConnection.getResponseCode();
+ assertEquals(200, completeMultipartUploadConnectionResponseCode,
+ "CompleteMultipartUploadPresignRequest should return 200 OK");
+ } finally {
+ if (completeMultipartUploadConnection != null) {
+ completeMultipartUploadConnection.disconnect();
+ }
+ }
+ }
+
+ private void mpuWithSdkHttpClient(PresignedCreateMultipartUploadRequest
presignCreateMultipartUpload,
+ File multipartUploadFile, String
bucketName, String keyName,
+ S3Presigner presigner, Map<String, String>
userMetadata, List<Tag> tags)
+ throws Exception {
+ // create MPU using presigned URL
+ String uploadId;
+ try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) {
+ uploadId = createMPUWithSdkHttpClient(presignCreateMultipartUpload,
sdkHttpClient);
+
+ // Upload parts using presigned URL
+ List<CompletedPart> completedParts =
+ uploadPartWithSdkHttpClient(multipartUploadFile, bucketName,
keyName, presigner, uploadId, sdkHttpClient);
+
+ // complete MPU using presigned URL
+ completeMPUWithSdkHttpClient(bucketName, keyName, presigner, uploadId,
completedParts, sdkHttpClient);
+
+ // verify upload result
+ HeadObjectResponse headObjectResponse = s3Client.headObject(b ->
b.bucket(bucketName).key(keyName));
+ assertTrue(headObjectResponse.hasMetadata());
+ assertEquals(userMetadata, headObjectResponse.metadata());
+
+ ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b ->
b.bucket(bucketName).key(keyName));
+ assertEquals(tags.size(), object.response().tagCount());
+ String actualContent = IoUtils.toUtf8String(object);
+ String originalContent = new
String(Files.readAllBytes(multipartUploadFile.toPath()),
StandardCharsets.UTF_8);
+ assertEquals(originalContent, actualContent, "Uploaded file content
should match original file content");
+ }
+ }
+
+ private void completeMPUWithSdkHttpClient(String bucketName, String keyName,
S3Presigner presigner, String uploadId,
+ List<CompletedPart> completedParts, SdkHttpClient
sdkHttpClient) throws Exception {
+ CompleteMultipartUploadRequest completeRequest =
CompleteMultipartUploadRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .uploadId(uploadId)
+
.multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
+ .build();
+
+ CompleteMultipartUploadPresignRequest
completeMultipartUploadPresignRequest =
+ CompleteMultipartUploadPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(10))
+ .completeMultipartUploadRequest(completeRequest)
+ .build();
+
+ PresignedCompleteMultipartUploadRequest
presignedCompleteMultipartUploadRequest =
+
presigner.presignCompleteMultipartUpload(completeMultipartUploadPresignRequest);
+
+ String xmlPayload = buildCompleteMultipartUploadXml(completedParts);
+ byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8);
+
+ SdkHttpRequest completeMultipartUploadRequest = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.POST)
+ .uri(presignedCompleteMultipartUploadRequest.url().toURI())
+ .build();
+
+ HttpExecuteRequest completeMultipartUploadExecuteRequest =
HttpExecuteRequest.builder()
+ .request(completeMultipartUploadRequest)
+ .contentStreamProvider(() -> new ByteArrayInputStream(payloadBytes))
+ .build();
+
+ HttpExecuteResponse completeMultipartUploadResponse =
+
sdkHttpClient.prepareRequest(completeMultipartUploadExecuteRequest).call();
+ assertEquals(200,
completeMultipartUploadResponse.httpResponse().statusCode(),
+ "CompleteMultipartUploadPresignRequest should return 200 OK");
+ }
+
+ private List<CompletedPart> uploadPartWithSdkHttpClient(File
multipartUploadFile, String bucketName, String keyName,
+ S3Presigner presigner,
String uploadId,
+ SdkHttpClient
httpClient) throws Exception {
+ List<CompletedPart> completedParts = new ArrayList<>();
+ int partNumber = 1;
+ ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB));
+
+ try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
+ long fileSize = file.length();
+ long position = 0;
+
+ while (position < fileSize) {
+ file.seek(position);
+ long read = file.getChannel().read(bb);
+
+ bb.flip();
+
+ // Generate presigned URL for each part
+ UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
+ .bucket(bucketName)
+ .key(keyName)
+ .uploadId(uploadId)
+ .partNumber(partNumber)
+ .contentLength((long) bb.remaining())
+ .build();
+
+ UploadPartPresignRequest presignRequest =
UploadPartPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(10))
+ .uploadPartRequest(uploadPartRequest)
+ .build();
+
+ PresignedUploadPartRequest presignedRequest =
presigner.presignUploadPart(presignRequest);
+
+ // upload each part using presigned URL
+ SdkHttpRequest uploadPartSdkRequest = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.PUT)
+ .uri(presignedRequest.url().toURI())
+ .build();
+
+ byte[] bytes = new byte[bb.remaining()];
+ bb.get(bytes);
+
+ HttpExecuteRequest uploadPartExecuteRequest =
HttpExecuteRequest.builder()
+ .request(uploadPartSdkRequest)
+ .contentStreamProvider(() -> new ByteArrayInputStream(bytes))
+ .build();
+
+ HttpExecuteResponse uploadPartResponse =
httpClient.prepareRequest(uploadPartExecuteRequest).call();
+
+ String etag = uploadPartResponse.httpResponse()
+ .firstMatchingHeader("ETag")
+ .orElseThrow(() -> new RuntimeException("ETag missing in
response"));
+
+ CompletedPart part = CompletedPart.builder()
+ .partNumber(partNumber)
+ .eTag(etag)
+ .build();
+ completedParts.add(part);
+
+ bb.clear();
+ position += read;
+ partNumber++;
+ }
+ }
+ return completedParts;
+ }
+
+ private String
createMPUWithSdkHttpClient(PresignedCreateMultipartUploadRequest request,
+ SdkHttpClient httpClient) throws Exception {
+ String uploadId;
+ SdkHttpRequest createMultipartUploadRequest = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.POST)
+ .uri(request.url().toURI())
+ .headers(request.signedHeaders())
+ .build();
+
+ HttpExecuteRequest createMultipartUploadExecuteRequest =
HttpExecuteRequest.builder()
+ .request(createMultipartUploadRequest)
+ .build();
+
+ HttpExecuteResponse createMultipartUploadResponse =
+ httpClient.prepareRequest(createMultipartUploadExecuteRequest).call();
+
+ try (InputStream is = createMultipartUploadResponse.responseBody().get()) {
+ String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
+ uploadId = S3SDKTestUtils.extractUploadId(responseXml);
+ }
+ return uploadId;
+ }
+
+ private String buildCompleteMultipartUploadXml(List<CompletedPart> parts) {
+ StringBuilder xml = new StringBuilder();
+ xml.append("<CompleteMultipartUpload>\n");
+ for (CompletedPart part : parts) {
+ xml.append(" <Part>\n");
+ xml.append("
<PartNumber>").append(part.partNumber()).append("</PartNumber>\n");
+ xml.append(" <ETag>").append(part.eTag()).append("</ETag>\n");
+ xml.append(" </Part>\n");
+ }
+ xml.append("</CompleteMultipartUpload>");
+ return xml.toString();
+ }
+
+ private S3Presigner createS3Presigner() {
+ return S3Presigner.builder()
+ // TODO: Find a way to retrieve the path style configuration from
S3Client instead
+
.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
+
.endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get())
+ .region(s3Client.serviceClientConfiguration().region())
+
.credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build();
+ }
+
/**
* Tests the functionality to create a snapshot of an Ozone bucket and then
read files
* from the snapshot directory using the S3 SDK.
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 0f598648bea..d15cf5c427f 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -89,8 +89,9 @@ public abstract class EndpointBase implements Auditor {
@Inject
private OzoneClient client;
+ @SuppressWarnings("checkstyle:VisibilityModifier")
@Inject
- private SignatureInfo signatureInfo;
+ protected SignatureInfo signatureInfo;
@Inject
private RequestIdentifier requestIdentifier;
@@ -501,6 +502,11 @@ public void setRequestIdentifier(RequestIdentifier
requestIdentifier) {
this.requestIdentifier = requestIdentifier;
}
+ @VisibleForTesting
+ public void setSignatureInfo(SignatureInfo signatureInfo) {
+ this.signatureInfo = signatureInfo;
+ }
+
public OzoneClient getClient() {
return client;
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index a76d6456eb6..7b8f8e99b49 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -1556,7 +1556,7 @@ private int getIOBufferSize(long fileLength) {
*/
private S3ChunkInputStreamInfo getS3ChunkInputStreamInfo(
InputStream body, long contentLength, String amzDecodedLength, String
keyPath) throws OS3Exception {
- final String amzContentSha256Header = validateSignatureHeader(headers,
keyPath);
+ final String amzContentSha256Header = validateSignatureHeader(headers,
keyPath, signatureInfo.isSignPayload());
final InputStream chunkInputStream;
final long effectiveLength;
if (hasMultiChunksPayload(amzContentSha256Header)) {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
index ea460f62d49..e2f8d64a4d1 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java
@@ -368,6 +368,12 @@ private static void validateCanonicalHeaders(
.filter(s -> s.startsWith("x-amz-"))
.collect(Collectors.toSet())) {
if (!(canonicalHeaders.contains(header + ":"))) {
+ // According to AWS Signature V4 documentation using Authorization
Header
+ //
https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+ // The x-amz-content-sha256 header is not required for CanonicalHeaders
+ if (X_AMZ_CONTENT_SHA256.equals(header)) {
+ continue;
+ }
LOG.error("The SignedHeaders list must include all "
+ "x-amz-* headers in the request");
throw S3_AUTHINFO_CREATION_ERROR;
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
index d9461bc2fb7..2b698c50272 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
@@ -164,9 +164,13 @@ public static void validateMultiChunksUpload(HttpHeaders
headers, String amzDeco
}
}
- public static String validateSignatureHeader(HttpHeaders headers, String
resource) throws OS3Exception {
+ public static String validateSignatureHeader(HttpHeaders headers, String
resource, boolean isSignedPayload)
+ throws OS3Exception {
String xAmzContentSha256Header =
headers.getHeaderString(X_AMZ_CONTENT_SHA256);
if (xAmzContentSha256Header == null) {
+ if (!isSignedPayload) {
+ return UNSIGNED_PAYLOAD;
+ }
OS3Exception ex = S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT,
resource);
ex.setErrorMessage("An error occurred (InvalidArgument): " +
"The " + X_AMZ_CONTENT_SHA256 + " header is not specified");
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java
index 8ae4f941ac9..6f430df7892 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java
@@ -17,12 +17,16 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.util.function.Supplier;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.core.HttpHeaders;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.s3.RequestIdentifier;
+import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
/**
* Base builder class for S3 endpoints in tests.
@@ -37,11 +41,14 @@ public class EndpointBuilder<T extends EndpointBase> {
private HttpHeaders httpHeaders;
private ContainerRequestContext requestContext;
private RequestIdentifier identifier;
+ private SignatureInfo signatureInfo;
protected EndpointBuilder(Supplier<T> constructor) {
this.constructor = constructor;
this.ozoneConfig = new OzoneConfiguration();
this.identifier = new RequestIdentifier();
+ this.signatureInfo = mock(SignatureInfo.class);
+ when(signatureInfo.isSignPayload()).thenReturn(true);
}
public EndpointBuilder<T> setBase(T base) {
@@ -74,6 +81,11 @@ public EndpointBuilder<T> setRequestId(RequestIdentifier
newRequestId) {
return this;
}
+ public EndpointBuilder<T> setSignatureInfo(SignatureInfo newSignatureInfo) {
+ this.signatureInfo = newSignatureInfo;
+ return this;
+ }
+
public T build() {
T endpoint = base != null ? base : constructor.get();
@@ -82,6 +94,7 @@ public T build() {
}
endpoint.setRequestIdentifier(identifier);
+ endpoint.setSignatureInfo(signatureInfo);
return endpoint;
}
@@ -106,6 +119,10 @@ protected RequestIdentifier getRequestId() {
return identifier;
}
+ protected SignatureInfo getSignatureInfo() {
+ return signatureInfo;
+ }
+
public static EndpointBuilder<RootEndpoint> newRootEndpointBuilder() {
return new EndpointBuilder<>(RootEndpoint::new);
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
index a06811434ff..fd83523214c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
@@ -55,6 +55,7 @@
import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -128,6 +129,9 @@ public static void setUp() throws Exception {
REST.setClient(CLIENT);
REST.setOzoneConfiguration(new OzoneConfiguration());
REST.setRequestIdentifier(new RequestIdentifier());
+ SignatureInfo signatureInfo = mock(SignatureInfo.class);
+ when(signatureInfo.isSignPayload()).thenReturn(true);
+ REST.setSignatureInfo(signatureInfo);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
