This is an automated email from the ASF dual-hosted git repository.
ivandika pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0a8edfd515c HDDS-13678. Refactor presigned URL integration tests
(#9037)
0a8edfd515c is described below
commit 0a8edfd515ca9e6e9e88ec2a76604b54d523b9d3
Author: Hsu Han Wen <[email protected]>
AuthorDate: Wed Sep 17 11:59:32 2025 +0800
HDDS-13678. Refactor presigned URL integration tests (#9037)
---
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 34 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 545 ++++++-------
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 865 +++++++++------------
3 files changed, 646 insertions(+), 798 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
index bb5ba27a28e..ec42a0d7b4f 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
@@ -20,10 +20,16 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.io.RandomAccessFile;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.security.MessageDigest;
+import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.ozone.test.InputSubstream;
@@ -94,4 +100,32 @@ public static String extractUploadId(String xml) {
}
return null;
}
+
+ /**
+ * Open an HttpURLConnection with the given parameters.
+ *
+ * @param url The URL to connect to.
+ * @param httpMethod The HTTP method to use (e.g., "GET", "PUT", "POST",
etc.).
+ * @param headers A map of request headers to set. Can be null.
+ * @param body The request body as a byte array. Can be null.
+ * @return An open HttpURLConnection.
+ * @throws IOException If an I/O error occurs.
+ */
+ public static HttpURLConnection openHttpURLConnection(URL url, String
httpMethod, Map<String, List<String>> headers,
+ byte[] body) throws
IOException {
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod(httpMethod);
+ if (headers != null) {
+ headers.forEach((key, values) -> values.forEach(value ->
connection.addRequestProperty(key, value)));
+ }
+
+ if (body != null) {
+ connection.setDoOutput(true);
+ try (OutputStream os = connection.getOutputStream()) {
+ IOUtils.write(body, os);
+ os.flush();
+ }
+ }
+ return connection;
+ }
}
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index 54bdddf6e1c..3fe4f6ab5a1 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -77,7 +77,6 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -122,8 +121,11 @@
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ozone.test.OzoneTestBase;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.io.TempDir;
@@ -1059,196 +1061,159 @@ public void testQuotaExceeded() throws IOException {
assertEquals("QuotaExceeded", ase.getErrorCode());
}
- @Test
- public void testPresignedUrlGet() throws IOException {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final String content = "bar";
- s3Client.createBucket(bucketName);
+ @Nested
+ @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+ class PresignedUrlTests {
+ private static final String BUCKET_NAME = "presigned-url-bucket";
+ private static final String CONTENT = "bar";
+ // Set the presigned URL to expire after one hour.
+ private final Date expiration = Date.from(Instant.now().plusMillis(1000 *
60 * 60));
- InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ @BeforeAll
+ public void setup() {
+ s3Client.createBucket(BUCKET_NAME);
+ }
- s3Client.putObject(bucketName, keyName, is, new ObjectMetadata());
+ @Test
+ public void testPresignedUrlGet() throws IOException {
+ final String keyName = getKeyName();
- // Set the presigned URL to expire after one hour.
- Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60));
-
- // Generate the presigned URL
- GeneratePresignedUrlRequest generatePresignedUrlRequest =
- new GeneratePresignedUrlRequest(bucketName, keyName)
- .withMethod(HttpMethod.GET)
- .withExpiration(expiration);
- generatePresignedUrlRequest.addRequestParameter("x-custom-parameter",
"custom-value");
- URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
-
- // Download the object using HttpUrlConnection (since v1.1)
- // Capture the response body to a byte array.
- URL presignedUrl = new URL(url.toExternalForm());
- HttpURLConnection connection = (HttpURLConnection)
presignedUrl.openConnection();
- connection.setRequestMethod("GET");
- // Download the result of executing the request.
- try (InputStream s3is = connection.getInputStream();
- ByteArrayOutputStream bos = new ByteArrayOutputStream(
- content.getBytes(StandardCharsets.UTF_8).length)) {
- IOUtils.copy(s3is, bos);
- assertEquals(content, bos.toString("UTF-8"));
- }
- }
+ InputStream is = new
ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8));
- @Test
- public void testPresignedUrlHead() throws IOException {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final String content = "bar";
- s3Client.createBucket(bucketName);
+ s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata());
- InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
- s3Client.putObject(bucketName, keyName, is, new ObjectMetadata());
+ // Generate the presigned URL
+ GeneratePresignedUrlRequest generatePresignedUrlRequest =
+ new GeneratePresignedUrlRequest(BUCKET_NAME,
keyName).withMethod(HttpMethod.GET).withExpiration(expiration);
+ generatePresignedUrlRequest.addRequestParameter("x-custom-parameter",
"custom-value");
+ URL presignedUrl =
s3Client.generatePresignedUrl(generatePresignedUrlRequest);
- // Set the presigned URL to expire after one hour.
- Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60));
-
- // Test HeadObject presigned URL
- GeneratePresignedUrlRequest generatePresignedUrlRequest =
- new GeneratePresignedUrlRequest(bucketName, keyName)
- .withMethod(HttpMethod.HEAD)
- .withExpiration(expiration);
- URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
-
- URL presignedUrl = new URL(url.toExternalForm());
- HttpURLConnection connection = null;
- try {
- connection = (HttpURLConnection) presignedUrl.openConnection();
- connection.setRequestMethod("HEAD");
-
- int responseCode = connection.getResponseCode();
- assertEquals(200, responseCode, "HeadObject presigned URL should return
200 OK");
- } finally {
- if (connection != null) {
- connection.disconnect();
+ // Download the object using HttpUrlConnection (since v1.1)
+ // Capture the response body to a byte array.
+ HttpURLConnection connection =
S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null);
+ // Download the result of executing the request.
+ try (InputStream s3is = connection.getInputStream();
+ ByteArrayOutputStream bos = new
ByteArrayOutputStream(CONTENT.getBytes(StandardCharsets.UTF_8).length)) {
+ IOUtils.copy(s3is, bos);
+ assertEquals(CONTENT, bos.toString("UTF-8"));
}
}
- // Test HeadBucket presigned URL
- GeneratePresignedUrlRequest generateBucketPresignedUrlRequest =
- new GeneratePresignedUrlRequest(bucketName, null)
- .withMethod(HttpMethod.HEAD)
- .withExpiration(expiration);
- URL bucketUrl =
s3Client.generatePresignedUrl(generateBucketPresignedUrlRequest);
-
- URL presignedBucketUrl = new URL(bucketUrl.toExternalForm());
- HttpURLConnection bucketConnection = null;
- try {
- bucketConnection = (HttpURLConnection)
presignedBucketUrl.openConnection();
- bucketConnection.setRequestMethod("HEAD");
-
- int bucketResponseCode = bucketConnection.getResponseCode();
- assertEquals(200, bucketResponseCode, "HeadBucket presigned URL should
return 200 OK");
- } finally {
- if (bucketConnection != null) {
- bucketConnection.disconnect();
+ @Test
+ public void testPresignedUrlHead() throws IOException {
+ final String keyName = getKeyName();
+
+ InputStream is = new
ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8));
+ s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata());
+
+ // Test HeadObject presigned URL
+ GeneratePresignedUrlRequest generatePresignedUrlRequest =
+ new GeneratePresignedUrlRequest(BUCKET_NAME,
keyName).withMethod(HttpMethod.HEAD).withExpiration(expiration);
+ URL presignedUrl =
s3Client.generatePresignedUrl(generatePresignedUrlRequest);
+
+ HttpURLConnection connection = null;
+ try {
+ connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl,
"HEAD", null, null);
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode, "HeadObject presigned URL should
return 200 OK");
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
}
- }
- }
-
- @Test
- public void testPresignedUrlPutObject() throws Exception {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final String expectedContent = "bar";
- s3Client.createBucket(bucketName);
- // Set the presigned URL to expire after one hour.
- Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60));
-
- // Test PutObjectRequest presigned URL
- GeneratePresignedUrlRequest generatePresignedUrlRequest =
- new GeneratePresignedUrlRequest(bucketName, keyName)
- .withMethod(HttpMethod.PUT)
- .withExpiration(expiration);
- URL presignedUrl =
s3Client.generatePresignedUrl(generatePresignedUrlRequest);
-
- HttpURLConnection connection = null;
- try {
- connection = (HttpURLConnection) presignedUrl.openConnection();
- connection.setRequestMethod("PUT");
- connection.setDoOutput(true);
- try (OutputStream os = connection.getOutputStream()) {
- os.write(expectedContent.getBytes(StandardCharsets.UTF_8));
- os.flush();
+ // Test HeadBucket presigned URL
+ GeneratePresignedUrlRequest generateBucketPresignedUrlRequest =
+ new GeneratePresignedUrlRequest(BUCKET_NAME,
null).withMethod(HttpMethod.HEAD).withExpiration(expiration);
+ URL presignedBucketUrl =
s3Client.generatePresignedUrl(generateBucketPresignedUrlRequest);
+
+ HttpURLConnection bucketConnection = null;
+ try {
+ bucketConnection =
S3SDKTestUtils.openHttpURLConnection(presignedBucketUrl, "HEAD", null, null);
+ int bucketResponseCode = bucketConnection.getResponseCode();
+ assertEquals(200, bucketResponseCode, "HeadBucket presigned URL should
return 200 OK");
+ } finally {
+ if (bucketConnection != null) {
+ bucketConnection.disconnect();
+ }
}
+ }
- int responseCode = connection.getResponseCode();
- assertEquals(200, responseCode, "PutObject presigned URL should return
200 OK");
- String actualContent;
- S3Object s3Object = s3Client.getObject(bucketName, keyName);
- try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) {
- actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
- }
- assertEquals(expectedContent, actualContent, "Downloaded content should
match uploaded content");
- } finally {
- if (connection != null) {
- connection.disconnect();
+ @Test
+ public void testPresignedUrlPutObject() throws Exception {
+ final String keyName = getKeyName();
+
+ // Test PutObjectRequest presigned URL
+ GeneratePresignedUrlRequest generatePresignedUrlRequest =
+ new GeneratePresignedUrlRequest(BUCKET_NAME,
keyName).withMethod(HttpMethod.PUT).withExpiration(expiration);
+ URL presignedUrl =
s3Client.generatePresignedUrl(generatePresignedUrlRequest);
+
+ HttpURLConnection connection = null;
+ try {
+ connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT",
+ null, CONTENT.getBytes(StandardCharsets.UTF_8));
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode, "PutObject presigned URL should return
200 OK");
+ String actualContent;
+ S3Object s3Object = s3Client.getObject(BUCKET_NAME, keyName);
+ try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) {
+ actualContent = IOUtils.toString(inputStream,
StandardCharsets.UTF_8);
+ }
+ assertEquals(CONTENT, actualContent, "Downloaded content should match
uploaded content");
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
}
}
- }
- @Test
- public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws
Exception {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final Map<String, String> userMetadata = new HashMap<>();
- userMetadata.put("key1", "value1");
- userMetadata.put("key2", "value2");
- final Map<String, String> tags = new HashMap<>();
- tags.put("tag1", "value1");
- tags.put("tag2", "value2");
-
- s3Client.createBucket(bucketName);
+ @Test
+ public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws
Exception {
+ final String keyName = getKeyName();
+ final Map<String, String> userMetadata = new HashMap<>();
+ userMetadata.put("key1", "value1");
+ userMetadata.put("key2", "value2");
+ final Map<String, String> tags = new HashMap<>();
+ tags.put("tag1", "value1");
+ tags.put("tag2", "value2");
- File multipartUploadFile =
Files.createFile(tempDir.resolve("multipartupload.txt")).toFile();
- createFile(multipartUploadFile, (int) (10 * MB));
+ File multipartUploadFile =
Files.createFile(tempDir.resolve("multipartupload.txt")).toFile();
+ createFile(multipartUploadFile, (int) (10 * MB));
- // create MPU using presigned URL
- GeneratePresignedUrlRequest initMPUPresignedUrlRequest =
- createInitMPUPresignedUrlRequest(bucketName, keyName, userMetadata,
tags);
+ // create MPU using presigned URL
+ GeneratePresignedUrlRequest initMPUPresignedUrlRequest =
+ createInitMPUPresignedUrlRequest(keyName, userMetadata, tags);
- String uploadId = initMultipartUpload(initMPUPresignedUrlRequest);
+ String uploadId = initMultipartUpload(initMPUPresignedUrlRequest);
- // upload parts using presigned URL
- List<PartETag> completedParts = uploadParts(multipartUploadFile,
bucketName, keyName, uploadId);
+ // upload parts using presigned URL
+ List<PartETag> completedParts = uploadParts(multipartUploadFile,
keyName, uploadId);
- // Complete multipart upload using presigned URL
- completeMPU(bucketName, keyName, uploadId, completedParts);
+ // Complete multipart upload using presigned URL
+ completeMPU(keyName, uploadId, completedParts);
- // Verify upload result
- ObjectMetadata objectMeta = s3Client.getObjectMetadata(bucketName,
keyName);
- assertEquals(userMetadata, objectMeta.getUserMetadata());
+ // Verify upload result
+ ObjectMetadata objectMeta = s3Client.getObjectMetadata(BUCKET_NAME,
keyName);
+ assertEquals(userMetadata, objectMeta.getUserMetadata());
- // Verify content
- S3Object s3Object = s3Client.getObject(bucketName, keyName);
- assertEquals(tags.size(), s3Object.getTaggingCount());
- String actualContent;
- try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) {
- actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+ // Verify content
+ S3Object s3Object = s3Client.getObject(BUCKET_NAME, keyName);
+ assertEquals(tags.size(), s3Object.getTaggingCount());
+ String actualContent;
+ try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) {
+ actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+ }
+ String expectedContent = new
String(Files.readAllBytes(multipartUploadFile.toPath()),
StandardCharsets.UTF_8);
+ assertEquals(expectedContent, actualContent, "Downloaded content should
match uploaded content");
}
- String expectedContent = new
String(Files.readAllBytes(multipartUploadFile.toPath()),
StandardCharsets.UTF_8);
- assertEquals(expectedContent, actualContent, "Downloaded content should
match uploaded content");
- }
- private static void completeMPU(String bucketName, String keyName, String
uploadId, List<PartETag> completedParts)
- throws IOException {
- GeneratePresignedUrlRequest request = new
GeneratePresignedUrlRequest(bucketName, keyName)
- .withMethod(HttpMethod.POST)
- .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 * 60));
- request.addRequestParameter("uploadId", uploadId);
+ private void completeMPU(String keyName, String uploadId, List<PartETag>
completedParts) throws IOException {
+ GeneratePresignedUrlRequest request =
+ new GeneratePresignedUrlRequest(BUCKET_NAME,
keyName).withMethod(HttpMethod.POST).withExpiration(expiration);
+ request.addRequestParameter("uploadId", uploadId);
- HttpURLConnection httpConnection = null;
- try {
- httpConnection = (HttpURLConnection)
s3Client.generatePresignedUrl(request).openConnection();
- httpConnection.setRequestMethod("POST");
- httpConnection.setDoOutput(true);
+ URL presignedUrl = s3Client.generatePresignedUrl(request);
// Generate completion XML payload
StringBuilder completionXml = new StringBuilder();
@@ -1262,154 +1227,150 @@ private static void completeMPU(String bucketName,
String keyName, String upload
completionXml.append("</CompleteMultipartUpload>");
byte[] completionPayloadBytes =
completionXml.toString().getBytes(StandardCharsets.UTF_8);
- try (OutputStream os = httpConnection.getOutputStream()) {
- IOUtils.write(completionPayloadBytes, os);
- }
- int responseCode = httpConnection.getResponseCode();
- assertEquals(200, responseCode, "Complete multipart upload should return
200 OK");
- } finally {
- if (httpConnection != null) {
- httpConnection.disconnect();
+ HttpURLConnection httpConnection = null;
+ try {
+ httpConnection = S3SDKTestUtils.openHttpURLConnection(presignedUrl,
"POST", null, completionPayloadBytes);
+ int responseCode = httpConnection.getResponseCode();
+ assertEquals(200, responseCode, "Complete multipart upload should
return 200 OK");
+ } finally {
+ if (httpConnection != null) {
+ httpConnection.disconnect();
+ }
}
}
- }
- private static List<PartETag> uploadParts(File multipartUploadFile, String
bucketName, String keyName,
- String uploadId) throws
IOException {
- List<PartETag> completedParts = new ArrayList<>();
- ByteBuffer byteBuffer = ByteBuffer.allocate((int) (5 * MB));
- long filePosition = 0;
- long fileLength = multipartUploadFile.length();
- int partNumber = 1;
-
- try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
- while (filePosition < fileLength) {
- file.seek(filePosition);
- long bytesRead = file.getChannel().read(byteBuffer);
- byteBuffer.flip();
-
- // generate presigned URL for each part
- GeneratePresignedUrlRequest request = new
GeneratePresignedUrlRequest(bucketName, keyName)
- .withMethod(HttpMethod.PUT)
- .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 *
60));
- request.addRequestParameter("partNumber", String.valueOf(partNumber));
- request.addRequestParameter("uploadId", uploadId);
-
- URL presignedUrl = s3Client.generatePresignedUrl(request);
-
- // upload each part using presigned URL
- HttpURLConnection connection = null;
- try {
- connection = (HttpURLConnection) presignedUrl.openConnection();
- connection.setDoOutput(true);
- connection.setRequestMethod("PUT");
- connection.setRequestProperty("Content-Length",
String.valueOf(byteBuffer.remaining()));
-
- try (OutputStream os = connection.getOutputStream()) {
- os.write(byteBuffer.array(), 0, byteBuffer.remaining());
- os.flush();
+ private List<PartETag> uploadParts(File multipartUploadFile, String
keyName, String uploadId) throws IOException {
+ List<PartETag> completedParts = new ArrayList<>();
+ ByteBuffer byteBuffer = ByteBuffer.allocate((int) (5 * MB));
+ long filePosition = 0;
+ long fileLength = multipartUploadFile.length();
+ int partNumber = 1;
+
+ try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
+ while (filePosition < fileLength) {
+ file.seek(filePosition);
+ long bytesRead = file.getChannel().read(byteBuffer);
+ byteBuffer.flip();
+
+ // generate presigned URL for each part
+ GeneratePresignedUrlRequest request =
+ new GeneratePresignedUrlRequest(BUCKET_NAME,
keyName).withMethod(HttpMethod.PUT)
+ .withExpiration(expiration);
+ request.addRequestParameter("partNumber",
String.valueOf(partNumber));
+ request.addRequestParameter("uploadId", uploadId);
+
+ URL presignedUrl = s3Client.generatePresignedUrl(request);
+
+ // upload each part using presigned URL
+ HttpURLConnection connection = null;
+ try {
+ Map<String, List<String>> headers = new HashMap<>();
+ List<String> header =
Collections.singletonList(String.valueOf(byteBuffer.remaining()));
+ headers.put("Content-Length", header);
+ byte[] body = new byte[byteBuffer.remaining()];
+ byteBuffer.get(body);
+
+ connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl,
"PUT", headers, body);
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode, String.format("Upload part %d
should return 200 OK", partNumber));
+
+ String etag = connection.getHeaderField("ETag");
+ PartETag partETag = new PartETag(partNumber, etag);
+ completedParts.add(partETag);
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
}
- int responseCode = connection.getResponseCode();
- assertEquals(200, responseCode, String.format("Upload part %d should
return 200 OK", partNumber));
+ byteBuffer.clear();
+ filePosition += bytesRead;
+ partNumber++;
+ }
+ }
+ return completedParts;
+ }
- String etag = connection.getHeaderField("ETag");
- PartETag partETag = new PartETag(partNumber, etag);
- completedParts.add(partETag);
- } finally {
- if (connection != null) {
- connection.disconnect();
+ private String initMultipartUpload(GeneratePresignedUrlRequest request)
throws IOException {
+ URL presignedUrl = s3Client.generatePresignedUrl(request);
+ String uploadId;
+ HttpURLConnection httpConnection = null;
+ try {
+ Map<String, String> customRequestHeaders =
request.getCustomRequestHeaders();
+ Map<String, List<String>> headers = new HashMap<>();
+ if (customRequestHeaders != null) {
+ for (Map.Entry<String, String> entry :
customRequestHeaders.entrySet()) {
+ headers.put(entry.getKey(),
Collections.singletonList(entry.getValue()));
}
}
+ httpConnection = S3SDKTestUtils.openHttpURLConnection(presignedUrl,
"POST", headers, null);
+ int initMPUConnectionResponseCode = httpConnection.getResponseCode();
+ assertEquals(200, initMPUConnectionResponseCode);
- byteBuffer.clear();
- filePosition += bytesRead;
- partNumber++;
+ try (InputStream is = httpConnection.getInputStream()) {
+ String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
+ uploadId = S3SDKTestUtils.extractUploadId(responseXml);
+ }
+ } finally {
+ if (httpConnection != null) {
+ httpConnection.disconnect();
+ }
}
+ return uploadId;
}
- return completedParts;
- }
- private String initMultipartUpload(GeneratePresignedUrlRequest request)
throws IOException {
- URL presignedUrl = s3Client.generatePresignedUrl(request);
- String uploadId;
- HttpURLConnection httpConnection = null;
- try {
- httpConnection = (HttpURLConnection) presignedUrl.openConnection();
- httpConnection.setRequestMethod("POST");
- httpConnection.setDoOutput(true);
-
request.getCustomRequestHeaders().forEach(httpConnection::setRequestProperty);
-
- httpConnection.connect();
- int initMPUConnectionResponseCode = httpConnection.getResponseCode();
- assertEquals(200, initMPUConnectionResponseCode);
- try (InputStream is = httpConnection.getInputStream()) {
- String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
- uploadId = S3SDKTestUtils.extractUploadId(responseXml);
- }
- } finally {
- if (httpConnection != null) {
- httpConnection.disconnect();
+ private GeneratePresignedUrlRequest
createInitMPUPresignedUrlRequest(String keyName,
+
Map<String, String> userMetadata,
+
Map<String, String> tags) throws Exception {
+ GeneratePresignedUrlRequest initMPUPresignUrlRequest =
+ new GeneratePresignedUrlRequest(BUCKET_NAME,
keyName).withMethod(HttpMethod.POST).withExpiration(expiration);
+
+ userMetadata.forEach((k, v) -> {
+
initMPUPresignUrlRequest.putCustomRequestHeader(CUSTOM_METADATA_HEADER_PREFIX +
k, v);
+ });
+
+ StringBuilder tagValueBuilder = new StringBuilder();
+ for (Map.Entry<String, String> entry : tags.entrySet()) {
+ if (tagValueBuilder.length() > 0) {
+ tagValueBuilder.append('&');
+ }
+
tagValueBuilder.append(entry.getKey()).append('=').append(URLEncoder.encode(entry.getValue(),
"UTF-8"));
}
+ initMPUPresignUrlRequest.putCustomRequestHeader(S3Consts.TAG_HEADER,
tagValueBuilder.toString());
+ return initMPUPresignUrlRequest;
}
- return uploadId;
- }
- private GeneratePresignedUrlRequest createInitMPUPresignedUrlRequest(String
bucketName, String keyName,
-
Map<String, String> userMetadata,
-
Map<String, String> tags) throws Exception {
- GeneratePresignedUrlRequest initMPUPresignUrlRequest = new
GeneratePresignedUrlRequest(bucketName, keyName)
- .withMethod(HttpMethod.POST)
- .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 * 60));
-
- userMetadata.forEach((k, v) -> {
-
initMPUPresignUrlRequest.putCustomRequestHeader(CUSTOM_METADATA_HEADER_PREFIX +
k, v);
- });
-
- StringBuilder tagValueBuilder = new StringBuilder();
- for (Map.Entry<String, String> entry : tags.entrySet()) {
- if (tagValueBuilder.length() > 0) {
- tagValueBuilder.append('&');
+ @Test
+ public void testPresignedUrlDelete() throws IOException {
+ final String keyName = getKeyName();
+
+ try (InputStream is = new
ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8))) {
+ s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata());
}
-
tagValueBuilder.append(entry.getKey()).append('=').append(URLEncoder.encode(entry.getValue(),
"UTF-8"));
- }
- initMPUPresignUrlRequest.putCustomRequestHeader(S3Consts.TAG_HEADER,
tagValueBuilder.toString());
- return initMPUPresignUrlRequest;
- }
- @Test
- public void testPresignedUrlDelete() throws IOException {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final String content = "bar";
+ // Generate the presigned URL for DELETE
+ GeneratePresignedUrlRequest generatePresignedUrlRequest =
+ new GeneratePresignedUrlRequest(BUCKET_NAME,
keyName).withMethod(HttpMethod.DELETE)
+ .withExpiration(expiration);
+ URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
+
+ // Execute the DELETE request using HttpUrlConnection
+ HttpURLConnection connection = null;
+ try {
+ connection = S3SDKTestUtils.openHttpURLConnection(url, "DELETE", null,
null);
+ int responseCode = connection.getResponseCode();
+ // Verify the response code is 204 (No Content)
+ assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode);
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
- s3Client.createBucket(bucketName);
- try (InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))) {
- s3Client.putObject(bucketName, keyName, is, new ObjectMetadata());
+ // Verify the object is deleted
+ assertFalse(s3Client.doesObjectExist(BUCKET_NAME, keyName));
}
-
- // Set the presigned URL to expire after one hour.
- Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60));
-
- // Generate the presigned URL for DELETE
- GeneratePresignedUrlRequest generatePresignedUrlRequest =
- new GeneratePresignedUrlRequest(bucketName, keyName)
- .withMethod(HttpMethod.DELETE)
- .withExpiration(expiration);
- URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
-
- // Execute the DELETE request using HttpUrlConnection
- URL presignedUrl = new URL(url.toExternalForm());
- HttpURLConnection connection = (HttpURLConnection)
presignedUrl.openConnection();
- connection.setRequestMethod("DELETE");
- int responseCode = connection.getResponseCode();
-
- // Verify the response code is 204 (No Content)
- assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode);
-
- // Verify the object is deleted
- assertFalse(s3Client.doesObjectExist(bucketName, keyName));
}
/**
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 31f58c6d9a2..79928c3264f 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -33,7 +33,6 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -72,6 +71,7 @@
import org.apache.hadoop.ozone.s3.endpoint.S3Owner;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ozone.test.OzoneTestBase;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Nested;
@@ -456,26 +456,46 @@ public void testResumableDownloadWithEtagMismatch()
throws Exception {
}
}
- @Test
- public void testPresignedUrlGet() throws Exception {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final String content = "bar";
- s3Client.createBucket(b -> b.bucket(bucketName));
+ @Nested
+ @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+ class PresignedUrlTests {
+ private static final String CONTENT = "bar";
+ private static final String BUCKET_NAME = "presigned-url-bucket";
+ private final SdkHttpClient sdkHttpClient = ApacheHttpClient.create();
+ // The URL will expire in 10 minutes.
+ private final Duration duration = Duration.ofMinutes(10);
+ private S3Presigner presigner;
- s3Client.putObject(b -> b
- .bucket(bucketName)
- .key(keyName),
- RequestBody.fromString(content));
+ @BeforeAll
+ public void setup() {
+ s3Client.createBucket(b -> b.bucket(BUCKET_NAME));
+ presigner = S3Presigner.builder()
+ // TODO: Find a way to retrieve the path style configuration from
S3Client instead
+
.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
+
.endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get())
+ .region(s3Client.serviceClientConfiguration().region())
+
.credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build();
+ }
- try (S3Presigner presigner = createS3Presigner()) {
- GetObjectRequest objectRequest = GetObjectRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .build();
+ @AfterAll
+ public void tearDown() {
+ presigner.close();
+ sdkHttpClient.close();
+ }
+
+ @Test
+ public void testPresignedUrlGet() throws Exception {
+ final String keyName = getKeyName();
+
+ s3Client.putObject(b -> b
+ .bucket(BUCKET_NAME)
+ .key(keyName),
+ RequestBody.fromString(CONTENT));
+
+ GetObjectRequest objectRequest =
GetObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build();
GetObjectPresignRequest presignRequest =
GetObjectPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10)) // The URL will expire
in 10 minutes.
+ .signatureDuration(duration)
.getObjectRequest(objectRequest)
.build();
@@ -484,66 +504,46 @@ public void testPresignedUrlGet() throws Exception {
// Download the object using HttpUrlConnection (since v1.1)
// Capture the response body to a byte array.
URL presignedUrl = presignedRequest.url();
- HttpURLConnection connection = (HttpURLConnection)
presignedUrl.openConnection();
- connection.setRequestMethod("GET");
+ HttpURLConnection connection =
S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null);
// Download the result of executing the request.
try (InputStream s3is = connection.getInputStream();
- ByteArrayOutputStream bos = new ByteArrayOutputStream(
- content.getBytes(StandardCharsets.UTF_8).length)) {
+ ByteArrayOutputStream bos = new
ByteArrayOutputStream(CONTENT.getBytes(StandardCharsets.UTF_8).length)) {
IoUtils.copy(s3is, bos);
- assertEquals(content, bos.toString("UTF-8"));
+ assertEquals(CONTENT, bos.toString("UTF-8"));
+ } finally {
+ connection.disconnect();
}
// Use the AWS SDK for Java SdkHttpClient class to do the download
- SdkHttpRequest request = SdkHttpRequest.builder()
- .method(SdkHttpMethod.GET)
- .uri(presignedUrl.toURI())
- .build();
+ SdkHttpRequest request =
SdkHttpRequest.builder().method(SdkHttpMethod.GET).uri(presignedUrl.toURI()).build();
- HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
- .request(request)
- .build();
+ HttpExecuteRequest executeRequest =
HttpExecuteRequest.builder().request(request).build();
- try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create();
- ByteArrayOutputStream bos = new ByteArrayOutputStream(
- content.getBytes(StandardCharsets.UTF_8).length)) {
+ try (ByteArrayOutputStream bos = new
ByteArrayOutputStream(CONTENT.getBytes(StandardCharsets.UTF_8).length)) {
HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
- assertTrue(response.responseBody().isPresent(), () -> "The presigned
url download request " +
- "should have a response body");
- response.responseBody().ifPresent(
- abortableInputStream -> {
- try {
- IoUtils.copy(abortableInputStream, bos);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- assertEquals(content, bos.toString("UTF-8"));
+ assertTrue(response.responseBody().isPresent(),
+ () -> "The presigned url download request " + "should have a
response body");
+ response.responseBody().ifPresent(abortableInputStream -> {
+ try {
+ IoUtils.copy(abortableInputStream, bos);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ assertEquals(CONTENT, bos.toString("UTF-8"));
}
}
- }
- @Test
- public void testPresignedUrlHead() throws Exception {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final String content = "bar";
- s3Client.createBucket(b -> b.bucket(bucketName));
-
- s3Client.putObject(b -> b
- .bucket(bucketName)
- .key(keyName),
- RequestBody.fromString(content));
+ @Test
+ public void testPresignedUrlHead() throws Exception {
+ final String keyName = getKeyName();
- try (S3Presigner presigner = createS3Presigner()) {
+ s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName),
RequestBody.fromString(CONTENT));
- HeadObjectRequest objectRequest = HeadObjectRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .build();
+ HeadObjectRequest objectRequest =
HeadObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build();
HeadObjectPresignRequest presignRequest =
HeadObjectPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
+ .signatureDuration(duration)
.headObjectRequest(objectRequest)
.build();
@@ -552,40 +552,29 @@ public void testPresignedUrlHead() throws Exception {
URL presignedUrl = presignedRequest.url();
HttpURLConnection connection = null;
try {
- connection = (HttpURLConnection) presignedUrl.openConnection();
- connection.setRequestMethod("HEAD");
-
+ connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl,
"HEAD", null, null);
int responseCode = connection.getResponseCode();
assertEquals(200, responseCode, "HeadObject presigned URL should
return 200 OK");
-
- // Use the AWS SDK for Java SdkHttpClient class to test the HEAD
request
- SdkHttpRequest request = SdkHttpRequest.builder()
- .method(SdkHttpMethod.HEAD)
- .uri(presignedUrl.toURI())
- .build();
-
- HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
- .request(request)
- .build();
-
- try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) {
- HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
- assertEquals(200, response.httpResponse().statusCode(),
- "HeadObject presigned URL should return 200 OK via
SdkHttpClient");
- }
} finally {
if (connection != null) {
connection.disconnect();
}
}
+ // Use the AWS SDK for Java SdkHttpClient class to test the HEAD request
+ SdkHttpRequest request =
SdkHttpRequest.builder().method(SdkHttpMethod.HEAD).uri(presignedUrl.toURI()).build();
+
+ HttpExecuteRequest executeRequest =
HttpExecuteRequest.builder().request(request).build();
+
+ HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
+ assertEquals(200, response.httpResponse().statusCode(),
+ "HeadObject presigned URL should return 200 OK via SdkHttpClient");
+
// Test HeadBucket presigned URL
- HeadBucketRequest bucketRequest = HeadBucketRequest.builder()
- .bucket(bucketName)
- .build();
+ HeadBucketRequest bucketRequest =
HeadBucketRequest.builder().bucket(BUCKET_NAME).build();
HeadBucketPresignRequest headBucketPresignRequest =
HeadBucketPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
+ .signatureDuration(duration)
.headBucketRequest(bucketRequest)
.build();
@@ -594,50 +583,35 @@ public void testPresignedUrlHead() throws Exception {
URL presignedBucketUrl = presignedBucketRequest.url();
HttpURLConnection bucketConnection = null;
try {
- bucketConnection = (HttpURLConnection)
presignedBucketUrl.openConnection();
- bucketConnection.setRequestMethod("HEAD");
-
+ bucketConnection =
S3SDKTestUtils.openHttpURLConnection(presignedBucketUrl, "HEAD", null, null);
int bucketResponseCode = bucketConnection.getResponseCode();
assertEquals(200, bucketResponseCode, "HeadBucket presigned URL should
return 200 OK");
-
- // Use the AWS SDK for Java SdkHttpClient class to test the HEAD
request for bucket
- SdkHttpRequest bucketSdkRequest = SdkHttpRequest.builder()
- .method(SdkHttpMethod.HEAD)
- .uri(presignedBucketUrl.toURI())
- .build();
-
- HttpExecuteRequest bucketExecuteRequest = HttpExecuteRequest.builder()
- .request(bucketSdkRequest)
- .build();
-
- try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) {
- HttpExecuteResponse response =
sdkHttpClient.prepareRequest(bucketExecuteRequest).call();
- assertEquals(200, response.httpResponse().statusCode(),
- "HeadBucket presigned URL should return 200 OK via
SdkHttpClient");
- }
} finally {
if (bucketConnection != null) {
bucketConnection.disconnect();
}
}
- }
- }
- @Test
- public void testPresignedUrlPut() throws Exception {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- s3Client.createBucket(b -> b.bucket(bucketName));
+ // Use the AWS SDK for Java SdkHttpClient class to test the HEAD request
for bucket
+ SdkHttpRequest bucketSdkRequest = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.HEAD)
+ .uri(presignedBucketUrl.toURI())
+ .build();
+ HttpExecuteRequest bucketExecuteRequest =
HttpExecuteRequest.builder().request(bucketSdkRequest).build();
- try (S3Presigner presigner = createS3Presigner()) {
+ HttpExecuteResponse bucketResponse =
sdkHttpClient.prepareRequest(bucketExecuteRequest).call();
+ assertEquals(200, bucketResponse.httpResponse().statusCode(),
+ "HeadBucket presigned URL should return 200 OK via SdkHttpClient");
+ }
- PutObjectRequest objectRequest = PutObjectRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .build();
+ @Test
+ public void testPresignedUrlPut() throws Exception {
+ final String keyName = getKeyName();
+
+ PutObjectRequest objectRequest =
PutObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build();
PutObjectPresignRequest presignRequest =
PutObjectPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
+ .signatureDuration(duration)
.putObjectRequest(objectRequest)
.build();
@@ -645,460 +619,371 @@ public void testPresignedUrlPut() throws Exception {
// use http url connection
HttpURLConnection connection = null;
- String expectedContent;
String actualContent;
try {
- expectedContent = "This is a test content for presigned PUT URL.";
- connection = (HttpURLConnection)
presignedRequest.url().openConnection();
- connection.setRequestMethod("PUT");
- connection.setDoOutput(true);
- try (OutputStream os = connection.getOutputStream()) {
- os.write(expectedContent.getBytes(StandardCharsets.UTF_8));
- os.flush();
- }
-
+ connection =
S3SDKTestUtils.openHttpURLConnection(presignedRequest.url(), "PUT",
+ presignedRequest.signedHeaders(),
CONTENT.getBytes(StandardCharsets.UTF_8));
int responseCode = connection.getResponseCode();
assertEquals(200, responseCode, "PutObject presigned URL should return
200 OK");
-
- //verify the object was uploaded
- ResponseInputStream<GetObjectResponse> object1 = s3Client.getObject(b1
-> b1.bucket(bucketName).key(keyName));
- actualContent = IoUtils.toUtf8String(object1);
- assertEquals(expectedContent, actualContent);
-
- // Use the AWS SDK for Java SdkHttpClient class to test the PUT request
- expectedContent = "This content is for testing the SdkHttpClient PUT
request.";
- SdkHttpRequest request = SdkHttpRequest.builder()
- .method(SdkHttpMethod.PUT)
- .uri(presignedRequest.url().toURI())
- .build();
-
- byte[] bytes = expectedContent.getBytes(StandardCharsets.UTF_8);
- HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
- .request(request)
- .contentStreamProvider(() -> new ByteArrayInputStream(bytes))
- .build();
-
- try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) {
- HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
- assertEquals(200, response.httpResponse().statusCode(),
- "PutObject presigned URL should return 200 OK via
SdkHttpClient");
- }
-
- //verify the object was uploaded
- ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b
-> b.bucket(bucketName).key(keyName));
- actualContent = IoUtils.toUtf8String(object);
- assertEquals(expectedContent, actualContent);
} finally {
if (connection != null) {
connection.disconnect();
}
}
- }
- }
+ //verify the object was uploaded
+ ResponseInputStream<GetObjectResponse> object1 = s3Client.getObject(b1
-> b1.bucket(BUCKET_NAME).key(keyName));
+ actualContent = IoUtils.toUtf8String(object1);
+ assertEquals(CONTENT, actualContent);
- @Test
- public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws
Exception {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final Map<String, String> userMetadata = new HashMap<>();
- userMetadata.put("key1", "value1");
- userMetadata.put("key2", "value2");
+ // Use the AWS SDK for Java SdkHttpClient class to test the PUT request
+ SdkHttpRequest request = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.PUT)
+ .uri(presignedRequest.url().toURI())
+ .build();
- List<Tag> tags = Arrays.asList(
- Tag.builder().key("tag1").value("value1").build(),
- Tag.builder().key("tag2").value("value2").build()
- );
+ byte[] bytes = CONTENT.getBytes(StandardCharsets.UTF_8);
+ HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
+ .request(request)
+ .contentStreamProvider(() -> new ByteArrayInputStream(bytes))
+ .build();
- s3Client.createBucket(b -> b.bucket(bucketName));
+ HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
+ assertEquals(200, response.httpResponse().statusCode(),
+ "PutObject presigned URL should return 200 OK via SdkHttpClient");
- File multipartUploadFile =
Files.createFile(tempDir.resolve("multipartupload.txt")).toFile();
- createFile(multipartUploadFile, (int) (10 * MB));
+ //verify the object was uploaded
+ ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b ->
b.bucket(BUCKET_NAME).key(keyName));
+ actualContent = IoUtils.toUtf8String(object);
+ assertEquals(CONTENT, actualContent);
+ }
+
+ @Test
+ public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws
Exception {
+ final String keyName = getKeyName();
+ final Map<String, String> userMetadata = new HashMap<>();
+ userMetadata.put("key1", "value1");
+ userMetadata.put("key2", "value2");
- try (S3Presigner presigner = createS3Presigner()) {
+ List<Tag> tags =
Arrays.asList(Tag.builder().key("tag1").value("value1").build(),
+ Tag.builder().key("tag2").value("value2").build());
+
+ File multipartUploadFile =
Files.createFile(tempDir.resolve("multipartupload.txt")).toFile();
+ createFile(multipartUploadFile, (int) (10 * MB));
// generate create MPU presigned URL
CreateMultipartUploadRequest createRequest =
CreateMultipartUploadRequest.builder()
- .bucket(bucketName)
+ .bucket(BUCKET_NAME)
.key(keyName)
.metadata(userMetadata)
.tagging(Tagging.builder().tagSet(tags).build())
.build();
CreateMultipartUploadPresignRequest createMPUPresignRequest =
CreateMultipartUploadPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
+ .signatureDuration(duration)
.createMultipartUploadRequest(createRequest)
.build();
PresignedCreateMultipartUploadRequest presignCreateMultipartUpload =
presigner.presignCreateMultipartUpload(createMPUPresignRequest);
- mpuWithHttpURLConnection(presignCreateMultipartUpload,
multipartUploadFile, bucketName, keyName, presigner,
- userMetadata, tags);
- mpuWithSdkHttpClient(presignCreateMultipartUpload, multipartUploadFile,
bucketName, keyName, presigner,
- userMetadata, tags);
+ mpuWithHttpURLConnection(presignCreateMultipartUpload,
multipartUploadFile, keyName, userMetadata, tags);
+ mpuWithSdkHttpClient(presignCreateMultipartUpload, multipartUploadFile,
keyName, userMetadata, tags);
}
- }
- private void mpuWithHttpURLConnection(PresignedCreateMultipartUploadRequest
presignCreateMultipartUpload,
- File multipartUploadFile, String
bucketName, String keyName,
- S3Presigner presigner, Map<String,
String> userMetadata,
- List<Tag> tags) throws IOException {
- // create MPU using presigned URL
- String uploadId;
- HttpURLConnection createMultiPartUploadConnection = null;
- try {
- createMultiPartUploadConnection =
openHttpUrlConnection(presignCreateMultipartUpload);
- int createMultiPartUploadConnectionResponseCode =
createMultiPartUploadConnection.getResponseCode();
- assertEquals(200, createMultiPartUploadConnectionResponseCode,
- "CreateMultipartUploadPresignRequest should return 200 OK");
-
- try (InputStream is = createMultiPartUploadConnection.getInputStream()) {
- String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
- uploadId = S3SDKTestUtils.extractUploadId(responseXml);
- }
- } finally {
- if (createMultiPartUploadConnection != null) {
- createMultiPartUploadConnection.disconnect();
+ private void
mpuWithHttpURLConnection(PresignedCreateMultipartUploadRequest
presignCreateMultipartUpload,
+ File multipartUploadFile, String
keyName, Map<String, String> userMetadata,
+ List<Tag> tags) throws IOException {
+ // create MPU using presigned URL
+ String uploadId;
+ HttpURLConnection createMPUConnection = null;
+ try {
+ createMPUConnection =
S3SDKTestUtils.openHttpURLConnection(presignCreateMultipartUpload.url(), "POST",
+ presignCreateMultipartUpload.signedHeaders(), null);
+ int createMultiPartUploadConnectionResponseCode =
createMPUConnection.getResponseCode();
+ assertEquals(200, createMultiPartUploadConnectionResponseCode,
+ "CreateMultipartUploadPresignRequest should return 200 OK");
+
+ try (InputStream is = createMPUConnection.getInputStream()) {
+ String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
+ uploadId = S3SDKTestUtils.extractUploadId(responseXml);
+ }
+ } finally {
+ if (createMPUConnection != null) {
+ createMPUConnection.disconnect();
+ }
}
- }
-
- // Upload parts using presigned URL
- List<CompletedPart> completedParts =
- uploadPartWithHttpURLConnection(multipartUploadFile, bucketName,
keyName, presigner, uploadId);
- // complete MPU using presigned URL
- CompleteMultipartUploadRequest completeRequest =
CompleteMultipartUploadRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .uploadId(uploadId)
-
.multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
- .build();
- CompleteMultipartUploadPresignRequest
completeMultipartUploadPresignRequest =
- CompleteMultipartUploadPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
- .completeMultipartUploadRequest(completeRequest)
- .build();
+ // Upload parts using presigned URL
+ List<CompletedPart> completedParts =
uploadPartWithHttpURLConnection(multipartUploadFile, keyName, uploadId);
- PresignedCompleteMultipartUploadRequest
presignedCompleteMultipartUploadRequest =
-
presigner.presignCompleteMultipartUpload(completeMultipartUploadPresignRequest);
+ // complete MPU using presigned URL
+ CompleteMultipartUploadRequest completeRequest =
CompleteMultipartUploadRequest.builder()
+ .bucket(BUCKET_NAME)
+ .key(keyName)
+ .uploadId(uploadId)
+
.multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
+ .build();
+ CompleteMultipartUploadPresignRequest completeMPUPresignRequest =
CompleteMultipartUploadPresignRequest.builder()
+ .signatureDuration(duration)
+ .completeMultipartUploadRequest(completeRequest)
+ .build();
- completeMPUWithHttpUrlConnection(presignedCompleteMultipartUploadRequest,
completedParts);
+ PresignedCompleteMultipartUploadRequest
presignedCompleteMultipartUploadRequest =
+ presigner.presignCompleteMultipartUpload(completeMPUPresignRequest);
- // verify upload result
- HeadObjectResponse headObjectResponse = s3Client.headObject(b ->
b.bucket(bucketName).key(keyName));
- assertTrue(headObjectResponse.hasMetadata());
- assertEquals(userMetadata, headObjectResponse.metadata());
-
- ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b ->
b.bucket(bucketName).key(keyName));
- assertEquals(tags.size(), object.response().tagCount());
- String actualContent = IoUtils.toUtf8String(object);
- String originalContent = new
String(Files.readAllBytes(multipartUploadFile.toPath()),
StandardCharsets.UTF_8);
- assertEquals(originalContent, actualContent, "Uploaded file content should
match original file content");
- }
+
completeMPUWithHttpUrlConnection(presignedCompleteMultipartUploadRequest,
completedParts);
- private List<CompletedPart> uploadPartWithHttpURLConnection(File
multipartUploadFile, String bucketName,
- String keyName,
S3Presigner presigner, String uploadId)
- throws IOException {
- List<CompletedPart> completedParts = new ArrayList<>();
- int partNumber = 1;
- ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB));
+ // verify upload result
+ HeadObjectResponse headObjectResponse = s3Client.headObject(b ->
b.bucket(BUCKET_NAME).key(keyName));
+ assertTrue(headObjectResponse.hasMetadata());
+ assertEquals(userMetadata, headObjectResponse.metadata());
- try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
- long fileSize = file.length();
- long position = 0;
+ ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b ->
b.bucket(BUCKET_NAME).key(keyName));
+ assertEquals(tags.size(), object.response().tagCount());
+ String actualContent = IoUtils.toUtf8String(object);
+ String originalContent = new
String(Files.readAllBytes(multipartUploadFile.toPath()),
StandardCharsets.UTF_8);
+ assertEquals(originalContent, actualContent, "Uploaded file content
should match original file content");
+ }
- while (position < fileSize) {
- file.seek(position);
- long read = file.getChannel().read(bb);
+ private List<CompletedPart> uploadPartWithHttpURLConnection(File
multipartUploadFile, String keyName,
+ String
uploadId) throws IOException {
+ List<CompletedPart> completedParts = new ArrayList<>();
+ int partNumber = 1;
+ ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB));
- bb.flip();
+ try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
+ long fileSize = file.length();
+ long position = 0;
- // First create an UploadPartRequest
- UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .uploadId(uploadId)
- .partNumber(partNumber)
- .contentLength((long) bb.remaining())
- .build();
+ while (position < fileSize) {
+ file.seek(position);
+ long read = file.getChannel().read(bb);
- // Generate presigned URL for each part
- UploadPartPresignRequest presignRequest =
UploadPartPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
- .uploadPartRequest(uploadPartRequest)
- .build();
+ bb.flip();
- PresignedUploadPartRequest presignedRequest =
presigner.presignUploadPart(presignRequest);
+ // First create an UploadPartRequest
+ UploadPartRequest request = UploadPartRequest.builder()
+ .bucket(BUCKET_NAME)
+ .key(keyName)
+ .uploadId(uploadId)
+ .partNumber(partNumber)
+ .contentLength((long) bb.remaining())
+ .build();
- // use presigned URL to upload the part
- HttpURLConnection connection = null;
- try {
- connection = (HttpURLConnection)
presignedRequest.url().openConnection();
- connection.setDoOutput(true);
- connection.setRequestMethod("PUT");
+ // Generate presigned URL for each part
+ UploadPartPresignRequest presignRequest =
UploadPartPresignRequest.builder()
+ .signatureDuration(duration)
+ .uploadPartRequest(request)
+ .build();
- try (OutputStream os = connection.getOutputStream()) {
- os.write(bb.array(), 0, bb.remaining());
- os.flush();
+ PresignedUploadPartRequest presignedRequest =
presigner.presignUploadPart(presignRequest);
+
+ // use presigned URL to upload the part
+ HttpURLConnection connection = null;
+ try {
+ byte[] body = new byte[bb.remaining()];
+ bb.get(body);
+ connection =
S3SDKTestUtils.openHttpURLConnection(presignedRequest.url(), "PUT",
+ presignedRequest.signedHeaders(), body);
+
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode, String.format("Upload part %d
should return 200 OK", partNumber));
+
+ String etag = connection.getHeaderField("ETag");
+ CompletedPart part =
CompletedPart.builder().partNumber(partNumber).eTag(etag).build();
+ completedParts.add(part);
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
}
- int responseCode = connection.getResponseCode();
- assertEquals(200, responseCode,
- String.format("Upload part %d should return 200 OK",
partNumber));
-
- String etag = connection.getHeaderField("ETag");
- CompletedPart part = CompletedPart.builder()
- .partNumber(partNumber)
- .eTag(etag)
- .build();
- completedParts.add(part);
- } finally {
- if (connection != null) {
- connection.disconnect();
- }
+ bb.clear();
+ position += read;
+ partNumber++;
}
-
- bb.clear();
- position += read;
- partNumber++;
}
+ return completedParts;
}
- return completedParts;
- }
- private HttpURLConnection
openHttpUrlConnection(PresignedCreateMultipartUploadRequest request) throws
IOException {
- HttpURLConnection connection = (HttpURLConnection)
request.url().openConnection();
- connection.setDoOutput(true);
- connection.setRequestMethod("POST");
- for (Map.Entry<String, List<String>> entry :
request.signedHeaders().entrySet()) {
- String key = entry.getKey();
- for (String value : entry.getValue()) {
- connection.setRequestProperty(key, value);
- }
- }
- return connection;
- }
+ private void
completeMPUWithHttpUrlConnection(PresignedCompleteMultipartUploadRequest
request,
+ List<CompletedPart>
completedParts) throws IOException {
+ HttpURLConnection connection = null;
+ try {
+ String xmlPayload = buildCompleteMultipartUploadXml(completedParts);
+ byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8);
+ connection = S3SDKTestUtils.openHttpURLConnection(request.url(),
"POST", request.signedHeaders(), payloadBytes);
- private void
completeMPUWithHttpUrlConnection(PresignedCompleteMultipartUploadRequest
request,
- List<CompletedPart>
completedParts) throws IOException {
- HttpURLConnection completeMultipartUploadConnection = null;
- try {
- completeMultipartUploadConnection = (HttpURLConnection)
request.url().openConnection();
- completeMultipartUploadConnection.setDoOutput(true);
- completeMultipartUploadConnection.setRequestMethod("POST");
-
- // copy headers
- for (Map.Entry<String, List<String>> entry :
request.signedHeaders().entrySet()) {
- String key = entry.getKey();
- for (String value : entry.getValue()) {
- completeMultipartUploadConnection.setRequestProperty(key, value);
+ int responseCode = connection.getResponseCode();
+ assertEquals(200, responseCode, "CompleteMultipartUploadPresignRequest
should return 200 OK");
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
}
}
-
- String xmlPayload = buildCompleteMultipartUploadXml(completedParts);
- byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8);
- try (OutputStream os =
completeMultipartUploadConnection.getOutputStream()) {
- IOUtils.write(payloadBytes, os);
- os.flush();
- }
-
- int completeMultipartUploadConnectionResponseCode =
completeMultipartUploadConnection.getResponseCode();
- assertEquals(200, completeMultipartUploadConnectionResponseCode,
- "CompleteMultipartUploadPresignRequest should return 200 OK");
- } finally {
- if (completeMultipartUploadConnection != null) {
- completeMultipartUploadConnection.disconnect();
- }
}
- }
- private void mpuWithSdkHttpClient(PresignedCreateMultipartUploadRequest
presignCreateMultipartUpload,
- File multipartUploadFile, String
bucketName, String keyName,
- S3Presigner presigner, Map<String, String>
userMetadata, List<Tag> tags)
- throws Exception {
- // create MPU using presigned URL
- String uploadId;
- try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) {
- uploadId = createMPUWithSdkHttpClient(presignCreateMultipartUpload,
sdkHttpClient);
+ private void mpuWithSdkHttpClient(PresignedCreateMultipartUploadRequest
presignCreateMultipartUpload,
+ File multipartUploadFile, String
keyName, Map<String, String> userMetadata,
+ List<Tag> tags) throws Exception {
+ // create MPU using presigned URL
+ String uploadId =
createMPUWithSdkHttpClient(presignCreateMultipartUpload);
// Upload parts using presigned URL
- List<CompletedPart> completedParts =
- uploadPartWithSdkHttpClient(multipartUploadFile, bucketName,
keyName, presigner, uploadId, sdkHttpClient);
+ List<CompletedPart> completedParts =
uploadPartWithSdkHttpClient(multipartUploadFile, keyName, uploadId);
// complete MPU using presigned URL
- completeMPUWithSdkHttpClient(bucketName, keyName, presigner, uploadId,
completedParts, sdkHttpClient);
+ completeMPUWithSdkHttpClient(keyName, uploadId, completedParts);
// verify upload result
- HeadObjectResponse headObjectResponse = s3Client.headObject(b ->
b.bucket(bucketName).key(keyName));
+ HeadObjectResponse headObjectResponse = s3Client.headObject(b ->
b.bucket(BUCKET_NAME).key(keyName));
assertTrue(headObjectResponse.hasMetadata());
assertEquals(userMetadata, headObjectResponse.metadata());
- ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b ->
b.bucket(bucketName).key(keyName));
+ ResponseInputStream<GetObjectResponse> object = s3Client.getObject(b ->
b.bucket(BUCKET_NAME).key(keyName));
assertEquals(tags.size(), object.response().tagCount());
String actualContent = IoUtils.toUtf8String(object);
String originalContent = new
String(Files.readAllBytes(multipartUploadFile.toPath()),
StandardCharsets.UTF_8);
assertEquals(originalContent, actualContent, "Uploaded file content
should match original file content");
}
- }
- private void completeMPUWithSdkHttpClient(String bucketName, String keyName,
S3Presigner presigner, String uploadId,
- List<CompletedPart> completedParts, SdkHttpClient
sdkHttpClient) throws Exception {
- CompleteMultipartUploadRequest completeRequest =
CompleteMultipartUploadRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .uploadId(uploadId)
-
.multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
- .build();
+ private void completeMPUWithSdkHttpClient(String keyName, String uploadId,
List<CompletedPart> completedParts)
+ throws Exception {
+ CompleteMultipartUploadRequest request =
CompleteMultipartUploadRequest.builder()
+ .bucket(BUCKET_NAME)
+ .key(keyName)
+ .uploadId(uploadId)
+
.multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build())
+ .build();
- CompleteMultipartUploadPresignRequest
completeMultipartUploadPresignRequest =
- CompleteMultipartUploadPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
- .completeMultipartUploadRequest(completeRequest)
- .build();
+ CompleteMultipartUploadPresignRequest presignRequest =
CompleteMultipartUploadPresignRequest.builder()
+ .signatureDuration(duration)
+ .completeMultipartUploadRequest(request)
+ .build();
- PresignedCompleteMultipartUploadRequest
presignedCompleteMultipartUploadRequest =
-
presigner.presignCompleteMultipartUpload(completeMultipartUploadPresignRequest);
+ PresignedCompleteMultipartUploadRequest
presignedCompleteMultipartUploadRequest =
+ presigner.presignCompleteMultipartUpload(presignRequest);
- String xmlPayload = buildCompleteMultipartUploadXml(completedParts);
- byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8);
+ String xmlPayload = buildCompleteMultipartUploadXml(completedParts);
+ byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8);
- SdkHttpRequest completeMultipartUploadRequest = SdkHttpRequest.builder()
- .method(SdkHttpMethod.POST)
- .uri(presignedCompleteMultipartUploadRequest.url().toURI())
- .build();
+ SdkHttpRequest sdkHttpRequest = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.POST)
+ .uri(presignedCompleteMultipartUploadRequest.url().toURI())
+ .build();
- HttpExecuteRequest completeMultipartUploadExecuteRequest =
HttpExecuteRequest.builder()
- .request(completeMultipartUploadRequest)
- .contentStreamProvider(() -> new ByteArrayInputStream(payloadBytes))
- .build();
+ HttpExecuteRequest httpExecuteRequest = HttpExecuteRequest.builder()
+ .request(sdkHttpRequest)
+ .contentStreamProvider(() -> new ByteArrayInputStream(payloadBytes))
+ .build();
- HttpExecuteResponse completeMultipartUploadResponse =
-
sdkHttpClient.prepareRequest(completeMultipartUploadExecuteRequest).call();
- assertEquals(200,
completeMultipartUploadResponse.httpResponse().statusCode(),
- "CompleteMultipartUploadPresignRequest should return 200 OK");
- }
+ HttpExecuteResponse response =
sdkHttpClient.prepareRequest(httpExecuteRequest).call();
+ assertEquals(200, response.httpResponse().statusCode(),
+ "CompleteMultipartUploadPresignRequest should return 200 OK");
+ }
- private List<CompletedPart> uploadPartWithSdkHttpClient(File
multipartUploadFile, String bucketName, String keyName,
- S3Presigner presigner,
String uploadId,
- SdkHttpClient
httpClient) throws Exception {
- List<CompletedPart> completedParts = new ArrayList<>();
- int partNumber = 1;
- ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB));
+ private List<CompletedPart> uploadPartWithSdkHttpClient(File
multipartUploadFile, String keyName, String uploadId)
+ throws Exception {
+ List<CompletedPart> completedParts = new ArrayList<>();
+ int partNumber = 1;
+ ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB));
- try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
- long fileSize = file.length();
- long position = 0;
+ try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile,
"r")) {
+ long fileSize = file.length();
+ long position = 0;
- while (position < fileSize) {
- file.seek(position);
- long read = file.getChannel().read(bb);
+ while (position < fileSize) {
+ file.seek(position);
+ long read = file.getChannel().read(bb);
- bb.flip();
+ bb.flip();
- // Generate presigned URL for each part
- UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .uploadId(uploadId)
- .partNumber(partNumber)
- .contentLength((long) bb.remaining())
- .build();
+ // Generate presigned URL for each part
+ UploadPartRequest request = UploadPartRequest.builder()
+ .bucket(BUCKET_NAME)
+ .key(keyName)
+ .uploadId(uploadId)
+ .partNumber(partNumber)
+ .contentLength((long) bb.remaining())
+ .build();
- UploadPartPresignRequest presignRequest =
UploadPartPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
- .uploadPartRequest(uploadPartRequest)
- .build();
+ UploadPartPresignRequest presignRequest =
UploadPartPresignRequest.builder()
+ .signatureDuration(duration)
+ .uploadPartRequest(request)
+ .build();
- PresignedUploadPartRequest presignedRequest =
presigner.presignUploadPart(presignRequest);
+ PresignedUploadPartRequest presignedRequest =
presigner.presignUploadPart(presignRequest);
- // upload each part using presigned URL
- SdkHttpRequest uploadPartSdkRequest = SdkHttpRequest.builder()
- .method(SdkHttpMethod.PUT)
- .uri(presignedRequest.url().toURI())
- .build();
+ // upload each part using presigned URL
+ SdkHttpRequest uploadPartSdkRequest = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.PUT)
+ .uri(presignedRequest.url().toURI())
+ .build();
- byte[] bytes = new byte[bb.remaining()];
- bb.get(bytes);
+ byte[] bytes = new byte[bb.remaining()];
+ bb.get(bytes);
- HttpExecuteRequest uploadPartExecuteRequest =
HttpExecuteRequest.builder()
- .request(uploadPartSdkRequest)
- .contentStreamProvider(() -> new ByteArrayInputStream(bytes))
- .build();
+ HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
+ .request(uploadPartSdkRequest)
+ .contentStreamProvider(() -> new ByteArrayInputStream(bytes))
+ .build();
- HttpExecuteResponse uploadPartResponse =
httpClient.prepareRequest(uploadPartExecuteRequest).call();
+ HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
- String etag = uploadPartResponse.httpResponse()
- .firstMatchingHeader("ETag")
- .orElseThrow(() -> new RuntimeException("ETag missing in
response"));
+ String etag = response.httpResponse().firstMatchingHeader("ETag")
+ .orElseThrow(() -> new RuntimeException("ETag missing in
response"));
- CompletedPart part = CompletedPart.builder()
- .partNumber(partNumber)
- .eTag(etag)
- .build();
- completedParts.add(part);
+ CompletedPart part =
CompletedPart.builder().partNumber(partNumber).eTag(etag).build();
+ completedParts.add(part);
- bb.clear();
- position += read;
- partNumber++;
+ bb.clear();
+ position += read;
+ partNumber++;
+ }
}
+ return completedParts;
}
- return completedParts;
- }
-
- private String
createMPUWithSdkHttpClient(PresignedCreateMultipartUploadRequest request,
- SdkHttpClient httpClient) throws Exception {
- String uploadId;
- SdkHttpRequest createMultipartUploadRequest = SdkHttpRequest.builder()
- .method(SdkHttpMethod.POST)
- .uri(request.url().toURI())
- .headers(request.signedHeaders())
- .build();
- HttpExecuteRequest createMultipartUploadExecuteRequest =
HttpExecuteRequest.builder()
- .request(createMultipartUploadRequest)
- .build();
+ private String
createMPUWithSdkHttpClient(PresignedCreateMultipartUploadRequest request)
throws Exception {
+ String uploadId;
+ SdkHttpRequest sdkHttpRequest = SdkHttpRequest.builder()
+ .method(SdkHttpMethod.POST)
+ .uri(request.url().toURI())
+ .headers(request.signedHeaders())
+ .build();
- HttpExecuteResponse createMultipartUploadResponse =
- httpClient.prepareRequest(createMultipartUploadExecuteRequest).call();
+ HttpExecuteRequest httpExecuteRequest = HttpExecuteRequest.builder()
+ .request(sdkHttpRequest)
+ .build();
- try (InputStream is = createMultipartUploadResponse.responseBody().get()) {
- String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
- uploadId = S3SDKTestUtils.extractUploadId(responseXml);
+ HttpExecuteResponse response =
sdkHttpClient.prepareRequest(httpExecuteRequest).call();
+ try (InputStream is = response.responseBody().get()) {
+ String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8);
+ uploadId = S3SDKTestUtils.extractUploadId(responseXml);
+ }
+ return uploadId;
}
- return uploadId;
- }
- private String buildCompleteMultipartUploadXml(List<CompletedPart> parts) {
- StringBuilder xml = new StringBuilder();
- xml.append("<CompleteMultipartUpload>\n");
- for (CompletedPart part : parts) {
- xml.append(" <Part>\n");
- xml.append("
<PartNumber>").append(part.partNumber()).append("</PartNumber>\n");
- xml.append(" <ETag>").append(part.eTag()).append("</ETag>\n");
- xml.append(" </Part>\n");
+ private String buildCompleteMultipartUploadXml(List<CompletedPart> parts) {
+ StringBuilder xml = new StringBuilder();
+ xml.append("<CompleteMultipartUpload>\n");
+ for (CompletedPart part : parts) {
+ xml.append(" <Part>\n");
+ xml.append("
<PartNumber>").append(part.partNumber()).append("</PartNumber>\n");
+ xml.append(" <ETag>").append(part.eTag()).append("</ETag>\n");
+ xml.append(" </Part>\n");
+ }
+ xml.append("</CompleteMultipartUpload>");
+ return xml.toString();
}
- xml.append("</CompleteMultipartUpload>");
- return xml.toString();
- }
- @Test
- public void testPresignedUrlDelete() throws Exception {
- final String bucketName = getBucketName();
- final String keyName = getKeyName();
- final String content = "bar";
- s3Client.createBucket(b -> b.bucket(bucketName));
-
- s3Client.putObject(b -> b
- .bucket(bucketName)
- .key(keyName),
- RequestBody.fromString(content));
+ @Test
+ public void testPresignedUrlDelete() throws Exception {
+ final String keyName = getKeyName();
- try (S3Presigner presigner = createS3Presigner()) {
+ s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName),
RequestBody.fromString(CONTENT));
- DeleteObjectRequest objectRequest = DeleteObjectRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .build();
+ DeleteObjectRequest objectRequest =
DeleteObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build();
DeleteObjectPresignRequest presignRequest =
DeleteObjectPresignRequest.builder()
.signatureDuration(Duration.ofMinutes(10))
@@ -1110,69 +995,37 @@ public void testPresignedUrlDelete() throws Exception {
// use http url connection
HttpURLConnection connection = null;
try {
- connection = (HttpURLConnection)
presignedRequest.url().openConnection();
- connection.setRequestMethod("DELETE");
-
+ connection =
S3SDKTestUtils.openHttpURLConnection(presignedRequest.url(), "DELETE", null,
null);
int responseCode = connection.getResponseCode();
assertEquals(204, responseCode, "DeleteObject presigned URL should
return 204 No Content");
//verify the object was deleted
- assertThrows(NoSuchKeyException.class, () -> s3Client.getObject(b ->
b.bucket(bucketName).key(keyName)));
+ assertThrows(NoSuchKeyException.class, () -> s3Client.getObject(b ->
b.bucket(BUCKET_NAME).key(keyName)));
} finally {
if (connection != null) {
connection.disconnect();
}
}
- }
- // use SdkHttpClient
- s3Client.putObject(b -> b
- .bucket(bucketName)
- .key(keyName),
- RequestBody.fromString(content));
-
- try (S3Presigner presigner = createS3Presigner();
- SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) {
-
- DeleteObjectRequest objectRequest = DeleteObjectRequest.builder()
- .bucket(bucketName)
- .key(keyName)
- .build();
-
- DeleteObjectPresignRequest presignRequest =
DeleteObjectPresignRequest.builder()
- .signatureDuration(Duration.ofMinutes(10))
- .deleteObjectRequest(objectRequest)
- .build();
-
- PresignedDeleteObjectRequest presignedRequest =
presigner.presignDeleteObject(presignRequest);
+ // use SdkHttpClient
+ s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName),
RequestBody.fromString(CONTENT));
SdkHttpRequest request = SdkHttpRequest.builder()
.method(SdkHttpMethod.DELETE)
.uri(presignedRequest.url().toURI())
.build();
- HttpExecuteRequest executeRequest = HttpExecuteRequest.builder()
- .request(request)
- .build();
+ HttpExecuteRequest executeRequest =
HttpExecuteRequest.builder().request(request).build();
HttpExecuteResponse response =
sdkHttpClient.prepareRequest(executeRequest).call();
assertEquals(204, response.httpResponse().statusCode(),
"DeleteObject presigned URL should return 204 No Content via
SdkHttpClient");
//verify the object was deleted
- assertThrows(NoSuchKeyException.class, () -> s3Client.getObject(b ->
b.bucket(bucketName).key(keyName)));
+ assertThrows(NoSuchKeyException.class, () -> s3Client.getObject(b ->
b.bucket(BUCKET_NAME).key(keyName)));
}
}
- private S3Presigner createS3Presigner() {
- return S3Presigner.builder()
- // TODO: Find a way to retrieve the path style configuration from
S3Client instead
-
.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
-
.endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get())
- .region(s3Client.serviceClientConfiguration().region())
-
.credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build();
- }
-
/**
* Tests the functionality to create a snapshot of an Ozone bucket and then
read files
* from the snapshot directory using the S3 SDK.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
