This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d437487 Pradeep/s3 credential chain fix (#5755)
d437487 is described below
commit d437487b08fd61ce5449bc5cfa9169352f54c6d5
Author: pradeepgv42 <66842697+pradeepg...@users.noreply.github.com>
AuthorDate: Sat Jul 25 12:47:00 2020 -0700
Pradeep/s3 credential chain fix (#5755)
* Add ProfileCredentialsProvider & instanceProfileCredentialsProvider
to the credentials provider chain for S3PinotFs
* Use a generic CredentialProvider and fix a bug in isDirectory func
* Remove HeadObjectRequest infavor of ListObjectsV2Request which limits the
number of keys returned back
Co-authored-by: Pradeep Gopanapalli Venkata
<pradeepvenk...@pradeeps-mbp.lan>
---
.../apache/pinot/plugin/filesystem/S3PinotFS.java | 22 +++++-----------------
1 file changed, 5 insertions(+), 17 deletions(-)
diff --git
a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
index 2e504ea..54f7912 100644
---
a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
+++
b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
@@ -41,10 +41,8 @@ import com.google.common.collect.ImmutableList;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
-import
software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import
software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.sync.ResponseTransformer;
import software.amazon.awssdk.regions.Region;
@@ -89,9 +87,7 @@ public class S3PinotFS extends PinotFS {
AwsBasicCredentials awsBasicCredentials =
AwsBasicCredentials.create(accessKey, secretKey);
awsCredentialsProvider =
StaticCredentialsProvider.create(awsBasicCredentials);
} else {
- awsCredentialsProvider =
-
AwsCredentialsProviderChain.builder().addCredentialsProvider(SystemPropertyCredentialsProvider.create())
-
.addCredentialsProvider(EnvironmentVariableCredentialsProvider.create()).build();
+ awsCredentialsProvider = DefaultCredentialsProvider.create();
}
_s3Client =
S3Client.builder().region(Region.of(region)).credentialsProvider(awsCredentialsProvider).build();
@@ -434,18 +430,10 @@ public class S3PinotFS extends PinotFS {
if (prefix.equals(DELIMITER)) {
return true;
}
- try {
- HeadObjectRequest headObjectRequest =
-
HeadObjectRequest.builder().bucket(uri.getHost()).key(uri.getPath()).build();
- HeadObjectResponse s3ObjectMetadata =
_s3Client.headObject(headObjectRequest);
-
- return s3ObjectMetadata.sdkHttpResponse().isSuccessful();
- } catch (NoSuchKeyException e) {
- LOGGER.error("Could not get directory entry for {}", uri);
- }
- ListObjectsV2Request listObjectsV2Request =
-
ListObjectsV2Request.builder().bucket(uri.getHost()).prefix(prefix).build();
+ ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request
+ .builder().bucket(uri.getHost())
+ .prefix(prefix).maxKeys(2).build();
ListObjectsV2Response listObjectsV2Response =
_s3Client.listObjectsV2(listObjectsV2Request);
return listObjectsV2Response.hasContents();
} catch (NoSuchKeyException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
