This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit afe5c517d992305c18b8ed3f8c6c199bfb19e44e
Author: Yulei-Yang <yulei.yang0...@gmail.com>
AuthorDate: Mon Dec 26 00:31:55 2022 +0800
[Improvement](S3) support access s3 via temporary security credentials
(#15340)
---
be/src/util/s3_util.cpp | 3 +++
be/src/util/s3_util.h | 1 +
.../import/import-scenes/external-storage-load.md | 16 +++++++++++++++-
.../data-operate/import/import-way/s3-load-manual.md | 16 +++++++++++++++-
.../Backup-and-Restore/CREATE-REPOSITORY.md | 17 +++++++++++++++++
.../import/import-scenes/external-storage-load.md | 16 +++++++++++++++-
.../data-operate/import/import-way/s3-load-manual.md | 14 +++++++++++++-
.../Backup-and-Restore/CREATE-REPOSITORY.md | 17 +++++++++++++++++
.../main/java/org/apache/doris/backup/S3Storage.java | 18 ++++++++++++++----
.../main/java/org/apache/doris/catalog/S3Resource.java | 8 ++++++++
10 files changed, 118 insertions(+), 8 deletions(-)
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index f9b67e57d7..26227cf63e 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -117,6 +117,9 @@ std::shared_ptr<Aws::S3::S3Client> ClientFactory::create(
Aws::Auth::AWSCredentials aws_cred(properties.find(S3_AK)->second,
properties.find(S3_SK)->second);
DCHECK(!aws_cred.IsExpiredOrEmpty());
+ if (properties.find(S3_TOKEN) != properties.end()) {
+ aws_cred.SetSessionToken(properties.find(S3_TOKEN)->second);
+ }
Aws::Client::ClientConfiguration aws_config;
aws_config.endpointOverride = properties.find(S3_ENDPOINT)->second;
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 74c723bbec..79deec6225 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -35,6 +35,7 @@ const static std::string S3_AK = "AWS_ACCESS_KEY";
const static std::string S3_SK = "AWS_SECRET_KEY";
const static std::string S3_ENDPOINT = "AWS_ENDPOINT";
const static std::string S3_REGION = "AWS_REGION";
+const static std::string S3_TOKEN = "AWS_TOKEN";
const static std::string S3_MAX_CONN_SIZE = "AWS_MAX_CONN_SIZE";
const static std::string S3_REQUEST_TIMEOUT_MS = "AWS_REQUEST_TIMEOUT_MS";
const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONN_TIMEOUT_MS";
diff --git
a/docs/en/docs/data-operate/import/import-scenes/external-storage-load.md
b/docs/en/docs/data-operate/import/import-scenes/external-storage-load.md
index 0074c591f3..e06db56de4 100644
--- a/docs/en/docs/data-operate/import/import-scenes/external-storage-load.md
+++ b/docs/en/docs/data-operate/import/import-scenes/external-storage-load.md
@@ -162,7 +162,7 @@ example:
### FAQ
-S3 SDK uses virtual-hosted style by default. However, some object storage
systems may not be enabled or support virtual-hosted style access. At this
time, we can add the `use_path_style` parameter to force the use of path style:
+1. S3 SDK uses virtual-hosted style by default. However, some object storage
systems may not be enabled or support virtual-hosted style access. At this
time, we can add the `use_path_style` parameter to force the use of path style:
```
WITH S3
@@ -174,3 +174,17 @@ S3 SDK uses virtual-hosted style by default. However, some
object storage system
"use_path_style" = "true"
)
```
+
+<version since="1.2"></version>
+2. Support using temporary security credentials to access object stores that
support the S3 protocol:
+
+```
+ WITH S3
+ (
+ "AWS_ENDPOINT" = "AWS_ENDPOINT",
+ "AWS_ACCESS_KEY" = "AWS_TEMP_ACCESS_KEY",
+ "AWS_SECRET_KEY" = "AWS_TEMP_SECRET_KEY",
+ "AWS_TOKEN" = "AWS_TEMP_TOKEN",
+ "AWS_REGION" = "AWS_REGION"
+ )
+```
diff --git a/docs/en/docs/data-operate/import/import-way/s3-load-manual.md
b/docs/en/docs/data-operate/import/import-way/s3-load-manual.md
index 815c2ba2fa..9f9a640324 100644
--- a/docs/en/docs/data-operate/import/import-way/s3-load-manual.md
+++ b/docs/en/docs/data-operate/import/import-way/s3-load-manual.md
@@ -80,7 +80,7 @@ example:
## FAQ
-S3 SDK uses virtual-hosted style by default. However, some object storage
systems may not be enabled or support virtual-hosted style access. At this
time, we can add the `use_path_style` parameter to force the use of path style:
+1. S3 SDK uses virtual-hosted style by default. However, some object storage
systems may not be enabled or support virtual-hosted style access. At this
time, we can add the `use_path_style` parameter to force the use of path style:
```text
WITH S3
@@ -92,3 +92,17 @@ S3 SDK uses virtual-hosted style by default. However, some
object storage system
"use_path_style" = "true"
)
```
+
+<version since="1.2"></version>
+2. Support using temporary security credentials to access object stores that
support the S3 protocol:
+
+```
+ WITH S3
+ (
+ "AWS_ENDPOINT" = "AWS_ENDPOINT",
+ "AWS_ACCESS_KEY" = "AWS_TEMP_ACCESS_KEY",
+ "AWS_SECRET_KEY" = "AWS_TEMP_SECRET_KEY",
+ "AWS_TOKEN" = "AWS_TEMP_TOKEN",
+ "AWS_REGION" = "AWS_REGION"
+ )
+```
diff --git
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
index 359aabd01f..f191be3157 100644
---
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
+++
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
@@ -139,6 +139,23 @@ PROPERTIES
);
```
+<version since="1.2"></version>
+7. Create a repository named minio_repo via temporary security credentials.
+
+```
+CREATE REPOSITORY `minio_repo`
+WITH S3
+ON LOCATION "s3://minio_repo"
+PROPERTIES
+(
+ "AWS_ENDPOINT" = "AWS_ENDPOINT",
+ "AWS_ACCESS_KEY" = "AWS_TEMP_ACCESS_KEY",
+ "AWS_SECRET_KEY" = "AWS_TEMP_SECRET_KEY",
+ "AWS_TOKEN" = "AWS_TEMP_TOKEN",
+ "AWS_REGION" = "AWS_REGION"
+)
+```
+
### Keywords
CREATE, REPOSITORY
diff --git
a/docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md
b/docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md
index 88c5a271e7..18b3002e01 100644
--- a/docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md
+++ b/docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md
@@ -168,7 +168,7 @@ Hdfs load 创建导入语句，导入方式和[Broker Load](../../../data-operat
### 常见问题
-S3 SDK 默认使用 `virtual-hosted style` 方式。但某些对象存储系统可能没开启或没支持 `virtual-hosted
style` 方式的访问，此时我们可以添加 `use_path_style` 参数来强制使用 `path style` 方式：
+1. S3 SDK 默认使用 `virtual-hosted style` 方式。但某些对象存储系统可能没开启或没支持 `virtual-hosted
style` 方式的访问，此时我们可以添加 `use_path_style` 参数来强制使用 `path style` 方式：
```
WITH S3
@@ -180,3 +180,17 @@ S3 SDK 默认使用 `virtual-hosted style` 方式。但某些对象存储系统
"use_path_style" = "true"
)
```
+
+<version since="1.2"></version>
+2. 支持使用临时秘钥（TOKEN) 访问所有支持 S3 协议的对象存储，用法如下：
+
+```
+ WITH S3
+ (
+ "AWS_ENDPOINT" = "AWS_ENDPOINT",
+ "AWS_ACCESS_KEY" = "AWS_TEMP_ACCESS_KEY",
+ "AWS_SECRET_KEY" = "AWS_TEMP_SECRET_KEY",
+ "AWS_TOKEN" = "AWS_TEMP_TOKEN",
+ "AWS_REGION" = "AWS_REGION"
+ )
+```
diff --git a/docs/zh-CN/docs/data-operate/import/import-way/s3-load-manual.md
b/docs/zh-CN/docs/data-operate/import/import-way/s3-load-manual.md
index 953370c596..b5736ec318 100644
--- a/docs/zh-CN/docs/data-operate/import/import-way/s3-load-manual.md
+++ b/docs/zh-CN/docs/data-operate/import/import-way/s3-load-manual.md
@@ -80,7 +80,7 @@ under the License.
## 常见问题
-S3 SDK 默认使用 virtual-hosted style 方式。但某些对象存储系统可能没开启或没支持 virtual-hosted style
方式的访问，此时我们可以添加 `use_path_style` 参数来强制使用 path style 方式：
+1. S3 SDK 默认使用 virtual-hosted style 方式。但某些对象存储系统可能没开启或没支持 virtual-hosted style
方式的访问，此时我们可以添加 `use_path_style` 参数来强制使用 path style 方式：
```text
WITH S3
@@ -93,5 +93,17 @@ S3 SDK 默认使用 virtual-hosted style 方式。但某些对象存储系统可
)
```
+<version since="1.2"></version>
+2. 支持使用临时秘钥（TOKEN) 访问所有支持 S3 协议的对象存储，用法如下：
+```
+ WITH S3
+ (
+ "AWS_ENDPOINT" = "AWS_ENDPOINT",
+ "AWS_ACCESS_KEY" = "AWS_TEMP_ACCESS_KEY",
+ "AWS_SECRET_KEY" = "AWS_TEMP_SECRET_KEY",
+ "AWS_TOKEN" = "AWS_TEMP_TOKEN",
+ "AWS_REGION" = "AWS_REGION"
+ )
+```
diff --git
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
index b61c24dcb4..710e9b46aa 100644
---
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
+++
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
@@ -136,6 +136,23 @@ PROPERTIES
"use_path_style" = "true"
);
```
+<version since="1.2"></version>
+7. 使用临时秘钥创建名为 minio_repo 的仓库
+
+```
+CREATE REPOSITORY `minio_repo`
+WITH S3
+ON LOCATION "s3://minio_repo"
+PROPERTIES
+(
+ "AWS_ENDPOINT" = "AWS_ENDPOINT",
+ "AWS_ACCESS_KEY" = "AWS_TEMP_ACCESS_KEY",
+ "AWS_SECRET_KEY" = "AWS_TEMP_SECRET_KEY",
+ "AWS_TOKEN" = "AWS_TEMP_TOKEN",
+ "AWS_REGION" = "AWS_REGION"
+)
+```
+
### Keywords
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/S3Storage.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/S3Storage.java
index 17833b1d68..d0828ca87e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/S3Storage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/S3Storage.java
@@ -34,6 +34,7 @@ import org.apache.http.client.utils.URIBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
@@ -148,10 +149,19 @@ public class S3Storage extends BlobStorage {
if (client == null) {
checkS3(caseInsensitiveProperties);
URI tmpEndpoint =
URI.create(caseInsensitiveProperties.get(S3Resource.S3_ENDPOINT));
- AwsBasicCredentials awsBasic = AwsBasicCredentials.create(
- caseInsensitiveProperties.get(S3Resource.S3_ACCESS_KEY),
- caseInsensitiveProperties.get(S3Resource.S3_SECRET_KEY));
- StaticCredentialsProvider scp =
StaticCredentialsProvider.create(awsBasic);
+ StaticCredentialsProvider scp;
+ if (!caseInsensitiveProperties.containsKey(S3Resource.S3_TOKEN)) {
+ AwsBasicCredentials awsBasic = AwsBasicCredentials.create(
+
caseInsensitiveProperties.get(S3Resource.S3_ACCESS_KEY),
+
caseInsensitiveProperties.get(S3Resource.S3_SECRET_KEY));
+ scp = StaticCredentialsProvider.create(awsBasic);
+ } else {
+ AwsSessionCredentials awsSession =
AwsSessionCredentials.create(
+
caseInsensitiveProperties.get(S3Resource.S3_ACCESS_KEY),
+
caseInsensitiveProperties.get(S3Resource.S3_SECRET_KEY),
+ caseInsensitiveProperties.get(S3Resource.S3_TOKEN));
+ scp = StaticCredentialsProvider.create(awsSession);
+ }
EqualJitterBackoffStrategy backoffStrategy =
EqualJitterBackoffStrategy
.builder()
.baseDelay(Duration.ofSeconds(1))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
index 18fae53ad3..b411c2f46c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
@@ -67,6 +67,7 @@ public class S3Resource extends Resource {
public static final String S3_BUCKET = "AWS_BUCKET";
// optional
+ public static final String S3_TOKEN = "AWS_TOKEN";
public static final String USE_PATH_STYLE = "use_path_style";
public static final String S3_MAX_CONNECTIONS = "AWS_MAX_CONNECTIONS";
public static final String S3_REQUEST_TIMEOUT_MS =
"AWS_REQUEST_TIMEOUT_MS";
@@ -191,6 +192,13 @@ public class S3Resource extends Resource {
} else {
s3Properties.put("fs.s3a.path.style.access", "false");
}
+ if (properties.containsKey(S3Resource.S3_TOKEN)) {
+ s3Properties.put("fs.s3a.session.token", properties.get(S3_TOKEN));
+ s3Properties.put("fs.s3a.aws.credentials.provider",
+
"org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider");
+ s3Properties.put("fs.s3a.impl.disable.cache", "true");
+ s3Properties.put("fs.s3.impl.disable.cache", "true");
+ }
for (Map.Entry<String, String> entry : properties.entrySet()) {
if (entry.getKey().startsWith(S3Resource.S3_FS_PREFIX)) {
s3Properties.put(entry.getKey(), entry.getValue());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
