This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0-beta
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d09de3bece12cc24b4625bdafb0accdd6c5b66b2
Author: slothever <18522955+w...@users.noreply.github.com>
AuthorDate: Fri Jun 9 09:18:43 2023 +0800
[fix](multi-catalog)fix hive catalog docs, obs impl, dlf properties (#20342)
1. fix hive catalog docs
2. fix dlf properties
3. fix obs impl
---
docs/en/docs/lakehouse/multi-catalog/hive.md | 20 ++++++-------
docs/zh-CN/docs/lakehouse/multi-catalog/hive.md | 20 ++++++-------
.../doris/catalog/HiveMetaStoreClientHelper.java | 2 ++
.../org/apache/doris/common/util/PrintableMap.java | 1 +
.../java/org/apache/doris/common/util/S3Util.java | 20 ++++++++-----
.../datasource/property/PropertyConverter.java | 35 ++++++++++++----------
.../property/constants/DLFProperties.java | 21 ++++++++++++-
.../org/apache/doris/fs/FileSystemFactory.java | 2 +-
.../datasource/property/PropertyConverterTest.java | 2 +-
9 files changed, 77 insertions(+), 46 deletions(-)
diff --git a/docs/en/docs/lakehouse/multi-catalog/hive.md
b/docs/en/docs/lakehouse/multi-catalog/hive.md
index 52269a648c..191bb7ce30 100644
--- a/docs/en/docs/lakehouse/multi-catalog/hive.md
+++ b/docs/en/docs/lakehouse/multi-catalog/hive.md
@@ -126,8 +126,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.uris" = "thrift://172.0.0.1:9083",
"s3.endpoint" = "s3.us-east-1.amazonaws.com",
- "s3.access-key" = "ak",
- "s3.secret-key" = "sk"
+ "s3.access_key" = "ak",
+ "s3.secret_key" = "sk"
"use_path_style" = "true"
);
```
@@ -145,8 +145,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.uris" = "thrift://172.0.0.1:9083",
"oss.endpoint" = "oss.oss-cn-beijing.aliyuncs.com",
- "oss.access-key" = "ak",
- "oss.secret-key" = "sk"
+ "oss.access_key" = "ak",
+ "oss.secret_key" = "sk"
);
```
@@ -157,8 +157,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.uris" = "thrift://172.0.0.1:9083",
"obs.endpoint" = "obs.cn-north-4.myhuaweicloud.com",
- "obs.access-key" = "ak",
- "obs.secret-key" = "sk"
+ "obs.access_key" = "ak",
+ "obs.secret_key" = "sk"
);
```
@@ -169,8 +169,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.uris" = "thrift://172.0.0.1:9083",
"cos.endpoint" = "cos.ap-beijing.myqcloud.com",
- "cos.access-key" = "ak",
- "cos.secret-key" = "sk"
+ "cos.access_key" = "ak",
+ "cos.secret_key" = "sk"
);
```
@@ -181,8 +181,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.type" = "glue",
"glue.endpoint" = "https://glue.us-east-1.amazonaws.com";,
- "glue.access-key" = "ak",
- "glue.secret-key" = "sk"
+ "glue.access_key" = "ak",
+ "glue.secret_key" = "sk"
);
```
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
index ff0ebdd157..a815f7eeba 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
@@ -125,8 +125,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.uris" = "thrift://172.0.0.1:9083",
"s3.endpoint" = "s3.us-east-1.amazonaws.com",
- "s3.access-key" = "ak",
- "s3.secret-key" = "sk"
+ "s3.access_key" = "ak",
+ "s3.secret_key" = "sk"
"use_path_style" = "true"
);
```
@@ -144,8 +144,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.uris" = "thrift://172.0.0.1:9083",
"oss.endpoint" = "oss.oss-cn-beijing.aliyuncs.com",
- "oss.access-key" = "ak",
- "oss.secret-key" = "sk"
+ "oss.access_key" = "ak",
+ "oss.secret_key" = "sk"
);
```
@@ -156,8 +156,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.uris" = "thrift://172.0.0.1:9083",
"obs.endpoint" = "obs.cn-north-4.myhuaweicloud.com",
- "obs.access-key" = "ak",
- "obs.secret-key" = "sk"
+ "obs.access_key" = "ak",
+ "obs.secret_key" = "sk"
);
```
@@ -168,8 +168,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.uris" = "thrift://172.0.0.1:9083",
"cos.endpoint" = "cos.ap-beijing.myqcloud.com",
- "cos.access-key" = "ak",
- "cos.secret-key" = "sk"
+ "cos.access_key" = "ak",
+ "cos.secret_key" = "sk"
);
```
@@ -180,8 +180,8 @@ CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.type" = "glue",
"glue.endpoint" = "https://glue.us-east-1.amazonaws.com";,
- "glue.access-key" = "ak",
- "glue.secret-key" = "sk"
+ "glue.access_key" = "ak",
+ "glue.secret_key" = "sk"
);
```
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
index 996d1f53da..eba26320b8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
@@ -43,6 +43,7 @@ import org.apache.doris.fs.remote.RemoteFileSystem;
import org.apache.doris.thrift.TBrokerFileStatus;
import org.apache.doris.thrift.TExprOpcode;
+import com.aliyun.datalake.metastore.common.DataLakeConfig;
import com.aliyun.datalake.metastore.hive2.ProxyMetaStoreClient;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
@@ -160,6 +161,7 @@ public class HiveMetaStoreClientHelper {
try {
if ("dlf".equalsIgnoreCase(type)) {
// For aliyun DLF
+ hiveConf.set(DataLakeConfig.CATALOG_CREATE_DEFAULT_DB,
"false");
metaStoreClient = new ProxyMetaStoreClient(hiveConf);
} else {
metaStoreClient = new HiveMetaStoreClient(hiveConf);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
index c0e731b997..3a5f5e0cd5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
@@ -60,6 +60,7 @@ public class PrintableMap<K, V> {
HIDDEN_KEY = Sets.newHashSet();
HIDDEN_KEY.addAll(S3Properties.Env.FS_KEYS);
HIDDEN_KEY.addAll(GlueProperties.META_KEYS);
+ HIDDEN_KEY.addAll(DLFProperties.META_KEYS);
}
public PrintableMap(Map<K, V> map, String keyValueSeparator,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
index 4620bdfa6c..f1c58feed0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -26,19 +26,22 @@ public class S3Util {
private static final Logger LOG = LogManager.getLogger(S3Util.class);
public static boolean isObjStorage(String location) {
+ return isS3CompatibleObjStorage(location) ||
location.startsWith(FeConstants.FS_PREFIX_OBS);
+ }
+
+ private static boolean isS3CompatibleObjStorage(String location) {
return location.startsWith(FeConstants.FS_PREFIX_S3)
- || location.startsWith(FeConstants.FS_PREFIX_S3A)
- || location.startsWith(FeConstants.FS_PREFIX_S3N)
- || location.startsWith(FeConstants.FS_PREFIX_GCS)
- || location.startsWith(FeConstants.FS_PREFIX_BOS)
- || location.startsWith(FeConstants.FS_PREFIX_COS)
- || location.startsWith(FeConstants.FS_PREFIX_OSS)
- || location.startsWith(FeConstants.FS_PREFIX_OBS);
+ || location.startsWith(FeConstants.FS_PREFIX_S3A)
+ || location.startsWith(FeConstants.FS_PREFIX_S3N)
+ || location.startsWith(FeConstants.FS_PREFIX_GCS)
+ || location.startsWith(FeConstants.FS_PREFIX_BOS)
+ || location.startsWith(FeConstants.FS_PREFIX_COS)
+ || location.startsWith(FeConstants.FS_PREFIX_OSS);
}
public static String convertToS3IfNecessary(String location) {
LOG.debug("try convert location to s3 prefix: " + location);
- if (isObjStorage(location)) {
+ if (isS3CompatibleObjStorage(location)) {
int pos = location.indexOf("://");
if (pos == -1) {
throw new RuntimeException("No '://' found in location: " +
location);
@@ -47,4 +50,5 @@ public class S3Util {
}
return location;
}
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index fad5b64e79..61a685abfa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -35,6 +35,7 @@ import com.amazonaws.glue.catalog.util.AWSGlueConfig;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.hadoop.fs.obs.OBSConstants;
+import org.apache.hadoop.fs.obs.OBSFileSystem;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider;
@@ -93,9 +94,7 @@ public class PropertyConverter {
* Support other cloud client here.
*/
public static Map<String, String> convertToHadoopFSProperties(Map<String,
String> props) {
- if (props.containsKey(S3Properties.ENDPOINT)) {
- return convertToS3Properties(props,
S3Properties.getCredential(props));
- } else if (props.containsKey(ObsProperties.ENDPOINT)) {
+ if (props.containsKey(ObsProperties.ENDPOINT)) {
return convertToOBSProperties(props,
ObsProperties.getCredential(props));
} else if (props.containsKey(GCSProperties.ENDPOINT)) {
return convertToGCSProperties(props,
GCSProperties.getCredential(props));
@@ -103,6 +102,8 @@ public class PropertyConverter {
return convertToOSSProperties(props,
OssProperties.getCredential(props));
} else if (props.containsKey(CosProperties.ENDPOINT)) {
return convertToCOSProperties(props,
CosProperties.getCredential(props));
+ } else if (props.containsKey(S3Properties.ENDPOINT)) {
+ return convertToS3Properties(props,
S3Properties.getCredential(props));
} else if (props.containsKey(S3Properties.Env.ENDPOINT)) {
// checkout env in the end
// compatible with the s3,obs,oss,cos when they use aws client.
@@ -117,6 +118,7 @@ public class PropertyConverter {
Map<String, String> obsProperties = Maps.newHashMap();
obsProperties.put(OBSConstants.ENDPOINT,
props.get(ObsProperties.ENDPOINT));
obsProperties.put(ObsProperties.FS.IMPL_DISABLE_CACHE, "true");
+ obsProperties.put("fs.obs.impl", OBSFileSystem.class.getName());
if (credential.isWhole()) {
obsProperties.put(OBSConstants.ACCESS_KEY,
credential.getAccessKey());
obsProperties.put(OBSConstants.SECRET_KEY,
credential.getSecretKey());
@@ -237,6 +239,7 @@ public class PropertyConverter {
getPropertiesFromDLFConf(props);
// if configure DLF properties in catalog properties, use them to
override config in hive-site.xml
getPropertiesFromDLFProps(props, credential);
+ props.put(DataLakeConfig.CATALOG_CREATE_DEFAULT_DB, "false");
return props;
}
@@ -267,7 +270,7 @@ public class PropertyConverter {
// And add "-internal" to access oss within vpc
props.put(S3Properties.REGION, "oss-" + region);
String publicAccess = hiveConf.get("dlf.catalog.accessPublic",
"false");
- props.put(S3Properties.ENDPOINT, getDLFEndpoint(region,
Boolean.parseBoolean(publicAccess)));
+ props.put(S3Properties.ENDPOINT, getOssEndpoint(region,
Boolean.parseBoolean(publicAccess)));
}
// 2. ak and sk
String ak = hiveConf.get(DataLakeConfig.CATALOG_ACCESS_KEY_ID);
@@ -296,34 +299,36 @@ public class PropertyConverter {
if (Strings.isNullOrEmpty(uid)) {
throw new IllegalArgumentException("Required dlf property: " +
DLFProperties.UID);
}
- props.put(DataLakeConfig.CATALOG_ENDPOINT,
props.get(DLFProperties.ENDPOINT));
+ String endpoint = props.get(DLFProperties.ENDPOINT);
+ props.put(DataLakeConfig.CATALOG_ENDPOINT, endpoint);
+ props.put(DataLakeConfig.CATALOG_REGION_ID,
props.getOrDefault(DLFProperties.REGION,
+ S3Properties.getRegionOfEndpoint(endpoint)));
props.put(DataLakeConfig.CATALOG_PROXY_MODE,
props.getOrDefault(DLFProperties.PROXY_MODE, "DLF_ONLY"));
props.put(DataLakeConfig.CATALOG_ACCESS_KEY_ID,
credential.getAccessKey());
props.put(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET,
credential.getSecretKey());
- props.put(DLFProperties.Site.ACCESS_PUBLIC,
props.get(DLFProperties.ACCESS_PUBLIC));
+ props.put(DLFProperties.Site.ACCESS_PUBLIC,
props.getOrDefault(DLFProperties.ACCESS_PUBLIC, "false"));
}
String uid = props.get(DataLakeConfig.CATALOG_USER_ID);
if (Strings.isNullOrEmpty(uid)) {
throw new IllegalArgumentException("Required dlf property: " +
DataLakeConfig.CATALOG_USER_ID);
}
// convert to s3 client property
- if
(props.containsKey(props.get(DataLakeConfig.CATALOG_ACCESS_KEY_ID))) {
- props.put(S3Properties.ACCESS_KEY,
props.get(DataLakeConfig.CATALOG_ACCESS_KEY_ID));
+ if (credential.isWhole()) {
+ props.put(S3Properties.ACCESS_KEY, credential.getAccessKey());
+ props.put(S3Properties.SECRET_KEY, credential.getSecretKey());
}
- if
(props.containsKey(props.get(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET))) {
- props.put(S3Properties.SECRET_KEY,
props.get(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET));
+ if (credential.isTemporary()) {
+ props.put(S3Properties.SESSION_TOKEN,
credential.getSessionToken());
}
String publicAccess =
props.getOrDefault(DLFProperties.Site.ACCESS_PUBLIC, "false");
- String region = props.get(DataLakeConfig.CATALOG_REGION_ID);
- String endpoint = props.getOrDefault(DataLakeConfig.CATALOG_ENDPOINT,
- getDLFEndpoint(region, Boolean.parseBoolean(publicAccess)));
+ String region = props.getOrDefault(DataLakeConfig.CATALOG_REGION_ID,
props.get(DLFProperties.REGION));
if (!Strings.isNullOrEmpty(region)) {
props.put(S3Properties.REGION, "oss-" + region);
- props.put(S3Properties.ENDPOINT, endpoint);
+ props.put(S3Properties.ENDPOINT, getOssEndpoint(region,
Boolean.parseBoolean(publicAccess)));
}
}
- private static String getDLFEndpoint(String region, boolean publicAccess) {
+ private static String getOssEndpoint(String region, boolean publicAccess) {
String prefix = "http://oss-";;
String suffix = ".aliyuncs.com";
if (!publicAccess) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java
index 4ec0ec78a5..dd0682114d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java
@@ -19,6 +19,10 @@ package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
+import com.aliyun.datalake.metastore.common.DataLakeConfig;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
import java.util.Map;
public class DLFProperties extends BaseProperties {
@@ -30,12 +34,27 @@ public class DLFProperties extends BaseProperties {
public static final String SECRET_KEY = "dlf.secret_key";
public static final String REGION = "dlf.region";
public static final String SESSION_TOKEN = "dlf.session_token";
+ public static final List<String> META_KEYS =
ImmutableList.of(DataLakeConfig.CATALOG_PROXY_MODE,
+ Site.ACCESS_PUBLIC,
+ DataLakeConfig.CATALOG_USER_ID,
+ DataLakeConfig.CATALOG_CREATE_DEFAULT_DB,
+ DataLakeConfig.CATALOG_ENDPOINT,
+ DataLakeConfig.CATALOG_REGION_ID,
+ DataLakeConfig.CATALOG_SECURITY_TOKEN,
+ DataLakeConfig.CATALOG_ACCESS_KEY_SECRET,
+ DataLakeConfig.CATALOG_ACCESS_KEY_ID);
public static class Site {
public static final String ACCESS_PUBLIC = "dlf.catalog.accessPublic";
}
public static CloudCredential getCredential(Map<String, String> props) {
- return getCloudCredential(props, ACCESS_KEY, SECRET_KEY,
SESSION_TOKEN);
+ CloudCredential credential = getCloudCredential(props, ACCESS_KEY,
SECRET_KEY, SESSION_TOKEN);
+ if (!credential.isWhole()) {
+ credential = getCloudCredential(props,
DataLakeConfig.CATALOG_ACCESS_KEY_ID,
+ DataLakeConfig.CATALOG_ACCESS_KEY_SECRET,
+ DataLakeConfig.CATALOG_SECURITY_TOKEN);
+ }
+ return credential;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
index 3765a589df..a27ccd88de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
@@ -56,7 +56,7 @@ public class FileSystemFactory {
// TODO: need optimize the method. the conf is converted many times.
Map<String, String> properties = new HashMap<>();
conf.iterator().forEachRemaining(e -> properties.put(e.getKey(),
e.getValue()));
- if (location.startsWith(FeConstants.FS_PREFIX_S3)) {
+ if (location.startsWith(FeConstants.FS_PREFIX_S3) ||
location.startsWith(FeConstants.FS_PREFIX_OBS)) {
return new S3FileSystem(properties);
} else if (location.startsWith(FeConstants.FS_PREFIX_HDFS) ||
location.startsWith(FeConstants.FS_PREFIX_GFS)) {
return new DFSFileSystem(properties);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index dcd4b3c3e6..7862347c19 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -312,7 +312,7 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals(properties.size(), 11);
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(hdProps.size(), 15);
+ Assertions.assertEquals(hdProps.size(), 16);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
