morningman commented on code in PR #18005:
URL: https://github.com/apache/doris/pull/18005#discussion_r1152765690
##########
docs/zh-CN/docs/lakehouse/multi-catalog/hive.md:
##########
@@ -114,23 +116,85 @@ CREATE CATALOG hive PROPERTIES (
);
```
-hive元数据存储在Glue，数据存储在S3，示例如下：
+### Hive On S3
+
+数据存储在S3，示例如下：
```sql
CREATE CATALOG hive PROPERTIES (
"type"="hms",
"hive.metastore.type" = "glue",
- "aws.region" = "us-east-1",
- "aws.glue.access-key" = "ak",
- "aws.glue.secret-key" = "sk",
- "AWS_ENDPOINT" = "s3.us-east-1.amazonaws.com",
- "AWS_REGION" = "us-east-1",
- "AWS_ACCESS_KEY" = "ak",
- "AWS_SECRET_KEY" = "sk",
+ "s3.endpoint" = "s3.us-east-1.amazonaws.com",
+ "s3.access-key" = "ak",
+ "s3.secret-key" = "sk"
"use_path_style" = "true"
);
```
+可选属性：
+
+* s3.connection.maximum： s3最大连接数，默认50
+* s3.connection.request.timeout：s3请求超时时间，默认3000ms
+* s3.connection.timeout： s3连接超时时间，默认1000ms
+
+### Hive On OSS
+
+数据存储在OSS，示例如下：
+
+```sql
+CREATE CATALOG hive PROPERTIES (
+ "type"="hms",
+ "hive.metastore.uris" = "thrift://172.21.0.44:7004",
+ "oss.endpoint" = "oss.oss-cn-beijing.aliyuncs.com",
+ "oss.access-key" = "ak",
+ "oss.secret-key" = "sk"
+);
+```
+
+### Hive On OBS
+
+数据存储在OBS，示例如下：
+
+```sql
+CREATE CATALOG hive PROPERTIES (
+ "type"="hms",
+ "hive.metastore.uris" = "thrift://172.21.0.44:7004",
+ "obs.endpoint" = "obs.cn-north-4.myhuaweicloud.com",
+ "obs.access-key" = "ak",
+ "obs.secret-key" = "sk"
+);
+```
+
+### Hive On COS
+
+数据存储在COS，示例如下：
+
+```sql
+CREATE CATALOG hive PROPERTIES (
+ "type"="hms",
+ "hive.metastore.uris" = "thrift://172.21.0.44:7004",
+ "cos.endpoint" = "cos.ap-beijing.myqcloud.com",
+ "cos.access-key" = "ak",
+ "cos.secret-key" = "sk"
+);
+```
+
+### Hive With Glue
+
+元数据存储在Glue，示例如下：
+
+```sql
+CREATE CATALOG hive PROPERTIES (
+ "type"="hms",
+ "hive.metastore.type" = "glue",
+ "glue.endpoint" = "https://glue.us-east-1.amazonaws.com";,
+ "glue.access-key" = "ak",
+ "glue.secret-key" = "sk"
Review Comment:
数据相关的信息也要给全
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java:
##########
@@ -31,26 +34,27 @@
public class IcebergGlueExternalCatalog extends IcebergExternalCatalog {
+ // As a default placeholder. The path just use for 'create table', query
stmt will not use it.
+ private static final String CHECKED_WAREHOUSE = "s3://doris";
+
public IcebergGlueExternalCatalog(long catalogId, String name, String
resource, Map<String, String> props) {
super(catalogId, name);
+ props = PropertyConverter.convertToMetaProperties(props);
catalogProperty = new CatalogProperty(resource, props);
}
@Override
protected void initLocalObjectsImpl() {
icebergCatalogType = ICEBERG_GLUE;
GlueCatalog glueCatalog = new GlueCatalog();
- // AWSGlueAsync glueClient;
Configuration conf = setGlueProperties(getConfiguration());
glueCatalog.setConf(conf);
// initialize glue catalog
- Map<String, String> catalogProperties =
catalogProperty.getProperties();
- // check AwsProperties.GLUE_CATALOG_ENDPOINT
- String metastoreUris =
catalogProperty.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, "");
- if (StringUtils.isEmpty(metastoreUris)) {
- throw new IllegalArgumentException("Missing glue properties
'warehouse'.");
- }
- catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION,
metastoreUris);
+ Map<String, String> catalogProperties =
catalogProperty.getHadoopProperties();
+ String warehouse =
catalogProperty.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION,
CHECKED_WAREHOUSE);
+ catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
+ catalogProperties.put(AwsProperties.S3FILEIO_ENDPOINT,
+ catalogProperties.getOrDefault(Constants.ENDPOINT,
conf.get(S3Properties.Env.ENDPOINT)));
Review Comment:
Why the key name in `conf` is `S3Properties.Env.ENDPOINT`?
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java:
##########
@@ -31,26 +34,27 @@
public class IcebergGlueExternalCatalog extends IcebergExternalCatalog {
+ // As a default placeholder. The path just use for 'create table', query
stmt will not use it.
+ private static final String CHECKED_WAREHOUSE = "s3://doris";
+
public IcebergGlueExternalCatalog(long catalogId, String name, String
resource, Map<String, String> props) {
super(catalogId, name);
+ props = PropertyConverter.convertToMetaProperties(props);
catalogProperty = new CatalogProperty(resource, props);
}
@Override
protected void initLocalObjectsImpl() {
icebergCatalogType = ICEBERG_GLUE;
GlueCatalog glueCatalog = new GlueCatalog();
- // AWSGlueAsync glueClient;
Configuration conf = setGlueProperties(getConfiguration());
Review Comment:
`setGlueProperties()` is an empty method
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java:
##########
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property;
+
+import org.apache.doris.datasource.property.constants.CosProperties;
+import org.apache.doris.datasource.property.constants.ObsProperties;
+import org.apache.doris.datasource.property.constants.OssProperties;
+import org.apache.doris.datasource.property.constants.S3Properties;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class S3ClientBEProperties {
+
+ public static Map<String, String> getBeFSProperties(Map<String, String>
properties) {
Review Comment:
Add comment to explain the method, better give an example.
##########
docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md:
##########
@@ -82,20 +82,12 @@ CREATE CATALOG glue PROPERTIES (
"type"="iceberg",
"iceberg.catalog.type" = "glue",
"glue.endpoint" = "https://glue.us-east-1.amazonaws.com";,
-"warehouse" = "s3://bucket/warehouse",
-"AWS_ENDPOINT" = "s3.us-east-1.amazonaws.com",
-"AWS_REGION" = "us-east-1",
-"AWS_ACCESS_KEY" = "ak",
-"AWS_SECRET_KEY" = "sk",
-"use_path_style" = "true"
+"glue.access_key" = "ak",
+"glue.secret_key" = "sk"
);
```
-`glue.endpoint`: Glue Endpoint. 参阅：[AWS Glue endpoints and
quotas](https://docs.aws.amazon.com/general/latest/gr/glue.html).
-
-`warehouse`: Glue Warehouse Location. Glue Catalog的根路径，用于指定数据存放位置。
-
-属性详情参见 [Iceberg Glue
Catalog](https://iceberg.apache.org/docs/latest/aws/#glue-catalog)
+Iceberg属性详情参见 [Iceberg Glue
Catalog](https://iceberg.apache.org/docs/latest/aws/#glue-catalog)
Review Comment:
这个是说里面的内容我们都支持么？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
