This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 91296ac75d3 branch-3.1: [test](iceberg) add iceberg rest obs gcs hdfs
test #56076 (#56144)
91296ac75d3 is described below
commit 91296ac75d3ae7804ec33fa0109d0aea8b60c334
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Sep 27 19:24:03 2025 -0700
branch-3.1: [test](iceberg) add iceberg rest obs gcs hdfs test #56076
(#56144)
Cherry-picked from #56076
Co-authored-by: zy-kkk <[email protected]>
---
.../iceberg-rest/docker-compose.yaml.tpl | 130 ++++++++++++++++++++-
.../iceberg-rest/iceberg-rest_settings.env | 40 ++++++-
.../property/metastore/IcebergRestProperties.java | 2 +-
regression-test/conf/regression-conf.groovy | 4 +
.../pipeline/external/conf/regression-conf.groovy | 4 +
.../nonConcurrent/conf/regression-conf.groovy | 4 +
.../pipeline/p0/conf/regression-conf.groovy | 4 +
.../iceberg_rest_s3_storage_vended_test.groovy | 0
.../iceberg_rest_storage_test.groovy} | 81 ++++++++++++-
9 files changed, 263 insertions(+), 6 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/iceberg-rest/docker-compose.yaml.tpl
b/docker/thirdparties/docker-compose/iceberg-rest/docker-compose.yaml.tpl
index 1c4b4f967d4..f9ecde13f0c 100644
--- a/docker/thirdparties/docker-compose/iceberg-rest/docker-compose.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg-rest/docker-compose.yaml.tpl
@@ -99,6 +99,134 @@ services:
networks:
- ${CONTAINER_UID}iceberg-rest
+ # OBS Catalog
+ iceberg-rest-obs:
+ image: apache/iceberg-rest-fixture:1.9.2
+ container_name: ${CONTAINER_UID}iceberg-rest-obs
+ ports:
+ - "${ICEBERG_REST_OBS_PORT}:8181"
+ environment:
+ - AWS_ACCESS_KEY_ID=${OBSAk}
+ - AWS_SECRET_ACCESS_KEY=${OBSSk}
+ - AWS_REGION=${OBSRegion}
+ - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog
+ - CATALOG_URI=jdbc:sqlite:/tmp/obs_catalog.db
+ - CATALOG_JDBC_USER=user
+ - CATALOG_JDBC_PASSWORD=password
+ - CATALOG_WAREHOUSE=s3://doris-build/iceberg_rest_warehouse/
+ - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+ - CATALOG_S3_ENDPOINT=https://${OBSEndpoint}
+ - CATALOG_S3_REGION=${OBSRegion}
+ - CATALOG_S3_PATH__STYLE__ACCESS=false
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:8181/v1/config";]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+ networks:
+ - ${CONTAINER_UID}iceberg-rest
+
+ # GCS Catalog
+ iceberg-rest-gcs:
+ image: apache/iceberg-rest-fixture:1.9.2
+ container_name: ${CONTAINER_UID}iceberg-rest-gcs
+ ports:
+ - "${ICEBERG_REST_GCS_PORT}:8181"
+ environment:
+ - AWS_ACCESS_KEY_ID=${GCSAk}
+ - AWS_SECRET_ACCESS_KEY=${GCSSk}
+ - AWS_REGION=auto
+ - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog
+ - CATALOG_URI=jdbc:sqlite:/tmp/gcs_catalog.db
+ - CATALOG_JDBC_USER=user
+ - CATALOG_JDBC_PASSWORD=password
+ -
CATALOG_WAREHOUSE=s3://selectdb-qa-datalake-test/iceberg_rest_warehouse/
+ - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+ - CATALOG_S3_ENDPOINT=https://${GCSEndpoint}
+ - CATALOG_S3_REGION=auto
+ - CATALOG_S3_PATH__STYLE__ACCESS=true
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:8181/v1/config";]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+ networks:
+ - ${CONTAINER_UID}iceberg-rest
+
+ # Built-in HDFS NameNode
+ hdfs-namenode:
+ image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
+ container_name: ${CONTAINER_UID}iceberg-hdfs-namenode
+ restart: always
+ ports:
+ - "${HDFS_NAMENODE_WEBUI_PORT}:9870"
+ - "${HDFS_NAMENODE_RPC_PORT}:8020"
+ environment:
+ - CLUSTER_NAME=iceberg-hdfs
+ - CORE_CONF_fs_defaultFS=hdfs://hdfs-namenode:8020
+ - CORE_CONF_hadoop_http_staticuser_user=root
+ - CORE_CONF_hadoop_proxyuser_hue_hosts=*
+ - CORE_CONF_hadoop_proxyuser_hue_groups=*
+ -
CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
+ - HDFS_CONF_dfs_webhdfs_enabled=true
+ - HDFS_CONF_dfs_permissions_enabled=false
+ - HDFS_CONF_dfs_namenode_rpc_address=hdfs-namenode:8020
+ - HDFS_CONF_dfs_namenode_http_address=0.0.0.0:9870
+ - HDFS_CONF_dfs_replication=1
+ - HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
+ -
HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+ networks:
+ - ${CONTAINER_UID}iceberg-rest
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:9870/";]
+ interval: 30s
+ timeout: 10s
+ retries: 5
+
+ # Built-in HDFS DataNode
+ hdfs-datanode:
+ image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
+ container_name: ${CONTAINER_UID}iceberg-hdfs-datanode
+ restart: always
+ ports:
+ - "${HDFS_DATANODE_WEBUI_PORT}:9864"
+ environment:
+ - CORE_CONF_fs_defaultFS=hdfs://hdfs-namenode:8020
+ - CORE_CONF_hadoop_http_staticuser_user=root
+ - CORE_CONF_hadoop_proxyuser_hue_hosts=*
+ - CORE_CONF_hadoop_proxyuser_hue_groups=*
+ -
CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
+ - HDFS_CONF_dfs_webhdfs_enabled=true
+ - HDFS_CONF_dfs_permissions_enabled=false
+ - HDFS_CONF_dfs_datanode_data_dir=/hadoop/dfs/data
+ - HDFS_CONF_dfs_replication=1
+ - SERVICE_PRECONDITION=hdfs-namenode:9870
+ networks:
+ - ${CONTAINER_UID}iceberg-rest
+ depends_on:
+ hdfs-namenode:
+ condition: service_healthy
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:9864/";]
+ interval: 30s
+ timeout: 10s
+ retries: 5
+
+ # HDFS Catalog (using built-in HDFS)
+ iceberg-rest-hdfs:
+ image: tabulario/iceberg-rest:1.6.0
+ container_name: ${CONTAINER_UID}iceberg-rest-hdfs
+ ports:
+ - "${ICEBERG_REST_HDFS_PORT}:8181"
+ environment:
+ -
CATALOG_WAREHOUSE=hdfs://${LOCAL_IP}:${HDFS_NAMENODE_RPC_PORT}/user/hive/iceberg_rest_warehouse/
+ - CATALOG_IO__IMPL=org.apache.iceberg.hadoop.HadoopFileIO
+ networks:
+ - ${CONTAINER_UID}iceberg-rest
+ depends_on:
+ - hdfs-namenode
+ - hdfs-datanode
+
networks:
${CONTAINER_UID}iceberg-rest:
- driver: bridge
\ No newline at end of file
+ driver: bridge
diff --git
a/docker/thirdparties/docker-compose/iceberg-rest/iceberg-rest_settings.env
b/docker/thirdparties/docker-compose/iceberg-rest/iceberg-rest_settings.env
index 2dcf5494f18..b0a075ac743 100644
--- a/docker/thirdparties/docker-compose/iceberg-rest/iceberg-rest_settings.env
+++ b/docker/thirdparties/docker-compose/iceberg-rest/iceberg-rest_settings.env
@@ -21,6 +21,32 @@
export ICEBERG_REST_S3_PORT=19181
export ICEBERG_REST_OSS_PORT=19182
export ICEBERG_REST_COS_PORT=19183
+export ICEBERG_REST_OBS_PORT=19184
+export ICEBERG_REST_GCS_PORT=19185
+export ICEBERG_REST_HDFS_PORT=19186
+
+# Built-in HDFS Configuration - using special ports to avoid conflicts
+# NameNode WebUI: 20870 (instead of 9870)
+# NameNode RPC: 20020 (instead of 8020)
+# DataNode WebUI: 20864 (instead of 9864)
+export HDFS_NAMENODE_WEBUI_PORT=20870
+export HDFS_NAMENODE_RPC_PORT=20020
+export HDFS_DATANODE_WEBUI_PORT=20864
+
+# Auto-detect local machine IP for HDFS access
+# This ensures both Doris FE and iceberg-rest can access HDFS using the same IP
+if command -v ip >/dev/null 2>&1; then
+ # Linux: get default route interface IP
+ export LOCAL_IP=$(ip route get 1.1.1.1 | grep -oP 'src \K\S+' 2>/dev/null
|| echo "127.0.0.1")
+elif command -v ifconfig >/dev/null 2>&1; then
+ # macOS: get en0 interface IP
+ export LOCAL_IP=$(ifconfig en0 | grep 'inet ' | awk '{print $2}'
2>/dev/null || echo "127.0.0.1")
+else
+ # Fallback to localhost
+ export LOCAL_IP="127.0.0.1"
+fi
+
+echo "Detected local IP for HDFS: $LOCAL_IP"
# AWS S3 Configuration
export AWSAk="*****************"
@@ -38,4 +64,16 @@ export OSSRegion="cn-beijing"
export COSAk="*****************"
export COSSk="*****************"
export COSEndpoint="cos.ap-beijing.myqcloud.com"
-export COSRegion="ap-beijing"
\ No newline at end of file
+export COSRegion="ap-beijing"
+
+# HUAWEI Cloud OBS Configuration
+export OBSAk="*****************"
+export OBSSk="*****************"
+export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com"
+export OBSRegion="cn-north-4"
+
+# Google Cloud Storage Configuration
+export GCSAk='*****************'
+export GCSSk='*****************'
+export GCSEndpoint='storage.googleapis.com'
+export GCSRegion='asia-east2'
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
index 6a93e72bdd5..ed9abc84e9f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
@@ -163,7 +163,7 @@ public class IcebergRestProperties extends
AbstractIcebergProperties {
@Override
public Catalog initCatalog(String catalogName, Map<String, String>
catalogProps,
- List<StorageProperties> storagePropertiesList) {
+ List<StorageProperties> storagePropertiesList) {
Map<String, String> fileIOProperties = Maps.newHashMap();
Configuration conf = new Configuration();
toFileIOProperties(storagePropertiesList, fileIOProperties, conf);
diff --git a/regression-test/conf/regression-conf.groovy
b/regression-test/conf/regression-conf.groovy
index a687a3c9fdf..ac6568c7646 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -234,6 +234,10 @@ iceberg_minio_port=19001
iceberg_rest_uri_port_s3=19181
iceberg_rest_uri_port_oss=19182
iceberg_rest_uri_port_cos=19183
+iceberg_rest_uri_port_obs=19184
+iceberg_rest_uri_port_gcs=19185
+iceberg_rest_uri_port_hdfs=19186
+iceberg_rest_hdfs_port=20020
// polaris rest catalog config
polaris_rest_uri_port=20181
diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy
b/regression-test/pipeline/external/conf/regression-conf.groovy
index 8b513458d0e..5b5dc431bb6 100644
--- a/regression-test/pipeline/external/conf/regression-conf.groovy
+++ b/regression-test/pipeline/external/conf/regression-conf.groovy
@@ -131,6 +131,10 @@ iceberg_rest_uri_port=18181
iceberg_rest_uri_port_s3=19181
iceberg_rest_uri_port_oss=19182
iceberg_rest_uri_port_cos=19183
+iceberg_rest_uri_port_obs=19184
+iceberg_rest_uri_port_gcs=19185
+iceberg_rest_uri_port_hdfs=19186
+iceberg_rest_hdfs_port=20020
iceberg_minio_port=19001
enableIcebergTest=true
diff --git a/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy
b/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy
index 922fe374a25..ac840ea3205 100644
--- a/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy
+++ b/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy
@@ -122,6 +122,10 @@ iceberg_minio_port=19001
iceberg_rest_uri_port_s3=19181
iceberg_rest_uri_port_oss=19182
iceberg_rest_uri_port_cos=19183
+iceberg_rest_uri_port_obs=19184
+iceberg_rest_uri_port_gcs=19185
+iceberg_rest_uri_port_hdfs=19186
+iceberg_rest_hdfs_port=20020
// polaris rest catalog config
polaris_rest_uri_port=20181
diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy
b/regression-test/pipeline/p0/conf/regression-conf.groovy
index 52ee29f0926..189882d8c0a 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -140,6 +140,10 @@ iceberg_minio_port=19001
iceberg_rest_uri_port_s3=19181
iceberg_rest_uri_port_oss=19182
iceberg_rest_uri_port_cos=19183
+iceberg_rest_uri_port_obs=19184
+iceberg_rest_uri_port_gcs=19185
+iceberg_rest_uri_port_hdfs=19186
+iceberg_rest_hdfs_port=20020
// polaris rest catalog config
polaris_rest_uri_port=20181
diff --git
a/regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_vended_test.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_rest_s3_storage_vended_test.groovy
similarity index 100%
rename from
regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_vended_test.groovy
rename to
regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_rest_s3_storage_vended_test.groovy
diff --git
a/regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_test.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_rest_storage_test.groovy
similarity index 78%
rename from
regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_test.groovy
rename to
regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_rest_storage_test.groovy
index c8944dcb597..1c9a03ff357 100644
---
a/regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_test.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_rest_storage_test.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("iceberg_rest_s3_storage_test",
"p2,external,iceberg,external_docker,external_docker_iceberg_rest,new_catalog_property")
{
+suite("iceberg_rest_storage_test",
"p2,external,iceberg,external_docker,external_docker_iceberg_rest,new_catalog_property")
{
def testQueryAndInsert = { String catalogProperties, String prefix ->
@@ -289,6 +289,27 @@ suite("iceberg_rest_s3_storage_test",
"p2,external,iceberg,external_docker,exter
'iceberg.rest.uri' = 'http://${externalEnvIp}:${rest_port_cos}',
"""
+ String rest_port_obs =
context.config.otherConfigs.get("iceberg_rest_uri_port_obs")
+ String iceberg_rest_type_prop_obs = """
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'iceberg.rest.uri' = 'http://${externalEnvIp}:${rest_port_obs}',
+ """
+
+ String rest_port_gcs =
context.config.otherConfigs.get("iceberg_rest_uri_port_gcs")
+ String iceberg_rest_type_prop_gcs = """
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'iceberg.rest.uri' = 'http://${externalEnvIp}:${rest_port_gcs}',
+ """
+
+ String rest_port_hdfs =
context.config.otherConfigs.get("iceberg_rest_uri_port_hdfs")
+ String iceberg_rest_type_prop_hdfs = """
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'iceberg.rest.uri' = 'http://${externalEnvIp}:${rest_port_hdfs}',
+ """
+
/*-----S3------*/
String s3_ak = context.config.otherConfigs.get("AWSAk")
String s3_sk = context.config.otherConfigs.get("AWSSk")
@@ -336,16 +357,49 @@ suite("iceberg_rest_s3_storage_test",
"p2,external,iceberg,external_docker,exter
'cos.endpoint' = '${cos_endpoint}'
"""
+ /****************OBS*******************/
+ String obs_ak = context.config.otherConfigs.get("hwYunAk")
+ String obs_sk = context.config.otherConfigs.get("hwYunSk")
+ String obs_parent_path = "doris-build"
+ String obs_endpoint = "https://obs.cn-north-4.myhuaweicloud.com";
+ String obs_region = "cn-north-4"
+ String obs_region_param = """
+ 'obs.region' = '${obs_region}',
+ """
+ String obs_storage_properties = """
+ 'obs.access_key' = '${obs_ak}',
+ 'obs.secret_key' = '${obs_sk}',
+ 'obs.endpoint' = '${obs_endpoint}'
+ """
+
+ /****************GCS*******************/
+ String gcs_ak = context.config.otherConfigs.get("GCSAk")
+ String gcs_sk = context.config.otherConfigs.get("GCSSk")
+ String gcs_parent_path = "selectdb-qa-datalake-test"
+ String gcs_endpoint = "https://storage.googleapis.com";
+ String gcs_storage_properties = """
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}',
+ 'gs.endpoint' = '${gcs_endpoint}'
+ """
+
+ /****************HDFS*******************/
+ String hdfsPort =
context.config.otherConfigs.get("iceberg_rest_hdfs_port")
+ String hdfs_parent_path = "/user/hive/iceberg_rest_warehouse"
+ String hdfs_storage_properties = """
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfsPort}'
+ """
+
// -------- REST on OSS --------
String warehouse = """
- 'warehouse' = 's3://${oss_parent_path}/iceberg_rest_warehouse',
+ 'warehouse' = 'oss://${oss_parent_path}/iceberg_rest_warehouse',
"""
testQueryAndInsert(iceberg_rest_type_prop_oss + warehouse +
oss_storage_properties, "iceberg_rest_on_oss")
testQueryAndInsert(iceberg_rest_type_prop_oss + warehouse +
oss_region_param + oss_storage_properties, "iceberg_rest_on_oss_region")
// -------- REST on COS --------
warehouse = """
- 'warehouse' = 's3://${cos_parent_path}/iceberg_rest_warehouse',
+ 'warehouse' = 'cos://${cos_parent_path}/iceberg_rest_warehouse',
"""
testQueryAndInsert(iceberg_rest_type_prop_cos + warehouse +
cos_storage_properties, "iceberg_rest_on_cos")
testQueryAndInsert(iceberg_rest_type_prop_cos + warehouse +
cos_region_param + cos_storage_properties, "iceberg_rest_on_cos_region")
@@ -361,5 +415,26 @@ suite("iceberg_rest_s3_storage_test",
"p2,external,iceberg,external_docker,exter
"""
testQueryAndInsert(iceberg_rest_type_prop_s3 + warehouse +
s3_storage_properties, "iceberg_rest_on_s3a")
testQueryAndInsert(iceberg_rest_type_prop_s3 + warehouse +
s3_region_param + s3_storage_properties, "iceberg_rest_on_s3a_region")
+
+ // -------- REST on OBS --------
+ warehouse = """
+ 'warehouse' = 'obs://${obs_parent_path}/iceberg_rest_warehouse',
+ """
+ testQueryAndInsert(iceberg_rest_type_prop_obs + warehouse +
obs_storage_properties, "iceberg_rest_on_obs")
+ testQueryAndInsert(iceberg_rest_type_prop_obs + warehouse +
obs_region_param + obs_storage_properties, "iceberg_rest_on_obs_region")
+
+ // -------- REST on GCS --------
+ if(context.config.otherConfigs.get("enableGCS")){
+ warehouse = """
+ 'warehouse' = 'gs://${gcs_parent_path}/iceberg_rest_warehouse',
+ """
+ testQueryAndInsert(iceberg_rest_type_prop_gcs + warehouse +
gcs_storage_properties, "iceberg_rest_on_gcs")
+ }
+
+ // -------- REST on HDFS --------
+ warehouse = """
+ 'warehouse' =
'hdfs://${externalEnvIp}:${hdfsPort}${hdfs_parent_path}',
+ """
+ testQueryAndInsert(iceberg_rest_type_prop_hdfs + warehouse +
hdfs_storage_properties, "iceberg_rest_on_hdfs")
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
