This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 219dc5654f5120147d0a3f0b46b85fca725e68b0
Author: Tiewei Fang <43782773+bepppo...@users.noreply.github.com>
AuthorDate: Mon Aug 26 13:01:57 2024 +0800
[enhencement](hive-catalog) add `hive.recursive_directories_table` and
`hive.ignore_absent_partitions` configs for HiveCatalog (#39494)
Add 2 configs for Hive Catalog:
1. `hive.recursive_directories_table`: Enable reading data from
subdirectories of table or partition locations. If disabled,
subdirectories are ignored. Default value is `false`.
2. `hive.ignore_absent_partitions`: Ignore partitions when the file
system location does not exist rather than failing the query. This skips
data that may be expected to be part of the table. Default value is
`true`.
---
.../hive_config_test/create_table.hql | 31 ++++++
.../data/multi_catalog/hive_config_test/run.sh | 14 +++
.../doris/datasource/hive/HiveMetaStoreCache.java | 8 +-
.../external_table_p0/hive/hive_config_test.out | 33 ++++++
.../external_table_p0/hive/hive_config_test.groovy | 124 +++++++++++++++++++++
5 files changed, 209 insertions(+), 1 deletion(-)
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_config_test/create_table.hql
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_config_test/create_table.hql
new file mode 100644
index 00000000000..2f193a2e3c1
--- /dev/null
+++
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_config_test/create_table.hql
@@ -0,0 +1,31 @@
+create database if not exists default;
+use default;
+
+CREATE TABLE `hive_recursive_directories_table`(
+ `id` int,
+ `name` string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
+LOCATION
+ '/user/doris/suites/default/hive_recursive_directories_table';
+
+
+CREATE TABLE `hive_ignore_absent_partitions_table`(
+ `id` int,
+ `name` string)
+PARTITIONED BY (country STRING, city STRING)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
+LOCATION
+ '/user/doris/suites/default/hive_ignore_absent_partitions_table';
+
+ALTER TABLE hive_ignore_absent_partitions_table ADD PARTITION (country='USA',
city='NewYork');
+ALTER TABLE hive_ignore_absent_partitions_table ADD PARTITION
(country='India', city='Delhi');
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_config_test/run.sh
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_config_test/run.sh
new file mode 100755
index 00000000000..7fc3d0555da
--- /dev/null
+++
b/docker/thirdparties/docker-compose/hive/scripts/data/multi_catalog/hive_config_test/run.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -x
+
+CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+## mkdir and put data to hdfs
+hadoop fs -mkdir -p /user/doris/suites/default/hive_recursive_directories_table
+hadoop fs -mkdir -p
/user/doris/suites/default/hive_ignore_absent_partitions_table
+
+# create table
+hive -f "${CUR_DIR}"/create_table.hql
+
+hadoop fs -rm -r
/user/doris/suites/default/hive_ignore_absent_partitions_table/country=India
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 312f2382b0d..3f448ca2195 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -364,7 +364,9 @@ public class HiveMetaStoreCache {
// So we need to recursively list data location.
//
https://blog.actorsfit.com/a?ID=00550-ce56ec63-1bff-4b0c-a6f7-447b93efaa31
List<RemoteFile> remoteFiles = new ArrayList<>();
- Status status = fs.listFiles(location, true, remoteFiles);
+ boolean isRecursiveDirectories = Boolean.valueOf(
+
catalog.getProperties().getOrDefault("hive.recursive_directories", "false"));
+ Status status = fs.listFiles(location, isRecursiveDirectories,
remoteFiles);
if (status.ok()) {
for (RemoteFile remoteFile : remoteFiles) {
String srcPath = remoteFile.getPath().toString();
@@ -380,6 +382,10 @@ public class HiveMetaStoreCache {
// Hive doesn't aware that the removed partition is missing.
// Here is to support this case without throw an exception.
LOG.warn(String.format("File %s not exist.", location));
+ if (!Boolean.valueOf(catalog.getProperties()
+ .getOrDefault("hive.ignore_absent_partitions", "true"))) {
+ throw new UserException("Partition location does not exist: "
+ location);
+ }
} else {
throw new RuntimeException(status.getErrMsg());
}
diff --git a/regression-test/data/external_table_p0/hive/hive_config_test.out
b/regression-test/data/external_table_p0/hive/hive_config_test.out
new file mode 100644
index 00000000000..1a000281dfc
--- /dev/null
+++ b/regression-test/data/external_table_p0/hive/hive_config_test.out
@@ -0,0 +1,33 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !check_outfile --
+1 doris
+2 nereids
+
+-- !check_outfile --
+1 doris
+2 nereids
+
+-- !check_outfile --
+1 doris
+2 nereids
+
+-- !1 --
+1 doris
+2 nereids
+
+-- !2 --
+1 doris
+1 doris
+1 doris
+2 nereids
+2 nereids
+2 nereids
+
+-- !check_outfile --
+1 doris
+2 nereids
+
+-- !3 --
+1 doris USA NewYork
+2 nereids USA NewYork
+
diff --git
a/regression-test/suites/external_table_p0/hive/hive_config_test.groovy
b/regression-test/suites/external_table_p0/hive/hive_config_test.groovy
new file mode 100644
index 00000000000..e75f3f5fbba
--- /dev/null
+++ b/regression-test/suites/external_table_p0/hive/hive_config_test.groovy
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("hive_config_test",
"p0,external,hive,external_docker,external_docker_hive") {
+ String db_name = "regression_test_external_table_p0_hive"
+ String internal_table = "hive_config_test"
+ String catalog_name = "docker_hive"
+
+ // create table and insert
+ sql """ DROP TABLE IF EXISTS ${internal_table} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${internal_table} (
+ `id` INT NOT NULL,
+ `name` STRING NOT NULL
+ )
+ DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
+ """
+ // insert data into interal table
+ sql """ INSERT INTO ${internal_table} VALUES (1, 'doris'), (2, 'nereids');
"""
+
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("diable Hive test.")
+ return;
+ }
+
+ for (String hivePrefix : ["hive2"]) {
+ String hdfs_port = context.config.otherConfigs.get(hivePrefix +
"HdfsPort")
+ String hms_port = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}"
+ // It's okay to use random `hdfsUser`, but can not be empty.
+ def hdfsUserName = "doris"
+
+
+ def test_outfile = {format, uri ->
+ def res = sql """
+ SELECT * FROM internal.${db_name}.${internal_table} t ORDER BY
id
+ INTO OUTFILE "${defaultFS}${uri}"
+ FORMAT AS ${format}
+ PROPERTIES (
+ "fs.defaultFS"="${defaultFS}",
+ "hadoop.username" = "${hdfsUserName}"
+ );
+ """
+
+ def outfile_url = res[0][3]
+ // check data correctness
+ order_qt_check_outfile """ select * from hdfs(
+ "uri" = "${outfile_url}.${format}",
+ "hadoop.username" = "${hdfsUserName}",
+ "format" = "${format}");
+ """
+ }
+
+
+ // 1. test hive.recursive_directories table config
+ test_outfile("orc",
"/user/doris/suites/default/hive_recursive_directories_table/exp_")
+ test_outfile("orc",
"/user/doris/suites/default/hive_recursive_directories_table/1/exp_")
+ test_outfile("orc",
"/user/doris/suites/default/hive_recursive_directories_table/2/exp_")
+
+ // test hive.recursive_directories_table = false
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+ 'hive.recursive_directories' = 'false'
+ );"""
+ sql """use `${catalog_name}`.`default`"""
+ order_qt_1 """ select * from hive_recursive_directories_table order by
id;"""
+
+ // test hive.recursive_directories_table = true
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+ 'hive.recursive_directories' = 'true'
+ );"""
+ sql """ use `${catalog_name}`.`default` """
+ order_qt_2 """ select * from hive_recursive_directories_table order by
id; """
+
+ // 2. test hive.ignore_absent_partitions-table
+ test_outfile("orc",
"/user/doris/suites/default/hive_ignore_absent_partitions_table/country=USA/city=NewYork/exp_")
+
+ // test 'hive.ignore_absent_partitions' = 'true'
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+ 'hive.ignore_absent_partitions' = 'true'
+ );"""
+ sql """use `${catalog_name}`.`default`"""
+ order_qt_3 """ select * from hive_ignore_absent_partitions_table order
by id;"""
+
+
+ // 'hive.ignore_absent_partitions' = 'false'
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+ 'hive.ignore_absent_partitions' = 'false'
+ );"""
+ sql """use `${catalog_name}`.`default`"""
+ test {
+ sql """ select * from hive_ignore_absent_partitions_table order by
id;"""
+
+ exception "Partition location does not exist"
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
