This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new a38ccb6a103 branch-3.1: [Fix](s3) Fix hdfs will throw an exception if
the URL verification fails #56268 (#56320)
a38ccb6a103 is described below
commit a38ccb6a10308830b0e9ec5fddfca40243e26ca5
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Sep 25 17:32:16 2025 +0800
branch-3.1: [Fix](s3) Fix hdfs will throw an exception if the URL
verification fails #56268 (#56320)
Cherry-picked from #56268
Co-authored-by: wudi <[email protected]>
---
.../property/storage/HdfsPropertiesUtils.java | 13 +++++-
.../external_table_p0/tvf/test_s3_tvf_globlist.out | 35 ++++++++++++++
.../tvf/test_s3_tvf_globlist.groovy | 54 ++++++++++++++++++++++
3 files changed, 101 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java
index 7fc091daa64..d7c2f34caca 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java
@@ -23,6 +23,8 @@ import
org.apache.doris.datasource.property.storage.exception.StoragePropertiesE
import com.google.common.base.Strings;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.UnsupportedEncodingException;
import java.net.URI;
@@ -38,6 +40,7 @@ import java.util.Set;
import java.util.stream.Collectors;
public class HdfsPropertiesUtils {
+ private static final Logger LOG =
LogManager.getLogger(HdfsPropertiesUtils.class);
private static final String URI_KEY = "uri";
private static final String STANDARD_HDFS_PREFIX = "hdfs://";
private static final String EMPTY_HDFS_PREFIX = "hdfs:///";
@@ -63,7 +66,15 @@ public class HdfsPropertiesUtils {
if (StringUtils.isBlank(uriStr)) {
return false;
}
- URI uri = URI.create(uriStr);
+ URI uri;
+ try {
+ uri = URI.create(uriStr);
+ } catch (Exception ex) {
+ // The glob syntax of s3 contains {, which will cause an error
here.
+ LOG.warn("Failed to validate uri is hdfs uri, {}",
ex.getMessage());
+ return false;
+ }
+
String schema = uri.getScheme();
if (StringUtils.isBlank(schema)) {
throw new IllegalArgumentException("Invalid uri: " + uriStr + ",
extract schema is null");
diff --git
a/regression-test/data/external_table_p0/tvf/test_s3_tvf_globlist.out
b/regression-test/data/external_table_p0/tvf/test_s3_tvf_globlist.out
new file mode 100644
index 00000000000..22663ec6322
--- /dev/null
+++ b/regression-test/data/external_table_p0/tvf/test_s3_tvf_globlist.out
@@ -0,0 +1,35 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_1 --
+1 Emily 25
+10 Liam 64
+2 Benjamin 35
+3 Olivia 28
+4 Alexander 60
+5 Ava 17
+6 William 69
+7 Sophia 32
+8 James 64
+9 Emma 37
+
+-- !select_2 --
+1 Emily 25
+10 Liam 64
+11 Alexander 34
+12 Isabella 43
+13 Benjamin 56
+14 Sophia 12
+15 Christopher 33
+16 Emma 23
+17 Michael 11
+18 Olivia 38
+19 Daniel 19
+2 Benjamin 35
+20 Ava 28
+3 Olivia 28
+4 Alexander 60
+5 Ava 17
+6 William 69
+7 Sophia 32
+8 James 64
+9 Emma 37
+
diff --git
a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_globlist.groovy
b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_globlist.groovy
new file mode 100644
index 00000000000..29bb4cc63d0
--- /dev/null
+++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_globlist.groovy
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_s3_tvf_globlist", "p0,external") {
+
+ String ak = getS3AK()
+ String sk = getS3SK()
+
+ try {
+ order_qt_select_1 """ SELECT * FROM S3
+ (
+ "uri" =
"s3://${s3BucketName}/regression/load/data/{example_0.csv}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "s3.endpoint" = "${getS3Endpoint()}",
+ "s3.region" = "${getS3Region()}",
+ "s3.access_key" = "${ak}",
+ "s3.secret_key" = "${sk}"
+ )
+ order by c1;
+ """
+ } finally {
+ }
+
+ try {
+ order_qt_select_2 """ SELECT * FROM S3
+ (
+ "uri" =
"s3://${s3BucketName}/regression/load/data/{example_0.csv,example_1.csv}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "s3.endpoint" = "${getS3Endpoint()}",
+ "s3.region" = "${getS3Region()}",
+ "s3.access_key" = "${ak}",
+ "s3.secret_key" = "${sk}"
+ )
+ order by c1;
+ """
+ } finally {
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
