This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new ddcec43d104 [Fix](tvf) Pass through user-defined properties (#39285)
ddcec43d104 is described below
commit ddcec43d1043cebf432120961e9d0e6a67894ae5
Author: Tiewei Fang <43782773+bepppo...@users.noreply.github.com>
AuthorDate: Wed Aug 14 21:12:15 2024 +0800
[Fix](tvf) Pass through user-defined properties (#39285)
bp: #35515
Previously, irrelevant properties passed by users when using TVF were
directly ignored by Doris.
Now, we retain and pass these additional user-defined properties to the
S3 SDK.
---
.../tablefunction/HdfsTableValuedFunction.java | 7 ++-
.../doris/tablefunction/S3TableValuedFunction.java | 9 +--
.../property/PropertyPassThroughTest.java | 68 ++++++++++++++++++++++
3 files changed, 78 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
index 051706ae474..bfc21e0b8fa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.analysis.StorageBackend.StorageType;
import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.URI;
import org.apache.doris.thrift.TFileType;
@@ -71,8 +72,10 @@ public class HdfsTableValuedFunction extends
ExternalFileTableValuedFunction {
locationProperties.put(HdfsResource.HADOOP_FS_NAME,
uri.getScheme() + "://" + uri.getAuthority());
}
- // 4. parse file
- parseFile();
+ if (!FeConstants.runningUnitTest) {
+ // 4. parse file
+ parseFile();
+ }
}
// =========== implement abstract methods of
ExternalFileTableValuedFunction =================
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
index bf87b0c8ba8..eb9592d6f7e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
@@ -78,16 +78,17 @@ public class S3TableValuedFunction extends
ExternalFileTableValuedFunction {
}
checkNecessaryS3Properties(otherProps);
CloudCredentialWithEndpoint credential = new
CloudCredentialWithEndpoint(endpoint,
- otherProps.get(S3Properties.REGION),
- otherProps.get(S3Properties.ACCESS_KEY),
- otherProps.get(S3Properties.SECRET_KEY));
+ getOrDefaultAndRemove(otherProps, S3Properties.REGION, ""),
+ getOrDefaultAndRemove(otherProps, S3Properties.ACCESS_KEY, ""),
+ getOrDefaultAndRemove(otherProps, S3Properties.SECRET_KEY,
""));
if (otherProps.containsKey(S3Properties.SESSION_TOKEN)) {
-
credential.setSessionToken(otherProps.get(S3Properties.SESSION_TOKEN));
+ credential.setSessionToken(getOrDefaultAndRemove(otherProps,
S3Properties.SESSION_TOKEN, ""));
}
locationProperties = S3Properties.credentialToMap(credential);
locationProperties.put(PropertyConverter.USE_PATH_STYLE, usePathStyle);
locationProperties.putAll(S3ClientBEProperties.getBeFSProperties(locationProperties));
+ locationProperties.putAll(otherProps);
if (forceVirtualHosted) {
filePath = NAME + S3URI.SCHEME_DELIM + virtualBucket +
S3URI.PATH_DELIM
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
new file mode 100644
index 00000000000..32a212c5cf7
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property;
+
+import org.apache.doris.analysis.SelectStmt;
+import org.apache.doris.analysis.TableValuedFunctionRef;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.tablefunction.HdfsTableValuedFunction;
+import org.apache.doris.tablefunction.S3TableValuedFunction;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class PropertyPassThroughTest extends TestWithFeService {
+ @Test
+ public void testS3TVFPropertiesPassThrough() throws Exception {
+ FeConstants.runningUnitTest = true;
+ String queryOld = "select * from s3(\n"
+ + " 'uri' =
'http://s3.us-east-1.amazonaws.com/my-bucket/test.parquet',\n"
+ + " 'access_key' = 'akk',\n"
+ + " 'secret_key' = 'skk',\n"
+ + " 'region' = 'us-east-1',\n"
+ + " 'format' = 'parquet',\n"
+ + " 'fs.s3a.list.version' = '1',\n"
+ + " 'test_property' = 'test',\n"
+ + " 'use_path_style' = 'true'\n"
+ + ") limit 10;";
+ SelectStmt analyzedStmt = createStmt(queryOld);
+ Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1);
+ TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef)
analyzedStmt.getTableRefs().get(0);
+ S3TableValuedFunction s3Tvf = (S3TableValuedFunction)
oldFuncTable.getTableFunction();
+
Assertions.assertTrue(s3Tvf.getBrokerDesc().getProperties().containsKey("fs.s3a.list.version"));
+
Assertions.assertTrue(s3Tvf.getBrokerDesc().getProperties().containsKey("test_property"));
+ }
+
+ @Test
+ public void testHdfsTVFPropertiesPassThrough() throws Exception {
+ FeConstants.runningUnitTest = true;
+ String queryOld = "select * from hdfs(\n"
+ + " 'uri' =
'hdfs://HDFS11111871/path/example_table/country=USA/city=NewYork/000000_0',\n"
+ + " 'hadoop.username' = 'hadoop',\n"
+ + " 'path_partition_keys' = 'country,city',\n"
+ + " 'format' = 'orc',\n"
+ + " 'test_property' = 'test'\n"
+ + ") limit 10;";
+ SelectStmt analyzedStmt = createStmt(queryOld);
+ Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1);
+ TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef)
analyzedStmt.getTableRefs().get(0);
+ HdfsTableValuedFunction hdfsTvf = (HdfsTableValuedFunction)
oldFuncTable.getTableFunction();
+
Assertions.assertTrue(hdfsTvf.getBrokerDesc().getProperties().containsKey("test_property"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
