This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 40f61597cf9 [fix](fe) Disable LOCAL TVF (branch-4.1) (#69593)
40f61597cf9 is described below
commit 40f61597cf912a8506d9b2b46108beefa2db098a
Author: Oliveira <[email protected]>
AuthorDate: Tue Oct 27 14:12:37 2026 +0800
[fix](fe) Disable LOCAL TVF (branch-4.1) (#69593)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #69591, #69528
Problem Summary:
LOCAL TVF can discover files on a backend during analysis. Reject LOCAL
unconditionally before property parsing, backend selection and file
discovery. FILE TVF with local storage delegates to the same entry
point. Explicit LOCAL INSERT and analyzed EXPLAIN INSERT use the same
rejection before sink property validation. No configuration option
enables these paths, including for ADMIN/OPERATOR. The underlying
storage implementation is retained.
Tests assert rejection before backend access and preserve remote
FILE/sink behavior where supported. Existing LOCAL-dependent success
suites explicitly return while the feature is unavailable. Export
completion/error checks remain active; LOCAL export-readback comparisons
are retired. Remote HDFS/S3 partition cases, the existing UUID S3
fixture branch, and ORC value checks through S3 TVF using the original
uploaded fixtures remain active where present on this branch.
### Release note
LOCAL TVF and FILE TVF with local storage are disabled for all users.
There is no configuration option to re-enable them. Explicit LOCAL TVF
INSERT is also disabled.
### Check List (For Author)
- Test
- [x] Unit Test — 8 targeted FE tests passed; zero failures, errors or
skips.
- [x] Regression test — rejection coverage and affected-suite
adaptations included; suite bodies have not been executed locally.
- [ ] Manual test
- [ ] No need to test
Validation: Checkstyle passed for all 5 changed Java files. Changed
Groovy suites passed syntax parsing with Groovy 4.0.19. Whitespace
checks passed. Runtime regression and formal local code review have not
been performed.
- Behavior changed:
- [ ] No.
- [x] Yes. LOCAL admission always fails; no startup opt-in is available.
- Does this need documentation?
- [ ] No.
- [x] Yes. Release note included; operator documentation should reflect
disabled LOCAL support.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../doris/nereids/rules/analysis/BindSink.java | 5 +
.../tablefunction/LocalTableValuedFunction.java | 7 +
.../rules/analysis/BindTVFTableSinkTest.java | 3 +-
.../rules/analysis/LocalTvfSinkDisabledTest.java | 75 +++++++
.../LocalTableValuedFunctionDisabledTest.java | 126 ++++++++++++
.../account_p0/test_local_tvf_disabled.groovy | 31 +++
.../suites/export_p0/test_export_basic.groovy | 183 +----------------
.../suites/export_p0/test_export_csv.groovy | 221 +--------------------
.../export_p0/test_export_data_consistency.groovy | 3 +
.../export_p0/test_export_empty_table.groovy | 25 +--
.../test_export_table_with_label_retry.groovy | 43 +---
.../suites/export_p0/test_export_view.groovy | 159 +--------------
.../export/test_export_external_table.groovy | 173 +---------------
.../tvf/insert/test_insert_into_local_tvf.groovy | 3 +
.../tvf/test_file_scanner_v2_review_fixes.groovy | 3 +
.../tvf/test_file_tvf_local.groovy | 3 +
.../tvf/test_local_tvf_compression.groovy | 3 +
.../test_local_tvf_csv_enclose_consistency.groovy | 3 +
.../tvf/test_local_tvf_enclose.groovy | 3 +
.../tvf/test_local_tvf_lzo.groovy | 3 +
...test_local_tvf_parquet_unsigned_integers.groovy | 3 +
.../tvf/test_local_tvf_with_complex_type.groovy | 3 +
...t_local_tvf_with_complex_type_element_at.groovy | 3 +
...l_tvf_with_complex_type_insertinto_doris.groovy | 3 +
.../tvf/test_parquet_meta_tvf.groovy | 19 +-
.../tvf/test_path_partition_keys.groovy | 49 +----
.../tvf/test_tvf_csv_line_end.groovy | 3 +
...st_array_with_single_row_max_than_uint16.groovy | 63 ++++--
.../complex_types/map_uniq_with_local_tvf.groovy | 59 ++++--
29 files changed, 418 insertions(+), 862 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java
index 1d2580e52fc..e88c0dc2d95 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java
@@ -116,6 +116,7 @@ import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.AutoCloseSessionVariable;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.tablefunction.LocalTableValuedFunction;
import org.apache.doris.thrift.TPartialUpdateNewRowPolicy;
import com.google.common.base.Preconditions;
@@ -659,6 +660,10 @@ public class BindSink implements AnalysisRuleFactory {
"INSERT INTO TVF only supports local/s3/hdfs, but got: " +
tvfName);
}
+ if (tvfName.equals(LocalTableValuedFunction.NAME)) {
+ LocalTableValuedFunction.checkAccess();
+ }
+
// Validate required properties
if (!properties.containsKey("file_path")) {
throw new AnalysisException("TVF sink requires 'file_path'
property");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java
index 4e4bf163145..a63fd79360c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java
@@ -61,6 +61,8 @@ public class LocalTableValuedFunction extends
ExternalFileTableValuedFunction {
private boolean sharedStorage = false;
public LocalTableValuedFunction(Map<String, String> properties) throws
AnalysisException {
+ checkAccess();
+
// 1. analyze common properties
Map<String, String> props = super.parseCommonProperties(properties);
@@ -135,6 +137,11 @@ public class LocalTableValuedFunction extends
ExternalFileTableValuedFunction {
}
}
+ /** Reject LOCAL before property parsing, file discovery or sink binding.
*/
+ public static void checkAccess() {
+ throw new org.apache.doris.nereids.exceptions.AnalysisException("LOCAL
TVF is disabled.");
+ }
+
@Override
public TFileType getTFileType() {
return TFileType.FILE_LOCAL;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java
index b76ba57ddb5..4ee7dcaa94f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java
@@ -38,9 +38,8 @@ class BindTVFTableSinkTest {
void duplicateOutputNamesKeepDistinctPositions() {
ConnectContext context = MemoTestUtils.createConnectContext();
InsertIntoTVFCommand command = (InsertIntoTVFCommand) new
NereidsParser().parseSingle(
- "INSERT INTO local("
+ "INSERT INTO s3("
+ "'file_path'='/tmp/tvf_duplicate_columns_',"
- + "'backend_id'='1',"
+ "'format'='csv') "
+ "SELECT 1 AS x, 2 AS x");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java
new file mode 100644
index 00000000000..142f35b4da2
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/LocalTvfSinkDisabledTest.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.analysis;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
+import
org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.qe.ConnectContext;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+class LocalTvfSinkDisabledTest {
+ @Test
+ void disabledLocalRejectsInsertAndExplainBeforeProperties() {
+ for (boolean authorized : new boolean[] {false, true}) {
+ for (String prefix : new String[] {"", "EXPLAIN "}) {
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> analyzeInsert(prefix + "INSERT INTO
local('format'='csv') SELECT 1", authorized));
+ Assertions.assertTrue(exception.getMessage().contains("LOCAL
TVF is disabled"));
+ }
+ }
+ }
+
+ @Test
+ void disabledLocalKeepsRemoteSinkValidation() {
+ for (String name : new String[] {"s3", "hdfs"}) {
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> analyzeInsert("INSERT INTO " + name +
"('format'='csv') SELECT 1", false));
+ Assertions.assertTrue(exception.getMessage().contains("requires
'file_path'"));
+ }
+ }
+
+ private Plan analyzeInsert(String sql, boolean authorized) {
+ ConnectContext context = MemoTestUtils.createConnectContext();
+ Env env = Mockito.mock(Env.class);
+ AccessControllerManager accessManager =
Mockito.mock(AccessControllerManager.class);
+ Mockito.when(env.getAccessManager()).thenReturn(accessManager);
+ Mockito.when(accessManager.checkGlobalPriv(context,
PrivPredicate.ADMIN)).thenReturn(authorized);
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class,
Mockito.CALLS_REAL_METHODS)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ LogicalPlan parsed = new NereidsParser().parseSingle(sql);
+ if (parsed instanceof ExplainCommand) {
+ parsed = ((ExplainCommand) parsed).getLogicalPlan();
+ }
+ InsertIntoTVFCommand command = (InsertIntoTVFCommand) parsed;
+ return
PlanChecker.from(context).analyze(command.getExplainPlan(context)).getPlan();
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionDisabledTest.java
b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionDisabledTest.java
new file mode 100644
index 00000000000..82a304e9752
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionDisabledTest.java
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.tablefunction;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.rpc.BackendServiceProxy;
+import org.apache.doris.system.SystemInfoService;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedConstruction;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.Map;
+
+public class LocalTableValuedFunctionDisabledTest {
+ private static final Map<String, String> PROPERTIES = ImmutableMap.of(
+ "file_path", "local_disabled_test.csv", "backend_id", "1",
"format", "csv");
+
+ private final Env env = Mockito.mock(Env.class);
+ private final ConnectContext context = Mockito.mock(ConnectContext.class);
+ private final AccessControllerManager accessManager =
Mockito.mock(AccessControllerManager.class);
+ private final SystemInfoService systemInfo =
Mockito.mock(SystemInfoService.class);
+ private final BackendServiceProxy proxy =
Mockito.mock(BackendServiceProxy.class);
+ private MockedStatic<Env> mockedEnv;
+ private MockedStatic<ConnectContext> mockedContext;
+ private MockedStatic<BackendServiceProxy> mockedProxy;
+
+ @BeforeEach
+ public void setUp() {
+ mockedEnv = Mockito.mockStatic(Env.class);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo);
+ mockedContext = Mockito.mockStatic(ConnectContext.class);
+ mockedContext.when(ConnectContext::get).thenReturn(context);
+ mockedProxy = Mockito.mockStatic(BackendServiceProxy.class);
+ mockedProxy.when(BackendServiceProxy::getInstance).thenReturn(proxy);
+ Mockito.when(env.getAccessManager()).thenReturn(accessManager);
+ }
+
+ @AfterEach
+ public void tearDown() {
+ mockedProxy.close();
+ mockedContext.close();
+ mockedEnv.close();
+ }
+
+ @Test
+ public void testLocalDisabledBeforePropertyParsing() {
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> new LocalTableValuedFunction(ImmutableMap.of()));
+ Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is
disabled"));
+ Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
+ }
+
+ @Test
+ public void testFormatsDisabledBeforeFileAccess() {
+ for (String format : new String[] {"csv", "json", "lance"}) {
+ Map<String, String> properties = ImmutableMap.of(
+ "file_path", "local_disabled_test", "backend_id", "1",
"format", format);
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> new LocalTableValuedFunction(properties));
+ Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF
is disabled"));
+ }
+ Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
+ }
+
+ @Test
+ public void testFileLocalDisabledBeforeFileAccess() {
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> new FileTableValuedFunction(PROPERTIES));
+ Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is
disabled"));
+ Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
+ }
+
+ @Test
+ public void testPrivilegedUsersCannotAccessLocalFiles() {
+ for (PrivPredicate privilege : new PrivPredicate[]
{PrivPredicate.ADMIN, PrivPredicate.OPERATOR}) {
+ Mockito.when(accessManager.checkGlobalPriv(context,
privilege)).thenReturn(true);
+ Assertions.assertThrows(AnalysisException.class, () -> new
LocalTableValuedFunction(PROPERTIES));
+ Assertions.assertThrows(AnalysisException.class, () -> new
FileTableValuedFunction(PROPERTIES));
+ }
+ Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
+ }
+
+ @Test
+ public void testFileS3RemainsAvailable() {
+ Map<String, String> properties = ImmutableMap.<String, String>builder()
+ .put("uri", "s3://test-bucket/local_disabled_test.csv")
+ .put("s3.endpoint", "s3.us-east-1.amazonaws.com")
+ .put("s3.region", "us-east-1")
+ .put("s3.access_key", "test-access-key")
+ .put("s3.secret_key", "test-secret-key")
+ .put("format", "csv")
+ .build();
+ try (MockedConstruction<S3TableValuedFunction> mockedS3 =
+ Mockito.mockConstruction(S3TableValuedFunction.class)) {
+ Assertions.assertDoesNotThrow(() -> new
FileTableValuedFunction(properties));
+ Assertions.assertEquals(1, mockedS3.constructed().size());
+ }
+ Mockito.verifyNoInteractions(accessManager, systemInfo, proxy);
+ }
+}
diff --git a/regression-test/suites/account_p0/test_local_tvf_disabled.groovy
b/regression-test/suites/account_p0/test_local_tvf_disabled.groovy
new file mode 100644
index 00000000000..5c4b2c9e7c5
--- /dev/null
+++ b/regression-test/suites/account_p0/test_local_tvf_disabled.groovy
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_local_tvf_disabled", "p0") {
+ for (def functionName : ["LOCAL", "FILE"]) {
+ for (def prefix : ["", "EXPLAIN "]) {
+ test {
+ sql """${prefix}SELECT * FROM ${functionName}(
+ "file_path" = "local_enable_test.csv",
+ "backend_id" = "-1",
+ "format" = "csv"
+ )"""
+ exception "LOCAL TVF is disabled"
+ }
+ }
+ }
+}
diff --git a/regression-test/suites/export_p0/test_export_basic.groovy
b/regression-test/suites/export_p0/test_export_basic.groovy
index c55fde0fac1..8daf920eee0 100644
--- a/regression-test/suites/export_p0/test_export_basic.groovy
+++ b/regression-test/suites/export_p0/test_export_basic.groovy
@@ -161,37 +161,8 @@ suite("test_export_basic", "p0") {
"""
waiting_export.call(db, label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `id` int(11) NULL,
- `Name` string NULL,
- `age` int(11) NULL
- )
- DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
- """
+ // LOCAL TVF readback is disabled; export completion is checked above.
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
-
- qt_select_load1 """ SELECT * FROM ${table_load_name} t ORDER BY id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -219,37 +190,8 @@ suite("test_export_basic", "p0") {
"""
waiting_export.call(db, label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `id` int(11) NULL,
- `Name` string NULL,
- `age` int(11) NULL
- )
- DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
- """
+ // LOCAL TVF readback is disabled; export completion is checked above.
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
-
- qt_select_load2 """ SELECT * FROM ${table_load_name} t ORDER BY id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -277,37 +219,8 @@ suite("test_export_basic", "p0") {
"""
waiting_export.call(db, label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `id` int(11) NULL,
- `Name` string NULL,
- `age` int(11) NULL
- )
- DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
- """
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- qt_select_load3 """ SELECT * FROM ${table_load_name} t ORDER BY id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -335,37 +248,8 @@ suite("test_export_basic", "p0") {
"""
waiting_export.call(db, label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `id` int(11) NULL,
- `Name` string NULL,
- `age` int(11) NULL
- )
- DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
- """
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- qt_select_load4 """ SELECT * FROM ${table_load_name} t ORDER BY id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -437,36 +321,8 @@ suite("test_export_basic", "p0") {
"""
waiting_export.call(db, label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `id` int(11) NULL,
- `Name` string NULL
- )
- DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
- """
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name} (id, Name)
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- qt_select_load6 """ SELECT * FROM ${table_load_name} t ORDER BY id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -495,35 +351,8 @@ suite("test_export_basic", "p0") {
"""
waiting_export.call(db, label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `id` int(11) NULL
- )
- DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
- """
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- qt_select_load7 """ SELECT * FROM ${table_load_name} t ORDER BY id; """
// test label
def label_db = "export_p0_test_label"
diff --git a/regression-test/suites/export_p0/test_export_csv.groovy
b/regression-test/suites/export_p0/test_export_csv.groovy
index dd9f807e69a..4bc5e87bddb 100644
--- a/regression-test/suites/export_p0/test_export_csv.groovy
+++ b/regression-test/suites/export_p0/test_export_csv.groovy
@@ -146,49 +146,8 @@ suite("test_export_csv", "p0") {
"""
waiting_export.call(label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `datetime` DATETIME NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `bool_col` boolean COMMENT "",
- `int_col` int COMMENT "",
- `bigint_col` bigint COMMENT "",
- `largeint_col` largeint COMMENT "",
- `float_col` float COMMENT "",
- `double_col` double COMMENT "",
- `char_col` CHAR(10) COMMENT "",
- `decimal_col` decimal COMMENT "",
- `ipv4_col` ipv4 COMMENT "",
- `ipv6_col` ipv6 COMMENT ""
- )
- DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
- """
+ // LOCAL TVF readback is disabled; export completion is checked above.
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ",");
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
-
- qt_select_load1 """ SELECT * FROM ${table_load_name} t ORDER BY
user_id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -216,50 +175,8 @@ suite("test_export_csv", "p0") {
"""
waiting_export.call(label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `datetime` DATETIME NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `bool_col` boolean COMMENT "",
- `int_col` int COMMENT "",
- `bigint_col` bigint COMMENT "",
- `largeint_col` largeint COMMENT "",
- `float_col` float COMMENT "",
- `double_col` double COMMENT "",
- `char_col` CHAR(10) COMMENT "",
- `decimal_col` decimal COMMENT "",
- `ipv4_col` ipv4 COMMENT "",
- `ipv6_col` ipv6 COMMENT ""
- )
- DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
- """
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = "ab",
- "line_delimiter" = "cc");
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- qt_select_load2 """ SELECT * FROM ${table_load_name} t ORDER BY
user_id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -286,51 +203,8 @@ suite("test_export_csv", "p0") {
"""
waiting_export.call(label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `datetime` DATETIME NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `bool_col` boolean COMMENT "",
- `int_col` int COMMENT "",
- `bigint_col` bigint COMMENT "",
- `largeint_col` largeint COMMENT "",
- `float_col` float COMMENT "",
- `double_col` double COMMENT "",
- `char_col` CHAR(10) COMMENT "",
- `decimal_col` decimal COMMENT "",
- `ipv4_col` ipv4 COMMENT "",
- `ipv6_col` ipv6 COMMENT ""
- )
- DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
- """
+ // LOCAL TVF readback is disabled; export completion is checked above.
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select *
- from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv_with_names",
- "line_delimiter" = "cc",
- "column_separator" = "ab");
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
-
- qt_select_load3 """ SELECT * FROM ${table_load_name} t ORDER BY
user_id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -357,51 +231,8 @@ suite("test_export_csv", "p0") {
"""
waiting_export.call(label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `datetime` DATETIME NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `bool_col` boolean COMMENT "",
- `int_col` int COMMENT "",
- `bigint_col` bigint COMMENT "",
- `largeint_col` largeint COMMENT "",
- `float_col` float COMMENT "",
- `double_col` double COMMENT "",
- `char_col` CHAR(10) COMMENT "",
- `decimal_col` decimal COMMENT "",
- `ipv4_col` ipv4 COMMENT "",
- `ipv6_col` ipv6 COMMENT ""
- )
- DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
- """
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select *
- from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv_with_names_and_types",
- "line_delimiter" = "cc",
- "column_separator" = "ab");
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- qt_select_load4 """ SELECT * FROM ${table_load_name} t ORDER BY
user_id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -427,50 +258,8 @@ suite("test_export_csv", "p0") {
"""
waiting_export.call(label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `datetime` DATETIME NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `bool_col` boolean COMMENT "",
- `int_col` int COMMENT "",
- `bigint_col` bigint COMMENT "",
- `largeint_col` largeint COMMENT "",
- `float_col` float COMMENT "",
- `double_col` double COMMENT "",
- `char_col` CHAR(10) COMMENT "",
- `decimal_col` decimal COMMENT "",
- `ipv4_col` ipv4 COMMENT "",
- `ipv6_col` ipv6 COMMENT ""
- )
- DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
- """
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select *
- from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "compress_type" = "gz");
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- qt_select_load5 """ SELECT * FROM ${table_load_name} t ORDER BY
user_id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
diff --git
a/regression-test/suites/export_p0/test_export_data_consistency.groovy
b/regression-test/suites/export_p0/test_export_data_consistency.groovy
index 45d2be4dfff..78d6eb7c6cb 100644
--- a/regression-test/suites/export_p0/test_export_data_consistency.groovy
+++ b/regression-test/suites/export_p0/test_export_data_consistency.groovy
@@ -22,6 +22,9 @@ import java.nio.file.Files
import java.nio.file.Paths
suite("test_export_data_consistency", "p0") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
// open nereids
sql """ set enable_nereids_planner=true """
sql """ set enable_fallback_to_original_planner=false """
diff --git a/regression-test/suites/export_p0/test_export_empty_table.groovy
b/regression-test/suites/export_p0/test_export_empty_table.groovy
index 76c665d0749..4e0b931bfd5 100644
--- a/regression-test/suites/export_p0/test_export_empty_table.groovy
+++ b/regression-test/suites/export_p0/test_export_empty_table.groovy
@@ -145,9 +145,8 @@ suite("test_export_empty_table", "p0") {
"column_separator" = ",");
"""
- // because we export the empty data,
- // there is no files.
- exception "get file list from backend failed."
+ // LOCAL is rejected before checking exported files.
+ exception "LOCAL TVF is disabled"
}
}
} finally {
@@ -187,9 +186,8 @@ suite("test_export_empty_table", "p0") {
"column_separator" = ",");
"""
- // because we export the empty data,
- // there is no files.
- exception "get file list from backend failed."
+ // LOCAL is rejected before checking exported files.
+ exception "LOCAL TVF is disabled"
}
}
} finally {
@@ -228,9 +226,8 @@ suite("test_export_empty_table", "p0") {
"column_separator" = ",");
"""
- // because we export the empty data,
- // there is no files.
- exception "get file list from backend failed."
+ // LOCAL is rejected before checking exported files.
+ exception "LOCAL TVF is disabled"
}
}
} finally {
@@ -270,9 +267,8 @@ suite("test_export_empty_table", "p0") {
"column_separator" = ",");
"""
- // because we export the empty data,
- // there is no files.
- exception "get file list from backend failed."
+ // LOCAL is rejected before checking exported files.
+ exception "LOCAL TVF is disabled"
}
}
} finally {
@@ -313,9 +309,8 @@ suite("test_export_empty_table", "p0") {
"column_separator" = ",");
"""
- // because we export the empty data,
- // there is no files.
- exception "get file list from backend failed."
+ // LOCAL is rejected before checking exported files.
+ exception "LOCAL TVF is disabled"
}
}
} finally {
diff --git
a/regression-test/suites/export_p0/test_export_table_with_label_retry.groovy
b/regression-test/suites/export_p0/test_export_table_with_label_retry.groovy
index bd7c85f4d8c..18b7e15f6d7 100644
--- a/regression-test/suites/export_p0/test_export_table_with_label_retry.groovy
+++ b/regression-test/suites/export_p0/test_export_table_with_label_retry.groovy
@@ -177,49 +177,8 @@ suite("test_export_table_with_label_retry", "p0") {
waiting_export_expect_success.call(label)
- // check data correctness
- sql """ DROP TABLE IF EXISTS ${table_load_name} """
- sql """
- CREATE TABLE IF NOT EXISTS ${table_load_name} (
- `user_id` LARGEINT NOT NULL COMMENT "用户id",
- `date` DATE NOT NULL COMMENT "数据灌入日期时间",
- `datetime` DATETIME NOT NULL COMMENT "数据灌入日期时间",
- `city` VARCHAR(20) COMMENT "用户所在城市",
- `age` SMALLINT COMMENT "用户年龄",
- `sex` TINYINT COMMENT "用户性别",
- `bool_col` boolean COMMENT "",
- `int_col` int COMMENT "",
- `bigint_col` bigint COMMENT "",
- `largeint_col` largeint COMMENT "",
- `float_col` float COMMENT "",
- `double_col` double COMMENT "",
- `char_col` CHAR(10) COMMENT "",
- `decimal_col` decimal COMMENT "",
- `ipv4_col` ipv4 COMMENT "",
- `ipv6_col` ipv6 COMMENT ""
- )
- DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
- """
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "column_separator" = ",",
- "format" = "csv");
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- qt_select_load1 """ SELECT * FROM ${table_load_name} t ORDER BY
user_id; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
diff --git a/regression-test/suites/export_p0/test_export_view.groovy
b/regression-test/suites/export_p0/test_export_view.groovy
index 1e35483d619..a4a3a24e862 100644
--- a/regression-test/suites/export_p0/test_export_view.groovy
+++ b/regression-test/suites/export_p0/test_export_view.groovy
@@ -184,29 +184,8 @@ suite("test_export_view", "p0") {
waiting_export.call(label)
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_view_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- order_qt_select_load1 """ SELECT * FROM ${table_load_name} t; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -235,29 +214,8 @@ suite("test_export_view", "p0") {
waiting_export.call(label)
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_view_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "column_separator" = ",",
- "format" = "csv_with_names"
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- order_qt_select_load2 """ SELECT * FROM ${table_load_name} t; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -288,29 +246,8 @@ suite("test_export_view", "p0") {
waiting_export.call(label)
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_view_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "column_separator" = ",",
- "format" = "csv_with_names"
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- order_qt_select_load3 """ SELECT * FROM ${table_load_name}; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -342,29 +279,8 @@ suite("test_export_view", "p0") {
waiting_export.call(label)
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select s1,k1, null as k2,k3 from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_view_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "column_separator" = ",",
- "format" = "csv_with_names"
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- order_qt_select_load4 """ SELECT * FROM ${table_load_name} t; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -393,29 +309,8 @@ suite("test_export_view", "p0") {
"""
waiting_export.call(label)
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_view_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "column_separator" = ",",
- "format" = "csv_with_names_and_types"
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- order_qt_select_load5 """ SELECT * FROM ${table_load_name} t; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -442,28 +337,8 @@ suite("test_export_view", "p0") {
"""
waiting_export.call(label)
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_view_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "orc"
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- order_qt_select_load6 """ SELECT * FROM ${table_load_name} t; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
@@ -493,28 +368,8 @@ suite("test_export_view", "p0") {
"""
waiting_export.call(label)
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- logger.info("Begin to insert into ${table_load_name} from local()")
- sql """
- insert into ${table_load_name}
- select s1,k1, null as k2,k3 from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_view_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "orc"
- );
- """
- insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result: " +
insert_res.toString())
- }
+ // LOCAL TVF readback is disabled; export completion is checked above.
- order_qt_select_load8 """ SELECT * FROM ${table_load_name} t; """
} finally {
try_sql("DROP TABLE IF EXISTS ${table_load_name}")
diff --git
a/regression-test/suites/external_table_p0/export/test_export_external_table.groovy
b/regression-test/suites/external_table_p0/export/test_export_external_table.groovy
index b9f18e83ebe..0c3f76508b3 100644
---
a/regression-test/suites/external_table_p0/export/test_export_external_table.groovy
+++
b/regression-test/suites/external_table_p0/export/test_export_external_table.groovy
@@ -165,31 +165,8 @@ suite("test_export_external_table",
"p0,external,mysql,external_docker,external_
logger.info("get export res ${export_res}")
def outfileinfo = export_res[0][11];
logger.info("get export outfile info ${outfileinfo}")
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- if (outfileinfo.contains(ip)) {
- logger.info("Begin to insert into
internal.${internal_db_name}.${table_load_name} from local()")
- sql """
- insert into
internal.${internal_db_name}.${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- def insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result:
" + insert_res.toString())
- }
- }
+ // LOCAL TVF readback is disabled; export completion is checked
above.
- order_qt_select_load1 """ SELECT * FROM
internal.${internal_db_name}.${table_load_name} order by k8; """
} finally {
delete_files.call("${outFilePath}")
@@ -222,31 +199,8 @@ suite("test_export_external_table",
"p0,external,mysql,external_docker,external_
def outfileinfo = export_res[0][11];
logger.info("get external export outfile info ${outfileinfo}")
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- if (outfileinfo.contains(ip)) {
- logger.info("Begin to insert into
internal.${internal_db_name}.${table_load_name} from local()")
- sql """
- insert into
internal.${internal_db_name}.${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv",
- "column_separator" = ","
- );
- """
- def insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result:
" + insert_res.toString())
- }
- }
+ // LOCAL TVF readback is disabled; export completion is checked
above.
- order_qt_select_load2 """ SELECT * FROM
internal.${internal_db_name}.${table_load_name} order by k8; """
} finally {
delete_files.call("${outFilePath}")
@@ -278,31 +232,8 @@ suite("test_export_external_table",
"p0,external,mysql,external_docker,external_
def outfileinfo = export_res[0][11];
logger.info("get csv_with_names export outfile info
${outfileinfo}")
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- if (outfileinfo.contains(ip)) {
- logger.info("Begin to insert into
internal.${internal_db_name}.${table_load_name} from local()")
- sql """
- insert into
internal.${internal_db_name}.${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "column_separator" = ",",
- "format" = "csv_with_names"
- );
- """
- def insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result:
" + insert_res.toString())
- }
- }
+ // LOCAL TVF readback is disabled; export completion is checked
above.
- order_qt_select_load3 """ SELECT * FROM
internal.${internal_db_name}.${table_load_name} order by k8; """
} finally {
delete_files.call("${outFilePath}")
@@ -334,31 +265,8 @@ suite("test_export_external_table",
"p0,external,mysql,external_docker,external_
def outfileinfo = export_res[0][11];
logger.info("get csv_with_names_and_types export outfile info
${outfileinfo}")
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- if (outfileinfo.contains(ip)) {
- logger.info("Begin to insert into
internal.${internal_db_name}.${table_load_name} from local()")
- sql """
- insert into
internal.${internal_db_name}.${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "column_separator" = ",",
- "format" = "csv_with_names_and_types"
- );
- """
- def insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result:
" + insert_res.toString())
- }
- }
+ // LOCAL TVF readback is disabled; export completion is checked
above.
- order_qt_select_load4 """ SELECT * FROM
internal.${internal_db_name}.${table_load_name} order by k8; """
} finally {
delete_files.call("${outFilePath}")
@@ -389,30 +297,8 @@ suite("test_export_external_table",
"p0,external,mysql,external_docker,external_
def outfileinfo = export_res[0][11];
logger.info("get orc export outfile info ${outfileinfo}")
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- if (outfileinfo.contains(ip)) {
- logger.info("Begin to insert into
internal.${internal_db_name}.${table_load_name} from local()")
- sql """
- insert into
internal.${internal_db_name}.${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "orc"
- );
- """
- def insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result:
" + insert_res.toString())
- }
- }
+ // LOCAL TVF readback is disabled; export completion is checked
above.
- order_qt_select_load5 """ SELECT * FROM
internal.${internal_db_name}.${table_load_name} order by k8; """
} finally {
delete_files.call("${outFilePath}")
@@ -442,30 +328,8 @@ suite("test_export_external_table",
"p0,external,mysql,external_docker,external_
def outfileinfo = export_res[0][11];
logger.info("get parquet export outfile info ${outfileinfo}")
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- if (outfileinfo.contains(ip)) {
- logger.info("Begin to insert into
internal.${internal_db_name}.${table_load_name} from local()")
- sql """
- insert into
internal.${internal_db_name}.${table_load_name}
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "parquet"
- );
- """
- def insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result:
" + insert_res.toString())
- }
- }
+ // LOCAL TVF readback is disabled; export completion is checked
above.
- order_qt_select_load6 """ SELECT * FROM
internal.${internal_db_name}.${table_load_name} order by k8; """
} finally {
delete_files.call("${outFilePath}")
@@ -497,31 +361,8 @@ suite("test_export_external_table",
"p0,external,mysql,external_docker,external_
def outfileinfo = export_res[0][11];
logger.info("get columns property export outfile info
${outfileinfo}")
- // check data correctness
- create_load_table(table_load_name)
-
- // use local() tvf to reload the data
- def ipList = [:]
- def portList = [:]
- getBackendIpHeartbeatPort(ipList, portList)
- ipList.each { beid, ip ->
- if (outfileinfo.contains(ip)) {
- logger.info("Begin to insert into
internal.${internal_db_name}.${table_load_name} from local()")
- sql """
- insert into
internal.${internal_db_name}.${table_load_name} (k8, k1, k5, k3, k7)
- select * from local(
- "file_path" =
"${local_tvf_prefix}/${table_export_name}_${uuid}/*",
- "backend_id" = "${beid}",
- "format" = "csv_with_names",
- "column_separator" = ","
- );
- """
- def insert_res = sql "show last insert;"
- logger.info("insert from local(), BE id = ${beid}, result:
" + insert_res.toString())
- }
- }
+ // LOCAL TVF readback is disabled; export completion is checked
above.
- order_qt_select_load7 """ SELECT * FROM
internal.${internal_db_name}.${table_load_name} order by k8; """
} finally {
delete_files.call("${outFilePath}")
diff --git
a/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_local_tvf.groovy
b/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_local_tvf.groovy
index 88a76251707..ac1bd8b216a 100644
---
a/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_local_tvf.groovy
+++
b/regression-test/suites/external_table_p0/tvf/insert/test_insert_into_local_tvf.groovy
@@ -16,6 +16,9 @@
// under the License.
suite("test_insert_into_local_tvf", "tvf,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
assertTrue(backends.size() > 0)
diff --git
a/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy
b/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy
index 6fa29c321c8..4359c46c244 100644
---
a/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_file_scanner_v2_review_fixes.groovy
@@ -16,6 +16,9 @@
// under the License.
suite("test_file_scanner_v2_review_fixes", "p0,external") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
def backends = sql "show backends"
assertTrue(backends.size() > 0)
def backendId = backends[0][0]
diff --git
a/regression-test/suites/external_table_p0/tvf/test_file_tvf_local.groovy
b/regression-test/suites/external_table_p0/tvf/test_file_tvf_local.groovy
index 606133d0c95..bba78d86b19 100644
--- a/regression-test/suites/external_table_p0/tvf/test_file_tvf_local.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_file_tvf_local.groovy
@@ -18,6 +18,9 @@ import org.junit.Assert
// under the License.
suite("test_file_tvf_local", "p0,tvf,external,external_docker") {
+ logger.info("Skip: this suite requires FILE TVF local storage, which is
disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
assertTrue(backends.size() > 0)
def be_id = backends[0][0]
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy
index a90b5f83fb5..459ff0f4e45 100644
---
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy
@@ -18,6 +18,9 @@ import org.junit.Assert
// under the License.
suite("test_local_tvf_compression", "p0,tvf,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
assertTrue(backends.size() > 0)
def be_id = backends[0][0]
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.groovy
index 331faa8ddb3..eb99a88bec1 100644
---
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_csv_enclose_consistency.groovy
@@ -16,6 +16,9 @@
// under the License.
suite("test_local_tvf_csv_enclose_consistency", "p0,external") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
def backends = sql "show backends"
assertTrue(backends.size() > 0)
def beId = backends[0][0]
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
index ea982c6b375..7f6ce130473 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_enclose.groovy
@@ -18,6 +18,9 @@ import org.junit.Assert
// under the License.
suite("test_local_tvf_enclose", "p0,tvf,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
assertTrue(backends.size() > 0)
def be_id = backends[0][0]
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_lzo.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_lzo.groovy
index a7eba83a970..0fa0afadbe4 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_lzo.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_lzo.groovy
@@ -19,6 +19,9 @@ import org.junit.Assert
// This suit test the `backends` tvf
suite("test_local_tvf_lzo", "p0,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/lzo"
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_parquet_unsigned_integers.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_parquet_unsigned_integers.groovy
index 48919d5ddc3..382fd6f758a 100644
---
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_parquet_unsigned_integers.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_parquet_unsigned_integers.groovy
@@ -19,6 +19,9 @@ import org.junit.Assert
// This suit test the `backends` tvf
suite("test_local_tvf_parquet_unsigned_integers",
"p0,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/"
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy
index 8d47c613c5a..ead1f026a30 100644
---
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy
@@ -19,6 +19,9 @@ import org.junit.Assert
// This suit test the `backends` tvf
suite("test_local_tvf_with_complex_type", "p0,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/"
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy
index 0250ab6c601..a7cdf77d409 100644
---
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy
@@ -19,6 +19,9 @@ import org.junit.Assert
// This suit test the `backends` tvf
suite("test_local_tvf_with_complex_type_element_at",
"p0,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
assertTrue(backends.size() > 0)
def be_id = backends[0][0]
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
index c197a138b3a..2a1220b6e5b 100644
---
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
@@ -19,6 +19,9 @@ import org.junit.Assert
// This suit test the `backends` tvf
suite("test_local_tvf_with_complex_type_insertinto_doris",
"p0,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ select * from backends(); """
assertTrue(backends.size() > 0)
def be_id = backends[0][0]
diff --git
a/regression-test/suites/external_table_p0/tvf/test_parquet_meta_tvf.groovy
b/regression-test/suites/external_table_p0/tvf/test_parquet_meta_tvf.groovy
index 1ac07b6463b..11693b8b481 100644
--- a/regression-test/suites/external_table_p0/tvf/test_parquet_meta_tvf.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_parquet_meta_tvf.groovy
@@ -267,13 +267,16 @@ suite("test_parquet_meta_tvf",
"p0,external,external_docker,tvf") {
);
"""
- // file metadata (local glob)
- order_qt_parquet_file_metadata_local_glob """
- select count(*) from parquet_meta(
- "file_path" = "${outFilePath}/*meta.parquet",
- "mode" = "parquet_file_metadata"
- );
- """
+ // Local wildcard expansion delegates to LOCAL TVF, which is disabled.
+ test {
+ sql """
+ select count(*) from parquet_meta(
+ "file_path" = "${outFilePath}/*meta.parquet",
+ "mode" = "parquet_file_metadata"
+ );
+ """
+ exception "LOCAL TVF is disabled"
+ }
order_qt_parquet_kv_metadata_local """
select * from parquet_meta(
@@ -520,6 +523,6 @@ suite("test_parquet_meta_tvf",
"p0,external,external_docker,tvf") {
"mode" = "parquet_metadata"
);
"""
- exception "failed to glob"
+ exception "LOCAL TVF is disabled"
}
}
diff --git
a/regression-test/suites/external_table_p0/tvf/test_path_partition_keys.groovy
b/regression-test/suites/external_table_p0/tvf/test_path_partition_keys.groovy
index 028632fc3dc..2f04697bf71 100644
---
a/regression-test/suites/external_table_p0/tvf/test_path_partition_keys.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_path_partition_keys.groovy
@@ -71,54 +71,7 @@ suite("test_path_partition_keys",
"p0,external,tvf,external_docker,hive") {
}
- List<List<Object>> backends = sql """ show backends """
- assertTrue(backends.size() > 0)
- def be_id = backends[0][0]
- def dataFilePath = context.config.dataPath +
"/external_table_p0/tvf/test_path_partition_keys/"
-
- def outFilePath="/test_path_partition_keys"
-
- for (List<Object> backend : backends) {
- def be_host = backend[1]
- scpFiles ("root", be_host, dataFilePath, outFilePath, false);
- }
-
- order_qt_local_1 """
- select * from local(
- "file_path" = "${outFilePath}/dt1=cyw/a.csv",
- "backend_id" = "${be_id}",
- "format" = "csv",
- "column_separator" = ",",
- "path_partition_keys"="dt1") order by c1,c2;
- """
-
- order_qt_local_2 """
- select * from local(
- "file_path" = "${outFilePath}/dt1=cyw/*",
- "backend_id" = "${be_id}",
- "format" = "csv",
- "column_separator" = ",",
- "path_partition_keys"="dt1") order by c1,c2 limit 2;
- """
-
- order_qt_local_3 """
- select c1,dt1 from local(
- "file_path" = "${outFilePath}/dt1=hello/c.csv",
- "backend_id" = "${be_id}",
- "format" = "csv",
- "column_separator" = ",",
- "path_partition_keys"="dt1") order by c1,c2 limit 7;
- """
-
- order_qt_local_4 """
- select dt2,dt1,c1,c2 from local(
- "file_path" = "${outFilePath}/dt2=two/dt1=hello/c.csv",
- "backend_id" = "${be_id}",
- "format" = "csv",
- "column_separator" = ",",
- "path_partition_keys"="dt2,dt1") order by c1,c2 limit 9;
- """
-
+ // LOCAL TVF is disabled; retain the HDFS and S3 partition-key cases.
String ak = getS3AK()
String sk = getS3SK()
diff --git
a/regression-test/suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy
b/regression-test/suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy
index 0f7bff999d1..29b0aff09f0 100644
--- a/regression-test/suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_tvf_csv_line_end.groovy
@@ -18,6 +18,9 @@ import org.junit.Assert
// under the License.
suite("test_tvf_csv_line_end", "p0,tvf,external,external_docker") {
+ logger.info("Skip: this suite requires LOCAL TVF, which is disabled.")
+ return
+
List<List<Object>> backends = sql """ show backends """
assertTrue(backends.size() > 0)
def be_id = backends[0][0]
diff --git
a/regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy
b/regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy
index 0294b215bf1..1deab9b13ad 100644
---
a/regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy
+++
b/regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy
@@ -15,25 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+import com.azure.storage.blob.BlobClientBuilder
+import com.azure.storage.common.StorageSharedKeyCredential
+
suite("test_array_with_single_row_max_than_uint16", "query") {
// define a sql table
- List<List<Object>> backends = sql """ select * from backends(); """
- assertTrue(backends.size() > 0)
- def be_id = backends[0][0]
- def dataFilePath = context.config.dataPath + "/query_p0"
def testTable = "arr_dd"
sql "DROP TABLE IF EXISTS ${testTable}"
- def outFilePath= dataFilePath
- if (backends.size() > 1) {
- // cluster mode need to make sure all be has this data
- outFilePath="/"
- def transFile01 = "${dataFilePath}/arr_max.orc"
- for (List<Object> backend : backends) {
- def be_host = backend[1]
- scpFiles("root", be_host, transFile01, outFilePath, false)
- }
- }
-
sql """
CREATE TABLE IF NOT EXISTS ${testTable} (
`k1` INT(11) NULL,
@@ -51,11 +39,46 @@ suite("test_array_with_single_row_max_than_uint16",
"query") {
)
"""
// prepare data
- qt_sql """
- insert into ${testTable} select * from local(
- "file_path" = "${outFilePath}/arr_max.orc",
- "backend_id" = "${be_id}",
- "format" = "orc");"""
+ String bucket = getS3BucketName()
+ String key =
"regression/local_tvf_migration/${UUID.randomUUID()}/arr_max.orc"
+ String provider = getS3Provider()
+ String endpoint = getS3Endpoint()
+ File fixture = new File(context.config.dataPath, "query_p0/arr_max.orc")
+ def deleteFixture
+ if (provider.equalsIgnoreCase("AZURE")) {
+ if (endpoint == "azure-endpoint") {
+ endpoint = "https://${getS3AK()}.blob.core.windows.net"
+ } else if (!endpoint.contains("://")) {
+ endpoint = "https://${endpoint}";
+ }
+ def blobClient = new BlobClientBuilder()
+ .endpoint(endpoint)
+ .containerName(bucket)
+ .blobName(key)
+ .credential(new StorageSharedKeyCredential(getS3AK(),
getS3SK()))
+ .buildClient()
+ blobClient.uploadFromFile(fixture.absolutePath)
+ deleteFixture = { blobClient.delete() }
+ } else {
+ def s3Client = getS3Client()
+ s3Client.putObject(bucket, key, fixture)
+ deleteFixture = { s3Client.deleteObject(bucket, key) }
+ }
+ try {
+ sql """INSERT INTO ${testTable} SELECT k1, k2, c_decimal FROM S3(
+ "uri" = "s3://${bucket}/${key}",
+ "format" = "orc",
+ "s3.endpoint" = "${endpoint}",
+ "provider" = "${provider}",
+ "s3.region" = "${getS3Region()}",
+ "s3.access_key" = "${getS3AK()}",
+ "s3.secret_key" = "${getS3SK()}"
+ )"""
+ } finally {
+ deleteFixture()
+ }
+ sql "sync"
+ qt_sql "SELECT COUNT(*) FROM ${testTable}"
qt_select """ select * from ${testTable} order by k1"""
}
diff --git
a/regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy
b/regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy
index ee28436e407..014350d7f3f 100644
--- a/regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy
+++ b/regression-test/suites/types/complex_types/map_uniq_with_local_tvf.groovy
@@ -15,19 +15,11 @@
// specific language governing permissions and limitations
// under the License.
+import com.azure.storage.blob.BlobClientBuilder
+import com.azure.storage.common.StorageSharedKeyCredential
+
suite("map_uniq_with_local_tvf", "p0") {
def table_name = "map_uniq"
- List<List<Object>> backends = sql """ show backends """
- def dataFilePath = context.config.dataPath + "/types/complex_types/"
- assertTrue(backends.size() > 0)
- def be_id = backends[0][0]
- // cluster mode need to make sure all be has this data
- def outFilePath="/"
- def transFile01="${dataFilePath}/mm.orc"
- for (List<Object> backend : backends) {
- def be_host = backend[1]
- scpFiles ("root", be_host, transFile01, outFilePath, false);
- }
sql "DROP TABLE IF EXISTS ${table_name};"
sql """
CREATE TABLE ${table_name} (
@@ -45,11 +37,46 @@ suite("map_uniq_with_local_tvf", "p0") {
"disable_auto_compaction" = "false"
); """
- qt_sql """
- insert into ${table_name} select * from local(
- "file_path" = "${outFilePath}/mm.orc",
- "backend_id" = "${be_id}",
- "format" = "orc");"""
+ String bucket = getS3BucketName()
+ String key = "regression/local_tvf_migration/${UUID.randomUUID()}/mm.orc"
+ String provider = getS3Provider()
+ String endpoint = getS3Endpoint()
+ File fixture = new File(context.config.dataPath,
"types/complex_types/mm.orc")
+ def deleteFixture
+ if (provider.equalsIgnoreCase("AZURE")) {
+ if (endpoint == "azure-endpoint") {
+ endpoint = "https://${getS3AK()}.blob.core.windows.net"
+ } else if (!endpoint.contains("://")) {
+ endpoint = "https://${endpoint}";
+ }
+ def blobClient = new BlobClientBuilder()
+ .endpoint(endpoint)
+ .containerName(bucket)
+ .blobName(key)
+ .credential(new StorageSharedKeyCredential(getS3AK(),
getS3SK()))
+ .buildClient()
+ blobClient.uploadFromFile(fixture.absolutePath)
+ deleteFixture = { blobClient.delete() }
+ } else {
+ def s3Client = getS3Client()
+ s3Client.putObject(bucket, key, fixture)
+ deleteFixture = { s3Client.deleteObject(bucket, key) }
+ }
+ try {
+ sql """INSERT INTO ${table_name} SELECT id, m FROM S3(
+ "uri" = "s3://${bucket}/${key}",
+ "format" = "orc",
+ "s3.endpoint" = "${endpoint}",
+ "provider" = "${provider}",
+ "s3.region" = "${getS3Region()}",
+ "s3.access_key" = "${getS3AK()}",
+ "s3.secret_key" = "${getS3SK()}"
+ )"""
+ } finally {
+ deleteFixture()
+ }
+ sql "sync"
+ qt_sql "SELECT COUNT(*) FROM ${table_name}"
qt_sql """ select count(m) from ${table_name}; """
qt_sql """ select count(m) from ${table_name} where map_size(m) > 0;"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
