This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 44119ab3044 branch-3.1: [fix](export) cancel job before adding task
#57488 (#57562)
44119ab3044 is described below
commit 44119ab30443ee58814759e284d38557af8424b3
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Nov 3 11:21:46 2025 +0800
branch-3.1: [fix](export) cancel job before adding task #57488 (#57562)
bp #57488
---
.../org/apache/doris/analysis/OutFileClause.java | 8 ++-
.../main/java/org/apache/doris/load/ExportMgr.java | 18 +++---
.../trees/plans/commands/ExportCommand.java | 22 ++++++-
.../export_p0/test_export_delete_disallow.groovy | 75 ++++++++++++++++++++++
4 files changed, 108 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index d38bf28e055..ec2544f320c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -526,8 +526,12 @@ public class OutFileClause {
}
if (copiedProps.containsKey(PROP_DELETE_EXISTING_FILES)) {
- deleteExistingFiles =
Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES))
- & Config.enable_delete_existing_files;
+ deleteExistingFiles =
Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES));
+ if (deleteExistingFiles && !Config.enable_delete_existing_files) {
+ throw new AnalysisException("Deleting existing files is not
allowed."
+ + " To enable this feature, you need to add
`enable_delete_existing_files=true`"
+ + " in fe.conf");
+ }
copiedProps.remove(PROP_DELETE_EXISTING_FILES);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index 89baa252d46..d9d0f75ce68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -111,17 +111,14 @@ public class ExportMgr {
} finally {
writeUnlock();
}
- // delete existing files
- if (Config.enable_delete_existing_files &&
Boolean.parseBoolean(job.getDeleteExistingFiles())) {
- if (job.getBrokerDesc() == null) {
- throw new AnalysisException("Local file system does not
support delete existing files");
- }
- String fullPath = job.getExportPath();
- BrokerUtil.deleteDirectoryWithFileSystem(fullPath.substring(0,
fullPath.lastIndexOf('/') + 1),
- job.getBrokerDesc());
- }
- // ATTN: Must add task after edit log, otherwise the job may finish
before adding job.
try {
+ // delete existing files
+ if (Boolean.parseBoolean(job.getDeleteExistingFiles())) {
+ String fullPath = job.getExportPath();
+ BrokerUtil.deleteDirectoryWithFileSystem(fullPath.substring(0,
fullPath.lastIndexOf('/') + 1),
+ job.getBrokerDesc());
+ }
+ // ATTN: Must add task after edit log, otherwise the job may
finish before adding job.
for (int i = 0; i < job.getCopiedTaskExecutors().size(); i++) {
Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(job.getCopiedTaskExecutors().get(i));
}
@@ -546,3 +543,4 @@ public class ExportMgr {
return size;
}
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index a19160379ec..35ddf3d86a2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.OutFileClause;
import org.apache.doris.analysis.Separator;
import org.apache.doris.analysis.StmtType;
import org.apache.doris.analysis.StorageBackend;
+import org.apache.doris.analysis.StorageBackend.StorageType;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.BrokerMgr;
import org.apache.doris.catalog.DatabaseIf;
@@ -305,9 +306,6 @@ public class ExportCommand extends Command implements
ForwardWithSync {
// set max_file_size
exportJob.setMaxFileSize(fileProperties.getOrDefault(OutFileClause.PROP_MAX_FILE_SIZE,
""));
- // set delete_existing_files
- exportJob.setDeleteExistingFiles(fileProperties.getOrDefault(
- OutFileClause.PROP_DELETE_EXISTING_FILES, ""));
// null means not specified
// "" means user specified zero columns
@@ -322,6 +320,23 @@ public class ExportCommand extends Command implements
ForwardWithSync {
// set broker desc
exportJob.setBrokerDesc(this.brokerDesc.get());
+ // set delete_existing_files
+ exportJob.setDeleteExistingFiles(fileProperties.getOrDefault(
+ OutFileClause.PROP_DELETE_EXISTING_FILES, ""));
+
+ if (!Config.enable_delete_existing_files &&
exportJob.getDeleteExistingFiles().equalsIgnoreCase("true")) {
+ throw new AnalysisException(("Deleting existing files is not
allowed."
+ + " To enable this feature, you need to add
`enable_delete_existing_files=true`"
+ + " in fe.conf"));
+ }
+
+ if (exportJob.getDeleteExistingFiles().equalsIgnoreCase("true")
+ && (exportJob.getBrokerDesc() == null
+ || exportJob.getBrokerDesc().storageType() ==
StorageType.LOCAL)) {
+ throw new org.apache.doris.common.AnalysisException(
+ "Local file system does not support delete existing
files");
+ }
+
// set sessions
exportJob.setQualifiedUser(ctx.getQualifiedUser());
exportJob.setUserIdentity(ctx.getCurrentUserIdentity());
@@ -391,3 +406,4 @@ public class ExportCommand extends Command implements
ForwardWithSync {
return StmtType.EXPORT;
}
}
+
diff --git
a/regression-test/suites/export_p0/test_export_delete_disallow.groovy
b/regression-test/suites/export_p0/test_export_delete_disallow.groovy
new file mode 100644
index 00000000000..f5f508afb52
--- /dev/null
+++ b/regression-test/suites/export_p0/test_export_delete_disallow.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_export_delete_disallow", "p0") {
+ def db = "regression_test_export_p0"
+
+ // check whether the FE config 'enable_outfile_to_local' is true
+ StringBuilder strBuilder = new StringBuilder()
+ strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser
+ ":" + context.config.jdbcPassword)
+ strBuilder.append(" http://"; + context.config.feHttpAddress +
"/rest/v1/config/fe")
+
+ String command = strBuilder.toString()
+ def process = command.toString().execute()
+ def code = process.waitFor()
+ def err = IOGroovyMethods.getText(new BufferedReader(new
InputStreamReader(process.getErrorStream())));
+ def out = process.getText()
+ logger.info("Request FE Config: code=" + code + ", out=" + out + ", err="
+ err)
+ assertEquals(code, 0)
+ def response = parseJson(out.trim())
+ assertEquals(response.code, 0)
+ assertEquals(response.msg, "success")
+ def configJson = response.data.rows
+ boolean enableOutfileToLocal = false
+ for (Object conf: configJson) {
+ assert conf instanceof Map
+ if (((Map<String, String>) conf).get("Name").toLowerCase() ==
"enable_outfile_to_local") {
+ enableOutfileToLocal = ((Map<String, String>)
conf).get("Value").toLowerCase() == "true"
+ }
+ }
+ if (!enableOutfileToLocal) {
+ logger.warn("Please set enable_outfile_to_local to true to run
test_outfile")
+ return
+ }
+
+ // create table and insert
+ sql """ DROP TABLE IF EXISTS test_export_delete_disallow_tbl"""
+ sql """
+ CREATE TABLE IF NOT EXISTS test_export_delete_disallow_tbl (
+ `id` int(11) NULL,
+ `Name` string NULL
+ )
+ PROPERTIES("replication_num" = "1");
+ """
+ sql """insert into test_export_delete_disallow_tbl values(1, "abc");"""
+
+ test {
+ sql """select * from test_export_delete_disallow_tbl limit 10 INTO
OUTFILE "file:///tmp/" properties("delete_existing_files" = "true");"""
+ exception """Deleting existing files is not allowed"""
+ }
+
+ test {
+ sql """EXPORT TABLE test_export_delete_disallow_tbl TO "file:///tmp"
PROPERTIES ("delete_existing_files" = "true");"""
+ exception """Deleting existing files is not allowe"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
