This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d0af6bad651 [fix](insert) do not register group commit and insert into
value job to load manager (#56852)
d0af6bad651 is described below
commit d0af6bad6513e2e3ee63af50d0f91d7fcc019c59
Author: hui lai <[email protected]>
AuthorDate: Mon Oct 13 14:28:45 2025 +0800
[fix](insert) do not register group commit and insert into value job to
load manager (#56852)
Register group commit to load manager when init will cause fe observer
core, introduced by https://github.com/apache/doris/pull/56412:
---
.../commands/insert/AbstractInsertExecutor.java | 5 +++-
.../commands/insert/InsertIntoTableCommand.java | 7 +++++-
.../plans/commands/insert/OlapInsertExecutor.java | 3 ++-
.../insert_overwrite_auto_detect.groovy | 16 +-----------
.../load_p0/insert/test_insert_statistic.groovy | 29 ++++++++++++++--------
.../nereids_p0/show/test_nereids_show_load.groovy | 4 +--
6 files changed, 34 insertions(+), 30 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
index 20b24fd9aed..3e393ff4d1b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/AbstractInsertExecutor.java
@@ -116,7 +116,10 @@ public abstract class AbstractInsertExecutor {
this.ctx = ctx;
this.database = table.getDatabase();
this.insertLoadJob = new InsertLoadJob(database.getId(), labelName,
jobId);
- ctx.getEnv().getLoadManager().addLoadJob(insertLoadJob);
+ // Do not add load job if job id is -1.
+ if (jobId != -1) {
+ ctx.getEnv().getLoadManager().addLoadJob(insertLoadJob);
+ }
this.coordinator = EnvFactory.getInstance().createCoordinator(
ctx, planner, ctx.getStatsErrorEstimator(),
insertLoadJob.getId());
this.labelName = labelName;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
index fd5ad2ac6aa..d0badaf2d78 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
@@ -47,6 +47,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Explainable;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.algebra.InlineTable;
import org.apache.doris.nereids.trees.plans.algebra.TVFRelation;
import org.apache.doris.nereids.trees.plans.commands.Command;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
@@ -386,10 +387,14 @@ public class InsertIntoTableCommand extends Command
implements NeedAuditEncrypti
dataSink,
physicalSink,
() -> new OlapGroupCommitInsertExecutor(
- ctx, olapTable, label, planner, insertCtx,
emptyInsert, groupCommitBackend, jobId
+ ctx, olapTable, label, planner, insertCtx,
emptyInsert, groupCommitBackend, -1
)
);
} else {
+ // Do not register insert into value to LoadManager.
+ if (getLogicalQuery().containsType(InlineTable.class)) {
+ jobId = -1;
+ }
executorFactory = ExecutorFactory.from(
planner,
dataSink,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
index e7d19f07577..78da636fe5b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertExecutor.java
@@ -312,7 +312,8 @@ public class OlapInsertExecutor extends
AbstractInsertExecutor {
if (0 != jobId) {
etlJobType = EtlJobType.INSERT_JOB;
}
- if (!Config.enable_nereids_load) {
+ // Do not register job if job id is -1.
+ if (!Config.enable_nereids_load && jobId != -1) {
// just record for loadv2 here
ctx.getEnv().getLoadManager()
.recordFinishedLoadJob(labelName, txnId,
database.getFullName(),
diff --git
a/regression-test/suites/insert_overwrite_p0/insert_overwrite_auto_detect.groovy
b/regression-test/suites/insert_overwrite_p0/insert_overwrite_auto_detect.groovy
index ffd3c3e642f..6c3c7e7cdd7 100644
---
a/regression-test/suites/insert_overwrite_p0/insert_overwrite_auto_detect.groovy
+++
b/regression-test/suites/insert_overwrite_p0/insert_overwrite_auto_detect.groovy
@@ -84,21 +84,7 @@ suite("test_iot_auto_detect") {
sql """ insert overwrite table list1 partition(*) with label
`iot_auto_txn${uniqueID2}` values ("SHANGHAI"), ("LIST"); """
sql """ insert overwrite table list1 partition(*) with label
`iot_auto_txn${uniqueID3}` values ("XXX"); """
- def max_try_milli_secs = 10000
- while(max_try_milli_secs) {
- def result = sql " show load where label like 'iot_auto_txn%' order by
LoadStartTime desc "
- // the last three loads are loads upper
- if(result[0][2] == "FINISHED" && result[1][2] == "FINISHED" &&
result[2][2] == "FINISHED" ) {
- break
- } else {
- sleep(1000) // wait 1 second every time
- max_try_milli_secs -= 1000
- if(max_try_milli_secs <= 0) {
- log.info("result: ${result[0][2]}, ${result[1][2]},
${result[2][2]}")
- fail()
- }
- }
- }
+ sleep(10000)
qt_sql " select * from list1 order by k0; "
diff --git a/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
b/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
index 61df308c58e..8e55b129074 100644
--- a/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
+++ b/regression-test/suites/load_p0/insert/test_insert_statistic.groovy
@@ -19,6 +19,7 @@ suite("test_insert_statistic", "p0") {
def dbName = "test_insert_statistic_db"
def insert_tbl = "test_insert_statistic_tbl"
def label = "test_insert_statistic_label"
+ sql """DROP DATABASE IF EXISTS ${dbName}"""
sql """CREATE DATABASE IF NOT EXISTS ${dbName}"""
sql """use ${dbName}"""
@@ -42,10 +43,18 @@ suite("test_insert_statistic", "p0") {
INSERT INTO ${insert_tbl}_1 values(1, 1, 1, 1)
"""
def result = sql "SHOW LOAD FROM ${dbName}"
- logger.info("JobDetails: " + result[0][14])
- def json = parseJson(result[0][14])
- assertEquals(json.ScannedRows, 1)
- assertTrue(json.LoadBytes > 0)
+ log.info("result size: " + result.size())
+ assertEquals(result.size(), 0)
+
+ // group commit
+ sql """ set group_commit = sync_mode; """
+ sql """
+ INSERT INTO ${insert_tbl}_1 values(1, 1, 1, 1)
+ """
+ sql """ set group_commit = off_mode; """
+ result = sql "SHOW LOAD FROM ${dbName}"
+ log.info("result size: " + result.size())
+ assertEquals(result.size(), 0)
// insert into select
sql """ DROP TABLE IF EXISTS ${insert_tbl}_2"""
@@ -67,9 +76,9 @@ suite("test_insert_statistic", "p0") {
INSERT INTO ${insert_tbl}_2 select * from ${insert_tbl}_1
"""
result = sql "SHOW LOAD FROM ${dbName}"
- logger.info("JobDetails: " + result[1][14])
- json = parseJson(result[1][14])
- assertEquals(json.ScannedRows, 1)
+ logger.info("JobDetails: " + result[0][14])
+ def json = parseJson(result[0][14])
+ assertEquals(json.ScannedRows, 2)
assertTrue(json.LoadBytes > 0)
// insert into s3 tvf
@@ -104,8 +113,8 @@ suite("test_insert_statistic", "p0") {
);
"""
result = sql "SHOW LOAD FROM ${dbName}"
- logger.info("JobDetails: " + result[2][14])
- json = parseJson(result[2][14])
- assertEquals(json.ScannedRows, 1)
+ logger.info("JobDetails: " + result[1][14])
+ json = parseJson(result[1][14])
+ assertEquals(json.ScannedRows, 2)
assertTrue(json.LoadBytes > 0)
}
\ No newline at end of file
diff --git
a/regression-test/suites/nereids_p0/show/test_nereids_show_load.groovy
b/regression-test/suites/nereids_p0/show/test_nereids_show_load.groovy
index afb928f2ab4..73a273b8d99 100644
--- a/regression-test/suites/nereids_p0/show/test_nereids_show_load.groovy
+++ b/regression-test/suites/nereids_p0/show/test_nereids_show_load.groovy
@@ -31,8 +31,8 @@ suite("test_nereids_show_load") {
PROPERTIES ("replication_allocation" = "tag.location.default: 1");
"""
- sql """ insert into test_nereids_show_load with label
`test_nereids_show_load_label_xyz` values (1,1); """
- sql """ insert into test_nereids_show_load with label
`new_test_nereids_show_load_label_xyz` values (2,2); """
+ sql """ insert into test_nereids_show_load with label
`test_nereids_show_load_label_xyz` select 1,1; """
+ sql """ insert into test_nereids_show_load with label
`new_test_nereids_show_load_label_xyz` select 2,2; """
checkNereidsExecute("SHOW LOAD")
checkNereidsExecute("SHOW LOAD FROM test_nereids_show_load_db")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
