This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 7890c35b380 branch-4.1: [opt](partition) Add global dynamic partition
drop protection #68844 (#69065)
7890c35b380 is described below
commit 7890c35b380c064e766914001b31b6717347bd8e
Author: Gavin Chou <[email protected]>
AuthorDate: Thu Oct 15 09:19:59 2026 +0800
branch-4.1: [opt](partition) Add global dynamic partition drop protection
#68844 (#69065)
Cherry-picked from #68844
Conflict resolution:
- Adapted dynamic partition cleanup to branch-4.1 `DropPartitionClause`
APIs.
- Adapted the global variable annotation and tests to branch-4.1 FE
APIs/JUnit 4.
Validation:
- `sh run-fe-ut.sh --run
"org.apache.doris.catalog.DynamicPartitionTableTest,org.apache.doris.qe.VariableMgrTest"`
(65 tests passed)
---
.../doris/clone/DynamicPartitionScheduler.java | 30 +++++--
.../java/org/apache/doris/qe/SessionVariable.java | 17 ++++
.../main/java/org/apache/doris/qe/VariableMgr.java | 30 +++++++
.../doris/catalog/DynamicPartitionTableTest.java | 97 ++++++++++++++++++++++
.../java/org/apache/doris/qe/VariableMgrTest.java | 26 ++++++
5 files changed, 194 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index a35ec5696bd..e0aa3aeb1b1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -59,6 +59,8 @@ import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.meta.MetaContext;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.doris.persist.PartitionPersistInfo;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.VariableMgr;
import org.apache.doris.thrift.TStorageMedium;
import com.google.common.annotations.VisibleForTesting;
@@ -777,13 +779,17 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
executeFirstTime);
}
clearDropPartitionFailedMsg(olapTable.getId());
- if (olapTable.getPartitionRetentionCount() > 0) {
- // Handle AUTO PARTITION cleanup based on
partition.retention_count
- dropPartitionClauses =
getDropPartitionClauseForAutoPartition(db, olapTable);
+ if (VariableMgr.isDynamicPartitionDropProtectionEnabled()) {
+ logDynamicPartitionDropProtection(db, olapTable);
} else {
- // Handle dynamic partition cleanup
- dropPartitionClauses =
getDropPartitionClauseForDynamic(db, olapTable, partitionColumn,
- partitionFormat);
+ if (olapTable.getPartitionRetentionCount() > 0) {
+ // Handle AUTO PARTITION cleanup based on
partition.retention_count
+ dropPartitionClauses =
getDropPartitionClauseForAutoPartition(db, olapTable);
+ } else {
+ // Handle dynamic partition cleanup
+ dropPartitionClauses =
getDropPartitionClauseForDynamic(db, olapTable, partitionColumn,
+ partitionFormat);
+ }
}
tableName = olapTable.getName();
} catch (Exception e) {
@@ -801,6 +807,10 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
continue;
}
try {
+ if (VariableMgr.isDynamicPartitionDropProtectionEnabled())
{
+ logDynamicPartitionDropProtection(db, olapTable);
+ break;
+ }
Env.getCurrentEnv().dropPartition(db, olapTable,
dropPartitionClause);
} catch (Exception e) {
recordDropPartitionFailedMsg(db.getFullName(), tableName,
e.getMessage(), olapTable.getId());
@@ -1003,6 +1013,14 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
createOrUpdateRuntimeInfo(tableId, CREATE_PARTITION_MSG,
DEFAULT_RUNTIME_VALUE);
}
+ private void logDynamicPartitionDropProtection(Database db, OlapTable
olapTable) {
+ LOG.warn("Skip automatic partition drop for db [{}-{}], table [{}-{}]
because global session variable "
+ + "'{}' is enabled. Existing partitions are retained;
set it to false to resume automatic "
+ + "partition cleanup",
+ db.getId(), db.getName(), olapTable.getId(),
olapTable.getName(),
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION);
+ }
+
private void recordDropPartitionFailedMsg(String dbName, String tableName,
String msg, long tableId) {
LOG.warn("dynamic drop partition failed: {}, db: {}, table: {}", msg,
dbName, tableName);
createOrUpdateRuntimeInfo(tableId, DYNAMIC_PARTITION_STATE,
State.ERROR.toString());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index f6eaf55e5f1..d947e837094 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -204,6 +204,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_REWRITE_ELEMENT_AT_TO_SLOT =
"enable_rewrite_element_at_to_slot";
public static final String ENABLE_ODBC_TRANSCATION =
"enable_odbc_transcation";
public static final String ENABLE_BINARY_SEARCH_FILTERING_PARTITIONS =
"enable_binary_search_filtering_partitions";
+ public static final String ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION =
+ "enable_dynamic_partition_drop_protection";
public static final String CACHE_SORTED_PARTITION_INTERVAL_SECOND =
"cache_sorted_partition_interval_second";
public static final String SKIP_PRUNE_PREDICATE = "skip_prune_predicate";
public static final String ENABLE_SQL_CACHE = "enable_sql_cache";
@@ -1546,6 +1548,21 @@ public class SessionVariable implements Serializable,
Writable {
)
public boolean enableBinarySearchFilteringPartitions = true;
+ @VariableMgr.VarAttr(
+ name = ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
+ flag = VariableMgr.GLOBAL,
+ description = {
+ "是否阻止后台动态分区任务自动删除分区。默认关闭。",
+ "Whether to prevent background dynamic partition tasks
from automatically dropping "
+ + "partitions. OFF by default."
+ }
+ )
+ private boolean enableDynamicPartitionDropProtection = false;
+
+ public boolean isEnableDynamicPartitionDropProtection() {
+ return enableDynamicPartitionDropProtection;
+ }
+
@VariableMgr.VarAttr(
name = CACHE_SORTED_PARTITION_INTERVAL_SECOND,
fuzzy = false,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
index c293aa0d4e6..e3f6eb151be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
@@ -158,6 +158,36 @@ public class VariableMgr {
return defaultSessionVariable;
}
+ public static boolean isDynamicPartitionDropProtectionEnabled() {
+ rlock.lock();
+ try {
+ return
defaultSessionVariable.isEnableDynamicPartitionDropProtection();
+ } finally {
+ rlock.unlock();
+ }
+ }
+
+ /**
+ * Sets a persistent global-only session variable while constructing an
image.
+ *
+ * <p>The caller is responsible for persisting the resulting image. This
method intentionally does not write an
+ * edit log because image construction may happen without a running edit
log.</p>
+ */
+ public static void setGlobalSessionVariableForImage(String name, String
value) throws DdlException {
+ wlock.lock();
+ try {
+ VarContext ctx = getVarContext(name);
+ Preconditions.checkArgument(ctx != null, "Unknown session
variable: %s", name);
+ Preconditions.checkArgument(ctx.getObj() == defaultSessionVariable,
+ "Not a session variable: %s", name);
+ Preconditions.checkArgument((ctx.getFlag() & GLOBAL) != 0,
+ "Session variable is not global-only: %s", name);
+ setValue(defaultSessionVariable, new
SessionVariableField(ctx.getField()), value);
+ } finally {
+ wlock.unlock();
+ }
+ }
+
// Set value to a variable
private static boolean setValue(Object obj, SessionVariableField
sessionVariableField, String value)
throws DdlException {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index a9a818733d0..03394cbba4a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -30,19 +30,23 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.mysql.authenticate.TestLogAppender;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.qe.VariableMgr;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;
+import org.apache.logging.log4j.Level;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -133,6 +137,99 @@ public class DynamicPartitionTableTest {
}
}
+ private void setDynamicPartitionDropProtection(boolean enabled) throws
DdlException {
+ VariableMgr.setGlobalSessionVariableForImage(
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
Boolean.toString(enabled));
+ }
+
+ @Test
+ public void testDynamicPartitionDropProtection() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_drop_protection` (\n"
+ + " `k1` date NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "DUPLICATE KEY(`k1`)\n"
+ + "PARTITION BY RANGE (`k1`) ()\n"
+ + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\",\n"
+ + "\"dynamic_partition.enable\" = \"false\",\n"
+ + "\"dynamic_partition.start\" = \"-1\",\n"
+ + "\"dynamic_partition.end\" = \"1\",\n"
+ + "\"dynamic_partition.time_unit\" = \"day\",\n"
+ + "\"dynamic_partition.prefix\" = \"p\",\n"
+ + "\"dynamic_partition.buckets\" = \"1\"\n"
+ + ");";
+ createTable(createOlapTblStmt);
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("dynamic_partition_drop_protection");
+ alterTable("ALTER TABLE test.dynamic_partition_drop_protection ADD
PARTITION p_old "
+ + "VALUES [('2000-01-01'), ('2000-01-02'))");
+
+ try {
+ setDynamicPartitionDropProtection(true);
+ alterTable("ALTER TABLE test.dynamic_partition_drop_protection SET
"
+ + "('dynamic_partition.enable' = 'true')");
+ try (TestLogAppender appender =
TestLogAppender.attach(DynamicPartitionScheduler.class, Level.WARN)) {
+ Env.getCurrentEnv().getDynamicPartitionScheduler()
+ .executeDynamicPartitionFirstTime(db.getId(),
table.getId());
+ Assert.assertTrue(appender.contains(Level.WARN,
+ "table [" + table.getId() + "-" + table.getName()
+ + "] because global session variable '"
+ +
SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION + "' is enabled"));
+ }
+
+ Assert.assertTrue(table.getPartitionNames().contains("p_old"));
+ Assert.assertTrue("Dynamic partition creation should continue
while drop protection is enabled",
+ table.getPartitionNames().size() > 1);
+
+ setDynamicPartitionDropProtection(false);
+ Env.getCurrentEnv().getDynamicPartitionScheduler()
+ .executeDynamicPartitionFirstTime(db.getId(),
table.getId());
+ Assert.assertFalse(table.getPartitionNames().contains("p_old"));
+ } finally {
+ setDynamicPartitionDropProtection(false);
+ }
+ }
+
+ @Test
+ public void testAutoPartitionRetentionDropProtection() throws Exception {
+ String createOlapTblStmt = "CREATE TABLE
test.`auto_partition_drop_protection` (\n"
+ + " `k1` datetime NOT NULL\n"
+ + ") ENGINE=OLAP\n"
+ + "DUPLICATE KEY(`k1`)\n"
+ + "AUTO PARTITION BY RANGE (date_trunc(k1, 'day')) ()\n"
+ + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"replication_num\" = \"1\",\n"
+ + "\"partition.retention_count\" = \"1\"\n"
+ + ");";
+ createTable(createOlapTblStmt);
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("test");
+ OlapTable table = (OlapTable)
db.getTableOrAnalysisException("auto_partition_drop_protection");
+ alterTable("ALTER TABLE test.auto_partition_drop_protection ADD
PARTITION p_old "
+ + "VALUES [('2000-01-01'), ('2000-01-02'))");
+ alterTable("ALTER TABLE test.auto_partition_drop_protection ADD
PARTITION p_new "
+ + "VALUES [('2020-01-01'), ('2020-01-02'))");
+
+ try {
+ setDynamicPartitionDropProtection(true);
+ Env.getCurrentEnv().getDynamicPartitionScheduler()
+ .executeDynamicPartitionFirstTime(db.getId(),
table.getId());
+ Assert.assertTrue(table.getPartitionNames().contains("p_old"));
+ Assert.assertTrue(table.getPartitionNames().contains("p_new"));
+
+ setDynamicPartitionDropProtection(false);
+ Env.getCurrentEnv().getDynamicPartitionScheduler()
+ .executeDynamicPartitionFirstTime(db.getId(),
table.getId());
+ Assert.assertFalse(table.getPartitionNames().contains("p_old"));
+ Assert.assertTrue(table.getPartitionNames().contains("p_new"));
+ } finally {
+ setDynamicPartitionDropProtection(false);
+ }
+ }
+
@Test
public void testNormal() throws Exception {
String createOlapTblStmt = "CREATE TABLE
test.`dynamic_partition_normal` (\n"
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
index 1bdd89b5e4f..5219437fdc1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
@@ -112,6 +112,32 @@ public class VariableMgrTest {
Assert.fail("No exception throws.");
}
+ @Test
+ public void testDynamicPartitionDropProtectionIsGlobalOnly() throws
Exception {
+ try {
+ VariableMgr.setGlobalSessionVariableForImage(
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
"false");
+
Assertions.assertFalse(VariableMgr.isDynamicPartitionDropProtectionEnabled());
+
+ SetOptionsCommand sessionStmt = (SetOptionsCommand)
UtFrameUtils.parseStmt(
+ "set enable_dynamic_partition_drop_protection=true", ctx);
+ Assertions.assertThrows(DdlException.class, () ->
sessionStmt.run(ctx, null));
+
+ VariableMgr.setGlobalSessionVariableForImage(
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
"true");
+
Assertions.assertTrue(VariableMgr.isDynamicPartitionDropProtectionEnabled());
+
Assertions.assertTrue(VariableMgr.newSessionVariable().isEnableDynamicPartitionDropProtection());
+
+ SetOptionsCommand globalStmt = (SetOptionsCommand)
UtFrameUtils.parseStmt(
+ "set global
enable_dynamic_partition_drop_protection=false", ctx);
+ globalStmt.run(ctx, null);
+
Assertions.assertFalse(VariableMgr.isDynamicPartitionDropProtectionEnabled());
+ } finally {
+ VariableMgr.setGlobalSessionVariableForImage(
+ SessionVariable.ENABLE_DYNAMIC_PARTITION_DROP_PROTECTION,
"false");
+ }
+ }
+
@Test
public void testAuthenticationPolicyIsTheReadOnlyMysqlDefault() throws
Exception {
SessionVariable sessionVariable = new SessionVariable();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
