This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 833e679a6af branch-3.0: [fix](dynamic partition) data lost with very
small dynamic_partition.start #43898 (#44032)
833e679a6af is described below
commit 833e679a6afbae59e8bf7b5f1886bfe0be8cc93b
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Nov 18 14:20:49 2024 +0800
branch-3.0: [fix](dynamic partition) data lost with very small
dynamic_partition.start #43898 (#44032)
Cherry-picked from #43898
Co-authored-by: camby <camby...@tencent.com>
---
.../doris/clone/DynamicPartitionScheduler.java | 9 ++++
.../test_dynamic_partition_small_start.out | 4 ++
.../test_dynamic_partition_small_start.groovy | 54 ++++++++++++++++++++++
3 files changed, 67 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index c34d4417f2b..e875bcf47d8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -460,12 +460,21 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
ZonedDateTime now =
ZonedDateTime.now(dynamicPartitionProperty.getTimeZone().toZoneId());
String lowerBorder =
DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty,
now, realStart, partitionFormat);
+ String limitBorder =
DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty,
+ now, 0, partitionFormat);
PartitionValue lowerPartitionValue = new PartitionValue(lowerBorder);
+ PartitionValue limitPartitionValue = new PartitionValue(limitBorder);
List<Range<PartitionKey>> reservedHistoryPartitionKeyRangeList = new
ArrayList<Range<PartitionKey>>();
Range<PartitionKey> reservePartitionKeyRange;
try {
PartitionKey lowerBound =
PartitionKey.createPartitionKey(Collections.singletonList(lowerPartitionValue),
Collections.singletonList(partitionColumn));
+ PartitionKey limitBound =
PartitionKey.createPartitionKey(Collections.singletonList(limitPartitionValue),
+ Collections.singletonList(partitionColumn));
+ // if start offset very small, then lowerBound may be very large,
such as '6031-07-01 00:00:00'
+ if (lowerBound.compareTo(limitBound) >= 0) {
+ return dropPartitionClauses;
+ }
reservePartitionKeyRange = Range.atLeast(lowerBound);
reservedHistoryPartitionKeyRangeList.add(reservePartitionKeyRange);
} catch (Exception e) {
diff --git
a/regression-test/data/partition_p0/dynamic_partition/test_dynamic_partition_small_start.out
b/regression-test/data/partition_p0/dynamic_partition/test_dynamic_partition_small_start.out
new file mode 100644
index 00000000000..095c7b20356
--- /dev/null
+++
b/regression-test/data/partition_p0/dynamic_partition/test_dynamic_partition_small_start.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1
+
diff --git
a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_small_start.groovy
b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_small_start.groovy
new file mode 100644
index 00000000000..97d638230eb
--- /dev/null
+++
b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_small_start.groovy
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_dynamic_partition_small_start', 'nonConcurrent') {
+ def old_check_interval =
getFeConfig('dynamic_partition_check_interval_seconds')
+ try {
+ setFeConfig('dynamic_partition_check_interval_seconds', 1);
+ sql 'DROP TABLE IF EXISTS test_dynamic_partition_small_start FORCE'
+ sql '''CREATE TABLE test_dynamic_partition_small_start
+ ( `k1` datetime NULL )
+ PARTITION BY RANGE (k1)()
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+ PROPERTIES
+ (
+ "replication_num" = "1",
+ "dynamic_partition.replication_num" = "1",
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.end" = "3",
+ "dynamic_partition.time_unit" = "month",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "1",
+ "dynamic_partition.start" = "-100000",
+ "dynamic_partition.history_partition_num" = "5",
+ "dynamic_partition.create_history_partition" = "true"
+ )'''
+
+ def partitions = sql_return_maparray "SHOW PARTITIONS FROM
test_dynamic_partition_small_start"
+ assertEquals(9, partitions.size());
+
+ sql ''' insert into test_dynamic_partition_small_start values(now())
'''
+ sql ''' sync '''
+
+ // wait dynamic partition scheduler, to make sure old partitions will
not be dropped
+ Thread.sleep(5000);
+ qt_sql ''' select count(1) from test_dynamic_partition_small_start '''
+ } finally {
+ setFeConfig('dynamic_partition_check_interval_seconds',
old_check_interval);
+ sql 'DROP TABLE IF EXISTS test_dynamic_partition_small_start'
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
