This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new b1767128699 [backport][branch-3.0] unpartition table need to keep in
recycle bin in insert overwrite case (#42404)
b1767128699 is described below
commit b17671286993d2f13105a569ccc279282dfe6b90
Author: Vallish Pai <vallish...@gmail.com>
AuthorDate: Mon Oct 28 07:52:55 2024 +0530
[backport][branch-3.0] unpartition table need to keep in recycle bin in
insert overwrite case (#42404)
bp #42130
follow up for https://github.com/apache/doris/pull/40512
Issue Number: close #xxx
if a insert overwrite happens to un-partitioned table, partition info
needs to be kept inside the recycle bin. (without this tablets may leak)
once the data kept in recycle bin, it can be recovered using below step.
recover the partition to table as another partition in the table.
create another temp user table , and select the recover partition info
and insert to it.
replace original user table from temp user table.
---
.../java/org/apache/doris/catalog/OlapTable.java | 19 ++++++
.../test_insert_overwrite_recover_no_partition.out | 14 ++++
...st_insert_overwrite_recover_no_partition.groovy | 78 ++++++++++++++++++++++
3 files changed, 111 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index d77e2744dbd..18d546b04d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1062,6 +1062,25 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
partitionInfo.getReplicaAllocation(partition.getId()),
partitionInfo.getIsInMemory(partition.getId()),
partitionInfo.getIsMutable(partition.getId()));
+ } else {
+ // unpartition
+ // construct a dummy range and dummy list.
+ List<Column> dummyColumns = new ArrayList<>();
+ dummyColumns.add(new Column("dummy", PrimitiveType.INT));
+ PartitionKey dummyKey = null;
+ try {
+ dummyKey =
PartitionKey.createInfinityPartitionKey(dummyColumns, false);
+ } catch (AnalysisException e) {
+ LOG.warn("should not happen", e);
+ }
+ Range<PartitionKey> dummyRange = Range.open(new
PartitionKey(), dummyKey);
+ Env.getCurrentRecycleBin().recyclePartition(dbId, id,
name, partition,
+ dummyRange,
+ new ListPartitionItem(Lists.newArrayList(new
PartitionKey())),
+ partitionInfo.getDataProperty(partition.getId()),
+
partitionInfo.getReplicaAllocation(partition.getId()),
+ partitionInfo.getIsInMemory(partition.getId()),
+ partitionInfo.getIsMutable(partition.getId()));
}
} else if (!reserveTablets) {
Env.getCurrentEnv().onErasePartition(partition);
diff --git
a/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.out
b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.out
new file mode 100644
index 00000000000..a3effb99a08
--- /dev/null
+++
b/regression-test/data/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.out
@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_check_1 --
+1 a 2022-01-02
+2 a 2023-01-02
+3 a 2024-01-02
+
+-- !select_check_2 --
+3 a 2024-01-02
+
+-- !select_check_3 --
+1 a 2022-01-02
+2 a 2023-01-02
+3 a 2024-01-02
+
diff --git
a/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
new file mode 100644
index 00000000000..f3123d71b8d
--- /dev/null
+++
b/regression-test/suites/catalog_recycle_bin_p0/test_insert_overwrite_recover_no_partition.groovy
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_insert_overwrite_recover_no_partition") {
+ def table = "test_insert_overwrite_recover_no_partition"
+
+ // create table and insert data for range.
+ sql """ drop table if exists ${table} force"""
+ sql """
+ create table ${table} (
+ `id` int(11),
+ `name` varchar(128),
+ `da` date
+ )
+ engine=olap
+ duplicate key(id)
+ distributed by hash(id) buckets 2
+ properties(
+ "replication_num"="1",
+ "light_schema_change"="true"
+ );
+ """
+
+ sql """ insert into ${table} values(1, 'a', '2022-01-02'); """
+ sql """ insert into ${table} values(2, 'a', '2023-01-02'); """
+ sql """ insert into ${table} values(3, 'a', '2024-01-02'); """
+ sql """ SYNC;"""
+
+ qt_select_check_1 """ select * from ${table} order by id,name,da; """
+
+ sql """ insert overwrite table ${table} values(3, 'a', '2024-01-02'); """
+
+
+ qt_select_check_2 """ select * from ${table} order by id,name,da; """
+
+ // now unpartition data is kept inside the recycle bin.
+ // we need to recover it as another partition in the table.
+ sql """ recover partition ${table} as p2 from ${table}; """
+
+ // create a table to copy the data only for partition p2.
+ table_bk = "test_insert_overwrite_recover_no_partition_backup"
+ sql """ drop table if exists ${table_bk} force"""
+ sql """
+ create table ${table_bk} (
+ `id` int(11),
+ `name` varchar(128),
+ `da` date
+ )
+ engine=olap
+ duplicate key(id)
+ distributed by hash(id) buckets 2
+ properties(
+ "replication_num"="1",
+ "light_schema_change"="true"
+ );
+ """
+ sql """ insert into ${table_bk} select * from ${table} partition p2; """
+
+ sql """ alter table ${table} replace with table ${table_bk}; """
+
+ // data from the select should be same as data before overwrite.
+ qt_select_check_3 """ select * from ${table} order by id,name,da; """
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
