This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 81dd00f6e4 [Feature](Compaction) Support do full compaction by table
id (#22010)
81dd00f6e4 is described below
commit 81dd00f6e446ccbd40b64dd07d0cdd847c24d7b1
Author: abmdocrt <yukang.lian2...@gmail.com>
AuthorDate: Mon Aug 21 11:54:51 2023 +0800
[Feature](Compaction) Support do full compaction by table id (#22010)
---
be/src/http/action/compaction_action.cpp | 108 ++++++++-----
be/src/http/action/compaction_action.h | 2 +-
be/src/olap/olap_define.h | 1 +
be/src/olap/storage_engine.cpp | 6 +
be/src/olap/storage_engine.h | 3 +
be/src/olap/tablet.cpp | 39 ++++-
be/src/olap/tablet.h | 2 +
.../docs/admin-manual/data-admin/data-recovery.md | 54 +++++++
.../admin-manual/http-actions/be/compaction-run.md | 6 +-
docs/sidebars.json | 1 +
.../docs/admin-manual/data-admin/data-recovery.md | 54 +++++++
.../admin-manual/http-actions/be/compaction-run.md | 6 +-
.../test_full_compaction_by_table_id.out | 41 +++++
.../plugins/plugin_curl_requester.groovy | 4 +
.../test_full_compaction_by_table_id.groovy | 176 +++++++++++++++++++++
15 files changed, 459 insertions(+), 44 deletions(-)
diff --git a/be/src/http/action/compaction_action.cpp
b/be/src/http/action/compaction_action.cpp
index d94ca60432..c81c65da1f 100644
--- a/be/src/http/action/compaction_action.cpp
+++ b/be/src/http/action/compaction_action.cpp
@@ -53,25 +53,43 @@ const static std::string HEADER_JSON = "application/json";
CompactionAction::CompactionAction(CompactionActionType ctype, ExecEnv*
exec_env,
TPrivilegeHier::type hier,
TPrivilegeType::type ptype)
: HttpHandlerWithAuth(exec_env, hier, ptype), _type(ctype) {}
-Status CompactionAction::_check_param(HttpRequest* req, uint64_t* tablet_id) {
+Status CompactionAction::_check_param(HttpRequest* req, uint64_t* tablet_id,
uint64_t* table_id) {
+ // req tablet id and table id, we have to set only one of them.
std::string req_tablet_id = req->param(TABLET_ID_KEY);
+ std::string req_table_id = req->param(TABLE_ID_KEY);
if (req_tablet_id == "") {
- return Status::OK();
- }
-
- try {
- *tablet_id = std::stoull(req_tablet_id);
- } catch (const std::exception& e) {
- return Status::InternalError("convert tablet_id failed, {}", e.what());
+ if (req_table_id == "") {
+ // both tablet id and table id are empty, return error.
+ return Status::InternalError(
+ "tablet id and table id can not be empty at the same
time!");
+ } else {
+ try {
+ *table_id = std::stoull(req_table_id);
+ } catch (const std::exception& e) {
+ return Status::InternalError("convert tablet_id or table_id
failed, {}", e.what());
+ }
+ return Status::OK();
+ }
+ } else {
+ if (req_table_id == "") {
+ try {
+ *tablet_id = std::stoull(req_tablet_id);
+ } catch (const std::exception& e) {
+ return Status::InternalError("convert tablet_id or table_id
failed, {}", e.what());
+ }
+ return Status::OK();
+ } else {
+ // both tablet id and table id are not empty, return err.
+ return Status::InternalError("tablet id and table id can not be
set at the same time!");
+ }
}
-
- return Status::OK();
}
// for viewing the compaction status
Status CompactionAction::_handle_show_compaction(HttpRequest* req,
std::string* json_result) {
uint64_t tablet_id = 0;
- RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param
failed");
+ uint64_t table_id = 0;
+ RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id, &table_id),
"check param failed");
TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id);
if (tablet == nullptr) {
@@ -84,9 +102,10 @@ Status
CompactionAction::_handle_show_compaction(HttpRequest* req, std::string*
Status CompactionAction::_handle_run_compaction(HttpRequest* req, std::string*
json_result) {
// 1. param check
- // check req_tablet_id is not empty
+ // check req_tablet_id or req_table_id is not empty and can not be set
together.
uint64_t tablet_id = 0;
- RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param
failed");
+ uint64_t table_id = 0;
+ RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id, &table_id),
"check param failed");
// check compaction_type equals 'base' or 'cumulative'
std::string compaction_type = req->param(PARAM_COMPACTION_TYPE);
@@ -96,44 +115,57 @@ Status
CompactionAction::_handle_run_compaction(HttpRequest* req, std::string* j
return Status::NotSupported("The compaction type '{}' is not
supported", compaction_type);
}
- // 2. fetch the tablet by tablet_id
- TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id);
- if (tablet == nullptr) {
- return Status::NotFound("Tablet not found. tablet_id={}", tablet_id);
- }
+ if (tablet_id == 0 && table_id != 0) {
+ std::vector<TabletSharedPtr> tablet_vec =
+ StorageEngine::instance()->tablet_manager()->get_all_tablet(
+ [table_id](Tablet* tablet) -> bool {
+ return tablet->get_table_id() == table_id;
+ });
+ for (const auto& tablet : tablet_vec) {
+ StorageEngine::instance()->submit_compaction_task(
+ tablet, CompactionType::FULL_COMPACTION, false);
+ }
+ } else {
+ // 2. fetch the tablet by tablet_id
+ TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id);
+ if (tablet == nullptr) {
+ return Status::NotFound("Tablet not found. tablet_id={}",
tablet_id);
+ }
- // 3. execute compaction task
- std::packaged_task<Status()> task([this, tablet, compaction_type]() {
- return _execute_compaction_callback(tablet, compaction_type);
- });
- std::future<Status> future_obj = task.get_future();
- std::thread(std::move(task)).detach();
+ // 3. execute compaction task
+ std::packaged_task<Status()> task([this, tablet, compaction_type]() {
+ return _execute_compaction_callback(tablet, compaction_type);
+ });
+ std::future<Status> future_obj = task.get_future();
+ std::thread(std::move(task)).detach();
- // 4. wait for result for 2 seconds by async
- std::future_status status = future_obj.wait_for(std::chrono::seconds(2));
- if (status == std::future_status::ready) {
- // fetch execute result
- Status olap_status = future_obj.get();
- if (!olap_status.ok()) {
- return olap_status;
+ // 4. wait for result for 2 seconds by async
+ std::future_status status =
future_obj.wait_for(std::chrono::seconds(2));
+ if (status == std::future_status::ready) {
+ // fetch execute result
+ Status olap_status = future_obj.get();
+ if (!olap_status.ok()) {
+ return olap_status;
+ }
+ } else {
+ LOG(INFO) << "Manual compaction task is timeout for waiting "
+ << (status == std::future_status::timeout);
}
- } else {
- LOG(INFO) << "Manual compaction task is timeout for waiting "
- << (status == std::future_status::timeout);
}
-
LOG(INFO) << "Manual compaction task is successfully triggered";
*json_result =
- "{\"status\": \"Success\", \"msg\": \"compaction task is
successfully triggered.\"}";
-
+ "{\"status\": \"Success\", \"msg\": \"compaction task is
successfully triggered. Table "
+ "id: " +
+ std::to_string(table_id) + ". Tablet id: " +
std::to_string(tablet_id) + "\"}";
return Status::OK();
}
Status CompactionAction::_handle_run_status_compaction(HttpRequest* req,
std::string* json_result) {
uint64_t tablet_id = 0;
+ uint64_t table_id = 0;
// check req_tablet_id is not empty
- RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param
failed");
+ RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id, &table_id),
"check param failed");
if (tablet_id == 0) {
// overall compaction status
diff --git a/be/src/http/action/compaction_action.h
b/be/src/http/action/compaction_action.h
index 9f52c625f1..1f0e67c6e9 100644
--- a/be/src/http/action/compaction_action.h
+++ b/be/src/http/action/compaction_action.h
@@ -66,7 +66,7 @@ private:
Status _handle_run_status_compaction(HttpRequest* req, std::string*
json_result);
/// check param and fetch tablet_id from req
- Status _check_param(HttpRequest* req, uint64_t* tablet_id);
+ Status _check_param(HttpRequest* req, uint64_t* tablet_id, uint64_t*
table_id);
private:
CompactionActionType _type;
diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h
index ff896e0ff8..e0e1d919a5 100644
--- a/be/src/olap/olap_define.h
+++ b/be/src/olap/olap_define.h
@@ -149,6 +149,7 @@ static const std::string END_ROWSET_ID = "end_rowset_id";
static const std::string CONVERTED_FLAG = "true";
static const std::string TABLET_CONVERT_FINISHED = "tablet_convert_finished";
const std::string TABLET_ID_KEY = "tablet_id";
+const std::string TABLE_ID_KEY = "table_id";
const std::string ENABLE_BYTE_TO_BASE64 = "byte_to_base64";
const std::string TABLET_ID_PREFIX = "t_";
const std::string ROWSET_ID_PREFIX = "s_";
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 572e545c96..7956d41cd2 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -54,6 +54,7 @@
#include "olap/binlog.h"
#include "olap/cumulative_compaction.h"
#include "olap/data_dir.h"
+#include "olap/full_compaction.h"
#include "olap/memtable_flush_executor.h"
#include "olap/olap_define.h"
#include "olap/olap_meta.h"
@@ -1142,6 +1143,11 @@ void
StorageEngine::create_base_compaction(TabletSharedPtr best_tablet,
base_compaction.reset(new BaseCompaction(best_tablet));
}
+void StorageEngine::create_full_compaction(TabletSharedPtr best_tablet,
+ std::shared_ptr<FullCompaction>&
full_compaction) {
+ full_compaction.reset(new FullCompaction(best_tablet));
+}
+
void StorageEngine::create_single_replica_compaction(
TabletSharedPtr best_tablet,
std::shared_ptr<SingleReplicaCompaction>& single_replica_compaction,
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index c4912ea5bb..9ab23463f2 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -187,6 +187,9 @@ public:
void create_base_compaction(TabletSharedPtr best_tablet,
std::shared_ptr<BaseCompaction>&
base_compaction);
+ void create_full_compaction(TabletSharedPtr best_tablet,
+ std::shared_ptr<FullCompaction>&
full_compaction);
+
void create_single_replica_compaction(
TabletSharedPtr best_tablet,
std::shared_ptr<SingleReplicaCompaction>&
single_replica_compaction,
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 485c434bff..dd3ae58136 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -79,6 +79,7 @@
#include "olap/cumulative_compaction_policy.h"
#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/delete_bitmap_calculator.h"
+#include "olap/full_compaction.h"
#include "olap/memtable.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
@@ -1723,7 +1724,7 @@ Status
Tablet::prepare_compaction_and_calculate_permits(CompactionType compactio
return Status::OK();
}
compaction_rowsets = _cumulative_compaction->get_input_rowsets();
- } else {
+ } else if (compaction_type == CompactionType::BASE_COMPACTION) {
DCHECK_EQ(compaction_type, CompactionType::BASE_COMPACTION);
MonotonicStopWatch watch;
watch.start();
@@ -1754,6 +1755,24 @@ Status
Tablet::prepare_compaction_and_calculate_permits(CompactionType compactio
return Status::OK();
}
compaction_rowsets = _base_compaction->get_input_rowsets();
+ } else {
+ DCHECK_EQ(compaction_type, CompactionType::FULL_COMPACTION);
+ MonotonicStopWatch watch;
+ watch.start();
+ StorageEngine::instance()->create_full_compaction(tablet,
_full_compaction);
+ Status res = _full_compaction->prepare_compact();
+ if (!res.ok()) {
+ set_last_full_compaction_failure_time(UnixMillis());
+ *permits = 0;
+ if (!res.is<BE_NO_SUITABLE_VERSION>()) {
+ return Status::InternalError("prepare full compaction with
err: {}", res);
+ }
+ // return OK if OLAP_ERR_BE_NO_SUITABLE_VERSION, so that we don't
need to
+ // print too much useless logs.
+ // And because we set permits to 0, so even if we return OK here,
nothing will be done.
+ return Status::OK();
+ }
+ compaction_rowsets = _full_compaction->get_input_rowsets();
}
*permits = 0;
for (auto rowset : compaction_rowsets) {
@@ -1839,7 +1858,7 @@ void Tablet::execute_compaction(CompactionType
compaction_type) {
return;
}
set_last_cumu_compaction_failure_time(0);
- } else {
+ } else if (compaction_type == CompactionType::BASE_COMPACTION) {
DCHECK_EQ(compaction_type, CompactionType::BASE_COMPACTION);
MonotonicStopWatch watch;
watch.start();
@@ -1863,14 +1882,28 @@ void Tablet::execute_compaction(CompactionType
compaction_type) {
return;
}
set_last_base_compaction_failure_time(0);
+ } else {
+ DCHECK_EQ(compaction_type, CompactionType::FULL_COMPACTION);
+ MonotonicStopWatch watch;
+ watch.start();
+ Status res = _full_compaction->execute_compact();
+ if (!res.ok()) {
+ set_last_full_compaction_failure_time(UnixMillis());
+ LOG(WARNING) << "failed to do full compaction. res=" << res
+ << ", tablet=" << full_name();
+ return;
+ }
+ set_last_full_compaction_failure_time(0);
}
}
void Tablet::reset_compaction(CompactionType compaction_type) {
if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
_cumulative_compaction.reset();
- } else {
+ } else if (compaction_type == CompactionType::BASE_COMPACTION) {
_base_compaction.reset();
+ } else {
+ _full_compaction.reset();
}
}
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index a7e4749636..8ba3f60c3d 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -62,6 +62,7 @@ class Tablet;
class CumulativeCompactionPolicy;
class CumulativeCompaction;
class BaseCompaction;
+class FullCompaction;
class SingleReplicaCompaction;
class RowsetWriter;
struct TabletTxnInfo;
@@ -665,6 +666,7 @@ private:
std::shared_ptr<CumulativeCompaction> _cumulative_compaction;
std::shared_ptr<BaseCompaction> _base_compaction;
+ std::shared_ptr<FullCompaction> _full_compaction;
std::shared_ptr<SingleReplicaCompaction> _single_replica_compaction;
// whether clone task occurred during the tablet is in thread pool queue
to wait for compaction
diff --git a/docs/en/docs/admin-manual/data-admin/data-recovery.md
b/docs/en/docs/admin-manual/data-admin/data-recovery.md
new file mode 100644
index 0000000000..c4439b1bd5
--- /dev/null
+++ b/docs/en/docs/admin-manual/data-admin/data-recovery.md
@@ -0,0 +1,54 @@
+---
+{
+ "title": "Data Recovery",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Data Recovery
+
+For the Unique Key Merge on Write table, there are bugs in some Doris
versions, which may cause errors when the system calculates the delete bitmap,
resulting in duplicate primary keys. At this time, the full compaction function
can be used to repair the data. This function is invalid for non-Unique Key
Merge on Write tables.
+
+This feature requires Doris version 2.0+.
+
+To use this function, it is necessary to stop the import as much as possible,
otherwise problems such as import timeout may occur.
+
+## Brief principle explanation
+
+After the full compaction is executed, the delete bitmap will be recalculated,
and the wrong delete bitmap data will be deleted to complete the data
restoration.
+
+## Instructions for use
+
+`POST /api/compaction/run?tablet_id={int}&compact_type=full`
+
+or
+
+`POST /api/compaction/run?table_id={int}&compact_type=full`
+
+Note that only one tablet_id and table_id can be specified, and cannot be
specified at the same time. After specifying table_id, full_compaction will be
automatically executed for all tablets under this table.
+
+## Example of use
+
+```
+curl -X POST
"http://127.0.0.1:8040/api/compaction/run?tablet_id=10015&compact_type=full";
+curl -X POST
"http://127.0.0.1:8040/api/compaction/run?table_id=10104&compact_type=full";
+```
\ No newline at end of file
diff --git a/docs/en/docs/admin-manual/http-actions/be/compaction-run.md
b/docs/en/docs/admin-manual/http-actions/be/compaction-run.md
index a1944acd9b..858eda5256 100644
--- a/docs/en/docs/admin-manual/http-actions/be/compaction-run.md
+++ b/docs/en/docs/admin-manual/http-actions/be/compaction-run.md
@@ -29,6 +29,7 @@ under the License.
## Request
`POST /api/compaction/run?tablet_id={int}&compact_type={enum}`
+`POST /api/compaction/run?table_id={int}&compact_type=full` Note that
table_id=xxx will take effect only when compact_type=full is specified.
`GET /api/compaction/run_status?tablet_id={int}`
@@ -41,8 +42,11 @@ Used to manually trigger the comparison and show status.
* `tablet_id`
- ID of the tablet
+* `table_id`
+ - ID of table. Note that table_id=xxx will take effect only when
compact_type=full is specified, and only one tablet_id and table_id can be
specified, and cannot be specified at the same time. After specifying table_id,
full_compaction will be automatically executed for all tablets under this table.
+
* `compact_type`
- - The value is `base` or `cumulative`
+ - The value is `base` or `cumulative` or `full`. For usage scenarios of
full_compaction, please refer to [Data
Recovery](../../data-admin/data-recovery.md).
## Request body
diff --git a/docs/sidebars.json b/docs/sidebars.json
index fd507c4f69..c1329d4f47 100644
--- a/docs/sidebars.json
+++ b/docs/sidebars.json
@@ -1103,6 +1103,7 @@
"items": [
"admin-manual/data-admin/backup",
"admin-manual/data-admin/restore",
+ "admin-manual/data-admin/data-recovery",
"admin-manual/data-admin/delete-recover"
]
},
diff --git a/docs/zh-CN/docs/admin-manual/data-admin/data-recovery.md
b/docs/zh-CN/docs/admin-manual/data-admin/data-recovery.md
new file mode 100644
index 0000000000..1cff225d7e
--- /dev/null
+++ b/docs/zh-CN/docs/admin-manual/data-admin/data-recovery.md
@@ -0,0 +1,54 @@
+---
+{
+ "title": "数据恢复",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# 数据恢复
+
+对于Unique Key Merge on Write表，在某些Doris的版本中存在bug，可能会导致系统在计算delete
bitmap时出现错误，导致出现重复主键，此时可以利用full compaction功能进行数据的修复。本功能对于非Unique Key Merge on
Write表无效。
+
+该功能需要 Doris 版本 2.0+。
+
+使用该功能，需要尽可能停止导入，否则可能会出现导入超时等问题。
+
+## 简要原理说明
+
+执行full compaction后，会对delete bitmap进行重新计算，将错误的delete bitmap数据删除，以完成数据的修复。
+
+## 使用说明
+
+`POST /api/compaction/run?tablet_id={int}&compact_type=full`
+
+或
+
+`POST /api/compaction/run?table_id={int}&compact_type=full`
+
+注意，tablet_id和table_id只能指定一个，不能够同时指定，指定table_id后会自动对此table下所有tablet执行full_compaction。
+
+## 使用例子
+
+```
+curl -X POST
"http://127.0.0.1:8040/api/compaction/run?tablet_id=10015&compact_type=full";
+curl -X POST
"http://127.0.0.1:8040/api/compaction/run?table_id=10104&compact_type=full";
+```
\ No newline at end of file
diff --git a/docs/zh-CN/docs/admin-manual/http-actions/be/compaction-run.md
b/docs/zh-CN/docs/admin-manual/http-actions/be/compaction-run.md
index f922ec310f..8120fbaaaf 100644
--- a/docs/zh-CN/docs/admin-manual/http-actions/be/compaction-run.md
+++ b/docs/zh-CN/docs/admin-manual/http-actions/be/compaction-run.md
@@ -29,6 +29,7 @@ under the License.
## Request
`POST /api/compaction/run?tablet_id={int}&compact_type={enum}`
+`POST /api/compaction/run?table_id={int}&compact_type=full`
注意，table_id=xxx只有在compact_type=full时指定才会生效。
`GET /api/compaction/run_status?tablet_id={int}`
@@ -41,8 +42,11 @@ under the License.
* `tablet_id`
- tablet的id
+* `table_id`
+ -
table的id。注意，table_id=xxx只有在compact_type=full时指定才会生效，并且tablet_id和table_id只能指定一个，不能够同时指定，指定table_id后会自动对此table下所有tablet执行full_compaction。
+
* `compact_type`
- - 取值为`base`或`cumulative`
+ -
取值为`base`或`cumulative`或`full`。full_compaction的使用场景请参考[数据恢复](../../data-admin/data-recovery.md)。
## Request body
diff --git
a/regression-test/data/compaction/test_full_compaction_by_table_id.out
b/regression-test/data/compaction/test_full_compaction_by_table_id.out
new file mode 100644
index 0000000000..b5ed2dffc1
--- /dev/null
+++ b/regression-test/data/compaction/test_full_compaction_by_table_id.out
@@ -0,0 +1,41 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !1 --
+1 1
+2 2
+
+-- !2 --
+1 10
+2 20
+
+-- !3 --
+1 100
+2 200
+
+-- !4 --
+1 100
+2 200
+3 300
+
+-- !5 --
+1 100
+2 200
+3 100
+
+-- !6 --
+1 100
+2 200
+
+-- !skip_delete --
+1 1
+1 10
+1 100
+2 2
+2 20
+2 200
+3 300
+3 100
+
+-- !select_final --
+1 100
+2 200
+
diff --git a/regression-test/plugins/plugin_curl_requester.groovy
b/regression-test/plugins/plugin_curl_requester.groovy
index a268339e63..d9b9913f4f 100644
--- a/regression-test/plugins/plugin_curl_requester.groovy
+++ b/regression-test/plugins/plugin_curl_requester.groovy
@@ -66,4 +66,8 @@ Suite.metaClass.be_run_full_compaction = { String ip, String
port, String tablet
return curl("POST",
String.format("http://%s:%s/api/compaction/run?tablet_id=%s&compact_type=full";,
ip, port, tablet_id))
}
+Suite.metaClass.be_run_full_compaction_by_table_id = { String ip, String port,
String table_id /* param */->
+ return curl("POST",
String.format("http://%s:%s/api/compaction/run?table_id=%s&compact_type=full";,
ip, port, table_id))
+}
+
logger.info("Added 'be_run_full_compaction' function to Suite")
diff --git
a/regression-test/suites/compaction/test_full_compaction_by_table_id.groovy
b/regression-test/suites/compaction/test_full_compaction_by_table_id.groovy
new file mode 100644
index 0000000000..9170051cf2
--- /dev/null
+++ b/regression-test/suites/compaction/test_full_compaction_by_table_id.groovy
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_full_compaction_by_table_id") {
+ def tableName = "test_full_compaction_by_table_id"
+
+ try {
+ String backend_id;
+
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP,
backendId_to_backendHttpPort);
+
+ backend_id = backendId_to_backendIP.keySet()[0]
+ def (code, out, err) =
show_be_config(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id))
+ logger.info("Show config: code=" + code + ", out=" + out + ", err=" +
err)
+ assertEquals(code, 0)
+ def configList = parseJson(out.trim())
+ assert configList instanceof List
+
+ boolean disableAutoCompaction = true
+ for (Object ele in (List) configList) {
+ assert ele instanceof List<String>
+ if (((List<String>) ele)[0] == "disable_auto_compaction") {
+ disableAutoCompaction = Boolean.parseBoolean(((List<String>)
ele)[2])
+ }
+ }
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE ${tableName} (
+ `user_id` INT NOT NULL, `value` INT NOT NULL)
+ UNIQUE KEY(`user_id`)
+ DISTRIBUTED BY HASH(`user_id`)
+ BUCKETS 32
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true",
+ "enable_unique_key_merge_on_write" = "true");"""
+
+ // version1 (1,1)(2,2)
+ sql """ INSERT INTO ${tableName} VALUES
+ (1,1),(2,2)
+ """
+ qt_1 """select * from ${tableName} order by user_id"""
+
+
+ // version2 (1,10)(2,20)
+ sql """ INSERT INTO ${tableName} VALUES
+ (1,10),(2,20)
+ """
+ qt_2 """select * from ${tableName} order by user_id"""
+
+
+ // version3 (1,100)(2,200)
+ sql """ INSERT INTO ${tableName} VALUES
+ (1,100),(2,200)
+ """
+ qt_3 """select * from ${tableName} order by user_id"""
+
+
+ // version4 (1,100)(2,200)(3,300)
+ sql """ INSERT INTO ${tableName} VALUES
+ (3,300)
+ """
+ qt_4 """select * from ${tableName} order by user_id"""
+
+
+ // version5 (1,100)(2,200)(3,100)
+ sql """update ${tableName} set value = 100 where user_id = 3"""
+ qt_5 """select * from ${tableName} order by user_id"""
+
+
+ // version6 (1,100)(2,200)
+ sql """delete from ${tableName} where user_id = 3"""
+ qt_6 """select * from ${tableName} order by user_id"""
+
+ sql "SET skip_delete_predicate = true"
+ sql "SET skip_delete_sign = true"
+ sql "SET skip_delete_bitmap = true"
+ // show all hidden data
+ // (1,10)(1,100)(2,2)(2,20)(2,200)(3,300)(3,100)
+ qt_skip_delete """select * from ${tableName} order by user_id"""
+
+
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+ String[][] tablets = sql """ show tablets from ${tableName}; """
+
+ // before full compaction, there are 7 rowsets in all tablets.
+ for (int i=0; i<tablets.size(); ++i) {
+ int rowsetCount = 0
+ (code, out, err) = curl("GET", tablets[i][18])
+ logger.info("Show tablets status: code=" + code + ", out=" + out +
", err=" + err)
+ assertEquals(code, 0)
+ def tabletJson = parseJson(out.trim())
+ assert tabletJson.rowsets instanceof List
+ rowsetCount =((List<String>) tabletJson.rowsets).size()
+ assertEquals (rowsetCount, 7)
+ }
+
+ // trigger full compactions for all tablets by table id in ${tableName}
+ // TODO: get table id
+ String tablet_id = tablets[0][0]
+ String[][] tablet_info = sql """ show tablet ${tablet_id}; """
+ logger.info("tablet"+tablet_info)
+ table_id = tablet_info[0][5]
+ backend_id = tablets[0][2]
+ times = 1
+
+ do{
+ (code, out, err) =
be_run_full_compaction_by_table_id(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id), table_id)
+ logger.info("Run compaction: code=" + code + ", out=" + out + ",
err=" + err)
+ ++times
+ sleep(2000)
+ } while (parseJson(out.trim()).status.toLowerCase()!="success" &&
times<=10)
+
+ def compactJson = parseJson(out.trim())
+ if (compactJson.status.toLowerCase() == "fail") {
+ assertEquals(disableAutoCompaction, false)
+ logger.info("Compaction was done automatically!")
+ }
+ if (disableAutoCompaction) {
+ assertEquals("success", compactJson.status.toLowerCase())
+ }
+
+ // wait for full compaction done
+ {
+ boolean running = true
+ do {
+ Thread.sleep(1000)
+ tablet_id = tablets[0][0]
+ backend_id = tablets[0][2]
+ (code, out, err) =
be_get_compaction_status(backendId_to_backendIP.get(backend_id),
backendId_to_backendHttpPort.get(backend_id), tablet_id)
+ logger.info("Get compaction status: code=" + code + ", out=" +
out + ", err=" + err)
+ assertEquals(code, 0)
+ def compactionStatus = parseJson(out.trim())
+ assertEquals("success", compactionStatus.status.toLowerCase())
+ running = compactionStatus.run_status
+ } while (running)
+ }
+
+ // after full compaction, there is only 1 rowset.
+
+ for (int i=0; i<tablets.size(); ++i) {
+ int rowsetCount = 0
+ (code, out, err) = curl("GET", tablets[i][18])
+ logger.info("Show tablets status: code=" + code + ", out=" + out +
", err=" + err)
+ assertEquals(code, 0)
+ def tabletJson = parseJson(out.trim())
+ assert tabletJson.rowsets instanceof List
+ rowsetCount =((List<String>) tabletJson.rowsets).size()
+ assertEquals (rowsetCount, 1)
+ }
+
+
+ // make sure all hidden data has been deleted
+ // (1,100)(2,200)
+ qt_select_final """select * from ${tableName} order by user_id"""
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
