This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3072e17b39 [Bugfix](primary-key) fix calc delete bitmap bug in
concurrent memtable flush (#12605)
3072e17b39 is described below
commit 3072e17b39ddb44a901701bb8eaf6e5961e8c1f6
Author: yixiutt <102007456+yixi...@users.noreply.github.com>
AuthorDate: Thu Sep 15 21:50:24 2022 +0800
[Bugfix](primary-key) fix calc delete bitmap bug in concurrent memtable
flush (#12605)
Co-authored-by: yixiutt <yi...@selectdb.com>
---
be/src/olap/delta_writer.cpp | 7 +++++--
be/src/olap/delta_writer.h | 2 +-
be/src/olap/memtable_flush_executor.cpp | 7 ++++---
be/src/olap/memtable_flush_executor.h | 2 +-
4 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index 7ea8a00966..d89d9e7023 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -141,8 +141,11 @@ Status DeltaWriter::init() {
_reset_mem_table();
// create flush handler
+ // unique key merge on write should flush serial cause calc delete bitmap
should load segment serial
+ bool should_serial = (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
+ _tablet->enable_unique_key_merge_on_write());
RETURN_NOT_OK(_storage_engine->memtable_flush_executor()->create_flush_token(
- &_flush_token, _rowset_writer->type(), _req.is_high_priority));
+ &_flush_token, _rowset_writer->type(), should_serial,
_req.is_high_priority));
_is_init = true;
return Status::OK();
@@ -281,7 +284,7 @@ Status DeltaWriter::wait_flush() {
}
void DeltaWriter::_reset_mem_table() {
- if (_tablet->enable_unique_key_merge_on_write()) {
+ if (_tablet->enable_unique_key_merge_on_write() && _delete_bitmap ==
nullptr) {
_delete_bitmap.reset(new DeleteBitmap(_tablet->tablet_id()));
}
_mem_table.reset(new MemTable(_tablet, _schema.get(),
_tablet_schema.get(), _req.slots,
diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h
index 476a77655b..181f9d5c1e 100644
--- a/be/src/olap/delta_writer.h
+++ b/be/src/olap/delta_writer.h
@@ -168,7 +168,7 @@ private:
PSuccessSlaveTabletNodeIds _success_slave_node_ids;
std::shared_mutex _slave_node_lock;
- DeleteBitmapPtr _delete_bitmap;
+ DeleteBitmapPtr _delete_bitmap = nullptr;
// current rowset_ids, used to do diff in publish_version
RowsetIdUnorderedSet _rowset_ids;
};
diff --git a/be/src/olap/memtable_flush_executor.cpp
b/be/src/olap/memtable_flush_executor.cpp
index 2cf159a963..bf1b6819a5 100644
--- a/be/src/olap/memtable_flush_executor.cpp
+++ b/be/src/olap/memtable_flush_executor.cpp
@@ -133,9 +133,10 @@ void MemTableFlushExecutor::init(const
std::vector<DataDir*>& data_dirs) {
// NOTE: we use SERIAL mode here to ensure all mem-tables from one tablet are
flushed in order.
Status MemTableFlushExecutor::create_flush_token(std::unique_ptr<FlushToken>*
flush_token,
- RowsetTypePB rowset_type,
bool is_high_priority) {
+ RowsetTypePB rowset_type,
bool should_serial,
+ bool is_high_priority) {
if (!is_high_priority) {
- if (rowset_type == BETA_ROWSET) {
+ if (rowset_type == BETA_ROWSET && !should_serial) {
// beta rowset can be flush in CONCURRENT, because each memtable
using a new segment writer.
flush_token->reset(
new
FlushToken(_flush_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT)));
@@ -145,7 +146,7 @@ Status
MemTableFlushExecutor::create_flush_token(std::unique_ptr<FlushToken>* fl
new
FlushToken(_flush_pool->new_token(ThreadPool::ExecutionMode::SERIAL)));
}
} else {
- if (rowset_type == BETA_ROWSET) {
+ if (rowset_type == BETA_ROWSET && !should_serial) {
// beta rowset can be flush in CONCURRENT, because each memtable
using a new segment writer.
flush_token->reset(new FlushToken(
_high_prio_flush_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT)));
diff --git a/be/src/olap/memtable_flush_executor.h
b/be/src/olap/memtable_flush_executor.h
index 71a1f3ae62..53e2cfaf98 100644
--- a/be/src/olap/memtable_flush_executor.h
+++ b/be/src/olap/memtable_flush_executor.h
@@ -107,7 +107,7 @@ public:
void init(const std::vector<DataDir*>& data_dirs);
Status create_flush_token(std::unique_ptr<FlushToken>* flush_token,
RowsetTypePB rowset_type,
- bool is_high_priority);
+ bool should_serial, bool is_high_priority);
private:
std::unique_ptr<ThreadPool> _flush_pool;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
