This is an automated email from the ASF dual-hosted git repository. zhangchen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 6913d68ba0 [Enhancement](merge-on-write) use delete bitmap to mark delete for rows with delete sign when sequence column doesn't exist (#24011) 6913d68ba0 is described below commit 6913d68ba0c40dfe2a2ee43600804b69c8d0bc12 Author: bobhan1 <bh2444151...@outlook.com> AuthorDate: Tue Sep 12 08:56:46 2023 +0800 [Enhancement](merge-on-write) use delete bitmap to mark delete for rows with delete sign when sequence column doesn't exist (#24011) --- be/src/olap/delete_bitmap_calculator.cpp | 3 +- be/src/olap/rowset/segment_v2/segment_writer.cpp | 41 ++++++++- be/src/olap/tablet.cpp | 25 ++++-- be/src/olap/tablet_meta.h | 7 ++ .../test_partial_update_delete_sign.out | 34 +++++++- .../test_delete_sign_delete_bitmap.out | 54 ++++++++++++ .../test_partial_update_delete_sign.groovy | 61 +++++++++++++- .../test_delete_sign_delete_bitmap.groovy | 96 ++++++++++++++++++++++ 8 files changed, 305 insertions(+), 16 deletions(-) diff --git a/be/src/olap/delete_bitmap_calculator.cpp b/be/src/olap/delete_bitmap_calculator.cpp index bfdb506c06..b1d9226eaf 100644 --- a/be/src/olap/delete_bitmap_calculator.cpp +++ b/be/src/olap/delete_bitmap_calculator.cpp @@ -192,7 +192,8 @@ Status MergeIndexDeleteBitmapCalculator::calculate_all(DeleteBitmapPtr delete_bi break; } RETURN_IF_ERROR(st); - delete_bitmap->add({_rowset_id, loc.segment_id, 0}, loc.row_id); + delete_bitmap->add({_rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, + loc.row_id); } return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 775d9b5a8a..0fb4ee0cd5 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -423,7 +423,8 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* if (_tablet_schema->is_strict_mode()) { ++num_rows_filtered; // delete the invalid newly inserted row - _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id, 0}, + _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id, + DeleteBitmap::TEMP_VERSION_COMMON}, segment_pos); } @@ -447,6 +448,13 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* if (delete_sign_column_data != nullptr && delete_sign_column_data[block_pos] != 0) { has_default_or_nullable = true; use_default_or_null_flag.emplace_back(true); + if (!_tablet_schema->has_sequence_col() && !have_input_seq_column) { + // we can directly use delete bitmap to mark the rows with delete sign as deleted + // if sequence column doesn't exist to eliminate reading delete sign columns in later reads + _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id, + DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN}, + segment_pos); + } } else { // partial update should not contain invisible columns use_default_or_null_flag.emplace_back(false); @@ -457,10 +465,12 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* if (st.is<KEY_ALREADY_EXISTS>()) { // although we need to mark delete current row, we still need to read missing columns // for this row, we need to ensure that each column is aligned - _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id, 0}, - segment_pos); + _mow_context->delete_bitmap->add( + {_opts.rowset_ctx->rowset_id, _segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, + segment_pos); } else { - _mow_context->delete_bitmap->add({loc.rowset_id, loc.segment_id, 0}, loc.row_id); + _mow_context->delete_bitmap->add( + {loc.rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, loc.row_id); } } CHECK(use_default_or_null_flag.size() == num_rows); @@ -638,6 +648,29 @@ Status SegmentWriter::append_block(const vectorized::Block* block, size_t row_po _serialize_block_to_row_column(*const_cast<vectorized::Block*>(block)); } + if (_opts.write_type == DataWriteType::TYPE_DIRECT && _opts.enable_unique_key_merge_on_write && + !_tablet_schema->has_sequence_col() && _tablet_schema->delete_sign_idx() != -1) { + const vectorized::ColumnWithTypeAndName& delete_sign_column = + block->get_by_position(_tablet_schema->delete_sign_idx()); + auto& delete_sign_col = + reinterpret_cast<const vectorized::ColumnInt8&>(*(delete_sign_column.column)); + if (delete_sign_col.size() >= row_pos + num_rows) { + const vectorized::Int8* delete_sign_column_data = delete_sign_col.get_data().data(); + uint32_t segment_start_pos = + _column_writers[_tablet_schema->delete_sign_idx()]->get_next_rowid(); + for (size_t block_pos = row_pos, seg_pos = segment_start_pos; + seg_pos < segment_start_pos + num_rows; block_pos++, seg_pos++) { + // we can directly use delete bitmap to mark the rows with delete sign as deleted + // if sequence column doesn't exist to eliminate reading delete sign columns in later reads + if (delete_sign_column_data[block_pos]) { + _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id, + DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN}, + seg_pos); + } + } + } + } + _olap_data_convertor->set_source_content(block, row_pos, num_rows); // find all row pos for short key indexes diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 7c737e4d3c..d4029f22be 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2971,7 +2971,8 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, Slice key = Slice(index_column->get_data_at(i).data, index_column->get_data_at(i).size); RowLocation loc; // same row in segments should be filtered - if (delete_bitmap->contains({rowset_id, seg->id(), 0}, row_id)) { + if (delete_bitmap->contains({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, + row_id)) { continue; } @@ -2989,7 +2990,8 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, // sequence id smaller than the previous one, so delete current row if (st.is<KEY_ALREADY_EXISTS>()) { - delete_bitmap->add({rowset_id, seg->id(), 0}, row_id); + delete_bitmap->add({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, + row_id); continue; } else if (is_partial_update && rowset_writer != nullptr) { // In publish version, record rows to be deleted for concurrent update @@ -3011,12 +3013,16 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, // rowset. // just set 0 as a unified temporary version number, and update to // the real version number later. - delete_bitmap->add({loc.rowset_id, loc.segment_id, 0}, loc.row_id); - delete_bitmap->add({rowset_id, seg->id(), 0}, row_id); + delete_bitmap->add( + {loc.rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, + loc.row_id); + delete_bitmap->add({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, + row_id); continue; } // when st = ok - delete_bitmap->add({loc.rowset_id, loc.segment_id, 0}, loc.row_id); + delete_bitmap->add({loc.rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, + loc.row_id); } remaining -= num_read; } @@ -3717,8 +3723,9 @@ Status Tablet::calc_delete_bitmap_between_segments( void Tablet::add_sentinel_mark_to_delete_bitmap(DeleteBitmap* delete_bitmap, const RowsetIdUnorderedSet& rowsetids) { for (const auto& rowsetid : rowsetids) { - delete_bitmap->add({rowsetid, DeleteBitmap::INVALID_SEGMENT_ID, 0}, - DeleteBitmap::ROWSET_SENTINEL_MARK); + delete_bitmap->add( + {rowsetid, DeleteBitmap::INVALID_SEGMENT_ID, DeleteBitmap::TEMP_VERSION_COMMON}, + DeleteBitmap::ROWSET_SENTINEL_MARK); } } @@ -3739,8 +3746,8 @@ Status Tablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, in std::vector<RowsetSharedPtr>* rowsets) { RowsetIdUnorderedSet missing_ids; for (const auto& rowsetid : rowset_ids) { - if (!delete_bitmap->delete_bitmap.contains( - {rowsetid, DeleteBitmap::INVALID_SEGMENT_ID, 0})) { + if (!delete_bitmap->delete_bitmap.contains({rowsetid, DeleteBitmap::INVALID_SEGMENT_ID, + DeleteBitmap::TEMP_VERSION_COMMON})) { missing_ids.insert(rowsetid); } } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index c0dcdc249a..42fec6489b 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -340,6 +340,13 @@ public: constexpr static inline uint32_t ROWSET_SENTINEL_MARK = std::numeric_limits<uint32_t>::max() - 1; + // When a delete bitmap is merged into tablet's delete bitmap, the version of entries in the delete bitmap + // will be replaced to the correspoding correct version. So before we finally merge a delete bitmap into + // tablet's delete bitmap we can use arbitary version number in BitmapKey. Here we define some version numbers + // for specific usage during this periods to avoid conflicts + constexpr static inline uint64_t TEMP_VERSION_COMMON = 0; + constexpr static inline uint64_t TEMP_VERSION_FOR_DELETE_SIGN = 1; + /** * * @param tablet_id the tablet which this delete bitmap associates with diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out index f66f4b6873..0f1cd09e7d 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out @@ -10,7 +10,7 @@ 2 2 2 2 2 4 4 4 4 4 --- !with_delete_sign -- +-- !1 -- 1 \N \N \N \N 1 1 1 1 1 1 0 2 2 2 2 2 0 @@ -20,3 +20,35 @@ 5 \N \N \N \N 1 5 5 5 5 5 0 +-- !2 -- +2 2 2 2 2 0 +4 4 4 4 4 0 + +-- !sql -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !after_delete -- +2 2 2 2 2 +4 4 4 4 4 + +-- !1 -- +1 \N \N \N \N 1 +1 1 1 1 1 0 +2 2 2 2 2 0 +3 \N \N \N \N 1 +3 3 3 3 3 0 +4 4 4 4 4 0 +5 \N \N \N \N 1 +5 5 5 5 5 0 + +-- !2 -- +1 \N \N \N \N 1 +2 2 2 2 2 0 +3 \N \N \N \N 1 +4 4 4 4 4 0 +5 \N \N \N \N 1 + diff --git a/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out b/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out new file mode 100644 index 0000000000..687aeab54a --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !after_delete -- +2 2 2 2 2 +4 4 4 4 4 + +-- !1 -- +1 1 1 1 1 0 +1 1 1 1 1 1 +2 2 2 2 2 0 +3 3 3 3 3 0 +3 3 3 3 3 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +5 5 5 5 5 1 + +-- !2 -- +2 2 2 2 2 0 +4 4 4 4 4 0 + +-- !sql -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !after_delete -- +2 2 2 2 2 +4 4 4 4 4 + +-- !1 -- +1 1 1 1 1 0 +1 1 1 1 1 1 +2 2 2 2 2 0 +3 3 3 3 3 0 +3 3 3 3 3 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +5 5 5 5 5 1 + +-- !2 -- +1 1 1 1 1 1 +2 2 2 2 2 0 +3 3 3 3 3 1 +4 4 4 4 4 0 +5 5 5 5 5 1 + diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy index 1aa651c48c..f8369f0ed3 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy @@ -55,6 +55,65 @@ suite('test_partial_update_delete_sign') { sql "set skip_storage_engine_merge=true;" sql "set skip_delete_bitmap=true;" sql "sync" - qt_with_delete_sign "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + // skip_delete_bitmap=true, skip_delete_sign=true + qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=false;" + sql "sync" + // skip_delete_bitmap=false, skip_delete_sign=true + qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" sql "drop table if exists ${tableName1};" + + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + def tableName2 = "test_partial_update_delete_sign2" + sql "DROP TABLE IF EXISTS ${tableName2};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "function_column.sequence_col" = 'c4' + );""" + + sql "insert into ${tableName2} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" + qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;" + streamLoad { + table "${tableName2}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'partial_columns', 'true' + set 'columns', 'k1,__DORIS_DELETE_SIGN__' + + file 'delete_sign.csv' + time 10000 // limit inflight 10s + } + sql "sync" + qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + // skip_delete_bitmap=true, skip_delete_sign=true + qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=false;" + sql "sync" + // skip_delete_bitmap=false, skip_delete_sign=true + qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName2};" } diff --git a/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy b/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy new file mode 100644 index 0000000000..f9b89c1eea --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/test_delete_sign_delete_bitmap.groovy @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_delete_sign_delete_bitmap') { + + def tableName1 = "test_delete_sign_delete_bitmap1" + sql "DROP TABLE IF EXISTS ${tableName1};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1" + );""" + + sql "insert into ${tableName1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" + qt_sql "select * from ${tableName1} order by k1,c1,c2,c3,c4;" + // sql "insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__) select k1,c1,c2,c3,c4,1 from ${tableName1} where k1 in (1,3,5);" + sql """insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__) values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);""" + sql "sync" + qt_after_delete "select * from ${tableName1} order by k1,c1,c2,c3,c4;" + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + // skip_delete_bitmap=true, skip_delete_sign=true + qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=false;" + sql "sync" + // skip_delete_bitmap=false, skip_delete_sign=true + qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName1};" + + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + def tableName2 = "test_delete_sign_delete_bitmap2" + sql "DROP TABLE IF EXISTS ${tableName2};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "function_column.sequence_col" = 'c4' + );""" + + sql "insert into ${tableName2} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" + qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;" + sql """insert into ${tableName2}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__) values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);""" + sql "sync" + qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;" + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + // skip_delete_bitmap=true, skip_delete_sign=true + qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=false;" + sql "sync" + // skip_delete_bitmap=false, skip_delete_sign=true + qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName2};" +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org