This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 790b63932e5612ec34e75d73ab36091f8cd43d44 Author: zhannngchen <[email protected]> AuthorDate: Sat May 13 08:43:36 2023 +0800 [fix](mow) revert 17147 and 18750 (#19583) --- be/src/olap/delta_writer.cpp | 3 +-- be/src/olap/rowset/beta_rowset_writer.cpp | 9 --------- be/src/olap/rowset/beta_rowset_writer.h | 8 +------- be/src/olap/rowset/segment_v2/segment_writer.cpp | 7 ------- be/src/olap/rowset/segment_v2/segment_writer.h | 6 ------ be/src/olap/tablet.cpp | 18 ------------------ be/src/olap/tablet_meta.cpp | 8 -------- be/src/olap/tablet_meta.h | 2 -- be/src/olap/txn_manager.cpp | 12 ++++++------ be/src/olap/txn_manager.h | 3 +-- 10 files changed, 9 insertions(+), 67 deletions(-) diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index ef29020ad9..cdf572f4dc 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -396,8 +396,7 @@ Status DeltaWriter::close_wait(const PSlaveTabletNodes& slave_tablet_nodes, if (_tablet->enable_unique_key_merge_on_write()) { _storage_engine->txn_manager()->set_txn_related_delete_bitmap( _req.partition_id, _req.txn_id, _tablet->tablet_id(), _tablet->schema_hash(), - _tablet->tablet_uid(), true, _delete_bitmap, _rowset_ids, - dynamic_cast<BetaRowsetWriter*>(_rowset_writer.get())->get_num_mow_keys()); + _tablet->tablet_uid(), true, _delete_bitmap, _rowset_ids); } _delta_written_success = true; diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 37fb5de5e0..d2b761b5ed 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -832,7 +832,6 @@ void BetaRowsetWriter::_build_rowset_meta(std::shared_ptr<RowsetMeta> rowset_met int64_t total_data_size = 0; int64_t total_index_size = 0; std::vector<KeyBoundsPB> segments_encoded_key_bounds; - std::unordered_set<std::string> key_set; { std::lock_guard<std::mutex> lock(_segid_statistics_map_mutex); for (const auto& itr : _segid_statistics_map) { @@ -840,15 +839,8 @@ void BetaRowsetWriter::_build_rowset_meta(std::shared_ptr<RowsetMeta> rowset_met total_data_size += itr.second.data_size; total_index_size += itr.second.index_size; segments_encoded_key_bounds.push_back(itr.second.key_bounds); -#ifndef NDEBUG - if (_context.enable_unique_key_merge_on_write) { - DCHECK(itr.second.key_set.get() != nullptr); - key_set.insert(itr.second.key_set->begin(), itr.second.key_set->end()); - } -#endif } } - _num_mow_keys = key_set.size(); for (auto itr = _segments_encoded_key_bounds.begin(); itr != _segments_encoded_key_bounds.end(); ++itr) { segments_encoded_key_bounds.push_back(*itr); @@ -993,7 +985,6 @@ Status BetaRowsetWriter::_flush_segment_writer(std::unique_ptr<segment_v2::Segme segstat.data_size = segment_size; segstat.index_size = index_size; segstat.key_bounds = key_bounds; - segstat.key_set = (*writer)->get_key_set(); { std::lock_guard<std::mutex> lock(_segid_statistics_map_mutex); CHECK_EQ(_segid_statistics_map.find(segid) == _segid_statistics_map.end(), true); diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h index 9668d2e7fb..8a44efc0fc 100644 --- a/be/src/olap/rowset/beta_rowset_writer.h +++ b/be/src/olap/rowset/beta_rowset_writer.h @@ -84,8 +84,6 @@ public: int32_t get_atomic_num_segment() const override { return _num_segment.load(); } - uint64_t get_num_mow_keys() { return _num_mow_keys; } - private: template <typename RowType> Status _add_row(const RowType& row); @@ -168,13 +166,9 @@ protected: int64_t data_size; int64_t index_size; KeyBoundsPB key_bounds; - std::shared_ptr<std::unordered_set<std::string>> key_set; }; - std::mutex _segid_statistics_map_mutex; std::map<uint32_t, Statistics> _segid_statistics_map; - - // used for check correctness of unique key mow keys. - std::atomic<uint64_t> _num_mow_keys; + std::mutex _segid_statistics_map_mutex; bool _is_pending = false; bool _already_built = false; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 2cc6f0a361..e9a5f11285 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -148,9 +148,6 @@ Status SegmentWriter::init(const std::vector<uint32_t>& col_ids, bool has_key) { _primary_key_index_builder.reset( new PrimaryKeyIndexBuilder(_file_writer, seq_col_length)); RETURN_IF_ERROR(_primary_key_index_builder->init()); -#ifndef NDEBUG - _key_set.reset(new std::unordered_set<std::string>()); -#endif } else { _short_key_index_builder.reset( new ShortKeyIndexBuilder(_segment_id, _opts.num_rows_per_block)); @@ -209,10 +206,6 @@ Status SegmentWriter::append_block(const vectorized::Block* block, size_t row_po std::string last_key; for (size_t pos = 0; pos < num_rows; pos++) { std::string key = _full_encode_keys(key_columns, pos); -#ifndef NDEBUG - DCHECK(_key_set.get() != nullptr); - _key_set->insert(key); -#endif if (_tablet_schema->has_sequence_col()) { _encode_seq_column(seq_column, pos, &key); } diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h index a5b32ceca1..43c74eb2c9 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.h +++ b/be/src/olap/rowset/segment_v2/segment_writer.h @@ -56,8 +56,6 @@ class ColumnWriter; extern const char* k_segment_magic; extern const uint32_t k_segment_magic_length; -using KeySetPtr = std::shared_ptr<std::unordered_set<std::string>>; - struct SegmentWriterOptions { uint32_t num_rows_per_block = 1024; bool enable_unique_key_merge_on_write = false; @@ -99,8 +97,6 @@ public: Slice min_encoded_key(); Slice max_encoded_key(); - KeySetPtr get_key_set() { return _key_set; } - DataDir* get_data_dir() { return _data_dir; } bool is_unique_key() { return _tablet_schema->keys_type() == UNIQUE_KEYS; } @@ -154,8 +150,6 @@ private: const KeyCoder* _seq_coder = nullptr; std::vector<uint16_t> _key_index_size; size_t _short_key_row_pos = 0; - // used to check if there's duplicate key in aggregate key and unique key data model - KeySetPtr _key_set; std::vector<uint32_t> _column_ids; bool _has_key = true; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index b39d284ced..20e6c1fd5d 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2241,24 +2241,6 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, const TabletT RETURN_IF_ERROR(calc_delete_bitmap(rowset->rowset_id(), segments, &rowset_ids_to_add, delete_bitmap, cur_version - 1, true)); - // Check the delete_bitmap correctness, now the check is only enabled in DEBUG env. - if (load_info->num_keys != 0) { - DeleteBitmap rs_bm(tablet_id()); - delete_bitmap->subset({rowset->rowset_id(), 0, 0}, - {rowset->rowset_id(), UINT32_MAX, INT64_MAX}, &rs_bm); - auto num_rows = rowset->num_rows(); - auto bitmap_cardinality = rs_bm.cardinality(); - std::string err_msg = fmt::format( - "The delete bitmap of unique key table may not correct, expect num unique keys:" - "{}, " - "now the num_rows: {}, delete bitmap cardinality: {}, num sgements: {}", - load_info->num_keys, num_rows, bitmap_cardinality, rowset->num_segments()); - DCHECK_EQ(load_info->num_keys, num_rows - bitmap_cardinality) << err_msg; - if (load_info->num_keys != num_rows - bitmap_cardinality) { - return Status::InternalError(err_msg); - } - } - // update version without write lock, compaction and publish_txn // will update delete bitmap, handle compaction with _rowset_update_lock // and publish_txn runs sequential so no need to lock here diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 4978d25fb6..8fc037b85e 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -963,14 +963,6 @@ void DeleteBitmap::merge(const DeleteBitmap& other) { } } -uint64_t DeleteBitmap::cardinality() { - uint64_t cardinality = 0; - for (auto entry : delete_bitmap) { - cardinality += entry.second.cardinality(); - } - return cardinality; -} - // We cannot just copy the underlying memory to construct a string // due to equivalent objects may have different padding bytes. // Reading padding bytes is undefined behavior, neither copy nor diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index d1b25b03d3..b4dc52a7fd 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -377,8 +377,6 @@ public: */ void merge(const DeleteBitmap& other); - uint64_t cardinality(); - /** * Checks if the given row is marked deleted in bitmap with the condition: * all the bitmaps that diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index 9010e5686e..945302fa18 100644 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -169,10 +169,12 @@ Status TxnManager::prepare_txn(TPartitionId partition_id, TTransactionId transac return Status::OK(); } -void TxnManager::set_txn_related_delete_bitmap( - TPartitionId partition_id, TTransactionId transaction_id, TTabletId tablet_id, - SchemaHash schema_hash, TabletUid tablet_uid, bool unique_key_merge_on_write, - DeleteBitmapPtr delete_bitmap, const RowsetIdUnorderedSet& rowset_ids, uint64_t num_keys) { +void TxnManager::set_txn_related_delete_bitmap(TPartitionId partition_id, + TTransactionId transaction_id, TTabletId tablet_id, + SchemaHash schema_hash, TabletUid tablet_uid, + bool unique_key_merge_on_write, + DeleteBitmapPtr delete_bitmap, + const RowsetIdUnorderedSet& rowset_ids) { pair<int64_t, int64_t> key(partition_id, transaction_id); TabletInfo tablet_info(tablet_id, schema_hash, tablet_uid); @@ -194,7 +196,6 @@ void TxnManager::set_txn_related_delete_bitmap( load_info.unique_key_merge_on_write = unique_key_merge_on_write; load_info.delete_bitmap = delete_bitmap; load_info.rowset_ids = rowset_ids; - load_info.num_keys = num_keys; } } @@ -280,7 +281,6 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id, if (tablet != nullptr && tablet->enable_unique_key_merge_on_write()) { load_info.unique_key_merge_on_write = true; load_info.delete_bitmap.reset(new DeleteBitmap(tablet->tablet_id())); - load_info.num_keys = 0; } } txn_tablet_map_t& txn_tablet_map = _get_txn_tablet_map(transaction_id); diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h index 7319234b27..197307f589 100644 --- a/be/src/olap/txn_manager.h +++ b/be/src/olap/txn_manager.h @@ -58,7 +58,6 @@ struct TabletTxnInfo { // records rowsets calc in commit txn RowsetIdUnorderedSet rowset_ids; int64_t creation_time; - uint64_t num_keys; TabletTxnInfo(PUniqueId load_id, RowsetSharedPtr rowset) : load_id(load_id), @@ -168,7 +167,7 @@ public: TTabletId tablet_id, SchemaHash schema_hash, TabletUid tablet_uid, bool unique_key_merge_on_write, DeleteBitmapPtr delete_bitmap, - const RowsetIdUnorderedSet& rowset_ids, uint64_t num_keys); + const RowsetIdUnorderedSet& rowset_ids); private: using TxnKey = std::pair<int64_t, int64_t>; // partition_id, transaction_id; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
