This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b4a798240a84dfb9b7b5953f45de14c4c68e2aaf
Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com>
AuthorDate: Mon May 20 20:05:23 2024 +0800
[fix](inverted_index) donot use int32_t for index id to avoid overflow
(#35062)
---
be/src/olap/rowset/beta_rowset.cpp | 2 +-
be/src/olap/rowset/beta_rowset.h | 2 +-
be/src/olap/rowset/rowset.h | 2 +-
be/src/olap/tablet.cpp | 2 +-
be/src/olap/tablet.h | 2 +-
be/src/olap/tablet_schema.cpp | 4 ++--
be/src/olap/tablet_schema.h | 4 ++--
be/src/olap/task/index_builder.cpp | 2 +-
be/src/olap/task/index_builder.h | 2 +-
be/test/testutil/mock_rowset.h | 2 +-
10 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index fc5275eca53..decb172956c 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -262,7 +262,7 @@ void BetaRowset::do_close() {
Status BetaRowset::link_files_to(const std::string& dir, RowsetId
new_rowset_id,
size_t new_rowset_start_seg_id,
- std::set<int32_t>* without_index_uids) {
+ std::set<int64_t>* without_index_uids) {
DCHECK(is_local());
auto fs = _rowset_meta->fs();
if (!fs) {
diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h
index 404a45a1be1..abf03da7f45 100644
--- a/be/src/olap/rowset/beta_rowset.h
+++ b/be/src/olap/rowset/beta_rowset.h
@@ -71,7 +71,7 @@ public:
Status link_files_to(const std::string& dir, RowsetId new_rowset_id,
size_t new_rowset_start_seg_id = 0,
- std::set<int32_t>* without_index_uids = nullptr)
override;
+ std::set<int64_t>* without_index_uids = nullptr)
override;
Status copy_files_to(const std::string& dir, const RowsetId&
new_rowset_id) override;
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index e7675c1916a..dffa4a7a3ee 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -205,7 +205,7 @@ public:
// hard link all files in this rowset to `dir` to form a new rowset with
id `new_rowset_id`.
virtual Status link_files_to(const std::string& dir, RowsetId
new_rowset_id,
size_t new_rowset_start_seg_id = 0,
- std::set<int32_t>* without_index_uids =
nullptr) = 0;
+ std::set<int64_t>* without_index_uids =
nullptr) = 0;
// copy all files to `dir`
virtual Status copy_files_to(const std::string& dir, const RowsetId&
new_rowset_id) = 0;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 61a1961edf9..f0867b4a19d 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1373,7 +1373,7 @@ std::vector<RowsetSharedPtr>
Tablet::pick_first_consecutive_empty_rowsets(int li
}
std::vector<RowsetSharedPtr>
Tablet::pick_candidate_rowsets_to_build_inverted_index(
- const std::set<int32_t>& alter_index_uids, bool is_drop_op) {
+ const std::set<int64_t>& alter_index_uids, bool is_drop_op) {
std::vector<RowsetSharedPtr> candidate_rowsets;
{
std::shared_lock rlock(_meta_lock);
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 7a959f7272f..6eb63648b9a 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -290,7 +290,7 @@ public:
std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_base_compaction();
std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_full_compaction();
std::vector<RowsetSharedPtr>
pick_candidate_rowsets_to_build_inverted_index(
- const std::set<int32_t>& alter_index_uids, bool is_drop_op);
+ const std::set<int64_t>& alter_index_uids, bool is_drop_op);
// used for single compaction to get the local versions
// Single compaction does not require remote rowsets and cannot violate
the cooldown semantics
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 0e9376e09cf..d423ec4d709 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1273,7 +1273,7 @@ bool TabletSchema::has_inverted_index(const TabletColumn&
col) const {
return false;
}
-bool TabletSchema::has_inverted_index_with_index_id(int32_t index_id,
+bool TabletSchema::has_inverted_index_with_index_id(int64_t index_id,
const std::string&
suffix_name) const {
for (size_t i = 0; i < _indexes.size(); i++) {
if (_indexes[i].index_type() == IndexType::INVERTED &&
@@ -1285,7 +1285,7 @@ bool
TabletSchema::has_inverted_index_with_index_id(int32_t index_id,
}
const TabletIndex* TabletSchema::get_inverted_index_with_index_id(
- int32_t index_id, const std::string& suffix_name) const {
+ int64_t index_id, const std::string& suffix_name) const {
for (size_t i = 0; i < _indexes.size(); i++) {
if (_indexes[i].index_type() == IndexType::INVERTED &&
_indexes[i].get_index_suffix() == suffix_name &&
_indexes[i].index_id() == index_id) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index ef2a6808d45..384772375d1 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -346,8 +346,8 @@ public:
}
std::vector<const TabletIndex*> get_indexes_for_column(const TabletColumn&
col) const;
bool has_inverted_index(const TabletColumn& col) const;
- bool has_inverted_index_with_index_id(int32_t index_id, const std::string&
suffix_path) const;
- const TabletIndex* get_inverted_index_with_index_id(int32_t index_id,
+ bool has_inverted_index_with_index_id(int64_t index_id, const std::string&
suffix_path) const;
+ const TabletIndex* get_inverted_index_with_index_id(int64_t index_id,
const std::string&
suffix_name) const;
const TabletIndex* get_inverted_index(const TabletColumn& col) const;
const TabletIndex* get_inverted_index(int32_t col_unique_id,
diff --git a/be/src/olap/task/index_builder.cpp
b/be/src/olap/task/index_builder.cpp
index 09f745833d1..a62763b8af8 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -59,7 +59,7 @@ Status IndexBuilder::update_inverted_index_info() {
LOG(INFO) << "begin to update_inverted_index_info, tablet=" <<
_tablet->tablet_id()
<< ", is_drop_op=" << _is_drop_op;
// index ids that will not be linked
- std::set<int32_t> without_index_uids;
+ std::set<int64_t> without_index_uids;
_output_rowsets.reserve(_input_rowsets.size());
_pending_rs_guards.reserve(_input_rowsets.size());
for (auto&& input_rowset : _input_rowsets) {
diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h
index 1ed16886315..75f3b5d2ff0 100644
--- a/be/src/olap/task/index_builder.h
+++ b/be/src/olap/task/index_builder.h
@@ -69,7 +69,7 @@ private:
std::vector<TColumn> _columns;
std::vector<doris::TOlapTableIndex> _alter_inverted_indexes;
bool _is_drop_op;
- std::set<int32_t> _alter_index_ids;
+ std::set<int64_t> _alter_index_ids;
std::vector<RowsetSharedPtr> _input_rowsets;
std::vector<RowsetSharedPtr> _output_rowsets;
std::vector<PendingRowsetGuard> _pending_rs_guards;
diff --git a/be/test/testutil/mock_rowset.h b/be/test/testutil/mock_rowset.h
index 50065ebe6b4..7761b2c6b03 100644
--- a/be/test/testutil/mock_rowset.h
+++ b/be/test/testutil/mock_rowset.h
@@ -30,7 +30,7 @@ class MockRowset : public Rowset {
Status remove() override { return Status::NotSupported("MockRowset not
support this method."); }
Status link_files_to(const std::string& dir, RowsetId new_rowset_id,
size_t start_seg_id,
- std::set<int32_t>* without_index_uids) override {
+ std::set<int64_t>* without_index_uids) override {
return Status::NotSupported("MockRowset not support this method.");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
