This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0bc757088af [chore](inverted index) fix narrow conversion compilation
failure (#53826)
0bc757088af is described below
commit 0bc757088af3ed6ddd49d667429db45b1d6c5af9
Author: airborne12 <[email protected]>
AuthorDate: Thu Jul 24 21:06:36 2025 +0800
[chore](inverted index) fix narrow conversion compilation failure (#53826)
---
.../rowset/segment_v2/index_storage_format.cpp | 8 ++--
.../rowset/segment_v2/index_storage_format_v1.cpp | 9 +++--
.../rowset/segment_v2/index_storage_format_v2.cpp | 19 +++++----
.../segment_v2/inverted_index/token_stream.h | 9 +++--
.../segment_v2/inverted_index/util/mock_iterator.h | 12 ++++--
.../inverted_index/util/priority_queue.h | 8 +++-
.../inverted_index/util/union_term_iterator.h | 9 +++--
.../rowset/segment_v2/inverted_index_iterator.cpp | 5 ++-
.../rowset/segment_v2/inverted_index_iterator.h | 2 +-
.../rowset/segment_v2/inverted_index_reader.cpp | 45 ++++++++++++----------
.../olap/rowset/segment_v2/inverted_index_reader.h | 10 ++---
.../rowset/segment_v2/inverted_index_writer.cpp | 24 +++++++-----
.../segment_v2/inverted_index_reader_test.cpp | 8 ++--
13 files changed, 100 insertions(+), 68 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/index_storage_format.cpp
b/be/src/olap/rowset/segment_v2/index_storage_format.cpp
index d32558422a2..72ca72718b7 100644
--- a/be/src/olap/rowset/segment_v2/index_storage_format.cpp
+++ b/be/src/olap/rowset/segment_v2/index_storage_format.cpp
@@ -17,12 +17,13 @@
#include "index_storage_format.h"
+#include "common/cast_set.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
#include "util/debug_points.h"
namespace doris::segment_v2 {
-
+#include "common/compile_check_begin.h"
IndexStorageFormat::IndexStorageFormat(IndexFileWriter* index_file_writer)
: _index_file_writer(index_file_writer) {}
@@ -84,7 +85,7 @@ void IndexStorageFormat::copy_file(const char* fileName,
lucene::store::Director
int64_t chunk = bufferLength;
while (remainder > 0) {
- int64_t len = std::min({chunk, length, remainder});
+ auto len = cast_set<int32_t>(std::min({chunk, length, remainder}));
input->readBytes(buffer, len);
output->writeBytes(buffer, len);
remainder -= len;
@@ -121,4 +122,5 @@ void IndexStorageFormat::copy_file(const char* fileName,
lucene::store::Director
}
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/index_storage_format_v1.cpp
b/be/src/olap/rowset/segment_v2/index_storage_format_v1.cpp
index 332a336ced7..2825271bb07 100644
--- a/be/src/olap/rowset/segment_v2/index_storage_format_v1.cpp
+++ b/be/src/olap/rowset/segment_v2/index_storage_format_v1.cpp
@@ -17,12 +17,14 @@
#include "index_storage_format_v1.h"
+#include "common/cast_set.h"
#include "olap/rowset/segment_v2/index_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
#include "util/debug_points.h"
namespace doris::segment_v2 {
+#include "common/compile_check_begin.h"
IndexStorageFormatV1::IndexStorageFormatV1(IndexFileWriter* index_file_writer)
: IndexStorageFormat(index_file_writer) {}
@@ -94,7 +96,7 @@ std::pair<int64_t, int32_t>
IndexStorageFormatV1::calculate_header_length(
_CLTHROWA(CL_ERR_IO, "Create RAMDirectory output error");
}
std::unique_ptr<lucene::store::IndexOutput> ram_output(out_idx);
- int32_t file_count = sorted_files.size();
+ auto file_count = cast_set<int32_t>(sorted_files.size());
ram_output->writeVInt(file_count);
int64_t header_file_length = 0;
@@ -148,7 +150,7 @@ void
IndexStorageFormatV1::write_header_and_data(lucene::store::IndexOutput* out
const std::vector<FileInfo>&
sorted_files,
lucene::store::Directory*
directory,
int64_t header_length,
int32_t header_file_count) {
- output->writeVInt(sorted_files.size());
+ output->writeVInt(cast_set<int32_t>(sorted_files.size()));
int64_t data_offset = header_length;
const int64_t buffer_length = 16384;
uint8_t buffer[buffer_length];
@@ -188,4 +190,5 @@ void IndexStorageFormatV1::add_index_info(int64_t index_id,
const std::string& i
*new_index_info = index_info;
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/index_storage_format_v2.cpp
b/be/src/olap/rowset/segment_v2/index_storage_format_v2.cpp
index 779d96c2398..2b4c8672710 100644
--- a/be/src/olap/rowset/segment_v2/index_storage_format_v2.cpp
+++ b/be/src/olap/rowset/segment_v2/index_storage_format_v2.cpp
@@ -17,12 +17,14 @@
#include "index_storage_format_v2.h"
+#include "common/cast_set.h"
#include "olap/rowset/segment_v2/index_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
#include "util/debug_points.h"
namespace doris::segment_v2 {
+#include "common/compile_check_begin.h"
FileMetadata::FileMetadata(int64_t id, std::string suffix, std::string file,
int64_t off,
int64_t len, lucene::store::Directory* dir)
@@ -119,8 +121,8 @@ std::vector<FileMetadata>
IndexStorageFormatV2::prepare_file_metadata(int64_t& c
for (const auto& file : sorted_files) {
bool is_meta = false;
- for (const auto& entry :
InvertedIndexDescriptor::index_file_info_map) {
- if (file.filename.find(entry.first) != std::string::npos) {
+ for (const auto& file_info :
InvertedIndexDescriptor::index_file_info_map) {
+ if (file.filename.find(file_info.first) != std::string::npos) {
meta_files.emplace_back(index_id, index_suffix,
file.filename, 0, file.filesize,
dir);
is_meta = true;
@@ -217,16 +219,16 @@ void
IndexStorageFormatV2::write_index_headers_and_metadata(
// Write the index ID and the number of files
output->writeLong(index_id);
- output->writeInt(static_cast<int32_t>(index_suffix.length()));
+ output->writeInt(cast_set<int32_t>(index_suffix.length()));
output->writeBytes(reinterpret_cast<const
uint8_t*>(index_suffix.data()),
- index_suffix.length());
- output->writeInt(static_cast<int32_t>(files.size()));
+ cast_set<int32_t>(index_suffix.length()));
+ output->writeInt(cast_set<int32_t>(files.size()));
// Write file metadata
for (const auto& file : files) {
- output->writeInt(static_cast<int32_t>(file.filename.length()));
+ output->writeInt(cast_set<int32_t>(file.filename.length()));
output->writeBytes(reinterpret_cast<const
uint8_t*>(file.filename.data()),
- file.filename.length());
+ cast_set<int32_t>(file.filename.length()));
output->writeLong(file.offset);
output->writeLong(file.length);
}
@@ -243,4 +245,5 @@ void
IndexStorageFormatV2::copy_files_data(lucene::store::IndexOutput* output,
}
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/token_stream.h
b/be/src/olap/rowset/segment_v2/inverted_index/token_stream.h
index 6435a4cbfbd..c2850f77992 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/token_stream.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/token_stream.h
@@ -25,12 +25,14 @@
#include "CLucene.h"
#include "CLucene/analysis/AnalysisHeader.h"
+#include "common/cast_set.h"
#include "common/exception.h"
#include "common/logging.h"
using namespace lucene::analysis;
namespace doris::segment_v2::inverted_index {
+#include "common/compile_check_begin.h"
/**
* All custom tokenizers and token_filters must use the following functions
@@ -45,16 +47,17 @@ public:
virtual ~DorisTokenStream() = default;
void set(Token* t, const std::string_view& term, int32_t pos = 1) {
- t->setTextNoCopy(term.data(), term.size());
+ t->setTextNoCopy(term.data(), cast_set<int32_t>(term.size()));
t->setPositionIncrement(pos);
}
void set_text(Token* t, const std::string_view& term) {
- t->setTextNoCopy(term.data(), term.size());
+ t->setTextNoCopy(term.data(), cast_set<int32_t>(term.size()));
}
int32_t get_position_increment(Token* t) { return
t->getPositionIncrement(); }
void set_position_increment(Token* t, int32_t pos) {
t->setPositionIncrement(pos); }
};
-}; // namespace doris::segment_v2::inverted_index
\ No newline at end of file
+}; // namespace doris::segment_v2::inverted_index
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/util/mock_iterator.h
b/be/src/olap/rowset/segment_v2/inverted_index/util/mock_iterator.h
index ec825caa8b3..6ae28d51bea 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/util/mock_iterator.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/util/mock_iterator.h
@@ -21,7 +21,10 @@
#include <memory>
#include <vector>
+#include "common/cast_set.h"
+
namespace doris::segment_v2::inverted_index {
+#include "common/compile_check_begin.h"
class MockIterator {
public:
@@ -34,7 +37,7 @@ public:
return _impl->current_doc != _impl->postings.end() ?
_impl->current_doc->first : INT_MAX;
}
- int32_t freq() const { return _impl->current_freq; }
+ int32_t freq() const { return cast_set<int32_t>(_impl->current_freq); }
int32_t next_doc() {
auto& postings = _impl->postings;
@@ -71,7 +74,7 @@ public:
return INT_MAX;
}
- int32_t doc_freq() const { return
static_cast<int32_t>(_impl->postings.size()); }
+ int32_t doc_freq() const { return
cast_set<int32_t>(_impl->postings.size()); }
int32_t next_position() {
auto& current_doc = _impl->current_doc;
@@ -97,7 +100,7 @@ private:
std::map<int32_t, std::vector<int32_t>> postings;
std::map<int32_t, std::vector<int32_t>>::iterator current_doc;
size_t pos_idx = 0;
- int32_t current_freq = 0;
+ size_t current_freq = 0;
Impl() {
current_doc = postings.begin();
@@ -119,4 +122,5 @@ private:
};
using MockIterPtr = std::shared_ptr<MockIterator>;
-} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/util/priority_queue.h
b/be/src/olap/rowset/segment_v2/inverted_index/util/priority_queue.h
index 24cd5298f62..21591e1673b 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/util/priority_queue.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/util/priority_queue.h
@@ -22,7 +22,10 @@
#include <functional>
#include <limits>
+#include "common/cast_set.h"
+
namespace doris::segment_v2::inverted_index {
+#include "common/compile_check_begin.h"
template <typename T>
class PriorityQueue {
@@ -89,7 +92,7 @@ public:
return update_top();
}
- int32_t size() { return _size; }
+ int32_t size() { return cast_set<int32_t>(_size); }
void clear() {
for (size_t i = 1; i <= _size; i++) {
@@ -155,4 +158,5 @@ private:
std::vector<T> _heap;
};
-} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/util/union_term_iterator.h
b/be/src/olap/rowset/segment_v2/inverted_index/util/union_term_iterator.h
index a82f34a08bf..0ed9ec2f539 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/util/union_term_iterator.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/util/union_term_iterator.h
@@ -17,6 +17,7 @@
#pragma once
+#include "common/cast_set.h"
#include "olap/rowset/segment_v2/inverted_index/util/mock_iterator.h"
#include "olap/rowset/segment_v2/inverted_index/util/term_position_iterator.h"
#include "priority_queue.h"
@@ -24,6 +25,7 @@
namespace doris::segment_v2 {
using namespace inverted_index;
+#include "common/compile_check_begin.h"
template <typename T>
class DocsQueue : public PriorityQueue<T*> {
@@ -87,7 +89,7 @@ public:
UnionTermIterator(std::vector<IterPtr> subs) : _subs(std::move(subs)) {
_pos_queue = std::make_unique<PositionsQueue>();
_docs_queue = std::make_unique<DocsQueue<T>>(_subs.size());
- int64_t cost = 0;
+ int32_t cost = 0;
for (auto& sub : _subs) {
_docs_queue->add(sub.get());
cost += sub->doc_freq();
@@ -112,7 +114,7 @@ public:
_pos_queue->sort();
pos_queue_doc = doc;
}
- return _pos_queue->size();
+ return cast_set<int32_t>(_pos_queue->size());
}
int32_t next_position() const { return _pos_queue->next(); }
@@ -150,4 +152,5 @@ private:
};
using UnionTermIterPtr =
std::shared_ptr<UnionTermIterator<TermPositionsIterator>>;
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_iterator.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_iterator.cpp
index dc5fbe0791b..50beb066688 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_iterator.cpp
@@ -19,6 +19,7 @@
#include <memory>
+#include "common/cast_set.h"
#include "olap/rowset/segment_v2/inverted_index_cache.h"
#include "olap/rowset/segment_v2/inverted_index_reader.h"
@@ -49,7 +50,7 @@ Status InvertedIndexIterator::read_from_index(const
IndexParam& param) {
_runtime_state->query_options().inverted_index_skip_threshold <
100) {
auto query_bkd_limit_percent =
_runtime_state->query_options().inverted_index_skip_threshold;
- uint32_t hit_count = 0;
+ size_t hit_count = 0;
RETURN_IF_ERROR(try_read_from_inverted_index(i_param->column_name,
i_param->query_value,
i_param->query_type,
&hit_count));
if (hit_count > i_param->num_rows * query_bkd_limit_percent / 100)
{
@@ -93,7 +94,7 @@ bool InvertedIndexIterator::has_null() {
Status InvertedIndexIterator::try_read_from_inverted_index(const std::string&
column_name,
const void*
query_value,
InvertedIndexQueryType query_type,
- uint32_t* count) {
+ size_t* count) {
// NOTE: only bkd index support try read now.
if (query_type == InvertedIndexQueryType::GREATER_EQUAL_QUERY ||
query_type == InvertedIndexQueryType::GREATER_THAN_QUERY ||
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_iterator.h
b/be/src/olap/rowset/segment_v2/inverted_index_iterator.h
index fe7466e8b80..e4ce9125fdf 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_iterator.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_iterator.h
@@ -46,7 +46,7 @@ public:
private:
Status try_read_from_inverted_index(const std::string& column_name, const
void* query_value,
- InvertedIndexQueryType query_type,
uint32_t* count);
+ InvertedIndexQueryType query_type,
size_t* count);
InvertedIndexReaderPtr _index_reader;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 93328f79b39..3e0594ebf38 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -57,6 +57,7 @@
#include "vec/common/string_ref.h"
namespace doris::segment_v2 {
+#include "common/compile_check_begin.h"
template <PrimitiveType PT>
Status InvertedIndexQueryParamFactory::create_query_value(
@@ -138,10 +139,10 @@ Status InvertedIndexReader::read_null_bitmap(const
io::IOContext* io_ctx,
InvertedIndexDescriptor::get_temporary_null_bitmap_file_name();
if (dir->fileExists(null_bitmap_file_name)) {
null_bitmap_in = dir->openInput(null_bitmap_file_name);
- size_t null_bitmap_size = null_bitmap_in->length();
+ auto null_bitmap_size =
cast_set<int32_t>(null_bitmap_in->length());
faststring buf;
buf.resize(null_bitmap_size);
- null_bitmap_in->readBytes(reinterpret_cast<uint8_t*>(buf.data()),
null_bitmap_size);
+ null_bitmap_in->readBytes(buf.data(), null_bitmap_size);
*null_bitmap =
roaring::Roaring::read(reinterpret_cast<char*>(buf.data()), false);
null_bitmap->runOptimize();
cache->insert(cache_key, null_bitmap, cache_handle);
@@ -579,7 +580,7 @@ Status BkdIndexReader::construct_bkd_query_value(const
void* query_value,
Status BkdIndexReader::invoke_bkd_try_query(const io::IOContext* io_ctx, const
void* query_value,
InvertedIndexQueryType query_type,
std::shared_ptr<lucene::util::bkd::bkd_reader> r,
- uint32_t* count) {
+ size_t* count) {
switch (query_type) {
case InvertedIndexQueryType::LESS_THAN_QUERY: {
auto visitor =
@@ -680,7 +681,7 @@ Status BkdIndexReader::invoke_bkd_query(const
io::IOContext* io_ctx, OlapReaderS
Status BkdIndexReader::try_query(const io::IOContext* io_ctx,
OlapReaderStatistics* stats,
RuntimeState* runtime_state, const
std::string& column_name,
const void* query_value,
InvertedIndexQueryType query_type,
- uint32_t* count) {
+ size_t* count) {
try {
std::shared_ptr<lucene::util::bkd::bkd_reader> r;
auto st = get_bkd_reader(r, io_ctx, stats, runtime_state);
@@ -778,7 +779,7 @@ Status BkdIndexReader::get_bkd_reader(BKDIndexSearcherPtr&
bkd_reader, const io:
_value_key_coder = get_key_coder(_type_info->type());
bkd_reader = *bkd_searcher;
if (bkd_reader->bytes_per_dim_ == 0) {
- bkd_reader->bytes_per_dim_ = _type_info->size();
+ bkd_reader->bytes_per_dim_ = cast_set<int32_t>(_type_info->size());
}
return Status::OK();
}
@@ -1115,24 +1116,25 @@
InvertedIndexVisitor<InvertedIndexQueryType::GREATER_THAN_QUERY>::compare(
}
template <InvertedIndexQueryType QT>
-lucene::util::bkd::relation
InvertedIndexVisitor<QT>::compare_prefix(std::vector<uint8_t>& prefix) {
- if (lucene::util::FutureArrays::CompareUnsigned(prefix.data(), 0,
prefix.size(),
- (const
uint8_t*)query_max.c_str(), 0,
- prefix.size()) > 0 ||
- lucene::util::FutureArrays::CompareUnsigned(prefix.data(), 0,
prefix.size(),
- (const
uint8_t*)query_min.c_str(), 0,
- prefix.size()) < 0) {
- return lucene::util::bkd::relation::CELL_OUTSIDE_QUERY;
+bkd::relation InvertedIndexVisitor<QT>::compare_prefix(std::vector<uint8_t>&
prefix) {
+ const int32_t length = cast_set<int32_t>(prefix.size());
+ const uint8_t* data = prefix.data();
+
+ auto cmp = [&](const std::string& bound) {
+ return lucene::util::FutureArrays::CompareUnsigned(
+ data, 0, length, reinterpret_cast<const
uint8_t*>(bound.data()), 0, length);
+ };
+
+ int32_t cmpMax = cmp(query_max);
+ int32_t cmpMin = cmp(query_min);
+
+ if (cmpMax > 0 || cmpMin < 0) {
+ return bkd::relation::CELL_OUTSIDE_QUERY;
}
- if (lucene::util::FutureArrays::CompareUnsigned(prefix.data(), 0,
prefix.size(),
- (const
uint8_t*)query_min.c_str(), 0,
- prefix.size()) > 0 &&
- lucene::util::FutureArrays::CompareUnsigned(prefix.data(), 0,
prefix.size(),
- (const
uint8_t*)query_max.c_str(), 0,
- prefix.size()) < 0) {
- return lucene::util::bkd::relation::CELL_INSIDE_QUERY;
+ if (cmpMin > 0 && cmpMax < 0) {
+ return bkd::relation::CELL_INSIDE_QUERY;
}
- return lucene::util::bkd::relation::CELL_CROSSES_QUERY;
+ return bkd::relation::CELL_CROSSES_QUERY;
}
template <InvertedIndexQueryType QT>
@@ -1177,3 +1179,4 @@ template class
InvertedIndexVisitor<InvertedIndexQueryType::GREATER_THAN_QUERY>;
template class
InvertedIndexVisitor<InvertedIndexQueryType::GREATER_EQUAL_QUERY>;
} // namespace doris::segment_v2
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index f97470c2789..faf08385935 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -188,7 +188,7 @@ public:
virtual Status try_query(const io::IOContext* io_ctx,
OlapReaderStatistics* stats,
RuntimeState* runtime_state, const std::string&
column_name,
const void* query_value, InvertedIndexQueryType
query_type,
- uint32_t* count) = 0;
+ size_t* count) = 0;
Status read_null_bitmap(const io::IOContext* io_ctx, OlapReaderStatistics*
stats,
InvertedIndexQueryCacheHandle* cache_handle,
@@ -258,7 +258,7 @@ public:
Status try_query(const io::IOContext* io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state, const std::string&
column_name,
const void* query_value, InvertedIndexQueryType
query_type,
- uint32_t* count) override {
+ size_t* count) override {
return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
"FullTextIndexReader not support try_query");
}
@@ -285,7 +285,7 @@ public:
Status try_query(const io::IOContext* io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state, const std::string&
column_name,
const void* query_value, InvertedIndexQueryType
query_type,
- uint32_t* count) override {
+ size_t* count) override {
return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
"StringTypeInvertedIndexReader not support try_query");
}
@@ -349,10 +349,10 @@ public:
Status try_query(const io::IOContext* io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state, const std::string&
column_name,
const void* query_value, InvertedIndexQueryType
query_type,
- uint32_t* count) override;
+ size_t* count) override;
Status invoke_bkd_try_query(const io::IOContext* io_ctx, const void*
query_value,
InvertedIndexQueryType query_type,
- std::shared_ptr<lucene::util::bkd::bkd_reader>
r, uint32_t* count);
+ std::shared_ptr<lucene::util::bkd::bkd_reader>
r, size_t* count);
Status invoke_bkd_query(const io::IOContext* io_ctx, OlapReaderStatistics*
stats,
const void* query_value, InvertedIndexQueryType
query_type,
std::shared_ptr<lucene::util::bkd::bkd_reader> r,
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 5ed7a17a25f..63074fa46c9 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -29,6 +29,7 @@
#include <string>
#include <vector>
+#include "common/cast_set.h"
#include "common/exception.h"
#include "io/fs/local_file_system.h"
@@ -65,6 +66,8 @@
#include "util/string_util.h"
namespace doris::segment_v2 {
+#include "common/compile_check_begin.h"
+
const int32_t MAX_FIELD_LEN = 0x7FFFFFFFL;
const int32_t MERGE_FACTOR = 100000000;
const int32_t MAX_LEAF_COUNT = 1024;
@@ -199,7 +202,8 @@ public:
{ index_writer->setMaxBufferedDocs(1); })
DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_index_writer_setMergeFactor_error",
{ index_writer->setMergeFactor(1); })
-
index_writer->setRAMBufferSizeMB(config::inverted_index_ram_buffer_size);
+ index_writer->setRAMBufferSizeMB(
+ static_cast<float>(config::inverted_index_ram_buffer_size));
index_writer->setMaxBufferedDocs(config::inverted_index_max_buffered_docs);
index_writer->setMaxFieldLength(MAX_FIELD_LEN);
index_writer->setMergeFactor(MERGE_FACTOR);
@@ -346,7 +350,7 @@ public:
// so we need to subtract num_rows to get the row id in segment
for (size_t i = 0; i < num_rows; i++) {
if (null_map[i] == 1) {
- null_indices.push_back(_rid - num_rows +
static_cast<uint32_t>(i));
+ null_indices.push_back(cast_set<uint32_t>(_rid - num_rows +
i));
}
}
@@ -372,7 +376,7 @@ public:
}
void new_char_token_stream(const char* s, size_t len,
lucene::document::Field* field) {
- _char_string_reader->init(s, len, false);
+ _char_string_reader->init(s, cast_set<int32_t>(len), false);
DBUG_EXECUTE_IF(
"InvertedIndexColumnWriterImpl::new_char_token_stream__char_string_reader_init_"
"error",
@@ -406,8 +410,8 @@ public:
return Status::InternalError(
"field or index writer is null in inverted index
writer");
}
- auto* v = (Slice*)values;
- for (int i = 0; i < count; ++i) {
+ const auto* v = (Slice*)values;
+ for (size_t i = 0; i < count; ++i) {
// only ignore_above UNTOKENIZED strings and empty strings not
tokenized
if ((!_should_analyzer && v->get_size() > _ignore_above) ||
(_should_analyzer && v->empty())) {
@@ -442,7 +446,7 @@ public:
return Status::InternalError("index writer is null in inverted
index writer");
}
size_t start_off = 0;
- for (int i = 0; i < count; ++i) {
+ for (size_t i = 0; i < count; ++i) {
// nullmap & value ptr-array may not from offsets[i] because
olap_convertor make offsets accumulate from _base_offset which may not is 0,
but nullmap & value in this segment is from 0, we only need
// every single array row element size to go through the
nullmap & value ptr-array, and also can go through the every row in array to
keep with _rid++
auto array_elem_size = offsets[i + 1] - offsets[i];
@@ -485,7 +489,8 @@ public:
std::unique_ptr<lucene::util::Reader>
char_string_reader =
DORIS_TRY(create_char_string_reader(
_inverted_index_ctx->char_filter_map));
- char_string_reader->init(v->get_data(),
v->get_size(), false);
+ char_string_reader->init(v->get_data(),
+
cast_set<int32_t>(v->get_size()), false);
_analyzer->set_ownReader(own_reader);
ts = _analyzer->tokenStream(new_field->name(),
char_string_reader.release());
@@ -630,7 +635,7 @@ public:
Status add_value(const CppType& value) {
try {
std::string new_value;
- size_t value_length = sizeof(CppType);
+ uint32_t value_length = sizeof(CppType);
DBUG_EXECUTE_IF(
"InvertedIndexColumnWriterImpl::add_value_bkd_writer_add_throw_"
@@ -659,7 +664,7 @@ public:
faststring buf;
buf.resize(size);
_null_bitmap.write(reinterpret_cast<char*>(buf.data()), false);
- null_bitmap_out->writeBytes(buf.data(), size);
+ null_bitmap_out->writeBytes(buf.data(), cast_set<int32_t>(size));
}
}
@@ -842,3 +847,4 @@ Status InvertedIndexColumnWriter::create(const Field* field,
return Status::OK();
}
} // namespace doris::segment_v2
+#include "common/compile_check_end.h"
\ No newline at end of file
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp
b/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp
index 280df514e88..cf54d4ac297 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp
@@ -2317,7 +2317,7 @@ public:
EXPECT_EQ(bitmap->cardinality(), 1);
// Test try_read functionality with non-BKD index (should succeed)
- uint32_t count = 0;
+ size_t count = 0;
auto* inverted_index_iterator =
static_cast<InvertedIndexIterator*>(iterator.get());
status = inverted_index_iterator->try_read_from_inverted_index(
"c2", &str_ref, InvertedIndexQueryType::EQUAL_QUERY, &count);
@@ -2755,7 +2755,7 @@ public:
for (auto& test_case : test_cases) {
// Test try_query path
- uint32_t count = 0;
+ size_t count = 0;
auto status = bkd_reader->try_query(&io_ctx, &stats,
&runtime_state, "c1",
&test_case.second,
test_case.first, &count);
EXPECT_TRUE(status.ok()) << "Try query type: " <<
static_cast<int>(test_case.first);
@@ -2844,7 +2844,7 @@ public:
EXPECT_TRUE(status.ok());
// Test try_read_from_inverted_index with non-BKD compatible query
- uint32_t count = 0;
+ size_t count = 0;
status = inverted_index_iterator->try_read_from_inverted_index(
"c1", &query_value, InvertedIndexQueryType::MATCH_ANY_QUERY,
&count);
EXPECT_TRUE(status.ok()); // Should succeed but not do anything for
non-BKD queries
@@ -2987,7 +2987,7 @@ public:
}
for (auto& test_case : test_cases) {
- uint32_t count = 0;
+ size_t count = 0;
auto status = bkd_reader->try_query(&io_ctx, &stats,
&runtime_state, "c_int",
&test_case.second,
test_case.first, &count);
EXPECT_TRUE(status.ok()) << "Try query type: " <<
static_cast<int>(test_case.first);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
