This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 2c0021a2f51 branch-3.1: [opt](segment) SegmentFooterPB cache #48924
(#53517)
2c0021a2f51 is described below
commit 2c0021a2f51e2e28e6b52928d1871bcd608fbd45
Author: lihangyu <[email protected]>
AuthorDate: Fri Jul 18 16:49:21 2025 +0800
branch-3.1: [opt](segment) SegmentFooterPB cache #48924 (#53517)
cherry-pick #48924
Co-authored-by: zhiqiang <[email protected]>
---
be/src/olap/page_cache.cpp | 85 +++++++++--
be/src/olap/page_cache.h | 78 +++++++---
be/src/olap/rowset/segment_v2/page_io.cpp | 2 +
be/src/olap/rowset/segment_v2/segment.cpp | 80 ++++++++--
be/src/olap/rowset/segment_v2/segment.h | 11 +-
be/test/olap/date_bloom_filter_test.cpp | 11 +-
be/test/olap/delete_bitmap_calculator_test.cpp | 113 +++++++-------
be/test/olap/segment_footer_cache_test.cpp | 195 +++++++++++++++++++++++++
8 files changed, 467 insertions(+), 108 deletions(-)
diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp
index b386da4d7c6..7e1129ba4de 100644
--- a/be/src/olap/page_cache.cpp
+++ b/be/src/olap/page_cache.cpp
@@ -17,6 +17,7 @@
#include "olap/page_cache.h"
+#include <gen_cpp/segment_v2.pb.h>
#include <glog/logging.h>
#include <ostream>
@@ -24,32 +25,51 @@
#include "runtime/exec_env.h"
namespace doris {
-template <typename TAllocator>
-PageBase<TAllocator>::PageBase(size_t b, bool use_cache,
segment_v2::PageTypePB page_type)
- : LRUCacheValueBase(), _size(b), _capacity(b) {
+
+template <typename T>
+MemoryTrackedPageBase<T>::MemoryTrackedPageBase(size_t size, bool use_cache,
+ segment_v2::PageTypePB
page_type)
+ : _size(size) {
if (use_cache) {
_mem_tracker_by_allocator =
StoragePageCache::instance()->mem_tracker(page_type);
} else {
_mem_tracker_by_allocator =
thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker_sptr();
}
+}
+
+MemoryTrackedPageWithPageEntity::MemoryTrackedPageWithPageEntity(size_t size,
bool use_cache,
+
segment_v2::PageTypePB page_type)
+ : MemoryTrackedPageBase<char*>(size, use_cache, page_type),
_capacity(size) {
{
- SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
- _data = reinterpret_cast<char*>(TAllocator::alloc(_capacity,
ALLOCATOR_ALIGNMENT_16));
+
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(this->_mem_tracker_by_allocator);
+ this->_data = reinterpret_cast<char*>(
+ Allocator<false>::alloc(this->_capacity,
ALLOCATOR_ALIGNMENT_16));
}
}
-template <typename TAllocator>
-PageBase<TAllocator>::~PageBase() {
- if (_data != nullptr) {
- DCHECK(_capacity != 0 && _size != 0);
- SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker_by_allocator);
- TAllocator::free(_data, _capacity);
+MemoryTrackedPageWithPageEntity::~MemoryTrackedPageWithPageEntity() {
+ if (this->_data != nullptr) {
+ DCHECK(this->_capacity != 0 && this->_size != 0);
+
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(this->_mem_tracker_by_allocator);
+ Allocator<false>::free(this->_data, this->_capacity);
}
}
-template class PageBase<Allocator<true>>;
-template class PageBase<Allocator<false>>;
+template <typename T>
+MemoryTrackedPageWithPagePtr<T>::MemoryTrackedPageWithPagePtr(size_t size,
+
segment_v2::PageTypePB page_type)
+ : MemoryTrackedPageBase<std::shared_ptr<T>>(size, true, page_type) {
+ DCHECK(this->_size > 0);
+ this->_size = size;
+ this->_mem_tracker_by_allocator->consume(this->_size);
+}
+
+template <typename T>
+MemoryTrackedPageWithPagePtr<T>::~MemoryTrackedPageWithPagePtr() {
+ DCHECK(this->_size > 0);
+ this->_mem_tracker_by_allocator->release(this->_size);
+}
StoragePageCache* StoragePageCache::create_global_cache(size_t capacity,
int32_t
index_cache_percentage,
@@ -98,7 +118,46 @@ void StoragePageCache::insert(const CacheKey& key,
DataPage* data, PageCacheHand
auto* cache = _get_page_cache(page_type);
auto* lru_handle = cache->insert(key.encode(), data, data->capacity(), 0,
priority);
+ DCHECK(lru_handle != nullptr);
+ *handle = PageCacheHandle(cache, lru_handle);
+}
+
+template <typename T>
+void StoragePageCache::insert(const CacheKey& key, T data, size_t size,
PageCacheHandle* handle,
+ segment_v2::PageTypePB page_type, bool
in_memory) {
+ static_assert(std::is_same<typename std::remove_cv<T>::type,
+ std::shared_ptr<typename
T::element_type>>::value,
+ "Second argument must be a std::shared_ptr");
+ using ValueType = typename T::element_type; // Type that shared_ptr points
to
+
+ CachePriority priority = CachePriority::NORMAL;
+ if (in_memory) {
+ priority = CachePriority::DURABLE;
+ }
+
+ auto* cache = _get_page_cache(page_type);
+ // Lify cycle of page will be managed by StoragePageCache
+ auto page =
std::make_unique<MemoryTrackedPageWithPagePtr<ValueType>>(size, page_type);
+ // Lify cycle of data will be managed by StoragePageCache and user at the
same time.
+ page->set_data(data);
+
+ auto* lru_handle = cache->insert(key.encode(), page.get(), size, 0,
priority);
+ DCHECK(lru_handle != nullptr);
*handle = PageCacheHandle(cache, lru_handle);
+ // Now page is managed by StoragePageCache.
+ page.release();
+}
+
+Slice PageCacheHandle::data() const {
+ auto* cache_value = (DataPage*)_cache->value(_handle);
+ return {cache_value->data(), cache_value->size()};
}
+template void StoragePageCache::insert(const CacheKey& key,
+
std::shared_ptr<segment_v2::SegmentFooterPB> data,
+ size_t size, PageCacheHandle* handle,
+ segment_v2::PageTypePB page_type, bool
in_memory);
+
+template class MemoryTrackedPageWithPagePtr<segment_v2::SegmentFooterPB>;
+
} // namespace doris
diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h
index 32b6683e782..73b20f1d5f2 100644
--- a/be/src/olap/page_cache.h
+++ b/be/src/olap/page_cache.h
@@ -37,33 +37,54 @@ namespace doris {
class PageCacheHandle;
-template <typename TAllocator>
-class PageBase : private TAllocator, public LRUCacheValueBase {
+template <typename T>
+class MemoryTrackedPageBase : public LRUCacheValueBase {
public:
- PageBase() = default;
- PageBase(size_t b, bool use_cache, segment_v2::PageTypePB page_type);
- PageBase(const PageBase&) = delete;
- PageBase& operator=(const PageBase&) = delete;
- ~PageBase() override;
+ MemoryTrackedPageBase() = default;
+ MemoryTrackedPageBase(size_t b, bool use_cache, segment_v2::PageTypePB
page_type);
- char* data() { return _data; }
+ MemoryTrackedPageBase(const MemoryTrackedPageBase&) = delete;
+ MemoryTrackedPageBase& operator=(const MemoryTrackedPageBase&) = delete;
+ ~MemoryTrackedPageBase() = default;
+
+ T data() { return _data; }
size_t size() { return _size; }
- size_t capacity() { return _capacity; }
+
+protected:
+ T _data;
+ size_t _size = 0;
+ std::shared_ptr<MemTrackerLimiter> _mem_tracker_by_allocator;
+};
+
+class MemoryTrackedPageWithPageEntity : Allocator<false>, public
MemoryTrackedPageBase<char*> {
+public:
+ MemoryTrackedPageWithPageEntity(size_t b, bool use_cache,
segment_v2::PageTypePB page_type);
+
+ size_t capacity() { return this->_capacity; }
+
+ ~MemoryTrackedPageWithPageEntity() override;
void reset_size(size_t n) {
- DCHECK(n <= _capacity);
- _size = n;
+ DCHECK(n <= this->_capacity);
+ this->_size = n;
}
private:
- char* _data = nullptr;
- // Effective size, smaller than capacity, such as data page remove
checksum suffix.
- size_t _size = 0;
size_t _capacity = 0;
- std::shared_ptr<MemTrackerLimiter> _mem_tracker_by_allocator;
};
-using DataPage = PageBase<Allocator<false>>;
+template <typename T>
+class MemoryTrackedPageWithPagePtr : public
MemoryTrackedPageBase<std::shared_ptr<T>> {
+public:
+ MemoryTrackedPageWithPagePtr(size_t b, segment_v2::PageTypePB page_type);
+
+ ~MemoryTrackedPageWithPagePtr() override;
+
+ void set_data(std::shared_ptr<T> data) { this->_data = data; }
+};
+
+using SemgnetFooterPBPage =
MemoryTrackedPageWithPagePtr<segment_v2::SegmentFooterPB>;
+using DataPage = MemoryTrackedPageWithPageEntity;
// Wrapper around Cache, and used for cache page of column data
// in Segment.
@@ -149,6 +170,17 @@ public:
void insert(const CacheKey& key, DataPage* data, PageCacheHandle* handle,
segment_v2::PageTypePB page_type, bool in_memory = false);
+ // Insert a std::share_ptr which points to a page into this cache.
+ // size should be the size of the page instead of shared_ptr.
+ // Internal implementation will wrap shared_ptr with
MemoryTrackedPageWithPagePtr
+ // Since we are using std::shared_ptr, so lify cycle of the page is not
managed by
+ // this cache alone.
+ // User could store a weak_ptr to the page, and lock it when needed.
+ // See Segment::_get_segment_footer for example.
+ template <typename T>
+ void insert(const CacheKey& key, T data, size_t size, PageCacheHandle*
handle,
+ segment_v2::PageTypePB page_type, bool in_memory = false);
+
std::shared_ptr<MemTrackerLimiter> mem_tracker(segment_v2::PageTypePB
page_type) {
return _get_page_cache(page_type)->mem_tracker();
}
@@ -211,9 +243,17 @@ public:
}
LRUCachePolicy* cache() const { return _cache; }
- Slice data() const {
- auto* cache_value = (DataPage*)_cache->value(_handle);
- return {cache_value->data(), cache_value->size()};
+ Slice data() const;
+
+ template <typename T>
+ T get() const {
+ static_assert(std::is_same<typename std::remove_cv<T>::type,
+ std::shared_ptr<typename
T::element_type>>::value,
+ "T must be a std::shared_ptr");
+ using ValueType = typename T::element_type; // Type that shared_ptr
points to
+ MemoryTrackedPageWithPagePtr<ValueType>* page =
+
(MemoryTrackedPageWithPagePtr<ValueType>*)_cache->value(_handle);
+ return page->data();
}
private:
diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp
b/be/src/olap/rowset/segment_v2/page_io.cpp
index 0e3d891aa07..4e1069888dc 100644
--- a/be/src/olap/rowset/segment_v2/page_io.cpp
+++ b/be/src/olap/rowset/segment_v2/page_io.cpp
@@ -134,6 +134,8 @@ Status PageIO::read_and_decompress_page_(const
PageReadOptions& opts, PageHandle
PageCacheHandle cache_handle;
StoragePageCache::CacheKey cache_key(opts.file_reader->path().native(),
opts.file_reader->size(),
opts.page_pointer.offset);
+ VLOG_DEBUG << fmt::format("Reading page {}:{}:{}", cache_key.fname,
cache_key.fsize,
+ cache_key.offset);
if (opts.use_page_cache && cache && cache->lookup(cache_key,
&cache_handle, opts.type)) {
// we find page in cache, use it
*handle = PageHandle(std::move(cache_handle));
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index 2b8e19c7ce3..c9a60bee032 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -167,8 +167,9 @@ io::UInt128Wrapper Segment::file_cache_key(std::string_view
rowset_id, uint32_t
}
int64_t Segment::get_metadata_size() const {
- return sizeof(Segment) + (_footer_pb ? _footer_pb->ByteSizeLong() : 0) +
- (_pk_index_meta ? _pk_index_meta->ByteSizeLong() : 0);
+ std::shared_ptr<SegmentFooterPB> footer_pb_shared = _footer_pb.lock();
+ return sizeof(Segment) + (_pk_index_meta ? _pk_index_meta->ByteSizeLong()
: 0) +
+ (footer_pb_shared ? footer_pb_shared->ByteSizeLong() : 0);
}
void Segment::update_metadata_size() {
@@ -178,18 +179,20 @@ void Segment::update_metadata_size() {
}
Status Segment::_open(OlapReaderStatistics* stats) {
- _footer_pb = std::make_unique<SegmentFooterPB>();
- RETURN_IF_ERROR(_parse_footer(_footer_pb.get(), stats));
- _pk_index_meta.reset(_footer_pb->has_primary_key_index_meta()
- ? new
PrimaryKeyIndexMetaPB(_footer_pb->primary_key_index_meta())
- : nullptr);
+ std::shared_ptr<SegmentFooterPB> footer_pb_shared;
+ RETURN_IF_ERROR(_get_segment_footer(footer_pb_shared, stats));
+
+ _pk_index_meta.reset(
+ footer_pb_shared->has_primary_key_index_meta()
+ ? new
PrimaryKeyIndexMetaPB(footer_pb_shared->primary_key_index_meta())
+ : nullptr);
// delete_bitmap_calculator_test.cpp
// DCHECK(footer.has_short_key_index_page());
- _sk_index_page = _footer_pb->short_key_index_page();
- _num_rows = _footer_pb->num_rows();
+ _sk_index_page = footer_pb_shared->short_key_index_page();
+ _num_rows = footer_pb_shared->num_rows();
// An estimated memory usage of a segment
- _meta_mem_usage += _footer_pb->ByteSizeLong();
+ _meta_mem_usage += footer_pb_shared->ByteSizeLong();
if (_pk_index_meta != nullptr) {
_meta_mem_usage += _pk_index_meta->ByteSizeLong();
}
@@ -392,7 +395,8 @@ Status Segment::_write_error_file(size_t file_size, size_t
offset, size_t bytes_
return Status::OK(); // already exists
};
-Status Segment::_parse_footer(SegmentFooterPB* footer, OlapReaderStatistics*
stats) {
+Status Segment::_parse_footer(std::shared_ptr<SegmentFooterPB>& footer,
+ OlapReaderStatistics* stats) {
// Footer := SegmentFooterPB, FooterPBSize(4), FooterPBChecksum(4),
MagicNumber(4)
auto file_size = _file_reader->size();
if (file_size < 12) {
@@ -459,6 +463,7 @@ Status Segment::_parse_footer(SegmentFooterPB* footer,
OlapReaderStatistics* sta
}
// deserialize footer PB
+ footer = std::make_shared<SegmentFooterPB>();
if (!footer->ParseFromString(footer_buf)) {
Status st = _write_error_file(file_size, file_size - 12 -
footer_length, bytes_read,
footer_buf.data(), io_ctx);
@@ -470,6 +475,9 @@ Status Segment::_parse_footer(SegmentFooterPB* footer,
OlapReaderStatistics* sta
_file_reader->path().native(), file_size,
file_cache_key_str(_file_reader->path().native()));
}
+
+ VLOG_DEBUG << fmt::format("Loading segment footer from {} finished",
+ _file_reader->path().native());
return Status::OK();
}
@@ -637,9 +645,9 @@ Status
Segment::_create_column_readers_once(OlapReaderStatistics* stats) {
SCOPED_RAW_TIMER(&stats->segment_create_column_readers_timer_ns);
}
return _create_column_readers_once_call.call([&] {
- DCHECK(_footer_pb);
- Defer defer([&]() { _footer_pb.reset(); });
- return _create_column_readers(*_footer_pb);
+ std::shared_ptr<SegmentFooterPB> footer_pb_shared;
+ RETURN_IF_ERROR(_get_segment_footer(footer_pb_shared, stats));
+ return _create_column_readers(*footer_pb_shared);
});
}
@@ -1015,4 +1023,48 @@ Status Segment::seek_and_read_by_rowid(const
TabletSchema& schema, SlotDescripto
return Status::OK();
}
+Status Segment::_get_segment_footer(std::shared_ptr<SegmentFooterPB>&
footer_pb,
+ OlapReaderStatistics* stats) {
+ std::shared_ptr<SegmentFooterPB> footer_pb_shared = _footer_pb.lock();
+ if (footer_pb_shared != nullptr) {
+ footer_pb = footer_pb_shared;
+ return Status::OK();
+ }
+
+ VLOG_DEBUG << fmt::format("Segment footer of {}:{}:{} is missing, try to
load it",
+ _file_reader->path().native(),
_file_reader->size(),
+ _file_reader->size() - 12);
+
+ StoragePageCache* segment_footer_cache =
ExecEnv::GetInstance()->get_storage_page_cache();
+ DCHECK(segment_footer_cache != nullptr);
+
+ auto cache_key = get_segment_footer_cache_key();
+
+ PageCacheHandle cache_handle;
+
+ if (!segment_footer_cache->lookup(cache_key, &cache_handle,
+ segment_v2::PageTypePB::DATA_PAGE)) {
+ RETURN_IF_ERROR(_parse_footer(footer_pb_shared, stats));
+ segment_footer_cache->insert(cache_key, footer_pb_shared,
footer_pb_shared->ByteSizeLong(),
+ &cache_handle,
segment_v2::PageTypePB::DATA_PAGE);
+ } else {
+ VLOG_DEBUG << fmt::format("Segment footer of {}:{}:{} is found in
cache",
+ _file_reader->path().native(),
_file_reader->size(),
+ _file_reader->size() - 12);
+ }
+ footer_pb_shared = cache_handle.get<std::shared_ptr<SegmentFooterPB>>();
+ _footer_pb = footer_pb_shared;
+ footer_pb = footer_pb_shared;
+ return Status::OK();
+}
+
+StoragePageCache::CacheKey Segment::get_segment_footer_cache_key() const {
+ DCHECK(_file_reader != nullptr);
+ // The footer is always at the end of the segment file.
+ // The size of footer is 12.
+ // So we use the size of file minus 12 as the cache key, which is unique
for each segment file.
+ return StoragePageCache::CacheKey(_file_reader->path().native(),
_file_reader->size(),
+ _file_reader->size() - 12);
+}
+
} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index b3b1a72c345..27237e71a63 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -35,6 +35,7 @@
#include "io/fs/file_system.h"
#include "olap/field.h"
#include "olap/olap_common.h"
+#include "olap/page_cache.h"
#include "olap/rowset/segment_v2/column_reader.h" // ColumnReader
#include "olap/rowset/segment_v2/page_handle.h"
#include "olap/schema.h"
@@ -219,7 +220,8 @@ private:
OlapReaderStatistics* stats = nullptr);
// open segment file and read the minimum amount of necessary information
(footer)
Status _open(OlapReaderStatistics* stats);
- Status _parse_footer(SegmentFooterPB* footer, OlapReaderStatistics* stats
= nullptr);
+ Status _parse_footer(std::shared_ptr<SegmentFooterPB>& footer,
+ OlapReaderStatistics* stats = nullptr);
Status _create_column_readers(const SegmentFooterPB& footer);
Status _load_pk_bloom_filter(OlapReaderStatistics* stats);
ColumnReader* _get_column_reader(const TabletColumn& col);
@@ -231,7 +233,10 @@ private:
Status _create_column_readers_once(OlapReaderStatistics* stats);
-private:
+ Status _get_segment_footer(std::shared_ptr<SegmentFooterPB>&,
OlapReaderStatistics* stats);
+
+ StoragePageCache::CacheKey get_segment_footer_cache_key() const;
+
friend class SegmentIterator;
io::FileSystemSPtr _fs;
io::FileReaderSPtr _file_reader;
@@ -268,7 +273,7 @@ private:
DorisCallOnce<Status> _create_column_readers_once_call;
- std::unique_ptr<SegmentFooterPB> _footer_pb;
+ std::weak_ptr<SegmentFooterPB> _footer_pb;
// used to hold short key index page in memory
PageHandle _sk_index_handle;
diff --git a/be/test/olap/date_bloom_filter_test.cpp
b/be/test/olap/date_bloom_filter_test.cpp
index c528862c87b..04a0eaaefd0 100644
--- a/be/test/olap/date_bloom_filter_test.cpp
+++ b/be/test/olap/date_bloom_filter_test.cpp
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+#include <gen_cpp/segment_v2.pb.h>
#include <gtest/gtest.h>
#include "olap/comparison_predicate.h"
@@ -150,7 +151,10 @@ TEST_F(DateBloomFilterTest, query_index_test) {
segment_v2::SegmentSharedPtr segment;
EXPECT_TRUE(((BetaRowset*)rowset.get())->load_segment(0, &segment).ok());
- auto st = segment->_create_column_readers(*(segment->_footer_pb));
+ std::shared_ptr<SegmentFooterPB> footer_pb_shared;
+ auto st = segment->_get_segment_footer(footer_pb_shared, nullptr);
+ EXPECT_TRUE(st.ok());
+ st = segment->_create_column_readers(*footer_pb_shared);
EXPECT_TRUE(st.ok());
// date
@@ -227,7 +231,10 @@ TEST_F(DateBloomFilterTest, in_list_predicate_test) {
segment_v2::SegmentSharedPtr segment;
EXPECT_TRUE(((BetaRowset*)rowset.get())->load_segment(0, &segment).ok());
- auto st = segment->_create_column_readers(*(segment->_footer_pb));
+ std::shared_ptr<SegmentFooterPB> footer_pb_shared;
+ auto st = segment->_get_segment_footer(footer_pb_shared, nullptr);
+ EXPECT_TRUE(st.ok());
+ st = segment->_create_column_readers(*(footer_pb_shared));
EXPECT_TRUE(st.ok());
// Test DATE column with IN predicate
diff --git a/be/test/olap/delete_bitmap_calculator_test.cpp
b/be/test/olap/delete_bitmap_calculator_test.cpp
index ee54a061363..42e1d279f0f 100644
--- a/be/test/olap/delete_bitmap_calculator_test.cpp
+++ b/be/test/olap/delete_bitmap_calculator_test.cpp
@@ -50,9 +50,9 @@ static RowsetId rowset_id {0};
using Generator = std::function<void(size_t rid, int cid, RowCursorCell&
cell)>;
-static TabletColumnPtr create_int_sequence_value(int32_t id, bool is_nullable
= true,
- bool is_bf_column = false,
- bool has_bitmap_index =
false) {
+TabletColumnPtr create_int_sequence_value(int32_t id, bool is_nullable = true,
+ bool is_bf_column = false,
+ bool has_bitmap_index = false) {
TabletColumnPtr column = std::make_shared<TabletColumn>();
column->_unique_id = id;
column->_col_name = std::to_string(id);
@@ -67,6 +67,58 @@ static TabletColumnPtr create_int_sequence_value(int32_t id,
bool is_nullable =
return column;
}
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema,
size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator
generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir) {
+ std::string filename = fmt::format("{}_{}.dat", rowset_id.to_string(),
segment_id);
+ std::string path = fmt::format("{}/{}", segment_dir, filename);
+ auto fs = io::global_local_filesystem();
+
+ io::FileWriterPtr file_writer;
+ Status st = fs->create_file(path, &file_writer);
+ EXPECT_TRUE(st.ok()) << st.to_string();
+ SegmentWriter writer(file_writer.get(), segment_id, build_schema, nullptr,
nullptr, opts,
+ nullptr);
+ st = writer.init();
+ EXPECT_TRUE(st.ok());
+
+ RowCursor row;
+ auto olap_st = row.init(build_schema);
+ EXPECT_EQ(Status::OK(), olap_st);
+
+ for (size_t rid = 0; rid < nrows; ++rid) {
+ for (int cid = 0; cid < build_schema->num_columns(); ++cid) {
+ RowCursorCell cell = row.cell(cid);
+ generator(rid, cid, cell);
+ }
+ EXPECT_TRUE(writer.append_row(row).ok());
+ }
+
+ uint64_t file_size, index_size;
+ st = writer.finalize(&file_size, &index_size);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(file_writer->close().ok());
+
+ EXPECT_NE("", writer.min_encoded_key().to_string());
+ EXPECT_NE("", writer.max_encoded_key().to_string());
+
+ int64_t tablet_id = 100;
+ st = segment_v2::Segment::open(fs, path, tablet_id, segment_id, rowset_id,
query_schema,
+ io::FileReaderOptions {}, res);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(nrows, (*res)->num_rows());
+}
+
+TabletSchemaSPtr create_schema(const std::vector<TabletColumnPtr>& columns,
+ KeysType keys_type = UNIQUE_KEYS) {
+ TabletSchemaSPtr res = std::make_shared<TabletSchema>();
+
+ for (auto& col : columns) {
+ res->append_column(*col);
+ }
+ res->_keys_type = keys_type;
+ return res;
+}
class DeleteBitmapCalculatorTest : public testing::Test {
public:
void SetUp() override {
@@ -82,59 +134,6 @@ public:
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(kSegmentDir).ok());
}
- TabletSchemaSPtr create_schema(const std::vector<TabletColumnPtr>& columns,
- KeysType keys_type = UNIQUE_KEYS) {
- TabletSchemaSPtr res = std::make_shared<TabletSchema>();
-
- for (auto& col : columns) {
- res->append_column(*col);
- }
- res->_keys_type = keys_type;
- return res;
- }
-
- void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr
build_schema, size_t segment_id,
- TabletSchemaSPtr query_schema, size_t nrows, Generator
generator,
- std::shared_ptr<Segment>* res) {
- std::string filename = fmt::format("{}_{}.dat", rowset_id.to_string(),
segment_id);
- std::string path = fmt::format("{}/{}", kSegmentDir, filename);
- auto fs = io::global_local_filesystem();
-
- io::FileWriterPtr file_writer;
- Status st = fs->create_file(path, &file_writer);
- EXPECT_TRUE(st.ok());
- SegmentWriter writer(file_writer.get(), segment_id, build_schema,
nullptr, nullptr, opts,
- nullptr);
- st = writer.init();
- EXPECT_TRUE(st.ok());
-
- RowCursor row;
- auto olap_st = row.init(build_schema);
- EXPECT_EQ(Status::OK(), olap_st);
-
- for (size_t rid = 0; rid < nrows; ++rid) {
- for (int cid = 0; cid < build_schema->num_columns(); ++cid) {
- RowCursorCell cell = row.cell(cid);
- generator(rid, cid, cell);
- }
- EXPECT_TRUE(writer.append_row(row).ok());
- }
-
- uint64_t file_size, index_size;
- st = writer.finalize(&file_size, &index_size);
- EXPECT_TRUE(st.ok());
- EXPECT_TRUE(file_writer->close().ok());
-
- EXPECT_NE("", writer.min_encoded_key().to_string());
- EXPECT_NE("", writer.max_encoded_key().to_string());
-
- int64_t tablet_id = 100;
- st = segment_v2::Segment::open(fs, path, tablet_id, segment_id,
rowset_id, query_schema,
- io::FileReaderOptions {}, res);
- EXPECT_TRUE(st.ok());
- EXPECT_EQ(nrows, (*res)->num_rows());
- }
-
void run_test(size_t const num_segments, size_t const max_rows_per_segment,
size_t const num_key_columns, bool has_sequence_col,
size_t const num_value_columns, int const random_seed, int
const min_value,
@@ -213,7 +212,7 @@ public:
*(int*)cell.mutable_cell_ptr() = data_map[{sid, rid}][cid];
};
build_segment(opts, tablet_schema, sid, tablet_schema,
datas[sid].size(), generator,
- &segment);
+ &segment, kSegmentDir);
}
// find the location of rows to be deleted using
`MergeIndexDeleteBitmapCalculator`
diff --git a/be/test/olap/segment_footer_cache_test.cpp
b/be/test/olap/segment_footer_cache_test.cpp
new file mode 100644
index 00000000000..670df87424c
--- /dev/null
+++ b/be/test/olap/segment_footer_cache_test.cpp
@@ -0,0 +1,195 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <random>
+
+#include "olap/page_cache.h"
+#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/segment_writer.h"
+#include "olap/storage_engine.h"
+#include "olap/tablet_schema_helper.h"
+
+namespace doris {
+
+TabletColumnPtr create_int_sequence_value(int32_t id, bool is_nullable = true,
+ bool is_bf_column = false, bool
has_bitmap_index = false);
+
+TabletSchemaSPtr create_schema(const std::vector<TabletColumnPtr>& columns,
+ KeysType keys_type = UNIQUE_KEYS);
+
+using Generator = std::function<void(size_t rid, int cid, RowCursorCell&
cell)>;
+
+void build_segment(SegmentWriterOptions opts, TabletSchemaSPtr build_schema,
size_t segment_id,
+ TabletSchemaSPtr query_schema, size_t nrows, Generator
generator,
+ std::shared_ptr<Segment>* res, std::string segment_dir);
+
+static std::string segment_footer_cache_test_segment_dir =
"./ut_dir/segment_footer_cache_test";
+
+class SegmentFooterCacheTest : public ::testing::Test {
+ using Segments = std::vector<std::shared_ptr<segment_v2::Segment>>;
+ Segments create(size_t const num_segments, size_t const
max_rows_per_segment,
+ size_t const num_key_columns, bool has_sequence_col,
+ size_t const num_value_columns, int const random_seed, int
const min_value,
+ int const max_value) {
+ Segments segments(num_segments);
+ segment_v2::SegmentWriterOptions opts;
+ opts.enable_unique_key_merge_on_write = true;
+
+ size_t const num_columns = num_key_columns + has_sequence_col +
num_value_columns;
+ size_t const seq_col_idx = has_sequence_col ? num_key_columns : -1;
+
+ std::vector<TabletColumnPtr> columns;
+
+ for (int i = 0; i < num_key_columns; ++i) {
+ columns.emplace_back(create_int_key(i));
+ }
+ if (has_sequence_col) {
+ columns.emplace_back(create_int_sequence_value(num_key_columns));
+ }
+ for (int i = 0; i < num_value_columns; ++i) {
+ columns.emplace_back(create_int_value(num_key_columns +
has_sequence_col));
+ }
+
+ TabletSchemaSPtr tablet_schema = create_schema(columns, UNIQUE_KEYS);
+
+ std::mt19937 rng(random_seed);
+ std::uniform_int_distribution<int> gen(min_value, max_value);
+
+ std::vector<std::vector<std::vector<int>>> datas(num_segments);
+ std::map<std::pair<size_t, size_t>, std::vector<int>> data_map;
+ // each flat_data of data will be a tuple of (column1, column2, ...,
segment_id, row_id)
+ std::vector<std::vector<int>> flat_data;
+ size_t seq_counter = 0;
+
+ // Generate random data, ensuring that there are no identical keys
within each segment
+ // and the keys within each segment are ordered.
+ // Also, ensure that the sequence values are not equal.
+ for (size_t sid = 0; sid < num_segments; ++sid) {
+ auto& segment_data = datas[sid];
+ for (size_t rid = 0; rid < max_rows_per_segment; ++rid) {
+ std::vector<int> row;
+ for (size_t cid = 0; cid < num_columns; ++cid) {
+ if (cid == seq_col_idx) {
+ row.emplace_back(++seq_counter);
+ } else {
+ row.emplace_back(gen(rng));
+ }
+ }
+ segment_data.emplace_back(row);
+ }
+ std::sort(segment_data.begin(), segment_data.end());
+ segment_data.erase(
+ std::unique(segment_data.begin(), segment_data.end(),
+ [&](std::vector<int> const& lhs,
std::vector<int> const& rhs) {
+ return std::vector<int>(lhs.begin(),
+ lhs.begin() +
num_key_columns) ==
+ std::vector<int>(rhs.begin(),
+ rhs.begin() +
num_key_columns);
+ }),
+ segment_data.end());
+ for (size_t rid = 0; rid < segment_data.size(); ++rid) {
+ data_map[{sid, rid}] = segment_data[rid];
+ auto row = segment_data[rid];
+ row.emplace_back(sid);
+ row.emplace_back(rid);
+ flat_data.emplace_back(row);
+ }
+ }
+
+ // Construct segments using the data generated before.
+ for (size_t sid = 0; sid < num_segments; ++sid) {
+ auto& segment = segments[sid];
+ std::vector<int> row_data;
+ auto generator = [&](size_t rid, int cid, RowCursorCell& cell) {
+ cell.set_not_null();
+ *(int*)cell.mutable_cell_ptr() = data_map[{sid, rid}][cid];
+ };
+ build_segment(opts, tablet_schema, sid, tablet_schema,
datas[sid].size(), generator,
+ &segment, segment_footer_cache_test_segment_dir);
+ }
+
+ return segments;
+ }
+
+ void SetUp() override {
+ auto st = io::global_local_filesystem()->delete_directory(
+ segment_footer_cache_test_segment_dir);
+ ASSERT_TRUE(st.ok()) << st;
+ st =
io::global_local_filesystem()->create_directory(segment_footer_cache_test_segment_dir);
+ ASSERT_TRUE(st.ok()) << st;
+ ExecEnv::GetInstance()->set_storage_engine(
+ std::make_unique<StorageEngine>(EngineOptions {}));
+ _segments = create(2, 10, 2, false, 1, 4933, 1, 3);
+ }
+
+ void TearDown() override {
+ EXPECT_TRUE(io::global_local_filesystem()
+
->delete_directory(segment_footer_cache_test_segment_dir)
+ .ok());
+ }
+
+private:
+ Segments _segments;
+};
+
+TEST_F(SegmentFooterCacheTest, TestGetSegmentFooter) {
+ for (auto segment_ptr : _segments) {
+ std::shared_ptr<segment_v2::SegmentFooterPB> footer;
+ Status st = segment_ptr->_get_segment_footer(footer, nullptr);
+ ASSERT_TRUE(st.ok());
+ }
+
+ for (auto segment_ptr : _segments) {
+ std::shared_ptr<segment_v2::SegmentFooterPB> footer;
+ Status st = segment_ptr->_get_segment_footer(footer, nullptr);
+ ASSERT_TRUE(st.ok());
+ }
+}
+
+TEST_F(SegmentFooterCacheTest, TestGetSegmentFooterCacheKey) {
+ for (auto segment_ptr : _segments) {
+ StoragePageCache::CacheKey cache_key =
segment_ptr->get_segment_footer_cache_key();
+ std::string path_native = segment_ptr->_file_reader->path().native();
+ size_t fsize = segment_ptr->_file_reader->size();
+ size_t offset = fsize - 12;
+ std::cout << "cache_key: " << cache_key.encode() << std::endl;
+ ASSERT_EQ(path_native, cache_key.fname);
+ ASSERT_EQ(fsize, cache_key.fsize);
+ ASSERT_EQ(offset, cache_key.offset);
+ }
+}
+
+TEST_F(SegmentFooterCacheTest, TestSemgnetFooterPBPage) {
+ StoragePageCache cache(16 * 2048, 0, 0, 16);
+ for (auto segment_ptr : _segments) {
+ std::shared_ptr<segment_v2::SegmentFooterPB> footer;
+ Status st = segment_ptr->_get_segment_footer(footer, nullptr);
+ ASSERT_TRUE(st.ok());
+ PageCacheHandle cache_handle;
+ cache.insert(segment_ptr->get_segment_footer_cache_key(), footer,
footer->ByteSizeLong(),
+ &cache_handle, segment_v2::PageTypePB::DATA_PAGE);
+
+
EXPECT_EQ(cache_handle.get<std::shared_ptr<segment_v2::SegmentFooterPB>>(),
footer);
+ auto found = cache.lookup(segment_ptr->get_segment_footer_cache_key(),
&cache_handle,
+ segment_v2::PageTypePB::DATA_PAGE);
+ ASSERT_TRUE(found);
+ }
+}
+
+} // namespace doris
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
