This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 4e06f136f00 [fix](segment cache) estimate momory consumed by segment
(#35647) (#35751)
4e06f136f00 is described below
commit 4e06f136f0096a8348678a56f0300851fc7034bb
Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com>
AuthorDate: Sat Jun 1 09:34:32 2024 +0800
[fix](segment cache) estimate momory consumed by segment (#35647) (#35751)
The memory consumed in segment cache is 0 after
https://github.com/apache/doris/pull/35432/files.
The pr also tracks memory usage of column readers.
---
be/src/olap/base_tablet.cpp | 4 ++++
be/src/olap/primary_key_index.cpp | 2 ++
be/src/olap/rowset/rowset.cpp | 7 +++++++
be/src/olap/rowset/rowset.h | 2 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 11 ++++++++++-
be/src/olap/rowset/segment_v2/column_reader.h | 2 +-
.../olap/rowset/segment_v2/indexed_column_reader.cpp | 8 ++++++++
be/src/olap/rowset/segment_v2/indexed_column_reader.h | 2 ++
be/src/olap/rowset/segment_v2/ordinal_page_index.cpp | 14 ++++++++++++++
be/src/olap/rowset/segment_v2/ordinal_page_index.h | 2 ++
be/src/olap/rowset/segment_v2/segment.cpp | 18 ++++++++++++++----
be/src/olap/rowset/segment_v2/segment.h | 1 +
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 11 +++++++++++
be/src/olap/rowset/segment_v2/zone_map_index.h | 2 ++
be/src/olap/short_key_index.cpp | 16 ++++++++++++++++
be/src/olap/short_key_index.h | 1 +
be/src/olap/tablet_schema.cpp | 10 ++++++++++
be/src/olap/tablet_schema.h | 4 +++-
18 files changed, 109 insertions(+), 8 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 43f514906cf..8901aee0e5a 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -33,6 +33,8 @@ extern MetricPrototype METRIC_query_scan_count;
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(flush_bytes, MetricUnit::BYTES);
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(flush_finish_count,
MetricUnit::OPERATIONS);
+static bvar::Adder<size_t> g_total_tablet_num("doris_total_tablet_num");
+
BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) :
_tablet_meta(std::move(tablet_meta)) {
_metric_entity =
DorisMetrics::instance()->metric_registry()->register_entity(
fmt::format("Tablet.{}", tablet_id()), {{"tablet_id",
std::to_string(tablet_id())}},
@@ -42,10 +44,12 @@ BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) :
_tablet_meta(std::move
INT_COUNTER_METRIC_REGISTER(_metric_entity, query_scan_count);
INT_COUNTER_METRIC_REGISTER(_metric_entity, flush_bytes);
INT_COUNTER_METRIC_REGISTER(_metric_entity, flush_finish_count);
+ g_total_tablet_num << 1;
}
BaseTablet::~BaseTablet() {
DorisMetrics::instance()->metric_registry()->deregister_entity(_metric_entity);
+ g_total_tablet_num << -1;
}
Status BaseTablet::set_tablet_state(TabletState state) {
diff --git a/be/src/olap/primary_key_index.cpp
b/be/src/olap/primary_key_index.cpp
index 6d1b1772a91..6ea4bc30d0a 100644
--- a/be/src/olap/primary_key_index.cpp
+++ b/be/src/olap/primary_key_index.cpp
@@ -32,6 +32,8 @@
namespace doris {
+static bvar::Adder<size_t>
g_primary_key_index_memory_bytes("doris_primary_key_index_memory_bytes");
+
Status PrimaryKeyIndexBuilder::init() {
// TODO(liaoxin) using the column type directly if there's only one column
in unique key columns
const auto* type_info =
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index c9fa1c6a3b9..f4667d3fb63 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -26,6 +26,8 @@
namespace doris {
+static bvar::Adder<size_t> g_total_rowset_num("doris_total_rowset_num");
+
Rowset::Rowset(const TabletSchemaSPtr& schema, const RowsetMetaSharedPtr&
rowset_meta)
: _rowset_meta(rowset_meta), _refs_by_reader(0) {
_is_pending = !_rowset_meta->has_version();
@@ -37,6 +39,11 @@ Rowset::Rowset(const TabletSchemaSPtr& schema, const
RowsetMetaSharedPtr& rowset
}
// build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema
_schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() :
schema;
+ g_total_rowset_num << 1;
+}
+
+Rowset::~Rowset() {
+ g_total_rowset_num << -1;
}
Status Rowset::load(bool use_cache) {
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index dffa4a7a3ee..6194703176f 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -118,7 +118,7 @@ private:
class Rowset : public std::enable_shared_from_this<Rowset> {
public:
- virtual ~Rowset() = default;
+ virtual ~Rowset();
// Open all segment files in this rowset and load necessary metadata.
// - `use_cache` : whether to use fd cache, only applicable to alpha
rowset now
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 6d5c28b8d6d..24b755d70cd 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -76,6 +76,9 @@
namespace doris {
namespace segment_v2 {
+static bvar::Adder<size_t>
g_column_reader_memory_bytes("doris_column_reader_memory_bytes");
+static bvar::Adder<size_t> g_column_reader_num("doris_column_reader_num");
+
Status ColumnReader::create(const ColumnReaderOptions& opts, const
ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr&
file_reader,
std::unique_ptr<ColumnReader>* reader) {
@@ -205,9 +208,15 @@ ColumnReader::ColumnReader(const ColumnReaderOptions&
opts, const ColumnMetaPB&
_meta_is_nullable = meta.is_nullable();
_meta_dict_page = meta.dict_page();
_meta_compression = meta.compression();
+
+ g_column_reader_memory_bytes << sizeof(*this);
+ g_column_reader_num << 1;
}
-ColumnReader::~ColumnReader() = default;
+ColumnReader::~ColumnReader() {
+ g_column_reader_memory_bytes << -sizeof(*this);
+ g_column_reader_num << -1;
+}
Status ColumnReader::init(const ColumnMetaPB* meta) {
_type_info = get_type_info(meta);
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h
b/be/src/olap/rowset/segment_v2/column_reader.h
index 30f916d00cd..f7330b1727b 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -116,7 +116,7 @@ public:
enum DictEncodingType { UNKNOWN_DICT_ENCODING, PARTIAL_DICT_ENCODING,
ALL_DICT_ENCODING };
- ~ColumnReader();
+ virtual ~ColumnReader();
// create a new column iterator. Client should delete returned iterator
Status new_iterator(ColumnIterator** iterator);
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index b5f974fb8a0..59251b5595d 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -56,6 +56,8 @@ static bvar::Adder<uint64_t>
g_index_reader_pk_pages("doris_pk", "index_reader_p
static bvar::PerSecond<bvar::Adder<uint64_t>>
g_index_reader_pk_bytes_per_second(
"doris_pk", "index_reader_pk_pages_per_second",
&g_index_reader_pk_pages, 60);
+static bvar::Adder<uint64_t>
g_index_reader_memory_bytes("doris_index_reader_memory_bytes");
+
using strings::Substitute;
Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) {
@@ -91,6 +93,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool
kept_in_memory) {
}
}
_num_values = _meta.num_values();
+
+ g_index_reader_memory_bytes << sizeof(*this);
return Status::OK();
}
@@ -134,6 +138,10 @@ Status IndexedColumnReader::read_page(const PagePointer&
pp, PageHandle* handle,
return st;
}
+IndexedColumnReader::~IndexedColumnReader() {
+ g_index_reader_memory_bytes << -sizeof(*this);
+}
+
///////////////////////////////////////////////////////////////////////////////
Status IndexedColumnIterator::_read_data_page(const PagePointer& pp) {
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index ef6e2754609..d156643a21c 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -51,6 +51,8 @@ public:
explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const
IndexedColumnMetaPB& meta)
: _file_reader(std::move(file_reader)), _meta(meta) {}
+ ~IndexedColumnReader();
+
Status load(bool use_page_cache, bool kept_in_memory);
// read a page specified by `pp' from `file' into `handle'
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
index 5761cc42b9d..b93d461d8c7 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
@@ -29,9 +29,13 @@
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/page_handle.h"
#include "olap/rowset/segment_v2/page_io.h"
+#include "ordinal_page_index.h"
#include "util/slice.h"
namespace doris {
+
+static bvar::Adder<size_t>
g_ordinal_index_memory_bytes("doris_ordinal_index_memory_bytes");
+
namespace segment_v2 {
void OrdinalIndexWriter::append_entry(ordinal_t ordinal, const PagePointer&
data_pp) {
@@ -122,6 +126,10 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
_pages[i] = reader.get_value(i);
}
_ordinals[_num_pages] = _num_values;
+
+ g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() *
sizeof(ordinal_t) +
+ _pages.size() *
sizeof(PagePointer) +
+ sizeof(OrdinalIndexReader);
return Status::OK();
}
@@ -146,5 +154,11 @@ OrdinalPageIndexIterator
OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal
return OrdinalPageIndexIterator(this, left);
}
+OrdinalIndexReader::~OrdinalIndexReader() {
+ g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() *
sizeof(ordinal_t) -
+ _pages.size() *
sizeof(PagePointer) -
+ sizeof(OrdinalIndexReader);
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
index 77f0b851816..8f9e0afe1bf 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
@@ -72,6 +72,8 @@ public:
_meta_pb.reset(new OrdinalIndexPB(meta_pb));
}
+ virtual ~OrdinalIndexReader();
+
// load and parse the index page into memory
Status load(bool use_page_cache, bool kept_in_memory);
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index c18eec3ad6e..e121639ac9e 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -113,6 +113,17 @@ Status Segment::_open() {
// DCHECK(footer.has_short_key_index_page());
_sk_index_page = _footer_pb->short_key_index_page();
_num_rows = _footer_pb->num_rows();
+
+ // An estimated memory usage of a segment
+ _meta_mem_usage += _footer_pb->ByteSizeLong();
+ _meta_mem_usage += sizeof(*this);
+ _meta_mem_usage += _tablet_schema->num_columns() *
config::estimated_mem_per_column_reader;
+
+ // 1024 comes from SegmentWriterOptions
+ _meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
+ // 0.01 comes from PrimaryKeyIndexBuilder::init
+ _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8;
+
return Status::OK();
}
@@ -301,7 +312,7 @@ Status Segment::_load_pk_bloom_filter() {
auto status = [this]() {
return _load_pk_bf_once.call([this] {
RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader,
*_pk_index_meta));
- _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
+ // _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
return Status::OK();
});
}();
@@ -338,7 +349,7 @@ Status Segment::_load_index_impl() {
if (_tablet_schema->keys_type() == UNIQUE_KEYS && _pk_index_meta !=
nullptr) {
_pk_index_reader.reset(new PrimaryKeyIndexReader());
RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader,
*_pk_index_meta));
- _meta_mem_usage += _pk_index_reader->get_memory_size();
+ // _meta_mem_usage += _pk_index_reader->get_memory_size();
return Status::OK();
} else {
// read and parse short key index page
@@ -360,7 +371,7 @@ Status Segment::_load_index_impl() {
DCHECK_EQ(footer.type(), SHORT_KEY_PAGE);
DCHECK(footer.has_short_key_page_footer());
- _meta_mem_usage += body.get_size();
+ // _meta_mem_usage += body.get_size();
_sk_index_decoder.reset(new ShortKeyIndexDecoder);
return _sk_index_decoder->parse(body,
footer.short_key_page_footer());
}
@@ -430,7 +441,6 @@ Status Segment::_create_column_readers(const
SegmentFooterPB& footer) {
RETURN_IF_ERROR(ColumnReader::create(opts,
footer.columns(iter->second), footer.num_rows(),
_file_reader, &reader));
_column_readers.emplace(column.unique_id(), std::move(reader));
- _meta_mem_usage += config::estimated_mem_per_column_reader;
}
// init by column path
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index 269d5f86364..1afe0311ffe 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -256,6 +256,7 @@ private:
// used to hold short key index page in memory
PageHandle _sk_index_handle;
// short key index decoder
+ // all content is in memory
std::unique_ptr<ShortKeyIndexDecoder> _sk_index_decoder;
// primary key index reader
std::unique_ptr<PrimaryKeyIndexReader> _pk_index_reader;
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
index e232e448ff6..6a1dee39cd7 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
@@ -39,6 +39,8 @@
namespace doris {
struct uint24_t;
+static bvar::Adder<size_t>
g_zone_map_memory_bytes("doris_zone_map_memory_bytes");
+
namespace segment_v2 {
template <PrimitiveType Type>
@@ -173,9 +175,18 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
return Status::Corruption("Failed to parse zone map");
}
}
+
+ g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) *
_page_zone_maps.size() +
+ sizeof(IndexedColumnMetaPB);
+
return Status::OK();
}
+ZoneMapIndexReader::~ZoneMapIndexReader() {
+ // Maybe wrong due to load failures.
+ g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) *
_page_zone_maps.size() -
+ sizeof(IndexedColumnMetaPB);
+}
#define APPLY_FOR_PRIMITITYPE(M) \
M(TYPE_TINYINT) \
M(TYPE_SMALLINT) \
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h
b/be/src/olap/rowset/segment_v2/zone_map_index.h
index eeb87eb7da7..923bd2c2046 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.h
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.h
@@ -151,6 +151,8 @@ public:
_page_zone_maps_meta.reset(new IndexedColumnMetaPB(page_zone_maps));
}
+ virtual ~ZoneMapIndexReader();
+
// load all page zone maps into memory
Status load(bool use_page_cache, bool kept_in_memory);
diff --git a/be/src/olap/short_key_index.cpp b/be/src/olap/short_key_index.cpp
index 69622cee454..3ef235e3018 100644
--- a/be/src/olap/short_key_index.cpp
+++ b/be/src/olap/short_key_index.cpp
@@ -22,12 +22,16 @@
#include <ostream>
#include "gutil/strings/substitute.h"
+#include "short_key_index.h"
+#include "util/bvar_helper.h"
#include "util/coding.h"
using strings::Substitute;
namespace doris {
+static bvar::Adder<size_t>
g_short_key_index_memory_bytes("doris_short_key_index_memory_bytes");
+
Status ShortKeyIndexBuilder::add_item(const Slice& key) {
put_varint32(&_offset_buf, _key_buf.size());
_key_buf.append(key.data, key.size);
@@ -85,7 +89,19 @@ Status ShortKeyIndexDecoder::parse(const Slice& body, const
segment_v2::ShortKey
return Status::Corruption("Still has data after parse all key offset");
}
_parsed = true;
+
+ g_short_key_index_memory_bytes << sizeof(_footer) + _key_data.size +
+ _offsets.size() *
sizeof(uint32_t) + sizeof(*this);
+
return Status::OK();
}
+ShortKeyIndexDecoder::~ShortKeyIndexDecoder() {
+ if (_parsed) {
+ g_short_key_index_memory_bytes << -sizeof(_footer) - _key_data.size -
+ _offsets.size() *
sizeof(uint32_t) -
+ sizeof(*this);
+ }
+}
+
} // namespace doris
diff --git a/be/src/olap/short_key_index.h b/be/src/olap/short_key_index.h
index 79303a3edbb..93f4b614908 100644
--- a/be/src/olap/short_key_index.h
+++ b/be/src/olap/short_key_index.h
@@ -135,6 +135,7 @@ private:
class ShortKeyIndexDecoder {
public:
ShortKeyIndexDecoder() : _parsed(false) {}
+ virtual ~ShortKeyIndexDecoder();
// client should assure that body is available when this class is used
Status parse(const Slice& body, const segment_v2::ShortKeyFooterPB&
footer);
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index a44abedc354..99bdf5a8973 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -53,6 +53,8 @@
namespace doris {
+static bvar::Adder<size_t>
g_total_tablet_schema_num("doris_total_tablet_schema_num");
+
FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) {
switch (primitiveType) {
case PrimitiveType::INVALID_TYPE:
@@ -809,6 +811,14 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const
{
}
}
+TabletSchema::TabletSchema() {
+ g_total_tablet_schema_num << 1;
+}
+
+TabletSchema::~TabletSchema() {
+ g_total_tablet_schema_num << -1;
+}
+
void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
if (column.is_key()) {
_num_key_columns++;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index c9e8fdce54e..c6fc428195d 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -269,7 +269,9 @@ public:
// TODO(yingchun): better to make constructor as private to avoid
// manually init members incorrectly, and define a new function like
// void create_from_pb(const TabletSchemaPB& schema, TabletSchema*
tablet_schema).
- TabletSchema() = default;
+ TabletSchema();
+ virtual ~TabletSchema();
+
void init_from_pb(const TabletSchemaPB& schema, bool
ignore_extracted_columns = false);
// Notice: Use deterministic way to serialize protobuf,
// since serialize Map in protobuf may could lead to un-deterministic by
default
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
