This is an automated email from the ASF dual-hosted git repository.
wangbo pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 977da5ca28a [branch-2.1]count tablet meta's static memory load from
disk (#41429) (#45128)
977da5ca28a is described below
commit 977da5ca28ae5b5341e95304126dde599a9fe697
Author: wangbo <wan...@selectdb.com>
AuthorDate: Mon Dec 9 14:44:20 2024 +0800
[branch-2.1]count tablet meta's static memory load from disk (#41429)
(#45128)
pick #41429
---
be/src/olap/metadata_adder.h | 227 +++++++++++++++++++++
be/src/olap/rowset/rowset_meta.cpp | 5 +
be/src/olap/rowset/rowset_meta.h | 5 +-
.../olap/rowset/segment_v2/bitmap_index_reader.h | 2 +-
.../segment_v2/bloom_filter_index_reader.cpp | 6 +
.../rowset/segment_v2/bloom_filter_index_reader.h | 4 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 15 +-
be/src/olap/rowset/segment_v2/column_reader.h | 4 +-
be/src/olap/rowset/segment_v2/index_page.cpp | 9 +
be/src/olap/rowset/segment_v2/index_page.h | 6 +-
.../rowset/segment_v2/indexed_column_reader.cpp | 12 +-
.../olap/rowset/segment_v2/indexed_column_reader.h | 4 +-
.../olap/rowset/segment_v2/inverted_index_reader.h | 3 +-
.../olap/rowset/segment_v2/ordinal_page_index.cpp | 21 +-
be/src/olap/rowset/segment_v2/ordinal_page_index.h | 4 +-
be/src/olap/rowset/segment_v2/segment.cpp | 14 +-
be/src/olap/rowset/segment_v2/segment.h | 4 +-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 17 +-
be/src/olap/rowset/segment_v2/zone_map_index.h | 5 +-
be/src/olap/tablet_meta.cpp | 3 +-
be/src/olap/tablet_meta.h | 3 +-
be/src/olap/tablet_schema.cpp | 18 +-
be/src/olap/tablet_schema.h | 11 +-
23 files changed, 334 insertions(+), 68 deletions(-)
diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h
new file mode 100644
index 00000000000..bdc9e7a398d
--- /dev/null
+++ b/be/src/olap/metadata_adder.h
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <bvar/bvar.h>
+#include <stdint.h>
+
+namespace doris {
+
+inline bvar::Adder<int64_t>
g_rowset_meta_mem_bytes("doris_rowset_meta_mem_bytes");
+inline bvar::Adder<int64_t> g_rowset_meta_num("doris_rowset_meta_num");
+
+inline bvar::Adder<int64_t>
g_tablet_meta_mem_bytes("doris_tablet_meta_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_meta_num("doris_tablet_meta_num");
+
+inline bvar::Adder<int64_t>
g_tablet_column_mem_bytes("doris_tablet_column_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_column_num("doris_tablet_column_num");
+
+inline bvar::Adder<int64_t>
g_tablet_index_mem_bytes("doris_tablet_index_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_index_num("doris_tablet_index_num");
+
+inline bvar::Adder<int64_t>
g_tablet_schema_mem_bytes("doris_tablet_schema_mem_bytes");
+inline bvar::Adder<int64_t> g_tablet_schema_num("doris_tablet_schema_num");
+
+inline bvar::Adder<int64_t> g_segment_mem_bytes("doris_segment_mem_bytes");
+inline bvar::Adder<int64_t> g_segment_num("doris_segment_num");
+
+inline bvar::Adder<int64_t>
g_column_reader_mem_bytes("doris_column_reader_mem_bytes");
+inline bvar::Adder<int64_t> g_column_reader_num("doris_column_reader_num");
+
+inline bvar::Adder<int64_t>
g_bitmap_index_reader_mem_bytes("doris_bitmap_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_bitmap_index_reader_num("doris_bitmap_index_reader_num");
+
+inline bvar::Adder<int64_t> g_bloom_filter_index_reader_mem_bytes(
+ "doris_bloom_filter_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_bloom_filter_index_reader_num("doris_bloom_filter_index_reader_num");
+
+inline bvar::Adder<int64_t>
g_index_page_reader_mem_bytes("doris_index_page_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_index_page_reader_num("doris_index_page_reader_num");
+
+inline bvar::Adder<int64_t> g_indexed_column_reader_mem_bytes(
+ "doris_indexed_column_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_indexed_column_reader_num("doris_indexed_column_reader_num");
+
+inline bvar::Adder<int64_t> g_inverted_index_reader_mem_bytes(
+ "doris_inverted_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_inverted_index_reader_num("doris_inverted_index_reader_num");
+
+inline bvar::Adder<int64_t> g_ordinal_index_reader_mem_bytes(
+ "doris_ordinal_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_ordinal_index_reader_num("doris_ordinal_index_reader_num");
+
+inline bvar::Adder<int64_t> g_zone_map_index_reader_mem_bytes(
+ "doris_zone_map_index_reader_mem_bytes");
+inline bvar::Adder<int64_t>
g_zone_map_index_reader_num("doris_zone_map_index_reader_num");
+
+class RowsetMeta;
+class TabletMeta;
+class TabletColumn;
+class TabletIndex;
+class TabletSchema;
+
+namespace segment_v2 {
+class Segment;
+class ColumnReader;
+class BitmapIndexReader;
+class BloomFilterIndexReader;
+class IndexPageReader;
+class IndexedColumnReader;
+class InvertedIndexReader;
+class OrdinalIndexReader;
+class ZoneMapIndexReader;
+}; // namespace segment_v2
+
+/*
+ When a derived Class extends MetadataAdder, then the Class's number and
fixed length field's memory can be counted automatically.
+ But if the Class has variable length field, then you should overwrite
get_metadata_size and call update_metadata_size when the Class's memory changes.
+
+ There are some special situations that need to be noted:
+ 1. when the derived Class override copy constructor, you'd better update
memory size(call update_metadata_size) if derived class's
+ memory changed in its copy constructor or you not call MetadataAdder's
copy constructor.
+ 2. when the derived Class override operator=, you'd better update memory
size(call update_metadata_size) if the derived Class has variable length field;
+
+ Anyway, you should update mem size whenever derived Class's memory changes.
+*/
+
+template <typename T>
+class MetadataAdder {
+public:
+ MetadataAdder();
+
+protected:
+ MetadataAdder(const MetadataAdder& other);
+
+ virtual ~MetadataAdder();
+
+ virtual int64_t get_metadata_size() const { return sizeof(T); }
+
+ void update_metadata_size();
+
+ MetadataAdder<T>& operator=(const MetadataAdder<T>& other) = default;
+
+private:
+ int64_t _current_meta_size {0};
+
+ void add_mem_size(int64_t val);
+
+ void add_num(int64_t val);
+};
+
+template <typename T>
+MetadataAdder<T>::MetadataAdder(const MetadataAdder<T>& other) {
+ this->_current_meta_size = other._current_meta_size;
+ add_num(1);
+ add_mem_size(this->_current_meta_size);
+}
+
+template <typename T>
+MetadataAdder<T>::MetadataAdder() {
+ this->_current_meta_size = sizeof(T);
+ add_mem_size(this->_current_meta_size);
+ add_num(1);
+}
+
+template <typename T>
+MetadataAdder<T>::~MetadataAdder() {
+ add_mem_size(-_current_meta_size);
+ add_num(-1);
+}
+
+template <typename T>
+void MetadataAdder<T>::update_metadata_size() {
+ int64_t old_size = _current_meta_size;
+ _current_meta_size = get_metadata_size();
+ int64_t size_diff = _current_meta_size - old_size;
+
+ add_mem_size(size_diff);
+}
+
+template <typename T>
+void MetadataAdder<T>::add_mem_size(int64_t val) {
+ if (val == 0) {
+ return;
+ }
+ if constexpr (std::is_same_v<T, RowsetMeta>) {
+ g_rowset_meta_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletMeta>) {
+ g_tablet_meta_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletColumn>) {
+ g_tablet_column_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletIndex>) {
+ g_tablet_index_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, TabletSchema>) {
+ g_tablet_schema_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
+ g_segment_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
+ g_column_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
+ g_bitmap_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T,
segment_v2::BloomFilterIndexReader>) {
+ g_bloom_filter_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexPageReader>) {
+ g_index_page_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexedColumnReader>) {
+ g_indexed_column_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::InvertedIndexReader>) {
+ g_inverted_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::OrdinalIndexReader>) {
+ g_ordinal_index_reader_mem_bytes << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
+ g_zone_map_index_reader_mem_bytes << val;
+ }
+}
+
+template <typename T>
+void MetadataAdder<T>::add_num(int64_t val) {
+ if (val == 0) {
+ return;
+ }
+ if constexpr (std::is_same_v<T, RowsetMeta>) {
+ g_rowset_meta_num << val;
+ } else if constexpr (std::is_same_v<T, TabletMeta>) {
+ g_tablet_meta_num << val;
+ } else if constexpr (std::is_same_v<T, TabletColumn>) {
+ g_tablet_column_num << val;
+ } else if constexpr (std::is_same_v<T, TabletIndex>) {
+ g_tablet_index_num << val;
+ } else if constexpr (std::is_same_v<T, TabletSchema>) {
+ g_tablet_schema_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::Segment>) {
+ g_segment_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ColumnReader>) {
+ g_column_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::BitmapIndexReader>) {
+ g_bitmap_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T,
segment_v2::BloomFilterIndexReader>) {
+ g_bloom_filter_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexPageReader>) {
+ g_index_page_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexedColumnReader>) {
+ g_indexed_column_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::InvertedIndexReader>) {
+ g_inverted_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::OrdinalIndexReader>) {
+ g_ordinal_index_reader_num << val;
+ } else if constexpr (std::is_same_v<T, segment_v2::ZoneMapIndexReader>) {
+ g_zone_map_index_reader_num << val;
+ }
+}
+
+}; // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/rowset_meta.cpp
b/be/src/olap/rowset/rowset_meta.cpp
index d37f5757064..8dffeadfc53 100644
--- a/be/src/olap/rowset/rowset_meta.cpp
+++ b/be/src/olap/rowset/rowset_meta.cpp
@@ -171,6 +171,11 @@ void RowsetMeta::_init() {
} else {
_rowset_id.init(_rowset_meta_pb.rowset_id_v2());
}
+ update_metadata_size();
+}
+
+int64_t RowsetMeta::get_metadata_size() const {
+ return sizeof(RowsetMeta) + _rowset_meta_pb.ByteSizeLong();
}
bool operator==(const RowsetMeta& a, const RowsetMeta& b) {
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index 99221789b81..b327b01110c 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -25,6 +25,7 @@
#include <vector>
#include "io/fs/file_system.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_fwd.h"
#include "olap/tablet_fwd.h"
@@ -32,11 +33,13 @@
namespace doris {
-class RowsetMeta {
+class RowsetMeta : public MetadataAdder<RowsetMeta> {
public:
RowsetMeta() = default;
~RowsetMeta();
+ int64_t get_metadata_size() const override;
+
bool init(const std::string& pb_rowset_meta);
bool init(const RowsetMeta* rowset_meta);
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
index 9753972583e..8d344e43ac7 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
@@ -41,7 +41,7 @@ namespace segment_v2 {
class BitmapIndexIterator;
class BitmapIndexPB;
-class BitmapIndexReader {
+class BitmapIndexReader : public MetadataAdder<BitmapIndexReader> {
public:
explicit BitmapIndexReader(io::FileReaderSPtr file_reader, const
BitmapIndexPB& index_meta)
: _file_reader(std::move(file_reader)),
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
index 0857c1890c4..3a1c9f53813 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
@@ -38,11 +38,17 @@ Status BloomFilterIndexReader::load(bool use_page_cache,
bool kept_in_memory) {
});
}
+int64_t BloomFilterIndexReader::get_metadata_size() const {
+ return sizeof(BloomFilterIndexReader) +
+ (_bloom_filter_index_meta ?
_bloom_filter_index_meta->ByteSizeLong() : 0);
+}
+
Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory)
{
const IndexedColumnMetaPB& bf_index_meta =
_bloom_filter_index_meta->bloom_filter();
_bloom_filter_reader.reset(new IndexedColumnReader(_file_reader,
bf_index_meta));
RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache,
kept_in_memory));
+ update_metadata_size();
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
index c2617ef4e4e..a10a910b2e1 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
@@ -38,7 +38,7 @@ class BloomFilterIndexIterator;
class BloomFilter;
class BloomFilterIndexPB;
-class BloomFilterIndexReader {
+class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
public:
explicit BloomFilterIndexReader(io::FileReaderSPtr file_reader,
const BloomFilterIndexPB&
bloom_filter_index_meta)
@@ -59,6 +59,8 @@ public:
private:
Status _load(bool use_page_cache, bool kept_in_memory);
+ int64_t get_metadata_size() const override;
+
private:
friend class BloomFilterIndexIterator;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index e8108faeafd..d3d7a3d2f62 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -76,9 +76,6 @@
namespace doris {
namespace segment_v2 {
-static bvar::Adder<size_t>
g_column_reader_memory_bytes("doris_column_reader_memory_bytes");
-static bvar::Adder<size_t> g_column_reader_num("doris_column_reader_num");
-
Status ColumnReader::create(const ColumnReaderOptions& opts, const
ColumnMetaPB& meta,
uint64_t num_rows, const io::FileReaderSPtr&
file_reader,
std::unique_ptr<ColumnReader>* reader) {
@@ -209,14 +206,12 @@ ColumnReader::ColumnReader(const ColumnReaderOptions&
opts, const ColumnMetaPB&
_meta_is_nullable = meta.is_nullable();
_meta_dict_page = meta.dict_page();
_meta_compression = meta.compression();
-
- g_column_reader_memory_bytes << sizeof(*this);
- g_column_reader_num << 1;
}
-ColumnReader::~ColumnReader() {
- g_column_reader_memory_bytes << -sizeof(*this);
- g_column_reader_num << -1;
+ColumnReader::~ColumnReader() = default;
+
+int64_t ColumnReader::get_metadata_size() const {
+ return sizeof(ColumnReader) + (_segment_zone_map ?
_segment_zone_map->ByteSizeLong() : 0);
}
Status ColumnReader::init(const ColumnMetaPB* meta) {
@@ -252,6 +247,8 @@ Status ColumnReader::init(const ColumnMetaPB* meta) {
}
}
+ update_metadata_size();
+
// ArrayColumnWriter writes a single empty array and flushes. In this
scenario,
// the item writer doesn't write any data and the corresponding ordinal
index is empty.
if (_ordinal_index == nullptr && !is_empty()) {
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h
b/be/src/olap/rowset/segment_v2/column_reader.h
index 0478427bcd9..2541ac02390 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -106,7 +106,7 @@ struct ColumnIteratorOptions {
// we should do our best to reduce resource usage through share
// same information, such as OrdinalPageIndex and Page data.
// This will cache data shared by all reader
-class ColumnReader {
+class ColumnReader : public MetadataAdder<ColumnReader> {
public:
// Create an initialized ColumnReader in *reader.
// This should be a lightweight operation without I/O.
@@ -224,6 +224,8 @@ private:
Status _calculate_row_ranges(const std::vector<uint32_t>& page_indexes,
RowRanges* row_ranges);
+ int64_t get_metadata_size() const override;
+
private:
int64_t _meta_length;
FieldType _meta_type;
diff --git a/be/src/olap/rowset/segment_v2/index_page.cpp
b/be/src/olap/rowset/segment_v2/index_page.cpp
index 9af7047c49b..1b033a9ff62 100644
--- a/be/src/olap/rowset/segment_v2/index_page.cpp
+++ b/be/src/olap/rowset/segment_v2/index_page.cpp
@@ -64,6 +64,10 @@ Status IndexPageBuilder::get_first_key(Slice* key) const {
///////////////////////////////////////////////////////////////////////////////
+int64_t IndexPageReader::get_metadata_size() const {
+ return sizeof(IndexPageReader) + _vl_field_mem_size;
+}
+
Status IndexPageReader::parse(const Slice& body, const IndexPageFooterPB&
footer) {
_footer = footer;
size_t num_entries = _footer.num_entries();
@@ -80,8 +84,13 @@ Status IndexPageReader::parse(const Slice& body, const
IndexPageFooterPB& footer
}
_keys.push_back(key);
_values.push_back(value);
+ _vl_field_mem_size += sizeof(char) * key.size;
}
+ _vl_field_mem_size +=
+ _keys.capacity() * sizeof(Slice) + _values.capacity() *
sizeof(PagePointer);
+ _vl_field_mem_size += _footer.ByteSizeLong();
+ update_metadata_size();
_parsed = true;
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/index_page.h
b/be/src/olap/rowset/segment_v2/index_page.h
index 7b15ef66391..0ebf425fc5c 100644
--- a/be/src/olap/rowset/segment_v2/index_page.h
+++ b/be/src/olap/rowset/segment_v2/index_page.h
@@ -26,6 +26,7 @@
#include <vector>
#include "common/status.h"
+#include "olap/metadata_adder.h"
#include "olap/rowset/segment_v2/page_pointer.h"
#include "util/faststring.h"
#include "util/slice.h"
@@ -79,7 +80,7 @@ private:
uint32_t _count = 0;
};
-class IndexPageReader {
+class IndexPageReader : public MetadataAdder<IndexPageReader> {
public:
IndexPageReader() : _parsed(false) {}
@@ -110,11 +111,14 @@ public:
void reset();
private:
+ int64_t get_metadata_size() const override;
+
bool _parsed;
IndexPageFooterPB _footer;
std::vector<Slice> _keys;
std::vector<PagePointer> _values;
+ int64_t _vl_field_mem_size {0};
};
class IndexPageIterator {
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index 59251b5595d..cce35d0b8d6 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -56,10 +56,12 @@ static bvar::Adder<uint64_t>
g_index_reader_pk_pages("doris_pk", "index_reader_p
static bvar::PerSecond<bvar::Adder<uint64_t>>
g_index_reader_pk_bytes_per_second(
"doris_pk", "index_reader_pk_pages_per_second",
&g_index_reader_pk_pages, 60);
-static bvar::Adder<uint64_t>
g_index_reader_memory_bytes("doris_index_reader_memory_bytes");
-
using strings::Substitute;
+int64_t IndexedColumnReader::get_metadata_size() const {
+ return sizeof(IndexedColumnReader) + _meta.ByteSizeLong();
+}
+
Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) {
_use_page_cache = use_page_cache;
_kept_in_memory = kept_in_memory;
@@ -94,7 +96,7 @@ Status IndexedColumnReader::load(bool use_page_cache, bool
kept_in_memory) {
}
_num_values = _meta.num_values();
- g_index_reader_memory_bytes << sizeof(*this);
+ update_metadata_size();
return Status::OK();
}
@@ -138,9 +140,7 @@ Status IndexedColumnReader::read_page(const PagePointer&
pp, PageHandle* handle,
return st;
}
-IndexedColumnReader::~IndexedColumnReader() {
- g_index_reader_memory_bytes << -sizeof(*this);
-}
+IndexedColumnReader::~IndexedColumnReader() = default;
///////////////////////////////////////////////////////////////////////////////
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
index d156643a21c..8a57383cd04 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h
@@ -46,7 +46,7 @@ namespace segment_v2 {
class EncodingInfo;
// thread-safe reader for IndexedColumn (see comments of `IndexedColumnWriter`
to understand what IndexedColumn is)
-class IndexedColumnReader {
+class IndexedColumnReader : public MetadataAdder<IndexedColumnReader> {
public:
explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const
IndexedColumnMetaPB& meta)
: _file_reader(std::move(file_reader)), _meta(meta) {}
@@ -72,6 +72,8 @@ public:
private:
Status load_index_page(const PagePointerPB& pp, PageHandle* handle,
IndexPageReader* reader);
+ int64_t get_metadata_size() const override;
+
friend class IndexedColumnIterator;
io::FileReaderSPtr _file_reader;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index 629af669e3c..662136d0782 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -171,7 +171,8 @@ public:
bool is_empty() const { return (_data_bitmap == nullptr && _null_bitmap ==
nullptr); }
};
-class InvertedIndexReader : public
std::enable_shared_from_this<InvertedIndexReader> {
+class InvertedIndexReader : public
std::enable_shared_from_this<InvertedIndexReader>,
+ public MetadataAdder<InvertedIndexReader> {
public:
explicit InvertedIndexReader(
const TabletIndex* index_meta,
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
index 24b2e337996..9ee82bacdd7 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp
@@ -34,8 +34,6 @@
namespace doris {
-static bvar::Adder<size_t>
g_ordinal_index_memory_bytes("doris_ordinal_index_memory_bytes");
-
namespace segment_v2 {
void OrdinalIndexWriter::append_entry(ordinal_t ordinal, const PagePointer&
data_pp) {
@@ -116,10 +114,6 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
_ordinals.resize(_num_pages + 1);
_pages.resize(_num_pages);
- g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() *
sizeof(ordinal_t) +
- _pages.size() *
sizeof(PagePointer) +
- sizeof(OrdinalIndexReader);
-
for (int i = 0; i < _num_pages; i++) {
Slice key = reader.get_key(i);
ordinal_t ordinal = 0;
@@ -132,9 +126,16 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
}
_ordinals[_num_pages] = _num_values;
+ update_metadata_size();
+
return Status::OK();
}
+int64_t OrdinalIndexReader::get_metadata_size() const {
+ return sizeof(OrdinalIndexReader) + _ordinals.capacity() *
sizeof(ordinal_t) +
+ _pages.capacity() * sizeof(PagePointer);
+}
+
OrdinalPageIndexIterator OrdinalIndexReader::seek_at_or_before(ordinal_t
ordinal) {
int32_t left = 0;
int32_t right = _num_pages - 1;
@@ -156,13 +157,7 @@ OrdinalPageIndexIterator
OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal
return OrdinalPageIndexIterator(this, left);
}
-OrdinalIndexReader::~OrdinalIndexReader() {
- if (_ordinals.size() > 0) {
- g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() *
sizeof(ordinal_t) -
- _pages.size() *
sizeof(PagePointer) -
- sizeof(OrdinalIndexReader);
- }
-}
+OrdinalIndexReader::~OrdinalIndexReader() = default;
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
index 8f9e0afe1bf..1d74cf98952 100644
--- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h
+++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h
@@ -64,7 +64,7 @@ private:
class OrdinalPageIndexIterator;
-class OrdinalIndexReader {
+class OrdinalIndexReader : public MetadataAdder<OrdinalIndexReader> {
public:
explicit OrdinalIndexReader(io::FileReaderSPtr file_reader, ordinal_t
num_values,
const OrdinalIndexPB& meta_pb)
@@ -96,6 +96,8 @@ private:
Status _load(bool use_page_cache, bool kept_in_memory,
std::unique_ptr<OrdinalIndexPB> index_meta);
+ int64_t get_metadata_size() const override;
+
private:
friend OrdinalPageIndexIterator;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index 554604d4f0e..dd77a14e150 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -73,7 +73,6 @@ namespace doris {
namespace segment_v2 {
-bvar::Adder<size_t> g_total_segment_num("doris_total_segment_num");
class InvertedIndexIterator;
Status Segment::open(io::FileSystemSPtr fs, const std::string& path, uint32_t
segment_id,
@@ -93,12 +92,13 @@ Segment::Segment(uint32_t segment_id, RowsetId rowset_id,
TabletSchemaSPtr table
: _segment_id(segment_id),
_meta_mem_usage(0),
_rowset_id(rowset_id),
- _tablet_schema(tablet_schema) {
- g_total_segment_num << 1;
-}
+ _tablet_schema(tablet_schema) {}
+
+Segment::~Segment() = default;
-Segment::~Segment() {
- g_total_segment_num << -1;
+int64_t Segment::get_metadata_size() const {
+ return sizeof(Segment) + (_footer_pb ? _footer_pb->ByteSizeLong() : 0) +
+ (_pk_index_meta ? _pk_index_meta->ByteSizeLong() : 0);
}
Status Segment::_open() {
@@ -120,6 +120,8 @@ Status Segment::_open() {
_meta_mem_usage += sizeof(*this);
_meta_mem_usage += _tablet_schema->num_columns() *
config::estimated_mem_per_column_reader;
+ update_metadata_size();
+
// 1024 comes from SegmentWriterOptions
_meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
// 0.01 comes from PrimaryKeyIndexBuilder::init
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index 3fbc54dcc30..442dab3393e 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -77,7 +77,7 @@ using SegmentSharedPtr = std::shared_ptr<Segment>;
// NOTE: This segment is used to a specified TabletSchema, when TabletSchema
// is changed, this segment can not be used any more. For example, after a
schema
// change finished, client should disable all cached Segment for old
TabletSchema.
-class Segment : public std::enable_shared_from_this<Segment> {
+class Segment : public std::enable_shared_from_this<Segment>, public
MetadataAdder<Segment> {
public:
static Status open(io::FileSystemSPtr fs, const std::string& path,
uint32_t segment_id,
RowsetId rowset_id, TabletSchemaSPtr tablet_schema,
@@ -85,6 +85,8 @@ public:
std::shared_ptr<Segment>* output);
~Segment();
+ int64_t get_metadata_size() const override;
+
Status new_iterator(SchemaSPtr schema, const StorageReadOptions&
read_options,
std::unique_ptr<RowwiseIterator>* iter);
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
index 991df2f9475..c2139ff0899 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
@@ -39,8 +39,6 @@
namespace doris {
struct uint24_t;
-static bvar::Adder<size_t>
g_zone_map_memory_bytes("doris_zone_map_memory_bytes");
-
namespace segment_v2 {
template <PrimitiveType Type>
@@ -157,9 +155,6 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
_page_zone_maps.resize(reader.num_values());
- g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) *
_page_zone_maps.size() +
- sizeof(IndexedColumnMetaPB);
-
// read and cache all page zone maps
for (int i = 0; i < reader.num_values(); ++i) {
size_t num_to_read = 1;
@@ -177,18 +172,18 @@ Status ZoneMapIndexReader::_load(bool use_page_cache,
bool kept_in_memory,
column->get_data_at(0).size)) {
return Status::Corruption("Failed to parse zone map");
}
+ _pb_meta_size += _page_zone_maps[i].ByteSizeLong();
}
+ update_metadata_size();
return Status::OK();
}
-ZoneMapIndexReader::~ZoneMapIndexReader() {
- // Maybe wrong due to load failures.
- if (_page_zone_maps.size() > 0) {
- g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) *
_page_zone_maps.size() -
- sizeof(IndexedColumnMetaPB);
- }
+int64_t ZoneMapIndexReader::get_metadata_size() const {
+ return sizeof(ZoneMapIndexReader) + _pb_meta_size;
}
+
+ZoneMapIndexReader::~ZoneMapIndexReader() = default;
#define APPLY_FOR_PRIMITITYPE(M) \
M(TYPE_TINYINT) \
M(TYPE_SMALLINT) \
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h
b/be/src/olap/rowset/segment_v2/zone_map_index.h
index 923bd2c2046..34869bbbfee 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.h
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.h
@@ -143,7 +143,7 @@ private:
uint64_t _estimated_size = 0;
};
-class ZoneMapIndexReader {
+class ZoneMapIndexReader : public MetadataAdder<ZoneMapIndexReader> {
public:
explicit ZoneMapIndexReader(io::FileReaderSPtr file_reader,
const IndexedColumnMetaPB& page_zone_maps)
@@ -163,12 +163,15 @@ public:
private:
Status _load(bool use_page_cache, bool kept_in_memory,
std::unique_ptr<IndexedColumnMetaPB>);
+ int64_t get_metadata_size() const override;
+
private:
DorisCallOnce<Status> _load_once;
// TODO: yyq, we shoud remove file_reader from here.
io::FileReaderSPtr _file_reader;
std::unique_ptr<IndexedColumnMetaPB> _page_zone_maps_meta;
std::vector<ZoneMapPB> _page_zone_maps;
+ int64_t _pb_meta_size {0};
};
} // namespace segment_v2
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index dd17041b6b1..b7c1d0b041f 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -318,7 +318,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
}
TabletMeta::TabletMeta(const TabletMeta& b)
- : _table_id(b._table_id),
+ : MetadataAdder(b),
+ _table_id(b._table_id),
_partition_id(b._partition_id),
_tablet_id(b._tablet_id),
_replica_id(b._replica_id),
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 3c36cad53ab..4a67f6b0f9d 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -43,6 +43,7 @@
#include "io/fs/file_system.h"
#include "olap/binlog_config.h"
#include "olap/lru_cache.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/rowset_meta.h"
#include "olap/tablet_schema.h"
@@ -90,7 +91,7 @@ class TBinlogConfig;
// Class encapsulates meta of tablet.
// The concurrency control is handled in Tablet Class, not in this class.
-class TabletMeta {
+class TabletMeta : public MetadataAdder<TabletMeta> {
public:
static TabletMetaSharedPtr create(
const TCreateTabletReq& request, const TabletUid& tablet_uid,
uint64_t shard_id,
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 7e83e15eac4..7da0f99537a 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -55,8 +55,6 @@
namespace doris {
-static bvar::Adder<size_t>
g_total_tablet_schema_num("doris_total_tablet_schema_num");
-
FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) {
switch (primitiveType) {
case PrimitiveType::INVALID_TYPE:
@@ -845,13 +843,12 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index)
const {
}
}
-TabletSchema::TabletSchema() {
- g_total_tablet_schema_num << 1;
-}
+TabletSchema::TabletSchema() = default;
-TabletSchema::~TabletSchema() {
- g_total_tablet_schema_num << -1;
- clear_column_cache_handlers();
+TabletSchema::~TabletSchema() = default;
+
+int64_t TabletSchema::get_metadata_size() const {
+ return sizeof(TabletSchema) + _vl_field_mem_size;
}
void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
@@ -996,8 +993,11 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema, bool ignore_extrac
_cols.emplace_back(std::move(column));
if (!_cols.back()->is_extracted_column()) {
+ _vl_field_mem_size += sizeof(StringRef) + sizeof(char) *
_cols.back()->name().size() +
+ sizeof(int32_t);
_field_name_to_index.emplace(StringRef(_cols.back()->name()),
_num_columns);
_field_id_to_index[_cols.back()->unique_id()] = _num_columns;
+ _vl_field_mem_size += sizeof(int32_t) * 2;
}
_num_columns++;
}
@@ -1036,6 +1036,8 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema, bool ignore_extrac
} else {
_inverted_index_storage_format =
schema.inverted_index_storage_format();
}
+
+ update_metadata_size();
}
void TabletSchema::copy_from(const TabletSchema& tablet_schema) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 79a885dfa1e..c9a0d45bd9b 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -35,6 +35,7 @@
#include "common/status.h"
#include "gutil/stringprintf.h"
+#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/options.h"
#include "runtime/define_primitive_type.h"
@@ -61,7 +62,7 @@ class TabletColumn;
using TabletColumnPtr = std::shared_ptr<TabletColumn>;
-class TabletColumn {
+class TabletColumn : public MetadataAdder<TabletColumn> {
public:
TabletColumn();
TabletColumn(const ColumnPB& column);
@@ -223,7 +224,7 @@ bool operator!=(const TabletColumn& a, const TabletColumn&
b);
class TabletSchema;
-class TabletIndex {
+class TabletIndex : public MetadataAdder<TabletIndex> {
public:
TabletIndex() = default;
void init_from_thrift(const TOlapTableIndex& index, const TabletSchema&
tablet_schema);
@@ -265,7 +266,7 @@ private:
std::map<string, string> _properties;
};
-class TabletSchema {
+class TabletSchema : public MetadataAdder<TabletSchema> {
public:
enum ColumnType { NORMAL = 0, DROPPED = 1, VARIANT = 2 };
// TODO(yingchun): better to make constructor as private to avoid
@@ -475,6 +476,8 @@ public:
return _inverted_index_storage_format;
}
+ int64_t get_metadata_size() const override;
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
@@ -520,6 +523,8 @@ private:
bool _store_row_column = false;
bool _skip_write_index_on_load = false;
InvertedIndexStorageFormatPB _inverted_index_storage_format =
InvertedIndexStorageFormatPB::V1;
+
+ int64_t _vl_field_mem_size {0}; // variable length field
};
bool operator==(const TabletSchema& a, const TabletSchema& b);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
