This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 34edc578f10 [opt](MergeIO) use equivalent merge size to measure merge
effectiveness (#26741)
34edc578f10 is described below
commit 34edc578f105d8c1e6a141ea372df28251ee9a0b
Author: Ashin Gau <ashin...@users.noreply.github.com>
AuthorDate: Tue Nov 14 10:07:14 2023 +0800
[opt](MergeIO) use equivalent merge size to measure merge effectiveness
(#26741)
`MergeRangeFileReader` is used to merge small IOs, and
`max_amplified_read_ratio` controls the proportion of read amplification.
However, in some extreme cases(eg. `orc strip size`/`parquet row group size` is
less than 3MB), the control effect of `max_amplified_read_ratio` is not good,
resulting in a large amount of small IOs.
After testing, the return time of a single IO for IO size smaller than 4kb
in hdfs(512kb in oss) remains basically unchanged. Therefore, equivalent IO
size is used to measure merge effectiveness:
```
EquivalentIOSize = MergeSize / Request IOs
```
When `EquivalentIOSize` is greater than 4kb in hdfs, or 512kb in oss, we
believe that this kind of merge is effective.
---
be/src/io/fs/buffered_reader.cpp | 8 +++++---
be/src/io/fs/buffered_reader.h | 11 ++++++++++-
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp
index f32e8a3454f..caa80616d35 100644
--- a/be/src/io/fs/buffered_reader.cpp
+++ b/be/src/io/fs/buffered_reader.cpp
@@ -151,7 +151,6 @@ Status MergeRangeFileReader::read_at_impl(size_t offset,
Slice result, size_t* b
}
content_size = 0;
hollow_size = 0;
- double amplified_ratio = config::max_amplified_read_ratio;
std::vector<std::pair<double, size_t>> ratio_and_size;
// Calculate the read amplified ratio for each merge operation and the
size of the merged data.
// Find the largest size of the merged data whose amplified ratio is less
than config::max_amplified_read_ratio
@@ -167,9 +166,12 @@ Status MergeRangeFileReader::read_at_impl(size_t offset,
Slice result, size_t* b
}
}
size_t best_merged_size = 0;
- for (const std::pair<double, size_t>& rs : ratio_and_size) {
+ for (int i = 0; i < ratio_and_size.size(); ++i) {
+ const std::pair<double, size_t>& rs = ratio_and_size[i];
+ size_t equivalent_size = rs.second / (i + 1);
if (rs.second > best_merged_size) {
- if (rs.first < amplified_ratio || rs.second <= MIN_READ_SIZE) {
+ if (rs.first <= _max_amplified_ratio ||
+ (_max_amplified_ratio < 1 && equivalent_size <=
_equivalent_io_size)) {
best_merged_size = rs.second;
}
}
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index 586aa9546fe..0bde47d79d9 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -130,8 +130,9 @@ public:
static constexpr size_t READ_SLICE_SIZE = 8 * 1024 * 1024; // 8MB
static constexpr size_t BOX_SIZE = 1 * 1024 * 1024; // 1MB
static constexpr size_t SMALL_IO = 2 * 1024 * 1024; // 2MB
+ static constexpr size_t HDFS_MIN_IO_SIZE = 4 * 1024; // 4KB
+ static constexpr size_t OSS_MIN_IO_SIZE = 512 * 1024; // 512KB
static constexpr size_t NUM_BOX = TOTAL_BUFFER_SIZE / BOX_SIZE; // 128
- static constexpr size_t MIN_READ_SIZE = 4096; // 4KB
MergeRangeFileReader(RuntimeProfile* profile, io::FileReaderSPtr reader,
const std::vector<PrefetchRange>&
random_access_ranges)
@@ -141,6 +142,11 @@ public:
_range_cached_data.resize(random_access_ranges.size());
_size = _reader->size();
_remaining = TOTAL_BUFFER_SIZE;
+ _is_oss = typeid_cast<io::S3FileReader*>(_reader.get()) != nullptr;
+ _max_amplified_ratio = config::max_amplified_read_ratio;
+ // Equivalent min size of each IO that can reach the maximum storage
speed limit:
+ // 512KB for oss, 4KB for hdfs
+ _equivalent_io_size = _is_oss ? OSS_MIN_IO_SIZE : HDFS_MIN_IO_SIZE;
if (_profile != nullptr) {
const char* random_profile = "MergedSmallIO";
ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1);
@@ -237,6 +243,9 @@ private:
int16 _last_box_ref = -1;
uint32 _last_box_usage = 0;
std::vector<int16> _box_ref;
+ bool _is_oss;
+ double _max_amplified_ratio;
+ size_t _equivalent_io_size;
Statistics _statistics;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
