This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d33f580e388 [fix](cloud-balance) use right file size instead of block
size (#38939)
d33f580e388 is described below
commit d33f580e3882af57ff88df5718af06ee20f3ada1
Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com>
AuthorDate: Wed Aug 7 10:40:21 2024 +0800
[fix](cloud-balance) use right file size instead of block size (#38939)
introduced by #34191
---
be/src/cloud/cloud_internal_service.cpp | 1 +
be/src/io/cache/block_file_cache_downloader.cpp | 3 ++-
be/src/olap/rowset/segment_v2/segment.cpp | 18 ++++++++++--------
gensrc/proto/internal_service.proto | 1 +
4 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/be/src/cloud/cloud_internal_service.cpp
b/be/src/cloud/cloud_internal_service.cpp
index 63e3d4f3081..66e089c22e9 100644
--- a/be/src/cloud/cloud_internal_service.cpp
+++ b/be/src/cloud/cloud_internal_service.cpp
@@ -94,6 +94,7 @@ void
CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id(
meta->set_rowset_id(rowset_id);
meta->set_segment_id(segment_id);
meta->set_file_name(file_name);
+
meta->set_file_size(rowset->rowset_meta()->segment_file_size(segment_id));
meta->set_offset(std::get<0>(tuple));
meta->set_size(std::get<1>(tuple));
meta->set_cache_type(cache_type_to_pb(std::get<2>(tuple)));
diff --git a/be/src/io/cache/block_file_cache_downloader.cpp
b/be/src/io/cache/block_file_cache_downloader.cpp
index 02e8f736828..585c0ff0159 100644
--- a/be/src/io/cache/block_file_cache_downloader.cpp
+++ b/be/src/io/cache/block_file_cache_downloader.cpp
@@ -171,7 +171,8 @@ void FileCacheBlockDownloader::download_file_cache_block(
DownloadFileMeta download_meta {
.path =
storage_resource.value()->remote_segment_path(*find_it->second,
meta.segment_id()),
- .file_size = meta.offset() + meta.size(), // To avoid trigger
get file size IO
+ .file_size = meta.has_file_size() ? meta.file_size()
+ : -1, // To avoid trigger
get file size IO
.offset = meta.offset(),
.download_size = meta.size(),
.file_system = storage_resource.value()->fs,
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index 1b31117f126..7f353cc0add 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -315,9 +315,10 @@ Status Segment::_parse_footer(SegmentFooterPB* footer) {
DCHECK_EQ(bytes_read, 12);
if (memcmp(fixed_buf + 8, k_segment_magic, k_segment_magic_length) != 0) {
- return Status::Corruption("Bad segment file {}: magic number not
match, cache_key: {}",
- _file_reader->path().native(),
-
file_cache_key_str(_file_reader->path().native()));
+ return Status::Corruption(
+ "Bad segment file {}: file_size: {}, magic number not match,
cache_key: {}",
+ _file_reader->path().native(), file_size,
+ file_cache_key_str(_file_reader->path().native()));
}
// read footer PB
@@ -339,17 +340,18 @@ Status Segment::_parse_footer(SegmentFooterPB* footer) {
uint32_t actual_checksum = crc32c::Value(footer_buf.data(),
footer_buf.size());
if (actual_checksum != expect_checksum) {
return Status::Corruption(
- "Bad segment file {}: footer checksum not match, actual={} vs
expect={}, "
- "cache_key: {}",
- _file_reader->path().native(), actual_checksum,
expect_checksum,
+ "Bad segment file {}: file_size = {}, footer checksum not
match, actual={} "
+ "vs expect={}, cache_key: {}",
+ _file_reader->path().native(), file_size, actual_checksum,
expect_checksum,
file_cache_key_str(_file_reader->path().native()));
}
// deserialize footer PB
if (!footer->ParseFromString(footer_buf)) {
return Status::Corruption(
- "Bad segment file {}: failed to parse SegmentFooterPB,
cache_key: ",
- _file_reader->path().native(),
file_cache_key_str(_file_reader->path().native()));
+ "Bad segment file {}: file_size = {}, failed to parse
SegmentFooterPB, cache_key: ",
+ _file_reader->path().native(), file_size,
+ file_cache_key_str(_file_reader->path().native()));
}
return Status::OK();
}
diff --git a/gensrc/proto/internal_service.proto
b/gensrc/proto/internal_service.proto
index b08308f2144..e188001e491 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -805,6 +805,7 @@ message FileCacheBlockMeta {
required int64 size = 6;
required FileCacheType cache_type = 7;
required int64 expiration_time = 8;
+ optional int64 file_size = 9;
}
message PGetFileCacheMetaResponse {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
