This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 4c9bafadbe6 [improvement](cooldown) skip cooling down tablets without
cold data #29690 (#29751)
4c9bafadbe6 is described below
commit 4c9bafadbe646dba29c66f6a4662ef8d4ed87726
Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com>
AuthorDate: Wed Jan 10 23:29:03 2024 +0800
[improvement](cooldown) skip cooling down tablets without cold data #29690
(#29751)
---
be/src/olap/tablet.cpp | 53 +++++++++++++++++++++++++++--------
be/src/olap/tablet.h | 2 ++
be/test/olap/tablet_cooldown_test.cpp | 1 +
3 files changed, 45 insertions(+), 11 deletions(-)
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index b2c4302c929..61fe618f032 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2365,8 +2365,33 @@ Status Tablet::_follow_cooldowned_data() {
return Status::OK();
}
+bool Tablet::_has_data_to_cooldown() {
+ int64_t min_local_version = std::numeric_limits<int64_t>::max();
+ RowsetSharedPtr rowset;
+ std::shared_lock meta_rlock(_meta_lock);
+ for (auto& [v, rs] : _rs_version_map) {
+ if (rs->is_local() && v.first < min_local_version &&
rs->data_disk_size() > 0) {
+ // this is a local rowset and has data
+ min_local_version = v.first;
+ rowset = rs;
+ }
+ }
+
+ int64_t newest_cooldown_time = 0;
+ if (rowset != nullptr) {
+ newest_cooldown_time = _get_newest_cooldown_time(rowset);
+ }
+
+ return (newest_cooldown_time != 0) && (newest_cooldown_time <
UnixSeconds());
+}
+
RowsetSharedPtr Tablet::pick_cooldown_rowset() {
RowsetSharedPtr rowset;
+
+ if (!_has_data_to_cooldown()) {
+ return nullptr;
+ }
+
// TODO(plat1ko): should we maintain `cooldowned_version` in `Tablet`?
int64_t cooldowned_version = -1;
// We pick the rowset with smallest start version in local.
@@ -2400,26 +2425,21 @@ RowsetSharedPtr Tablet::pick_cooldown_rowset() {
return rowset;
}
-bool Tablet::need_cooldown(int64_t* cooldown_timestamp, size_t* file_size) {
+int64_t Tablet::_get_newest_cooldown_time(const RowsetSharedPtr& rowset) {
int64_t id = storage_policy_id();
if (id <= 0) {
VLOG_DEBUG << "tablet does not need cooldown, tablet id: " <<
tablet_id();
- return false;
+ return 0;
}
auto storage_policy = get_storage_policy(id);
if (!storage_policy) {
LOG(WARNING) << "Cannot get storage policy: " << id;
- return false;
+ return 0;
}
auto cooldown_ttl_sec = storage_policy->cooldown_ttl;
auto cooldown_datetime = storage_policy->cooldown_datetime;
- RowsetSharedPtr rowset = pick_cooldown_rowset();
- if (!rowset) {
- VLOG_DEBUG << "pick cooldown rowset, get null, tablet id: " <<
tablet_id();
- return false;
- }
-
int64_t newest_cooldown_time = std::numeric_limits<int64_t>::max();
+
if (cooldown_ttl_sec >= 0) {
newest_cooldown_time = rowset->newest_write_timestamp() +
cooldown_ttl_sec;
}
@@ -2427,9 +2447,21 @@ bool Tablet::need_cooldown(int64_t* cooldown_timestamp,
size_t* file_size) {
newest_cooldown_time = std::min(newest_cooldown_time,
cooldown_datetime);
}
+ return newest_cooldown_time;
+}
+
+bool Tablet::need_cooldown(int64_t* cooldown_timestamp, size_t* file_size) {
+ RowsetSharedPtr rowset = pick_cooldown_rowset();
+ if (!rowset) {
+ VLOG_DEBUG << "pick cooldown rowset, get null, tablet id: " <<
tablet_id();
+ return false;
+ }
+
+ auto newest_cooldown_time = _get_newest_cooldown_time(rowset);
+
// the rowset should do cooldown job only if it's cooldown ttl plus newest
write time is less than
// current time or it's datatime is less than current time
- if (newest_cooldown_time < UnixSeconds()) {
+ if (newest_cooldown_time != 0 && newest_cooldown_time < UnixSeconds()) {
*cooldown_timestamp = newest_cooldown_time;
*file_size = rowset->data_disk_size();
VLOG_DEBUG << "tablet need cooldown, tablet id: " << tablet_id()
@@ -2438,7 +2470,6 @@ bool Tablet::need_cooldown(int64_t* cooldown_timestamp,
size_t* file_size) {
}
VLOG_DEBUG << "tablet does not need cooldown, tablet id: " << tablet_id()
- << " ttl sec: " << cooldown_ttl_sec << " cooldown datetime: "
<< cooldown_datetime
<< " newest write time: " << rowset->newest_write_timestamp();
return false;
}
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 626f11031dc..9ed7e5937b2 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -612,6 +612,8 @@ private:
Status _follow_cooldowned_data();
Status _read_cooldown_meta(const std::shared_ptr<io::RemoteFileSystem>& fs,
TabletMetaPB* tablet_meta_pb);
+ bool _has_data_to_cooldown();
+ int64_t _get_newest_cooldown_time(const RowsetSharedPtr& rowset);
////////////////////////////////////////////////////////////////////////////
// end cooldown functions
////////////////////////////////////////////////////////////////////////////
diff --git a/be/test/olap/tablet_cooldown_test.cpp
b/be/test/olap/tablet_cooldown_test.cpp
index d02586592c2..2387f5af3d9 100644
--- a/be/test/olap/tablet_cooldown_test.cpp
+++ b/be/test/olap/tablet_cooldown_test.cpp
@@ -239,6 +239,7 @@ public:
storage_policy->name = "TabletCooldownTest";
storage_policy->version = 1;
storage_policy->resource_id = kResourceId;
+ storage_policy->cooldown_datetime = UnixSeconds() - 1;
put_storage_policy(kStoragePolicyId, storage_policy);
constexpr uint32_t MAX_PATH_LEN = 1024;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
