gavinchou commented on code in PR #42066:
URL: https://github.com/apache/doris/pull/42066#discussion_r1814896216
##########
be/src/cloud/cloud_tablet_mgr.cpp:
##########
@@ -181,8 +181,14 @@ Result<std::shared_ptr<CloudTablet>>
CloudTabletMgr::get_tablet(int64_t tablet_i
auto* handle = _cache->insert(key, value.release(), 1,
sizeof(CloudTablet),
CachePriority::NORMAL);
- auto ret = std::shared_ptr<CloudTablet>(
- tablet.get(), [this, handle](...) {
_cache->release(handle); });
+ auto ret =
+ std::shared_ptr<CloudTablet>(tablet.get(), [this,
handle](CloudTablet* tablet) {
+ int64_t now = duration_cast<std::chrono::milliseconds>(
+
std::chrono::system_clock::now().time_since_epoch())
+ .count();
+ tablet->last_cache_release_ms = now;
Review Comment:
both before and after release to refresh
last_access_time_ms
##########
be/src/cloud/cloud_tablet.h:
##########
@@ -196,10 +196,13 @@ class CloudTablet final : public BaseTablet {
int64_t last_base_compaction_success_time_ms = 0;
int64_t last_cumu_compaction_success_time_ms = 0;
int64_t last_cumu_no_suitable_version_ms = 0;
+ int64_t last_cache_release_ms = 0;
Review Comment:
last_access_time_ms
##########
be/src/cloud/cloud_tablet_mgr.h:
##########
@@ -65,6 +68,11 @@ class CloudTabletMgr {
std::vector<std::shared_ptr<CloudTablet>>* tablets,
int64_t* max_score);
+ void build_all_report_tablets_info(std::map<TTabletId, TTablet>*
tablets_info,
+ uint64_t* tablet_num);
+
+ void obtain_specific_quantity_tablets(std::vector<TabletInfo>&
tablets_info, int64_t num);
Review Comment:
get_tablet_info(int64_t num_tablets, std::vector<TabletInfo>* tablets_info)
##########
be/src/agent/task_worker_pool.cpp:
##########
@@ -1134,6 +1140,46 @@ void report_tablet_callback(StorageEngine& engine, const
TMasterInfo& master_inf
}
}
+void report_tablet_callback(CloudStorageEngine& engine, const TMasterInfo&
master_info) {
+ // Random sleep 1~5 seconds before doing report.
+ // In order to avoid the problem that the FE receives many report requests
at the same time
+ // and can not be processed.
+ if (config::report_random_wait) {
+ random_sleep(5);
+ }
+
+ TReportRequest request;
+ request.__set_backend(BackendOptions::get_local_backend());
+ request.__isset.tablets = true;
+
+ increase_report_version();
+ uint64_t report_version;
+ uint64_t tablet_num = 0;
Review Comment:
total_num_tablets
##########
be/src/agent/task_worker_pool.cpp:
##########
@@ -1134,6 +1140,46 @@ void report_tablet_callback(StorageEngine& engine, const
TMasterInfo& master_inf
}
}
+void report_tablet_callback(CloudStorageEngine& engine, const TMasterInfo&
master_info) {
+ // Random sleep 1~5 seconds before doing report.
+ // In order to avoid the problem that the FE receives many report requests
at the same time
+ // and can not be processed.
+ if (config::report_random_wait) {
+ random_sleep(5);
+ }
+
+ TReportRequest request;
+ request.__set_backend(BackendOptions::get_local_backend());
+ request.__isset.tablets = true;
+
+ increase_report_version();
+ uint64_t report_version;
+ uint64_t tablet_num = 0;
+ for (int i = 0; i < 5; i++) {
Review Comment:
why for 5 times?
##########
be/src/cloud/cloud_tablet_mgr.cpp:
##########
@@ -357,4 +368,57 @@ Status CloudTabletMgr::get_topn_tablets_to_compact(
return Status::OK();
}
+void CloudTabletMgr::build_all_report_tablets_info(std::map<TTabletId,
TTablet>* tablets_info,
+ uint64_t* tablet_num) {
+ DCHECK(tablets_info != nullptr);
+ VLOG_NOTICE << "begin to build all report cloud tablets info";
+
+ HistogramStat tablet_version_num_hist;
+
+ auto handler = [&](const std::weak_ptr<CloudTablet>& tablet_wk) {
+ auto tablet = tablet_wk.lock();
+ if (!tablet) return;
+ (*tablet_num)++;
+ TTabletInfo tablet_info;
+ tablet->build_tablet_report_info(&tablet_info);
+ int64_t now = duration_cast<std::chrono::milliseconds>(
+
std::chrono::system_clock::now().time_since_epoch())
+ .count();
+ if (now - config::cloud_tablet_report_exceed_time_limit * 1000 <
Review Comment:
tablet_report_inactive_duration_ms
##########
be/src/cloud/cloud_tablet_mgr.cpp:
##########
@@ -357,4 +368,57 @@ Status CloudTabletMgr::get_topn_tablets_to_compact(
return Status::OK();
}
+void CloudTabletMgr::build_all_report_tablets_info(std::map<TTabletId,
TTablet>* tablets_info,
+ uint64_t* tablet_num) {
+ DCHECK(tablets_info != nullptr);
+ VLOG_NOTICE << "begin to build all report cloud tablets info";
+
+ HistogramStat tablet_version_num_hist;
+
+ auto handler = [&](const std::weak_ptr<CloudTablet>& tablet_wk) {
+ auto tablet = tablet_wk.lock();
+ if (!tablet) return;
+ (*tablet_num)++;
+ TTabletInfo tablet_info;
+ tablet->build_tablet_report_info(&tablet_info);
+ int64_t now = duration_cast<std::chrono::milliseconds>(
+
std::chrono::system_clock::now().time_since_epoch())
+ .count();
+ if (now - config::cloud_tablet_report_exceed_time_limit * 1000 <
+ tablet->last_cache_release_ms) {
+ // the tablet is still being accessed and used in recently, so not
report it
+ return;
+ }
+ auto& t_tablet = (*tablets_info)[tablet->tablet_id()];
+ // On the cloud, a specific BE has only one tablet replica;
+ // there are no multiple replicas for a specific BE.
+ // This is only to reuse the non-cloud report protocol.
+ tablet_version_num_hist.add(tablet_info.total_version_count);
+ t_tablet.tablet_infos.emplace_back(std::move(tablet_info));
+ };
+
+ auto weak_tablets = get_weak_tablets();
+ std::for_each(weak_tablets.begin(), weak_tablets.end(), handler);
+
+ DorisMetrics::instance()->tablet_version_num_distribution->set_histogram(
+ tablet_version_num_hist);
+ LOG(INFO) << "success to build all cloud report tablets info.
all_tablet_count=" << *tablet_num
Review Comment:
all_tablet_count -> num_tablets
number inactive tablets
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
