csun5285 commented on code in PR #31488:
URL: https://github.com/apache/doris/pull/31488#discussion_r1507196225
##########
be/src/olap/cumulative_compaction_time_series_policy.cpp:
##########
@@ -47,8 +50,13 @@ uint32_t
TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(
continue;
} else {
// collect the rowsets of cumulative part
- total_size += rs_meta->total_disk_size();
score += rs_meta->get_compaction_score();
+ if (rs_meta->compaction_level() == 0) {
+ level0_total_size += rs_meta->total_disk_size();
+ level0_score += rs_meta->get_compaction_score();
+ } else {
+ checked_rs_metas.push_back(rs_meta);
Review Comment:
这个函数只是计算下分数，可以在这直接计算 level1 是否满足条件
##########
be/src/olap/compaction.cpp:
##########
@@ -133,10 +133,13 @@ void Compaction::init_profile(const std::string& label) {
Status Compaction::merge_input_rowsets() {
std::vector<RowsetReaderSharedPtr> input_rs_readers;
input_rs_readers.reserve(_input_rowsets.size());
+ int64_t max_compaction_level = 0;
for (auto& rowset : _input_rowsets) {
RowsetReaderSharedPtr rs_reader;
RETURN_IF_ERROR(rowset->create_reader(&rs_reader));
input_rs_readers.push_back(std::move(rs_reader));
+ max_compaction_level =
Review Comment:
每次 compaction 执行时，应该同一层的合并，还是level0 和 level1 的可以混合？
##########
be/src/olap/cumulative_compaction_time_series_policy.cpp:
##########
@@ -193,6 +223,41 @@ int
TimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
return 0;
}
+ int64_t compaction_goal_size_mbytes =
+ tablet->tablet_meta()->time_series_compaction_goal_size_mbytes();
+
+ std::vector<RowsetSharedPtr> level1_rowsets;
+ if (tablet->tablet_meta()->time_series_compaction_level_threshold() == 2) {
+ int64_t continuous_size = 0;
+ int64_t cumulative_point = tablet->cumulative_layer_point();
+ for (const auto& rowset : candidate_rowsets) {
+ const auto& rs_meta = rowset->rowset_meta();
+ if (rs_meta->end_version() < cumulative_point) {
Review Comment:
可以删掉，传进来的candidate_rowsets 的版本一定是大于等于cumulative_point的
##########
be/src/olap/cumulative_compaction_time_series_policy.cpp:
##########
@@ -193,6 +223,41 @@ int
TimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
return 0;
}
+ int64_t compaction_goal_size_mbytes =
+ tablet->tablet_meta()->time_series_compaction_goal_size_mbytes();
+
+ std::vector<RowsetSharedPtr> level1_rowsets;
+ if (tablet->tablet_meta()->time_series_compaction_level_threshold() == 2) {
+ int64_t continuous_size = 0;
+ int64_t cumulative_point = tablet->cumulative_layer_point();
+ for (const auto& rowset : candidate_rowsets) {
+ const auto& rs_meta = rowset->rowset_meta();
+ if (rs_meta->end_version() < cumulative_point) {
+ continue;
+ }
+ if (rs_meta->compaction_level() == 0) {
+ break;
+ }
+ int64_t rs_meta_size = rs_meta->total_disk_size();
+ if (rs_meta_size >= compaction_goal_size_mbytes * 2 * 1024 * 1024)
{
Review Comment:
为什么要乘以2
##########
be/src/olap/compaction.cpp:
##########
@@ -133,10 +133,13 @@ void Compaction::init_profile(const std::string& label) {
Status Compaction::merge_input_rowsets() {
std::vector<RowsetReaderSharedPtr> input_rs_readers;
input_rs_readers.reserve(_input_rowsets.size());
+ int64_t max_compaction_level = 0;
for (auto& rowset : _input_rowsets) {
RowsetReaderSharedPtr rs_reader;
RETURN_IF_ERROR(rowset->create_reader(&rs_reader));
input_rs_readers.push_back(std::move(rs_reader));
+ max_compaction_level =
+ std::max(max_compaction_level,
rowset->rowset_meta()->compaction_level());
}
Review Comment:
如果所有的输入版本都为空，不需要提升 compation_level
##########
be/src/olap/cumulative_compaction_time_series_policy.cpp:
##########
@@ -70,15 +78,37 @@ uint32_t
TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(
return score;
}
- // Condition 1: the size of input files for compaction meets the
requirement of parameter compaction_goal_size
int64_t compaction_goal_size_mbytes =
tablet->tablet_meta()->time_series_compaction_goal_size_mbytes();
- if (total_size >= compaction_goal_size_mbytes * 1024 * 1024) {
+ if (tablet->tablet_meta()->time_series_compaction_level_threshold() == 2) {
+ int32_t rs_meta_count = 0;
+ int64_t continuous_size = 0;
+ for (const auto& rs_meta : checked_rs_metas) {
Review Comment:
上面 rs_metas 是无序的，checked_rs_metas 里面也是无序的
##########
be/src/olap/compaction.cpp:
##########
@@ -133,10 +133,13 @@ void Compaction::init_profile(const std::string& label) {
Status Compaction::merge_input_rowsets() {
std::vector<RowsetReaderSharedPtr> input_rs_readers;
input_rs_readers.reserve(_input_rowsets.size());
+ int64_t max_compaction_level = 0;
for (auto& rowset : _input_rowsets) {
RowsetReaderSharedPtr rs_reader;
RETURN_IF_ERROR(rowset->create_reader(&rs_reader));
input_rs_readers.push_back(std::move(rs_reader));
+ max_compaction_level =
Review Comment:
如果是只能同一层的合并，这里校验下level是否相等？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
