yixiutt commented on code in PR #11542:
URL: https://github.com/apache/doris/pull/11542#discussion_r938408836
##########
be/src/olap/base_compaction.cpp:
##########
@@ -81,16 +84,61 @@ Status BaseCompaction::execute_compact_impl() {
return Status::OK();
}
+bool BaseCompaction::_filter_input_rowset() {
+ bool exceed = false;
+ // if enable dup key skip big file and no delete predicate
+ // we skip big files too save resources
+ if (!config::enable_dup_key_base_compaction_skip_big_file ||
+ _tablet->keys_type() != KeysType::DUP_KEYS ||
_tablet->delete_predicates().size() != 0) {
+ return false;
+ }
+ int32_t total_rowsets = 0;
+ int64_t total_size = 0;
+ int64_t max_size = config::base_compaction_dup_key_max_file_size_mbytes;
+ // first find a proper rowset for start
+ auto rs_iter = _input_rowsets.begin();
+ while (rs_iter != _input_rowsets.end()) {
+ if ((*rs_iter)->rowset_meta()->total_disk_size() >= max_size) {
+ rs_iter = _input_rowsets.erase(rs_iter);
+ ;
+ } else {
+ break;
+ }
+ }
+ // find input rowset
+ while (rs_iter != _input_rowsets.end()) {
+ ++total_rowsets;
+ total_size += (*rs_iter)->rowset_meta()->total_disk_size();
+ ++rs_iter;
+ if (total_rowsets <= 1) {
+ continue;
+ }
+ if (total_size > max_size) {
+ exceed = true;
+ break;
+ }
+ }
+ // remove rowset left
+ while (rs_iter != _input_rowsets.end()) {
+ rs_iter = _input_rowsets.erase(rs_iter);
+ }
Review Comment:
fixed, we will compact four 512M rowset
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
