dataroaring commented on code in PR #56361:
URL: https://github.com/apache/doris/pull/56361#discussion_r2428541528
##########
be/src/vec/sink/vtablet_finder.cpp:
##########
@@ -88,10 +91,46 @@ Status OlapTabletFinder::find_tablets(RuntimeState* state,
Block* block, int row
_vpartition->find_tablets(block, qualified_rows, partitions,
tablet_index,
&_partition_to_tablet_map);
if (_find_tablet_mode == FindTabletMode::FIND_TABLET_EVERY_BATCH) {
+ // Count rows per partition for accurate tablet switching
+ std::unordered_map<VOlapTablePartition*, int64_t>
partition_row_counts;
+ for (size_t i = 0; i < qualified_rows.size(); ++i) {
+ auto row_idx = qualified_rows[i];
+ auto* partition = partitions[row_idx];
+ if (partition != nullptr) {
+ partition_row_counts[partition]++;
+ }
+ }
+
+ // Update local state for each partition involved in this batch
for (auto it : _partition_to_tablet_map) {
- // do round-robin for next batch
- if (it.first->load_tablet_idx != -1) {
- it.first->load_tablet_idx++;
+ auto* partition = it.first;
+ if (partition->load_tablet_idx == -1) continue;
+
+ // Use actual row count for this partition
+ int64_t partition_rows = partition_row_counts[partition];
+ partition->current_tablet_rows += partition_rows;
+
+ LOG(INFO) << "Tablet switching: partition_id=" << partition->id
+ << ", current_tablet_rows=" <<
partition->current_tablet_rows
+ << ", switching_threshold=" <<
partition->switching_threshold
+ << ", load_tablet_idx=" << partition->load_tablet_idx
+ << ", num_buckets=" << partition->num_buckets
+ << ", partition_rows_this_batch=" << partition_rows;
+
+ // Check if we need to switch to next tablet based on threshold
+ if (partition->switching_threshold > 0 &&
+ partition->current_tablet_rows >=
partition->switching_threshold) {
+ // Switch to next tablet in round-robin fashion
+ int64_t old_tablet_idx = partition->load_tablet_idx;
+ partition->load_tablet_idx =
+ (partition->load_tablet_idx + 1) %
partition->num_buckets;
+ partition->current_tablet_rows = 0;
+ LOG(INFO) << "Tablet switched: partition_id=" <<
partition->id
+ << ", from tablet_idx=" << old_tablet_idx
+ << " to tablet_idx=" <<
partition->load_tablet_idx;
+ } else if (partition->switching_threshold == 0) {
+ // Legacy behavior: increment tablet index for each batch
+ partition->load_tablet_idx++;
Review Comment:
We'd better put these logic to vpartition->find_tablets, because it has
found partition.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
