airborne12 commented on code in PR #68758:
URL: https://github.com/apache/doris/pull/68758#discussion_r4282776843
##########
be/src/storage/index/inverted/query/engine/collect/top_k_collector.cpp:
##########
@@ -15,48 +15,46 @@
// specific language governing permissions and limitations
// under the License.
-#include "storage/index/inverted/query_v2/collect/top_k_collector.h"
-
-#include "storage/index/inverted/query_v2/collect/multi_segment_util.h"
+#include "storage/index/inverted/query/engine/collect/top_k_collector.h"
namespace doris::segment_v2::inverted_index::query_v2 {
void collect_multi_segment_top_k(const WeightPtr& weight, const
QueryExecutionContext& context,
const std::string& binding_key, size_t k,
const std::shared_ptr<roaring::Roaring>&
roaring,
const CollectionSimilarityPtr& similarity,
bool use_wand,
- const std::shared_ptr<const
roaring::Roaring>& delete_bitmap) {
- TopKCollector final_collector(k);
-
- for_each_index_segment(
- context, binding_key, [&](const QueryExecutionContext& seg_ctx,
uint32_t seg_base) {
- float initial_threshold = final_collector.threshold();
-
- TopKCollector seg_collector(k);
- float threshold = initial_threshold;
- auto callback = [&](uint32_t doc_id, float score) -> float {
- if (delete_bitmap != nullptr &&
delete_bitmap->contains(doc_id + seg_base)) {
- return threshold;
- }
- threshold = seg_collector.collect(doc_id, score);
- return threshold;
- };
-
- if (use_wand) {
- weight->for_each_pruning(seg_ctx, binding_key,
initial_threshold, callback);
- } else {
- auto scorer = weight->scorer(seg_ctx, binding_key);
- if (scorer) {
- Weight::for_each_pruning_scorer(scorer,
initial_threshold, callback);
- }
- }
-
- for (const auto& doc : seg_collector.into_sorted_vec()) {
- final_collector.collect(doc.doc_id + seg_base, doc.score);
+ const std::shared_ptr<const
roaring::Roaring>& delete_bitmap,
+ roaring::Roaring* null_rows) {
+ if (context.segment_num_rows == 0) {
+ return;
+ }
+ TopKCollector collector(k);
+ const float initial_threshold = collector.threshold();
+ float threshold = initial_threshold;
+ auto callback = [&](uint32_t doc_id, float score) -> float {
+ if (delete_bitmap != nullptr && delete_bitmap->contains(doc_id)) {
+ return threshold;
+ }
+ threshold = collector.collect(doc_id, score);
+ return threshold;
+ };
+ if (use_wand && null_rows == nullptr) {
Review Comment:
Fixed in 891428cdbad. An occur Boolean is two-valued and never reports
UNKNOWN rows, so `collect_multi_segment_top_k` now keeps the weight's own
pruning (`block_wand`) for it even when the caller asks for UNKNOWN rows.
`Weight::is_two_valued()` marks those weights. Weights that can report UNKNOWN
rows still rank with the scorer they read those rows from.
Test: `FunctionSearchNullTest.TopKOverATermThresholdKeepsWand` runs
`body:foo/bar` with `minimum_should_match=1`, scoring, `LIMIT 1` and WAND
enabled through SEARCH. It counts the block skips requested from the fake
cursors (`FakeIndexSource::shallow_seeks`). With the change reverted the count
is 0, because every row is scored; with it, block-max WAND skips the lagging
term's block. The rows returned are the same either way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
