guptashivam04 commented on code in PR #69438:
URL: https://github.com/apache/doris/pull/69438#discussion_r4367231524
##########
be/src/storage/index/inverted/reader/collection_statistics.cpp:
##########
@@ -89,13 +114,171 @@ void add_term_doc_frequency(
(*logical_frequencies)[field][logical_term] += doc_frequency;
}
+Status sum_non_null_norm_lengths(index_query::IndexSource& source,
+ std::span<const uint32_t> null_docs,
uint64_t* out) {
+ if (!null_docs.empty() && null_docs.back() >= source.doc_count()) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>(
+ "NULL row {} is beyond the {} documents of the index",
null_docs.back(),
+ source.doc_count());
+ }
+ uint64_t all_lengths = 0;
+ RETURN_IF_ERROR(source.sum_norm_lengths(&all_lengths));
+ uint64_t null_lengths = 0;
+ RETURN_IF_ERROR(source.sum_norm_lengths_of(null_docs, &null_lengths));
+ DORIS_CHECK(null_lengths <= all_lengths);
+ *out = all_lengths - null_lengths;
+ return Status::OK();
+}
+
+Status exclude_hidden_docs(index_query::IndexSource& source, std::span<const
uint32_t> hidden_docs,
+ const roaring::Roaring& null_docs, uint64_t
non_null_length_sum,
+ index_query::IndexScoringStats* stats) {
+ return exclude_hidden_docs(
+ hidden_docs, null_docs, non_null_length_sum,
+ [&source](std::span<const uint32_t> docs, uint64_t* out) {
+ return source.sum_norm_lengths_of(docs, out);
+ },
+ stats);
+}
+
+Status exclude_hidden_docs(std::span<const uint32_t> hidden_docs, const
roaring::Roaring& null_docs,
+ uint64_t non_null_length_sum,
+ const SumNormLengthsOf& sum_norm_lengths_of,
+ index_query::IndexScoringStats* stats) {
+ DORIS_CHECK(stats != nullptr);
+ if (!hidden_docs.empty() && hidden_docs.back() >= stats->doc_count) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>(
+ "hidden row {} is beyond the {} documents of the index",
hidden_docs.back(),
+ stats->doc_count);
+ }
+ stats->doc_count -= hidden_docs.size();
+ if (!stats->has_norms) {
+ // Without norms every document stands for one token.
+ stats->token_count = stats->doc_count;
+ return Status::OK();
+ }
+ // The exact token count of the hidden rows is not stored anywhere, and
norms hold lengths
+ // only approximately, so count the lengths the norms of the returned rows
stand for. A
+ // replica that has compacted the hidden rows away counts the same rows
with the same norms.
+ std::vector<uint32_t> hidden_non_null_docs;
+ hidden_non_null_docs.reserve(hidden_docs.size());
Review Comment:
Fixed in da1c9d25f0: the hidden and NULL row-ID lists are now `DorisVector`s
(charged to the query's memory through the Doris allocator), the SNII reader's
NULL-ID vector is charged with `CONSUME_THREAD_MEM_TRACKER` while it lives, and
a segment without NULL rows sums its hidden rows in place instead of copying
them.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
