This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 5c2991d958a branch-4.2: [improvement](be) Cut per-document work in
SNII docid decode #69909 (#69955)
5c2991d958a is described below
commit 5c2991d958a60f942186933a87eb7bdc986c98d1
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Nov 3 09:57:15 2026 +0800
branch-4.2: [improvement](be) Cut per-document work in SNII docid decode
#69909 (#69955)
Cherry-picked from #69909
Co-authored-by: Jack <[email protected]>
---
.../storage/index/inverted/snii/encoding/pfor.cpp | 12 ++---
.../storage/index/inverted/snii/format/frq_pod.cpp | 37 ++++++++++-----
.../inverted/snii/query/docid_posting_reader.cpp | 2 +-
.../index/inverted/snii/format/frq_pod_test.cpp | 55 ++++++++++++++++++++++
4 files changed, 86 insertions(+), 20 deletions(-)
diff --git a/be/src/storage/index/inverted/snii/encoding/pfor.cpp
b/be/src/storage/index/inverted/snii/encoding/pfor.cpp
index f8ad4f15959..2a05fdcddbf 100644
--- a/be/src/storage/index/inverted/snii/encoding/pfor.cpp
+++ b/be/src/storage/index/inverted/snii/encoding/pfor.cpp
@@ -417,15 +417,15 @@ Status pfor_decode(ByteSource* src, size_t n, uint32_t*
out) {
"pfor bit width {} exceeds 32", w);
}
uint32_t n_exc;
- RETURN_IF_ERROR(src->get_varint32(&n_exc));
+ RETURN_IF_ERROR(src->get_varint32_fast(&n_exc));
RETURN_IF_ERROR(bitunpack(src, n, w, out));
// 64 bits wide: a delta from the file is at most 2^32 - 1 and idx is
below n before every add, so the
// sum cannot wrap back into [0, n) the way a 32-bit one can.
uint64_t idx = 0;
for (uint32_t i = 0; i < n_exc; ++i) {
uint32_t d, val;
- RETURN_IF_ERROR(src->get_varint32(&d));
- RETURN_IF_ERROR(src->get_varint32(&val));
+ RETURN_IF_ERROR(src->get_varint32_fast(&d));
+ RETURN_IF_ERROR(src->get_varint32_fast(&val));
idx += d;
if (idx >= n) {
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED,
false>(
@@ -440,7 +440,7 @@ Status pfor_skip(ByteSource* src, size_t n) {
uint8_t w = 0;
RETURN_IF_ERROR(src->get_u8(&w));
uint32_t n_exc = 0;
- RETURN_IF_ERROR(src->get_varint32(&n_exc));
+ RETURN_IF_ERROR(src->get_varint32_fast(&n_exc));
const size_t packed = (static_cast<size_t>(w) * n + 7) / 8;
Slice unused;
RETURN_IF_ERROR(src->get_bytes(packed, &unused));
@@ -448,8 +448,8 @@ Status pfor_skip(ByteSource* src, size_t n) {
for (uint32_t i = 0; i < n_exc; ++i) {
uint32_t d = 0;
uint32_t val = 0;
- RETURN_IF_ERROR(src->get_varint32(&d));
- RETURN_IF_ERROR(src->get_varint32(&val));
+ RETURN_IF_ERROR(src->get_varint32_fast(&d));
+ RETURN_IF_ERROR(src->get_varint32_fast(&val));
idx += d;
if (idx >= n) {
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED,
false>(
diff --git a/be/src/storage/index/inverted/snii/format/frq_pod.cpp
b/be/src/storage/index/inverted/snii/format/frq_pod.cpp
index 2685cf01c86..3be6087f259 100644
--- a/be/src/storage/index/inverted/snii/format/frq_pod.cpp
+++ b/be/src/storage/index/inverted/snii/format/frq_pod.cpp
@@ -358,23 +358,34 @@ Status decode_dd_region_impl(Slice dd_disk, const
FrqRegionMeta& meta, uint64_t
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED, false>(
"frq: trailing bytes after dd region payload");
}
- uint64_t cur = win_base;
- if (n != 0 && cur > std::numeric_limits<uint32_t>::max()) {
+ if (n == 0) {
+ return Status::OK();
+ }
+ if (win_base > std::numeric_limits<uint32_t>::max()) {
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED, false>(
"frq: window base exceeds uint32 docid range");
}
+ // A window is checked whole, so the sum below takes no branch per
document: only the first
+ // delta may be zero, and the last docid must fit 32 bits.
+ uint32_t* const docs = docids->data();
+ uint64_t last = win_base + docs[0];
+ uint32_t smallest_delta = std::numeric_limits<uint32_t>::max();
+ for (uint32_t i = 1; i < n; ++i) {
+ last += docs[i];
+ smallest_delta = std::min(smallest_delta, docs[i]);
+ }
+ if (smallest_delta == 0) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED, false>(
+ "frq: zero docid delta");
+ }
+ if (last > std::numeric_limits<uint32_t>::max()) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED, false>(
+ "frq: docid accumulation overflow");
+ }
+ auto docid = static_cast<uint32_t>(win_base);
for (uint32_t i = 0; i < n; ++i) {
- const uint32_t delta = (*docids)[i];
- if (i != 0 && delta == 0) {
- return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED,
false>(
- "frq: zero docid delta");
- }
- if (delta > std::numeric_limits<uint32_t>::max() - cur) {
- return Status::Error<ErrorCode::INVERTED_INDEX_FILE_CORRUPTED,
false>(
- "frq: docid accumulation overflow");
- }
- cur += delta;
- (*docids)[i] = static_cast<uint32_t>(cur);
+ docid += docs[i];
+ docs[i] = docid;
}
return Status::OK();
}
diff --git a/be/src/storage/index/inverted/snii/query/docid_posting_reader.cpp
b/be/src/storage/index/inverted/snii/query/docid_posting_reader.cpp
index 5193d2aa712..3add9c2d607 100644
--- a/be/src/storage/index/inverted/snii/query/docid_posting_reader.cpp
+++ b/be/src/storage/index/inverted/snii/query/docid_posting_reader.cpp
@@ -172,7 +172,7 @@ Status decode_window_prefix_plan(const
io::BatchRangeFetcher& fetcher, const Win
RETURN_IF_ERROR(sink->append_range(first,
static_cast<uint64_t>(meta.last_docid) + 1));
continue;
}
- docs.clear();
+ // docs is decoded over, not cleared: a resize zero-fills only what it
adds.
positions.clear();
RETURN_IF_ERROR(reader::decode_window_slices(meta, dd_region, Slice(),
/*want_positions=*/false,
&docs, &positions));
diff --git a/be/test/storage/index/inverted/snii/format/frq_pod_test.cpp
b/be/test/storage/index/inverted/snii/format/frq_pod_test.cpp
index 961abad1958..1626c99a671 100644
--- a/be/test/storage/index/inverted/snii/format/frq_pod_test.cpp
+++ b/be/test/storage/index/inverted/snii/format/frq_pod_test.cpp
@@ -22,6 +22,7 @@
#include <algorithm>
#include <cstdint>
#include <limits>
+#include <string>
#include <utility>
#include <vector>
@@ -207,6 +208,60 @@ TEST(SniiFrqPod,
DecodeRejectsZeroInteriorDeltaAndDocidOverflow) {
EXPECT_TRUE(overflow.is<doris::ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>()) <<
overflow;
}
+// A window is checked whole before its deltas are summed. Only the first
delta may be zero, a zero
+// anywhere else in any PFOR run is found, and the sum may reach the last
docid but not pass it.
+TEST(SniiFrqPod, DecodeChecksEveryDeltaOfTheWindow) {
+ constexpr uint32_t kLastDocid = std::numeric_limits<uint32_t>::max();
+ constexpr size_t kRun = doris::snii::format::kFrqBaseUnit;
+ U32Vec docids;
+ auto decode = [&docids](const U32Vec& deltas, uint64_t win_base) {
+ auto [bytes, meta] = MakeRawDdFromDeltas(deltas);
+ return decode_dd_region(Slice(bytes), meta, win_base, &docids);
+ };
+ auto expect_corrupt = [](const Status& status, const std::string& message)
{
+
EXPECT_TRUE(status.is<doris::ErrorCode::INVERTED_INDEX_FILE_CORRUPTED>()) <<
status;
+ EXPECT_NE(status.to_string().find(message), std::string::npos) <<
status;
+ };
+
+ U32Vec deltas(3 * kRun, 3);
+ deltas[0] = 0;
+ ASSERT_TRUE(decode(deltas, /*win_base=*/7).ok());
+ EXPECT_EQ(docids.front(), 7U);
+ EXPECT_EQ(docids[1], 10U);
+ EXPECT_EQ(docids.back(), 7U + 3U * (3 * kRun - 1));
+ for (const size_t at : {size_t {1}, kRun - 1, kRun, 3 * kRun - 1}) {
+ U32Vec with_zero = deltas;
+ with_zero[at] = 0;
+ expect_corrupt(decode(with_zero, 7), "zero docid delta");
+ }
+
+ ASSERT_TRUE(decode({1, kLastDocid - 2}, /*win_base=*/1).ok());
+ EXPECT_EQ(docids, (U32Vec {2, kLastDocid}));
+ ASSERT_TRUE(decode({0}, kLastDocid).ok());
+ EXPECT_EQ(docids, (U32Vec {kLastDocid}));
+ expect_corrupt(decode({1, kLastDocid - 1}, 1), "docid accumulation
overflow");
+ // These add up to 2^32, which is 0 in 32 bits.
+ expect_corrupt(decode(U32Vec(16, 1U << 28), 0), "docid accumulation
overflow");
+ expect_corrupt(decode({0}, static_cast<uint64_t>(kLastDocid) + 1),
+ "window base exceeds uint32 docid range");
+}
+
+// The docid reader decodes each window over the vector of the one before.
+TEST(SniiFrqPod, DecodeOverAVectorOfAnotherSize) {
+ U32Vec large(3000);
+ for (uint32_t i = 0; i < large.size(); ++i) {
+ large[i] = i * 5;
+ }
+ const U32Vec small = {5, 9, 12};
+ U32Vec out;
+ ASSERT_TRUE(RoundTrip(large, 0, 0, &out).ok());
+ EXPECT_EQ(out, large);
+ ASSERT_TRUE(RoundTrip(small, 0, 0, &out).ok());
+ EXPECT_EQ(out, small);
+ ASSERT_TRUE(RoundTrip(large, 0, 0, &out).ok());
+ EXPECT_EQ(out, large);
+}
+
// first_docid < win_base (dd[0] would underflow) -> InvalidArgument.
TEST(SniiFrqPod, FirstDocBelowWinBaseRejected) {
U32Vec docs = {100, 200};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
