romseygeek commented on code in PR #16669:
URL: https://github.com/apache/lucene/pull/16669#discussion_r4080709493
##########
lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.java:
##########
@@ -18,12 +18,15 @@
import java.io.IOException;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
final class ReqExclBulkScorer extends BulkScorer {
+ private static final int WINDOW_SIZE = 4096;
private final BulkScorer req;
private final DocIdSetIterator exclApproximation;
private final TwoPhaseIterator exclTwoPhase;
+ private WindowBits windowBits;
Review Comment:
Can we make this final? I don't think it needs to be lazily instantiated.
##########
lucene/core/src/test/org/apache/lucene/search/TestReqExclBulkScorer.java:
##########
@@ -134,7 +142,153 @@ public void collect(int doc) throws IOException {
FixedBitSet excludedSet = new FixedBitSet(maxDoc);
excludedSet.or(excl.iterator());
expectedMatches.andNot(excludedSet);
+ if (acceptDocs != null) {
+ expectedMatches.and(acceptedDocs);
+ }
assertArrayEquals(expectedMatches.getBits(), actualMatches.getBits());
}
+
+ public void testDenseScore() throws IOException {
+ int maxDoc = 10_000;
+ FixedBitSet required = new FixedBitSet(maxDoc);
+ FixedBitSet excluded = new FixedBitSet(maxDoc);
+ FixedBitSet accepted = null;
+ FixedBitSet expected = new FixedBitSet(maxDoc);
+
+ boolean hasDelete = random().nextBoolean();
+ if (hasDelete) {
+ accepted = new FixedBitSet(maxDoc);
+ }
+
+ for (int doc = 0; doc < maxDoc; doc++) {
+ if (doc % 3 == 1) {
+ required.set(doc);
+ }
+ if ((doc & 1) == 0) {
+ excluded.set(doc);
+ }
+
+ if (required.get(doc) && excluded.get(doc) == false) {
+ expected.set(doc);
+ }
+
+ if (hasDelete) {
+ if (doc % 5 != 0) {
+ accepted.set(doc);
+ } else {
+ expected.clear(doc);
+ }
+ }
+ }
+ Bits acceptDocs = hasDelete ? accepted.asReadOnlyBits() : null;
+
+ BitSetIterator exclusion = new BitSetIterator(excluded,
excluded.cardinality());
+ ReqExclBulkScorer scorer =
+ new ReqExclBulkScorer(
+ new ConstantScoreBulkScorer(
+ 0f,
+ ScoreMode.COMPLETE_NO_SCORES,
+ new BitSetIterator(required, required.cardinality())),
+ exclusion);
+ FixedBitSet actual = new FixedBitSet(maxDoc);
+ assertEquals(
+ DocIdSetIterator.NO_MORE_DOCS,
+ scorer.score(
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ actual.set(doc);
+ }
+ },
+ acceptDocs,
+ 0,
+ maxDoc));
+
+ assertArrayEquals(expected.getBits(), actual.getBits());
+ }
+
+ public void testSparseScore() throws IOException {
+ int maxDoc = 10_000;
+ FixedBitSet excluded = new FixedBitSet(maxDoc);
+ FixedBitSet accepted = null;
+ FixedBitSet expected = new FixedBitSet(maxDoc);
+ expected.set(0, maxDoc);
+
+ excluded.set(42);
+ excluded.set(600);
+
+ expected.clear(42);
+ expected.clear(600);
+
+ boolean hasDelete = random().nextBoolean();
+ if (hasDelete) {
+ accepted = new FixedBitSet(maxDoc);
+ accepted.set(0, maxDoc);
+ accepted.clear(42);
+ accepted.clear(601);
+ expected.clear(42);
+ expected.clear(600);
+ }
+
+ BitSetIterator exclusion = new BitSetIterator(excluded,
excluded.cardinality());
+ ReqExclBulkScorer scorer =
+ new ReqExclBulkScorer(
+ new ConstantScoreBulkScorer(
+ 0f, ScoreMode.COMPLETE_NO_SCORES,
DocIdSetIterator.all(maxDoc)),
+ exclusion);
+ FixedBitSet collected = new FixedBitSet(maxDoc);
+ assertEquals(
+ DocIdSetIterator.NO_MORE_DOCS,
+ scorer.score(
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ collected.set(doc);
+ }
+ },
+ null,
Review Comment:
This needs to be an acceptDocs generated from `accept.asReadOnlyBits()` I
think?
##########
lucene/CHANGES.txt:
##########
@@ -524,6 +524,8 @@ Optimizations
* GITHUB#16520: Support two-phase filters in MaxScoreBulkScorer bitset path.
(Ke Wei)
+* GITHUB#1669: Use intoBitSet to speed up ReqExclBulkScorer with dense
exclusions. (Ke Wei)
Review Comment:
```suggestion
* GITHUB#16669: Use intoBitSet to speed up ReqExclBulkScorer with dense
exclusions. (Ke Wei)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
