jpountz commented on a change in pull request #541:
URL: https://github.com/apache/lucene/pull/541#discussion_r795686801
##########
File path: lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java
##########
@@ -24,89 +24,73 @@
import org.apache.lucene.util.DocBaseBitSetIterator;
import org.apache.lucene.util.FixedBitSet;
-class DocIdsWriter {
+final class DocIdsWriter {
- private DocIdsWriter() {}
+ private static final byte CONTINUOUS_IDS = (byte) -2;
+ private static final byte BITSET_IDS = (byte) -1;
+ private static final byte DELTA_FOR_UTIL = (byte) 32 + 16;
+ private static final byte BPV_24 = (byte) 32 + 24;
+ private static final byte BPV_32 = (byte) 32;
+ // These signs are legacy, should no longer be used in the writing side.
+ private static final byte LEGACY_DELTA_VINT = (byte) 0;
+ private static final byte LEGACY_BPV_24 = (byte) 24;
- static void writeDocIds(int[] docIds, int start, int count, DataOutput out)
throws IOException {
+ private final BKDForUtil forUtil;
+ private final int[] scratch;
+
+ DocIdsWriter(int maxPointsInLeaf) {
+ scratch = new int[maxPointsInLeaf];
+ forUtil = new BKDForUtil(maxPointsInLeaf);
+ }
+
+ void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws
IOException {
// docs can be sorted either when all docs in a block have the same value
// or when a segment is sorted
- boolean sorted = true;
boolean strictlySorted = true;
+ int min = docIds[0];
+ int max = docIds[0];
for (int i = 1; i < count; ++i) {
int last = docIds[start + i - 1];
int current = docIds[start + i];
- if (last > current) {
- sorted = strictlySorted = false;
- break;
- } else if (last == current) {
+ if (last >= current) {
strictlySorted = false;
}
+ min = Math.min(min, current);
+ max = Math.max(max, current);
}
- int min2max = docIds[start + count - 1] - docIds[start] + 1;
+ int min2max = max - min + 1;
if (strictlySorted) {
if (min2max == count) {
// continuous ids, typically happens when segment is sorted
- out.writeByte((byte) -2);
+ out.writeByte(CONTINUOUS_IDS);
out.writeVInt(docIds[start]);
return;
} else if (min2max <= (count << 4)) {
assert min2max > count : "min2max: " + min2max + ", count: " + count;
// Only trigger bitset optimization when max - min + 1 <= 16 * count
in order to avoid
// expanding too much storage.
// A field with lower cardinality will have higher probability to
trigger this optimization.
- out.writeByte((byte) -1);
+ out.writeByte(BITSET_IDS);
writeIdsAsBitSet(docIds, start, count, out);
return;
}
}
- if (sorted) {
- out.writeByte((byte) 0);
- int previous = 0;
- for (int i = 0; i < count; ++i) {
- int doc = docIds[start + i];
- out.writeVInt(doc - previous);
- previous = doc;
+
+ if (min2max <= 0xFFFFL) {
Review comment:
```suggestion
if (min2max <= 0xFFFF) {
```
##########
File path: lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java
##########
@@ -24,89 +24,73 @@
import org.apache.lucene.util.DocBaseBitSetIterator;
import org.apache.lucene.util.FixedBitSet;
-class DocIdsWriter {
+final class DocIdsWriter {
- private DocIdsWriter() {}
+ private static final byte CONTINUOUS_IDS = (byte) -2;
+ private static final byte BITSET_IDS = (byte) -1;
+ private static final byte DELTA_FOR_UTIL = (byte) 32 + 16;
+ private static final byte BPV_24 = (byte) 32 + 24;
+ private static final byte BPV_32 = (byte) 32;
+ // These signs are legacy, should no longer be used in the writing side.
+ private static final byte LEGACY_DELTA_VINT = (byte) 0;
+ private static final byte LEGACY_BPV_24 = (byte) 24;
- static void writeDocIds(int[] docIds, int start, int count, DataOutput out)
throws IOException {
+ private final BKDForUtil forUtil;
+ private final int[] scratch;
+
+ DocIdsWriter(int maxPointsInLeaf) {
+ scratch = new int[maxPointsInLeaf];
+ forUtil = new BKDForUtil(maxPointsInLeaf);
+ }
+
+ void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws
IOException {
// docs can be sorted either when all docs in a block have the same value
// or when a segment is sorted
- boolean sorted = true;
boolean strictlySorted = true;
+ int min = docIds[0];
+ int max = docIds[0];
for (int i = 1; i < count; ++i) {
int last = docIds[start + i - 1];
int current = docIds[start + i];
- if (last > current) {
- sorted = strictlySorted = false;
- break;
- } else if (last == current) {
+ if (last >= current) {
strictlySorted = false;
}
+ min = Math.min(min, current);
+ max = Math.max(max, current);
}
- int min2max = docIds[start + count - 1] - docIds[start] + 1;
+ int min2max = max - min + 1;
if (strictlySorted) {
if (min2max == count) {
// continuous ids, typically happens when segment is sorted
- out.writeByte((byte) -2);
+ out.writeByte(CONTINUOUS_IDS);
out.writeVInt(docIds[start]);
return;
} else if (min2max <= (count << 4)) {
assert min2max > count : "min2max: " + min2max + ", count: " + count;
// Only trigger bitset optimization when max - min + 1 <= 16 * count
in order to avoid
// expanding too much storage.
// A field with lower cardinality will have higher probability to
trigger this optimization.
- out.writeByte((byte) -1);
+ out.writeByte(BITSET_IDS);
writeIdsAsBitSet(docIds, start, count, out);
return;
}
}
- if (sorted) {
- out.writeByte((byte) 0);
- int previous = 0;
- for (int i = 0; i < count; ++i) {
- int doc = docIds[start + i];
- out.writeVInt(doc - previous);
- previous = doc;
+
+ if (min2max <= 0xFFFFL) {
+ out.writeByte(DELTA_FOR_UTIL);
+ for (int i = 0; i < count; i++) {
+ scratch[i] = docIds[start + i] - min;
}
+ out.writeVInt(min);
+ forUtil.encode16(count, scratch, out);
} else {
- long max = 0;
- for (int i = 0; i < count; ++i) {
- max |= Integer.toUnsignedLong(docIds[start + i]);
- }
- if (max <= 0xffffff) {
- out.writeByte((byte) 24);
- // write them the same way we are reading them.
- int i;
- for (i = 0; i < count - 7; i += 8) {
- int doc1 = docIds[start + i];
- int doc2 = docIds[start + i + 1];
- int doc3 = docIds[start + i + 2];
- int doc4 = docIds[start + i + 3];
- int doc5 = docIds[start + i + 4];
- int doc6 = docIds[start + i + 5];
- int doc7 = docIds[start + i + 6];
- int doc8 = docIds[start + i + 7];
- long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 |
((doc3 >>> 8) & 0xffffL);
- long l2 =
- (doc3 & 0xffL) << 56
- | (doc4 & 0xffffffL) << 32
- | (doc5 & 0xffffffL) << 8
- | ((doc6 >> 16) & 0xffL);
- long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8
& 0xffffffL);
- out.writeLong(l1);
- out.writeLong(l2);
- out.writeLong(l3);
- }
- for (; i < count; ++i) {
- out.writeShort((short) (docIds[start + i] >>> 8));
- out.writeByte((byte) docIds[start + i]);
- }
+ if (max <= 0xFFFFFFL) {
Review comment:
```suggestion
if (max <= 0xFFFFFF) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]