jpountz commented on a change in pull request #541:
URL: https://github.com/apache/lucene/pull/541#discussion_r794688777
##########
File path: lucene/core/src/java/org/apache/lucene/store/DataInput.java
##########
@@ -169,6 +169,13 @@ public void readLongs(long[] dst, int offset, int length)
throws IOException {
}
}
+ public void readInts(int[] dst, int offset, int length) throws IOException {
Review comment:
Can you add javadocs?
##########
File path: lucene/core/src/java/org/apache/lucene/util/bkd/BKDForUtil.java
##########
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util.bkd;
+
+import java.io.IOException;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+final class BKDForUtil {
+
+ private final int[] tmp;
+
+ BKDForUtil(int maxPointsInLeaf) {
+ // For encode16/decode16, we do not need to use tmp array.
+ // For encode24/decode24, we need a (3/4 * maxPointsInLeaf) length tmp
array.
+ // For encode32/decode32, we reuse the scratch in DocIdsWriter.
+ // So (3/4 * maxPointsInLeaf) is enough here.
+ final int len = (maxPointsInLeaf >>> 2) * 3;
+ tmp = new int[len];
+ }
+
+ void encode16(int len, int[] ints, DataOutput out) throws IOException {
+ final int halfLen = len >>> 1;
+ for (int i = 0; i < halfLen; ++i) {
+ ints[i] = ints[halfLen + i] | (ints[i] << 16);
+ }
+ for (int i = 0; i < halfLen; i++) {
+ out.writeInt(ints[i]);
+ }
+ if ((len & 1) == 1) {
+ out.writeShort((short) ints[len - 1]);
+ }
+ }
+
+ void encode32(int off, int len, int[] ints, DataOutput out) throws
IOException {
+ for (int i = 0; i < len; i++) {
+ out.writeInt(ints[off + i]);
+ }
+ }
+
+ void encode24(int off, int len, int[] ints, DataOutput out) throws
IOException {
+ final int quarterLen = len >>> 2;
+ final int quarterLen3 = quarterLen * 3;
+ for (int i = 0; i < quarterLen3; ++i) {
+ tmp[i] = ints[off + i] << 8;
+ }
+ for (int i = 0; i < quarterLen; i++) {
+ final int longIdx = off + i + quarterLen3;
+ tmp[i] |= (ints[longIdx] >>> 16) & 0xFF;
Review comment:
I think we don't even need toe mask since values use 24 bits?
```suggestion
tmp[i] |= ints[longIdx] >>> 16;
```
##########
File path: lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java
##########
@@ -24,89 +24,73 @@
import org.apache.lucene.util.DocBaseBitSetIterator;
import org.apache.lucene.util.FixedBitSet;
-class DocIdsWriter {
+final class DocIdsWriter {
- private DocIdsWriter() {}
+ private static final byte CONTINUOUS_IDS = (byte) -2;
+ private static final byte BITSET_IDS = (byte) -1;
+ private static final byte DELTA_FOR_UTIL = (byte) 32 + 16;
+ private static final byte BPV_24 = (byte) 32 + 24;
+ private static final byte BPV_32 = (byte) 32;
+ // These signs are legacy, should no longer be used in the writing side.
+ private static final byte LEGACY_DELTA_VINT = (byte) 0;
+ private static final byte LEGACY_BPV_24 = (byte) 24;
- static void writeDocIds(int[] docIds, int start, int count, DataOutput out)
throws IOException {
+ private final BKDForUtil forUtil;
+ private final int[] scratch;
+
+ DocIdsWriter(int maxPointsInLeaf) {
+ scratch = new int[maxPointsInLeaf];
+ forUtil = new BKDForUtil(maxPointsInLeaf);
+ }
+
+ void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws
IOException {
// docs can be sorted either when all docs in a block have the same value
// or when a segment is sorted
- boolean sorted = true;
boolean strictlySorted = true;
+ int min = docIds[0];
+ int max = docIds[0];
for (int i = 1; i < count; ++i) {
int last = docIds[start + i - 1];
int current = docIds[start + i];
- if (last > current) {
- sorted = strictlySorted = false;
- break;
- } else if (last == current) {
+ if (last >= current) {
strictlySorted = false;
}
+ min = Math.min(min, current);
+ max = Math.max(max, current);
}
- int min2max = docIds[start + count - 1] - docIds[start] + 1;
+ int min2max = max - min + 1;
if (strictlySorted) {
if (min2max == count) {
// continuous ids, typically happens when segment is sorted
- out.writeByte((byte) -2);
+ out.writeByte(CONTINUOUS_IDS);
out.writeVInt(docIds[start]);
return;
} else if (min2max <= (count << 4)) {
assert min2max > count : "min2max: " + min2max + ", count: " + count;
// Only trigger bitset optimization when max - min + 1 <= 16 * count
in order to avoid
// expanding too much storage.
// A field with lower cardinality will have higher probability to
trigger this optimization.
- out.writeByte((byte) -1);
+ out.writeByte(BITSET_IDS);
writeIdsAsBitSet(docIds, start, count, out);
return;
}
}
- if (sorted) {
- out.writeByte((byte) 0);
- int previous = 0;
- for (int i = 0; i < count; ++i) {
- int doc = docIds[start + i];
- out.writeVInt(doc - previous);
- previous = doc;
+
+ if (Integer.toUnsignedLong(min2max) <= 0xFFFFL) {
Review comment:
since doc IDs are integers in [0, MAX_VALUE) we don't need to convert to
an unsigned long (MAX_VALUE is not a legal doc ID)
```suggestion
if (min2max <= 0xFFFF) {
```
##########
File path: lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java
##########
@@ -24,89 +24,73 @@
import org.apache.lucene.util.DocBaseBitSetIterator;
import org.apache.lucene.util.FixedBitSet;
-class DocIdsWriter {
+final class DocIdsWriter {
- private DocIdsWriter() {}
+ private static final byte CONTINUOUS_IDS = (byte) -2;
+ private static final byte BITSET_IDS = (byte) -1;
+ private static final byte DELTA_FOR_UTIL = (byte) 32 + 16;
+ private static final byte BPV_24 = (byte) 32 + 24;
+ private static final byte BPV_32 = (byte) 32;
+ // These signs are legacy, should no longer be used in the writing side.
+ private static final byte LEGACY_DELTA_VINT = (byte) 0;
+ private static final byte LEGACY_BPV_24 = (byte) 24;
- static void writeDocIds(int[] docIds, int start, int count, DataOutput out)
throws IOException {
+ private final BKDForUtil forUtil;
+ private final int[] scratch;
+
+ DocIdsWriter(int maxPointsInLeaf) {
+ scratch = new int[maxPointsInLeaf];
+ forUtil = new BKDForUtil(maxPointsInLeaf);
+ }
+
+ void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws
IOException {
// docs can be sorted either when all docs in a block have the same value
// or when a segment is sorted
- boolean sorted = true;
boolean strictlySorted = true;
+ int min = docIds[0];
+ int max = docIds[0];
for (int i = 1; i < count; ++i) {
int last = docIds[start + i - 1];
int current = docIds[start + i];
- if (last > current) {
- sorted = strictlySorted = false;
- break;
- } else if (last == current) {
+ if (last >= current) {
strictlySorted = false;
}
+ min = Math.min(min, current);
+ max = Math.max(max, current);
}
- int min2max = docIds[start + count - 1] - docIds[start] + 1;
+ int min2max = max - min + 1;
if (strictlySorted) {
if (min2max == count) {
// continuous ids, typically happens when segment is sorted
- out.writeByte((byte) -2);
+ out.writeByte(CONTINUOUS_IDS);
out.writeVInt(docIds[start]);
return;
} else if (min2max <= (count << 4)) {
assert min2max > count : "min2max: " + min2max + ", count: " + count;
// Only trigger bitset optimization when max - min + 1 <= 16 * count
in order to avoid
// expanding too much storage.
// A field with lower cardinality will have higher probability to
trigger this optimization.
- out.writeByte((byte) -1);
+ out.writeByte(BITSET_IDS);
writeIdsAsBitSet(docIds, start, count, out);
return;
}
}
- if (sorted) {
- out.writeByte((byte) 0);
- int previous = 0;
- for (int i = 0; i < count; ++i) {
- int doc = docIds[start + i];
- out.writeVInt(doc - previous);
- previous = doc;
+
+ if (Integer.toUnsignedLong(min2max) <= 0xFFFFL) {
+ out.writeByte(DELTA_FOR_UTIL);
+ for (int i = 0; i < count; i++) {
+ scratch[i] = docIds[start + i] - min;
}
+ out.writeVInt(min);
+ forUtil.encode16(count, scratch, out);
} else {
- long max = 0;
- for (int i = 0; i < count; ++i) {
- max |= Integer.toUnsignedLong(docIds[start + i]);
- }
- if (max <= 0xffffff) {
- out.writeByte((byte) 24);
- // write them the same way we are reading them.
- int i;
- for (i = 0; i < count - 7; i += 8) {
- int doc1 = docIds[start + i];
- int doc2 = docIds[start + i + 1];
- int doc3 = docIds[start + i + 2];
- int doc4 = docIds[start + i + 3];
- int doc5 = docIds[start + i + 4];
- int doc6 = docIds[start + i + 5];
- int doc7 = docIds[start + i + 6];
- int doc8 = docIds[start + i + 7];
- long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 |
((doc3 >>> 8) & 0xffffL);
- long l2 =
- (doc3 & 0xffL) << 56
- | (doc4 & 0xffffffL) << 32
- | (doc5 & 0xffffffL) << 8
- | ((doc6 >> 16) & 0xffL);
- long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8
& 0xffffffL);
- out.writeLong(l1);
- out.writeLong(l2);
- out.writeLong(l3);
- }
- for (; i < count; ++i) {
- out.writeShort((short) (docIds[start + i] >>> 8));
- out.writeByte((byte) docIds[start + i]);
- }
+ if (Integer.toUnsignedLong(max) <= 0xFFFFFFL) {
Review comment:
```suggestion
if (max <= 0xFFFFFF) {
```
##########
File path:
lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
##########
@@ -503,6 +538,8 @@ private void unsetBuffers() {
curBuf = null;
curBufIndex = 0;
curLongBufferViews = null;
+ curFloatBufferViews = null;
+ curIntBufferViews = null;
Review comment:
+1 to open a separate issue
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]