thecoop commented on code in PR #16696:
URL: https://github.com/apache/lucene/pull/16696#discussion_r4061804507
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java:
##########
@@ -432,21 +459,172 @@ private void mergeOneFlatVectorFieldCentered(FieldInfo
fieldInfo, MergeState mer
segmentWriteState.infoStream.message(
QUANTIZED_VECTOR_COMPONENT, "Vectors' count:" + vectorCount);
}
- QuantizedByteVectorValues quantizedVectorValues =
- mergedQuantizedVectorValues(fieldInfo, mergeState, mergedCentroid);
+ // Only asymmetric encodings have query-side records, and only FLOAT32
fields score graphs with
+ // them. FLOAT16 fields use fp16 vectors. The predicate sees vectorCount
before deletions, so it
+ // can request data even when the merged field is too small to build a
graph.
+ boolean prepareQueryData =
+ needsMergeScorer != null
+ && encoding.isAsymmetric()
+ && fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32
+ && needsMergeScorer.test(vectorCount);
long vectorDataOffset = vectorData.alignFilePointer(Float.BYTES);
- DocsWithFieldSet docsWithField = writeVectorData(vectorData,
quantizedVectorValues);
- long vectorDataLength = vectorData.getFilePointer() - vectorDataOffset;
- float centroidDp =
- docsWithField.cardinality() > 0 ?
VectorUtil.dotProduct(mergedCentroid, mergedCentroid) : 0;
- writeMeta(
+ DocsWithFieldSet docsWithField;
+ MergeScorerData mergeScorerData = null;
+ if (prepareQueryData) {
+ docsWithField = new DocsWithFieldSet();
+ mergeScorerData =
+ writeVectorAndQueryData(fieldInfo, mergeState, mergedCentroid,
docsWithField);
+ } else {
+ QuantizedByteVectorValues quantizedVectorValues =
+ mergedQuantizedVectorValues(fieldInfo, mergeState, mergedCentroid);
+ docsWithField = writeVectorData(vectorData, quantizedVectorValues);
+ }
+ boolean success = false;
+ try {
+ long vectorDataLength = vectorData.getFilePointer() - vectorDataOffset;
+ float centroidDp =
+ docsWithField.cardinality() > 0
+ ? VectorUtil.dotProduct(mergedCentroid, mergedCentroid)
+ : 0;
+ writeMeta(
+ fieldInfo,
+ segmentWriteState.segmentInfo.maxDoc(),
+ vectorDataOffset,
+ vectorDataLength,
+ mergedCentroid,
+ centroidDp,
+ docsWithField);
+ success = true;
+ return mergeScorerData;
+ } finally {
+ // Delete the records on failure because their handle has not been
returned.
+ if (success == false && mergeScorerData != null) {
+ mergeScorerData.close();
Review Comment:
Should use `IOUtils` here (also get rid of the `success` boolean, and use
catch blocks, as done elsewhere)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
