mccullocht commented on code in PR #16030:
URL: https://github.com/apache/lucene/pull/16030#discussion_r4049370617
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java:
##########
@@ -320,39 +351,167 @@ private QuantizedByteVectorValues
mergedQuantizedVectorValues(
return new QuantizedFloatVectorValues(vectorValues, quantizer, encoding,
centroid);
}
+ /**
+ * Returns a view that quantizes a single segment's float vectors against
{@code centroid} using
+ * this writer's encoding, without consulting any quantized bytes the
segment may already store.
+ */
+ private QuantizedFloatVectorValues quantizeFromFloats(
+ KnnVectorsReader reader, FieldInfo fieldInfo, float[] centroid) throws
IOException {
+ OptimizedScalarQuantizer quantizer =
+ new OptimizedScalarQuantizer(fieldInfo.getVectorSimilarityFunction());
+ FloatVectorValues vectorValues =
+ fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT16
+ ? new
Float16AsFloatVectorValues(reader.getFloat16VectorValues(fieldInfo.name))
+ : reader.getFloatVectorValues(fieldInfo.name);
+ if (fieldInfo.getVectorSimilarityFunction() == COSINE) {
+ vectorValues = new NormalizedFloatVectorValues(vectorValues);
+ }
+ return new QuantizedFloatVectorValues(vectorValues, quantizer, encoding,
centroid);
+ }
+
@Override
public void mergeOneFlatVectorField(FieldInfo fieldInfo, MergeState
mergeState)
throws IOException {
- // Don't need access to the random vectors, we can just use the merged
- rawVectorDelegate.mergeOneFlatVectorField(fieldInfo, mergeState);
if (fieldInfo.getVectorEncoding().isFloatingPoint() == false) {
+ rawVectorDelegate.mergeOneFlatVectorField(fieldInfo, mergeState);
return;
}
- final float[] centroid;
+ switch (mode) {
+ case CENTERED -> {
+ failIfFloatVectorsMissing(fieldInfo, mergeState);
+ mergeOneFlatVectorFieldCentered(fieldInfo, mergeState);
+ }
+ case DATA_BLIND_WITH_FLOATS -> {
+ // The output segment stores full-precision floats, so every
contributing segment must
+ // provide them; the quantized merge then re-quantizes them against
the zero centroid.
+ failIfFloatVectorsMissing(fieldInfo, mergeState);
+ rawVectorDelegate.mergeOneFlatVectorField(fieldInfo, mergeState);
+ mergeOneFlatVectorFieldDataBlind(fieldInfo, mergeState);
+ }
+ case DATA_BLIND_WITHOUT_FLOATS ->
mergeOneFlatVectorFieldDataBlind(fieldInfo, mergeState);
+ }
+ }
+
+ /**
+ * Fails the merge when the output mode stores full-precision float vectors
({@code mode} is
+ * {@link Mode#CENTERED} or {@link Mode#DATA_BLIND_WITH_FLOATS}) but a
contributing segment cannot
+ * supply them, e.g. a segment written in {@link
Mode#DATA_BLIND_WITHOUT_FLOATS}. Such segments
+ * can only offer dequantized (already-quantized) values in place of true
full-precision vectors;
+ * this fails loudly rather than silently degrading quality.
+ */
+ private void failIfFloatVectorsMissing(FieldInfo fieldInfo, MergeState
mergeState)
+ throws IOException {
+ for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) {
+ KnnVectorsReader reader = mergeState.knnVectorsReaders[i];
+ if (reader == null || hasRawVectorValues(reader, fieldInfo)) {
+ continue;
+ }
+ // Tolerate segments that have no vectors for the field at all; they
contribute nothing.
+ FloatVectorValues values = floatingPointVectorValues(reader, fieldInfo);
+ if (values == null || values.size() == 0) {
+ continue;
+ }
+ throw new IllegalStateException(
Review Comment:
I'd prefer to be consistent here -- allowing "empty" segments this way might
lead to some confusion about why it is or isn't working.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
