This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 274d1de9c8 [lumina] Use VECTOR length for index dimension (#8235)
274d1de9c8 is described below
commit 274d1de9c89b61a6edf935c4416491924480ce02
Author: QuakeWang <[email protected]>
AuthorDate: Tue Jun 16 08:59:39 2026 +0800
[lumina] Use VECTOR length for index dimension (#8235)
Lumina vector writers used `lumina.index.dimension` for both ARRAY and
VECTOR fields. For VECTOR fields, the schema already carries the fixed
length, so falling back to the default dimension could make write
validation and index metadata disagree with the actual field type.
This PR resolves the effective dimension from `VectorType#getLength()`,
rejects explicit dimension conflicts, and builds native Lumina options
from the resolved dimension so PQ options are capped against the real
dimension. `ArrayType<FLOAT>` continues to use `lumina.index.dimension`.
---
.../index/LuminaVectorGlobalIndexWriter.java | 31 ++--
.../lumina/index/LuminaVectorIndexOptions.java | 23 ++-
.../index/LuminaVectorGlobalIndexWriterTest.java | 161 +++++++++++++++++++++
3 files changed, 200 insertions(+), 15 deletions(-)
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
index 6e1a31902e..206da270a4 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
@@ -78,6 +78,7 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingletonWriter
private final GlobalIndexFileWriter fileWriter;
private final LuminaVectorIndexOptions options;
+ private final Map<String, String> luminaOptions;
private final int dim;
/** Temporary file storing records as [rowId (long)][float * dim] in
native byte order. */
@@ -99,14 +100,13 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingletonWriter
LuminaVectorIndexOptions options) {
this.fileWriter = fileWriter;
this.options = options;
- this.dim = options.dimension();
+ this.dim = validateAndResolveDimension(fieldType, options);
+ this.luminaOptions = options.toLuminaOptions(dim);
this.count = 0;
this.closed = false;
this.recordSizeInBytes = checkedRecordSize(dim, IO_BUFFER_SIZE);
this.vectorBuf = new float[dim];
- validateFieldType(fieldType);
-
try {
this.tempVectorFile = File.createTempFile("lumina-vectors-",
".bin");
this.tempVectorFile.deleteOnExit();
@@ -120,14 +120,25 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingletonWriter
}
}
- private void validateFieldType(DataType dataType) {
+ private static int validateAndResolveDimension(
+ DataType dataType, LuminaVectorIndexOptions options) {
if (dataType instanceof VectorType) {
- DataType elementType = ((VectorType) dataType).getElementType();
+ VectorType vectorType = (VectorType) dataType;
+ DataType elementType = vectorType.getElementType();
if (!(elementType instanceof FloatType)) {
throw new IllegalArgumentException(
"Lumina vector index requires float vector, but got: "
+ elementType);
}
- return;
+ int typeDimension = vectorType.getLength();
+ if (options.isDimensionExplicitlyConfigured() &&
options.dimension() != typeDimension) {
+ throw new IllegalArgumentException(
+ String.format(
+ "%s configured %d conflicts with VECTOR length
%d.",
+ LuminaVectorIndexOptions.DIMENSION.key(),
+ options.dimension(),
+ typeDimension));
+ }
+ return typeDimension;
}
if (dataType instanceof ArrayType) {
DataType elementType = ((ArrayType) dataType).getElementType();
@@ -135,7 +146,7 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingletonWriter
throw new IllegalArgumentException(
"Lumina vector index requires float array, but got: "
+ elementType);
}
- return;
+ return options.dimension();
}
throw new IllegalArgumentException(
"Lumina vector index requires VectorType or ArrayType<FLOAT>,
but got: "
@@ -255,7 +266,7 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingletonWriter
try (LuminaIndex index =
LuminaIndex.createForBuild(
- options.indexType(), dim, options.metric(),
options.toLuminaOptions())) {
+ options.indexType(), dim, options.metric(),
luminaOptions)) {
// Pretrain and insert via streaming file-backed Dataset API
long phaseStart = System.currentTimeMillis();
@@ -288,7 +299,7 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingletonWriter
"Lumina index build completed in {} ms",
System.currentTimeMillis() - buildStart);
- LuminaIndexMeta meta = new
LuminaIndexMeta(options.toLuminaOptions());
+ LuminaIndexMeta meta = new LuminaIndexMeta(luminaOptions);
// rowCount = logical rows including nulls (not just indexed
vectors)
return new ResultEntry(fileName, logicalRowId, meta.serialize());
}
@@ -300,7 +311,7 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingletonWriter
* thread pool is sized correctly before the builder is created.
*/
private void configureExecutorThreadCount() {
- Map<String, String> luminaOpts = options.toLuminaOptions();
+ Map<String, String> luminaOpts = luminaOptions;
String threadCountKey =
LuminaVectorIndexOptions.toLuminaKey(
LuminaVectorIndexOptions.DISKANN_BUILD_THREAD_COUNT);
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
index 2a23e8bbf8..80c6b8e28e 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
@@ -110,16 +110,18 @@ public class LuminaVectorIndexOptions {
.withDescription("The parallel number for search.");
private final int dimension;
+ private final boolean dimensionExplicitlyConfigured;
private final LuminaVectorMetric metric;
private final String indexType;
private final Map<String, String> luminaOptions;
public LuminaVectorIndexOptions(Options options) {
this.dimension = validatePositive(options.get(DIMENSION),
DIMENSION.key());
+ this.dimensionExplicitlyConfigured = options.contains(DIMENSION);
this.metric = parseMetric(options.get(DISTANCE_METRIC));
this.indexType = options.get(INDEX_TYPE);
validateEncodingMetricCombination(options.get(ENCODING_TYPE),
this.metric);
- this.luminaOptions = buildLuminaOptions(options, this.dimension);
+ this.luminaOptions = buildLuminaOptions(options);
}
/**
@@ -128,13 +130,26 @@ public class LuminaVectorIndexOptions {
* diskann.build.ef_construction}.
*/
public Map<String, String> toLuminaOptions() {
- return new LinkedHashMap<>(luminaOptions);
+ return toLuminaOptions(dimension);
+ }
+
+ public Map<String, String> toLuminaOptions(int dimension) {
+ Map<String, String> result = new LinkedHashMap<>(luminaOptions);
+ result.put(
+ toLuminaKey(DIMENSION),
+ String.valueOf(validatePositive(dimension, DIMENSION.key())));
+ capPqM(result, dimension);
+ return result;
}
public int dimension() {
return dimension;
}
+ public boolean isDimensionExplicitlyConfigured() {
+ return dimensionExplicitlyConfigured;
+ }
+
public LuminaVectorMetric metric() {
return metric;
}
@@ -177,7 +192,7 @@ public class LuminaVectorIndexOptions {
* like {@code index.type} are always present in the metadata, matching
paimon-cpp behavior.
*/
@SuppressWarnings("unchecked")
- private static Map<String, String> buildLuminaOptions(Options options, int
dimension) {
+ private static Map<String, String> buildLuminaOptions(Options options) {
Map<String, String> result = new LinkedHashMap<>();
// Populate all known options with their resolved values (user-set or
default).
for (ConfigOption<?> opt : ALL_OPTIONS) {
@@ -193,8 +208,6 @@ public class LuminaVectorIndexOptions {
result.putIfAbsent(key.substring(LUMINA_PREFIX.length()),
entry.getValue());
}
}
- // PQ encoding requires pq.m <= dimension; auto-cap to avoid native
init failures.
- capPqM(result, dimension);
return result;
}
diff --git
a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java
new file mode 100644
index 0000000000..b0b1996e73
--- /dev/null
+++
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriterTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.lumina.index;
+
+import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.FloatType;
+import org.apache.paimon.types.VectorType;
+
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Field;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Unit tests for {@link LuminaVectorGlobalIndexWriter}. */
+public class LuminaVectorGlobalIndexWriterTest {
+
+ @Test
+ public void testVectorTypeUsesTypeDimensionByDefault() {
+ DataType vecFieldType = new VectorType(2, new FloatType());
+ Options options = new Options();
+ options.setString(LuminaVectorIndexOptions.DISTANCE_METRIC.key(),
"l2");
+
+ LuminaVectorIndexOptions indexOptions = new
LuminaVectorIndexOptions(options);
+
+ try (LuminaVectorGlobalIndexWriter writer =
+ new LuminaVectorGlobalIndexWriter(
+ createNoopFileWriter(), vecFieldType, indexOptions)) {
+ assertThat(writerLuminaOptions(writer))
+ .containsEntry(
+ LuminaVectorIndexOptions.toLuminaKey(
+ LuminaVectorIndexOptions.DIMENSION),
+ "2");
+
+ writer.write(new float[] {1.0f, 0.0f});
+
+ assertThatThrownBy(() -> writer.write(new float[] {1.0f, 0.0f,
0.0f}))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("expected 2")
+ .hasMessageContaining("got 3");
+ }
+ }
+
+ @Test
+ public void testVectorTypePqMUsesTypeDimensionForCap() {
+ assertVectorTypePqMOption(256, 192, 192);
+ assertVectorTypePqMOption(256, 300, 256);
+ }
+
+ @Test
+ public void testVectorTypeRejectsExplicitDimensionConflict() {
+ DataType vecFieldType = new VectorType(2, new FloatType());
+ Options options = createDefaultOptions(3);
+ LuminaVectorIndexOptions indexOptions = new
LuminaVectorIndexOptions(options);
+
+ assertThatThrownBy(
+ () ->
+ new LuminaVectorGlobalIndexWriter(
+ createNoopFileWriter(), vecFieldType,
indexOptions))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(LuminaVectorIndexOptions.DIMENSION.key())
+ .hasMessageContaining("configured 3")
+ .hasMessageContaining("VECTOR length 2");
+ }
+
+ @Test
+ public void testArrayTypeUsesOptionDimension() {
+ DataType arrayFieldType = new ArrayType(new FloatType());
+ Options options = createDefaultOptions(3);
+ LuminaVectorIndexOptions indexOptions = new
LuminaVectorIndexOptions(options);
+
+ try (LuminaVectorGlobalIndexWriter writer =
+ new LuminaVectorGlobalIndexWriter(
+ createNoopFileWriter(), arrayFieldType, indexOptions))
{
+ writer.write(new float[] {1.0f, 0.0f, 0.0f});
+
+ assertThatThrownBy(() -> writer.write(new float[] {1.0f, 0.0f}))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("expected 3")
+ .hasMessageContaining("got 2");
+ }
+ }
+
+ private static Options createDefaultOptions(int dimension) {
+ Options options = new Options();
+ options.setInteger(LuminaVectorIndexOptions.DIMENSION.key(),
dimension);
+ options.setString(LuminaVectorIndexOptions.DISTANCE_METRIC.key(),
"l2");
+ return options;
+ }
+
+ private static GlobalIndexFileWriter createNoopFileWriter() {
+ return new GlobalIndexFileWriter() {
+ @Override
+ public String newFileName(String prefix) {
+ return prefix;
+ }
+
+ @Override
+ public PositionOutputStream newOutputStream(String fileName) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ private static void assertVectorTypePqMOption(
+ int vectorDimension, int configuredPqM, int expectedPqM) {
+ DataType vecFieldType = new VectorType(vectorDimension, new
FloatType());
+ Options options = new Options();
+ options.setString(LuminaVectorIndexOptions.DISTANCE_METRIC.key(),
"l2");
+ options.setInteger(LuminaVectorIndexOptions.ENCODING_PQ_M.key(),
configuredPqM);
+ LuminaVectorIndexOptions indexOptions = new
LuminaVectorIndexOptions(options);
+
+ try (LuminaVectorGlobalIndexWriter writer =
+ new LuminaVectorGlobalIndexWriter(
+ createNoopFileWriter(), vecFieldType, indexOptions)) {
+ Map<String, String> luminaOptions = writerLuminaOptions(writer);
+ assertThat(luminaOptions)
+ .containsEntry(
+ LuminaVectorIndexOptions.toLuminaKey(
+ LuminaVectorIndexOptions.DIMENSION),
+ String.valueOf(vectorDimension))
+ .containsEntry(
+ LuminaVectorIndexOptions.toLuminaKey(
+ LuminaVectorIndexOptions.ENCODING_PQ_M),
+ String.valueOf(expectedPqM));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, String>
writerLuminaOptions(LuminaVectorGlobalIndexWriter writer) {
+ try {
+ Field field =
LuminaVectorGlobalIndexWriter.class.getDeclaredField("luminaOptions");
+ field.setAccessible(true);
+ return (Map<String, String>) field.get(writer);
+ } catch (ReflectiveOperationException e) {
+ throw new AssertionError(e);
+ }
+ }
+}
