This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 4276e0cdd7 [feature] Support per-field vector index options (#8239)
4276e0cdd7 is described below
commit 4276e0cdd7ecd9bf997a1f92e54c00266ee29e23
Author: jerry <[email protected]>
AuthorDate: Tue Jun 16 08:59:12 2026 +0800
[feature] Support per-field vector index options (#8239)
Support per-field vector index options with
`fields.<field-name>.<option>`.
This allows tables with multiple vector columns using the same index
type to configure different dimensions or index parameters for each
column. Field-level options do not include the index-type prefix, and
they override index-type-level defaults such as `<index-type>.<option>`
for the matching stored table column name.
---
docs/docs/multimodal-table/global-index.mdx | 32 ++++++
.../vector/index/VectorGlobalIndexerFactory.java | 25 ++++-
.../paimon/vector/index/VectorGlobalIndexTest.java | 5 +-
.../index/VectorGlobalIndexerFactoryTest.java | 111 ++++++++++++++++++++-
4 files changed, 165 insertions(+), 8 deletions(-)
diff --git a/docs/docs/multimodal-table/global-index.mdx
b/docs/docs/multimodal-table/global-index.mdx
index 747ccbaa9a..a7698f124a 100644
--- a/docs/docs/multimodal-table/global-index.mdx
+++ b/docs/docs/multimodal-table/global-index.mdx
@@ -127,6 +127,38 @@ Supported vector index options:
| `<index-type>.hnsw.ef-construction` | `150` | HNSW construction search width
for `ivf-hnsw-flat` and `ivf-hnsw-sq`. |
| `<index-type>.hnsw.max-level` | `7` | Maximum HNSW level for `ivf-hnsw-flat`
and `ivf-hnsw-sq`. |
+**Per-Field Options**
+
+The options above can also be set at the table level (in `TBLPROPERTIES`),
where they are shared
+by every vector column of the same index type. When a table has multiple
vector columns, you can
+scope an option to a single column with `fields.<field-name>.<option>`. The
field-level
+form takes precedence over the column-agnostic `<index-type>.<option>` for
that column. Use the
+stored table column name exactly as `<field-name>`. Field-level vector options
do not include the
+index-type prefix; for example, use `fields.image_embedding.nlist` to override
the shared
+`ivf-pq.nlist` option for `image_embedding`:
+
+```sql
+CREATE TABLE my_table (
+ id INT,
+ title_embedding ARRAY<FLOAT>,
+ image_embedding ARRAY<FLOAT>
+) TBLPROPERTIES (
+ 'bucket' = '-1',
+ 'row-tracking.enabled' = 'true',
+ 'data-evolution.enabled' = 'true',
+ 'global-index.enabled' = 'true',
+ -- per-column dimensions
+ 'fields.title_embedding.dimension' = '768',
+ 'fields.image_embedding.dimension' = '512',
+ -- shared by every ivf-pq column, overridden only for 'image_embedding'
+ 'ivf-pq.nlist' = '256',
+ 'fields.image_embedding.nlist' = '512'
+);
+```
+
+With the properties above, `title_embedding` is indexed with `nlist=256` while
`image_embedding`
+uses `nlist=512`.
+
**Vector Search**
Search-time options are passed with each vector search request:
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactory.java
b/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactory.java
index 354c967b56..9367723114 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactory.java
+++
b/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactory.java
@@ -37,14 +37,20 @@ public abstract class VectorGlobalIndexerFactory implements
GlobalIndexerFactory
public GlobalIndexer create(DataField field, Options options) {
String identifier = identifier();
return new VectorGlobalIndexer(
- field.type(), nativeOptions(field.type(), options,
identifier), identifier);
+ field.type(),
+ nativeOptions(field.type(), options, identifier, field.name()),
+ identifier);
}
static Map<String, String> nativeOptions(
- DataType fieldType, Options tableOptions, String identifier) {
+ DataType fieldType, Options tableOptions, String identifier,
String fieldName) {
Map<String, String> nativeOptions = new LinkedHashMap<>();
String optionPrefix = identifier + ".";
- for (Map.Entry<String, String> entry :
tableOptions.toMap().entrySet()) {
+ String fieldPrefix = "fields." + fieldName + ".";
+ Map<String, String> tableOptionsMap = tableOptions.toMap();
+
+ // First collect index-type level options, e.g. <index-type>.xxx.
+ for (Map.Entry<String, String> entry : tableOptionsMap.entrySet()) {
String optionKey = entry.getKey();
if (optionKey.startsWith(optionPrefix)) {
String nativeKey =
nativeOptionKey(optionKey.substring(optionPrefix.length()));
@@ -53,6 +59,19 @@ public abstract class VectorGlobalIndexerFactory implements
GlobalIndexerFactory
}
}
}
+
+ // Then collect field level options, e.g. fields.<field-name>.xxx,
which take precedence
+ // over the index-type level options for this field.
+ for (Map.Entry<String, String> entry : tableOptionsMap.entrySet()) {
+ String optionKey = entry.getKey();
+ if (optionKey.startsWith(fieldPrefix)) {
+ String nativeKey =
nativeOptionKey(optionKey.substring(fieldPrefix.length()));
+ if (nativeKey != null) {
+ nativeOptions.put(nativeKey, entry.getValue());
+ }
+ }
+ }
+
nativeOptions.put("index.type", identifier.replace('-', '_'));
nativeOptions.put(
"dimension", String.valueOf(dimension(fieldType,
nativeOptions, identifier)));
diff --git
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexTest.java
b/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexTest.java
index 73f8a4b28f..b10c843ef2 100644
---
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexTest.java
+++
b/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexTest.java
@@ -347,7 +347,7 @@ public class VectorGlobalIndexTest {
new VectorGlobalIndexer(
vectorType,
VectorGlobalIndexerFactory.nativeOptions(
- vectorType, options, IVF_PQ_IDENTIFIER),
+ vectorType, options, IVF_PQ_IDENTIFIER,
fieldName),
IVF_PQ_IDENTIFIER);
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
@@ -373,7 +373,8 @@ public class VectorGlobalIndexTest {
return new VectorGlobalIndexWriter(
fileWriter,
fieldType,
- VectorGlobalIndexerFactory.nativeOptions(fieldType, options,
IVF_PQ_IDENTIFIER),
+ VectorGlobalIndexerFactory.nativeOptions(
+ fieldType, options, IVF_PQ_IDENTIFIER, fieldName),
IVF_PQ_IDENTIFIER);
}
diff --git
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactoryTest.java
b/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactoryTest.java
index 1caf082f2c..b0f33d7706 100644
---
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactoryTest.java
+++
b/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactoryTest.java
@@ -70,7 +70,8 @@ public class VectorGlobalIndexerFactoryTest {
VectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
- IvfFlatVectorGlobalIndexerFactory.IDENTIFIER);
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec");
assertThat(nativeOptions)
.containsEntry("index.type", "ivf_flat")
@@ -92,7 +93,8 @@ public class VectorGlobalIndexerFactoryTest {
VectorGlobalIndexerFactory.nativeOptions(
new VectorType(8, new FloatType()),
options,
- IvfFlatVectorGlobalIndexerFactory.IDENTIFIER);
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec");
assertThat(nativeOptions).containsEntry("dimension", "8");
}
@@ -107,9 +109,112 @@ public class VectorGlobalIndexerFactoryTest {
VectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
-
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER))
+
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("ivf-flat.dimension")
.hasMessageContaining("positive integer");
}
+
+ @Test
+ public void testFieldLevelOptionsOverrideIndexTypeOptions() {
+ Options options = new Options();
+ options.setString("ivf-flat.dimension", "32");
+ options.setString("ivf-flat.nlist", "128");
+ options.setString("fields.vec.nlist", "256");
+
+ Map<String, String> nativeOptions =
+ VectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ options,
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec");
+
+ assertThat(nativeOptions)
+ .containsEntry("dimension", "32")
+ .containsEntry("nlist", "256")
+ .doesNotContainEntry("nlist", "128");
+ }
+
+ @Test
+ public void testFieldLevelDimensionOverridesIndexTypeDimension() {
+ Options options = new Options();
+ options.setString("ivf-flat.dimension", "32");
+ options.setString("fields.vec.dimension", "64");
+
+ Map<String, String> nativeOptions =
+ VectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ options,
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec");
+
+ assertThat(nativeOptions).containsEntry("dimension", "64");
+ }
+
+ @Test
+ public void testFieldLevelOptionsOnlyApplyToMatchingField() {
+ Options options = new Options();
+ options.setString("ivf-flat.nlist", "128");
+ options.setString("fields.vec.nlist", "256");
+
+ Map<String, String> nativeOptions =
+ VectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ options,
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "other");
+
+ assertThat(nativeOptions).containsEntry("nlist", "128");
+ }
+
+ @Test
+ public void testFieldLevelOptionsRequireExactFieldName() {
+ Options options = new Options();
+ options.setString("ivf-flat.nlist", "128");
+ options.setString("fields.vec_extra.nlist", "512");
+
+ Map<String, String> nativeOptions =
+ VectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ options,
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec");
+
+ assertThat(nativeOptions).containsEntry("nlist", "128");
+ }
+
+ @Test
+ public void testFieldLevelOptionsWithoutIndexTypeOption() {
+ Options options = new Options();
+ options.setString("fields.vec.distance.metric", "cosine");
+
+ Map<String, String> nativeOptions =
+ VectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ options,
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec");
+
+ assertThat(nativeOptions).containsEntry("metric", "cosine");
+ }
+
+ @Test
+ public void testFieldLevelVectorOptionsCoexistWithCoreFieldOptions() {
+ Options options = new Options();
+ options.setString("ivf-flat.nlist", "128");
+ options.setString("fields.vec.nlist", "256");
+ options.setString("fields.vec.aggregate-function", "sum");
+
+ Map<String, String> nativeOptions =
+ VectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ options,
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec");
+
+ assertThat(nativeOptions)
+ .containsEntry("nlist", "256")
+ .doesNotContainKey("aggregate-function");
+ }
}
