This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c4308ca177 [vector] Expose IVF search tuning options (#9367)
c4308ca177 is described below
commit c4308ca17768bf72c1fa65ad487abd0820eb3639
Author: shyjsarah <[email protected]>
AuthorDate: Mon Aug 24 11:45:00 2026 +0800
[vector] Expose IVF search tuning options (#9367)
---
docs/docs/multimodal-table/global-index/vector.mdx | 9 ++++
.../index/NativeVectorGlobalIndexReader.java | 62 ++++++++++++++++------
.../vector/index/NativeVectorGlobalIndexTest.java | 50 +++++++++++++++++
3 files changed, 106 insertions(+), 15 deletions(-)
diff --git a/docs/docs/multimodal-table/global-index/vector.mdx
b/docs/docs/multimodal-table/global-index/vector.mdx
index 217390ac63..8bcd42f7cb 100644
--- a/docs/docs/multimodal-table/global-index/vector.mdx
+++ b/docs/docs/multimodal-table/global-index/vector.mdx
@@ -284,13 +284,22 @@ Search-time options are passed with each vector search
request:
| Option | Default | Description |
|---|---|---|
| `ivf.nprobe` | Automatic | Explicit number of IVF clusters to probe. When
omitted, paimon-vindex derives the width from the index, `top_k`, and filter
selectivity. |
+| `ivf.max_initial_filter_expansion_factor` | Disabled | Positive integer
limiting filter-driven expansion of the initial automatic IVF probe width. A
factor of `1` disables initial filter expansion. Progressive retries may still
probe more clusters when fewer than `top_k` filtered results are found. |
| `ivf.refine_factor` | Disabled | Retrieves `top_k * refine_factor` IVF
candidates and reranks them with the original vectors stored in the Paimon
table. It is most useful for compressed indexes such as `ivf-pq`, `ivf-sq`, and
`ivf-rq` when recall is more important than latency. |
+| `ivf_pq.batch_table_reuse` | `auto` | IVF-PQ batch search distance-table
reuse mode: `auto`, `on`, or `off`. Other index types and scalar searches
ignore it. |
+| `ivf_pq.batch_table_reuse.max_bytes` | 512 MiB | Positive long integer
limiting the memory used by IVF-PQ batch distance-table reuse. Search falls
back to direct table construction when the reusable tables exceed the budget. |
| `diskann.l_search` | Automatic | paimon-vindex DiskANN graph candidate
width. The automatic value uses calibration when available, otherwise `max(100,
2 * top_k)`. |
Use the same distance metric at build time and query time. Search options can
be passed per query,
so you can use a larger `ivf.nprobe` or `diskann.l_search` for higher recall
queries and a smaller
value for latency-sensitive queries. Do not set both in one query.
+`ivf.max_initial_filter_expansion_factor` applies only to automatic IVF search
and cannot be combined
+with `ivf.nprobe` or `diskann.l_search`. Lower factors reduce initial
filtered-search work but may
+reduce Recall@K compared with uncapped automatic search. Progressive expansion
occurs only when
+fewer than `top_k` valid results are returned; if the capped initial search
already fills `top_k`,
+probing stops.
+
`ivf.refine_factor` can also be configured with `refine_factor`,
`rerank_factor`, and hyphenated
spellings such as `ivf.refine-factor`. Setting `ivf.refine_factor=1` still
performs the raw-vector
rerank for the indexed candidates; leaving it unset skips the rerank stage.
diff --git
a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
index 4056212e92..7b9c2d7872 100644
---
a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
@@ -65,6 +65,11 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
private static final String NPROBE_PARAMETER = "ivf.nprobe";
private static final String L_SEARCH_PARAMETER = "diskann.l_search";
+ private static final String MAX_INITIAL_FILTER_EXPANSION_FACTOR_PARAMETER =
+ "ivf.max_initial_filter_expansion_factor";
+ private static final String IVF_PQ_BATCH_TABLE_REUSE_PARAMETER =
"ivf_pq.batch_table_reuse";
+ private static final String IVF_PQ_BATCH_TABLE_REUSE_MAX_BYTES_PARAMETER =
+ "ivf_pq.batch_table_reuse.max_bytes";
private static final int VECTOR_INDEX_MIN_SEEK_FOR_VECTOR_READS = 16 *
1024;
private static final int VECTOR_INDEX_PARALLELISM_FOR_VECTOR_READS = 32;
@@ -151,6 +156,8 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
if (scope == null) {
return emptyResults(n);
}
+ VectorSearchParams searchParams =
+ batchSearchParams(batchVectorSearch.options(),
scope.effectiveK);
// Flatten query vectors into one contiguous array for a single native
call.
float[] queries = new float[n * dim];
@@ -160,15 +167,8 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
VectorSearchBatchResult batchResult =
scope.filterBytes != null
- ? vectorReader.searchBatch(
- queries,
- n,
- searchParams(batchVectorSearch.options(),
scope.effectiveK),
- scope.filterBytes)
- : vectorReader.searchBatch(
- queries,
- n,
- searchParams(batchVectorSearch.options(),
scope.effectiveK));
+ ? vectorReader.searchBatch(queries, n, searchParams,
scope.filterBytes)
+ : vectorReader.searchBatch(queries, n, searchParams);
// result i corresponds to vectors[i], matching input order.
List<Optional<ScoredGlobalIndexResult>> results = new ArrayList<>(n);
@@ -287,17 +287,36 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
static VectorSearchParams searchParams(Map<String, String> parameters, int
topK) {
Integer nprobe = intParameter(parameters, NPROBE_PARAMETER);
Integer lSearch = intParameter(parameters, L_SEARCH_PARAMETER);
+ Integer maxInitialFilterExpansionFactor =
+ intParameter(parameters,
MAX_INITIAL_FILTER_EXPANSION_FACTOR_PARAMETER);
if (nprobe != null && lSearch != null) {
throw new IllegalArgumentException(
"Cannot set both '" + NPROBE_PARAMETER + "' and '" +
L_SEARCH_PARAMETER + "'.");
}
+ VectorSearchParams searchParams;
if (nprobe != null) {
- return VectorSearchParams.ivf(topK, nprobe);
- }
- if (lSearch != null) {
- return VectorSearchParams.diskAnn(topK, lSearch);
- }
- return VectorSearchParams.automatic(topK);
+ searchParams = VectorSearchParams.ivf(topK, nprobe);
+ } else if (lSearch != null) {
+ searchParams = VectorSearchParams.diskAnn(topK, lSearch);
+ } else {
+ searchParams = VectorSearchParams.automatic(topK);
+ }
+ return maxInitialFilterExpansionFactor == null
+ ? searchParams
+ :
searchParams.withMaxInitialFilterExpansionFactor(maxInitialFilterExpansionFactor);
+ }
+
+ static VectorSearchParams batchSearchParams(Map<String, String>
parameters, int topK) {
+ VectorSearchParams searchParams = searchParams(parameters, topK);
+ String reuseMode = parameters.get(IVF_PQ_BATCH_TABLE_REUSE_PARAMETER);
+ if (reuseMode != null) {
+ searchParams = searchParams.withIvfPqBatchTableReuse(reuseMode);
+ }
+ Long reuseMaxBytes =
+ longParameter(parameters,
IVF_PQ_BATCH_TABLE_REUSE_MAX_BYTES_PARAMETER);
+ return reuseMaxBytes == null
+ ? searchParams
+ : searchParams.withIvfPqBatchTableReuseMaxBytes(reuseMaxBytes);
}
private static Integer intParameter(Map<String, String> parameters, String
key) {
@@ -313,6 +332,19 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
}
}
+ private static Long longParameter(Map<String, String> parameters, String
key) {
+ String value = parameters.get(key);
+ if (value == null) {
+ return null;
+ }
+ try {
+ return Long.parseLong(value);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(
+ "Invalid value for '" + key + "': " + value + ". Must be a
long integer.", e);
+ }
+ }
+
private void validateSearchVector(Object vector) {
if (!(vector instanceof float[])) {
throw new IllegalArgumentException(
diff --git
a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
index eb7750915a..d4aeb7f316 100644
---
a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
+++
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
@@ -27,6 +27,7 @@ import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
+import org.apache.paimon.index.vector.IvfPqBatchTableReuseMode;
import org.apache.paimon.index.vector.VectorSearchParams;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.BatchVectorSearch;
@@ -255,6 +256,55 @@ public class NativeVectorGlobalIndexTest {
assertThat(diskAnnParams.topK()).isEqualTo(10);
}
+ @Test
+ public void testIvfInitialFilterExpansionFactorValidationIsPropagated() {
+ assertThatThrownBy(
+ () ->
+ NativeVectorGlobalIndexReader.searchParams(
+ Collections.singletonMap(
+
"ivf.max_initial_filter_expansion_factor", "0"),
+ 10))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("greater than 0");
+
+ Map<String, String> parameters = new HashMap<>();
+ parameters.put("ivf.nprobe", "16");
+ parameters.put("ivf.max_initial_filter_expansion_factor", "4");
+ assertThatThrownBy(() ->
NativeVectorGlobalIndexReader.searchParams(parameters, 10))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("requires automatic IVF search");
+ }
+
+ @Test
+ public void testIvfPqBatchTableReuseIsPropagatedToBatchSearchParams() {
+ VectorSearchParams params =
+ NativeVectorGlobalIndexReader.batchSearchParams(
+ Collections.singletonMap("ivf_pq.batch_table_reuse",
"on"), 10);
+
+
assertThat(params.ivfPqBatchTableReuse()).isEqualTo(IvfPqBatchTableReuseMode.ON);
+ }
+
+ @Test
+ public void testIvfPqBatchTableReuseMaxBytesIsPropagated() {
+ VectorSearchParams params =
+ NativeVectorGlobalIndexReader.batchSearchParams(
+
Collections.singletonMap("ivf_pq.batch_table_reuse.max_bytes", "134217728"),
+ 10);
+
+ assertThat(params.ivfPqBatchTableReuseMaxBytes()).isEqualTo(128L *
1024 * 1024);
+ }
+
+ @Test
+ public void testIvfPqBatchTableReuseMaxBytesSupportsLongValues() {
+ VectorSearchParams params =
+ NativeVectorGlobalIndexReader.batchSearchParams(
+ Collections.singletonMap(
+ "ivf_pq.batch_table_reuse.max_bytes",
"5368709120"),
+ 10);
+
+ assertThat(params.ivfPqBatchTableReuseMaxBytes()).isEqualTo(5L * 1024
* 1024 * 1024);
+ }
+
@Test
public void testVectorSearchParameterRangeValidationDelegatedToNative() {
assertThat(
