This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-vector-index.git
The following commit(s) were added to refs/heads/main by this push:
new ba41a5f feat: make initial IVF filter expansion configurable (#68)
ba41a5f is described below
commit ba41a5f730397553a10806e72074031298464bf5
Author: shyjsarah <[email protected]>
AuthorDate: Tue Aug 4 11:26:19 2026 +0800
feat: make initial IVF filter expansion configurable (#68)
---
core/src/autotune.rs | 70 +++++++++++++-
core/src/index.rs | 106 ++++++++++++++++++++-
docs/api.html | 13 ++-
ffi/src/lib.rs | 1 +
.../paimon/index/vector/VectorSearchParams.java | 51 +++++++++-
.../index/vector/VectorIndexJavaApiTest.java | 37 +++++++
.../vector/VectorIndexNativeValidationTest.java | 17 +++-
jni/src/lib.rs | 21 ++++
8 files changed, 306 insertions(+), 10 deletions(-)
diff --git a/core/src/autotune.rs b/core/src/autotune.rs
index a7adba0..d15d145 100644
--- a/core/src/autotune.rs
+++ b/core/src/autotune.rs
@@ -139,13 +139,24 @@ pub fn default_training_vector_count(vector_count: usize,
nlist: usize) -> io::R
///
/// The policy scans at least 1/16 of coarse lists and enough average list rows
/// for four candidates per requested result. Filtering scales this initial
-/// width by inverse selectivity; search wrappers may still expand
progressively
-/// when invalid/padded results remain.
+/// width by inverse selectivity. Callers may cap that initial filter
expansion;
+/// search wrappers may still expand progressively when invalid/padded results
+/// remain.
pub fn infer_ivf_nprobe(
nlist: usize,
vector_count: usize,
top_k: usize,
matching_count: Option<usize>,
+) -> io::Result<usize> {
+ infer_ivf_nprobe_with_filter_expansion_cap(nlist, vector_count, top_k,
matching_count, None)
+}
+
+pub(crate) fn infer_ivf_nprobe_with_filter_expansion_cap(
+ nlist: usize,
+ vector_count: usize,
+ top_k: usize,
+ matching_count: Option<usize>,
+ max_initial_filter_expansion_factor: Option<usize>,
) -> io::Result<usize> {
if nlist == 0 {
return Err(invalid_input("nlist must be greater than 0"));
@@ -167,14 +178,30 @@ pub fn infer_ivf_nprobe(
.max(candidate_lists)
.min(nlist);
+ if matches!(max_initial_filter_expansion_factor, Some(0)) {
+ return Err(invalid_input(
+ "maximum initial filter expansion factor must be greater than 0",
+ ));
+ }
+
if let Some(matching_count) = matching_count {
if matching_count == 0 {
return Ok(1);
}
- nprobe = ((nprobe as u128)
+ let base_nprobe = nprobe;
+ let matching_count = matching_count.min(vector_count);
+ let selectivity_scaled = (base_nprobe as u128)
.saturating_mul(vector_count as u128)
.div_ceil(matching_count as u128)
- .min(nlist as u128)) as usize;
+ .min(nlist as u128);
+ let expansion_cap = max_initial_filter_expansion_factor
+ .map(|factor| {
+ (base_nprobe as u128)
+ .saturating_mul(factor as u128)
+ .min(nlist as u128)
+ })
+ .unwrap_or(nlist as u128);
+ nprobe = selectivity_scaled.min(expansion_cap) as usize;
}
Ok(nprobe.clamp(1, nlist))
}
@@ -403,6 +430,41 @@ mod tests {
assert_eq!(infer_ivf_nprobe(1024, 1_000_000, 10, Some(0)).unwrap(), 1);
}
+ #[test]
+ fn automatic_nprobe_caps_initial_filter_expansion() {
+ assert_eq!(
+ infer_ivf_nprobe_with_filter_expansion_cap(256, 2_560_000, 3,
Some(256_000), Some(4))
+ .unwrap(),
+ 64
+ );
+ }
+
+ #[test]
+ fn automatic_nprobe_filter_expansion_cap_preserves_bounds() {
+ assert_eq!(
+ infer_ivf_nprobe_with_filter_expansion_cap(256, 2_560_000, 3,
Some(256_000), Some(1))
+ .unwrap(),
+ 16
+ );
+ assert_eq!(
+ infer_ivf_nprobe(256, 2_560_000, 3, Some(256_000)).unwrap(),
+ 160
+ );
+ assert_eq!(
+ infer_ivf_nprobe_with_filter_expansion_cap(256, 2_560_000, 3,
Some(5_120_000), Some(4))
+ .unwrap(),
+ 16
+ );
+ assert!(infer_ivf_nprobe_with_filter_expansion_cap(
+ 256,
+ 2_560_000,
+ 3,
+ Some(256_000),
+ Some(0)
+ )
+ .is_err());
+ }
+
#[test]
fn calibrated_candidate_never_hides_an_unsatisfied_target() {
let candidates = [
diff --git a/core/src/index.rs b/core/src/index.rs
index cc2ed7c..f5b35f8 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -17,7 +17,7 @@
use crate::autotune::{
default_training_vector_count, diskann_build_preset,
infer_diskann_l_search, infer_ivf_nlist,
- infer_ivf_nprobe, infer_rq_bits, DiskAnnBuildPreset, TuningObjective,
+ infer_ivf_nprobe_with_filter_expansion_cap, infer_rq_bits,
DiskAnnBuildPreset, TuningObjective,
};
use crate::diskann::{
diskann_training_sample_limit, validate_diskann_format_configuration,
@@ -991,6 +991,13 @@ pub struct VectorSearchParams {
pub top_k: usize,
pub search_width: SearchWidth,
pub width: usize,
+ /// Caps inverse-selectivity expansion of the initial automatic IVF nprobe.
+ ///
+ /// `None` preserves unlimited expansion. Lower factors reduce initial
search
+ /// work but may reduce recall compared with uncapped automatic search.
+ /// Progressive search may exceed this initial cap only when filtered
results
+ /// do not fill `top_k`.
+ pub max_initial_filter_expansion_factor: Option<usize>,
pub ivfpq_batch_table_reuse: IvfPqBatchTableReuseMode,
pub ivfpq_batch_table_reuse_max_bytes: usize,
}
@@ -1001,6 +1008,7 @@ impl VectorSearchParams {
top_k,
search_width: SearchWidth::IvfNProbe,
width: nprobe,
+ max_initial_filter_expansion_factor: None,
ivfpq_batch_table_reuse: IvfPqBatchTableReuseMode::Auto,
ivfpq_batch_table_reuse_max_bytes:
DEFAULT_IVFPQ_BATCH_TABLE_REUSE_MAX_BYTES,
}
@@ -1011,6 +1019,7 @@ impl VectorSearchParams {
top_k,
search_width: SearchWidth::DiskAnnLSearch,
width: l_search,
+ max_initial_filter_expansion_factor: None,
ivfpq_batch_table_reuse: IvfPqBatchTableReuseMode::Auto,
ivfpq_batch_table_reuse_max_bytes:
DEFAULT_IVFPQ_BATCH_TABLE_REUSE_MAX_BYTES,
}
@@ -1021,11 +1030,23 @@ impl VectorSearchParams {
top_k,
search_width: SearchWidth::Auto,
width: 0,
+ max_initial_filter_expansion_factor: None,
ivfpq_batch_table_reuse: IvfPqBatchTableReuseMode::Auto,
ivfpq_batch_table_reuse_max_bytes:
DEFAULT_IVFPQ_BATCH_TABLE_REUSE_MAX_BYTES,
}
}
+ /// Limits filter-driven expansion of the initial automatic IVF nprobe.
+ ///
+ /// A factor of 1 keeps the unfiltered automatic width. Lower factors
reduce
+ /// initial search work but may reduce recall compared with uncapped
automatic
+ /// search. This setting applies only to automatic IVF search; progressive
+ /// expansion occurs only when fewer than `top_k` filtered results are
found.
+ pub fn with_max_initial_filter_expansion_factor(mut self, factor: usize)
-> Self {
+ self.max_initial_filter_expansion_factor = Some(factor);
+ self
+ }
+
pub fn with_ivfpq_batch_table_reuse(mut self, mode:
IvfPqBatchTableReuseMode) -> Self {
self.ivfpq_batch_table_reuse = mode;
self
@@ -1046,6 +1067,14 @@ impl VectorSearchParams {
fn validate(self) -> io::Result<()> {
validate_positive(self.top_k, "top_k")?;
+ if let Some(factor) = self.max_initial_filter_expansion_factor {
+ validate_positive(factor, "maximum initial filter expansion
factor")?;
+ if self.search_width != SearchWidth::Auto {
+ return Err(invalid_input(
+ "maximum initial filter expansion factor requires
automatic IVF search",
+ ));
+ }
+ }
validate_positive(
self.ivfpq_batch_table_reuse_max_bytes,
"IVF-PQ batch table reuse max bytes",
@@ -1059,7 +1088,13 @@ impl VectorSearchParams {
matching_count: Option<usize>,
) -> io::Result<usize> {
match self.search_width {
- SearchWidth::Auto => infer_ivf_nprobe(nlist, vector_count,
self.top_k, matching_count),
+ SearchWidth::Auto => infer_ivf_nprobe_with_filter_expansion_cap(
+ nlist,
+ vector_count,
+ self.top_k,
+ matching_count,
+ self.max_initial_filter_expansion_factor,
+ ),
SearchWidth::IvfNProbe if self.width > 0 =>
Ok(self.width.min(nlist)),
SearchWidth::IvfNProbe => Err(invalid_input("nprobe must be
greater than 0")),
SearchWidth::DiskAnnLSearch => Err(invalid_input(
@@ -1074,6 +1109,11 @@ impl VectorSearchParams {
}
fn resolve_diskann_l_search_with(self, calibrated: Option<usize>) ->
io::Result<usize> {
+ if self.max_initial_filter_expansion_factor.is_some() {
+ return Err(invalid_input(
+ "maximum initial filter expansion factor is only valid for IVF
indexes",
+ ));
+ }
match self.search_width {
SearchWidth::Auto => Ok(calibrated
.unwrap_or(infer_diskann_l_search(self.top_k)?)
@@ -2932,6 +2972,48 @@ mod tests {
.unwrap_err()
.to_string()
.contains("cannot be used with a DiskANN"));
+ assert!(VectorSearchParams::automatic(10)
+ .with_max_initial_filter_expansion_factor(4)
+ .resolve_diskann_l_search()
+ .unwrap_err()
+ .to_string()
+ .contains("only valid for IVF"));
+ }
+
+ #[test]
+ fn automatic_search_params_configure_initial_filter_expansion_cap() {
+ let params = VectorSearchParams::automatic(3)
+ .with_ivfpq_batch_table_reuse(IvfPqBatchTableReuseMode::Off)
+ .with_ivfpq_batch_table_reuse_max_bytes(128 * 1024 * 1024)
+ .with_max_initial_filter_expansion_factor(4);
+ assert_eq!(params.max_initial_filter_expansion_factor, Some(4));
+ assert_eq!(
+ params.ivfpq_batch_table_reuse,
+ IvfPqBatchTableReuseMode::Off
+ );
+ assert_eq!(params.ivfpq_batch_table_reuse_max_bytes, 128 * 1024 *
1024);
+ assert_eq!(
+ params
+ .resolve_ivf_nprobe(256, 2_560_000, Some(256_000))
+ .unwrap(),
+ 64
+ );
+ }
+
+ #[test]
+ fn initial_filter_expansion_cap_requires_positive_automatic_search() {
+ assert!(VectorSearchParams::automatic(3)
+ .with_max_initial_filter_expansion_factor(0)
+ .validate()
+ .unwrap_err()
+ .to_string()
+ .contains("greater than 0"));
+ assert!(VectorSearchParams::new(3, 16)
+ .with_max_initial_filter_expansion_factor(4)
+ .validate()
+ .unwrap_err()
+ .to_string()
+ .contains("automatic IVF search"));
}
#[test]
@@ -2984,6 +3066,26 @@ mod tests {
assert_eq!(result.0, vec![7, 8]);
}
+ #[test]
+ fn capped_automatic_filtered_search_can_expand_past_the_initial_cap() {
+ let params =
VectorSearchParams::automatic(2).with_max_initial_filter_expansion_factor(4);
+ let initial_nprobe = params
+ .resolve_ivf_nprobe(256, 2_560_000, Some(256_000))
+ .unwrap();
+ let mut observed = Vec::new();
+ let result = progressive_ivf_search(params, 256, initial_nprobe, 1, 2,
256_000, |nprobe| {
+ observed.push(nprobe);
+ if nprobe < 128 {
+ Ok((vec![7, -1], vec![1.0, f32::MAX]))
+ } else {
+ Ok((vec![7, 8], vec![1.0, 2.0]))
+ }
+ })
+ .unwrap();
+ assert_eq!(observed, vec![64, 128]);
+ assert_eq!(result.0, vec![7, 8]);
+ }
+
#[test]
fn automatic_search_treats_negative_row_ids_as_valid_results() {
let mut observed = Vec::new();
diff --git a/docs/api.html b/docs/api.html
index 0fdde75..af7063d 100644
--- a/docs/api.html
+++ b/docs/api.html
@@ -56,6 +56,7 @@
<div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Applies
to</th><th>Description</th></tr></thead><tbody>
<tr><td><code>top_k</code></td><td>All indexes</td><td>Number of
nearest neighbors returned for each query.</td></tr>
<tr><td><code>Auto</code> width</td><td>All indexes</td><td>IVF
starts at <code>max(8, ceil(nlist/16))</code>, adds enough average-list
capacity for at least <code>4 × top_k</code> candidates, scales for filter
selectivity, and progressively doubles when a filtered result is short. DiskANN
uses a calibrated width when present, otherwise <code>max(100, 2 ×
top_k)</code>.</td></tr>
+
<tr><td><code>max_initial_filter_expansion_factor</code></td><td>Automatic IVF
search in Rust and Java</td><td>Optional cap on inverse-selectivity expansion
of the initial <code>nprobe</code>. Unset preserves the current unlimited
behavior. A value of 1 keeps the unfiltered automatic width. Lower factors
reduce initial search work but may reduce recall compared with uncapped
automatic search. Progressive expansion occurs only when fewer than
<code>top_k</code> valid results a [...]
<tr><td><code>nprobe</code></td><td>IVF families</td><td>Explicit
expert override for the number of lists probed. A tagged IVF width is rejected
by DiskANN rather than silently ignored.</td></tr>
<tr><td><code>l_search</code></td><td>DiskANN</td><td>Explicit
expert override for graph search-list size; it is clamped to at least
<code>top_k</code>. A tagged DiskANN width is rejected by IVF indexes.</td></tr>
</tbody></table></div>
@@ -118,6 +119,10 @@ let mut reader = VectorIndexReader::open(file)?;
reader.optimize_for_search()?;
let params = VectorSearchParams::automatic(10);
let (ids, distances) = reader.search(&query, params)?;</code></pre></div>
+ <div class="code-block"><span class="code-label">Rust · optional
filtered-IVF tuning</span><pre><code>// Example only: select the factor using
workload-specific
+// latency and Recall@K measurements.
+let params = VectorSearchParams::automatic(10)
+ .with_max_initial_filter_expansion_factor(4);</code></pre></div>
<div class="code-block"><span class="code-label">Rust · other
configurations</span><pre><code>VectorIndexConfig::IvfFlat {
dimension: 128, nlist: 1024, metric: MetricType::L2,
};
@@ -217,8 +222,14 @@ try (VectorIndexReader reader = new
VectorIndexReader(vectorIndexInput)) {
VectorIndexMetadata metadata = reader.metadata();
reader.optimizeForSearch();
VectorSearchResult result = reader.search(
- query, VectorSearchParams.automatic(10));
+ query,
+ VectorSearchParams.automatic(10));
}</code></pre></div>
+ <div class="code-block"><span class="code-label">Java · optional
filtered-IVF tuning</span><pre><code>// Example only: select the factor using
workload-specific
+// latency and Recall@K measurements.
+VectorSearchParams params =
+ VectorSearchParams.automatic(10)
+ .withMaxInitialFilterExpansionFactor(4);</code></pre></div>
<h3>Batching a large training set</h3>
<div class="code-block"><span class="code-label">Java ·
Trainer-owned native staging</span><pre><code>try (VectorIndexTrainer trainer =
VectorIndexTrainer.create(options)) {
for (float[] batch : trainingBatches) {
diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs
index 807c322..31bcc36 100644
--- a/ffi/src/lib.rs
+++ b/ffi/src/lib.rs
@@ -541,6 +541,7 @@ fn search_params_from_ffi(params: PaimonVindexSearchParams)
-> Result<VectorSear
top_k: params.top_k,
search_width,
width: params.width,
+ max_initial_filter_expansion_factor: None,
ivfpq_batch_table_reuse: IvfPqBatchTableReuseMode::Auto,
ivfpq_batch_table_reuse_max_bytes:
DEFAULT_IVFPQ_BATCH_TABLE_REUSE_MAX_BYTES,
})
diff --git
a/java/src/main/java/org/apache/paimon/index/vector/VectorSearchParams.java
b/java/src/main/java/org/apache/paimon/index/vector/VectorSearchParams.java
index a491af5..d5aa4f5 100644
--- a/java/src/main/java/org/apache/paimon/index/vector/VectorSearchParams.java
+++ b/java/src/main/java/org/apache/paimon/index/vector/VectorSearchParams.java
@@ -28,6 +28,7 @@ public final class VectorSearchParams {
private final int topK;
private final int searchWidth;
private final int width;
+ private final int maxInitialFilterExpansionFactor;
private final int ivfPqBatchTableReuseMode;
private final long ivfPqBatchTableReuseMaxBytes;
@@ -36,6 +37,7 @@ public final class VectorSearchParams {
topK,
SEARCH_WIDTH_IVF_NPROBE,
nprobe,
+ 0,
IvfPqBatchTableReuseMode.AUTO.code(),
DEFAULT_IVF_PQ_BATCH_TABLE_REUSE_MAX_BYTES);
}
@@ -44,11 +46,13 @@ public final class VectorSearchParams {
int topK,
int searchWidth,
int width,
+ int maxInitialFilterExpansionFactor,
int ivfPqBatchTableReuseMode,
long ivfPqBatchTableReuseMaxBytes) {
this.topK = topK;
this.searchWidth = searchWidth;
this.width = width;
+ this.maxInitialFilterExpansionFactor = maxInitialFilterExpansionFactor;
this.ivfPqBatchTableReuseMode = ivfPqBatchTableReuseMode;
this.ivfPqBatchTableReuseMaxBytes = ivfPqBatchTableReuseMaxBytes;
}
@@ -58,6 +62,7 @@ public final class VectorSearchParams {
topK,
SEARCH_WIDTH_AUTO,
0,
+ 0,
IvfPqBatchTableReuseMode.AUTO.code(),
DEFAULT_IVF_PQ_BATCH_TABLE_REUSE_MAX_BYTES);
}
@@ -67,6 +72,7 @@ public final class VectorSearchParams {
topK,
SEARCH_WIDTH_IVF_NPROBE,
nprobe,
+ 0,
IvfPqBatchTableReuseMode.AUTO.code(),
DEFAULT_IVF_PQ_BATCH_TABLE_REUSE_MAX_BYTES);
}
@@ -76,6 +82,7 @@ public final class VectorSearchParams {
topK,
SEARCH_WIDTH_DISKANN_L_SEARCH,
lSearch,
+ 0,
IvfPqBatchTableReuseMode.AUTO.code(),
DEFAULT_IVF_PQ_BATCH_TABLE_REUSE_MAX_BYTES);
}
@@ -92,6 +99,35 @@ public final class VectorSearchParams {
return width;
}
+ int maxInitialFilterExpansionFactor() {
+ return maxInitialFilterExpansionFactor;
+ }
+
+ /**
+ * Limits filter-driven expansion of the initial automatic IVF nprobe.
+ *
+ * <p>A factor of 1 keeps the unfiltered automatic width. Lower factors
reduce initial search
+ * work but may reduce recall compared with uncapped automatic search.
Progressive expansion
+ * occurs only when fewer than {@code topK} filtered results are found.
+ */
+ public VectorSearchParams withMaxInitialFilterExpansionFactor(int factor) {
+ if (factor <= 0) {
+ throw new IllegalArgumentException(
+ "Maximum initial filter expansion factor must be greater
than 0");
+ }
+ if (searchWidth != SEARCH_WIDTH_AUTO) {
+ throw new IllegalStateException(
+ "Maximum initial filter expansion factor requires
automatic IVF search");
+ }
+ return new VectorSearchParams(
+ topK,
+ searchWidth,
+ width,
+ factor,
+ ivfPqBatchTableReuseMode,
+ ivfPqBatchTableReuseMaxBytes);
+ }
+
public IvfPqBatchTableReuseMode ivfPqBatchTableReuse() {
return IvfPqBatchTableReuseMode.fromCode(ivfPqBatchTableReuseMode);
}
@@ -109,7 +145,12 @@ public final class VectorSearchParams {
throw new IllegalArgumentException("IVF-PQ batch table reuse mode
is null");
}
return new VectorSearchParams(
- topK, searchWidth, width, mode.code(),
ivfPqBatchTableReuseMaxBytes);
+ topK,
+ searchWidth,
+ width,
+ maxInitialFilterExpansionFactor,
+ mode.code(),
+ ivfPqBatchTableReuseMaxBytes);
}
public VectorSearchParams withIvfPqBatchTableReuse(String mode) {
@@ -122,7 +163,12 @@ public final class VectorSearchParams {
"IVF-PQ batch table reuse max bytes must be positive");
}
return new VectorSearchParams(
- topK, searchWidth, width, ivfPqBatchTableReuseMode, maxBytes);
+ topK,
+ searchWidth,
+ width,
+ maxInitialFilterExpansionFactor,
+ ivfPqBatchTableReuseMode,
+ maxBytes);
}
public VectorSearchParams withLSearch(int lSearch) {
@@ -130,6 +176,7 @@ public final class VectorSearchParams {
topK,
SEARCH_WIDTH_DISKANN_L_SEARCH,
lSearch,
+ 0,
ivfPqBatchTableReuseMode,
ivfPqBatchTableReuseMaxBytes);
}
diff --git
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java
index fed0feb..761762e 100644
---
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java
+++
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexJavaApiTest.java
@@ -29,6 +29,7 @@ public class VectorIndexJavaApiTest {
testBatchResultCopiesArraysAndSlicesRows();
testMetadata();
testSearchParametersRemainAlgorithmSpecific();
+ testAutomaticInitialFilterExpansionFactor();
testIvfPqBatchTableReuseMode();
testReaderRejectsNegativeAdjacencyCacheBudget();
testClosedReaderRejectsOperations();
@@ -71,6 +72,42 @@ public class VectorIndexJavaApiTest {
new VectorSearchParams(10, 4).searchWidth());
}
+ private static void testAutomaticInitialFilterExpansionFactor() {
+ VectorSearchParams defaults = VectorSearchParams.automatic(10);
+ assertEquals(0, defaults.maxInitialFilterExpansionFactor());
+
+ VectorSearchParams capped =
+ defaults
+ .withIvfPqBatchTableReuse(IvfPqBatchTableReuseMode.ON)
+ .withIvfPqBatchTableReuseMaxBytes(128L * 1024 * 1024)
+ .withMaxInitialFilterExpansionFactor(4);
+ assertEquals(4, capped.maxInitialFilterExpansionFactor());
+ assertEquals(IvfPqBatchTableReuseMode.ON,
capped.ivfPqBatchTableReuse());
+ assertEquals(128L * 1024 * 1024,
capped.ivfPqBatchTableReuseMaxBytes());
+
+ VectorSearchParams diskAnn = capped.withLSearch(200);
+ assertEquals(0, diskAnn.maxInitialFilterExpansionFactor());
+ assertEquals(IvfPqBatchTableReuseMode.ON,
diskAnn.ivfPqBatchTableReuse());
+ assertEquals(128L * 1024 * 1024,
diskAnn.ivfPqBatchTableReuseMaxBytes());
+
+ assertThrows(
+ IllegalArgumentException.class,
+ new ThrowingRunnable() {
+ @Override
+ public void run() {
+ defaults.withMaxInitialFilterExpansionFactor(0);
+ }
+ });
+ assertThrows(
+ IllegalStateException.class,
+ new ThrowingRunnable() {
+ @Override
+ public void run() {
+ VectorSearchParams.ivf(10,
16).withMaxInitialFilterExpansionFactor(4);
+ }
+ });
+ }
+
private static void testIvfPqBatchTableReuseMode() {
VectorSearchParams defaults = new VectorSearchParams(10, 4);
assertEquals(IvfPqBatchTableReuseMode.AUTO,
defaults.ivfPqBatchTableReuse());
diff --git
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java
index 06cbff9..a7fd445 100644
---
a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java
+++
b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeValidationTest.java
@@ -120,7 +120,10 @@ public class VectorIndexNativeValidationTest {
assertEquals(ROUNDTRIP_DIMENSION, metadata.dimension());
assertEquals(8, metadata.nlist());
VectorSearchResult result =
- reader.search(queryForCenter(0.0f),
VectorSearchParams.automatic(2));
+ reader.search(
+ queryForCenter(0.0f),
+ VectorSearchParams.automatic(2)
+ .withMaxInitialFilterExpansionFactor(4));
assertIdInCluster(result.ids()[0], 0);
} finally {
reader.close();
@@ -517,6 +520,18 @@ public class VectorIndexNativeValidationTest {
if (plan.randomReadLatencyNanos() <= 0 || plan.windowBytes()
<= 0) {
throw new AssertionError("DiskANN read plan was not
resolved during open");
}
+ assertThrowsMessage(
+ RuntimeException.class,
+ "only valid for IVF",
+ new ThrowingRunnable() {
+ @Override
+ public void run() {
+ reader.search(
+ queryForCenter(0.0f),
+ VectorSearchParams.automatic(1)
+
.withMaxInitialFilterExpansionFactor(4));
+ }
+ });
}
reader.optimizeForSearch();
diff --git a/jni/src/lib.rs b/jni/src/lib.rs
index 9fb9afa..db6eb38 100644
--- a/jni/src/lib.rs
+++ b/jni/src/lib.rs
@@ -413,6 +413,9 @@ fn search_params(env: &mut JNIEnv, params: JObject) ->
Result<VectorSearchParams
let top_k = call_int_method(env, ¶ms, "topK")?;
let search_width = call_int_method(env, ¶ms, "searchWidth")?;
let width = call_int_method(env, ¶ms, "width")?;
+ let max_initial_filter_expansion_factor =
max_initial_filter_expansion_factor(
+ call_int_method(env, ¶ms, "maxInitialFilterExpansionFactor")?,
+ )?;
let ivfpq_batch_table_reuse =
ivfpq_batch_table_reuse_mode(call_int_method(env, ¶ms,
"ivfPqBatchTableReuseMode")?)?;
let ivfpq_batch_table_reuse_max_bytes = positive_jlong_to_usize(
@@ -438,11 +441,22 @@ fn search_params(env: &mut JNIEnv, params: JObject) ->
Result<VectorSearchParams
top_k: top_k as usize,
search_width,
width: width as usize,
+ max_initial_filter_expansion_factor,
ivfpq_batch_table_reuse,
ivfpq_batch_table_reuse_max_bytes,
})
}
+fn max_initial_filter_expansion_factor(value: jint) -> Result<Option<usize>,
String> {
+ match value {
+ 0 => Ok(None),
+ value if value > 0 => Ok(Some(value as usize)),
+ value => Err(format!(
+ "invalid maximum initial filter expansion factor: {value}"
+ )),
+ }
+}
+
fn ivfpq_batch_table_reuse_mode(code: jint) ->
Result<IvfPqBatchTableReuseMode, String> {
match code {
0 => Ok(IvfPqBatchTableReuseMode::Off),
@@ -1084,4 +1098,11 @@ mod tests {
assert!(positive_jlong_to_usize(0, "reuse max bytes").is_err());
assert!(positive_jlong_to_usize(-1, "reuse max bytes").is_err());
}
+
+ #[test]
+ fn initial_filter_expansion_factor_maps_zero_to_unlimited() {
+ assert_eq!(max_initial_filter_expansion_factor(0).unwrap(), None);
+ assert_eq!(max_initial_filter_expansion_factor(4).unwrap(), Some(4));
+ assert!(max_initial_filter_expansion_factor(-1).is_err());
+ }
}
