This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 9cd773a8 feat(vindex): add DiskANN and IVF-SQ/RQ support (#726)
9cd773a8 is described below
commit 9cd773a87426b47c3961e4a079758503f0b218ca
Author: jerry <[email protected]>
AuthorDate: Tue Aug 18 15:16:35 2026 +0800
feat(vindex): add DiskANN and IVF-SQ/RQ support (#726)
---
crates/integrations/datafusion/src/procedures.rs | 2 +-
crates/integrations/datafusion/tests/procedures.rs | 2 +-
crates/paimon/src/table/global_index_types.rs | 22 +-
crates/paimon/src/table/vector_search_builder.rs | 42 ++-
crates/paimon/src/vindex/mod.rs | 403 +++++++++++++++++++--
crates/paimon/src/vindex/reader.rs | 204 +++++++++--
docs/src/sql.md | 119 +++++-
7 files changed, 714 insertions(+), 80 deletions(-)
diff --git a/crates/integrations/datafusion/src/procedures.rs
b/crates/integrations/datafusion/src/procedures.rs
index 672300ae..91074351 100644
--- a/crates/integrations/datafusion/src/procedures.rs
+++ b/crates/integrations/datafusion/src/procedures.rs
@@ -585,7 +585,7 @@ async fn proc_create_global_index(
// Echo the raw argument, not the normalized one, so a typo stays
visible.
return Err(DataFusionError::NotImplemented(format!(
"create_global_index only supports index_type => 'btree',
'bitmap', or vindex types \
- ('ivf-flat', 'ivf-pq'), got '{index_type_arg}'"
+ ('ivf-flat', 'ivf-pq', 'ivf-sq', 'ivf-rq', 'diskann'), got
'{index_type_arg}'"
)));
}
ok_result(ctx)
diff --git a/crates/integrations/datafusion/tests/procedures.rs
b/crates/integrations/datafusion/tests/procedures.rs
index 2f0c2d1f..458567d8 100644
--- a/crates/integrations/datafusion/tests/procedures.rs
+++ b/crates/integrations/datafusion/tests/procedures.rs
@@ -198,7 +198,7 @@ async fn
test_create_global_index_rejects_unsupported_index_types() {
index_type => '{index_type}'\
)"
),
- "only supports index_type => 'btree', 'bitmap', or vindex types
('ivf-flat', 'ivf-pq')",
+ "only supports index_type => 'btree', 'bitmap', or vindex types
('ivf-flat', 'ivf-pq', 'ivf-sq', 'ivf-rq', 'diskann')",
)
.await;
}
diff --git a/crates/paimon/src/table/global_index_types.rs
b/crates/paimon/src/table/global_index_types.rs
index eb83ad72..2fa65e54 100644
--- a/crates/paimon/src/table/global_index_types.rs
+++ b/crates/paimon/src/table/global_index_types.rs
@@ -16,7 +16,10 @@
// under the License.
use crate::lumina::{is_lumina_index_type, LUMINA_IDENTIFIER};
-use crate::vindex::{is_vindex_index_type, IVF_FLAT_IDENTIFIER,
IVF_PQ_IDENTIFIER};
+use crate::vindex::{
+ is_vindex_index_type, DISKANN_IDENTIFIER, IVF_FLAT_IDENTIFIER,
IVF_PQ_IDENTIFIER,
+ IVF_RQ_IDENTIFIER, IVF_SQ_IDENTIFIER,
+};
pub(crate) const BTREE_GLOBAL_INDEX_TYPE: &str = "btree";
pub(crate) const BITMAP_GLOBAL_INDEX_TYPE: &str = "bitmap";
@@ -35,7 +38,7 @@ pub(crate) fn normalize_sorted_global_index_type(index_type:
&str) -> Option<&'s
/// Used verbatim in the unsupported-type error of both the builder and the
/// DataFusion procedure so the two messages stay in sync.
pub const SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP: &str =
- "btree, bitmap, lumina, lumina-vector-ann, ivf-flat, ivf-pq";
+ "btree, bitmap, lumina, lumina-vector-ann, ivf-flat, ivf-pq, ivf-sq,
ivf-rq, diskann";
/// Canonicalize any supported global index type to a stable `&'static str`, or
/// `None` if unsupported. Case-insensitive. Order: sorted -> lumina -> vindex.
@@ -66,6 +69,9 @@ fn canonical_vindex_identifier(lowered: &str) ->
Option<&'static str> {
match lowered {
IVF_FLAT_IDENTIFIER => Some(IVF_FLAT_IDENTIFIER),
IVF_PQ_IDENTIFIER => Some(IVF_PQ_IDENTIFIER),
+ IVF_SQ_IDENTIFIER => Some(IVF_SQ_IDENTIFIER),
+ IVF_RQ_IDENTIFIER => Some(IVF_RQ_IDENTIFIER),
+ DISKANN_IDENTIFIER => Some(DISKANN_IDENTIFIER),
_ => None,
}
}
@@ -109,6 +115,18 @@ mod tests {
normalize_global_index_type_for_drop("IVF-PQ"),
Some("ivf-pq")
);
+ assert_eq!(
+ normalize_global_index_type_for_drop("IVF-SQ"),
+ Some("ivf-sq")
+ );
+ assert_eq!(
+ normalize_global_index_type_for_drop("IVF-RQ"),
+ Some("ivf-rq")
+ );
+ assert_eq!(
+ normalize_global_index_type_for_drop("DiskANN"),
+ Some("diskann")
+ );
}
#[test]
diff --git a/crates/paimon/src/table/vector_search_builder.rs
b/crates/paimon/src/table/vector_search_builder.rs
index 5919b4c7..12e60d3b 100644
--- a/crates/paimon/src/table/vector_search_builder.rs
+++ b/crates/paimon/src/table/vector_search_builder.rs
@@ -2085,10 +2085,20 @@ fn pk_vector_query_dimension(
let dimension =
LuminaVectorIndexOptions::new(&merged)?.dimension;
Ok(Some(dimension as usize))
} else {
+ let mut dimension_options = HashMap::new();
+ for key in [
+ "dimension".to_string(),
+ format!("{index_type}.dimension"),
+ format!("fields.{}.dimension", vector_field.name()),
+ ] {
+ if let Some(value) = query_options.get(&key) {
+ dimension_options.insert(key, value.clone());
+ }
+ }
Ok(Some(
VindexVectorIndexOptions::new(
table_options,
- query_options,
+ &dimension_options,
index_type,
vector_field,
)?
@@ -3453,6 +3463,28 @@ mod tests {
assert_eq!(vindex_index_parallelism(4, 8), 4);
}
+ #[test]
+ fn vindex_array_dimension_accepts_diskann_search_options() {
+ let field = DataField::new(
+ 1,
+ "embedding".to_string(),
+ DataType::Array(ArrayType::new(DataType::Float(FloatType::new()))),
+ );
+ let query_options = HashMap::from([
+ ("diskann.dimension".to_string(), "8".to_string()),
+ ("diskann.l_search".to_string(), "64".to_string()),
+ (
+ "vindex.reader.memory-budget-bytes".to_string(),
+ "1048576".to_string(),
+ ),
+ ]);
+
+ assert_eq!(
+ pk_vector_query_dimension(&HashMap::new(), &query_options,
"diskann", &field).unwrap(),
+ Some(8)
+ );
+ }
+
fn make_field(id: i32, name: &str) -> DataField {
DataField::new(id, name.to_string(), DataType::Int(IntType::default()))
}
@@ -4933,8 +4965,12 @@ mod tests {
VectorIndexBackend::from_index_type("ivf-flat"),
Some(VectorIndexBackend::Vindex)
);
- // `diskann` is Lumina's internal index type, not a top-level index
type.
- assert_eq!(VectorIndexBackend::from_index_type("diskann"), None);
+ for index_type in ["ivf-sq", "ivf-rq", "diskann"] {
+ assert_eq!(
+ VectorIndexBackend::from_index_type(index_type),
+ Some(VectorIndexBackend::Vindex)
+ );
+ }
}
#[test]
diff --git a/crates/paimon/src/vindex/mod.rs b/crates/paimon/src/vindex/mod.rs
index da89d257..8ec93a13 100644
--- a/crates/paimon/src/vindex/mod.rs
+++ b/crates/paimon/src/vindex/mod.rs
@@ -30,6 +30,9 @@ use std::sync::OnceLock;
pub const IVF_FLAT_IDENTIFIER: &str = "ivf-flat";
pub const IVF_PQ_IDENTIFIER: &str = "ivf-pq";
+pub const IVF_SQ_IDENTIFIER: &str = "ivf-sq";
+pub const IVF_RQ_IDENTIFIER: &str = "ivf-rq";
+pub const DISKANN_IDENTIFIER: &str = "diskann";
const DEFAULT_DIMENSION: &str = "128";
const DEFAULT_METRIC: &str = "inner_product";
@@ -38,6 +41,23 @@ const DEFAULT_PQ_M: &str = "16";
const DEFAULT_PQ_USE_OPQ: &str = "false";
const DEFAULT_TRAIN_SAMPLE_RATIO: f64 = 1.0;
const VECTOR_SEARCH_TIMING_ENV: &str = "PAIMON_LOG_VECTOR_SEARCH_TIMING";
+const DISKANN_OPTION_KEYS: &[(&str, &str)] = &[
+ ("deployment-profile", "deployment-profile"),
+ ("target-recall", "target-recall"),
+ ("max-bytes-per-vector", "max-bytes-per-vector"),
+ ("pq.code-ratio", "pq.code-ratio"),
+ ("pq.m", "pq.m"),
+ ("pq.bits", "pq.bits"),
+ ("diskann.build-preset", "build-preset"),
+ ("diskann.seed", "seed"),
+ ("diskann.memory-budget-bytes", "memory-budget-bytes"),
+ ("diskann.max-degree", "max-degree"),
+ ("diskann.build-search-list-size", "build-search-list-size"),
+ ("diskann.alpha", "alpha"),
+ ("diskann.storage-layout", "storage-layout"),
+ ("diskann.raw-vector-encoding", "raw-vector-encoding"),
+ ("diskann.build-distance", "build-distance"),
+];
#[cfg(test)]
static VECTOR_SEARCH_TIMING_TEST_GUARDS: AtomicUsize = AtomicUsize::new(0);
@@ -68,13 +88,23 @@ pub(crate) fn vector_search_timing_enabled() -> bool {
}
pub fn is_vindex_index_type(index_type: &str) -> bool {
- matches!(index_type, IVF_FLAT_IDENTIFIER | IVF_PQ_IDENTIFIER)
+ matches!(
+ index_type,
+ IVF_FLAT_IDENTIFIER
+ | IVF_PQ_IDENTIFIER
+ | IVF_SQ_IDENTIFIER
+ | IVF_RQ_IDENTIFIER
+ | DISKANN_IDENTIFIER
+ )
}
pub(crate) fn native_index_type(index_type: &str) -> Option<&'static str> {
match index_type {
IVF_FLAT_IDENTIFIER => Some("ivf_flat"),
IVF_PQ_IDENTIFIER => Some("ivf_pq"),
+ IVF_SQ_IDENTIFIER => Some("ivf_sq"),
+ IVF_RQ_IDENTIFIER => Some("ivf_rq"),
+ DISKANN_IDENTIFIER => Some("diskann"),
_ => None,
}
}
@@ -108,18 +138,20 @@ impl VindexVectorIndexOptions {
"dimension".to_string(),
resolve_dimension(table_options, user_options, index_type, field)?,
);
- native_options.insert(
- "nlist".to_string(),
- option_value(
- table_options,
- user_options,
- field.name(),
- index_type,
- "nlist",
- "nlist",
- DEFAULT_NLIST,
- ),
- );
+ if index_type != DISKANN_IDENTIFIER {
+ native_options.insert(
+ "nlist".to_string(),
+ option_value(
+ table_options,
+ user_options,
+ field.name(),
+ index_type,
+ "nlist",
+ "nlist",
+ DEFAULT_NLIST,
+ ),
+ );
+ }
native_options.insert(
"metric".to_string(),
normalize_metric(&option_value(
@@ -159,6 +191,34 @@ impl VindexVectorIndexOptions {
),
);
}
+ if index_type == IVF_RQ_IDENTIFIER {
+ for key in ["rq.bits", "max-bytes-per-vector"] {
+ if let Some(value) = optional_value(
+ table_options,
+ user_options,
+ field.name(),
+ index_type,
+ key,
+ key,
+ ) {
+ native_options.insert(key.to_string(), value);
+ }
+ }
+ }
+ if index_type == DISKANN_IDENTIFIER {
+ for &(native_key, paimon_suffix) in DISKANN_OPTION_KEYS {
+ if let Some(value) = optional_value(
+ table_options,
+ user_options,
+ field.name(),
+ index_type,
+ native_key,
+ paimon_suffix,
+ ) {
+ native_options.insert(native_key.to_string(), value);
+ }
+ }
+ }
let config =
VectorIndexConfig::from_options(&native_options).map_err(|e| {
crate::Error::DataInvalid {
@@ -249,17 +309,40 @@ fn is_supported_user_option_key(key: &str, index_type:
&str, field_name: &str) -
fn is_allowed_native_key(key: &str, index_type: &str) -> bool {
match key {
- "dimension" | "nlist" | "metric" => true,
- "pq.m" | "use-opq" => index_type == IVF_PQ_IDENTIFIER,
- _ => false,
+ "dimension" | "metric" => true,
+ "nlist" => index_type != DISKANN_IDENTIFIER,
+ "use-opq" => index_type == IVF_PQ_IDENTIFIER,
+ "rq.bits" => index_type == IVF_RQ_IDENTIFIER,
+ "max-bytes-per-vector" => {
+ matches!(index_type, IVF_RQ_IDENTIFIER | DISKANN_IDENTIFIER)
+ }
+ "pq.m" if index_type == IVF_PQ_IDENTIFIER => true,
+ _ => {
+ index_type == DISKANN_IDENTIFIER
+ && DISKANN_OPTION_KEYS
+ .iter()
+ .any(|(native_key, _)| *native_key == key)
+ }
}
}
fn is_allowed_paimon_suffix(suffix: &str, index_type: &str) -> bool {
match suffix {
- "dimension" | "nlist" | "distance.metric" | "train.sample-ratio" =>
true,
- "pq.m" | "pq.use-opq" => index_type == IVF_PQ_IDENTIFIER,
- _ => false,
+ "dimension" | "distance.metric" => true,
+ "nlist" => index_type != DISKANN_IDENTIFIER,
+ "train.sample-ratio" => true,
+ "pq.use-opq" => index_type == IVF_PQ_IDENTIFIER,
+ "rq.bits" => index_type == IVF_RQ_IDENTIFIER,
+ "max-bytes-per-vector" => {
+ matches!(index_type, IVF_RQ_IDENTIFIER | DISKANN_IDENTIFIER)
+ }
+ "pq.m" if index_type == IVF_PQ_IDENTIFIER => true,
+ _ => {
+ index_type == DISKANN_IDENTIFIER
+ && DISKANN_OPTION_KEYS
+ .iter()
+ .any(|(_, paimon_suffix)| *paimon_suffix == suffix)
+ }
}
}
@@ -374,6 +457,11 @@ fn normalize_metric(metric: &str) -> String {
mod tests {
use super::*;
use crate::spec::{ArrayType, FloatType, VectorType};
+ use paimon_vindex_core::index::{
+ IndexType, VectorIndexMetadata, VectorIndexReader, VectorIndexTrainer,
VectorIndexWriter,
+ };
+ use paimon_vindex_core::io::PosWriter;
+ use std::io::Cursor;
fn array_float_field() -> DataField {
DataField::new(
@@ -383,10 +471,46 @@ mod tests {
)
}
+ fn roundtrip_metadata(index_type: &str, user_options: &[(&str, &str)]) ->
VectorIndexMetadata {
+ let user_options = user_options
+ .iter()
+ .map(|(key, value)| ((*key).to_string(), (*value).to_string()))
+ .collect();
+ let options = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &user_options,
+ index_type,
+ &array_float_field(),
+ )
+ .unwrap();
+ let n = 512;
+ let dimension = options.config.dimension();
+ let data = (0..n * dimension)
+ .map(|offset| {
+ let row = offset / dimension;
+ let column = offset % dimension;
+ (row % 4) as f32 * 20.0 + column as f32 * 0.01 + row as f32 *
0.0001
+ })
+ .collect::<Vec<_>>();
+ let training = VectorIndexTrainer::train(options.config, &data,
n).unwrap();
+ let mut writer = VectorIndexWriter::new(training);
+ writer
+ .add_vectors(&(0..n as i64).collect::<Vec<_>>(), &data, n)
+ .unwrap();
+ let mut bytes = Vec::new();
+ writer.write(&mut PosWriter::new(&mut bytes)).unwrap();
+ VectorIndexReader::open(Cursor::new(bytes))
+ .unwrap()
+ .metadata()
+ }
+
#[test]
fn test_vindex_index_type_identifier_helper() {
assert!(is_vindex_index_type(IVF_FLAT_IDENTIFIER));
assert!(is_vindex_index_type(IVF_PQ_IDENTIFIER));
+ assert!(is_vindex_index_type("ivf-sq"));
+ assert!(is_vindex_index_type("ivf-rq"));
+ assert!(is_vindex_index_type("diskann"));
assert!(!is_vindex_index_type("ivf-hnsw-flat"));
assert!(!is_vindex_index_type("ivf-hnsw-sq"));
assert!(!is_vindex_index_type(""));
@@ -433,6 +557,201 @@ mod tests {
);
}
+ #[test]
+ fn test_vindex_options_map_ivf_rq_bits() {
+ let user_options = HashMap::from([
+ ("dimension".to_string(), "8".to_string()),
+ ("nlist".to_string(), "4".to_string()),
+ ("rq.bits".to_string(), "3".to_string()),
+ ]);
+
+ let options = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &user_options,
+ IVF_RQ_IDENTIFIER,
+ &array_float_field(),
+ )
+ .unwrap();
+
+ assert_eq!(
+ options.native_options.get("rq.bits").map(String::as_str),
+ Some("3")
+ );
+ assert_eq!(options.config.resolved().rq_bits, Some(3));
+
+ let defaults = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &HashMap::new(),
+ IVF_RQ_IDENTIFIER,
+ &array_float_field(),
+ )
+ .unwrap();
+ assert_eq!(defaults.config.resolved().rq_bits, Some(4));
+ assert!(!defaults.native_options.contains_key("rq.bits"));
+
+ let capacity_goal = HashMap::from([
+ ("dimension".to_string(), "100".to_string()),
+ ("nlist".to_string(), "16".to_string()),
+ ("ivf-rq.max-bytes-per-vector".to_string(), "88".to_string()),
+ ]);
+ let inferred = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &capacity_goal,
+ IVF_RQ_IDENTIFIER,
+ &array_float_field(),
+ )
+ .unwrap();
+ assert_eq!(inferred.config.resolved().rq_bits, Some(3));
+ assert_eq!(
+ inferred
+ .native_options
+ .get("max-bytes-per-vector")
+ .map(String::as_str),
+ Some("88")
+ );
+ assert!(!inferred.native_options.contains_key("rq.bits"));
+
+ let invalid = HashMap::from([("ivf-rq.rq.bits".to_string(),
"9".to_string())]);
+ let error = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &invalid,
+ IVF_RQ_IDENTIFIER,
+ &array_float_field(),
+ )
+ .expect_err("rq.bits outside 1..=8 must be rejected");
+ assert!(error.to_string().contains("rq.bits"));
+ }
+
+ #[test]
+ fn test_vindex_options_map_diskann_config() {
+ let user_options = HashMap::from([
+ ("diskann.dimension".to_string(), "128".to_string()),
+ ("diskann.distance.metric".to_string(), "l2".to_string()),
+ (
+ "diskann.deployment-profile".to_string(),
+ "local_storage".to_string(),
+ ),
+ ("diskann.target-recall".to_string(), "0.9".to_string()),
+ (
+ "diskann.max-bytes-per-vector".to_string(),
+ "1024".to_string(),
+ ),
+ ("diskann.pq.code-ratio".to_string(), "0.125".to_string()),
+ ("diskann.pq.m".to_string(), "16".to_string()),
+ ("diskann.pq.bits".to_string(), "4".to_string()),
+ ("diskann.build-preset".to_string(), "balanced".to_string()),
+ ("diskann.seed".to_string(), "7".to_string()),
+ ("diskann.max-degree".to_string(), "32".to_string()),
+ (
+ "diskann.build-search-list-size".to_string(),
+ "64".to_string(),
+ ),
+ ("diskann.alpha".to_string(), "1.4".to_string()),
+ (
+ "diskann.memory-budget-bytes".to_string(),
+ "123456".to_string(),
+ ),
+ (
+ "diskann.storage-layout".to_string(),
+ "interleaved".to_string(),
+ ),
+ ("diskann.raw-vector-encoding".to_string(), "f16".to_string()),
+ (
+ "diskann.build-distance".to_string(),
+ "full_precision".to_string(),
+ ),
+ ]);
+
+ let options = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &user_options,
+ DISKANN_IDENTIFIER,
+ &array_float_field(),
+ )
+ .unwrap();
+ let resolved = options.config.resolved();
+
+ assert_eq!(resolved.pq_m, Some(16));
+ assert_eq!(resolved.pq_bits, Some(4));
+ let build = resolved.diskann_build.unwrap();
+ assert_eq!(build.max_degree, 32);
+ assert_eq!(build.build_search_list_size, 64);
+ assert_eq!(build.alpha, 1.4);
+ assert!(!options.native_options.contains_key("nlist"));
+ for key in [
+ "deployment-profile",
+ "target-recall",
+ "max-bytes-per-vector",
+ "pq.code-ratio",
+ "diskann.build-preset",
+ "diskann.seed",
+ "diskann.storage-layout",
+ "diskann.raw-vector-encoding",
+ "diskann.build-distance",
+ ] {
+ assert!(options.native_options.contains_key(key), "missing {key}");
+ }
+ }
+
+ #[test]
+ fn test_new_index_types_roundtrip_metadata() {
+ let sq = roundtrip_metadata(
+ IVF_SQ_IDENTIFIER,
+ &[("ivf-sq.dimension", "8"), ("ivf-sq.nlist", "4")],
+ );
+ assert_eq!(sq.index_type, IndexType::IvfSq);
+ assert_eq!(
+ (sq.dimension, sq.nlist, sq.total_vectors, sq.pq_bits),
+ (8, 4, 512, Some(8))
+ );
+
+ let rq = roundtrip_metadata(
+ IVF_RQ_IDENTIFIER,
+ &[
+ ("ivf-rq.dimension", "8"),
+ ("ivf-rq.nlist", "4"),
+ ("ivf-rq.rq.bits", "4"),
+ ],
+ );
+ assert_eq!(rq.index_type, IndexType::IvfRq);
+ assert_eq!(
+ (rq.dimension, rq.nlist, rq.total_vectors, rq.rq_bits),
+ (8, 4, 512, Some(4))
+ );
+
+ let diskann = roundtrip_metadata(
+ DISKANN_IDENTIFIER,
+ &[
+ ("diskann.dimension", "8"),
+ ("diskann.pq.m", "4"),
+ ("diskann.pq.bits", "4"),
+ ("diskann.max-degree", "8"),
+ ("diskann.build-search-list-size", "16"),
+ ("diskann.alpha", "1.2"),
+ ("diskann.raw-vector-encoding", "f32"),
+ ],
+ );
+ assert_eq!(diskann.index_type, IndexType::DiskAnn);
+ assert_eq!(
+ (
+ diskann.dimension,
+ diskann.total_vectors,
+ diskann.pq_m,
+ diskann.pq_bits
+ ),
+ (8, 512, Some(4), Some(4))
+ );
+ let diskann = diskann.diskann.unwrap();
+ assert_eq!(
+ (
+ diskann.max_degree,
+ diskann.build_search_list_size,
+ diskann.alpha
+ ),
+ (8, 16, 1.2)
+ );
+ }
+
#[test]
fn test_vindex_options_field_options_override_shared_table_options() {
let table_options = HashMap::from([
@@ -501,6 +820,15 @@ mod tests {
.unwrap();
assert_eq!(options.train_sample_ratio, 1.0);
assert!(!options.native_options.contains_key("train.sample-ratio"));
+
+ let diskann = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &HashMap::from([("diskann.train.sample-ratio".to_string(),
"0.5".to_string())]),
+ DISKANN_IDENTIFIER,
+ &array_float_field(),
+ )
+ .unwrap();
+ assert_eq!(diskann.train_sample_ratio, 0.5);
}
#[test]
@@ -617,24 +945,24 @@ mod tests {
#[test]
fn test_vindex_options_reject_non_applicable_user_options() {
- let table_options = HashMap::new();
- let user_options = HashMap::from([
- ("ivf-flat.dimension".to_string(), "8".to_string()),
- ("ivf-flat.nlist".to_string(), "4".to_string()),
- ("ivf-flat.pq.m".to_string(), "2".to_string()),
- ]);
-
- let err = VindexVectorIndexOptions::new(
- &table_options,
- &user_options,
- IVF_FLAT_IDENTIFIER,
- &array_float_field(),
- )
- .expect_err("non-applicable user option should be rejected");
+ for (index_type, key) in [
+ (IVF_FLAT_IDENTIFIER, "ivf-flat.pq.m"),
+ (IVF_FLAT_IDENTIFIER, "diskann.max-degree"),
+ (DISKANN_IDENTIFIER, "diskann.nlist"),
+ ] {
+ let user_options = HashMap::from([(key.to_string(),
"2".to_string())]);
+ let err = VindexVectorIndexOptions::new(
+ &HashMap::new(),
+ &user_options,
+ index_type,
+ &array_float_field(),
+ )
+ .expect_err("non-applicable user option should be rejected");
- assert!(
- matches!(err, crate::Error::ConfigInvalid { message } if
message.contains("ivf-flat.pq.m"))
- );
+ assert!(
+ matches!(err, crate::Error::ConfigInvalid { message } if
message.contains(key))
+ );
+ }
}
#[test]
@@ -665,6 +993,9 @@ mod tests {
fn test_native_index_type_helper() {
assert_eq!(native_index_type(IVF_FLAT_IDENTIFIER), Some("ivf_flat"));
assert_eq!(native_index_type(IVF_PQ_IDENTIFIER), Some("ivf_pq"));
+ assert_eq!(native_index_type("ivf-sq"), Some("ivf_sq"));
+ assert_eq!(native_index_type("ivf-rq"), Some("ivf_rq"));
+ assert_eq!(native_index_type("diskann"), Some("diskann"));
assert_eq!(native_index_type("ivf-hnsw-flat"), None);
assert_eq!(native_index_type("ivf-hnsw-sq"), None);
assert_eq!(native_index_type("btree"), None);
diff --git a/crates/paimon/src/vindex/reader.rs
b/crates/paimon/src/vindex/reader.rs
index f0cb0fe8..8f8eab35 100644
--- a/crates/paimon/src/vindex/reader.rs
+++ b/crates/paimon/src/vindex/reader.rs
@@ -19,7 +19,8 @@ use crate::vector_search::{GlobalIndexIOMeta, VectorSearch};
use crate::vindex::vector_search_timing_enabled;
use paimon_vindex_core::distance::MetricType;
use paimon_vindex_core::index::{
- VectorIndexMetadata, VectorIndexReader as VIndexReader, VectorSearchParams,
+ IndexType, VectorIndexMetadata, VectorIndexReader as VIndexReader,
VectorIndexReaderOptions,
+ VectorSearchParams,
};
use paimon_vindex_core::io::{ReadRequest, SeekRead, SeekReadCapabilities};
use std::collections::BinaryHeap;
@@ -30,6 +31,9 @@ use std::time::{Duration, Instant};
const DEFAULT_NPROBE: usize = 16;
const NPROBE_PARAMETER: &str = "ivf.nprobe";
+// Matches Java's NativeVectorGlobalIndexReader; l_search is intentionally
snake_case.
+const L_SEARCH_PARAMETER: &str = "diskann.l_search";
+const READER_MEMORY_BUDGET_PARAMETER: &str =
"vindex.reader.memory-budget-bytes";
const NATIVE_BATCH_PROCESS_WORKING_SET_BYTES: usize = 64 * 1024 * 1024;
// Native searches run on dedicated executor threads, so blocking here does
not block async I/O.
static NATIVE_BATCH_MEMORY_POOL: NativeBatchMemoryPool =
@@ -367,13 +371,17 @@ impl VindexVectorGlobalIndexReader {
}
let open_start = self.timing_enabled.then(Instant::now);
+ let reader_options = VectorIndexReaderOptions::new(int_parameter(
+ &self.options,
+ READER_MEMORY_BUDGET_PARAMETER,
+ VectorIndexReaderOptions::default().memory_budget_bytes,
+ )?);
let source = stream_fn(&self.io_meta.file_path)?;
- let mut reader =
VIndexReader::open(VindexInput::new(source)).map_err(|e| {
- crate::Error::DataInvalid {
+ let mut reader =
VIndexReader::open_with_options(VindexInput::new(source), reader_options)
+ .map_err(|e| crate::Error::DataInvalid {
message: format!("Failed to open paimon-vindex-core reader:
{}", e),
source: Some(Box::new(e)),
- }
- })?;
+ })?;
let vindex_open = open_start.map_or(Duration::ZERO, |start|
start.elapsed());
let metadata_start = self.timing_enabled.then(Instant::now);
let metadata = reader.metadata();
@@ -405,7 +413,7 @@ fn search_vindex(
return Ok(None);
};
let (labels, distances) = execute_scalar_search(reader, vector_search,
&prepared)?;
- let id_to_scores = collect_results(&labels, &distances, prepared.top_k,
metadata.metric);
+ let id_to_scores = collect_results(&labels, &distances,
prepared.params.top_k, metadata.metric);
if id_to_scores.is_empty() {
return Ok(None);
}
@@ -415,8 +423,7 @@ fn search_vindex(
#[derive(Clone, PartialEq, Eq)]
struct PreparedSearch {
- top_k: usize,
- nprobe: usize,
+ params: VectorSearchParams,
filter_bytes: Option<Vec<u8>>,
}
@@ -441,7 +448,29 @@ fn prepare_search(
if top_k == 0 {
return Ok(None);
}
- let nprobe = int_parameter(options, NPROBE_PARAMETER, DEFAULT_NPROBE)?;
+ let mut params = match metadata.index_type {
+ IndexType::DiskAnn => match options.get(L_SEARCH_PARAMETER) {
+ Some(value) => {
+ let l_search = value
+ .parse::<usize>()
+ .ok()
+ .filter(|value| *value > 0)
+ .ok_or_else(|| crate::Error::DataInvalid {
+ message: format!(
+ "Invalid value for '{}': {}. Must be a positive
integer.",
+ L_SEARCH_PARAMETER, value
+ ),
+ source: None,
+ })?;
+ VectorSearchParams::with_l_search(top_k, l_search)
+ }
+ None => VectorSearchParams::automatic(top_k),
+ },
+ _ => VectorSearchParams::new(
+ top_k,
+ int_parameter(options, NPROBE_PARAMETER, DEFAULT_NPROBE)?,
+ ),
+ };
let filter_bytes = if let Some(include_ids) =
&vector_search.include_row_ids {
if include_ids.is_empty() {
@@ -459,10 +488,10 @@ fn prepare_search(
} else {
None
};
+ params.top_k = top_k;
Ok(Some(PreparedSearch {
- top_k,
- nprobe,
+ params,
filter_bytes,
}))
}
@@ -472,7 +501,7 @@ fn execute_scalar_search(
vector_search: &VectorSearch,
prepared: &PreparedSearch,
) -> crate::Result<(Vec<i64>, Vec<f32>)> {
- let params = VectorSearchParams::new(prepared.top_k, prepared.nprobe);
+ let params = prepared.params;
match &prepared.filter_bytes {
Some(filter) => reader
.search_with_roaring_filter(&vector_search.vector, params, filter)
@@ -532,7 +561,8 @@ fn search_batch_vindex(
let index = indices[0];
let (labels, distances) =
execute_scalar_search(reader, &vector_searches[index],
&prepared)?;
- let map = collect_results(&labels, &distances, prepared.top_k,
metadata.metric);
+ let map =
+ collect_results(&labels, &distances,
prepared.params.top_k, metadata.metric);
if !map.is_empty() {
results[index] = Some(map);
}
@@ -550,7 +580,7 @@ fn search_batch_vindex(
for &index in indices {
queries.extend_from_slice(&vector_searches[index].vector);
}
- let params = VectorSearchParams::new(prepared.top_k,
prepared.nprobe);
+ let params = prepared.params;
let (labels, distances) = match &prepared.filter_bytes {
Some(filter) => reader
.search_batch_with_roaring_filter(&queries, indices.len(),
params, filter)
@@ -565,7 +595,7 @@ fn search_batch_vindex(
source: Some(Box::new(e)),
})?,
};
- let expected = indices.len() * prepared.top_k;
+ let expected = indices.len() * prepared.params.top_k;
if labels.len() != expected || distances.len() != expected {
return Err(crate::Error::DataInvalid {
message: format!(
@@ -577,12 +607,12 @@ fn search_batch_vindex(
});
}
for (query_index, &result_index) in indices.iter().enumerate() {
- let start = query_index * prepared.top_k;
- let end = start + prepared.top_k;
+ let start = query_index * prepared.params.top_k;
+ let end = start + prepared.params.top_k;
let map = collect_results(
&labels[start..end],
&distances[start..end],
- prepared.top_k,
+ prepared.params.top_k,
metadata.metric,
);
if !map.is_empty() {
@@ -635,12 +665,25 @@ fn native_batch_query_working_set_bytes(
.saturating_mul(std::mem::size_of::<f32>() * 2);
let centroid_products =
metadata.nlist.saturating_mul(std::mem::size_of::<f32>());
let probe_results = prepared
- .nprobe
+ .params
+ .configured_ivf_nprobe()
+ .unwrap_or(0)
.min(metadata.nlist)
.saturating_mul(std::mem::size_of::<usize>() +
std::mem::size_of::<f32>());
- let top_k_results = prepared.top_k.saturating_mul(
+ let top_k_results = prepared.params.top_k.saturating_mul(
std::mem::size_of::<i64>() + std::mem::size_of::<f32>() +
std::mem::size_of::<(f32, i64)>(),
);
+ // TODO: Mirrors vindex 0.3's live frontier; use upstream scratch-byte
reporting when exposed.
+ let diskann_candidates = if metadata.index_type == IndexType::DiskAnn {
+ prepared
+ .params
+ .configured_diskann_l_search()
+ .unwrap_or_else(||
prepared.params.top_k.saturating_mul(2).max(100))
+ .max(prepared.params.top_k)
+ .saturating_mul(std::mem::size_of::<(usize, f32)>() +
std::mem::size_of::<(i64, f32)>())
+ } else {
+ 0
+ };
let pq_tables = match (metadata.pq_m, metadata.pq_bits) {
(Some(m), Some(bits)) => 1usize
.checked_shl(bits as u32)
@@ -654,6 +697,7 @@ fn native_batch_query_working_set_bytes(
.saturating_add(centroid_products)
.saturating_add(probe_results)
.saturating_add(top_k_results)
+ .saturating_add(diskann_candidates)
.saturating_add(pq_tables)
.saturating_add(256)
.max(1)
@@ -740,6 +784,8 @@ mod tests {
use crate::vindex::range_reader::VindexFileReader;
use async_trait::async_trait;
use bytes::Bytes;
+ use paimon_vindex_core::diskann::DiskAnnBuildParams;
+ use paimon_vindex_core::diskann_io::DiskAnnHeader;
use paimon_vindex_core::index::{VectorIndexConfig, VectorIndexTrainer,
VectorIndexWriter};
use paimon_vindex_core::io::{PosWriter, SeekReadCapabilities};
use std::cell::Cell;
@@ -935,8 +981,7 @@ mod tests {
diskann: None,
};
let base_prepared = PreparedSearch {
- top_k: 10,
- nprobe: 16,
+ params: VectorSearchParams::new(10, 16),
filter_bytes: None,
};
let index_parallelism = 32;
@@ -948,7 +993,7 @@ mod tests {
assert!(native_batch_chunk_size(&larger_index, &base_prepared,
index_parallelism) < base);
let mut larger_top_k = base_prepared.clone();
- larger_top_k.top_k *= 4;
+ larger_top_k.params.top_k *= 4;
assert!(native_batch_chunk_size(&base_metadata, &larger_top_k,
index_parallelism) < base);
let mut pq_metadata = base_metadata.clone();
@@ -986,8 +1031,7 @@ mod tests {
diskann: None,
};
let prepared = PreparedSearch {
- top_k: 10,
- nprobe: 16,
+ params: VectorSearchParams::new(10, 16),
filter_bytes: Some(vec![0; 128]),
};
let chunk_size = native_batch_chunk_size(&metadata, &prepared, 1);
@@ -1068,6 +1112,116 @@ mod tests {
assert!(int_parameter(&options, NPROBE_PARAMETER,
DEFAULT_NPROBE).is_err());
}
+ #[test]
+ fn prepare_diskann_search_uses_automatic_or_explicit_l_search() {
+ let metadata = VectorIndexMetadata {
+ index_type: paimon_vindex_core::index::IndexType::DiskAnn,
+ dimension: TEST_DIMENSION,
+ nlist: 1,
+ metric: MetricType::L2,
+ total_vectors: 100,
+ pq_m: Some(2),
+ pq_bits: Some(8),
+ rq_bits: None,
+ diskann: None,
+ };
+
+ let automatic = prepare_search(&metadata, &HashMap::new(), &query())
+ .unwrap()
+ .unwrap();
+ let automatic_chunk_size = native_batch_chunk_size(&metadata,
&automatic, 1);
+ assert!(automatic_chunk_size > 1);
+ let wide_search = PreparedSearch {
+ params: VectorSearchParams::with_l_search(10, 4096),
+ filter_bytes: None,
+ };
+ assert!(native_batch_chunk_size(&metadata, &wide_search, 1) <
automatic_chunk_size);
+ let automatic_params = automatic.params;
+ assert_eq!(
+ automatic_params.search_width,
+ paimon_vindex_core::index::SearchWidth::Auto
+ );
+
+ let explicit = prepare_search(
+ &metadata,
+ &HashMap::from([("diskann.l_search".to_string(),
"64".to_string())]),
+ &query(),
+ )
+ .unwrap()
+ .unwrap();
+ assert!(automatic != explicit);
+ let explicit_params = explicit.params;
+ assert_eq!(
+ explicit_params.search_width,
+ paimon_vindex_core::index::SearchWidth::DiskAnnLSearch
+ );
+ assert_eq!(explicit_params.width, 64);
+
+ let zero_l_search = prepare_search(
+ &metadata,
+ &HashMap::from([("diskann.l_search".to_string(),
"0".to_string())]),
+ &query(),
+ )
+ .err()
+ .unwrap();
+ assert!(zero_l_search.to_string().contains("positive integer"));
+
+ let mixed_options = HashMap::from([
+ ("ivf.nprobe".to_string(), "4".to_string()),
+ ("diskann.l_search".to_string(), "64".to_string()),
+ ]);
+ let diskann = prepare_search(&metadata, &mixed_options, &query())
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ diskann.params.search_width,
+ paimon_vindex_core::index::SearchWidth::DiskAnnLSearch
+ );
+ assert_eq!(diskann.params.width, 64);
+
+ let mut ivf_metadata = metadata;
+ ivf_metadata.index_type = IndexType::IvfFlat;
+ let ivf = prepare_search(&ivf_metadata, &mixed_options, &query())
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ ivf.params.search_width,
+ paimon_vindex_core::index::SearchWidth::IvfNProbe
+ );
+ assert_eq!(ivf.params.width, 4);
+ }
+
+ #[test]
+ fn diskann_reader_enforces_configured_memory_budget_during_optimization() {
+ let header = DiskAnnHeader::for_layout(
+ 8,
+ 2,
+ 0,
+ 2,
+ DiskAnnBuildParams {
+ max_degree: 1,
+ build_search_list_size: 2,
+ ..DiskAnnBuildParams::default()
+ },
+ )
+ .unwrap();
+ let bytes = header.encode().to_vec();
+ let io_meta =
+ GlobalIndexIOMeta::new("budget.index".to_string(), bytes.len() as
u64, Vec::new());
+ let options = HashMap::from([(
+ "vindex.reader.memory-budget-bytes".to_string(),
+ "1".to_string(),
+ )]);
+ let mut reader = VindexVectorGlobalIndexReader::new(io_meta, options);
+
+ let error = reader
+ .load(|_| Ok(Cursor::new(bytes)))
+ .expect_err("resident state above the configured budget must
fail");
+ let message = error.to_string();
+ assert!(message.contains("resident"), "{message}");
+ assert!(message.contains("budget"), "{message}");
+ }
+
#[test]
fn erased_input_forwards_clone_and_capabilities() {
let capabilities = SeekReadCapabilities {
diff --git a/docs/src/sql.md b/docs/src/sql.md
index eaf9758a..239d61c0 100644
--- a/docs/src/sql.md
+++ b/docs/src/sql.md
@@ -986,8 +986,8 @@ CREATE TABLE paimon.my_db.items (
);
```
-For vector indexes backed by vindex, set `index_type` to `ivf-flat` or
-`ivf-pq`:
+For vector indexes backed by vindex, set `index_type` to `ivf-flat`,
+`ivf-pq`, `ivf-sq`, `ivf-rq`, or `diskann`:
```sql
CALL sys.create_global_index(
@@ -998,9 +998,41 @@ CALL sys.create_global_index(
);
```
+Examples for the additional vindex index types:
+
+```sql
+-- IVF-SQ uses a fixed 8-bit scalar code.
+CALL sys.create_global_index(
+ table => 'paimon.my_db.items',
+ index_column => 'embedding',
+ index_type => 'ivf-sq',
+ options =>
'ivf-sq.dimension=4,ivf-sq.nlist=256,ivf-sq.distance.metric=cosine'
+);
+
+CALL sys.create_global_index(
+ table => 'paimon.my_db.items',
+ index_column => 'embedding',
+ index_type => 'ivf-rq',
+ options =>
'ivf-rq.dimension=4,ivf-rq.nlist=256,ivf-rq.distance.metric=cosine,ivf-rq.rq.bits=4'
+);
+
+CALL sys.create_global_index(
+ table => 'paimon.my_db.items',
+ index_column => 'embedding',
+ index_type => 'diskann',
+ options =>
'diskann.dimension=4,diskann.distance.metric=cosine,diskann.deployment-profile=local_storage,diskann.build-preset=balanced'
+);
+```
+
+The vindex `diskann` index is separate from the Lumina index whose
+`lumina.index.type` is `diskann`. Use the `diskann.*` options below with
+`index_type => 'diskann'`; use `lumina.*` options only with a Lumina index.
+
The `options` argument is a comma-separated `key=value` string. User options
override table options. Use keys prefixed by the selected index type, or set
-field-level table options with `fields.<column>.<option>`:
+field-level table options with `fields.<column>.<option>`. For example,
+`diskann.max-degree` becomes `fields.embedding.max-degree` for an `embedding`
+column:
```sql
CREATE TABLE paimon.my_db.image_items (
@@ -1023,13 +1055,40 @@ Supported vindex options:
|---|---:|---|---|
| `<index-type>.dimension` | `128` | all vindex types | Vector dimension for
`ARRAY<FLOAT>` columns. Existing `VECTOR<FLOAT,N>` columns use `N` from the
type. |
| `<index-type>.distance.metric` | `inner_product` | all vindex types |
Distance metric: `inner_product`, `cosine`, or `l2`. |
-| `<index-type>.nlist` | `256` | all vindex types | Number of IVF lists. |
+| `<index-type>.nlist` | `256` | all IVF types | Number of IVF lists. DiskANN
rejects this option. |
| `<index-type>.train.sample-ratio` or `fields.<field>.train.sample-ratio` |
`1.0` | all vindex types | Fraction of shard rows selected evenly for training.
Must be in `(0, 1]`; all rows are still added to the index. The field-specific
option takes precedence. |
| `<index-type>.pq.m` | `16` | `ivf-pq` | Number of product-quantization
sub-vectors. The dimension must be divisible by this value. |
| `<index-type>.pq.use-opq` | `false` | `ivf-pq` | Whether to enable OPQ
before PQ encoding. |
+| `ivf-rq.rq.bits` | `4`, or inferred | `ivf-rq` | Residual-quantization width
in the range `1` to `8`. When omitted, `ivf-rq.max-bytes-per-vector` can select
it. |
+| `ivf-rq.max-bytes-per-vector` | unset | `ivf-rq` | Optional positive
persisted-code budget used to infer `rq.bits`. |
-Native vindex aliases are also accepted in the `options` string: `dimension`,
-`metric`, `nlist`, `pq.m`, and `use-opq`.
+IVF-SQ has no `sq.bits` option; it always stores one 8-bit scalar code per
+dimension. DiskANN accepts these build options:
+
+| Option | Default | Description |
+|---|---:|---|
+| `diskann.deployment-profile` | `auto` | Intended serving medium: `auto`,
`memory`, `local_storage`, `remote_storage`, or `object_store`. |
+| `diskann.target-recall` | unset | Value in `[0, 1]` used to choose a build
preset when one is not specified. This is a tuning hint, not a recall
guarantee. |
+| `diskann.max-bytes-per-vector` | unset | Optional positive persisted-size
budget that guides PQ width and raw-vector encoding. |
+| `diskann.pq.code-ratio` | `0.0625` | Ratio of resident PQ-code bytes to raw
`f32` vector bytes; must be in `(0, 0.25]` for 8-bit PQ or `(0, 0.125]` for
4-bit PQ. |
+| `diskann.pq.m` | automatic | Explicit PQ chunk count in `1..=dimension`;
overrides `pq.code-ratio`. |
+| `diskann.pq.bits` | `8` | PQ width; must be `4` or `8`. |
+| `diskann.build-preset` | inferred | `fast_build`, `balanced`, or
`high_recall`; without `target-recall` or an explicit value, uses `balanced`. |
+| `diskann.seed` | `42` | Reproducible graph-build seed. |
+| `diskann.memory-budget-bytes` | `8589934592` | Positive internal build-state
budget. This is not the query Reader budget or a process RSS limit. |
+| `diskann.max-degree` | preset value | Maximum graph out-degree in
`1..=1023`. |
+| `diskann.build-search-list-size` | preset value | Candidate-list width used
while building the graph; must be at least `max-degree`. |
+| `diskann.alpha` | preset value | Finite robust-pruning threshold at least
`1`. |
+| `diskann.storage-layout` | preset value | `auto`, `compact`, or
`interleaved`. |
+| `diskann.raw-vector-encoding` | preset value | `auto`, `f32`, or `f16`
rerank-vector encoding. |
+| `diskann.build-distance` | preset value | `auto`, `full_precision`, or
`product_quantized`. |
+
+For procedure calls, prefer the index-prefixed option names shown above. Native
+vindex aliases are also accepted in the `options` string: `dimension`,
`metric`,
+`nlist`, `pq.m`, `use-opq`, `rq.bits`, `max-bytes-per-vector`,
+`deployment-profile`, `target-recall`, `pq.code-ratio`, `pq.bits`, and the
+`diskann.*` build keys listed in the table. Build options for another index
+family are rejected rather than ignored.
Inspect committed index files with the `$table_indexes` system table:
@@ -1051,9 +1110,9 @@ CALL sys.drop_global_index(
```
`index_type` accepts every type the create procedures build: `btree`, `bitmap`,
-`lumina` (or `lumina-vector-ann`), and the vindex types `ivf-flat` and
`ivf-pq`.
-It defaults to `btree`, is case-insensitive and surrounding whitespace is
-ignored.
+`lumina` (or `lumina-vector-ann`), and the vindex types `ivf-flat`, `ivf-pq`,
+`ivf-sq`, `ivf-rq`, and `diskann`. It defaults to `btree`, is case-insensitive
+and surrounding whitespace is ignored.
### create_lumina_index
@@ -1335,9 +1394,45 @@ The distance metric is configured at index creation time
via table options:
### Vindex Index Options
For vindex-backed search, build the index with
-`CALL sys.create_global_index` and an index type such as `ivf-flat` or
-`ivf-pq`. See [create_global_index](#create_global_index) for the supported
-index types, table requirements, and option keys.
+`CALL sys.create_global_index` and an index type such as `ivf-flat`, `ivf-sq`,
+`ivf-pq`, `ivf-rq`, or `diskann`. See
+[create_global_index](#create_global_index) for the table requirements and
+build option keys.
+
+With Paimon's `SQLContext`, set query-time vindex options for the session
before
+calling `vector_search`, then reset them when they are no longer needed:
+
+```sql
+-- IVF only; defaults to 16.
+SET 'paimon.ivf.nprobe' = '32';
+SELECT * FROM vector_search('paimon.my_db.items', 'embedding', '[1.0, 0.0,
0.0, 0.0]', 10);
+RESET 'paimon.ivf.nprobe';
+
+-- DiskANN only; must be at least 1. Omit it to use the automatic search width.
+SET 'paimon.diskann.l_search' = '100';
+SELECT * FROM vector_search('paimon.my_db.items', 'embedding', '[1.0, 0.0,
0.0, 0.0]', 10);
+RESET 'paimon.diskann.l_search';
+```
+
+Query options are index-family specific and non-applicable options are ignored:
+IVF readers consume only `ivf.nprobe`, while DiskANN readers consume only
+`diskann.l_search`. Setting both options is allowed; each index uses its own.
+
+`vindex.reader.memory-budget-bytes` sets the per-Reader resident-data and cache
+budget (default 4 GiB). It is distinct from the DiskANN build option
+`diskann.memory-budget-bytes` and from process RSS:
+
+```sql
+SET 'paimon.vindex.reader.memory-budget-bytes' = '4294967296';
+SELECT * FROM vector_search('paimon.my_db.items', 'embedding', '[1.0, 0.0,
0.0, 0.0]', 10);
+RESET 'paimon.vindex.reader.memory-budget-bytes';
+```
+
+These `SET`/`RESET` values are provided by Paimon's `SQLContext`; registering
+`vector_search` directly on a raw DataFusion `SessionContext` does not install
+that dynamic-option path. Rust callers can pass the same unprefixed keys, such
+as `diskann.l_search`, through `VectorSearchBuilder::with_options` or
+`BatchVectorSearchBuilder::with_options`.
### Lumina Index Options
