This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 75c9ee01 fix(spec): widen index-manifest row count to i64 (#751)
75c9ee01 is described below
commit 75c9ee01081e5ee1987c883e234109fba12a05bc
Author: Junrui Lee <[email protected]>
AuthorDate: Thu Aug 27 15:44:54 2026 +0800
fix(spec): widen index-manifest row count to i64 (#751)
---
bindings/c/src/tests.rs | 2 +-
.../datafusion/src/system_tables/table_indexes.rs | 2 +-
.../datafusion/tests/dynamic_bucket_tables.rs | 5 +--
.../integrations/datafusion/tests/system_tables.rs | 5 +--
.../src/spec/avro/index_manifest_entry_decode.rs | 4 +--
crates/paimon/src/spec/index_file_meta.rs | 2 +-
crates/paimon/src/spec/index_manifest.rs | 34 +++++++++++++++++--
crates/paimon/src/table/data_evolution_writer.rs | 2 +-
.../paimon/src/table/full_text_search_builder.rs | 2 +-
.../paimon/src/table/global_index_drop_builder.rs | 2 +-
crates/paimon/src/table/hybrid_search_builder.rs | 4 +--
.../paimon/src/table/lumina_index_build_builder.rs | 38 +++++++++++++---------
.../paimon/src/table/pk_full_text_bucket_search.rs | 2 +-
.../paimon/src/table/pk_full_text_bucket_state.rs | 4 +--
crates/paimon/src/table/pk_full_text_read.rs | 2 +-
crates/paimon/src/table/pk_full_text_scan.rs | 4 +--
.../src/table/sorted_global_index_build_builder.rs | 38 +++++++++++++++-------
crates/paimon/src/table/table_commit.rs | 2 +-
crates/paimon/src/table/vector_search_builder.rs | 2 +-
.../paimon/src/table/vindex_index_build_builder.rs | 9 +++--
crates/paimon/tests/pk_vector_baseline_test.rs | 4 +--
crates/paimon/tests/pk_vector_batch_test.rs | 2 +-
22 files changed, 107 insertions(+), 64 deletions(-)
diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs
index e7da4306..37fa0c91 100644
--- a/bindings/c/src/tests.rs
+++ b/bindings/c/src/tests.rs
@@ -2459,7 +2459,7 @@ fn build_pk_vector_table(path: &str, vectors: &[[f32;
PK_DIM]]) -> Table {
index_type: INDEX_TYPE.to_string(),
file_name: index_file_name,
file_size: i64::try_from(index_file_size).unwrap(),
- row_count: i32::try_from(row_count).unwrap(),
+ row_count,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: 0,
diff --git a/crates/integrations/datafusion/src/system_tables/table_indexes.rs
b/crates/integrations/datafusion/src/system_tables/table_indexes.rs
index c1c900ee..cff4796a 100644
--- a/crates/integrations/datafusion/src/system_tables/table_indexes.rs
+++ b/crates/integrations/datafusion/src/system_tables/table_indexes.rs
@@ -132,7 +132,7 @@ impl TableProvider for TableIndexesTable {
index_types.push(index_file.index_type.as_str());
file_names.push(index_file.file_name.as_str());
file_sizes.push(index_file.file_size);
- row_counts.push(i64::from(index_file.row_count));
+ row_counts.push(index_file.row_count);
append_dv_ranges(
&mut dv_ranges,
index_file
diff --git a/crates/integrations/datafusion/tests/dynamic_bucket_tables.rs
b/crates/integrations/datafusion/tests/dynamic_bucket_tables.rs
index 34aea68f..186b7894 100644
--- a/crates/integrations/datafusion/tests/dynamic_bucket_tables.rs
+++ b/crates/integrations/datafusion/tests/dynamic_bucket_tables.rs
@@ -1000,10 +1000,7 @@ async fn
test_pk_dynamic_bucket_partitioned_insert_overwrite() {
}
// Total hash count: partition 'b' had 2 keys, partition 'a' now has 1 key
- let total_hashes: i64 = entries_after
- .iter()
- .map(|e| e.index_file.row_count as i64)
- .sum();
+ let total_hashes: i64 = entries_after.iter().map(|e|
e.index_file.row_count).sum();
assert_eq!(
total_hashes, 3,
"Total hash entries should be 3 (2 from 'b' + 1 from 'a'), got
{total_hashes}"
diff --git a/crates/integrations/datafusion/tests/system_tables.rs
b/crates/integrations/datafusion/tests/system_tables.rs
index 267f3c97..582d6ef1 100644
--- a/crates/integrations/datafusion/tests/system_tables.rs
+++ b/crates/integrations/datafusion/tests/system_tables.rs
@@ -287,10 +287,7 @@ async fn test_table_indexes_system_table() {
assert_eq!(index_types.value(row), expected.index_file.index_type);
assert_eq!(file_names.value(row), expected.index_file.file_name);
assert_eq!(file_sizes.value(row), expected.index_file.file_size);
- assert_eq!(
- row_counts.value(row),
- i64::from(expected.index_file.row_count)
- );
+ assert_eq!(row_counts.value(row), expected.index_file.row_count);
assert_eq!(
dv_ranges.is_null(row),
expected.index_file.deletion_vectors_ranges.is_none()
diff --git a/crates/paimon/src/spec/avro/index_manifest_entry_decode.rs
b/crates/paimon/src/spec/avro/index_manifest_entry_decode.rs
index efdd960f..38bf0c71 100644
--- a/crates/paimon/src/spec/avro/index_manifest_entry_decode.rs
+++ b/crates/paimon/src/spec/avro/index_manifest_entry_decode.rs
@@ -36,7 +36,7 @@ impl AvroRecordDecode for IndexManifestEntry {
let mut index_type: Option<String> = None;
let mut file_name: Option<String> = None;
let mut file_size: Option<i64> = None;
- let mut row_count: Option<i32> = None;
+ let mut row_count: Option<i64> = None;
let mut deletion_vectors_ranges: Option<IndexMap<String,
DeletionVectorMeta>> = None;
let mut global_index_meta: Option<GlobalIndexMeta> = None;
@@ -61,7 +61,7 @@ impl AvroRecordDecode for IndexManifestEntry {
"_INDEX_TYPE" => index_type = Some(read_string_field(cursor,
field.nullable)?),
"_FILE_NAME" => file_name = Some(read_string_field(cursor,
field.nullable)?),
"_FILE_SIZE" => file_size = Some(read_long_field(cursor,
field.nullable)?),
- "_ROW_COUNT" => row_count = Some(read_long_field(cursor,
field.nullable)? as i32),
+ "_ROW_COUNT" => row_count = Some(read_long_field(cursor,
field.nullable)?),
"_DELETIONS_VECTORS_RANGES" | "_DELETION_VECTORS_RANGES" => {
deletion_vectors_ranges =
decode_nullable_dv_ranges(cursor, field.nullable)?;
}
diff --git a/crates/paimon/src/spec/index_file_meta.rs
b/crates/paimon/src/spec/index_file_meta.rs
index d6e6e80e..3b1b2f56 100644
--- a/crates/paimon/src/spec/index_file_meta.rs
+++ b/crates/paimon/src/spec/index_file_meta.rs
@@ -66,7 +66,7 @@ pub struct IndexFileMeta {
pub file_size: i64,
#[serde(rename = "_ROW_COUNT")]
- pub row_count: i32,
+ pub row_count: i64,
// use Indexmap to ensure the order of deletion_vectors_ranges is
consistent.
#[serde(
diff --git a/crates/paimon/src/spec/index_manifest.rs
b/crates/paimon/src/spec/index_manifest.rs
index 5160b61e..24cac0f2 100644
--- a/crates/paimon/src/spec/index_manifest.rs
+++ b/crates/paimon/src/spec/index_manifest.rs
@@ -27,9 +27,8 @@ use crate::Result;
///
/// Must match the serde layout of `IndexManifestEntry`.
///
-/// Note: `_FILE_SIZE` is an Avro `long` and Rust `i64`, matching Java Paimon's
-/// schema. `_ROW_COUNT` remains an Avro `long` for Java compatibility while
-/// the Rust `IndexFileMeta` field is still `i32`.
+/// Note: `_FILE_SIZE` and `_ROW_COUNT` are Avro `long`s and Rust `i64`s,
+/// matching Java Paimon's schema.
pub const INDEX_MANIFEST_ENTRY_SCHEMA: &str = r#"{
"type": "record",
"name": "org.apache.paimon.avro.generated.record",
@@ -370,6 +369,35 @@ mod tests {
assert_eq!(decoded, vec![entry]);
}
+ #[test]
+ fn row_count_above_i32_max_round_trips_through_index_manifest() {
+ // _ROW_COUNT is an Avro `long`; a source-backed index over a large
data
+ // file can exceed i32::MAX. The decoder must not truncate it.
+ let row_count = i64::from(i32::MAX) + 100;
+ let entry: IndexManifestEntry =
serde_json::from_value(serde_json::json!({
+ "_VERSION": 1,
+ "_KIND": 0,
+ "_PARTITION": [0, 0, 0, 0],
+ "_BUCKET": 0,
+ "_INDEX_TYPE": "TEST",
+ "_FILE_NAME": "index",
+ "_FILE_SIZE": 42,
+ "_ROW_COUNT": row_count
+ }))
+ .unwrap();
+
+ let bytes = crate::spec::to_avro_bytes_with_compression(
+ INDEX_MANIFEST_ENTRY_SCHEMA,
+ std::slice::from_ref(&entry),
+ crate::spec::DEFAULT_AVRO_COMPRESSION,
+ )
+ .unwrap();
+
+ let decoded = IndexManifest::read_from_bytes(&bytes).unwrap();
+ assert_eq!(decoded[0].index_file.row_count, row_count);
+ assert_eq!(decoded, vec![entry]);
+ }
+
#[test]
fn legacy_five_field_global_index_decodes_without_source_meta() {
// 5-field _GLOBAL_INDEX schema (pre-#8549): no _SOURCE_META.
Identical to
diff --git a/crates/paimon/src/table/data_evolution_writer.rs
b/crates/paimon/src/table/data_evolution_writer.rs
index d8b66e86..1736626c 100644
--- a/crates/paimon/src/table/data_evolution_writer.rs
+++ b/crates/paimon/src/table/data_evolution_writer.rs
@@ -713,7 +713,7 @@ impl DataEvolutionDeleteWriter {
index_type: DELETION_VECTORS_INDEX_TYPE.to_string(),
file_name,
file_size,
- row_count,
+ row_count: i64::from(row_count),
deletion_vectors_ranges: Some(ranges),
global_index_meta: None,
})
diff --git a/crates/paimon/src/table/full_text_search_builder.rs
b/crates/paimon/src/table/full_text_search_builder.rs
index 532cd0f3..81beacc2 100644
--- a/crates/paimon/src/table/full_text_search_builder.rs
+++ b/crates/paimon/src/table/full_text_search_builder.rs
@@ -1270,7 +1270,7 @@ mod tests {
index_type: FULL_TEXT_INDEX_TYPE.to_string(),
file_name: name.to_string(),
file_size: 0,
- row_count: i32::try_from(end - start + 1).unwrap(),
+ row_count: end - start + 1,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: start,
diff --git a/crates/paimon/src/table/global_index_drop_builder.rs
b/crates/paimon/src/table/global_index_drop_builder.rs
index 6bc70e72..505139fe 100644
--- a/crates/paimon/src/table/global_index_drop_builder.rs
+++ b/crates/paimon/src/table/global_index_drop_builder.rs
@@ -259,7 +259,7 @@ mod tests {
index_type: index_type.to_string(),
file_name: name.to_string(),
file_size: 128,
- row_count: (row_range_end - row_range_start + 1) as i32,
+ row_count: (row_range_end - row_range_start + 1),
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start,
diff --git a/crates/paimon/src/table/hybrid_search_builder.rs
b/crates/paimon/src/table/hybrid_search_builder.rs
index 3f5006cb..5068dc80 100644
--- a/crates/paimon/src/table/hybrid_search_builder.rs
+++ b/crates/paimon/src/table/hybrid_search_builder.rs
@@ -1385,7 +1385,7 @@ mod pk_hybrid_tests {
index_type: VECTOR_INDEX_TYPE.to_string(),
file_name: vector_index_name,
file_size: i64::try_from(vector_index_size).unwrap(),
- row_count: i32::try_from(row_count).unwrap(),
+ row_count,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: 0,
@@ -1418,7 +1418,7 @@ mod pk_hybrid_tests {
index_type: PK_FULL_TEXT_INDEX_TYPE.to_string(),
file_name: ft_index_name,
file_size: 1,
- row_count: i32::try_from(row_count).unwrap(),
+ row_count,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: 0,
diff --git a/crates/paimon/src/table/lumina_index_build_builder.rs
b/crates/paimon/src/table/lumina_index_build_builder.rs
index 7bed2cc3..d340c97e 100644
--- a/crates/paimon/src/table/lumina_index_build_builder.rs
+++ b/crates/paimon/src/table/lumina_index_build_builder.rs
@@ -200,7 +200,15 @@ impl<'a> LuminaIndexBuildBuilder<'a> {
index_meta: Vec<u8>,
) -> Result<IndexFileMeta> {
let row_count = checked_row_count(shard.row_range_start,
shard.row_range_end)?;
- validate_vector_buffer(vectors, row_count, dimension)?;
+ // The native Lumina builder counts rows in an i32; the manifest keeps
the
+ // full width.
+ let native_row_count = i32::try_from(row_count).map_err(|_|
Error::DataInvalid {
+ message: format!(
+ "Lumina shard row count {row_count} exceeds what the native
builder accepts"
+ ),
+ source: None,
+ })?;
+ validate_vector_buffer(vectors, native_row_count, dimension)?;
let ids = (0..row_count as u64).collect::<Vec<_>>();
let native_options =
LuminaIndexMeta::deserialize(&index_meta)?.options().clone();
@@ -208,8 +216,8 @@ impl<'a> LuminaIndexBuildBuilder<'a> {
let temp_file = TempFileGuard::new(temp_path.clone());
let temp_path_str = temp_path.to_string_lossy().to_string();
let builder = LuminaBuilder::create(&native_options)?;
- builder.pretrain(vectors, row_count, dimension)?;
- builder.insert(vectors, &ids, row_count, dimension)?;
+ builder.pretrain(vectors, native_row_count, dimension)?;
+ builder.insert(vectors, &ids, native_row_count, dimension)?;
builder.dump(&temp_path_str)?;
let file_name = format!("lumina-global-index-{}.index",
uuid::Uuid::new_v4());
@@ -596,10 +604,7 @@ async fn extract_vectors(
index_column,
dimension,
shard.row_range_start,
- i64::from(checked_row_count(
- shard.row_range_start,
- shard.row_range_end,
- )?),
+ checked_row_count(shard.row_range_start, shard.row_range_end)?,
)
}
@@ -751,19 +756,22 @@ fn checked_i64(value: u64, context: &str) -> Result<i64> {
})
}
-fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result<i32> {
+fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result<i64> {
if row_range_end < row_range_start {
return Err(Error::DataInvalid {
message: format!("Invalid Lumina row range [{row_range_start},
{row_range_end}]"),
source: None,
});
}
- i32::try_from(row_range_end - row_range_start + 1).map_err(|_|
Error::DataInvalid {
- message: format!(
- "Lumina row count is too large for Rust IndexFileMeta:
[{row_range_start}, {row_range_end}]"
- ),
- source: None,
- })
+ row_range_end
+ .checked_sub(row_range_start)
+ .and_then(|span| span.checked_add(1))
+ .ok_or_else(|| Error::DataInvalid {
+ message: format!(
+ "Row count overflows for row range [{row_range_start},
{row_range_end}]"
+ ),
+ source: None,
+ })
}
fn validate_vector_buffer(vectors: &[f32], row_count: i32, dimension: i32) ->
Result<()> {
@@ -1701,7 +1709,7 @@ mod tests {
index_type: LUMINA_IDENTIFIER.to_string(),
file_name: format!("lumina-synthetic-{start}-{end}.index"),
file_size: 1,
- row_count: (end - start + 1) as i32,
+ row_count: (end - start + 1),
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: start,
diff --git a/crates/paimon/src/table/pk_full_text_bucket_search.rs
b/crates/paimon/src/table/pk_full_text_bucket_search.rs
index d774e237..ecb2ba23 100644
--- a/crates/paimon/src/table/pk_full_text_bucket_search.rs
+++ b/crates/paimon/src/table/pk_full_text_bucket_search.rs
@@ -369,7 +369,7 @@ mod tests {
index_type: PK_FULL_TEXT_INDEX_TYPE.into(),
file_name: file_name.into(),
file_size: 1,
- row_count: total as i32,
+ row_count: total,
deletion_vectors_ranges: None,
global_index_meta: Some(gim(7, frame(level, files))),
}
diff --git a/crates/paimon/src/table/pk_full_text_bucket_state.rs
b/crates/paimon/src/table/pk_full_text_bucket_state.rs
index 48ad4b21..c32cf6e4 100644
--- a/crates/paimon/src/table/pk_full_text_bucket_state.rs
+++ b/crates/paimon/src/table/pk_full_text_bucket_state.rs
@@ -66,7 +66,7 @@ fn payload_matches_source(
None => return false, // overflow → stale
}
}
- i64::from(payload.row_count) == total
+ payload.row_count == total
&& global_meta.row_range_start == 0
&& global_meta.row_range_end == total - 1
}
@@ -297,7 +297,7 @@ mod tests {
fn payload(
file_name: &str,
index_type: &str,
- row_count: i32,
+ row_count: i64,
global_index_meta: Option<GlobalIndexMeta>,
) -> IndexFileMeta {
IndexFileMeta {
diff --git a/crates/paimon/src/table/pk_full_text_read.rs
b/crates/paimon/src/table/pk_full_text_read.rs
index 63b58f84..d5eed2fc 100644
--- a/crates/paimon/src/table/pk_full_text_read.rs
+++ b/crates/paimon/src/table/pk_full_text_read.rs
@@ -684,7 +684,7 @@ mod read_tests {
index_type: PK_FULL_TEXT_INDEX_TYPE.into(),
file_name: file_name.into(),
file_size: 1,
- row_count: total as i32,
+ row_count: total,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: 0,
diff --git a/crates/paimon/src/table/pk_full_text_scan.rs
b/crates/paimon/src/table/pk_full_text_scan.rs
index 5a677655..2ffe0b49 100644
--- a/crates/paimon/src/table/pk_full_text_scan.rs
+++ b/crates/paimon/src/table/pk_full_text_scan.rs
@@ -508,7 +508,7 @@ mod tests {
fn payload(
file_name: &str,
index_type: &str,
- row_count: i32,
+ row_count: i64,
global_index_meta: Option<GlobalIndexMeta>,
) -> IndexFileMeta {
IndexFileMeta {
@@ -526,7 +526,7 @@ mod tests {
payload(
file_name,
PK_FULL_TEXT_INDEX_TYPE,
- total as i32,
+ total,
Some(gim(7, 0, total - 1, Some(frame(level, files)))),
)
}
diff --git a/crates/paimon/src/table/sorted_global_index_build_builder.rs
b/crates/paimon/src/table/sorted_global_index_build_builder.rs
index 94fc3165..de72364b 100644
--- a/crates/paimon/src/table/sorted_global_index_build_builder.rs
+++ b/crates/paimon/src/table/sorted_global_index_build_builder.rs
@@ -715,10 +715,7 @@ async fn extract_index_rows(
read_builder.with_projection(&[index_column, ROW_ID_FIELD_NAME])?;
let read = read_builder.new_read()?;
let batches = read.to_arrow(&splits)?.try_collect::<Vec<_>>().await?;
- let expected_row_count = i64::from(checked_row_count(
- shard.row_range_start,
- shard.row_range_end,
- )?);
+ let expected_row_count = checked_row_count(shard.row_range_start,
shard.row_range_end)?;
if index_type == MULTIVALUE_GLOBAL_INDEX_TYPE {
let DataType::Array(array_type) = index_field.data_type() else {
unreachable!("multivalue field was validated before extraction")
@@ -1017,7 +1014,7 @@ fn checked_i64(value: u64, context: &str) -> Result<i64> {
})
}
-fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result<i32> {
+fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result<i64> {
if row_range_end < row_range_start {
return Err(Error::DataInvalid {
message: format!(
@@ -1026,12 +1023,15 @@ fn checked_row_count(row_range_start: i64,
row_range_end: i64) -> Result<i32> {
source: None,
});
}
- i32::try_from(row_range_end - row_range_start + 1).map_err(|_|
Error::DataInvalid {
- message: format!(
- "Sorted global index row count is too large for Rust
IndexFileMeta: [{row_range_start}, {row_range_end}]"
- ),
- source: None,
- })
+ row_range_end
+ .checked_sub(row_range_start)
+ .and_then(|span| span.checked_add(1))
+ .ok_or_else(|| Error::DataInvalid {
+ message: format!(
+ "Row count overflows for row range [{row_range_start},
{row_range_end}]"
+ ),
+ source: None,
+ })
}
fn ranges_overlap(left_start: i64, left_end: i64, right_start: i64, right_end:
i64) -> bool {
@@ -1040,6 +1040,20 @@ fn ranges_overlap(left_start: i64, left_end: i64,
right_start: i64, right_end: i
#[cfg(test)]
mod tests {
+
+ /// A row range wider than `i32::MAX` yields the full count instead of
being
+ /// rejected, and an inverted or overflowing range is still an error.
+ #[test]
+ fn checked_row_count_spans_beyond_i32() {
+ let start = 0;
+ let end = i64::from(i32::MAX) + 10;
+ assert_eq!(
+ super::checked_row_count(start, end).unwrap(),
+ i64::from(i32::MAX) + 11
+ );
+ assert!(super::checked_row_count(5, 4).is_err());
+ assert!(super::checked_row_count(i64::MIN, i64::MAX).is_err());
+ }
use super::*;
use crate::btree::BTreeIndexMeta;
use crate::catalog::Identifier;
@@ -3105,7 +3119,7 @@ mod tests {
index_type: BTREE_GLOBAL_INDEX_TYPE.to_string(),
file_name: "btree-synthetic-hole.index".to_string(),
file_size: 1,
- row_count: (hole_end - hole_start + 1) as i32,
+ row_count: (hole_end - hole_start + 1),
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: hole_start,
diff --git a/crates/paimon/src/table/table_commit.rs
b/crates/paimon/src/table/table_commit.rs
index 96749512..6e5d78c3 100644
--- a/crates/paimon/src/table/table_commit.rs
+++ b/crates/paimon/src/table/table_commit.rs
@@ -3288,7 +3288,7 @@ mod tests {
index_type: "lumina".to_string(),
file_name: name.to_string(),
file_size: 128,
- row_count: (row_range_end - row_range_start + 1) as i32,
+ row_count: (row_range_end - row_range_start + 1),
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start,
diff --git a/crates/paimon/src/table/vector_search_builder.rs
b/crates/paimon/src/table/vector_search_builder.rs
index 9867c582..aa1fdc6d 100644
--- a/crates/paimon/src/table/vector_search_builder.rs
+++ b/crates/paimon/src/table/vector_search_builder.rs
@@ -5701,7 +5701,7 @@ mod tests {
index_type: IVF_FLAT_IDENTIFIER.to_string(),
file_name: index_file_name,
file_size: i64::try_from(index_file_size).unwrap(),
- row_count: i32::try_from(row_count).unwrap(),
+ row_count,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: 0,
diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs
b/crates/paimon/src/table/vindex_index_build_builder.rs
index 881c6308..b0680fe7 100644
--- a/crates/paimon/src/table/vindex_index_build_builder.rs
+++ b/crates/paimon/src/table/vindex_index_build_builder.rs
@@ -1203,7 +1203,7 @@ fn checked_i64(value: u64, context: &str) -> Result<i64> {
})
}
-fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result<i32> {
+fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result<i64> {
if row_range_end < row_range_start {
return Err(Error::DataInvalid {
message: format!("Invalid vindex row range [{row_range_start},
{row_range_end}]"),
@@ -1213,10 +1213,9 @@ fn checked_row_count(row_range_start: i64,
row_range_end: i64) -> Result<i32> {
row_range_end
.checked_sub(row_range_start)
.and_then(|count| count.checked_add(1))
- .and_then(|count| i32::try_from(count).ok())
.ok_or_else(|| Error::DataInvalid {
message: format!(
- "vindex row count is too large for Rust IndexFileMeta:
[{row_range_start}, {row_range_end}]"
+ "Row count overflows for row range [{row_range_start},
{row_range_end}]"
),
source: None,
})
@@ -1624,7 +1623,7 @@ mod tests {
index_type: IVF_FLAT_IDENTIFIER.to_string(),
file_name:
format!("vector-ivf-flat-synthetic-{start}-{end}.index"),
file_size: 1,
- row_count: (end - start + 1) as i32,
+ row_count: end - start + 1,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: start,
@@ -1895,7 +1894,7 @@ mod tests {
index_type: "lumina".to_string(),
file_name: "lumina-synthetic-0.index".to_string(),
file_size: 1,
- row_count: (coverage[0].to() - coverage[0].from() + 1) as i32,
+ row_count: (coverage[0].to() - coverage[0].from() + 1) as i64,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: coverage[0].from(),
diff --git a/crates/paimon/tests/pk_vector_baseline_test.rs
b/crates/paimon/tests/pk_vector_baseline_test.rs
index 058e499f..a5c4e90b 100644
--- a/crates/paimon/tests/pk_vector_baseline_test.rs
+++ b/crates/paimon/tests/pk_vector_baseline_test.rs
@@ -424,7 +424,7 @@ async fn build_table_with_first_row_id(
index_type: INDEX_TYPE.to_string(),
file_name: index_file_name,
file_size: i64::try_from(index_file_size).unwrap(),
- row_count: i32::try_from(row_count).unwrap(),
+ row_count,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: 0,
@@ -1271,7 +1271,7 @@ async fn
pk_vector_refine_factor_matches_exact_ground_truth() {
index_type: INDEX_TYPE.to_string(),
file_name: index_file_name,
file_size: i64::try_from(index_file_size).unwrap(),
- row_count: i32::try_from(row_count).unwrap(),
+ row_count,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: 0,
diff --git a/crates/paimon/tests/pk_vector_batch_test.rs
b/crates/paimon/tests/pk_vector_batch_test.rs
index 235d82d8..87e4ca59 100644
--- a/crates/paimon/tests/pk_vector_batch_test.rs
+++ b/crates/paimon/tests/pk_vector_batch_test.rs
@@ -285,7 +285,7 @@ async fn build_table(vectors: &[[f32; DIM]]) ->
(tempfile::TempDir, Table) {
index_type: INDEX_TYPE.to_string(),
file_name: index_file_name,
file_size: i64::try_from(index_file_size).unwrap(),
- row_count: i32::try_from(row_count).unwrap(),
+ row_count,
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
row_range_start: 0,
