This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 8c903819 fix: use i64 for index file size (#586)
8c903819 is described below
commit 8c9038195ae20fa4c0d5f8e812170a7497a83420
Author: XiaoHongbo <[email protected]>
AuthorDate: Wed Jul 22 22:47:13 2026 +0800
fix: use i64 for index file size (#586)
---
bindings/c/src/tests.rs | 2 +-
.../datafusion/src/system_tables/table_indexes.rs | 2 +-
.../integrations/datafusion/tests/system_tables.rs | 5 +---
.../src/spec/avro/index_manifest_entry_decode.rs | 4 +--
crates/paimon/src/spec/index_file_meta.rs | 2 +-
crates/paimon/src/spec/index_manifest.rs | 33 +++++++++++++++++++---
.../src/table/btree_global_index_build_builder.rs | 6 ++--
crates/paimon/src/table/bucket_assigner_dynamic.rs | 4 +--
crates/paimon/src/table/data_evolution_writer.rs | 2 +-
crates/paimon/src/table/global_index_scanner.rs | 2 +-
.../paimon/src/table/lumina_index_build_builder.rs | 18 ++++++++----
crates/paimon/src/table/referenced_files.rs | 2 +-
.../paimon/src/table/vindex_index_build_builder.rs | 9 +++++-
crates/paimon/tests/pk_vector_baseline_test.rs | 4 +--
crates/paimon/tests/pk_vector_batch_test.rs | 2 +-
15 files changed, 66 insertions(+), 31 deletions(-)
diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs
index 3897189c..58fb27ab 100644
--- a/bindings/c/src/tests.rs
+++ b/bindings/c/src/tests.rs
@@ -1764,7 +1764,7 @@ fn build_pk_vector_table(path: &str, vectors: &[[f32;
PK_DIM]]) -> Table {
let index_file = IndexFileMeta {
index_type: INDEX_TYPE.to_string(),
file_name: index_file_name,
- file_size: i32::try_from(index_file_size).unwrap(),
+ file_size: i64::try_from(index_file_size).unwrap(),
row_count: i32::try_from(row_count).unwrap(),
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
diff --git a/crates/integrations/datafusion/src/system_tables/table_indexes.rs
b/crates/integrations/datafusion/src/system_tables/table_indexes.rs
index b48a8a1c..c1c900ee 100644
--- a/crates/integrations/datafusion/src/system_tables/table_indexes.rs
+++ b/crates/integrations/datafusion/src/system_tables/table_indexes.rs
@@ -131,7 +131,7 @@ impl TableProvider for TableIndexesTable {
buckets.push(entry.bucket);
index_types.push(index_file.index_type.as_str());
file_names.push(index_file.file_name.as_str());
- file_sizes.push(i64::from(index_file.file_size));
+ file_sizes.push(index_file.file_size);
row_counts.push(i64::from(index_file.row_count));
append_dv_ranges(
&mut dv_ranges,
diff --git a/crates/integrations/datafusion/tests/system_tables.rs
b/crates/integrations/datafusion/tests/system_tables.rs
index be1d13ec..ca32da12 100644
--- a/crates/integrations/datafusion/tests/system_tables.rs
+++ b/crates/integrations/datafusion/tests/system_tables.rs
@@ -286,10 +286,7 @@ async fn test_table_indexes_system_table() {
assert_eq!(buckets.value(row), expected.bucket);
assert_eq!(index_types.value(row), expected.index_file.index_type);
assert_eq!(file_names.value(row), expected.index_file.file_name);
- assert_eq!(
- file_sizes.value(row),
- i64::from(expected.index_file.file_size)
- );
+ assert_eq!(file_sizes.value(row), expected.index_file.file_size);
assert_eq!(
row_counts.value(row),
i64::from(expected.index_file.row_count)
diff --git a/crates/paimon/src/spec/avro/index_manifest_entry_decode.rs
b/crates/paimon/src/spec/avro/index_manifest_entry_decode.rs
index 986b27b9..efdd960f 100644
--- a/crates/paimon/src/spec/avro/index_manifest_entry_decode.rs
+++ b/crates/paimon/src/spec/avro/index_manifest_entry_decode.rs
@@ -35,7 +35,7 @@ impl AvroRecordDecode for IndexManifestEntry {
let mut bucket: Option<i32> = None;
let mut index_type: Option<String> = None;
let mut file_name: Option<String> = None;
- let mut file_size: Option<i32> = None;
+ let mut file_size: Option<i64> = None;
let mut row_count: Option<i32> = None;
let mut deletion_vectors_ranges: Option<IndexMap<String,
DeletionVectorMeta>> = None;
let mut global_index_meta: Option<GlobalIndexMeta> = None;
@@ -60,7 +60,7 @@ impl AvroRecordDecode for IndexManifestEntry {
"_BUCKET" => bucket = Some(read_int_field(cursor,
field.nullable)?),
"_INDEX_TYPE" => index_type = Some(read_string_field(cursor,
field.nullable)?),
"_FILE_NAME" => file_name = Some(read_string_field(cursor,
field.nullable)?),
- "_FILE_SIZE" => file_size = Some(read_long_field(cursor,
field.nullable)? as i32),
+ "_FILE_SIZE" => file_size = Some(read_long_field(cursor,
field.nullable)?),
"_ROW_COUNT" => row_count = Some(read_long_field(cursor,
field.nullable)? as i32),
"_DELETIONS_VECTORS_RANGES" | "_DELETION_VECTORS_RANGES" => {
deletion_vectors_ranges =
decode_nullable_dv_ranges(cursor, field.nullable)?;
diff --git a/crates/paimon/src/spec/index_file_meta.rs
b/crates/paimon/src/spec/index_file_meta.rs
index ff1e63c0..d6e6e80e 100644
--- a/crates/paimon/src/spec/index_file_meta.rs
+++ b/crates/paimon/src/spec/index_file_meta.rs
@@ -63,7 +63,7 @@ pub struct IndexFileMeta {
pub file_name: String,
#[serde(rename = "_FILE_SIZE")]
- pub file_size: i32,
+ pub file_size: i64,
#[serde(rename = "_ROW_COUNT")]
pub row_count: i32,
diff --git a/crates/paimon/src/spec/index_manifest.rs
b/crates/paimon/src/spec/index_manifest.rs
index a0430b2f..5160b61e 100644
--- a/crates/paimon/src/spec/index_manifest.rs
+++ b/crates/paimon/src/spec/index_manifest.rs
@@ -27,10 +27,9 @@ use crate::Result;
///
/// Must match the serde layout of `IndexManifestEntry`.
///
-/// Note: `_FILE_SIZE` and `_ROW_COUNT` are declared as Avro `long` to match
-/// Java Paimon's schema, while the Rust `IndexFileMeta` fields are `i32`.
-/// `serde_avro_fast` transparently coerces between integer widths during
-/// serialization/deserialization, so the mismatch is intentional.
+/// Note: `_FILE_SIZE` is an Avro `long` and Rust `i64`, matching Java Paimon's
+/// schema. `_ROW_COUNT` remains an Avro `long` for Java compatibility while
+/// the Rust `IndexFileMeta` field is still `i32`.
pub const INDEX_MANIFEST_ENTRY_SCHEMA: &str = r#"{
"type": "record",
"name": "org.apache.paimon.avro.generated.record",
@@ -345,6 +344,32 @@ mod tests {
);
}
+ #[test]
+ fn file_size_above_i32_max_round_trips_through_index_manifest() {
+ let file_size = i64::from(i32::MAX) + 1;
+ let entry: IndexManifestEntry =
serde_json::from_value(serde_json::json!({
+ "_VERSION": 1,
+ "_KIND": 0,
+ "_PARTITION": [0, 0, 0, 0],
+ "_BUCKET": 0,
+ "_INDEX_TYPE": "TEST",
+ "_FILE_NAME": "index",
+ "_FILE_SIZE": file_size,
+ "_ROW_COUNT": 1
+ }))
+ .unwrap();
+
+ let bytes = crate::spec::to_avro_bytes_with_compression(
+ INDEX_MANIFEST_ENTRY_SCHEMA,
+ std::slice::from_ref(&entry),
+ crate::spec::DEFAULT_AVRO_COMPRESSION,
+ )
+ .unwrap();
+
+ let decoded = IndexManifest::read_from_bytes(&bytes).unwrap();
+ assert_eq!(decoded, vec![entry]);
+ }
+
#[test]
fn legacy_five_field_global_index_decodes_without_source_meta() {
// 5-field _GLOBAL_INDEX schema (pre-#8549): no _SOURCE_META.
Identical to
diff --git a/crates/paimon/src/table/btree_global_index_build_builder.rs
b/crates/paimon/src/table/btree_global_index_build_builder.rs
index 90b9891a..07bbe166 100644
--- a/crates/paimon/src/table/btree_global_index_build_builder.rs
+++ b/crates/paimon/src/table/btree_global_index_build_builder.rs
@@ -275,7 +275,7 @@ impl<'a> BTreeGlobalIndexBuildBuilder<'a> {
Ok(IndexFileMeta {
index_type: index_type.to_string(),
file_name,
- file_size: checked_i32(
+ file_size: checked_i64(
status.size,
"Index file is too large for Rust IndexFileMeta",
)?,
@@ -733,8 +733,8 @@ fn sort_index_rows(rows: &mut [BTreeKeyRow], cmp: &dyn
Fn(&[u8], &[u8]) -> Order
});
}
-fn checked_i32(value: u64, context: &str) -> Result<i32> {
- i32::try_from(value).map_err(|_| Error::DataInvalid {
+fn checked_i64(value: u64, context: &str) -> Result<i64> {
+ i64::try_from(value).map_err(|_| Error::DataInvalid {
message: format!("{context}: {value}"),
source: None,
})
diff --git a/crates/paimon/src/table/bucket_assigner_dynamic.rs
b/crates/paimon/src/table/bucket_assigner_dynamic.rs
index 9db4db9c..030385d9 100644
--- a/crates/paimon/src/table/bucket_assigner_dynamic.rs
+++ b/crates/paimon/src/table/bucket_assigner_dynamic.rs
@@ -80,10 +80,10 @@ impl HashIndexFile {
buf.extend_from_slice(&h.to_be_bytes());
}
- let file_size: i32 = buf
+ let file_size: i64 = buf
.len()
.try_into()
- .expect("hash index file size exceeds i32::MAX");
+ .expect("hash index file size exceeds i64::MAX");
let output = file_io.new_output(&path)?;
output.write(bytes::Bytes::from(buf)).await?;
diff --git a/crates/paimon/src/table/data_evolution_writer.rs
b/crates/paimon/src/table/data_evolution_writer.rs
index 49057e9d..47752e3f 100644
--- a/crates/paimon/src/table/data_evolution_writer.rs
+++ b/crates/paimon/src/table/data_evolution_writer.rs
@@ -689,7 +689,7 @@ impl DataEvolutionDeleteWriter {
bytes.extend_from_slice(&serialized);
}
- let file_size = i32::try_from(bytes.len()).map_err(|_|
crate::Error::DataInvalid {
+ let file_size = i64::try_from(bytes.len()).map_err(|_|
crate::Error::DataInvalid {
message: "Deletion-vector index file is too large".to_string(),
source: None,
})?;
diff --git a/crates/paimon/src/table/global_index_scanner.rs
b/crates/paimon/src/table/global_index_scanner.rs
index c535db7f..b6f50e88 100644
--- a/crates/paimon/src/table/global_index_scanner.rs
+++ b/crates/paimon/src/table/global_index_scanner.rs
@@ -211,7 +211,7 @@ impl GlobalIndexScanner {
BITMAP_GLOBAL_INDEX_TYPE => GlobalIndexFileKind::Bitmap,
_ => unreachable!("normalized sorted global index type"),
},
- file_size: i64::from(entry.index_file.file_size),
+ file_size: entry.index_file.file_size,
row_range_start: global_meta.row_range_start,
meta: sorted_meta,
};
diff --git a/crates/paimon/src/table/lumina_index_build_builder.rs
b/crates/paimon/src/table/lumina_index_build_builder.rs
index cbff6e7e..5e45abf8 100644
--- a/crates/paimon/src/table/lumina_index_build_builder.rs
+++ b/crates/paimon/src/table/lumina_index_build_builder.rs
@@ -230,7 +230,7 @@ impl<'a> LuminaIndexBuildBuilder<'a> {
Ok(IndexFileMeta {
index_type: LUMINA_IDENTIFIER.to_string(),
file_name,
- file_size: checked_i32(
+ file_size: checked_i64(
status.size,
"Index file is too large for Rust IndexFileMeta",
)?,
@@ -741,8 +741,8 @@ fn extract_vectors_from_batches(
Ok(vectors)
}
-fn checked_i32(value: u64, context: &str) -> Result<i32> {
- i32::try_from(value).map_err(|_| Error::DataInvalid {
+fn checked_i64(value: u64, context: &str) -> Result<i64> {
+ i64::try_from(value).map_err(|_| Error::DataInvalid {
message: format!("{context}: {value}"),
source: None,
})
@@ -1517,9 +1517,15 @@ mod tests {
}
#[test]
- fn test_checked_metadata_conversion_rejects_large_file_size() {
- let err = checked_i32(i32::MAX as u64 + 1, "Index file is too large")
- .expect_err("large file size should fail");
+ fn test_checked_metadata_conversion_supports_i64_file_size() {
+ let above_i32_max = i32::MAX as u64 + 1;
+ assert_eq!(
+ checked_i64(above_i32_max, "Index file is too large").unwrap(),
+ i64::from(i32::MAX) + 1
+ );
+
+ let err = checked_i64(i64::MAX as u64 + 1, "Index file is too large")
+ .expect_err("file size above i64::MAX should fail");
assert!(matches!(err, Error::DataInvalid { message, .. } if
message.contains("too large")));
}
diff --git a/crates/paimon/src/table/referenced_files.rs
b/crates/paimon/src/table/referenced_files.rs
index 8d1f1cac..10fe2437 100644
--- a/crates/paimon/src/table/referenced_files.rs
+++ b/crates/paimon/src/table/referenced_files.rs
@@ -499,7 +499,7 @@ async fn collect_snapshot_files(
file_set
.index_files
.entry(entry.index_file.file_name.clone())
- .or_insert(entry.index_file.file_size as i64);
+ .or_insert(entry.index_file.file_size);
}
}
diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs
b/crates/paimon/src/table/vindex_index_build_builder.rs
index 0f0d8c25..11579c6e 100644
--- a/crates/paimon/src/table/vindex_index_build_builder.rs
+++ b/crates/paimon/src/table/vindex_index_build_builder.rs
@@ -255,7 +255,7 @@ impl<'a> VindexIndexBuildBuilder<'a> {
Ok(IndexFileMeta {
index_type: self.index_type.clone(),
file_name,
- file_size: checked_i32(
+ file_size: checked_i64(
status.size,
"Index file is too large for Rust IndexFileMeta",
)?,
@@ -728,6 +728,13 @@ fn checked_i32(value: u64, context: &str) -> Result<i32> {
})
}
+fn checked_i64(value: u64, context: &str) -> Result<i64> {
+ i64::try_from(value).map_err(|_| Error::DataInvalid {
+ message: format!("{context}: {value}"),
+ source: None,
+ })
+}
+
fn checked_row_count(row_range_start: i64, row_range_end: i64) -> Result<i32> {
if row_range_end < row_range_start {
return Err(Error::DataInvalid {
diff --git a/crates/paimon/tests/pk_vector_baseline_test.rs
b/crates/paimon/tests/pk_vector_baseline_test.rs
index 97d9e18c..1328cddd 100644
--- a/crates/paimon/tests/pk_vector_baseline_test.rs
+++ b/crates/paimon/tests/pk_vector_baseline_test.rs
@@ -423,7 +423,7 @@ async fn build_table_with_first_row_id(
let index_file = IndexFileMeta {
index_type: INDEX_TYPE.to_string(),
file_name: index_file_name,
- file_size: i32::try_from(index_file_size).unwrap(),
+ file_size: i64::try_from(index_file_size).unwrap(),
row_count: i32::try_from(row_count).unwrap(),
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
@@ -1270,7 +1270,7 @@ async fn
pk_vector_refine_factor_matches_exact_ground_truth() {
let index_file = IndexFileMeta {
index_type: INDEX_TYPE.to_string(),
file_name: index_file_name,
- file_size: i32::try_from(index_file_size).unwrap(),
+ file_size: i64::try_from(index_file_size).unwrap(),
row_count: i32::try_from(row_count).unwrap(),
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
diff --git a/crates/paimon/tests/pk_vector_batch_test.rs
b/crates/paimon/tests/pk_vector_batch_test.rs
index 84790e77..235d82d8 100644
--- a/crates/paimon/tests/pk_vector_batch_test.rs
+++ b/crates/paimon/tests/pk_vector_batch_test.rs
@@ -284,7 +284,7 @@ async fn build_table(vectors: &[[f32; DIM]]) ->
(tempfile::TempDir, Table) {
let index_file = IndexFileMeta {
index_type: INDEX_TYPE.to_string(),
file_name: index_file_name,
- file_size: i32::try_from(index_file_size).unwrap(),
+ file_size: i64::try_from(index_file_size).unwrap(),
row_count: i32::try_from(row_count).unwrap(),
deletion_vectors_ranges: None,
global_index_meta: Some(GlobalIndexMeta {
