This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 85c9127f perf(parquet): prune IN predicates with page stats (#704)
85c9127f is described below
commit 85c9127f66c69a2986389bbfa1422a9b53cc9358
Author: QuakeWang <[email protected]>
AuthorDate: Wed Aug 12 12:03:57 2026 +0800
perf(parquet): prune IN predicates with page stats (#704)
---
crates/paimon/src/arrow/format/parquet.rs | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/crates/paimon/src/arrow/format/parquet.rs
b/crates/paimon/src/arrow/format/parquet.rs
index c039074d..960828ec 100644
--- a/crates/paimon/src/arrow/format/parquet.rs
+++ b/crates/paimon/src/arrow/format/parquet.rs
@@ -1395,6 +1395,10 @@ impl StatsAccessor for ParquetPageStats<'_> {
}
page_index_value_to_datum(self.column_index, self.page_idx, data_type,
false)
}
+
+ fn supports_in_min_max_pruning(&self) -> bool {
+ true
+ }
}
/// Decode a per-page min/max from a [`ColumnIndexMetaData`] into a [`Datum`].
@@ -3486,6 +3490,22 @@ mod tests {
assert_eq!(sel.row_count(), 0);
}
+ #[tokio::test]
+ async fn test_page_selection_in_keeps_only_matching_page() {
+ let bytes = write_multi_page_parquet(10, 80).await;
+ let metadata = load_metadata_with_page_index(&bytes, true);
+ let fields = vec![int_field("id"), int_field("value")];
+
+ let predicates = vec![id_leaf(
+ PredicateOperator::In,
+ vec![Datum::Int(35), Datum::Int(1000)],
+ )];
+ let sel = super::build_predicate_page_selection(&metadata,
&predicates, &fields)
+ .unwrap()
+ .expect("non-matching pages should be skipped");
+ assert_eq!(sel.row_count(), 10);
+ }
+
#[tokio::test]
async fn test_page_selection_range_keeps_overlapping_pages() {
let bytes = write_multi_page_parquet(10, 80).await;
