This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new b3af0c0 fix(scan): prune OR predicates using stats (#426)
b3af0c0 is described below
commit b3af0c05554a6aab895d05f8bdc9ad5bc6d6bfb4
Author: QuakeWang <[email protected]>
AuthorDate: Wed Jul 1 14:55:39 2026 +0800
fix(scan): prune OR predicates using stats (#426)
---
crates/integration_tests/tests/read_tables.rs | 9 +--
crates/paimon/src/predicate_stats.rs | 5 +-
crates/paimon/src/table/stats_filter.rs | 11 ++-
crates/paimon/src/table/table_scan.rs | 97 ++++++++++++++++++++++++++-
4 files changed, 114 insertions(+), 8 deletions(-)
diff --git a/crates/integration_tests/tests/read_tables.rs
b/crates/integration_tests/tests/read_tables.rs
index e03a123..ac45170 100644
--- a/crates/integration_tests/tests/read_tables.rs
+++ b/crates/integration_tests/tests/read_tables.rs
@@ -752,7 +752,7 @@ async fn
test_read_multi_partitioned_table_or_of_mixed_ands_prunes_partitions()
Predicate::and(vec![
pb.equal("dt", Datum::String("2024-01-01".into())).unwrap(),
pb.equal("hr", Datum::Int(10)).unwrap(),
- pb.greater_than("id", Datum::Int(10)).unwrap(),
+ pb.greater_than("id", Datum::Int(0)).unwrap(),
]),
Predicate::and(vec![
pb.equal("dt", Datum::String("2024-01-01".into())).unwrap(),
@@ -780,8 +780,9 @@ async fn
test_read_multi_partitioned_table_or_of_mixed_ands_prunes_partitions()
);
}
-/// A directly mixed OR like `dt = '...' OR id > 10` is still not safely
-/// splittable into a partition predicate, so no partitions should be pruned.
+/// A directly mixed OR like `dt = '...' OR id > 0` is still not safely
+/// splittable into a partition predicate. The data predicate branch may match
+/// all provisioned files, so this isolates partition projection behavior.
#[tokio::test]
async fn test_read_partitioned_table_mixed_or_filter_preserves_all() {
use paimon::spec::{Datum, Predicate, PredicateBuilder};
@@ -793,7 +794,7 @@ async fn
test_read_partitioned_table_mixed_or_filter_preserves_all() {
let filter = Predicate::or(vec![
pb.equal("dt", Datum::String("2024-01-01".into())).unwrap(),
- pb.greater_than("id", Datum::Int(10)).unwrap(),
+ pb.greater_than("id", Datum::Int(0)).unwrap(),
]);
let (plan, batches) = scan_and_read_with_filter(&table, filter).await;
diff --git a/crates/paimon/src/predicate_stats.rs
b/crates/paimon/src/predicate_stats.rs
index e0efd67..fc1a79a 100644
--- a/crates/paimon/src/predicate_stats.rs
+++ b/crates/paimon/src/predicate_stats.rs
@@ -300,7 +300,10 @@ fn predicate_may_match_with_schema<T: StatsAccessor>(
Predicate::And(children) => children
.iter()
.all(|child| predicate_may_match_with_schema(child, stats,
field_mapping, file_fields)),
- Predicate::Or(_) | Predicate::Not(_) => true,
+ Predicate::Or(children) => children
+ .iter()
+ .any(|child| predicate_may_match_with_schema(child, stats,
field_mapping, file_fields)),
+ Predicate::Not(_) => true,
Predicate::Leaf {
index,
data_type,
diff --git a/crates/paimon/src/table/stats_filter.rs
b/crates/paimon/src/table/stats_filter.rs
index 68a4d65..999213d 100644
--- a/crates/paimon/src/table/stats_filter.rs
+++ b/crates/paimon/src/table/stats_filter.rs
@@ -375,7 +375,16 @@ fn data_evolution_predicate_may_match(
row_count,
)
}),
- Predicate::Or(_) | Predicate::Not(_) => true,
+ Predicate::Or(children) => children.iter().any(|child| {
+ data_evolution_predicate_may_match(
+ child,
+ table_fields,
+ field_sources,
+ file_stats,
+ row_count,
+ )
+ }),
+ Predicate::Not(_) => true,
Predicate::Leaf {
index,
data_type,
diff --git a/crates/paimon/src/table/table_scan.rs
b/crates/paimon/src/table/table_scan.rs
index bd847d1..27bc8d8 100644
--- a/crates/paimon/src/table/table_scan.rs
+++ b/crates/paimon/src/table/table_scan.rs
@@ -843,7 +843,10 @@ mod tests {
use crate::table::bucket_filter::{compute_target_buckets,
extract_predicate_for_keys};
use crate::table::partition_filter::PartitionFilter;
use crate::table::source::{DataSplit, DataSplitBuilder, DeletionFile};
- use crate::table::stats_filter::{data_file_matches_predicates,
group_by_overlapping_row_id};
+ use crate::table::stats_filter::{
+ data_evolution_group_matches_predicates, data_file_matches_predicates,
+ group_by_overlapping_row_id,
+ };
use crate::table::Table;
use crate::Error;
use chrono::{DateTime, Utc};
@@ -1327,7 +1330,7 @@ mod tests {
}
#[test]
- fn test_data_file_matches_unsupported_predicate_fails_open() {
+ fn test_data_file_matches_or_prunes_when_no_child_matches() {
let fields = int_field();
let file = test_data_file_meta(
int_stats_row(Some(10)),
@@ -1341,6 +1344,29 @@ mod tests {
pb.greater_than("id", Datum::Int(25)).unwrap(),
]);
+ assert!(!data_file_matches_predicates(
+ &file,
+ &[predicate],
+ TEST_SCHEMA_ID,
+ &test_schema_fields(),
+ ));
+ }
+
+ #[test]
+ fn test_data_file_matches_or_keeps_when_any_child_matches() {
+ let fields = int_field();
+ let file = test_data_file_meta(
+ int_stats_row(Some(10)),
+ int_stats_row(Some(20)),
+ vec![Some(0)],
+ 5,
+ );
+ let pb = PredicateBuilder::new(&fields);
+ let predicate = Predicate::or(vec![
+ pb.less_than("id", Datum::Int(15)).unwrap(),
+ pb.greater_than("id", Datum::Int(25)).unwrap(),
+ ]);
+
assert!(data_file_matches_predicates(
&file,
&[predicate],
@@ -1349,6 +1375,73 @@ mod tests {
));
}
+ #[test]
+ fn test_data_file_matches_not_fails_open() {
+ let fields = int_field();
+ let file = test_data_file_meta(
+ int_stats_row(Some(10)),
+ int_stats_row(Some(20)),
+ vec![Some(0)],
+ 5,
+ );
+ let predicate = Predicate::negate(
+ PredicateBuilder::new(&fields)
+ .less_than("id", Datum::Int(5))
+ .unwrap(),
+ );
+
+ assert!(data_file_matches_predicates(
+ &file,
+ &[predicate],
+ TEST_SCHEMA_ID,
+ &test_schema_fields(),
+ ));
+ }
+
+ #[test]
+ fn test_data_evolution_group_matches_or_prunes_when_no_child_matches() {
+ let fields = int_field();
+ let file = test_data_file_meta(
+ int_stats_row(Some(10)),
+ int_stats_row(Some(20)),
+ vec![Some(0)],
+ 5,
+ );
+ let pb = PredicateBuilder::new(&fields);
+ let predicate = Predicate::or(vec![
+ pb.less_than("id", Datum::Int(5)).unwrap(),
+ pb.greater_than("id", Datum::Int(25)).unwrap(),
+ ]);
+
+ assert!(!data_evolution_group_matches_predicates(
+ &[file],
+ &[predicate],
+ &fields,
+ ));
+ }
+
+ #[test]
+ fn test_data_evolution_group_matches_or_keeps_when_any_child_matches() {
+ let fields = int_field();
+ let file = test_data_file_meta(
+ int_stats_row(Some(10)),
+ int_stats_row(Some(20)),
+ vec![Some(0)],
+ 5,
+ );
+ let pb = PredicateBuilder::new(&fields);
+ let predicate = Predicate::or(vec![
+ pb.less_than("id", Datum::Int(15)).unwrap(),
+ pb.greater_than("id", Datum::Int(25)).unwrap(),
+ ]);
+
+ assert!(data_evolution_group_matches_predicates(
+ &[file],
+ &[predicate],
+ &fields,
+ ));
+ }
+
#[test]
fn test_data_file_matches_corrupt_stats_fails_open() {
let fields = int_field();
