This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new bde536c134 [python] Exclude partition columns from value-stats pruning
(#8438)
bde536c134 is described below
commit bde536c134b612df3077d1e25a8b256226e41de3
Author: XiaoHongbo <[email protected]>
AuthorDate: Fri Jul 3 11:32:25 2026 +0800
[python] Exclude partition columns from value-stats pruning (#8438)
---
paimon-python/pypaimon/read/push_down_utils.py | 18 ++++++++++++++++
.../pypaimon/read/scanner/file_scanner.py | 4 ++++
paimon-python/pypaimon/tests/predicates_test.py | 11 ++++++++++
.../pypaimon/tests/reader_append_only_test.py | 24 ++++++++++++++++++++++
4 files changed, 57 insertions(+)
diff --git a/paimon-python/pypaimon/read/push_down_utils.py
b/paimon-python/pypaimon/read/push_down_utils.py
index 9f558f558a..ee564a4c79 100644
--- a/paimon-python/pypaimon/read/push_down_utils.py
+++ b/paimon-python/pypaimon/read/push_down_utils.py
@@ -161,3 +161,21 @@ def remove_row_id_filter(predicate: Predicate) ->
Optional[Predicate]:
return new_children[0]
return PredicateBuilder.or_predicates(new_children)
return predicate
+
+
+def exclude_predicate_with_fields(predicate: Optional[Predicate], fields:
Set[str]) -> Optional[Predicate]:
+ """Drop predicate parts referencing any of ``fields`` (mirrors Java
+ PredicateBuilder.excludePredicateWithFields)."""
+ if not predicate or not fields:
+ return predicate
+ if predicate.method == "and":
+ kept = []
+ for p in _split_and(predicate):
+ r = exclude_predicate_with_fields(p, fields)
+ if r is not None:
+ kept.append(r)
+ return PredicateBuilder.and_predicates(kept) if kept else None
+ # leaf or OR: drop the whole thing if it touches any field (OR isn't split
apart)
+ if _get_all_fields(predicate) & fields:
+ return None
+ return predicate
diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py
b/paimon-python/pypaimon/read/scanner/file_scanner.py
index d6c7ad6f36..46eb340b43 100755
--- a/paimon-python/pypaimon/read/scanner/file_scanner.py
+++ b/paimon-python/pypaimon/read/scanner/file_scanner.py
@@ -33,6 +33,7 @@ from pypaimon.manifest.simple_stats_evolutions import
SimpleStatsEvolutions
from pypaimon.schema.data_types import DataField
from pypaimon.read.plan import Plan
from pypaimon.read.push_down_utils import (_get_all_fields,
+ exclude_predicate_with_fields,
remove_row_id_filter,
trim_and_transform_predicate)
from pypaimon.read.scan_stats import ScanStats
@@ -221,6 +222,9 @@ class FileScanner:
self.manifest_scanner = manifest_scanner
self.predicate = predicate
self.predicate_for_stats = remove_row_id_filter(predicate) if
predicate else None
+ # Partition columns aren't in data files, so skip them for value-stats
pruning.
+ self.predicate_for_stats = exclude_predicate_with_fields(
+ self.predicate_for_stats, set(self.table.partition_keys))
self.limit = limit
self.snapshot_manager = table.snapshot_manager()
diff --git a/paimon-python/pypaimon/tests/predicates_test.py
b/paimon-python/pypaimon/tests/predicates_test.py
index 509efc79ca..9bcea3b39a 100644
--- a/paimon-python/pypaimon/tests/predicates_test.py
+++ b/paimon-python/pypaimon/tests/predicates_test.py
@@ -97,6 +97,17 @@ class PredicateTest(unittest.TestCase):
predicate_builder.equal('f2', 'a')
self.assertEqual(str(e.exception), "The field f2 is not in field list
['f0', 'f1'].")
+ def test_exclude_predicate_with_fields(self):
+ from pypaimon.read.push_down_utils import exclude_predicate_with_fields
+ pb =
self.catalog.get_table('default.test_append').new_read_builder().new_predicate_builder()
+ f0 = pb.is_null('f0')
+ f1 = pb.is_null('f1')
+
+ self.assertIsNone(exclude_predicate_with_fields(f0, {'f0'}))
+ self.assertIs(exclude_predicate_with_fields(f1, {'f0'}), f1)
+ self.assertIs(exclude_predicate_with_fields(pb.and_predicates([f0,
f1]), {'f0'}), f1)
+ self.assertIsNone(exclude_predicate_with_fields(pb.or_predicates([f0,
f1]), {'f0'}))
+
def test_append_with_duplicate(self):
pa_schema = pa.schema([
('f0', pa.int64()),
diff --git a/paimon-python/pypaimon/tests/reader_append_only_test.py
b/paimon-python/pypaimon/tests/reader_append_only_test.py
index 34509db74a..7e0c5a6748 100644
--- a/paimon-python/pypaimon/tests/reader_append_only_test.py
+++ b/paimon-python/pypaimon/tests/reader_append_only_test.py
@@ -101,6 +101,30 @@ class AoReaderTest(unittest.TestCase):
actual = self._read_test_table(read_builder).sort_by('user_id')
self.assertEqual(actual, self.expected)
+ def test_isnull_partition_filter(self):
+ schema = Schema.from_pyarrow_schema(self.pa_schema,
partition_keys=['dt'])
+ self.catalog.create_table('default.isnull_partition_filter', schema,
False)
+ table = self.catalog.get_table('default.isnull_partition_filter')
+ data = pa.Table.from_pydict({
+ 'user_id': [1, 2],
+ 'item_id': [1001, 1002],
+ 'behavior': ['a', 'b'],
+ 'dt': [None, 'p1'],
+ }, schema=self.pa_schema)
+ wb = table.new_batch_write_builder()
+ w, c = wb.new_write(), wb.new_commit()
+ w.write_arrow(data)
+ c.commit(w.prepare_commit())
+ w.close()
+ c.close()
+
+ pb = table.new_read_builder().new_predicate_builder()
+ rb = table.new_read_builder().with_filter(pb.is_null('dt'))
+ result = self._read_test_table(rb)
+
+ self.assertEqual(result.column('dt').to_pylist(), [None])
+ self.assertEqual(result.column('user_id').to_pylist(), [1])
+
def test_plan_snapshot_id_for_empty_and_non_empty_scan(self):
schema = Schema.from_pyarrow_schema(self.pa_schema,
partition_keys=['dt'])
self.catalog.create_table('default.test_plan_snapshot_id', schema,
False)
