This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new bd9e67c feat(predicate): add string and range leaf operators
(StartsWith/EndsWith/Contains/Like/Between) (#425)
bd9e67c is described below
commit bd9e67c0485faebaa9e2a38afcabc55852cc0038
Author: Junrui Lee <[email protected]>
AuthorDate: Wed Jul 1 14:33:35 2026 +0800
feat(predicate): add string and range leaf operators
(StartsWith/EndsWith/Contains/Like/Between) (#425)
---
Cargo.toml | 2 +
.../integrations/datafusion/src/filter_pushdown.rs | 286 +++++++++-
crates/paimon/Cargo.toml | 2 +
crates/paimon/src/arrow/filtering.rs | 6 +
crates/paimon/src/arrow/format/orc.rs | 8 +
crates/paimon/src/arrow/format/parquet.rs | 273 ++++++++-
crates/paimon/src/arrow/format/vortex.rs | 20 +-
crates/paimon/src/btree/query.rs | 47 ++
crates/paimon/src/predicate_stats.rs | 443 ++++++++++++++-
crates/paimon/src/spec/predicate.rs | 623 +++++++++++++++++++++
crates/paimon/src/table/global_index_scanner.rs | 100 +++-
11 files changed, 1765 insertions(+), 45 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index c72bf55..38591c8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -30,12 +30,14 @@ rust-version = "1.91.0"
[workspace.dependencies]
arrow = "58.0"
arrow-array = { version = "58.0", features = ["ffi"] }
+arrow-arith = "58.0"
arrow-buffer = "58.0"
arrow-schema = "58.0"
arrow-cast = "58.0"
arrow-ord = "58.0"
arrow-row = "58.0"
arrow-select = "58.0"
+arrow-string = "58.0"
datafusion = "53.0.0"
datafusion-ffi = "53.0.0"
paimon = { version = "0.3.0", path = "crates/paimon" }
diff --git a/crates/integrations/datafusion/src/filter_pushdown.rs
b/crates/integrations/datafusion/src/filter_pushdown.rs
index 0d41827..5a000a8 100644
--- a/crates/integrations/datafusion/src/filter_pushdown.rs
+++ b/crates/integrations/datafusion/src/filter_pushdown.rs
@@ -16,8 +16,10 @@
// under the License.
use datafusion::common::{Column, ScalarValue};
-use datafusion::logical_expr::expr::InList;
-use datafusion::logical_expr::{Between, BinaryExpr, Expr, Operator,
TableProviderFilterPushDown};
+use datafusion::logical_expr::expr::{InList, ScalarFunction};
+use datafusion::logical_expr::{
+ Between, BinaryExpr, Expr, Like, Operator, TableProviderFilterPushDown,
+};
use paimon::spec::{DataField, DataType, Datum, Predicate, PredicateBuilder};
#[derive(Debug)]
@@ -146,6 +148,8 @@ impl<'a> FilterTranslator<'a> {
}
Expr::InList(in_list) => self.translate_in_list(in_list),
Expr::Between(between) => self.translate_between(between),
+ Expr::ScalarFunction(func) => self.translate_scalar_function(func),
+ Expr::Like(like) => self.translate_like(like),
_ => None,
}
}
@@ -249,22 +253,56 @@ impl<'a> FilterTranslator<'a> {
field.data_type(),
)?;
- let predicate = Predicate::and(vec![
- self.predicate_builder
- .greater_or_equal(field.name(), low)
- .ok()?,
- self.predicate_builder
- .less_or_equal(field.name(), high)
- .ok()?,
- ]);
-
+ // Native Between / NotBetween leaf: lets the planner / b-tree
+ // recognize the range as a single op (see
`btree::query::extract_between`).
+ // NotBetween is safe to push because its evaluator, stats prune and
+ // Parquet row filter all treat a NULL operand as non-matching (SQL
+ // three-valued logic), and a data-column range stays Inexact so
+ // DataFusion keeps the residual filter.
if between.negated {
- // Same concern as Expr::Not: negation wraps in Predicate::Not
- // which has incorrect NULL semantics for Exact pushdown.
- None
+ self.predicate_builder
+ .not_between(field.name(), low, high)
+ .ok()
} else {
- Some(predicate)
+ self.predicate_builder.between(field.name(), low, high).ok()
+ }
+ }
+
+ fn translate_scalar_function(&self, func: &ScalarFunction) ->
Option<Predicate> {
+ // DataFusion built-in UDFs surfaced from `LIKE 'x%' / '%x' / '%x%'`
+ // rewrites and direct `starts_with(col, 'x') / ends_with / contains`
+ // calls. Only `(col, literal)` shapes are handled; anything else
+ // (transform on either side, non-string args) falls open to None.
+ if func.args.len() != 2 {
+ return None;
+ }
+ let field = self.resolve_field(&func.args[0])?;
+ let scalar = extract_scalar_literal(&func.args[1])?;
+ let datum = scalar_to_datum(scalar, field.data_type())?;
+
+ match func.name() {
+ "starts_with" => self.predicate_builder.starts_with(field.name(),
datum).ok(),
+ "ends_with" => self.predicate_builder.ends_with(field.name(),
datum).ok(),
+ "contains" => self.predicate_builder.contains(field.name(),
datum).ok(),
+ _ => None,
+ }
+ }
+
+ fn translate_like(&self, like: &Like) -> Option<Predicate> {
+ // Negated and case-insensitive (ILIKE) variants stay unsupported for
+ // now: NOT-LIKE has the same NULL-semantics concern as `Expr::Not`;
+ // ILIKE has no equivalent in paimon's predicate model.
+ if like.negated || like.case_insensitive {
+ return None;
}
+ let field = self.resolve_field(like.expr.as_ref())?;
+ let scalar = extract_scalar_literal(like.pattern.as_ref())?;
+ let datum = scalar_to_datum(scalar, field.data_type())?;
+ // PredicateBuilder::like rejects escape characters other than `\`,
+ // so unsupported escapes naturally fall open via `.ok() -> None`.
+ self.predicate_builder
+ .like(field.name(), datum, like.escape_char)
+ .ok()
}
fn resolve_field(&self, expr: &Expr) -> Option<&'a DataField> {
@@ -598,31 +636,219 @@ mod tests {
}
#[test]
- fn test_translate_negated_between_is_not_supported() {
+ fn test_translate_boolean_literal_is_not_supported() {
let fields = test_fields();
- let filter = Expr::Between(Between::new(
- Box::new(Expr::Column(Column::from_name("hr"))),
- true, // negated
- Box::new(lit(1)),
- Box::new(lit(20)),
- ));
+ for value in [true, false] {
+ let filter = Expr::Literal(ScalarValue::Boolean(Some(value)),
None);
+ assert!(
+ build_pushed_predicate(&[filter], &fields).is_none(),
+ "Boolean literal ({value}) is not a partition predicate and
must not be translated"
+ );
+ }
+ }
+
+ #[test]
+ fn test_translate_starts_with_udf() {
+ let fields = test_fields();
+ let filter = datafusion::functions::string::expr_fn::starts_with(
+ Expr::Column(Column::from_name("dt")),
+ lit("2024"),
+ );
+ let predicate =
+ build_pushed_predicate(&[filter], &fields).expect("starts_with
should translate");
+ match predicate {
+ Predicate::Leaf { op, literals, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::StartsWith);
+ assert_eq!(literals, vec![Datum::String("2024".to_string())]);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_translate_ends_with_udf() {
+ let fields = test_fields();
+ let filter = datafusion::functions::string::expr_fn::ends_with(
+ Expr::Column(Column::from_name("dt")),
+ lit("01-01"),
+ );
+ let predicate =
+ build_pushed_predicate(&[filter], &fields).expect("ends_with
should translate");
+ match predicate {
+ Predicate::Leaf { op, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::EndsWith);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_translate_contains_udf() {
+ let fields = test_fields();
+ let filter = datafusion::functions::string::expr_fn::contains(
+ Expr::Column(Column::from_name("dt")),
+ lit("01"),
+ );
+ let predicate =
+ build_pushed_predicate(&[filter], &fields).expect("contains should
translate");
+ match predicate {
+ Predicate::Leaf { op, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::Contains);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_translate_starts_with_on_non_string_column_falls_open() {
+ let fields = test_fields();
+ // `id` is Int — datum coercion fails and translation returns None.
+ let filter = datafusion::functions::string::expr_fn::starts_with(
+ Expr::Column(Column::from_name("id")),
+ lit("foo"),
+ );
assert!(
build_pushed_predicate(&[filter], &fields).is_none(),
- "Negated BETWEEN should not translate due to NULL semantics"
+ "starts_with on non-string column must not translate"
);
}
+ fn like_filter(pattern: &str, negated: bool, case_insensitive: bool) ->
Expr {
+ Expr::Like(Like::new(
+ negated,
+ Box::new(Expr::Column(Column::from_name("dt"))),
+ Box::new(lit(pattern)),
+ None,
+ case_insensitive,
+ ))
+ }
+
#[test]
- fn test_translate_boolean_literal_is_not_supported() {
+ fn test_translate_like_rewrites_to_starts_with() {
let fields = test_fields();
+ let predicate = build_pushed_predicate(&[like_filter("2024%", false,
false)], &fields)
+ .expect("LIKE prefix% should translate");
+ match predicate {
+ Predicate::Leaf { op, literals, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::StartsWith);
+ assert_eq!(literals, vec![Datum::String("2024".to_string())]);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
- for value in [true, false] {
- let filter = Expr::Literal(ScalarValue::Boolean(Some(value)),
None);
- assert!(
- build_pushed_predicate(&[filter], &fields).is_none(),
- "Boolean literal ({value}) is not a partition predicate and
must not be translated"
- );
+ #[test]
+ fn test_translate_like_rewrites_to_ends_with() {
+ let fields = test_fields();
+ let predicate = build_pushed_predicate(&[like_filter("%01-01", false,
false)], &fields)
+ .expect("LIKE %suffix should translate");
+ match predicate {
+ Predicate::Leaf { op, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::EndsWith);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_translate_like_rewrites_to_contains() {
+ let fields = test_fields();
+ let predicate = build_pushed_predicate(&[like_filter("%01%", false,
false)], &fields)
+ .expect("LIKE %mid% should translate");
+ match predicate {
+ Predicate::Leaf { op, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::Contains);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_translate_like_no_wildcards_rewrites_to_eq() {
+ let fields = test_fields();
+ let predicate = build_pushed_predicate(&[like_filter("2024-01-01",
false, false)], &fields)
+ .expect("LIKE without wildcards should translate to Eq");
+ match predicate {
+ Predicate::Leaf { op, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::Eq);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_translate_like_residual_keeps_like_leaf() {
+ let fields = test_fields();
+ let predicate = build_pushed_predicate(&[like_filter("a%b%c", false,
false)], &fields)
+ .expect("complex LIKE should translate as a Like leaf");
+ match predicate {
+ Predicate::Leaf { op, literals, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::Like);
+ assert_eq!(literals, vec![Datum::String("a%b%c".to_string())]);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_translate_negated_like_falls_open() {
+ let fields = test_fields();
+ assert!(
+ build_pushed_predicate(&[like_filter("a%", true, false)],
&fields).is_none(),
+ "NOT LIKE must not translate (NULL semantics)"
+ );
+ }
+
+ #[test]
+ fn test_translate_ilike_falls_open() {
+ let fields = test_fields();
+ assert!(
+ build_pushed_predicate(&[like_filter("a%", false, true)],
&fields).is_none(),
+ "ILIKE must not translate (case-insensitive not modeled)"
+ );
+ }
+
+ #[test]
+ fn test_translate_between_produces_native_between_leaf() {
+ let fields = test_fields();
+ let filter = Expr::Between(Between::new(
+ Box::new(Expr::Column(Column::from_name("hr"))),
+ false,
+ Box::new(lit(1)),
+ Box::new(lit(20)),
+ ));
+ let predicate =
+ build_pushed_predicate(&[filter], &fields).expect("BETWEEN should
translate");
+ match predicate {
+ Predicate::Leaf { op, literals, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::Between);
+ assert_eq!(literals, vec![Datum::Int(1), Datum::Int(20)]);
+ }
+ other => panic!(
+ "expected native Between leaf, got {other:?} (Stage 3 must not
produce \
+ the legacy GtEq+LtEq And shape)"
+ ),
+ }
+ }
+
+ #[test]
+ fn test_translate_not_between_produces_native_not_between_leaf() {
+ let fields = test_fields();
+ let filter = Expr::Between(Between::new(
+ Box::new(Expr::Column(Column::from_name("hr"))),
+ true,
+ Box::new(lit(1)),
+ Box::new(lit(20)),
+ ));
+ let predicate =
+ build_pushed_predicate(&[filter], &fields).expect("NOT BETWEEN
should translate");
+ match predicate {
+ Predicate::Leaf { op, literals, .. } => {
+ assert_eq!(op, paimon::spec::PredicateOperator::NotBetween);
+ assert_eq!(literals, vec![Datum::Int(1), Datum::Int(20)]);
+ }
+ other => panic!("expected native NotBetween leaf, got {other:?}"),
}
}
}
diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml
index e79c129..7755a8e 100644
--- a/crates/paimon/Cargo.toml
+++ b/crates/paimon/Cargo.toml
@@ -78,12 +78,14 @@ crc32fast = "1"
zstd = "0.13"
snap = "1"
arrow-array = { workspace = true }
+arrow-arith = { workspace = true }
arrow-buffer = { workspace = true }
arrow-cast = { workspace = true }
arrow-ord = { workspace = true }
arrow-row = { workspace = true }
arrow-schema = { workspace = true }
arrow-select = { workspace = true }
+arrow-string = { workspace = true }
futures = "0.3"
tokio-util = { workspace = true, features = ["compat"] }
parquet = { workspace = true, features = ["async", "zstd", "lz4", "snap"] }
diff --git a/crates/paimon/src/arrow/filtering.rs
b/crates/paimon/src/arrow/filtering.rs
index f88c59b..1f450bd 100644
--- a/crates/paimon/src/arrow/filtering.rs
+++ b/crates/paimon/src/arrow/filtering.rs
@@ -143,6 +143,12 @@ fn predicate_supported_for_reader_pruning(predicate:
&Predicate) -> bool {
| PredicateOperator::GtEq
| PredicateOperator::In
| PredicateOperator::NotIn
+ | PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like
+ | PredicateOperator::Between
+ | PredicateOperator::NotBetween
)
}
Predicate::AlwaysTrue | Predicate::And(_) | Predicate::Or(_) |
Predicate::Not(_) => false,
diff --git a/crates/paimon/src/arrow/format/orc.rs
b/crates/paimon/src/arrow/format/orc.rs
index f4b078c..a90cfb9 100644
--- a/crates/paimon/src/arrow/format/orc.rs
+++ b/crates/paimon/src/arrow/format/orc.rs
@@ -252,6 +252,14 @@ fn build_orc_leaf_predicate(
}
PredicateOperator::IsNull | PredicateOperator::NotEq |
PredicateOperator::NotIn => None,
PredicateOperator::IsNotNull => None,
+ // String/range ops are not pushed into ORC; returning None falls open
to
+ // the outer stats-prune + arrow row-filter path.
+ PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like
+ | PredicateOperator::Between
+ | PredicateOperator::NotBetween => None,
}
}
diff --git a/crates/paimon/src/arrow/format/parquet.rs
b/crates/paimon/src/arrow/format/parquet.rs
index d7bfc5b..0d70e41 100644
--- a/crates/paimon/src/arrow/format/parquet.rs
+++ b/crates/paimon/src/arrow/format/parquet.rs
@@ -22,14 +22,19 @@ use crate::spec::{DataField, DataType, Datum, Predicate,
PredicateOperator};
use crate::table::{ArrowRecordBatchStream, RowRange};
use crate::Error;
use arrow_array::{
- Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Decimal128Array,
Float32Array,
- Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, RecordBatch,
Scalar, StringArray,
+ Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Datum as
ArrowDatum, Decimal128Array,
+ Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array,
RecordBatch, Scalar,
+ StringArray,
};
use arrow_ord::cmp::{
eq as arrow_eq, gt as arrow_gt, gt_eq as arrow_gt_eq, lt as arrow_lt,
lt_eq as arrow_lt_eq,
neq as arrow_neq,
};
use arrow_schema::ArrowError;
+use arrow_string::like::{
+ contains as arrow_contains, ends_with as arrow_ends_with, like as
arrow_like,
+ starts_with as arrow_starts_with,
+};
use async_trait::async_trait;
use bytes::Bytes;
use futures::future::BoxFuture;
@@ -286,6 +291,12 @@ fn predicate_supported_for_parquet_row_filter(op:
PredicateOperator) -> bool {
| PredicateOperator::GtEq
| PredicateOperator::In
| PredicateOperator::NotIn
+ | PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like
+ | PredicateOperator::Between
+ | PredicateOperator::NotBetween
)
}
@@ -315,6 +326,32 @@ fn parquet_row_filter_literals_supported(
}
Ok(true)
}
+ PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like => {
+ // Substring kernels only run against string-typed columns; reject
+ // non-string file types early so the filter falls back to stats
+ // pruning + residual evaluation.
+ if !matches!(file_data_type, DataType::Char(_) |
DataType::VarChar(_)) {
+ return Ok(false);
+ }
+ let Some(literal) = literals.first() else {
+ return Ok(false);
+ };
+ Ok(literal_scalar_for_parquet_filter(literal,
file_data_type)?.is_some())
+ }
+ PredicateOperator::Between | PredicateOperator::NotBetween => {
+ if literals.len() != 2 {
+ return Ok(false);
+ }
+ for literal in literals {
+ if literal_scalar_for_parquet_filter(literal,
file_data_type)?.is_none() {
+ return Ok(false);
+ }
+ }
+ Ok(true)
+ }
}
}
@@ -354,7 +391,11 @@ fn evaluate_exact_leaf_predicate(
| PredicateOperator::Lt
| PredicateOperator::LtEq
| PredicateOperator::Gt
- | PredicateOperator::GtEq => {
+ | PredicateOperator::GtEq
+ | PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like => {
let Some(literal) = literals.first() else {
return Ok(BooleanArray::from(vec![true; array.len()]));
};
@@ -366,9 +407,48 @@ fn evaluate_exact_leaf_predicate(
let result = evaluate_column_predicate(array, &scalar, op)?;
Ok(sanitize_filter_mask(result))
}
+ PredicateOperator::Between | PredicateOperator::NotBetween => {
+ evaluate_between_predicate(array, data_type, op, literals)
+ }
}
}
+/// `Between` / `NotBetween` translate to `gt_eq(col, low) & lt_eq(col, high)`
+/// (and its negation). `arrow_ord::cmp` produces nullable masks: any null
+/// row makes the comparison null, so a fully-built `Between` mask preserves
+/// nulls. `NotBetween` then negates valid rows and leaves nulls null —
+/// matching SQL three-valued logic; `sanitize_filter_mask` collapses nulls
+/// into `false` to match the predicate evaluator's "NULL → false" rule.
+fn evaluate_between_predicate(
+ array: &ArrayRef,
+ data_type: &DataType,
+ op: PredicateOperator,
+ literals: &[Datum],
+) -> Result<BooleanArray, ArrowError> {
+ let (Some(low), Some(high)) = (literals.first(), literals.get(1)) else {
+ return Ok(BooleanArray::from(vec![true; array.len()]));
+ };
+ let Some(low_scalar) = literal_scalar_for_parquet_filter(low, data_type)
+ .map_err(|e| ArrowError::ComputeError(e.to_string()))?
+ else {
+ return Ok(BooleanArray::from(vec![true; array.len()]));
+ };
+ let Some(high_scalar) = literal_scalar_for_parquet_filter(high, data_type)
+ .map_err(|e| ArrowError::ComputeError(e.to_string()))?
+ else {
+ return Ok(BooleanArray::from(vec![true; array.len()]));
+ };
+ let lo_mask = arrow_gt_eq(array, &low_scalar)?;
+ let hi_mask = arrow_lt_eq(array, &high_scalar)?;
+ let between = arrow_arith::boolean::and_kleene(&lo_mask, &hi_mask)?;
+ let result = match op {
+ PredicateOperator::Between => between,
+ PredicateOperator::NotBetween => arrow_arith::boolean::not(&between)?,
+ _ => unreachable!(),
+ };
+ Ok(sanitize_filter_mask(result))
+}
+
fn evaluate_set_membership_predicate(
array: &ArrayRef,
data_type: &DataType,
@@ -423,13 +503,66 @@ fn evaluate_column_predicate(
PredicateOperator::LtEq => arrow_lt_eq(column, scalar),
PredicateOperator::Gt => arrow_gt(column, scalar),
PredicateOperator::GtEq => arrow_gt_eq(column, scalar),
+ PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like => {
+ let pattern = pattern_scalar_for_string_kernel(scalar,
column.data_type())?;
+ match op {
+ PredicateOperator::StartsWith => arrow_starts_with(column,
&pattern),
+ PredicateOperator::EndsWith => arrow_ends_with(column,
&pattern),
+ PredicateOperator::Contains => arrow_contains(column,
&pattern),
+ PredicateOperator::Like => arrow_like(column, &pattern),
+ _ => unreachable!(),
+ }
+ }
PredicateOperator::IsNull
| PredicateOperator::IsNotNull
| PredicateOperator::In
- | PredicateOperator::NotIn => Ok(BooleanArray::new_null(column.len())),
+ | PredicateOperator::NotIn
+ | PredicateOperator::Between
+ | PredicateOperator::NotBetween =>
Ok(BooleanArray::new_null(column.len())),
}
}
+/// `arrow_string::like::*` kernels reject mismatched string types — Utf8
column
+/// against Utf8 pattern is fine, but a LargeUtf8 / Utf8View column needs a
+/// pattern of the same flavour. The shared scalar built upstream is always
+/// `StringArray` (Utf8); promote it to match the column when needed.
+fn pattern_scalar_for_string_kernel(
+ scalar: &Scalar<ArrayRef>,
+ column_type: &arrow_schema::DataType,
+) -> Result<Scalar<ArrayRef>, ArrowError> {
+ use arrow_array::{LargeStringArray, StringArray, StringViewArray};
+ use arrow_schema::DataType as ArrowDataType;
+
+ let arr = scalar.get().0;
+ let value = arr
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .and_then(|s| (s.len() == 1 && s.is_valid(0)).then(||
s.value(0).to_string()));
+ let Some(value) = value else {
+ return Ok(scalar.clone());
+ };
+ Ok(match column_type {
+ ArrowDataType::Utf8 =>
Scalar::new(Arc::new(StringArray::from(vec![value])) as ArrayRef),
+ ArrowDataType::LargeUtf8 => {
+ Scalar::new(Arc::new(LargeStringArray::from(vec![value])) as
ArrayRef)
+ }
+ ArrowDataType::Utf8View => {
+ Scalar::new(Arc::new(StringViewArray::from(vec![value])) as
ArrayRef)
+ }
+ ArrowDataType::Dictionary(_, value_type) if value_type.as_ref() ==
&ArrowDataType::Utf8 => {
+ Scalar::new(Arc::new(StringArray::from(vec![value])) as ArrayRef)
+ }
+ other => {
+ return Err(ArrowError::InvalidArgumentError(format!(
+ "string predicate against non-string column type {other:?}"
+ )))
+ }
+ })
+}
+
fn sanitize_filter_mask(mask: BooleanArray) -> BooleanArray {
if mask.null_count() == 0 {
return mask;
@@ -1189,6 +1322,138 @@ mod tests {
assert!(row_filter.is_some());
}
+ // -----------------------------------------------------------------------
+ // String predicate tests (StartsWith / EndsWith / Contains)
+ // -----------------------------------------------------------------------
+
+ fn run_string_op(
+ op: super::PredicateOperator,
+ column: arrow_array::ArrayRef,
+ pattern: &str,
+ ) -> arrow_array::BooleanArray {
+ use crate::spec::VarCharType;
+ let dt = DataType::VarChar(VarCharType::default());
+ super::evaluate_exact_leaf_predicate(
+ &column,
+ &dt,
+ op,
+ &[Datum::String(pattern.to_string())],
+ )
+ .expect("string op should evaluate")
+ }
+
+ #[test]
+ fn test_evaluate_starts_with_string_array() {
+ use arrow_array::StringArray;
+ let arr: arrow_array::ArrayRef = Arc::new(StringArray::from(vec![
+ Some("foo"),
+ Some("foobar"),
+ Some("baz"),
+ None,
+ ]));
+ let mask = run_string_op(super::PredicateOperator::StartsWith, arr,
"foo");
+ let expected = arrow_array::BooleanArray::from(vec![true, true, false,
false]);
+ assert_eq!(mask, expected);
+ }
+
+ #[test]
+ fn test_evaluate_ends_with_large_string_array() {
+ use arrow_array::LargeStringArray;
+ let arr: arrow_array::ArrayRef = Arc::new(LargeStringArray::from(vec![
+ Some("hello"),
+ Some("world"),
+ Some("ello"),
+ None,
+ ]));
+ let mask = run_string_op(super::PredicateOperator::EndsWith, arr,
"ello");
+ let expected = arrow_array::BooleanArray::from(vec![true, false, true,
false]);
+ assert_eq!(mask, expected);
+ }
+
+ #[test]
+ fn test_evaluate_contains_string_view_array() {
+ use arrow_array::StringViewArray;
+ let arr: arrow_array::ArrayRef = Arc::new(StringViewArray::from(vec![
+ Some("apple pie"),
+ Some("banana"),
+ Some("crab apple"),
+ None,
+ ]));
+ let mask = run_string_op(super::PredicateOperator::Contains, arr,
"apple");
+ let expected = arrow_array::BooleanArray::from(vec![true, false, true,
false]);
+ assert_eq!(mask, expected);
+ }
+
+ #[test]
+ fn test_evaluate_like_pattern_with_underscore_and_percent() {
+ use arrow_array::StringArray;
+ let arr: arrow_array::ArrayRef = Arc::new(StringArray::from(vec![
+ Some("foobar"),
+ Some("foox"),
+ Some("zoobar"),
+ None,
+ ]));
+ // f_o% matches "foobar" (f-o-o then anything) and "foox" (f-o-o then
x)
+ // but not "zoobar".
+ let mask = run_string_op(super::PredicateOperator::Like, arr, "f_o%");
+ let expected = arrow_array::BooleanArray::from(vec![true, true, false,
false]);
+ assert_eq!(mask, expected);
+ }
+
+ #[test]
+ fn test_evaluate_like_escaped_percent_treated_literally() {
+ use arrow_array::StringArray;
+ let arr: arrow_array::ArrayRef =
+ Arc::new(StringArray::from(vec![Some("100%"), Some("1000"),
None]));
+ let mask = run_string_op(super::PredicateOperator::Like, arr,
r"100\%");
+ let expected = arrow_array::BooleanArray::from(vec![true, false,
false]);
+ assert_eq!(mask, expected);
+ }
+
+ // -----------------------------------------------------------------------
+ // BETWEEN / NOT BETWEEN row-filter tests
+ // -----------------------------------------------------------------------
+
+ fn run_between(
+ op: super::PredicateOperator,
+ column: arrow_array::ArrayRef,
+ low: i32,
+ high: i32,
+ ) -> arrow_array::BooleanArray {
+ let dt = DataType::Int(IntType::new());
+ super::evaluate_exact_leaf_predicate(&column, &dt, op,
&[Datum::Int(low), Datum::Int(high)])
+ .expect("BETWEEN should evaluate")
+ }
+
+ #[test]
+ fn test_evaluate_between_int_array() {
+ let arr: arrow_array::ArrayRef = Arc::new(Int32Array::from(vec![
+ Some(1),
+ Some(5),
+ Some(10),
+ Some(11),
+ None,
+ ]));
+ let mask = run_between(super::PredicateOperator::Between, arr, 5, 10);
+ let expected = arrow_array::BooleanArray::from(vec![false, true, true,
false, false]);
+ assert_eq!(mask, expected);
+ }
+
+ #[test]
+ fn test_evaluate_not_between_treats_null_as_false() {
+ let arr: arrow_array::ArrayRef = Arc::new(Int32Array::from(vec![
+ Some(1),
+ Some(5),
+ Some(10),
+ Some(11),
+ None,
+ ]));
+ let mask = run_between(super::PredicateOperator::NotBetween, arr, 5,
10);
+ // NULL → false (residual filter convention; matches
sanitize_filter_mask).
+ let expected = arrow_array::BooleanArray::from(vec![true, false,
false, true, false]);
+ assert_eq!(mask, expected);
+ }
+
// -----------------------------------------------------------------------
// merge_byte_ranges tests
// -----------------------------------------------------------------------
diff --git a/crates/paimon/src/arrow/format/vortex.rs
b/crates/paimon/src/arrow/format/vortex.rs
index 1d093a3..0a9717d 100644
--- a/crates/paimon/src/arrow/format/vortex.rs
+++ b/crates/paimon/src/arrow/format/vortex.rs
@@ -545,6 +545,15 @@ fn evaluate_arrow_leaf_predicate(
})?;
Ok(Some(sanitize_filter_mask(mask)))
}
+ // Vortex's scalar comparators don't cover substring/range predicates,
so
+ // fall open: returning None defers them to the outer stats-prune +
arrow
+ // row-filter path, which evaluates them with full NULL (Kleene)
semantics.
+ PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like
+ | PredicateOperator::Between
+ | PredicateOperator::NotBetween => Ok(None),
}
}
@@ -609,7 +618,16 @@ fn evaluate_column_predicate(
PredicateOperator::IsNull
| PredicateOperator::IsNotNull
| PredicateOperator::In
- | PredicateOperator::NotIn => Ok(BooleanArray::new_null(column.len())),
+ | PredicateOperator::NotIn
+ // String/range ops never reach the scalar comparator: the leaf
+ // dispatcher falls open (returns None) for them, so the value here is
+ // unused. Listed only to keep the match exhaustive.
+ | PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like
+ | PredicateOperator::Between
+ | PredicateOperator::NotBetween =>
Ok(BooleanArray::new_null(column.len())),
}
}
diff --git a/crates/paimon/src/btree/query.rs b/crates/paimon/src/btree/query.rs
index 3a86bad..0aed9d4 100644
--- a/crates/paimon/src/btree/query.rs
+++ b/crates/paimon/src/btree/query.rs
@@ -96,6 +96,26 @@ where
all_non_null -= excluded;
Ok(all_non_null)
}
+ PredicateOperator::Between => {
+ let from = serialize_datum(&literals[0], data_type);
+ let to = serialize_datum(&literals[1], data_type);
+ self.range_query(&from, &to, true, true).await
+ }
+ PredicateOperator::NotBetween => {
+ let mut all_non_null = self.all_non_null_rows().await?;
+ let from = serialize_datum(&literals[0], data_type);
+ let to = serialize_datum(&literals[1], data_type);
+ let inside = self.range_query(&from, &to, true, true).await?;
+ all_non_null -= inside;
+ Ok(all_non_null)
+ }
+ PredicateOperator::StartsWith
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Like => Err(io::Error::new(
+ io::ErrorKind::Unsupported,
+ format!("BTree index does not support op: {op}"),
+ )),
}
}
}
@@ -116,9 +136,36 @@ pub(crate) type ExtractBetweenResult<'a> = (
/// Try to extract a between pattern (lower + upper bound) from predicates.
/// Returns (between_info, remaining_predicates).
+///
+/// Recognizes two shapes:
+/// 1. A native `Between` leaf with two literals — preferred and emitted by the
+/// DataFusion translator since Stage 3.
+/// 2. A `GtEq` / `Gt` paired with a `LtEq` / `Lt` on the same column —
+/// legacy shape, kept for direct `PredicateBuilder` users that still build
+/// the conjunction explicitly.
pub(crate) fn extract_between<'a>(
predicates: &[(PredicateOperator, &'a [Datum], &'a DataType)],
) -> ExtractBetweenResult<'a> {
+ // Shape 1: native Between leaf — single tuple is enough.
+ for (i, (op, literals, dt)) in predicates.iter().enumerate() {
+ if matches!(op, PredicateOperator::Between) && literals.len() == 2 {
+ let between = BetweenInfo {
+ from: &literals[0],
+ to: &literals[1],
+ from_inclusive: true,
+ to_inclusive: true,
+ data_type: dt,
+ };
+ let remaining: Vec<_> = predicates
+ .iter()
+ .enumerate()
+ .filter(|(j, _)| *j != i)
+ .map(|(_, p)| *p)
+ .collect();
+ return (Some(between), remaining);
+ }
+ }
+
if predicates.len() < 2 {
return (None, predicates.to_vec());
}
diff --git a/crates/paimon/src/predicate_stats.rs
b/crates/paimon/src/predicate_stats.rs
index cb44c72..e0efd67 100644
--- a/crates/paimon/src/predicate_stats.rs
+++ b/crates/paimon/src/predicate_stats.rs
@@ -62,12 +62,30 @@ pub(crate) fn data_leaf_may_match<T: StatsAccessor>(
PredicateOperator::In | PredicateOperator::NotIn => {
return true;
}
+ PredicateOperator::EndsWith | PredicateOperator::Contains => {
+ // String min/max ordering carries no information about suffix /
+ // substring matches, so fail open.
+ return true;
+ }
+ PredicateOperator::Between | PredicateOperator::NotBetween => {
+ return between_may_match(
+ index,
+ stats_data_type,
+ predicate_data_type,
+ op,
+ literals,
+ stats,
+ all_null,
+ );
+ }
PredicateOperator::Eq
| PredicateOperator::NotEq
| PredicateOperator::Lt
| PredicateOperator::LtEq
| PredicateOperator::Gt
- | PredicateOperator::GtEq => {}
+ | PredicateOperator::GtEq
+ | PredicateOperator::StartsWith
+ | PredicateOperator::Like => {}
}
if all_null == Some(true) {
@@ -112,11 +130,100 @@ pub(crate) fn data_leaf_may_match<T: StatsAccessor>(
Some(Ordering::Less | Ordering::Equal)
),
PredicateOperator::GtEq => !matches!(max_value.partial_cmp(literal),
Some(Ordering::Less)),
+ PredicateOperator::StartsWith => {
+ // pat lives in [min, max] iff max >= pat AND min < pat_next, where
+ // pat_next is pat with its last codepoint incremented. If we can't
+ // compute pat_next (last char is char::MAX, increments into the
+ // UTF-16 surrogate range, etc.), fail open.
+ let (pat, min_str, max_str) = match (literal, &min_value,
&max_value) {
+ (Datum::String(p), Datum::String(lo), Datum::String(hi)) => {
+ (p.as_str(), lo.as_str(), hi.as_str())
+ }
+ _ => return true,
+ };
+ // If the file's max is below the pattern (lexicographically), no
+ // string in the file can start with `pat`.
+ if max_str < pat {
+ return false;
+ }
+ // Compute pat_next; if we can, use the [pat, pat_next) range to
+ // also rule out files whose min is already past every pat-prefixed
+ // string. Otherwise just trust the upper bound check.
+ match next_string_for_prefix(pat) {
+ Some(pat_next) => min_str.as_bytes() < pat_next.as_slice(),
+ None => true,
+ }
+ }
+ PredicateOperator::Like => {
+ // Try to extract a literal prefix from the LIKE pattern (the
+ // characters before the first unescaped wildcard). If we get one,
+ // prune as if it were StartsWith; otherwise fail open.
+ let (pattern, min_str, max_str) = match (literal, &min_value,
&max_value) {
+ (Datum::String(p), Datum::String(lo), Datum::String(hi)) => {
+ (p.as_str(), lo.as_str(), hi.as_str())
+ }
+ _ => return true,
+ };
+ let Some(pat) = like_pattern_literal_prefix(pattern) else {
+ return true;
+ };
+ if pat.is_empty() {
+ return true;
+ }
+ if max_str < pat.as_str() {
+ return false;
+ }
+ match next_string_for_prefix(&pat) {
+ Some(pat_next) => min_str.as_bytes() < pat_next.as_slice(),
+ None => true,
+ }
+ }
PredicateOperator::IsNull
| PredicateOperator::IsNotNull
| PredicateOperator::In
- | PredicateOperator::NotIn => true,
+ | PredicateOperator::NotIn
+ | PredicateOperator::EndsWith
+ | PredicateOperator::Contains
+ | PredicateOperator::Between
+ | PredicateOperator::NotBetween => true,
+ }
+}
+
+/// Return the literal prefix of a SQL LIKE pattern up to the first unescaped
+/// `%` or `_`. A backslash escapes the next character (which is appended
+/// literally, mirroring arrow's `like` kernel); a trailing backslash is a
+/// literal backslash.
+fn like_pattern_literal_prefix(pattern: &str) -> Option<String> {
+ let mut out = String::with_capacity(pattern.len());
+ let mut chars = pattern.chars().peekable();
+ while let Some(c) = chars.next() {
+ match c {
+ '%' | '_' => return Some(out),
+ '\\' => match chars.next() {
+ Some(next) => out.push(next),
+ None => out.push('\\'),
+ },
+ other => out.push(other),
+ }
+ }
+ Some(out)
+}
+
+/// Compute the smallest string strictly greater than every string with
`prefix`
+/// as a prefix, by incrementing the last codepoint. Returns `None` if the last
+/// codepoint cannot be incremented within valid Unicode (e.g. `char::MAX`).
+fn next_string_for_prefix(prefix: &str) -> Option<Vec<u8>> {
+ let last_char = prefix.chars().next_back()?;
+ let mut next_code = last_char as u32 + 1;
+ // Skip over the UTF-16 surrogate range, which is not valid scalar Unicode.
+ if (0xD800..=0xDFFF).contains(&next_code) {
+ next_code = 0xE000;
}
+ let next_char = char::from_u32(next_code)?;
+ let mut bytes = prefix.as_bytes()[..prefix.len() -
last_char.len_utf8()].to_vec();
+ let mut buf = [0u8; 4];
+ bytes.extend_from_slice(next_char.encode_utf8(&mut buf).as_bytes());
+ Some(bytes)
}
pub(crate) fn missing_field_may_match(op: PredicateOperator, row_count: i64)
-> bool {
@@ -127,6 +234,60 @@ pub(crate) fn missing_field_may_match(op:
PredicateOperator, row_count: i64) ->
matches!(op, PredicateOperator::IsNull)
}
+/// Stats-prune `field BETWEEN low AND high` (and its negation) by treating it
+/// as the conjunction `field >= low AND field <= high`:
+/// * `Between` may match iff the file's `[min, max]` overlaps `[low, high]`.
+/// * `NotBetween` may match iff some row could fall outside `[low, high]`,
+/// i.e. unless the file's `[min, max]` is entirely inside `[low, high]`.
+///
+/// All-null files are pruned for both ops (NULL comparisons resolve to NULL,
+/// which the evaluator treats as false).
+fn between_may_match<T: StatsAccessor>(
+ index: usize,
+ stats_data_type: &DataType,
+ predicate_data_type: &DataType,
+ op: PredicateOperator,
+ literals: &[Datum],
+ stats: &T,
+ all_null: Option<bool>,
+) -> bool {
+ if all_null == Some(true) {
+ return false;
+ }
+ let (Some(low), Some(high)) = (literals.first(), literals.get(1)) else {
+ return true;
+ };
+ let min_value = match stats
+ .min_value(index, stats_data_type)
+ .and_then(|datum| coerce_stats_datum_for_predicate(datum,
predicate_data_type))
+ {
+ Some(value) => value,
+ None => return true,
+ };
+ let max_value = match stats
+ .max_value(index, stats_data_type)
+ .and_then(|datum| coerce_stats_datum_for_predicate(datum,
predicate_data_type))
+ {
+ Some(value) => value,
+ None => return true,
+ };
+
+ let max_ge_low = !matches!(max_value.partial_cmp(low),
Some(Ordering::Less));
+ let min_le_high = !matches!(min_value.partial_cmp(high),
Some(Ordering::Greater));
+ let overlaps = max_ge_low && min_le_high;
+
+ match op {
+ PredicateOperator::Between => overlaps,
+ PredicateOperator::NotBetween => {
+ // Prune only when [min, max] is entirely inside [low, high].
+ let min_ge_low = !matches!(min_value.partial_cmp(low),
Some(Ordering::Less));
+ let max_le_high = !matches!(max_value.partial_cmp(high),
Some(Ordering::Greater));
+ !(min_ge_low && max_le_high)
+ }
+ _ => unreachable!("between_may_match is only called for
Between/NotBetween"),
+ }
+}
+
fn predicate_may_match_with_schema<T: StatsAccessor>(
predicate: &Predicate,
stats: &T,
@@ -193,3 +354,281 @@ fn coerce_stats_datum_for_predicate(datum: Datum,
predicate_data_type: &DataType
_ => None,
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::spec::{IntType, VarCharType};
+
+ struct MockStats {
+ row_count: i64,
+ null_count: Option<i64>,
+ min: Option<Datum>,
+ max: Option<Datum>,
+ }
+
+ impl StatsAccessor for MockStats {
+ fn row_count(&self) -> i64 {
+ self.row_count
+ }
+ fn null_count(&self, _index: usize) -> Option<i64> {
+ self.null_count
+ }
+ fn min_value(&self, _index: usize, _data_type: &DataType) ->
Option<Datum> {
+ self.min.clone()
+ }
+ fn max_value(&self, _index: usize, _data_type: &DataType) ->
Option<Datum> {
+ self.max.clone()
+ }
+ }
+
+ fn varchar() -> DataType {
+ DataType::VarChar(VarCharType::default())
+ }
+
+ fn string_stats(min: &str, max: &str) -> MockStats {
+ MockStats {
+ row_count: 10,
+ null_count: Some(0),
+ min: Some(Datum::String(min.to_string())),
+ max: Some(Datum::String(max.to_string())),
+ }
+ }
+
+ fn run(op: PredicateOperator, lit: &str, stats: &MockStats) -> bool {
+ let dt = varchar();
+ data_leaf_may_match(0, &dt, &dt, op,
&[Datum::String(lit.to_string())], stats)
+ }
+
+ #[test]
+ fn starts_with_prunes_when_max_below_pattern() {
+ let stats = string_stats("aaa", "fooa");
+ assert!(!run(PredicateOperator::StartsWith, "foob", &stats));
+ }
+
+ #[test]
+ fn starts_with_prunes_when_min_past_pattern_range() {
+ // [foo, fop) is the pat range. min "fop" is already past it.
+ let stats = string_stats("fop", "zzz");
+ assert!(!run(PredicateOperator::StartsWith, "foo", &stats));
+ }
+
+ #[test]
+ fn starts_with_keeps_when_pattern_inside_range() {
+ let stats = string_stats("aaa", "zzz");
+ assert!(run(PredicateOperator::StartsWith, "foo", &stats));
+ }
+
+ #[test]
+ fn starts_with_keeps_when_min_equals_pattern() {
+ let stats = string_stats("foo", "foozzz");
+ assert!(run(PredicateOperator::StartsWith, "foo", &stats));
+ }
+
+ #[test]
+ fn starts_with_falls_open_when_stats_missing() {
+ let stats = MockStats {
+ row_count: 5,
+ null_count: Some(0),
+ min: None,
+ max: None,
+ };
+ assert!(run(PredicateOperator::StartsWith, "foo", &stats));
+ }
+
+ #[test]
+ fn ends_with_and_contains_fall_open() {
+ let stats = string_stats("aaa", "zzz");
+ assert!(run(PredicateOperator::EndsWith, "foo", &stats));
+ assert!(run(PredicateOperator::Contains, "foo", &stats));
+ }
+
+ #[test]
+ fn like_with_literal_prefix_prunes_like_starts_with() {
+ // pattern "foo%" → prefix "foo"; max "fooa" already past pattern end?
+ // No: max = "fooa" >= "foo" and min "aaa" < "fop". So this case keeps.
+ let stats = string_stats("aaa", "fooa");
+ assert!(run(PredicateOperator::Like, "foo%", &stats));
+ // pattern "foo%": [foo, fop). file [zaa, zzz] — max < pat → prune.
+ let stats = string_stats("zaa", "zzz");
+ assert!(!run(PredicateOperator::Like, "foo%", &stats));
+ // file [fop, zzz] — min already past prefix range → prune.
+ let stats = string_stats("fop", "zzz");
+ assert!(!run(PredicateOperator::Like, "foo%", &stats));
+ }
+
+ #[test]
+ fn like_without_literal_prefix_falls_open() {
+ let stats = string_stats("aaa", "ccc");
+ // Leading wildcard → no prefix → fail open.
+ assert!(run(PredicateOperator::Like, "%foo%", &stats));
+ // Leading underscore → no prefix → fail open.
+ assert!(run(PredicateOperator::Like, "_oo", &stats));
+ }
+
+ #[test]
+ fn like_with_escaped_wildcard_in_prefix_is_decoded() {
+ // "100\%foo" → literal prefix "100%foo".
+ let stats = string_stats("100", "100%fzz");
+ assert!(run(PredicateOperator::Like, r"100\%foo", &stats));
+ let stats = string_stats("zzz0", "zzz9");
+ assert!(!run(PredicateOperator::Like, r"100\%foo", &stats));
+ }
+
+ #[test]
+ fn missing_field_returns_false_for_string_ops() {
+ // Only IsNull is allowed when the field is missing.
+ for op in [
+ PredicateOperator::StartsWith,
+ PredicateOperator::EndsWith,
+ PredicateOperator::Contains,
+ PredicateOperator::Like,
+ ] {
+ assert!(!missing_field_may_match(op, 5));
+ }
+ }
+
+ // Sanity check: integer ops keep their existing semantics after the new
+ // string variants are interleaved into the dispatcher.
+ #[test]
+ fn integer_eq_still_prunes_outside_range() {
+ let dt = DataType::Int(IntType::new());
+ let stats = MockStats {
+ row_count: 10,
+ null_count: Some(0),
+ min: Some(Datum::Int(0)),
+ max: Some(Datum::Int(100)),
+ };
+ assert!(!data_leaf_may_match(
+ 0,
+ &dt,
+ &dt,
+ PredicateOperator::Eq,
+ &[Datum::Int(500)],
+ &stats,
+ ));
+ }
+
+ fn int_stats(min: i32, max: i32) -> MockStats {
+ MockStats {
+ row_count: 10,
+ null_count: Some(0),
+ min: Some(Datum::Int(min)),
+ max: Some(Datum::Int(max)),
+ }
+ }
+
+ fn run_int(op: PredicateOperator, lits: &[Datum], stats: &MockStats) ->
bool {
+ let dt = DataType::Int(IntType::new());
+ data_leaf_may_match(0, &dt, &dt, op, lits, stats)
+ }
+
+ /// Stage 3 invariant: a `Between` leaf and the equivalent `GtEq+LtEq`
+ /// conjunction must produce identical stats-prune verdicts. If they
+ /// diverge, the DataFusion translator switch (And-of-comparisons →
+ /// Between leaf) silently changes pruning behavior in production.
+ #[test]
+ fn between_matches_gteq_lteq_conjunction() {
+ let cases: &[(i32, i32, i32, i32, bool)] = &[
+ // (min, max, low, high, expected_may_match)
+ (0, 100, 50, 60, true), // overlap inside
+ (0, 100, 200, 300, false), // entirely above
+ (0, 100, -50, -1, false), // entirely below
+ (0, 100, 100, 100, true), // boundary high
+ (0, 100, 0, 0, true), // boundary low
+ (50, 100, 0, 49, false), // low < min < high < max impossible —
fully below
+ (50, 100, 0, 200, true), // file fully inside [low, high]
+ ];
+ for &(min, max, low, high, expected) in cases {
+ let stats = int_stats(min, max);
+ let between = run_int(
+ PredicateOperator::Between,
+ &[Datum::Int(low), Datum::Int(high)],
+ &stats,
+ );
+ let gteq = run_int(PredicateOperator::GtEq, &[Datum::Int(low)],
&stats);
+ let lteq = run_int(PredicateOperator::LtEq, &[Datum::Int(high)],
&stats);
+ assert_eq!(
+ between,
+ gteq && lteq,
+ "Between vs GtEq+LtEq divergence at ({min},{max}) ∩
[{low},{high}]"
+ );
+ assert_eq!(
+ between, expected,
+ "Between unexpected at {min},{max} ∩ [{low},{high}]"
+ );
+ }
+ }
+
+ #[test]
+ fn not_between_prunes_only_when_file_fully_inside_range() {
+ // file [10, 20] ⊆ [0, 100] → all rows are within [0, 100], so NOT
+ // BETWEEN can prune.
+ let stats = int_stats(10, 20);
+ assert!(!run_int(
+ PredicateOperator::NotBetween,
+ &[Datum::Int(0), Datum::Int(100)],
+ &stats,
+ ));
+ // file [0, 100] ⊃ [10, 20] → some rows lie outside, can't prune.
+ let stats = int_stats(0, 100);
+ assert!(run_int(
+ PredicateOperator::NotBetween,
+ &[Datum::Int(10), Datum::Int(20)],
+ &stats,
+ ));
+ // file disjoint with [50, 60] → all rows are outside, can't prune.
+ let stats = int_stats(0, 10);
+ assert!(run_int(
+ PredicateOperator::NotBetween,
+ &[Datum::Int(50), Datum::Int(60)],
+ &stats,
+ ));
+ }
+
+ #[test]
+ fn between_with_all_null_file_is_pruned() {
+ let dt = DataType::Int(IntType::new());
+ let stats = MockStats {
+ row_count: 10,
+ null_count: Some(10),
+ min: None,
+ max: None,
+ };
+ assert!(!data_leaf_may_match(
+ 0,
+ &dt,
+ &dt,
+ PredicateOperator::Between,
+ &[Datum::Int(0), Datum::Int(100)],
+ &stats,
+ ));
+ assert!(!data_leaf_may_match(
+ 0,
+ &dt,
+ &dt,
+ PredicateOperator::NotBetween,
+ &[Datum::Int(0), Datum::Int(100)],
+ &stats,
+ ));
+ }
+
+ #[test]
+ fn between_falls_open_when_stats_missing() {
+ let dt = DataType::Int(IntType::new());
+ let stats = MockStats {
+ row_count: 5,
+ null_count: Some(0),
+ min: None,
+ max: None,
+ };
+ assert!(data_leaf_may_match(
+ 0,
+ &dt,
+ &dt,
+ PredicateOperator::Between,
+ &[Datum::Int(0), Datum::Int(100)],
+ &stats,
+ ));
+ }
+}
diff --git a/crates/paimon/src/spec/predicate.rs
b/crates/paimon/src/spec/predicate.rs
index 73b6c0c..a419f77 100644
--- a/crates/paimon/src/spec/predicate.rs
+++ b/crates/paimon/src/spec/predicate.rs
@@ -223,6 +223,12 @@ pub enum PredicateOperator {
GtEq,
In,
NotIn,
+ StartsWith,
+ EndsWith,
+ Contains,
+ Like,
+ Between,
+ NotBetween,
}
impl fmt::Display for PredicateOperator {
@@ -238,6 +244,12 @@ impl fmt::Display for PredicateOperator {
Self::GtEq => write!(f, ">="),
Self::In => write!(f, "IN"),
Self::NotIn => write!(f, "NOT IN"),
+ Self::StartsWith => write!(f, "STARTS_WITH"),
+ Self::EndsWith => write!(f, "ENDS_WITH"),
+ Self::Contains => write!(f, "CONTAINS"),
+ Self::Like => write!(f, "LIKE"),
+ Self::Between => write!(f, "BETWEEN"),
+ Self::NotBetween => write!(f, "NOT BETWEEN"),
}
}
}
@@ -650,6 +662,113 @@ impl PredicateBuilder {
self.leaf(field, PredicateOperator::NotIn, literals)
}
+ // -- string operators --
+
+ /// `field LIKE 'pat%'` shape. Empty pattern → `IsNotNull(field)` (every
+ /// non-null string starts with the empty string). Non-string `pattern`
+ /// → [`Error::ConfigInvalid`].
+ pub fn starts_with(&self, field: &str, pattern: Datum) ->
Result<Predicate> {
+ self.string_leaf(field, PredicateOperator::StartsWith, pattern)
+ }
+
+ /// `field LIKE '%pat'` shape. Empty pattern → `IsNotNull(field)`.
+ pub fn ends_with(&self, field: &str, pattern: Datum) -> Result<Predicate> {
+ self.string_leaf(field, PredicateOperator::EndsWith, pattern)
+ }
+
+ /// `field LIKE '%pat%'` shape. Empty pattern → `IsNotNull(field)`.
+ pub fn contains(&self, field: &str, pattern: Datum) -> Result<Predicate> {
+ self.string_leaf(field, PredicateOperator::Contains, pattern)
+ }
+
+ /// `field LIKE '<pattern>'` with optional `escape` character (default
`\`).
+ /// Mirrors Java `LikeOptimization`: rewrites `prefix%` / `%suffix` /
+ /// `%mid%` / no-wildcard patterns into [`PredicateOperator::StartsWith`] /
+ /// [`PredicateOperator::EndsWith`] / [`PredicateOperator::Contains`] /
+ /// [`PredicateOperator::Eq`]; falls back to a [`PredicateOperator::Like`]
+ /// leaf for anything more complex (`_`, multi-segment `%`, escaped
+ /// wildcards). The `Like` evaluator follows arrow_string `like` kernel
+ /// semantics for the residual cases.
+ ///
+ /// `escape == None` defaults to `\`. Any other ESCAPE character is
+ /// rejected with [`Error::ConfigInvalid`] (the DataFusion translator turns
+ /// that into a fall-open). Empty pattern → [`PredicateOperator::Eq`] of
the
+ /// empty string (SQL semantics: only the empty string matches).
+ pub fn like(&self, field: &str, pattern: Datum, escape: Option<char>) ->
Result<Predicate> {
+ let pattern_str = match &pattern {
+ Datum::String(s) => s.clone(),
+ other => {
+ return Err(Error::ConfigInvalid {
+ message: format!("LIKE requires a string pattern, got
{other}"),
+ });
+ }
+ };
+ let escape_char = escape.unwrap_or('\\');
+ if escape_char != '\\' {
+ return Err(Error::ConfigInvalid {
+ message: format!(
+ "LIKE escape character {escape_char:?} is not supported
(only '\\\\')"
+ ),
+ });
+ }
+
+ match optimize_like_pattern(&pattern_str) {
+ LikeShape::EmptyOrLiteral(s) => self.equal(field,
Datum::String(s)),
+ LikeShape::StartsWith(prefix) => self.starts_with(field,
Datum::String(prefix)),
+ LikeShape::EndsWith(suffix) => self.ends_with(field,
Datum::String(suffix)),
+ LikeShape::Contains(mid) => self.contains(field,
Datum::String(mid)),
+ LikeShape::Residual => self.leaf(field, PredicateOperator::Like,
vec![pattern]),
+ }
+ }
+
+ // -- range operators --
+
+ /// `field BETWEEN low AND high`. SQL semantics: inclusive on both ends.
+ /// `low > high` short-circuits to [`Predicate::AlwaysFalse`] (no value can
+ /// be between an empty range).
+ pub fn between(&self, field: &str, low: Datum, high: Datum) ->
Result<Predicate> {
+ if Self::low_strictly_above_high(&low, &high) {
+ return Ok(Predicate::AlwaysFalse);
+ }
+ self.leaf(field, PredicateOperator::Between, vec![low, high])
+ }
+
+ /// `field NOT BETWEEN low AND high`. SQL three-valued logic: NULL value
+ /// → NULL → treated as false, matching the existing `NotEq` evaluator.
+ /// `low > high` short-circuits to `IsNotNull(field)` (every non-null value
+ /// is "not between" an empty range).
+ pub fn not_between(&self, field: &str, low: Datum, high: Datum) ->
Result<Predicate> {
+ if Self::low_strictly_above_high(&low, &high) {
+ return self.is_not_null(field);
+ }
+ self.leaf(field, PredicateOperator::NotBetween, vec![low, high])
+ }
+
+ fn low_strictly_above_high(low: &Datum, high: &Datum) -> bool {
+ matches!(datum_cmp(low, high), Some(Ordering::Greater))
+ }
+
+ /// Shared body for the three string operators: empty-string short-circuit
+ /// and literal-type guard ([`leaf`] still cross-checks against the column
+ /// type, so non-string columns are rejected there).
+ fn string_leaf(&self, field: &str, op: PredicateOperator, pattern: Datum)
-> Result<Predicate> {
+ match &pattern {
+ // Every non-null string starts with / ends with / contains the
+ // empty string, and a NULL value matches none of them — i.e. the
+ // empty pattern is exactly `IsNotNull`. Folding to `AlwaysTrue`
+ // would wrongly retain NULL rows (and drop the field reference,
+ // keeping the predicate out of the data-pruning path).
+ Datum::String(s) if s.is_empty() => return self.is_not_null(field),
+ Datum::String(_) => {}
+ other => {
+ return Err(Error::ConfigInvalid {
+ message: format!("{op} requires a string pattern, got
{other}"),
+ });
+ }
+ }
+ self.leaf(field, op, vec![pattern])
+ }
+
// -- internal --
/// Resolve field name to index + type, validate literals, and build a
leaf predicate.
@@ -703,6 +822,7 @@ impl PredicateBuilder {
message: format!("{op} expects at least 1 literal, got 0"),
});
}
+ PredicateOperator::Between | PredicateOperator::NotBetween => (2,
literals.len()),
_ => (1, literals.len()),
};
if actual != expected {
@@ -914,6 +1034,40 @@ fn eval_leaf(op: PredicateOperator, datum:
Option<&Datum>, literals: &[Datum]) -
),
PredicateOperator::In => literals.iter().any(|lit|
datum_eq(val, lit)),
PredicateOperator::NotIn => !literals.iter().any(|lit|
datum_eq(val, lit)),
+ PredicateOperator::StartsWith => match (val, literals.first())
{
+ (Datum::String(haystack), Some(Datum::String(needle))) => {
+ haystack.starts_with(needle.as_str())
+ }
+ _ => unreachable!(
+ "STARTS_WITH must have Datum::String value and literal
(validated by builder)"
+ ),
+ },
+ PredicateOperator::EndsWith => match (val, literals.first()) {
+ (Datum::String(haystack), Some(Datum::String(needle))) => {
+ haystack.ends_with(needle.as_str())
+ }
+ _ => unreachable!(
+ "ENDS_WITH must have Datum::String value and literal
(validated by builder)"
+ ),
+ },
+ PredicateOperator::Contains => match (val, literals.first()) {
+ (Datum::String(haystack), Some(Datum::String(needle))) => {
+ haystack.contains(needle.as_str())
+ }
+ _ => unreachable!(
+ "CONTAINS must have Datum::String value and literal
(validated by builder)"
+ ),
+ },
+ PredicateOperator::Like => match (val, literals.first()) {
+ (Datum::String(haystack), Some(Datum::String(pattern))) =>
{
+ like_match(haystack, pattern)
+ }
+ _ => unreachable!(
+ "LIKE must have Datum::String value and literal
(validated by builder)"
+ ),
+ },
+ PredicateOperator::Between => eval_between(val, literals),
+ PredicateOperator::NotBetween => !eval_between(val, literals),
// IsNull/IsNotNull are handled in the outer match above.
PredicateOperator::IsNull | PredicateOperator::IsNotNull =>
unreachable!(),
}
@@ -921,6 +1075,158 @@ fn eval_leaf(op: PredicateOperator, datum:
Option<&Datum>, literals: &[Datum]) -
}
}
+// ---------------------------------------------------------------------------
+// LIKE pattern optimization & evaluation
+// ---------------------------------------------------------------------------
+
+/// Result of LIKE pattern shape analysis. The `String` payloads are the
+/// literal substrings extracted from the pattern (with escape sequences
+/// already decoded).
+enum LikeShape {
+ /// Empty pattern, or a pattern that contains no wildcards / escapes —
+ /// equivalent to `Eq <literal>` (where the literal is the unescaped
+ /// pattern, possibly empty).
+ EmptyOrLiteral(String),
+ /// `prefix%` (exactly one trailing `%`, no other wildcards or escapes).
+ StartsWith(String),
+ /// `%suffix`.
+ EndsWith(String),
+ /// `%mid%` (exactly one leading and one trailing `%`).
+ Contains(String),
+ /// Anything else: `_`, multi-segment `%`, or any escape sequence — must
+ /// fall through to a `Like` leaf evaluator.
+ Residual,
+}
+
+/// Classify a SQL LIKE pattern. The escape character is hardcoded to `\` —
+/// callers wanting any other escape should bypass optimization and surface a
+/// `Like` leaf directly. Any presence of `\` in the pattern forces
+/// [`LikeShape::Residual`] (the simple shape rules don't account for escaped
+/// wildcards).
+fn optimize_like_pattern(pattern: &str) -> LikeShape {
+ if pattern.contains('\\') || pattern.contains('_') {
+ return LikeShape::Residual;
+ }
+ let bytes = pattern.as_bytes();
+ let percent_count = bytes.iter().filter(|b| **b == b'%').count();
+ match percent_count {
+ 0 => LikeShape::EmptyOrLiteral(pattern.to_string()),
+ 1 => {
+ if let Some(prefix) = pattern.strip_suffix('%') {
+ LikeShape::StartsWith(prefix.to_string())
+ } else if let Some(suffix) = pattern.strip_prefix('%') {
+ LikeShape::EndsWith(suffix.to_string())
+ } else {
+ LikeShape::Residual
+ }
+ }
+ 2 if pattern.starts_with('%') && pattern.ends_with('%') => {
+ // `%%` reduces to `Contains('')`, which itself short-circuits to
+ // `IsNotNull` at the StartsWith/EndsWith/Contains builder boundary
+ // — so no special-casing here.
+ LikeShape::Contains(pattern[1..pattern.len() - 1].to_string())
+ }
+ _ => LikeShape::Residual,
+ }
+}
+
+/// Evaluate a SQL LIKE pattern against a value. Implements the backtracking
+/// matcher used by `arrow_string::like::like`:
+/// * `%` matches any (possibly empty) substring,
+/// * `_` matches exactly one character,
+/// * `\X` matches the literal `X` for any character `X` (the backslash is
+/// consumed); a trailing `\` matches a literal backslash.
+fn like_match(value: &str, pattern: &str) -> bool {
+ let value: Vec<char> = value.chars().collect();
+ let pattern: Vec<char> = pattern.chars().collect();
+ like_match_chars(&value, 0, &pattern, 0)
+}
+
+fn like_match_chars(value: &[char], mut vi: usize, pattern: &[char], mut pi:
usize) -> bool {
+ while pi < pattern.len() {
+ match pattern[pi] {
+ '%' => {
+ // Collapse runs of `%` and try every possible suffix.
+ while pi < pattern.len() && pattern[pi] == '%' {
+ pi += 1;
+ }
+ if pi == pattern.len() {
+ return true;
+ }
+ while vi <= value.len() {
+ if like_match_chars(value, vi, pattern, pi) {
+ return true;
+ }
+ if vi == value.len() {
+ return false;
+ }
+ vi += 1;
+ }
+ return false;
+ }
+ '_' => {
+ if vi == value.len() {
+ return false;
+ }
+ vi += 1;
+ pi += 1;
+ }
+ '\\' => {
+ // Mirror arrow's `like` kernel: a backslash consumes the next
+ // character and matches it literally, whatever it is (`%`,
`_`,
+ // `\`, or any other char such as `a`). A trailing backslash
+ // matches a literal backslash.
+ let expected = match pattern.get(pi + 1) {
+ Some(&next) => {
+ pi += 2;
+ next
+ }
+ None => {
+ pi += 1;
+ '\\'
+ }
+ };
+ if vi == value.len() || value[vi] != expected {
+ return false;
+ }
+ vi += 1;
+ }
+ other => {
+ if vi == value.len() || value[vi] != other {
+ return false;
+ }
+ vi += 1;
+ pi += 1;
+ }
+ }
+ }
+ vi == value.len()
+}
+
+// ---------------------------------------------------------------------------
+// BETWEEN evaluation
+// ---------------------------------------------------------------------------
+
+/// Evaluate `value BETWEEN low AND high` (inclusive). `NotBetween` is the
+/// boolean complement at the call site, which is correct for non-null
+/// `value` — null handling already short-circuits in the outer `eval_leaf`.
+/// Returns `false` when either comparison is incomparable (defensive: the
+/// builder validates types up front, so this is unreachable in practice).
+fn eval_between(value: &Datum, literals: &[Datum]) -> bool {
+ let (Some(low), Some(high)) = (literals.first(), literals.get(1)) else {
+ unreachable!("BETWEEN must have 2 literals (validated by builder)");
+ };
+ let above_low = matches!(
+ datum_cmp(value, low),
+ Some(Ordering::Greater | Ordering::Equal)
+ );
+ let below_high = matches!(
+ datum_cmp(value, high),
+ Some(Ordering::Less | Ordering::Equal)
+ );
+ above_low && below_high
+}
+
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
@@ -1893,4 +2199,321 @@ mod tests {
.project_field_index_inclusive(&mapping)
.is_none());
}
+
+ // ======================== string operators ========================
+
+ #[test]
+ fn test_builder_starts_with() {
+ let pb = PredicateBuilder::new(&test_fields());
+ let pred = pb
+ .starts_with("name", Datum::String("foo".to_string()))
+ .unwrap();
+ match &pred {
+ Predicate::Leaf { op, literals, .. } => {
+ assert_eq!(*op, PredicateOperator::StartsWith);
+ assert_eq!(literals, &[Datum::String("foo".to_string())]);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_builder_ends_with() {
+ let pb = PredicateBuilder::new(&test_fields());
+ let pred = pb
+ .ends_with("name", Datum::String("bar".to_string()))
+ .unwrap();
+ match &pred {
+ Predicate::Leaf { op, .. } => assert_eq!(*op,
PredicateOperator::EndsWith),
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_builder_contains() {
+ let pb = PredicateBuilder::new(&test_fields());
+ let pred = pb
+ .contains("name", Datum::String("baz".to_string()))
+ .unwrap();
+ match &pred {
+ Predicate::Leaf { op, .. } => assert_eq!(*op,
PredicateOperator::Contains),
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_builder_string_ops_empty_pattern_is_not_null() {
+ let pb = PredicateBuilder::new(&test_fields());
+ // An empty pattern matches every non-null string and no NULL, so it is
+ // exactly `IsNotNull` (not `AlwaysTrue`, which would retain NULL
rows).
+ for build in [
+ pb.starts_with("name", Datum::String(String::new())),
+ pb.ends_with("name", Datum::String(String::new())),
+ pb.contains("name", Datum::String(String::new())),
+ ] {
+ assert!(matches!(
+ build.unwrap(),
+ Predicate::Leaf {
+ op: PredicateOperator::IsNotNull,
+ ..
+ }
+ ));
+ }
+ }
+
+ #[test]
+ fn test_builder_string_ops_reject_non_string_pattern() {
+ let pb = PredicateBuilder::new(&test_fields());
+ assert!(pb.starts_with("name", Datum::Int(1)).is_err());
+ assert!(pb.ends_with("name", Datum::Int(1)).is_err());
+ assert!(pb.contains("name", Datum::Int(1)).is_err());
+ }
+
+ #[test]
+ fn test_builder_string_ops_reject_non_string_column() {
+ let pb = PredicateBuilder::new(&test_fields());
+ // `id` is Int, so a String literal fails the cross-check inside
leaf().
+ assert!(pb
+ .starts_with("id", Datum::String("x".to_string()))
+ .is_err());
+ }
+
+ #[test]
+ fn test_eval_string_operators() {
+ let lit = Datum::String("oo".to_string());
+ let val = Datum::String("foobar".to_string());
+
+ assert!(eval_leaf(
+ PredicateOperator::Contains,
+ Some(&val),
+ std::slice::from_ref(&lit),
+ ));
+ assert!(!eval_leaf(
+ PredicateOperator::StartsWith,
+ Some(&val),
+ std::slice::from_ref(&lit),
+ ));
+ assert!(eval_leaf(
+ PredicateOperator::StartsWith,
+ Some(&val),
+ &[Datum::String("foo".to_string())],
+ ));
+ assert!(eval_leaf(
+ PredicateOperator::EndsWith,
+ Some(&val),
+ &[Datum::String("bar".to_string())],
+ ));
+ assert!(!eval_leaf(
+ PredicateOperator::EndsWith,
+ Some(&val),
+ &[Datum::String("baz".to_string())],
+ ));
+ // NULL value → false (SQL three-valued logic).
+ assert!(!eval_leaf(
+ PredicateOperator::StartsWith,
+ None,
+ &[Datum::String("foo".to_string())],
+ ));
+ }
+
+ // ======================== LIKE operator ========================
+
+ fn assert_leaf(pred: &Predicate, expected_op: PredicateOperator,
expected_lit: &str) {
+ match pred {
+ Predicate::Leaf { op, literals, .. } => {
+ assert_eq!(*op, expected_op, "op mismatch");
+ assert_eq!(literals,
&[Datum::String(expected_lit.to_string())]);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_like_optimization_rewrites() {
+ let pb = PredicateBuilder::new(&test_fields());
+ // No wildcards → Eq.
+ assert_leaf(
+ &pb.like("name", Datum::String("foo".to_string()), None)
+ .unwrap(),
+ PredicateOperator::Eq,
+ "foo",
+ );
+ // Empty pattern → Eq("") (only empty string matches).
+ assert_leaf(
+ &pb.like("name", Datum::String(String::new()), None).unwrap(),
+ PredicateOperator::Eq,
+ "",
+ );
+ // prefix% → StartsWith.
+ assert_leaf(
+ &pb.like("name", Datum::String("foo%".to_string()), None)
+ .unwrap(),
+ PredicateOperator::StartsWith,
+ "foo",
+ );
+ // %suffix → EndsWith.
+ assert_leaf(
+ &pb.like("name", Datum::String("%bar".to_string()), None)
+ .unwrap(),
+ PredicateOperator::EndsWith,
+ "bar",
+ );
+ // %mid% → Contains.
+ assert_leaf(
+ &pb.like("name", Datum::String("%baz%".to_string()), None)
+ .unwrap(),
+ PredicateOperator::Contains,
+ "baz",
+ );
+ }
+
+ #[test]
+ fn test_like_residual_for_non_optimizable_patterns() {
+ let pb = PredicateBuilder::new(&test_fields());
+ // `_` keeps Like leaf.
+ assert_leaf(
+ &pb.like("name", Datum::String("f_o".to_string()), None)
+ .unwrap(),
+ PredicateOperator::Like,
+ "f_o",
+ );
+ // Multi-segment % keeps Like leaf.
+ assert_leaf(
+ &pb.like("name", Datum::String("a%b%c".to_string()), None)
+ .unwrap(),
+ PredicateOperator::Like,
+ "a%b%c",
+ );
+ // Escaped wildcards keep Like leaf (optimization is conservative).
+ assert_leaf(
+ &pb.like("name", Datum::String(r"foo\%".to_string()), None)
+ .unwrap(),
+ PredicateOperator::Like,
+ r"foo\%",
+ );
+ }
+
+ #[test]
+ fn test_like_rejects_custom_escape_char() {
+ let pb = PredicateBuilder::new(&test_fields());
+ assert!(pb
+ .like("name", Datum::String("a$%".to_string()), Some('$'))
+ .is_err());
+ }
+
+ #[test]
+ fn test_like_rejects_non_string_pattern() {
+ let pb = PredicateBuilder::new(&test_fields());
+ assert!(pb.like("name", Datum::Int(1), None).is_err());
+ }
+
+ #[test]
+ fn test_like_match_evaluator() {
+ // Patterns that fall back to Like leaf must evaluate correctly.
+ assert!(super::like_match("foobar", "f_o%"));
+ assert!(super::like_match("foobar", "%bar"));
+ assert!(super::like_match("foobar", "f%r"));
+ assert!(!super::like_match("foobar", "f_x%"));
+ // `_` requires exactly one character.
+ assert!(super::like_match("ab", "a_"));
+ assert!(!super::like_match("a", "a_"));
+ assert!(!super::like_match("abc", "a_"));
+ // Escape handling.
+ assert!(super::like_match("100%", "100\\%"));
+ assert!(!super::like_match("1000", "100\\%"));
+ assert!(super::like_match("a_b", "a\\_b"));
+ assert!(!super::like_match("axb", "a\\_b"));
+ // Escaped non-wildcard: `\X` matches literal `X`, not `\X` (arrow
+ // semantics, verified against arrow_string's like_escape test).
+ assert!(super::like_match("a", "\\a"));
+ assert!(!super::like_match("\\a", "\\a"));
+ // Trailing backslash matches a literal backslash.
+ assert!(super::like_match("\\", "\\"));
+ // Empty pattern only matches empty value.
+ assert!(super::like_match("", ""));
+ assert!(!super::like_match("a", ""));
+ }
+
+ // ======================== BETWEEN / NOT BETWEEN ========================
+
+ #[test]
+ fn test_builder_between_keeps_inclusive_range() {
+ let pb = PredicateBuilder::new(&test_fields());
+ let pred = pb.between("id", Datum::Int(1), Datum::Int(10)).unwrap();
+ match &pred {
+ Predicate::Leaf { op, literals, .. } => {
+ assert_eq!(*op, PredicateOperator::Between);
+ assert_eq!(literals, &[Datum::Int(1), Datum::Int(10)]);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_builder_between_low_above_high_short_circuits_to_always_false() {
+ let pb = PredicateBuilder::new(&test_fields());
+ let pred = pb.between("id", Datum::Int(10), Datum::Int(1)).unwrap();
+ assert!(matches!(pred, Predicate::AlwaysFalse));
+ }
+
+ #[test]
+ fn test_builder_not_between_low_above_high_short_circuits_to_is_not_null()
{
+ let pb = PredicateBuilder::new(&test_fields());
+ let pred = pb.not_between("id", Datum::Int(10),
Datum::Int(1)).unwrap();
+ match &pred {
+ Predicate::Leaf { op, .. } => assert_eq!(*op,
PredicateOperator::IsNotNull),
+ other => panic!("expected IsNotNull leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_builder_between_rejects_type_mismatch() {
+ let pb = PredicateBuilder::new(&test_fields());
+ // `id` is Int — String literal violates leaf() type cross-check.
+ assert!(pb
+ .between("id", Datum::String("a".to_string()), Datum::Int(10))
+ .is_err());
+ }
+
+ #[test]
+ fn test_eval_between_inclusive() {
+ let lits = [Datum::Int(5), Datum::Int(10)];
+ for v in [5, 7, 10] {
+ assert!(eval_leaf(
+ PredicateOperator::Between,
+ Some(&Datum::Int(v)),
+ &lits,
+ ));
+ }
+ for v in [4, 11] {
+ assert!(!eval_leaf(
+ PredicateOperator::Between,
+ Some(&Datum::Int(v)),
+ &lits,
+ ));
+ }
+ // NULL value → false.
+ assert!(!eval_leaf(PredicateOperator::Between, None, &lits));
+ }
+
+ #[test]
+ fn test_eval_not_between_complement_with_null_false() {
+ let lits = [Datum::Int(5), Datum::Int(10)];
+ for v in [4, 11] {
+ assert!(eval_leaf(
+ PredicateOperator::NotBetween,
+ Some(&Datum::Int(v)),
+ &lits,
+ ));
+ }
+ for v in [5, 7, 10] {
+ assert!(!eval_leaf(
+ PredicateOperator::NotBetween,
+ Some(&Datum::Int(v)),
+ &lits,
+ ));
+ }
+ // NULL value → false (matches existing NotEq null-semantics).
+ assert!(!eval_leaf(PredicateOperator::NotBetween, None, &lits));
+ }
}
diff --git a/crates/paimon/src/table/global_index_scanner.rs
b/crates/paimon/src/table/global_index_scanner.rs
index cdbccac..dd2ab09 100644
--- a/crates/paimon/src/table/global_index_scanner.rs
+++ b/crates/paimon/src/table/global_index_scanner.rs
@@ -134,6 +134,9 @@ impl GlobalIndexScanner {
data_type,
..
} => {
+ if !is_btree_supported_op(*op) {
+ return Ok(None);
+ }
let field_id = self.find_field_id_by_name(column)?;
let field_id = match field_id {
Some(id) => id,
@@ -161,14 +164,16 @@ impl GlobalIndexScanner {
..
} = child
{
- if let Some(field_id) =
self.find_field_id_by_name(column)? {
- if self.entries_for_field(field_id).is_some() {
-
leaf_groups.entry(field_id).or_default().push((
- *op,
- literals.as_slice(),
- data_type,
- ));
- continue;
+ if is_btree_supported_op(*op) {
+ if let Some(field_id) =
self.find_field_id_by_name(column)? {
+ if
self.entries_for_field(field_id).is_some() {
+
leaf_groups.entry(field_id).or_default().push((
+ *op,
+ literals.as_slice(),
+ data_type,
+ ));
+ continue;
+ }
}
}
}
@@ -270,6 +275,17 @@ impl GlobalIndexScanner {
entry.meta.may_match_between(&from_key, &to_key, &cmp)
});
+ // When a Between conjunct exists but the file does not overlap its
+ // range, the whole AND cannot match — drop the file regardless of
+ // how the remaining predicates evaluate. Without this guard, a
file
+ // outside the Between range but matched by some remaining
predicate
+ // (e.g. `BETWEEN 10 AND 20 AND id >= 0` on a file [30, 40]) would
+ // be retained because `file_result` is initialized from the
+ // remaining bitmap, silently dropping the Between conjunct.
+ if between.is_some() && !between_matches {
+ continue;
+ }
+
if matching_predicates.is_empty() && !between_matches {
continue;
}
@@ -383,6 +399,27 @@ impl GlobalIndexScanner {
}
}
+/// Whether the b-tree global index can evaluate this operator directly.
+/// Operators that fall outside this set bypass the index and are evaluated
+/// later in the read pipeline (stats prune + parquet row filter).
+fn is_btree_supported_op(op: PredicateOperator) -> bool {
+ matches!(
+ op,
+ PredicateOperator::Eq
+ | PredicateOperator::NotEq
+ | PredicateOperator::Lt
+ | PredicateOperator::LtEq
+ | PredicateOperator::Gt
+ | PredicateOperator::GtEq
+ | PredicateOperator::In
+ | PredicateOperator::NotIn
+ | PredicateOperator::IsNull
+ | PredicateOperator::IsNotNull
+ | PredicateOperator::Between
+ | PredicateOperator::NotBetween
+ )
+}
+
/// Convert a RoaringTreemap to merged RowRanges (already sorted and
deduplicated).
fn bitmap_to_ranges(bitmap: &RoaringTreemap) -> Vec<RowRange> {
if bitmap.is_empty() {
@@ -941,4 +978,51 @@ mod tests {
let ranges = result.unwrap();
assert_eq!(ranges, vec![RowRange::new(22, 26)]);
}
+
+ /// Regression for the Between+remaining bug in `evaluate_leaf`. When a
+ /// native `Between` leaf is paired with another conjunct (e.g. `id >= 0`),
+ /// and the file's b-tree key range falls **outside** the Between range
+ /// but is still matched by the remaining predicate, the whole AND must
+ /// produce zero rows. Before the fix, `file_result` was initialized from
+ /// the remaining predicate's bitmap and the Between conjunct was silently
+ /// dropped — the test would observe the file's full row id set instead of
+ /// the empty set.
+ #[tokio::test]
+ async fn test_between_unmatched_file_drops_remaining_match() {
+ let (file_io, table_path, file_name, _tmp) =
+ setup_testdata_table("btree_int_100_no_compress.bin");
+ // File covers keys [0, 198] (row_ids 0..99). Pick a Between range
+ // entirely below 0 so `may_match_between` is false, and a `>= 0`
+ // conjunct that would otherwise scoop up every row in the file.
+ let meta = BTreeIndexMeta::new(Some(le_int_key(0)),
Some(le_int_key(198)), false);
+ let entries = vec![make_global_index_entry(&file_name, 1, 0, 99,
&meta)];
+ let fields = int_schema_fields();
+
+ let predicates = vec![Predicate::and(vec![
+ Predicate::Leaf {
+ column: "id".to_string(),
+ index: 0,
+ data_type: DataType::Int(crate::spec::IntType::new()),
+ op: PredicateOperator::Between,
+ literals: vec![Datum::Int(-100), Datum::Int(-50)],
+ },
+ Predicate::Leaf {
+ column: "id".to_string(),
+ index: 0,
+ data_type: DataType::Int(crate::spec::IntType::new()),
+ op: PredicateOperator::GtEq,
+ literals: vec![Datum::Int(0)],
+ },
+ ])];
+
+ let result = evaluate_global_index(&file_io, &table_path, &entries,
&predicates, &fields)
+ .await
+ .unwrap();
+ let ranges = result.unwrap();
+ assert!(
+ ranges.is_empty(),
+ "Between(-100..-50) AND id>=0 must produce zero rows on a file \
+ whose key range is [0, 198] — got {ranges:?}"
+ );
+ }
}
