This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 9cf89f2 [python] Support Date/Time/Timestamp/Decimal filter literals
in py_to_datum (#443)
9cf89f2 is described below
commit 9cf89f278356813480788b018b9e62c3ae22d198
Author: Junrui Lee <[email protected]>
AuthorDate: Fri Jul 3 19:16:29 2026 +0800
[python] Support Date/Time/Timestamp/Decimal filter literals in py_to_datum
(#443)
---
bindings/python/Cargo.toml | 5 +-
bindings/python/src/predicate.rs | 647 ++++++++++++++++++++++++++++++++++++-
bindings/python/tests/test_read.py | 88 ++++-
3 files changed, 727 insertions(+), 13 deletions(-)
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index c2e853b..af0085d 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -28,16 +28,17 @@ doc = false
[dependencies]
arrow = { workspace = true, features = ["pyarrow"] }
+chrono = "0.4.38"
datafusion = { workspace = true }
datafusion-ffi = { workspace = true }
futures = "0.3"
paimon = { path = "../../crates/paimon", features = ["storage-all"] }
paimon-datafusion = { path = "../../crates/integrations/datafusion", features
= ["fulltext"] }
-pyo3 = { version = "0.28", features = ["abi3-py310"] }
+pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] }
serde_json = "1.0"
tokio = { workspace = true }
[dev-dependencies]
# Enables `Python::attach` to auto-initialize an interpreter for Rust unit
tests.
# Only applied to the test build; the maturin/extension-module build does not
pull dev-dependencies.
-pyo3 = { version = "0.28", features = ["abi3-py310", "auto-initialize"] }
+pyo3 = { version = "0.28", features = ["abi3-py310", "auto-initialize",
"chrono"] }
diff --git a/bindings/python/src/predicate.rs b/bindings/python/src/predicate.rs
index 0e9962b..08b2f5c 100644
--- a/bindings/python/src/predicate.rs
+++ b/bindings/python/src/predicate.rs
@@ -15,10 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-use paimon::spec::{DataField, DataType, Datum, Predicate, PredicateBuilder};
+use paimon::spec::{DataField, DataType, Datum, DecimalType, Predicate,
PredicateBuilder};
use pyo3::exceptions::{PyNotImplementedError, PyValueError};
use pyo3::prelude::*;
-use pyo3::types::{PyBool, PyDict, PyList, PyString};
+use pyo3::types::PyTzInfoAccess;
+use pyo3::types::{PyBool, PyDateTime, PyDict, PyInt, PyList, PyString, PyTime,
PyTzInfo};
/// Convert a single Python literal into a typed [`Datum`] driven by the target
/// [`DataType`].
@@ -32,11 +33,24 @@ use pyo3::types::{PyBool, PyDict, PyList, PyString};
/// (which is an `int` subclass) and enforce the target range.
/// - `Float`/`Double` accept Python `int` or `float` but reject `bool`.
/// - `Char`/`VarChar` accept only a Python `str` (no implicit
stringification).
-/// - All other types (Date/Time/Timestamp/Decimal/Bytes/complex) are not
-/// supported yet and raise `NotImplementedError`.
+/// - `Date` accepts only a `datetime.date` that is not a `datetime.datetime`
+/// (accepting the subclass would silently drop the time-of-day part).
+/// - `Time` accepts only a naive `datetime.time` with whole-millisecond
+/// microseconds (`Datum::Time` carries millis-of-day; sub-millisecond values
+/// would be silently truncated).
+/// - `Timestamp` accepts only a naive `datetime.datetime` (the type is
+/// timezone-less); `LocalZonedTimestamp` accepts only a timezone-aware one
+/// (converted to UTC). This mirrors the DataFusion pushdown path, which
+/// refuses zoned literals for TIMESTAMP and naive ones for TIMESTAMP_LTZ.
+/// - `Decimal` accepts a `decimal.Decimal` or an `int`, rescaled losslessly to
+/// the column's scale; anything needing rounding, exceeding the column's
+/// precision, non-finite, or a binary `float` is rejected.
+/// - All other types (Bytes/complex) are not supported yet and raise
+/// `NotImplementedError`.
///
/// Errors:
-/// - `ValueError` for type mismatches and out-of-range integers.
+/// - `ValueError` for type mismatches, out-of-range integers, zoned/naive
+/// timestamp mismatches, and lossy decimal/time conversions.
/// - `NotImplementedError` for unsupported field types (message names the
type).
pub(crate) fn py_to_datum(value: &Bound<'_, PyAny>, data_type: &DataType) ->
PyResult<Datum> {
match data_type {
@@ -64,6 +78,11 @@ pub(crate) fn py_to_datum(value: &Bound<'_, PyAny>,
data_type: &DataType) -> PyR
.map_err(|_| PyValueError::new_err("expected a str literal for
String field"))?;
Ok(Datum::String(s.to_str()?.to_string()))
}
+ DataType::Date(_) => date_datum(value),
+ DataType::Time(_) => time_datum(value),
+ DataType::Timestamp(_) => timestamp_datum(value),
+ DataType::LocalZonedTimestamp(_) => local_zoned_timestamp_datum(value),
+ DataType::Decimal(dec) => decimal_datum(value, dec),
other => Err(PyNotImplementedError::new_err(format!(
"literal conversion for type {other:?} is not supported yet"
))),
@@ -103,6 +122,221 @@ fn float_val(value: &Bound<'_, PyAny>) -> PyResult<f64> {
.map_err(|_| PyValueError::new_err("expected a numeric literal"))
}
+/// Convert a `datetime.date` into `Datum::Date` (epoch days).
+///
+/// `datetime.datetime` is a `date` subclass but is rejected: accepting it
would
+/// silently drop the time-of-day part.
+fn date_datum(value: &Bound<'_, PyAny>) -> PyResult<Datum> {
+ if value.is_instance_of::<PyDateTime>() {
+ return Err(PyValueError::new_err(
+ "expected a datetime.date literal for Date field, got
datetime.datetime \
+ (the time-of-day part would be dropped)",
+ ));
+ }
+ let date: chrono::NaiveDate = value
+ .extract()
+ .map_err(|_| PyValueError::new_err("expected a datetime.date literal
for Date field"))?;
+ let epoch = chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap();
+ let days = date.signed_duration_since(epoch).num_days();
+ let days = i32::try_from(days)
+ .map_err(|_| PyValueError::new_err(format!("date literal {date} out of
range")))?;
+ Ok(Datum::Date(days))
+}
+
+/// Convert a naive `datetime.time` into `Datum::Time` (millis of day).
+///
+/// Timezone-aware times are rejected (TIME has no timezone), as are
+/// sub-millisecond microseconds (`Datum::Time` carries millis; truncating
+/// would silently change comparison semantics).
+fn time_datum(value: &Bound<'_, PyAny>) -> PyResult<Datum> {
+ let time = value
+ .cast::<PyTime>()
+ .map_err(|_| PyValueError::new_err("expected a datetime.time literal
for Time field"))?;
+ if time.get_tzinfo().is_some() {
+ return Err(PyValueError::new_err(
+ "expected a naive datetime.time literal for Time field (tzinfo
must be None)",
+ ));
+ }
+ let time: chrono::NaiveTime = value
+ .extract()
+ .map_err(|_| PyValueError::new_err("expected a datetime.time literal
for Time field"))?;
+ use chrono::Timelike;
+ if !time.nanosecond().is_multiple_of(1_000_000) {
+ return Err(PyValueError::new_err(
+ "time literal has sub-millisecond microseconds, which TIME
comparisons \
+ cannot represent without truncation",
+ ));
+ }
+ let millis = time.num_seconds_from_midnight() * 1_000 + time.nanosecond()
/ 1_000_000;
+ Ok(Datum::Time(millis as i32))
+}
+
+/// Split epoch microseconds into `(millis, nanos-of-milli)` with a Euclidean
+/// split so pre-epoch instants keep a non-negative nano remainder, matching
+/// `BinaryRow::get_timestamp_raw` and the DataFusion pushdown conversion.
+fn micros_to_millis_nanos(micros: i64) -> (i64, i32) {
+ (
+ micros.div_euclid(1_000),
+ (micros.rem_euclid(1_000) * 1_000) as i32,
+ )
+}
+
+/// Convert a naive `datetime.datetime` into `Datum::Timestamp`.
+///
+/// TIMESTAMP (without time zone) is timezone-less, so timezone-aware datetimes
+/// are rejected rather than silently reinterpreted — parity with the
DataFusion
+/// pushdown path, which refuses zoned literals for this type.
+fn timestamp_datum(value: &Bound<'_, PyAny>) -> PyResult<Datum> {
+ let dt = value.cast::<PyDateTime>().map_err(|_| {
+ PyValueError::new_err("expected a datetime.datetime literal for
Timestamp field")
+ })?;
+ if dt.get_tzinfo().is_some() {
+ return Err(PyValueError::new_err(
+ "expected a naive datetime.datetime literal for Timestamp field
(tzinfo must \
+ be None); TIMESTAMP has no timezone — use a TIMESTAMP WITH LOCAL
TIME ZONE \
+ column for zoned instants",
+ ));
+ }
+ let naive: chrono::NaiveDateTime = value.extract().map_err(|_| {
+ PyValueError::new_err("expected a datetime.datetime literal for
Timestamp field")
+ })?;
+ let (millis, nanos) =
micros_to_millis_nanos(naive.and_utc().timestamp_micros());
+ Ok(Datum::Timestamp { millis, nanos })
+}
+
+/// Convert a timezone-aware `datetime.datetime` into
`Datum::LocalZonedTimestamp`
+/// (the UTC instant).
+///
+/// Aware follows Python's definition: `tzinfo` is set AND `utcoffset()` is not
+/// `None`. A tzinfo whose `utcoffset()` returns `None` is naive — passing it
to
+/// `astimezone` would interpret the value as process-local time, normalizing
+/// the same literal differently per machine. Naive datetimes are rejected
+/// rather than assumed to be in any timezone — parity with the DataFusion
+/// pushdown path, which refuses naive literals for TIMESTAMP WITH LOCAL TIME
+/// ZONE.
+fn local_zoned_timestamp_datum(value: &Bound<'_, PyAny>) -> PyResult<Datum> {
+ let py = value.py();
+ let dt = value.cast::<PyDateTime>().map_err(|_| {
+ PyValueError::new_err("expected a datetime.datetime literal for
LocalZonedTimestamp field")
+ })?;
+ // Python's aware/naive rule (datetime docs): aware iff tzinfo is not None
+ // and utcoffset() does not return None.
+ let aware =
+ dt.get_tzinfo().is_some() && !dt.call_method0(pyo3::intern!(py,
"utcoffset"))?.is_none();
+ if !aware {
+ return Err(PyValueError::new_err(
+ "expected a timezone-aware datetime.datetime literal for
LocalZonedTimestamp \
+ field (naive datetimes, including tzinfo with utcoffset() = None,
are \
+ ambiguous)",
+ ));
+ }
+ // Normalize through `astimezone(utc)` so any tzinfo implementation
+ // (zoneinfo, pytz, fixed offsets) is handled by Python itself.
+ let utc = PyTzInfo::utc(py)?;
+ let as_utc = dt.call_method1(pyo3::intern!(py, "astimezone"), (utc,))?;
+ let instant: chrono::DateTime<chrono::Utc> = as_utc.extract()?;
+ let (millis, nanos) = micros_to_millis_nanos(instant.timestamp_micros());
+ Ok(Datum::LocalZonedTimestamp { millis, nanos })
+}
+
+/// Convert a `decimal.Decimal` or `int` literal into `Datum::Decimal` at the
+/// column's precision and scale.
+///
+/// The conversion is exact: values that would need rounding at the column's
+/// scale, or whose unscaled magnitude exceeds the column's precision, are
+/// rejected. Binary `float`s are rejected outright (they are not exact decimal
+/// values), as are non-finite decimals (NaN/Infinity).
+fn decimal_datum(value: &Bound<'_, PyAny>, dec: &DecimalType) ->
PyResult<Datum> {
+ let precision = dec.precision();
+ let scale = dec.scale();
+ if value.is_instance_of::<PyBool>() {
+ return Err(PyValueError::new_err("bool is not a valid decimal
literal"));
+ }
+
+ let unscaled = if value.is_instance_of::<PyInt>() {
+ let v: i128 = value.extract().map_err(|_| {
+ PyValueError::new_err("int literal out of supported range for
Decimal field")
+ })?;
+ v.checked_mul(10i128.pow(scale)).ok_or_else(|| {
+ PyValueError::new_err("int literal out of supported range for
Decimal field")
+ })?
+ } else {
+ let decimal_cls = value.py().import("decimal")?.getattr("Decimal")?;
+ if !value.is_instance(&decimal_cls)? {
+ return Err(PyValueError::new_err(
+ "expected a decimal.Decimal or int literal for Decimal field
(float is \
+ not an exact decimal value)",
+ ));
+ }
+ decimal_to_unscaled(value, scale)?
+ };
+
+ // 10^precision fits i128 for the supported precision range (<= 38).
+ if unscaled.unsigned_abs() >= 10u128.pow(precision) {
+ return Err(PyValueError::new_err(format!(
+ "decimal literal does not fit DECIMAL({precision}, {scale})"
+ )));
+ }
+ Ok(Datum::Decimal {
+ unscaled,
+ precision,
+ scale,
+ })
+}
+
+/// Rescale a finite `decimal.Decimal` to `scale` exactly via `as_tuple()`
+/// (sign, digits, exponent), avoiding Decimal arithmetic whose context
+/// precision could silently round.
+fn decimal_to_unscaled(value: &Bound<'_, PyAny>, scale: u32) -> PyResult<i128>
{
+ let tuple = value.call_method0("as_tuple")?;
+ let sign: u8 = tuple.getattr("sign")?.extract()?;
+ // For NaN/Infinity the exponent is a string ('n'/'N'/'F') and extraction
fails.
+ let exponent: i64 = tuple.getattr("exponent")?.extract().map_err(|_| {
+ PyValueError::new_err("non-finite Decimal (NaN/Infinity) is not a
valid literal")
+ })?;
+ let digits: Vec<u32> = tuple.getattr("digits")?.extract()?;
+
+ let overflow =
+ || PyValueError::new_err("decimal literal out of supported range for
Decimal field");
+ let mut magnitude: i128 = 0;
+ for d in digits {
+ magnitude = magnitude
+ .checked_mul(10)
+ .and_then(|m| m.checked_add(d as i128))
+ .ok_or_else(overflow)?;
+ }
+
+ // value = ±magnitude * 10^exponent; unscaled = ±magnitude * 10^(exponent
+ scale).
+ let shift = exponent + scale as i64;
+ let magnitude = if shift >= 0 {
+ u32::try_from(shift)
+ .ok()
+ .filter(|s| *s <= 38)
+ .and_then(|s| magnitude.checked_mul(10i128.pow(s)))
+ .ok_or_else(overflow)?
+ } else if magnitude == 0 {
+ 0
+ } else {
+ let down = u32::try_from(-shift).ok().filter(|s| *s <= 38).ok_or_else(
+ // More than 38 digits would have to be truncated; magnitude != 0
+ // means that is always lossy.
+ || rounding_error(scale),
+ )?;
+ let divisor = 10i128.pow(down);
+ if magnitude % divisor != 0 {
+ return Err(rounding_error(scale));
+ }
+ magnitude / divisor
+ };
+ Ok(if sign == 1 { -magnitude } else { magnitude })
+}
+
+fn rounding_error(scale: u32) -> PyErr {
+ PyValueError::new_err(format!(
+ "decimal literal cannot be represented at scale {scale} without
rounding"
+ ))
+}
+
/// Operators recognized by the lightweight dict format but not translatable
to a
/// Rust [`Predicate`] for pushdown.
const METHOD_NOT_SUPPORTED: &[&str] = &["not"];
@@ -836,11 +1070,410 @@ mod tests {
}
#[test]
- fn timestamp_field_is_not_implemented() {
+ fn timestamp_field_rejects_non_datetime() {
Python::attach(|py| {
let v = 0i64.into_pyobject(py).unwrap();
let err = py_to_datum(&v,
&DataType::Timestamp(Default::default())).unwrap_err();
-
assert!(err.is_instance_of::<pyo3::exceptions::PyNotImplementedError>(py));
+ assert!(err.is_instance_of::<pyo3::exceptions::PyValueError>(py));
+ });
+ }
+
+ // ---- temporal literals ----
+
+ /// `datetime.date(2024, 1, 1)` epoch days (1970-01-01 = 0).
+ const D_2024_01_01: i32 = 19723;
+
+ fn py_date<'py>(py: Python<'py>, y: i32, m: u8, d: u8) -> Bound<'py,
PyAny> {
+ pyo3::types::PyDate::new(py, y, m, d).unwrap().into_any()
+ }
+
+ fn py_time<'py>(
+ py: Python<'py>,
+ h: u8,
+ min: u8,
+ s: u8,
+ micro: u32,
+ tz: Option<&Bound<'py, pyo3::types::PyTzInfo>>,
+ ) -> Bound<'py, PyAny> {
+ pyo3::types::PyTime::new(py, h, min, s, micro, tz)
+ .unwrap()
+ .into_any()
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ fn py_datetime<'py>(
+ py: Python<'py>,
+ y: i32,
+ mo: u8,
+ d: u8,
+ h: u8,
+ mi: u8,
+ s: u8,
+ micro: u32,
+ tz: Option<&Bound<'py, pyo3::types::PyTzInfo>>,
+ ) -> Bound<'py, PyAny> {
+ pyo3::types::PyDateTime::new(py, y, mo, d, h, mi, s, micro, tz)
+ .unwrap()
+ .into_any()
+ }
+
+ fn utc(py: Python<'_>) -> Bound<'_, pyo3::types::PyTzInfo> {
+ pyo3::types::PyTzInfo::utc(py).unwrap().to_owned()
+ }
+
+ fn fixed_offset(py: Python<'_>, seconds: i32) -> Bound<'_,
pyo3::types::PyTzInfo> {
+ let delta = pyo3::types::PyDelta::new(py, 0, seconds, 0,
false).unwrap();
+ pyo3::types::PyTzInfo::fixed_offset(py, &delta).unwrap()
+ }
+
+ #[test]
+ fn date_field_accepts_date() {
+ Python::attach(|py| {
+ let v = py_date(py, 2024, 1, 1);
+ assert_eq!(
+ py_to_datum(&v, &DataType::Date(Default::default())).unwrap(),
+ Datum::Date(D_2024_01_01)
+ );
+ });
+ }
+
+ #[test]
+ fn date_field_accepts_pre_epoch_date() {
+ Python::attach(|py| {
+ let v = py_date(py, 1969, 12, 31);
+ assert_eq!(
+ py_to_datum(&v, &DataType::Date(Default::default())).unwrap(),
+ Datum::Date(-1)
+ );
+ });
+ }
+
+ #[test]
+ fn date_field_rejects_datetime() {
+ Python::attach(|py| {
+ // datetime is a date subclass; accepting it would silently drop
the
+ // time-of-day part.
+ let v = py_datetime(py, 2024, 1, 1, 0, 0, 0, 0, None);
+ let err = py_to_datum(&v,
&DataType::Date(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn date_field_rejects_str() {
+ Python::attach(|py| {
+ let v = "2024-01-01".into_pyobject(py).unwrap();
+ let err = py_to_datum(v.as_any(),
&DataType::Date(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn time_field_accepts_naive_time() {
+ Python::attach(|py| {
+ let v = py_time(py, 12, 34, 56, 789_000, None);
+ assert_eq!(
+ py_to_datum(&v, &DataType::Time(Default::default())).unwrap(),
+ Datum::Time(45_296_789)
+ );
+ });
+ }
+
+ #[test]
+ fn time_field_rejects_sub_millisecond() {
+ Python::attach(|py| {
+ // Datum::Time carries millis-of-day; silently truncating micros
+ // would change equality semantics.
+ let v = py_time(py, 0, 0, 0, 500, None);
+ let err = py_to_datum(&v,
&DataType::Time(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn time_field_rejects_aware_time() {
+ Python::attach(|py| {
+ let tz = utc(py);
+ let v = py_time(py, 1, 2, 3, 0, Some(&tz));
+ let err = py_to_datum(&v,
&DataType::Time(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn timestamp_field_accepts_naive_datetime() {
+ Python::attach(|py| {
+ let v = py_datetime(py, 2024, 1, 1, 0, 0, 0, 123_456, None);
+ assert_eq!(
+ py_to_datum(&v,
&DataType::Timestamp(Default::default())).unwrap(),
+ Datum::Timestamp {
+ millis: 1_704_067_200_123,
+ nanos: 456_000,
+ }
+ );
+ });
+ }
+
+ #[test]
+ fn timestamp_field_pre_epoch_uses_euclidean_split() {
+ Python::attach(|py| {
+ // 1969-12-31 23:59:59.999999 = -1 µs from epoch
+ // → floor millis -1, non-negative nanos remainder 999_000.
+ let v = py_datetime(py, 1969, 12, 31, 23, 59, 59, 999_999, None);
+ assert_eq!(
+ py_to_datum(&v,
&DataType::Timestamp(Default::default())).unwrap(),
+ Datum::Timestamp {
+ millis: -1,
+ nanos: 999_000,
+ }
+ );
+ });
+ }
+
+ #[test]
+ fn timestamp_field_rejects_aware_datetime() {
+ Python::attach(|py| {
+ // TIMESTAMP is timezone-less; parity with the DataFusion path
which
+ // refuses zoned literals for it.
+ let tz = utc(py);
+ let v = py_datetime(py, 2024, 1, 1, 0, 0, 0, 0, Some(&tz));
+ let err = py_to_datum(&v,
&DataType::Timestamp(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn timestamp_field_rejects_date() {
+ Python::attach(|py| {
+ let v = py_date(py, 2024, 1, 1);
+ let err = py_to_datum(&v,
&DataType::Timestamp(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn local_zoned_timestamp_field_accepts_aware_datetime() {
+ Python::attach(|py| {
+ // 2024-01-01T08:00:00+08:00 == 2024-01-01T00:00:00Z.
+ let tz = fixed_offset(py, 8 * 3600);
+ let v = py_datetime(py, 2024, 1, 1, 8, 0, 0, 0, Some(&tz));
+ assert_eq!(
+ py_to_datum(&v,
&DataType::LocalZonedTimestamp(Default::default())).unwrap(),
+ Datum::LocalZonedTimestamp {
+ millis: 1_704_067_200_000,
+ nanos: 0,
+ }
+ );
+ });
+ }
+
+ #[test]
+ fn local_zoned_timestamp_field_rejects_naive_datetime() {
+ Python::attach(|py| {
+ let v = py_datetime(py, 2024, 1, 1, 0, 0, 0, 0, None);
+ let err =
+ py_to_datum(&v,
&DataType::LocalZonedTimestamp(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ /// Build a datetime whose tzinfo subclass returns None from utcoffset() —
+ /// naive by Python's definition (docs: "aware if tzinfo is not None AND
+ /// utcoffset() does not return None") despite tzinfo being set.
+ fn pseudo_naive_datetime(py: Python<'_>) -> Bound<'_, PyAny> {
+ let ns = PyDict::new(py);
+ py.run(
+ c"import datetime
+class FloatingTz(datetime.tzinfo):
+ def utcoffset(self, dt): return None
+ def dst(self, dt): return None
+ def tzname(self, dt): return 'FLOATING'
+value = datetime.datetime(2024, 1, 1, tzinfo=FloatingTz())",
+ None,
+ Some(&ns),
+ )
+ .unwrap();
+ ns.get_item("value").unwrap().unwrap()
+ }
+
+ #[test]
+ fn local_zoned_timestamp_field_rejects_pseudo_naive_tzinfo() {
+ Python::attach(|py| {
+ // tzinfo is set but utcoffset() is None: astimezone(utc) would
+ // interpret the value as process-local time, so the same literal
+ // would normalize differently per machine. Must be rejected.
+ let v = pseudo_naive_datetime(py);
+ let err =
+ py_to_datum(&v,
&DataType::LocalZonedTimestamp(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn timestamp_field_rejects_pseudo_naive_tzinfo() {
+ Python::attach(|py| {
+ // Timestamp deliberately rejects ANY tzinfo, even one whose
+ // utcoffset() is None: stricter than Python's naive/aware
+ // definition, but explicit — the caller strips tzinfo to state
+ // wall-clock intent.
+ let v = pseudo_naive_datetime(py);
+ let err = py_to_datum(&v,
&DataType::Timestamp(Default::default())).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ // ---- decimal literals ----
+
+ fn py_decimal<'py>(py: Python<'py>, repr: &str) -> Bound<'py, PyAny> {
+ py.import("decimal")
+ .unwrap()
+ .getattr("Decimal")
+ .unwrap()
+ .call1((repr,))
+ .unwrap()
+ }
+
+ fn decimal_type(precision: u32, scale: u32) -> DataType {
+ DataType::Decimal(paimon::spec::DecimalType::new(precision,
scale).unwrap())
+ }
+
+ #[test]
+ fn decimal_field_accepts_exact_scale() {
+ Python::attach(|py| {
+ let v = py_decimal(py, "12.34");
+ assert_eq!(
+ py_to_datum(&v, &decimal_type(10, 2)).unwrap(),
+ Datum::Decimal {
+ unscaled: 1234,
+ precision: 10,
+ scale: 2,
+ }
+ );
+ });
+ }
+
+ #[test]
+ fn decimal_field_rescales_losslessly() {
+ Python::attach(|py| {
+ let v = py_decimal(py, "1.5");
+ assert_eq!(
+ py_to_datum(&v, &decimal_type(10, 2)).unwrap(),
+ Datum::Decimal {
+ unscaled: 150,
+ precision: 10,
+ scale: 2,
+ }
+ );
+ });
+ }
+
+ #[test]
+ fn decimal_field_accepts_scientific_notation() {
+ Python::attach(|py| {
+ let v = py_decimal(py, "1E+2");
+ assert_eq!(
+ py_to_datum(&v, &decimal_type(10, 2)).unwrap(),
+ Datum::Decimal {
+ unscaled: 10_000,
+ precision: 10,
+ scale: 2,
+ }
+ );
+ });
+ }
+
+ #[test]
+ fn decimal_field_negative_value() {
+ Python::attach(|py| {
+ let v = py_decimal(py, "-12.34");
+ assert_eq!(
+ py_to_datum(&v, &decimal_type(10, 2)).unwrap(),
+ Datum::Decimal {
+ unscaled: -1234,
+ precision: 10,
+ scale: 2,
+ }
+ );
+ });
+ }
+
+ #[test]
+ fn decimal_field_accepts_int_literal() {
+ Python::attach(|py| {
+ let v = 7i64.into_pyobject(py).unwrap();
+ assert_eq!(
+ py_to_datum(&v, &decimal_type(10, 2)).unwrap(),
+ Datum::Decimal {
+ unscaled: 700,
+ precision: 10,
+ scale: 2,
+ }
+ );
+ });
+ }
+
+ #[test]
+ fn decimal_field_rejects_rounding() {
+ Python::attach(|py| {
+ // 0.005 cannot be represented at scale 2 without rounding.
+ let v = py_decimal(py, "0.005");
+ let err = py_to_datum(&v, &decimal_type(10, 2)).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn decimal_field_rejects_precision_overflow() {
+ Python::attach(|py| {
+ let v = py_decimal(py, "123.45");
+ let err = py_to_datum(&v, &decimal_type(4, 2)).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn decimal_field_rejects_non_finite() {
+ Python::attach(|py| {
+ for repr in ["NaN", "Infinity", "-Infinity"] {
+ let v = py_decimal(py, repr);
+ let err = py_to_datum(&v, &decimal_type(10, 2)).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py), "{repr}");
+ }
+ });
+ }
+
+ #[test]
+ fn decimal_field_rejects_float() {
+ Python::attach(|py| {
+ // Binary floats are not exact decimal values; requiring
+ // decimal.Decimal keeps the conversion lossless.
+ let v = 1.5f64.into_pyobject(py).unwrap();
+ let err = py_to_datum(&v, &decimal_type(10, 2)).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn decimal_field_rejects_bool() {
+ Python::attach(|py| {
+ let v = true.into_pyobject(py).unwrap();
+ let err = py_to_datum(v.as_any(), &decimal_type(10,
2)).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn unsupported_field_type_still_not_implemented() {
+ Python::attach(|py| {
+ // Bytes/complex types remain out of scope for literal conversion.
+ let v = 0i64.into_pyobject(py).unwrap();
+ for dt in [
+ DataType::Binary(Default::default()),
+
DataType::Array(paimon::spec::ArrayType::new(DataType::Int(IntType::new()))),
+ ] {
+ let err = py_to_datum(&v, &dt).unwrap_err();
+
assert!(err.is_instance_of::<pyo3::exceptions::PyNotImplementedError>(py));
+ }
});
}
diff --git a/bindings/python/tests/test_read.py
b/bindings/python/tests/test_read.py
index 8065746..f81d583 100644
--- a/bindings/python/tests/test_read.py
+++ b/bindings/python/tests/test_read.py
@@ -278,17 +278,97 @@ def test_filter_string_op_on_non_string_column_raises():
def test_filter_unsupported_type_raises():
- # Build a table with a timestamp column, filter on it ->
NotImplementedError
+ # Binary columns have no literal conversion -> NotImplementedError
with tempfile.TemporaryDirectory() as warehouse:
ctx = SQLContext()
ctx.register_catalog("paimon", {"warehouse": warehouse})
ctx.sql("CREATE SCHEMA paimon.tdb")
- ctx.sql("CREATE TABLE paimon.tdb.tt (id INT, created_at TIMESTAMP)")
- ctx.sql("INSERT INTO paimon.tdb.tt VALUES (1, TIMESTAMP '2024-01-01
00:00:00')")
+ ctx.sql("CREATE TABLE paimon.tdb.tt (id INT, payload BYTEA)")
+ ctx.sql("INSERT INTO paimon.tdb.tt VALUES (1, X'00')")
table = PaimonCatalog({"warehouse": warehouse}).get_table("tdb.tt")
with pytest.raises(NotImplementedError):
table.new_read_builder().with_filter(
- {"method": "equal", "field": "created_at", "literals": [0]})
+ {"method": "equal", "field": "payload", "literals": [0]})
+
+
+def _make_temporal_table(warehouse):
+ ctx = SQLContext()
+ ctx.register_catalog("paimon", {"warehouse": warehouse})
+ ctx.sql("CREATE SCHEMA paimon.tmp")
+ ctx.sql(
+ "CREATE TABLE paimon.tmp.tt (id INT, d DATE, ts TIMESTAMP, dec
DECIMAL(10, 2))")
+ # Two separate INSERTs -> two files, so file stats can prune per-file.
+ ctx.sql(
+ "INSERT INTO paimon.tmp.tt VALUES "
+ "(1, DATE '2024-01-01', TIMESTAMP '2024-01-01 00:00:00', 12.34)")
+ ctx.sql(
+ "INSERT INTO paimon.tmp.tt VALUES "
+ "(2, DATE '2024-06-15', TIMESTAMP '2024-06-15 12:30:00', 56.78)")
+ return PaimonCatalog({"warehouse": warehouse}).get_table("tmp.tt")
+
+
+def test_filter_date_literal_prunes_and_reads():
+ import datetime
+
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_temporal_table(warehouse)
+ b = table.new_read_builder().with_filter(
+ {"method": "equal", "field": "d", "literals": [datetime.date(2024,
1, 1)]})
+ splits = b.new_scan().plan().splits()
+ assert len(splits) == 1
+ t = pa.Table.from_batches(b.new_read().read(splits))
+ assert t.column("id").to_pylist() == [1]
+
+
+def test_filter_timestamp_literal_prunes_and_reads():
+ import datetime
+
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_temporal_table(warehouse)
+ b = table.new_read_builder().with_filter(
+ {"method": "greaterThan", "field": "ts",
+ "literals": [datetime.datetime(2024, 3, 1, 0, 0, 0)]})
+ splits = b.new_scan().plan().splits()
+ assert len(splits) == 1
+ t = pa.Table.from_batches(b.new_read().read(splits))
+ assert t.column("id").to_pylist() == [2]
+
+
+def test_filter_timestamp_rejects_aware_datetime():
+ import datetime
+
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_temporal_table(warehouse)
+ aware = datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc)
+ with pytest.raises(ValueError):
+ table.new_read_builder().with_filter(
+ {"method": "greaterThan", "field": "ts", "literals": [aware]})
+
+
+def test_filter_decimal_literal_prunes_and_reads():
+ from decimal import Decimal
+
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_temporal_table(warehouse)
+ b = table.new_read_builder().with_filter(
+ {"method": "equal", "field": "dec", "literals":
[Decimal("12.34")]})
+ splits = b.new_scan().plan().splits()
+ assert len(splits) == 1
+ t = pa.Table.from_batches(b.new_read().read(splits))
+ assert t.column("id").to_pylist() == [1]
+
+
+def test_filter_decimal_rejects_float_and_rounding():
+ from decimal import Decimal
+
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_temporal_table(warehouse)
+ b = table.new_read_builder()
+ with pytest.raises(ValueError):
+ b.with_filter({"method": "equal", "field": "dec", "literals":
[12.34]})
+ with pytest.raises(ValueError):
+ b.with_filter(
+ {"method": "equal", "field": "dec", "literals":
[Decimal("0.005")]})
def test_filter_unknown_field_raises():
