This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e3cd680847 [python] Align exact VARIANT integer and decimal types
(#9168)
e3cd680847 is described below
commit e3cd680847aa1dc37c5839299ba439f684013944
Author: XiaoHongbo <[email protected]>
AuthorDate: Tue Aug 11 17:21:44 2026 +0800
[python] Align exact VARIANT integer and decimal types (#9168)
---
paimon-python/pypaimon/data/variant_path.py | 20 +++++++++++--
paimon-python/pypaimon/data/variant_shredding.py | 6 ++--
paimon-python/pypaimon/tests/variant_path_test.py | 36 ++++++++++++++++++++++-
3 files changed, 55 insertions(+), 7 deletions(-)
diff --git a/paimon-python/pypaimon/data/variant_path.py
b/paimon-python/pypaimon/data/variant_path.py
index 9a958c3359..e7ce3d0adb 100644
--- a/paimon-python/pypaimon/data/variant_path.py
+++ b/paimon-python/pypaimon/data/variant_path.py
@@ -892,7 +892,7 @@ def _variant_array_children(value, pos, end):
def _supports_exact_get(data_type):
if (pa.types.is_boolean(data_type)
- or pa.types.is_int64(data_type)
+ or pa.types.is_signed_integer(data_type)
or pa.types.is_float32(data_type)
or pa.types.is_float64(data_type)
or pa.types.is_string(data_type)
@@ -917,13 +917,27 @@ def _supports_exact_get(data_type):
return False
+def _validate_decimal_scale(data_type):
+ if pa.types.is_decimal128(data_type) and data_type.scale < 0:
+ raise ValueError("VARIANT decimal scale must be non-negative")
+ if pa.types.is_struct(data_type):
+ for field in data_type:
+ _validate_decimal_scale(field.type)
+ elif (pa.types.is_list(data_type)
+ or pa.types.is_large_list(data_type)
+ or pa.types.is_fixed_size_list(data_type)):
+ _validate_decimal_scale(data_type.value_type)
+ elif pa.types.is_map(data_type):
+ _validate_decimal_scale(data_type.item_type)
+
+
def _exact_primitive_matches(value, pos, data_type):
variant_type = _variant_get_type(value, pos)
if variant_type == _Type.NULL:
return True
if pa.types.is_boolean(data_type):
return variant_type == _Type.BOOLEAN
- if pa.types.is_int64(data_type):
+ if pa.types.is_signed_integer(data_type):
return variant_type == _Type.LONG
if pa.types.is_float32(data_type):
return variant_type == _Type.FLOAT
@@ -1084,6 +1098,7 @@ class _Replacement:
else:
raise TypeError(
"VARIANT replacement must be an Arrow Scalar or Array")
+ _validate_decimal_scale(self.type)
if not _supported_replacement_type(self.type):
raise TypeError(
f"Unsupported exact VARIANT replacement type: {self.type}")
@@ -1331,6 +1346,7 @@ def _variant_get(column, paths: Mapping[str,
pa.DataType]):
for path, target_type in paths.items():
if not isinstance(target_type, pa.DataType):
raise TypeError("VARIANT data_type must be a PyArrow data type")
+ _validate_decimal_scale(target_type)
if not _supports_exact_get(target_type):
raise TypeError(
f"Unsupported exact VARIANT data type: {target_type}")
diff --git a/paimon-python/pypaimon/data/variant_shredding.py
b/paimon-python/pypaimon/data/variant_shredding.py
index 18692cbbe6..d67af9bf7a 100644
--- a/paimon-python/pypaimon/data/variant_shredding.py
+++ b/paimon-python/pypaimon/data/variant_shredding.py
@@ -224,6 +224,8 @@ def _encode_scalar_to_value_bytes(typed_value, arrow_type:
pa.DataType) -> bytes
def _append_scalar(builder, value, arrow_type: pa.DataType) -> None:
"""Dispatch a typed Python scalar into the appropriate builder method."""
+ if pa.types.is_decimal(arrow_type) and arrow_type.scale < 0:
+ raise ValueError('VARIANT decimal scale must be non-negative')
if value is None:
builder.append_null()
return
@@ -303,10 +305,6 @@ def _append_scalar(builder, value, arrow_type:
pa.DataType) -> None:
if precision > arrow_type.precision:
raise ValueError(
f'{decimal} exceeds Arrow precision {arrow_type.precision}')
- if scale < 0:
- unscaled *= 10 ** -scale
- scale = 0
- precision = max(1, len(str(abs(unscaled))))
builder.append_decimal_unscaled(unscaled, precision, scale)
else:
# Fallback: encode as string
diff --git a/paimon-python/pypaimon/tests/variant_path_test.py
b/paimon-python/pypaimon/tests/variant_path_test.py
index d5797c7f2d..9ddf75c700 100644
--- a/paimon-python/pypaimon/tests/variant_path_test.py
+++ b/paimon-python/pypaimon/tests/variant_path_test.py
@@ -128,7 +128,17 @@ class TestVariantGet(unittest.TestCase):
with self.assertRaisesRegex(TypeError, "does not match"):
variant_get(_variants([1.0]), '$', pa.string())
with self.assertRaisesRegex(TypeError, "Unsupported exact"):
- variant_get(_variants([1]), '$', pa.int32())
+ variant_get(_variants([1]), '$', pa.uint32())
+
+ def test_reads_all_signed_integer_widths(self):
+ column = _variants([-12, 34])
+
+ for data_type in (pa.int8(), pa.int16(), pa.int32(), pa.int64()):
+ with self.subTest(data_type=data_type):
+ self.assertEqual(
+ variant_get(column, '$', data_type).to_pylist(),
+ [-12, 34],
+ )
def test_reads_exact_primitive_types(self):
timestamp = datetime.datetime(2026, 8, 11, 1, 2, 3, 456000)
@@ -274,6 +284,30 @@ class TestVariantGet(unittest.TestCase):
class TestVariantReplace(unittest.TestCase):
+ def test_signed_integer_replacement_round_trips(self):
+ column = _variants([1])
+
+ for data_type in (pa.int8(), pa.int16(), pa.int32(), pa.int64()):
+ with self.subTest(data_type=data_type):
+ result = variant_replace(
+ column, '$', pa.scalar(-12, type=data_type))
+ self.assertEqual(
+ variant_get(result, '$', data_type).to_pylist(), [-12])
+
+ def test_rejects_negative_decimal_scale(self):
+ data_type = pa.decimal128(3, -2)
+ column = _variants([100])
+
+ with self.assertRaisesRegex(ValueError, "non-negative"):
+ variant_get(column, '$', data_type)
+ with self.assertRaisesRegex(ValueError, "non-negative"):
+ variant_replace(
+ column, '$', pa.scalar(Decimal('1E+2'), type=data_type))
+ with self.assertRaisesRegex(ValueError, "non-negative"):
+ _encode_scalar_to_value_bytes(Decimal('1E+2'), data_type)
+ with self.assertRaisesRegex(ValueError, "non-negative"):
+ _encode_scalar_to_value_bytes(None, data_type)
+
def test_replaces_exact_primitive_types(self):
original_timestamp = datetime.datetime(2026, 8, 11)
column = _typed_object({
