This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new eb9b95a92b [common] Fix invalid DecimalType when extracting a variant
decimal (#9672)
eb9b95a92b is described below
commit eb9b95a92b02ec2ee915a4e76a5c0b19315afb30
Author: jackylee <[email protected]>
AuthorDate: Fri Sep 11 14:02:58 2026 +0800
[common] Fix invalid DecimalType when extracting a variant decimal (#9672)
---
.../org/apache/paimon/data/variant/VariantGet.java | 10 ++++++-
.../paimon/data/variant/GenericVariantTest.java | 32 ++++++++++++++++++++++
.../data/variant/PaimonShreddingUtilsTest.java | 21 ++++++++++++++
3 files changed, 62 insertions(+), 1 deletion(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
index e053f330f4..37aa20b4af 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/VariantGet.java
@@ -138,8 +138,16 @@ public class VariantGet {
break;
case DECIMAL:
BigDecimal decimal = v.getDecimal();
- int precision = decimal.precision();
+ if (decimal.scale() < 0) {
+ // stripTrailingZeros folds trailing zeros into a
negative exponent,
+ // and a negative scale is not a Paimon decimal
+ decimal = decimal.setScale(0);
+ }
int scale = decimal.scale();
+ // precision() counts the digits of the unscaled value, so
it is smaller than
+ // the scale for a value below 0.1, which DecimalType
rejects. The variant
+ // writer caps both at MAX_DECIMAL16_PRECISION, so this
stays in range.
+ int precision = Math.max(decimal.precision(), scale);
input = Decimal.fromBigDecimal(decimal, precision, scale);
inputType = DataTypes.DECIMAL(precision, scale);
break;
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
index be8051cc72..e5e507c455 100644
---
a/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/data/variant/GenericVariantTest.java
@@ -276,6 +276,38 @@ public class GenericVariantTest {
assertThat(variant.variantGet("$.nullField", DataTypes.BOOLEAN(),
castArgs)).isNull();
}
+ @Test
+ public void testVariantGetDecimalWithScaleAbovePrecision() {
+ // precision() counts the digits of the unscaled value, so it is below
the scale for any
+ // value under 0.1. The scale 38 case is the widest the reader admits,
and it is the
+ // first one here that needs a non compact Decimal.
+ String tiny = "0.00000000000000000000000000000000000001";
+ Variant variant = GenericVariant.fromJson("{\"small\": 0.05, \"tiny\":
" + tiny + "}");
+ VariantCastArgs castArgs = new VariantCastArgs(false, ZoneOffset.UTC);
+
+ assertThat(variant.variantGet("$.small", DataTypes.STRING(), castArgs))
+ .isEqualTo(BinaryString.fromString("0.05"));
+ assertThat(variant.variantGet("$.small", DataTypes.DECIMAL(5, 3),
castArgs))
+ .isEqualTo(Decimal.fromBigDecimal(new BigDecimal("0.050"), 5,
3));
+ assertThat(variant.variantGet("$.tiny", DataTypes.STRING(), castArgs))
+ .isEqualTo(BinaryString.fromString(tiny));
+ assertThat(variant.variantGet("$.tiny", DataTypes.DECIMAL(38, 38),
castArgs))
+ .isEqualTo(Decimal.fromBigDecimal(new BigDecimal(tiny), 38,
38));
+ }
+
+ @Test
+ public void testVariantGetDecimalWithNegativeScale() {
+ // getDecimal() strips trailing zeros, which turns 100.00 into 1E+2, a
negative scale
+ Variant variant = GenericVariant.fromJson("{\"round\": 100.00}");
+ VariantCastArgs castArgs = new VariantCastArgs(false, ZoneOffset.UTC);
+
+ // rescaling rather than un-stripping keeps this in step with toJson
+ assertThat(variant.variantGet("$.round", DataTypes.STRING(), castArgs))
+ .isEqualTo(BinaryString.fromString("100"));
+ assertThat(variant.variantGet("$.round", DataTypes.DECIMAL(5, 1),
castArgs))
+ .isEqualTo(Decimal.fromBigDecimal(new BigDecimal("100.0"), 5,
1));
+ }
+
@Test
public void testObjectFieldOrderingCompatibility() {
String bmpKey = "\uE000";
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java
index 19ebca4fbf..b1952357d7 100644
---
a/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/data/variant/PaimonShreddingUtilsTest.java
@@ -291,6 +291,27 @@ public class PaimonShreddingUtilsTest {
})));
}
+ @Test
+ public void testAssembleDecimalWithScaleAbovePrecision() {
+ // the unshredded leg extracts through VariantGet, which used to build
an invalid
+ // DecimalType for a value below 0.1 or one whose trailing zeros were
stripped off
+ GenericVariant v = GenericVariant.fromJson("{\"round\": 100.00,
\"small\": 0.05}");
+ VariantCastArgs castArgs = new VariantCastArgs(true, ZoneOffset.UTC);
+
+ VariantSchema variantSchema =
buildVariantSchema(variantShreddingSchema(RowType.of()));
+ FieldToExtract[] fieldsToExtract = {
+ buildFieldsToExtract(DataTypes.STRING(), "$.round", castArgs,
variantSchema),
+ buildFieldsToExtract(DataTypes.STRING(), "$.small", castArgs,
variantSchema)
+ };
+
+ assertThat(
+ assembleVariantStruct(
+ castShredded(v, variantSchema), variantSchema,
fieldsToExtract))
+ .isEqualTo(
+ GenericRow.of(
+ BinaryString.fromString("100"),
BinaryString.fromString("0.05")));
+ }
+
private static void assertVariantStructEquals(
RowType shreddedType, RowType allTypes, GenericVariant v,
GenericRow expected) {
VariantCastArgs castArgs = new VariantCastArgs(true, ZoneOffset.UTC);
