This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 5a04a76f538 [FLINK-40832][core] Validate decimal range in
`VariantBuilder.of(BigDecimal)`
5a04a76f538 is described below
commit 5a04a76f538aad3afb721631acf42ce4233dd081
Author: Moritz Manner <[email protected]>
AuthorDate: Wed Sep 30 19:58:22 2026 +0200
[FLINK-40832][core] Validate decimal range in
`VariantBuilder.of(BigDecimal)`
---
.../variant/BinaryVariantInternalBuilder.java | 49 ++++++++++++++++++---
.../flink/types/variant/BinaryVariantUtil.java | 10 ++++-
.../apache/flink/types/variant/VariantBuilder.java | 8 +++-
.../variant/BinaryVariantInternalBuilderTest.java | 13 ++++++
.../flink/types/variant/BinaryVariantTest.java | 51 ++++++++++++++++++++++
5 files changed, 122 insertions(+), 9 deletions(-)
diff --git
a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantInternalBuilder.java
b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantInternalBuilder.java
index a61763a93b6..5e3520985d3 100644
---
a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantInternalBuilder.java
+++
b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantInternalBuilder.java
@@ -73,6 +73,7 @@ import static
org.apache.flink.types.variant.BinaryVariantUtil.U8_MAX;
import static org.apache.flink.types.variant.BinaryVariantUtil.VERSION;
import static org.apache.flink.types.variant.BinaryVariantUtil.arrayHeader;
import static org.apache.flink.types.variant.BinaryVariantUtil.checkIndex;
+import static
org.apache.flink.types.variant.BinaryVariantUtil.fitsVariantDecimal;
import static org.apache.flink.types.variant.BinaryVariantUtil.getMetadataKey;
import static org.apache.flink.types.variant.BinaryVariantUtil.handleArray;
import static org.apache.flink.types.variant.BinaryVariantUtil.handleObject;
@@ -237,23 +238,28 @@ public class BinaryVariantInternalBuilder {
writePos += 8;
}
- // Append a decimal value to the variant builder. The caller should
guarantee that its precision
- // and scale fit into `MAX_DECIMAL16_PRECISION`.
- public void appendDecimal(BigDecimal d) {
+ /**
+ * Appends a decimal value to the variant builder. A negative scale is
rescaled to 0, which
+ * keeps the numeric value.
+ *
+ * @throws VariantTypeException if the precision or the scale of the
rescaled decimal exceeds
+ * {@link BinaryVariantUtil#MAX_DECIMAL16_PRECISION}
+ */
+ public void appendDecimal(BigDecimal decimal) {
+ final BigDecimal d = toVariantDecimal(decimal);
checkCapacity(2 + 16);
BigInteger unscaled = d.unscaledValue();
- if (d.scale() <= MAX_DECIMAL4_PRECISION && d.precision() <=
MAX_DECIMAL4_PRECISION) {
+ if (fitsVariantDecimal(d, MAX_DECIMAL4_PRECISION)) {
writeBuffer[writePos++] = primitiveHeader(DECIMAL4);
writeBuffer[writePos++] = (byte) d.scale();
writeLong(writeBuffer, writePos, unscaled.intValueExact(), 4);
writePos += 4;
- } else if (d.scale() <= MAX_DECIMAL8_PRECISION && d.precision() <=
MAX_DECIMAL8_PRECISION) {
+ } else if (fitsVariantDecimal(d, MAX_DECIMAL8_PRECISION)) {
writeBuffer[writePos++] = primitiveHeader(DECIMAL8);
writeBuffer[writePos++] = (byte) d.scale();
writeLong(writeBuffer, writePos, unscaled.longValueExact(), 8);
writePos += 8;
} else {
- assert d.scale() <= MAX_DECIMAL16_PRECISION && d.precision() <=
MAX_DECIMAL16_PRECISION;
writeBuffer[writePos++] = primitiveHeader(DECIMAL16);
writeBuffer[writePos++] = (byte) d.scale();
// `toByteArray` returns a big-endian representation. We need to
copy it reversely and
@@ -271,6 +277,35 @@ public class BinaryVariantInternalBuilder {
}
}
+ // The variant spec requires a scale in [0, 38] and a precision of at most
38.
+ private static BigDecimal toVariantDecimal(BigDecimal d) {
+ BigDecimal result = d;
+ if (d.scale() < 0) {
+ // A non-zero value with a scale below -38 has more than 38 digits
after rescaling.
+ // Reject it upfront because setScale is slow for exponents like
1e9999999 and throws
+ // an ArithmeticException for exponents like 1e999999999.
+ if (d.signum() != 0 && d.scale() < -MAX_DECIMAL16_PRECISION) {
+ throw decimalOutOfRange(d);
+ }
+ // Rescaling a non-zero value gives it precision - scale digits.
For example, 12345e34
+ // has precision 5 and scale -34, so it has 5 - (-34) = 39 digits
after rescaling.
+ result = d.setScale(0);
+ }
+ if (!fitsVariantDecimal(result, MAX_DECIMAL16_PRECISION)) {
+ throw decimalOutOfRange(d);
+ }
+ return result;
+ }
+
+ private static VariantTypeException decimalOutOfRange(BigDecimal d) {
+ return new VariantTypeException(
+ String.format(
+ "Decimal with precision %d and scale %d is outside the
range supported by "
+ + "variant decimals. After rescaling a
negative scale to 0, the "
+ + "precision and scale must not exceed %d.",
+ d.precision(), d.scale(), MAX_DECIMAL16_PRECISION));
+ }
+
public void appendDate(int daysSinceEpoch) {
checkCapacity(1 + 4);
writeBuffer[writePos++] = primitiveHeader(DATE);
@@ -685,7 +720,7 @@ public class BinaryVariantInternalBuilder {
}
}
BigDecimal d = new BigDecimal(input);
- if (d.scale() <= MAX_DECIMAL16_PRECISION && d.precision() <=
MAX_DECIMAL16_PRECISION) {
+ if (fitsVariantDecimal(d, MAX_DECIMAL16_PRECISION)) {
appendDecimal(d);
return true;
}
diff --git
a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantUtil.java
b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantUtil.java
index 0451f370415..98866958125 100644
---
a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantUtil.java
+++
b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantUtil.java
@@ -626,9 +626,17 @@ public class BinaryVariantUtil {
return Double.longBitsToDouble(readLong(value, pos + 1, 8));
}
+ /**
+ * Returns whether a decimal can be stored as a variant decimal: its
precision and its
+ * non-negative scale must both be at most {@code maxPrecision}.
+ */
+ static boolean fitsVariantDecimal(BigDecimal d, int maxPrecision) {
+ return d.scale() >= 0 && d.scale() <= maxPrecision && d.precision() <=
maxPrecision;
+ }
+
/** Check whether the precision and scale of the decimal are within the
limit. */
private static void checkDecimal(BigDecimal d, int maxPrecision) {
- if (d.precision() > maxPrecision || d.scale() > maxPrecision) {
+ if (!fitsVariantDecimal(d, maxPrecision)) {
throw malformedVariant();
}
}
diff --git
a/flink-core/src/main/java/org/apache/flink/types/variant/VariantBuilder.java
b/flink-core/src/main/java/org/apache/flink/types/variant/VariantBuilder.java
index abda3b8491d..5774aa0856f 100644
---
a/flink-core/src/main/java/org/apache/flink/types/variant/VariantBuilder.java
+++
b/flink-core/src/main/java/org/apache/flink/types/variant/VariantBuilder.java
@@ -58,7 +58,13 @@ public interface VariantBuilder {
/** Create a variant from a boolean. */
Variant of(boolean b);
- /** Create a variant from a BigDecimal. */
+ /**
+ * Create a variant from a BigDecimal. A negative scale is rescaled to 0,
which keeps the
+ * numeric value.
+ *
+ * @throws VariantTypeException if the precision or the scale of the
rescaled decimal exceeds
+ * {@link BinaryVariantUtil#MAX_DECIMAL16_PRECISION}
+ */
Variant of(BigDecimal bigDecimal);
/** Create a variant from an Instant. */
diff --git
a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java
b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java
index 3ce271896a2..a1b3eb5ef06 100644
---
a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java
+++
b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java
@@ -143,6 +143,19 @@ class BinaryVariantInternalBuilderTest {
.isInstanceOf(IOException.class);
}
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ "123456789012345678901234567890123456789",
+ "0.000000000000000000000000000000000000001"
+ })
+ void testParseJsonStoresNumbersOutsideDecimalRangeAsDouble(final String
number)
+ throws IOException {
+ BinaryVariant variant = BinaryVariantInternalBuilder.parseJson(number,
false);
+ assertThat(variant.getType()).isSameAs(Variant.Type.DOUBLE);
+ assertThat(variant.getDouble()).isEqualTo(Double.parseDouble(number));
+ }
+
@Test
void testAppendFloat() {
BinaryVariantInternalBuilder builder = new
BinaryVariantInternalBuilder(false);
diff --git
a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java
b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java
index 5379d34cd86..e3725c1d202 100644
---
a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java
+++
b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java
@@ -23,6 +23,7 @@ import org.apache.flink.core.testutils.CommonTestUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
import java.math.BigDecimal;
@@ -114,6 +115,56 @@ class BinaryVariantTest {
assertThat(builder.ofNull().isNull()).isTrue();
}
+ @Test
+ void testDecimalWithNegativeScale() {
+ Variant variant = builder.of(new BigDecimal("-1e5"));
+ assertThat(variant.getType()).isSameAs(Variant.Type.DECIMAL);
+ assertThat(variant.getDecimal()).isEqualByComparingTo("-100000");
+ assertThat(variant.toJson()).isEqualTo("-100000");
+
+ // 38 digits after rescaling
+ assertThat(builder.of(new
BigDecimal("1e37")).getDecimal()).isEqualByComparingTo("1e37");
+ assertThat(builder.of(new BigDecimal("0e999999999")).getDecimal())
+ .isEqualByComparingTo(BigDecimal.ZERO);
+
+ // getDecimal() strips trailing zeros, so its result can have a
negative scale
+ BigDecimal stripped = builder.of(BigDecimal.valueOf(100)).getDecimal();
+ assertThat(stripped.scale()).isNegative();
+
assertThat(builder.of(stripped).getDecimal()).isEqualByComparingTo("100");
+ }
+
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ "12345678901234567890123456789012345678",
+ "-0.12345678901234567890123456789012345678"
+ })
+ void testDecimalWithMaxPrecisionAndScale(final String decimal) {
+ assertThat(builder.of(new
BigDecimal(decimal)).getDecimal()).isEqualByComparingTo(decimal);
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ // precision 39
+ "123456789012345678901234567890123456789, 39, 0",
+ // scale 39
+ "0.000000000000000000000000000000000000001, 1, 39",
+ // 39 digits after rescaling the negative scale
+ "1e38, 1, -38",
+ "-1e999999999, 1, -999999999",
+ // the scale does not fit into a byte
+ "1e2147483647, 1, -2147483647"
+ })
+ void testDecimalOutOfRange(final String decimal, final int precision,
final int scale) {
+ assertThatThrownBy(() -> builder.of(new BigDecimal(decimal)))
+ .isInstanceOf(VariantTypeException.class)
+ .hasMessage(
+ "Decimal with precision %d and scale %d is outside the
range supported by "
+ + "variant decimals. After rescaling a
negative scale to 0, the "
+ + "precision and scale must not exceed 38.",
+ precision, scale);
+ }
+
@Test
void testNanosecondPrecisionVariant() {
// Microsecond-precision values keep using the compact
TIMESTAMP/TIMESTAMP_LTZ encoding,
