This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ed613d6abc [format] Support numeric type widening when reading Avro
files (#9362)
ed613d6abc is described below
commit ed613d6abc27ff5d3b6d313346af23af47316030
Author: Arnav Balyan <[email protected]>
AuthorDate: Sun Aug 23 20:48:09 2026 +0530
[format] Support numeric type widening when reading Avro files (#9362)
---
.../paimon/format/avro/FieldReaderFactory.java | 44 ++++++++++++++++++
.../paimon/format/avro/AvroFileFormatTest.java | 52 ++++++++++++++++++++++
2 files changed, 96 insertions(+)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldReaderFactory.java
b/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldReaderFactory.java
index 1369a359eb..6c5a4d018e 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldReaderFactory.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/avro/FieldReaderFactory.java
@@ -76,10 +76,16 @@ public class FieldReaderFactory implements
AvroSchemaVisitor<FieldReader> {
private static final FieldReader INT_READER = new IntReader();
+ private static final FieldReader INT_TO_BIGINT_READER = new
IntToBigIntReader();
+
+ private static final FieldReader INT_TO_DOUBLE_READER = new
IntToDoubleReader();
+
private static final FieldReader BIGINT_READER = new BigIntReader();
private static final FieldReader FLOAT_READER = new FloatReader();
+ private static final FieldReader FLOAT_TO_DOUBLE_READER = new
FloatToDoubleReader();
+
private static final FieldReader DOUBLE_READER = new DoubleReader();
private static final FieldReader TIMESTAMP_MILLS_READER = new
TimestampMillsReader();
@@ -93,6 +99,20 @@ public class FieldReaderFactory implements
AvroSchemaVisitor<FieldReader> {
&& type.getTypeRoot() == DataTypeRoot.BLOB) {
return new BlobBytesReader(uriReader);
}
+ if (type != null && primitive.getLogicalType() == null) {
+ if (primitive.getType() == Schema.Type.INT) {
+ if (type.getTypeRoot() == DataTypeRoot.BIGINT) {
+ return INT_TO_BIGINT_READER;
+ }
+ if (type.getTypeRoot() == DataTypeRoot.DOUBLE) {
+ return INT_TO_DOUBLE_READER;
+ }
+ }
+ if (primitive.getType() == Schema.Type.FLOAT
+ && type.getTypeRoot() == DataTypeRoot.DOUBLE) {
+ return FLOAT_TO_DOUBLE_READER;
+ }
+ }
return AvroSchemaVisitor.super.primitive(primitive, type);
}
@@ -335,6 +355,22 @@ public class FieldReaderFactory implements
AvroSchemaVisitor<FieldReader> {
}
}
+ private static class IntToBigIntReader extends IntReader {
+
+ @Override
+ public Object read(Decoder decoder, Object reuse) throws IOException {
+ return (long) decoder.readInt();
+ }
+ }
+
+ private static class IntToDoubleReader extends IntReader {
+
+ @Override
+ public Object read(Decoder decoder, Object reuse) throws IOException {
+ return (double) decoder.readInt();
+ }
+ }
+
private static class BigIntReader implements FieldReader {
@Override
@@ -361,6 +397,14 @@ public class FieldReaderFactory implements
AvroSchemaVisitor<FieldReader> {
}
}
+ private static class FloatToDoubleReader extends FloatReader {
+
+ @Override
+ public Object read(Decoder decoder, Object reuse) throws IOException {
+ return (double) decoder.readFloat();
+ }
+ }
+
private static class DoubleReader implements FieldReader {
@Override
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
index ed60c016a7..0bcc799b3b 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java
@@ -37,6 +37,10 @@ import org.apache.paimon.types.RowType;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DecoderFactory;
@@ -52,6 +56,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.NoSuchElementException;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
@@ -271,6 +276,53 @@ public class AvroFileFormatTest {
assertThat(decoder.isEnd()).isTrue();
}
+ @Test
+ void testReadNumericTypeWidening() throws IOException {
+ Schema writerSchema =
+ SchemaBuilder.record("record")
+ .fields()
+ .requiredInt("int_to_bigint")
+ .requiredInt("int_to_double")
+ .requiredFloat("float_to_double")
+ .endRecord();
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path file = new Path(new Path(tempPath.toUri()),
UUID.randomUUID().toString());
+
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false);
+ DataFileWriter<GenericRecord> writer =
+ new DataFileWriter<>(new
GenericDatumWriter<>(writerSchema))) {
+ writer.create(writerSchema, out);
+ GenericRecord record = new GenericData.Record(writerSchema);
+ record.put("int_to_bigint", 42);
+ record.put("int_to_double", 21);
+ record.put("float_to_double", 10.5f);
+ writer.append(record);
+ }
+
+ RowType tableType =
+ RowType.builder()
+ .field("int_to_bigint", DataTypes.BIGINT().notNull())
+ .field("int_to_double", DataTypes.DOUBLE().notNull())
+ .field("float_to_double", DataTypes.DOUBLE().notNull())
+ .build();
+ List<Object> values = new ArrayList<>();
+ try (RecordReader<InternalRow> reader =
+ fileFormat
+ .createReaderFactory(tableType, tableType, new
ArrayList<>())
+ .createReader(
+ new FormatReaderContext(
+ fileIO, file,
fileIO.getFileSize(file), null, null))) {
+ reader.forEachRemaining(
+ row -> {
+ values.add(row.getLong(0));
+ values.add(row.getDouble(1));
+ values.add(row.getDouble(2));
+ });
+ }
+
+ assertThat(values).containsExactly(42L, 21.0d, 10.5d);
+ }
+
@Test
void testReadsLargeZstdBlock() throws IOException {
RowType rowType =
