This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 464c28278d [format][python] Support TIME keys for MAP BLOB (#9009)
464c28278d is described below
commit 464c28278dbe39094ba83fe4459197e7d16389d4
Author: QuakeWang <[email protected]>
AuthorDate: Wed Aug 5 13:16:42 2026 +0800
[format][python] Support TIME keys for MAP BLOB (#9009)
---
docs/docs/concepts/spec/fileformat.md | 4 ++-
docs/docs/multimodal-table/blob.mdx | 2 +-
docs/docs/primary-key-table/blob-storage.md | 2 +-
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 22 +++++++++++---
.../format/blob/MapBlobElementSerializer.java | 1 +
.../paimon/format/blob/BlobFileFormatTest.java | 3 ++
.../pypaimon/common/map_blob_key_serializer.py | 34 ++++++++++++++++++++++
paimon-python/pypaimon/tests/blob_test.py | 24 +++++++++++++++
.../pypaimon/tests/e2e/java_py_read_write_test.py | 15 ++++++++++
9 files changed, 100 insertions(+), 7 deletions(-)
diff --git a/docs/docs/concepts/spec/fileformat.md
b/docs/docs/concepts/spec/fileformat.md
index 7701184ac9..541498df57 100644
--- a/docs/docs/concepts/spec/fileformat.md
+++ b/docs/docs/concepts/spec/fileformat.md
@@ -895,10 +895,12 @@ their encodings are:
| `DECIMAL(p, s)`, `p <= 18` | Eight-byte little-endian signed unscaled
integer |
| `DECIMAL(p, s)`, `p > 18` | Minimal-length signed big-endian
two's-complement unscaled integer |
| `DATE` | Four-byte little-endian signed count of days since 1970-01-01 |
+| `TIME(p)` | Four-byte little-endian signed count of milliseconds since
midnight |
| `CHAR`, `VARCHAR` | UTF-8 bytes |
The DECIMAL scale is defined by the field type and is not stored in each key.
An empty
-map has an entry count of zero and is distinct from a null map.
+map has an entry count of zero and is distinct from a null map. The `TIME(p)`
encoding
+uses Paimon's millisecond internal representation and does not add nanosecond
precision.
At the outer file index level, `-1` represents a null field and `-2`
represents a
field placeholder used by data evolution.
diff --git a/docs/docs/multimodal-table/blob.mdx
b/docs/docs/multimodal-table/blob.mdx
index 400b83766a..3ee89a0910 100644
--- a/docs/docs/multimodal-table/blob.mdx
+++ b/docs/docs/multimodal-table/blob.mdx
@@ -87,7 +87,7 @@ Paimon supports three storage modes for BLOB fields, selected
via **comment dire
This allows one table to mix different storage modes for different BLOB
columns.
`ARRAY<BLOB>` and `MAP<K, BLOB>` are supported only by `__BLOB_FIELD`;
descriptor-only and blob-view comment directives accept scalar BLOB fields
only.
-Map keys support the integer family, `BOOLEAN`, `DECIMAL`, `DATE`, `CHAR`, and
+Map keys support the integer family, `BOOLEAN`, `DECIMAL`, `DATE`, `TIME`,
`CHAR`, and
`VARCHAR`. Use non-null keys for compatibility across Flink, Spark, and Python.
## Table Options
diff --git a/docs/docs/primary-key-table/blob-storage.md
b/docs/docs/primary-key-table/blob-storage.md
index 7469d88044..3eff81bb83 100644
--- a/docs/docs/primary-key-table/blob-storage.md
+++ b/docs/docs/primary-key-table/blob-storage.md
@@ -94,7 +94,7 @@ array order, a null array, and null elements are preserved.
An empty array write
`MAP<K, BLOB>` is externalized value by value. Keys remain in the normal data
file and every non-null value is replaced
with a descriptor to managed storage. A null map, an empty map, and null
values are preserved. Supported key types are
-the integer family, `BOOLEAN`, `DECIMAL`, `DATE`, `CHAR`, and `VARCHAR`;
`blob-descriptor-field` and
+the integer family, `BOOLEAN`, `DECIMAL`, `DATE`, `TIME`, `CHAR`, and
`VARCHAR`; `blob-descriptor-field` and
`blob-view-field` remain scalar-only declarations.
`blob.target-file-size` controls when a writer rolls to a new managed payload
pack. A pack can contain payloads from
diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
index 403ec3c2dc..a45990f158 100644
--- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -1522,6 +1522,7 @@ public class JavaPyE2ETest {
"high_decimal_payloads",
DataTypes.MAP(DataTypes.DECIMAL(20, 2),
DataTypes.BLOB()))
.column("date_payloads",
DataTypes.MAP(DataTypes.DATE(), DataTypes.BLOB()))
+ .column("time_payloads",
DataTypes.MAP(DataTypes.TIME(3), DataTypes.BLOB()))
.option(ROW_TRACKING_ENABLED.key(), "true")
.option(DATA_EVOLUTION_ENABLED.key(), "true")
.option(BUCKET.key(), "-1")
@@ -1546,6 +1547,8 @@ public class JavaPyE2ETest {
new
BlobData("java-high-decimal".getBytes(StandardCharsets.UTF_8)));
Map<Object, Object> datePayloads = new LinkedHashMap<>();
datePayloads.put(-1, new
BlobData("java-date".getBytes(StandardCharsets.UTF_8)));
+ Map<Object, Object> timePayloads = new LinkedHashMap<>();
+ timePayloads.put(45_296_789, new
BlobData("java-time".getBytes(StandardCharsets.UTF_8)));
FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
@@ -1558,12 +1561,19 @@ public class JavaPyE2ETest {
new GenericMap(booleanPayloads),
new GenericMap(compactDecimalPayloads),
new GenericMap(highDecimalPayloads),
- new GenericMap(datePayloads)));
+ new GenericMap(datePayloads),
+ new GenericMap(timePayloads)));
write.write(
GenericRow.of(
- 2, new GenericMap(Collections.emptyMap()), null,
null, null, null));
- write.write(GenericRow.of(3, null, null, null, null, null));
- write.write(GenericRow.of(4, new GenericMap(last), null, null,
null, null));
+ 2,
+ new GenericMap(Collections.emptyMap()),
+ null,
+ null,
+ null,
+ null,
+ null));
+ write.write(GenericRow.of(3, null, null, null, null, null, null));
+ write.write(GenericRow.of(4, new GenericMap(last), null, null,
null, null, null));
commit.commit(write.prepareCommit());
}
@@ -1653,6 +1663,10 @@ public class JavaPyE2ETest {
InternalMap dateMap = row.getMap(5);
assertThat(dateMap.keyArray().getInt(0)).isEqualTo(-1);
assertSingleBlobValue(dateMap, valuePrefix + "-date");
+
+ InternalMap timeMap = row.getMap(6);
+
assertThat(timeMap.keyArray().getInt(0)).isEqualTo(45_296_789);
+ assertSingleBlobValue(timeMap, valuePrefix + "-time");
});
}
assertThat(found[0]).isTrue();
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/blob/MapBlobElementSerializer.java
b/paimon-format/src/main/java/org/apache/paimon/format/blob/MapBlobElementSerializer.java
index 3c67077e3d..6d73bfc5d6 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/blob/MapBlobElementSerializer.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/blob/MapBlobElementSerializer.java
@@ -468,6 +468,7 @@ final class MapBlobElementSerializer implements
BlobElementSerializer {
DecimalType decimalType = (DecimalType) keyType;
return new DecimalKeySerializer(decimalType.getPrecision(),
decimalType.getScale());
case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
return new IntKeySerializer();
case CHAR:
case VARCHAR:
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/blob/BlobFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/blob/BlobFileFormatTest.java
index 0c22894ee5..a155100b24 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/blob/BlobFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/blob/BlobFileFormatTest.java
@@ -499,6 +499,7 @@ public class BlobFileFormatTest {
DataTypes.DECIMAL(10, 2),
DataTypes.DECIMAL(20, 2),
DataTypes.DATE(),
+ DataTypes.TIME(3),
DataTypes.CHAR(10),
DataTypes.VARCHAR(10)
};
@@ -512,6 +513,7 @@ public class BlobFileFormatTest {
Decimal.fromBigDecimal(new BigDecimal("12.34"), 10, 2),
Decimal.fromBigDecimal(new
BigDecimal("123456789012345678.90"), 20, 2),
-1,
+ 45_296_789,
BinaryString.fromString("char"),
BinaryString.fromString("varchar")
};
@@ -535,6 +537,7 @@ public class BlobFileFormatTest {
(byte) 0xd2
},
{(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff},
+ {(byte) 0x95, 0x2c, (byte) 0xb3, 0x02},
"char".getBytes(),
"varchar".getBytes()
};
diff --git a/paimon-python/pypaimon/common/map_blob_key_serializer.py
b/paimon-python/pypaimon/common/map_blob_key_serializer.py
index 51bc3123c6..58eaa02dd4 100644
--- a/paimon-python/pypaimon/common/map_blob_key_serializer.py
+++ b/paimon-python/pypaimon/common/map_blob_key_serializer.py
@@ -157,6 +157,38 @@ class DateMapBlobKeySerializer(MapBlobKeySerializer):
return _EPOCH_DATE + datetime.timedelta(days=struct.unpack('<i',
data)[0])
+class TimeMapBlobKeySerializer(MapBlobKeySerializer):
+
+ def __init__(self, type_name: str):
+ self._type_name = type_name
+ self.fixed_length = 4
+
+ def serialize(self, key) -> bytes:
+ if not isinstance(key, datetime.time):
+ raise ValueError(
+ f"MAP<X, BLOB> {self._type_name} key must be a datetime.time."
+ )
+ millis = (
+ (key.hour * 3600 + key.minute * 60 + key.second) * 1000
+ + key.microsecond // 1000
+ )
+ return struct.pack('<i', millis)
+
+ def deserialize(self, data: bytes):
+ if len(data) != self.fixed_length:
+ raise ValueError(
+ f"Expected {self.fixed_length} key bytes, but found
{len(data)}."
+ )
+ millis = struct.unpack('<i', data)[0]
+ seconds, millis = divmod(millis, 1000)
+ minutes, second = divmod(seconds, 60)
+ hour, minute = divmod(minutes, 60)
+ try:
+ return datetime.time(hour, minute, second, millis * 1000)
+ except ValueError as error:
+ raise ValueError("Invalid MAP<X, BLOB> TIME key.") from error
+
+
def create_map_blob_key_serializer(data_type: DataType) ->
MapBlobKeySerializer:
if not isinstance(data_type, AtomicType):
raise ValueError(f"Unsupported key type for MAP<X, BLOB>: {data_type}")
@@ -177,6 +209,8 @@ def create_map_blob_key_serializer(data_type: DataType) ->
MapBlobKeySerializer:
return DecimalMapBlobKeySerializer(type_name, precision, scale)
if type_name == 'DATE':
return DateMapBlobKeySerializer()
+ if type_name == 'TIME' or type_name.startswith('TIME('):
+ return TimeMapBlobKeySerializer(type_name)
if type_name == 'STRING' or type_name.startswith('CHAR') or
type_name.startswith('VARCHAR'):
return MapBlobKeySerializer(type_name)
raise ValueError(f"Unsupported key type for MAP<X, BLOB>: {data_type}")
diff --git a/paimon-python/pypaimon/tests/blob_test.py
b/paimon-python/pypaimon/tests/blob_test.py
index e7dfc9d179..70100d2d1a 100644
--- a/paimon-python/pypaimon/tests/blob_test.py
+++ b/paimon-python/pypaimon/tests/blob_test.py
@@ -3042,6 +3042,7 @@ class BlobEndToEndTest(unittest.TestCase):
self.assertEqual(result["tail"].to_data(), b"third")
def test_map_blob_key_types_and_rejections(self):
+ from pypaimon.common.map_blob_key_serializer import
create_map_blob_key_serializer
from pypaimon.write.blob_format_writer import BlobFormatWriter
file_io = LocalFileIO(self.temp_dir, Options({}))
@@ -3058,6 +3059,7 @@ class BlobEndToEndTest(unittest.TestCase):
Decimal("123456789012345678.90"),
),
(AtomicType("DATE"), datetime.date(1969, 12, 31)),
+ (AtomicType("TIME(3)"), datetime.time(12, 34, 56, 789000)),
(AtomicType("STRING"), "string"),
(AtomicType("CHAR(3)"), "abc"),
(AtomicType("VARCHAR(10)"), "varchar"),
@@ -3072,6 +3074,7 @@ class BlobEndToEndTest(unittest.TestCase):
b"\xd2\x04\x00\x00\x00\x00\x00\x00",
b"\x00\xab\x54\xa9\x8c\xeb\x1f\x0a\xd2",
b"\xff\xff\xff\xff",
+ b"\x95\x2c\xb3\x02",
b"string",
b"abc",
b"varchar",
@@ -3124,6 +3127,14 @@ class BlobEndToEndTest(unittest.TestCase):
finally:
reader.close()
+ time_serializer = create_map_blob_key_serializer(AtomicType("TIME(9)"))
+ serialized_time = time_serializer.serialize(datetime.time(12, 34, 56,
789999))
+ self.assertEqual(serialized_time, b"\x95\x2c\xb3\x02")
+ self.assertEqual(
+ time_serializer.deserialize(serialized_time),
+ datetime.time(12, 34, 56, 789000),
+ )
+
output = io.BytesIO()
unsupported_key_writer = BlobFormatWriter(output)
unsupported_key_field = DataField(
@@ -3164,6 +3175,19 @@ class BlobEndToEndTest(unittest.TestCase):
RowKind.INSERT,
))
+ invalid_time_key_writer = BlobFormatWriter(io.BytesIO())
+ time_key_field = DataField(
+ 0,
+ "blob_map",
+ MapType(True, AtomicType("TIME(3)"), AtomicType("BLOB")),
+ )
+ with self.assertRaisesRegex(ValueError, "key must be a datetime.time"):
+ invalid_time_key_writer.add_element(GenericRow(
+ [{"not-a-time": BlobData(b"value")}],
+ [time_key_field],
+ RowKind.INSERT,
+ ))
+
def test_reject_malformed_map_blob_payloads(self):
string_field = DataField(
0,
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index 5db6840a31..ce7b9d6038 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -1564,6 +1564,9 @@ class JavaPyReadWriteTest(unittest.TestCase):
'date_payloads': {
datetime.date(1969, 12, 31): b'java-date',
},
+ 'time_payloads': {
+ datetime.time(12, 34, 56, 789000): b'java-time',
+ },
}
for name, expected in expected_additional_payloads.items():
self.assertEqual(
@@ -1580,6 +1583,7 @@ class JavaPyReadWriteTest(unittest.TestCase):
high_decimal_map_blob_type = pa.map_(
pa.decimal128(20, 2), pa.large_binary())
date_map_blob_type = pa.map_(pa.date32(), pa.large_binary())
+ time_map_blob_type = pa.map_(pa.time32('ms'), pa.large_binary())
pa_schema = pa.schema([
('id', pa.int32()),
('payloads', map_blob_type),
@@ -1587,6 +1591,7 @@ class JavaPyReadWriteTest(unittest.TestCase):
('compact_decimal_payloads', compact_decimal_map_blob_type),
('high_decimal_payloads', high_decimal_map_blob_type),
('date_payloads', date_map_blob_type),
+ ('time_payloads', time_map_blob_type),
])
schema = Schema.from_pyarrow_schema(
pa_schema,
@@ -1635,6 +1640,13 @@ class JavaPyReadWriteTest(unittest.TestCase):
)], None, None, None],
type=date_map_blob_type,
),
+ 'time_payloads': pa.array(
+ [[(
+ datetime.time(12, 34, 56, 789000),
+ b'python-time',
+ )], None, None, None],
+ type=time_map_blob_type,
+ ),
}, schema=pa_schema)
write_builder = table.new_batch_write_builder()
table_write = write_builder.new_write()
@@ -1670,6 +1682,9 @@ class JavaPyReadWriteTest(unittest.TestCase):
'date_payloads': {
datetime.date(1969, 12, 31): b'python-date',
},
+ 'time_payloads': {
+ datetime.time(12, 34, 56, 789000): b'python-time',
+ },
}
for name, expected in expected_additional_payloads.items():
self.assertEqual(
