This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 19a933b7d9 [python] Fix BLOB CRC32 compatibility with Java (#9426)
19a933b7d9 is described below
commit 19a933b7d96658f19a87301b0990919702464fd3
Author: liangjie <[email protected]>
AuthorDate: Fri Aug 28 15:41:33 2026 +0800
[python] Fix BLOB CRC32 compatibility with Java (#9426)
---
paimon-python/pypaimon/tests/blob_test.py | 90 +++++++++++++++++++++-
paimon-python/pypaimon/write/blob_format_writer.py | 6 +-
2 files changed, 88 insertions(+), 8 deletions(-)
diff --git a/paimon-python/pypaimon/tests/blob_test.py
b/paimon-python/pypaimon/tests/blob_test.py
index 4c6a7b1932..bbc095cdf7 100644
--- a/paimon-python/pypaimon/tests/blob_test.py
+++ b/paimon-python/pypaimon/tests/blob_test.py
@@ -2228,12 +2228,29 @@ class BlobEndToEndTest(unittest.TestCase):
return output.getvalue(), payload
def test_blob_crc_fallback_matches_zlib(self):
+ record, _ = self._write_blob_record_with_crc_backend(zlib)
+ self._assert_record_crc(record)
+
+ def test_blob_array_crc_includes_record_length(self):
+ from pypaimon.write.blob_format_writer import BlobFormatWriter
+
+ output = io.BytesIO()
+ writer = BlobFormatWriter(output)
+ writer.add_blob_array(
+ 'blob_field', [BlobData(b'first'), None, BlobData(b'second')])
+ self._assert_record_crc(output.getvalue())
+
+ def test_blob_map_crc_includes_record_length(self):
from pypaimon.write.blob_format_writer import BlobFormatWriter
- record, payload = self._write_blob_record_with_crc_backend(zlib)
- expected_crc = zlib.crc32(
- struct.pack('<I', BlobFormatWriter.MAGIC_NUMBER))
- expected_crc = zlib.crc32(payload, expected_crc) & 0xffffffff
+ output = io.BytesIO()
+ writer = BlobFormatWriter(output)
+ writer.add_blob_map(
+ 'blob_field', {'key': BlobData(b'value')}, AtomicType('STRING'))
+ self._assert_record_crc(output.getvalue())
+
+ def _assert_record_crc(self, record):
+ expected_crc = zlib.crc32(record[:-4]) & 0xffffffff
actual_crc = struct.unpack('<I', record[-4:])[0]
self.assertEqual(expected_crc, actual_crc)
@@ -2765,6 +2782,71 @@ class BlobEndToEndTest(unittest.TestCase):
+ struct.pack('<I', index_length)
)
+ def test_blob_golden_bytes(self):
+ from pypaimon.write.blob_format_writer import BlobFormatWriter
+
+ source_file_path = os.path.join(self.temp_dir, "source.bin")
+ with open(source_file_path, 'wb') as source_file:
+ source_file.write(b"descriptor")
+
+ blob_file_path = os.path.join(self.temp_dir, "blob_golden.blob")
+ fields = [DataField(0, "blob", AtomicType("BLOB"))]
+ writer = BlobFormatWriter(open(blob_file_path, 'wb'))
+ writer.add_element(GenericRow([BlobData(b"inline")], fields,
RowKind.INSERT))
+ writer.add_element(
+ GenericRow([Blob.from_local(source_file_path)], fields,
RowKind.INSERT)
+ )
+ writer.add_element(GenericRow([None], fields, RowKind.INSERT))
+ writer.add_element(GenericRow([Blob.PLACE_HOLDER], fields,
RowKind.INSERT))
+ writer.close()
+
+ with open(blob_file_path, 'rb') as blob_file:
+ self.assertEqual(
+ blob_file.read().hex(),
+ "cf114e58696e6c696e6516000000000000002960c8e9"
+ "cf114e5864657363726970746f721a000000000000003f69146b"
+ "2c0835010400000001",
+ )
+
+ def test_array_blob_golden_bytes(self):
+ from pypaimon.write.blob_format_writer import BlobFormatWriter
+
+ source_file_path = os.path.join(self.temp_dir, "source.bin")
+ with open(source_file_path, 'wb') as source_file:
+ source_file.write(b"descriptor")
+
+ blob_file_path = os.path.join(self.temp_dir, "array_golden.blob")
+ fields = [DataField(
+ 0,
+ "blob_array",
+ ArrayType(True, AtomicType("BLOB")),
+ )]
+ writer = BlobFormatWriter(open(blob_file_path, 'wb'))
+ writer.add_element(GenericRow([[]], fields, RowKind.INSERT))
+ writer.add_element(GenericRow(
+ [[
+ BlobData(b"inline"),
+ None,
+ BlobData(b""),
+ Blob.from_local(source_file_path),
+ ]],
+ fields,
+ RowKind.INSERT,
+ ))
+ writer.add_element(GenericRow([None], fields, RowKind.INSERT))
+ writer.add_element(
+ GenericRow([Blob.ARRAY_PLACE_HOLDER], fields, RowKind.INSERT)
+ )
+ writer.close()
+
+ with open(blob_file_path, 'rb') as blob_file:
+ self.assertEqual(
+ blob_file.read().hex(),
+ "cf114e58424342410100000000000000001d000000000000009bd49157"
+ "cf114e58424342410104000000696e6c696e6564657363726970746f72"
+ "0c0d0214040000003100000000000000d08307713a2863010400000001",
+ )
+
def test_map_blob_golden_bytes(self):
from pypaimon.write.blob_format_writer import BlobFormatWriter
diff --git a/paimon-python/pypaimon/write/blob_format_writer.py
b/paimon-python/pypaimon/write/blob_format_writer.py
index 8a7969b271..0dfa44d180 100644
--- a/paimon-python/pypaimon/write/blob_format_writer.py
+++ b/paimon-python/pypaimon/write/blob_format_writer.py
@@ -129,8 +129,7 @@ class BlobFormatWriter:
# Write length (8 bytes, little endian)
length_bytes = struct.pack('<Q', bin_length)
- self.output_stream.write(length_bytes)
- self.position += 8
+ crc32 = self._write_with_crc(length_bytes, crc32)
# Write CRC32 (4 bytes, little endian)
crc_bytes = struct.pack('<I', crc32 & 0xffffffff)
@@ -195,8 +194,7 @@ class BlobFormatWriter:
bin_length = self.position - previous_pos + self.METADATA_SIZE
self.lengths.append(bin_length)
- self.output_stream.write(struct.pack('<Q', bin_length))
- self.position += 8
+ crc32 = self._write_with_crc(struct.pack('<Q', bin_length), crc32)
self.output_stream.write(struct.pack('<I', crc32 & 0xffffffff))
self.position += 4
