This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 0078e187 refactor(parquet): generalize read type adaptation for BLOB
(#322)
0078e187 is described below
commit 0078e18794e04b548c6e72872ac893da6c2ea7e0
Author: Zhang Jiawei <[email protected]>
AuthorDate: Fri Sep 11 14:11:49 2026 +0800
refactor(parquet): generalize read type adaptation for BLOB (#322)
---
src/paimon/core/schema/arrow_schema_validator.cpp | 8 +-
.../core/schema/arrow_schema_validator_test.cpp | 17 ++--
src/paimon/core/schema/schema_validation.cpp | 6 +-
src/paimon/core/schema/schema_validation_test.cpp | 3 +-
src/paimon/core/table/format/format_table_read.cpp | 2 +-
src/paimon/format/parquet/CMakeLists.txt | 4 +-
.../format/parquet/parquet_file_batch_reader.cpp | 39 +++----
.../format/parquet/parquet_file_batch_reader.h | 7 +-
...converter.cpp => parquet_read_type_adapter.cpp} | 113 ++++++++++++---------
...amp_converter.h => parquet_read_type_adapter.h} | 24 +++--
...test.cpp => parquet_read_type_adapter_test.cpp} | 69 ++++++++-----
test/inte/paimon_read_compat_inte_test.cpp | 26 ++++-
.../append_types_compatibility.db/README.md | 18 +++-
...8a49858f-c0a7-468e-8d71-45b65142d422-0.parquet} | Bin
...ta-8a49858f-c0a7-468e-8d71-45b65142d422-1.blob} | Bin
...anifest-4db45a0c-f933-4043-80de-f1d5a5107966-0} | Bin 2249 -> 2232 bytes
...st-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0} | Bin 1006 -> 1006 bytes
...st-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1} | Bin 1006 -> 1115 bytes
.../schema/schema-0 | 14 +--
.../snapshot/EARLIEST | 0
.../snapshot/LATEST | 0
.../java_array_blob_types/snapshot/snapshot-1 | 18 ++++
...d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet} | Bin
...ta-d2b0d467-f565-4453-b400-468f9ec22a54-1.blob} | Bin
...anifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0} | Bin 2249 -> 2233 bytes
...st-list-d3262408-2da8-408b-9c67-b487ecf8565b-0} | Bin 1111 -> 1006 bytes
...st-list-d3262408-2da8-408b-9c67-b487ecf8565b-1} | Bin 1006 -> 1115 bytes
.../schema/schema-0 | 13 +--
.../snapshot/EARLIEST | 0
.../snapshot/LATEST | 0
.../java_map_blob_types/snapshot/snapshot-1 | 18 ++++
.../java_nested_blob_types/snapshot/snapshot-1 | 18 ----
...ta-043fd246-38df-40f7-9147-c1c416282ecc-0.blob} | Bin
...3cb0cb1a-5db9-478e-a727-6f34d6e800e5-0.parquet} | Bin
...anifest-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0} | Bin 2333 -> 2296 bytes
...st-list-2f17c472-fea6-42d0-8300-12da4874d53a-0} | Bin 799 -> 799 bytes
...st-list-2f17c472-fea6-42d0-8300-12da4874d53a-1} | Bin 899 -> 899 bytes
.../schema/schema-0 | 18 +---
.../snapshot/LATEST | 0
.../python_array_blob_types/snapshot/snapshot-1 | 15 +++
...4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet} | Bin
...ta-6e33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob} | Bin
...anifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0} | Bin 2333 -> 2298 bytes
...st-list-c762398d-e22c-4676-b43d-4564909a0859-0} | Bin 799 -> 799 bytes
...st-list-c762398d-e22c-4676-b43d-4564909a0859-1} | Bin 799 -> 899 bytes
.../schema/schema-0 | 17 +---
.../snapshot/LATEST | 0
.../python_map_blob_types/snapshot/snapshot-1 | 15 +++
.../python_nested_blob_types/snapshot/snapshot-1 | 15 ---
...-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet | Bin 0 -> 1124 bytes
...manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0 | Bin 0 -> 2421 bytes
...est-list-edec5851-5eca-466b-9bde-780df761f932-0 | Bin 0 -> 1117 bytes
...st-list-edec5851-5eca-466b-9bde-780df761f932-1} | Bin 1117 -> 1219 bytes
.../rust_array_blob_types/schema/schema-0 | 1 +
.../snapshot/LATEST | 0
.../rust_array_blob_types/snapshot/snapshot-1 | 1 +
...-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet | Bin 0 -> 1503 bytes
...manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0 | Bin 0 -> 2418 bytes
...est-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0 | Bin 0 -> 1117 bytes
...st-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1} | Bin 1219 -> 1219 bytes
.../rust_map_blob_types/schema/schema-0 | 1 +
.../snapshot/LATEST | 0
.../rust_map_blob_types/snapshot/snapshot-1 | 1 +
...-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet | Bin 2015 -> 0 bytes
...manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0 | Bin 2432 -> 0 bytes
.../rust_nested_blob_types/schema/schema-0 | 1 -
.../rust_nested_blob_types/snapshot/snapshot-1 | 1 -
67 files changed, 291 insertions(+), 212 deletions(-)
diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp
b/src/paimon/core/schema/arrow_schema_validator.cpp
index 00997936..c008d222 100644
--- a/src/paimon/core/schema/arrow_schema_validator.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator.cpp
@@ -173,7 +173,9 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
case arrow::Type::type::LARGE_BINARY: {
if (BlobUtils::IsBlobMetadata(key_value_metadata)) {
if (!allow_blob) {
- return Status::Invalid("Blob field must be a top-level
field.");
+ return Status::Invalid(
+ "BLOB field must be a top-level field or the direct
value of a "
+ "top-level MAP field.");
}
break;
}
@@ -258,7 +260,9 @@ Status ArrowSchemaValidator::ValidateField(const
std::shared_ptr<arrow::Field>&
case arrow::Type::type::LARGE_BINARY: {
if (BlobUtils::IsBlobField(field)) {
if (!allow_blob) {
- return Status::Invalid("Blob field must be a top-level
field.");
+ return Status::Invalid(
+ "BLOB field must be a top-level field or the direct
value of a "
+ "top-level MAP field.");
}
break;
}
diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp
b/src/paimon/core/schema/arrow_schema_validator_test.cpp
index d644e563..283b61ec 100644
--- a/src/paimon/core/schema/arrow_schema_validator_test.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp
@@ -170,7 +170,7 @@ TEST(ArrowSchemaValidatorTest, TestInvalidDataType) {
}
}
-TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) {
+TEST(ArrowSchemaValidatorTest, TestBlobFieldPlacement) {
{
auto arrow_schema =
arrow::schema(arrow::FieldVector({BlobUtils::ToArrowField("blob",
true)}));
@@ -186,14 +186,16 @@ TEST(ArrowSchemaValidatorTest,
TestBlobFieldMustBeTopLevel) {
arrow::field("nested",
arrow::struct_({BlobUtils::ToArrowField("blob", true)}));
auto arrow_schema =
arrow::schema(arrow::FieldVector({nested_blob_field}));
ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
- "Blob field must be a top-level field.");
+ "BLOB field must be a top-level field or the
direct value of a "
+ "top-level MAP field.");
}
{
auto array_blob_field =
arrow::field("array_blob",
arrow::list(BlobUtils::ToArrowField("item", true)));
auto arrow_schema =
arrow::schema(arrow::FieldVector({array_blob_field}));
ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
- "Blob field must be a top-level field.");
+ "BLOB field must be a top-level field or the
direct value of a "
+ "top-level MAP field.");
}
{
auto map_blob_field = arrow::field(
@@ -211,7 +213,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel)
{
arrow::map(arrow::utf8(),
arrow::struct_({BlobUtils::ToArrowField("blob", true)})));
auto arrow_schema =
arrow::schema(arrow::FieldVector({map_blob_field}));
ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
- "Blob field must be a top-level field.");
+ "BLOB field must be a top-level field or the
direct value of a "
+ "top-level MAP field.");
}
{
auto nested_map_blob_field = arrow::field(
@@ -220,7 +223,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel)
{
arrow::map(arrow::utf8(),
BlobUtils::ToArrowField("value", true))));
auto arrow_schema =
arrow::schema(arrow::FieldVector({nested_map_blob_field}));
ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
- "Blob field must be a top-level field.");
+ "BLOB field must be a top-level field or the
direct value of a "
+ "top-level MAP field.");
}
{
std::vector<DataField> nested_fields = {
@@ -230,7 +234,8 @@ TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel)
{
arrow::field("nested",
DataField::ConvertDataFieldsToArrowStructType(nested_fields)))};
auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields);
ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema),
- "Blob field must be a top-level field.");
+ "BLOB field must be a top-level field or the
direct value of a "
+ "top-level MAP field.");
}
}
diff --git a/src/paimon/core/schema/schema_validation.cpp
b/src/paimon/core/schema/schema_validation.cpp
index 19fbdb13..0dd2533d 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -789,9 +789,9 @@ Status SchemaValidation::ValidateMosaicDataFields(const
TableSchema& schema,
const std::vector<std::string> inline_blob_fields =
options.GetBlobInlineFields();
const std::set<std::string>
inline_blob_field_set(inline_blob_fields.begin(),
inline_blob_fields.end());
- // Match Java SchemaValidation by validating only fields stored in the
normal data file. C++
- // permits BLOB only as a top-level field; descriptor and view fields are
inline, so Mosaic
- // must reject them here.
+ // Match Java SchemaValidation by validating only fields stored in the
normal data file.
+ // Top-level BLOB fields stored in separate files are skipped; descriptor
and view fields are
+ // inline, so Mosaic must reject them here.
for (const DataField& field : schema.Fields()) {
if (BlobUtils::IsBlobField(field.ArrowField()) &&
inline_blob_field_set.count(field.Name()) == 0) {
diff --git a/src/paimon/core/schema/schema_validation_test.cpp
b/src/paimon/core/schema/schema_validation_test.cpp
index fed7ea5a..11755578 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -1245,7 +1245,8 @@ TEST(SchemaValidationTest,
TestMapSharedShreddingRejectsBlobValue) {
});
ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, nested_schema,
/*partition_keys=*/{},
/*primary_keys=*/{}, options),
- "Blob field must be a top-level field.");
+ "BLOB field must be a top-level field or the direct
value of a "
+ "top-level MAP field.");
}
TEST(SchemaValidationTest, TestMapSharedShreddingCompression) {
diff --git a/src/paimon/core/table/format/format_table_read.cpp
b/src/paimon/core/table/format/format_table_read.cpp
index cda8cbfe..7be0e2f8 100644
--- a/src/paimon/core/table/format/format_table_read.cpp
+++ b/src/paimon/core/table/format/format_table_read.cpp
@@ -60,7 +60,7 @@ namespace {
/// `table_unit`. A file's unit means something else in each format, so the
rule does too:
///
/// - parquet records the unit, so the file's is the one its values are in.
Only `MILLI` under a
-/// `SECOND` column is rescaled, by `ParquetTimestampConverter`, which
parquet's own round trip
+/// `SECOND` column is rescaled, by `ParquetReadTypeAdapter`, which
parquet's own round trip
/// needs since it has no `SECOND`. Any other pair would be relabelled
rather than rescaled -
/// `us` under an `ms` type reads a thousand times too large - so it is
refused;
/// - orc records none: a value is a seconds-plus-nanoseconds pair that
`OrcAdapter` builds at
diff --git a/src/paimon/format/parquet/CMakeLists.txt
b/src/paimon/format/parquet/CMakeLists.txt
index 60d89851..937e7124 100644
--- a/src/paimon/format/parquet/CMakeLists.txt
+++ b/src/paimon/format/parquet/CMakeLists.txt
@@ -19,7 +19,7 @@ set(PAIMON_PARQUET_FILE_FORMAT
predicate_converter.cpp
file_reader_wrapper.cpp
page_filtered_row_group_reader.cpp
- parquet_timestamp_converter.cpp
+ parquet_read_type_adapter.cpp
parquet_file_batch_reader.cpp
parquet_file_format_factory.cpp
parquet_format_writer.cpp
@@ -54,7 +54,7 @@ if(PAIMON_BUILD_TESTS)
SOURCES
file_reader_wrapper_test.cpp
page_filtered_row_group_reader_test.cpp
- parquet_timestamp_converter_test.cpp
+ parquet_read_type_adapter_test.cpp
parquet_vector_io_test.cpp
parquet_field_id_converter_test.cpp
parquet_file_batch_reader_test.cpp
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
index 89758dc5..76f0df13 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
@@ -54,8 +54,8 @@
#include "paimon/core/utils/nested_projection_utils.h"
#include "paimon/format/parquet/parquet_field_id_converter.h"
#include "paimon/format/parquet/parquet_format_defs.h"
+#include "paimon/format/parquet/parquet_read_type_adapter.h"
#include "paimon/format/parquet/parquet_schema_util.h"
-#include "paimon/format/parquet/parquet_timestamp_converter.h"
#include "paimon/format/parquet/predicate_converter.h"
#include "paimon/reader/batch_reader.h"
#include "paimon/utils/roaring_bitmap32.h"
@@ -75,10 +75,9 @@ class Predicate;
namespace paimon::parquet {
namespace {
-// LIST/MAP do not support pruning fields from their nested value types, but
physical and
-// logical leaf types may still differ (for example, Parquet reports LTZ
timestamps as UTC
-// while Paimon exposes them in the local timezone). Compare only the nested
projection shape
-// here so those representation differences are handled by the normal cast
path.
+// LIST, MAP, and FIXED_SIZE_LIST do not support partial projection of their
nested values.
+// Require matching nested structures while allowing the leaf-type
representation differences
+// explicitly handled below.
bool HasSameNestedProjectionShape(const std::shared_ptr<arrow::DataType>&
read_type,
const std::shared_ptr<arrow::DataType>&
file_type) {
const bool read_is_nested = ArrowSchemaValidator::IsNestedType(read_type);
@@ -87,8 +86,8 @@ bool HasSameNestedProjectionShape(const
std::shared_ptr<arrow::DataType>& read_t
if (read_is_nested || file_is_nested) {
return false;
}
- // ParquetTimestampConverter explicitly supports timestamp unit and
timezone
- // conversion after reading. Other atomic type differences remain
unsupported here.
+ // ParquetReadTypeAdapter supports timezone retyping and the
millisecond-to-second
+ // conversion required for Paimon timestamps after reading.
if (read_type->id() == arrow::Type::TIMESTAMP &&
file_type->id() == arrow::Type::TIMESTAMP) {
const auto& read_timestamp = static_cast<const
arrow::TimestampType&>(*read_type);
@@ -356,11 +355,11 @@ Result<std::unique_ptr<::ArrowSchema>>
ParquetFileBatchReader::GetFileSchema() c
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> new_schema,
ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(file_schema));
PAIMON_ASSIGN_OR_RAISE(
- std::shared_ptr<arrow::DataType> new_type,
-
ParquetTimestampConverter::AdjustTimezone(arrow::struct_(new_schema->fields())));
+ std::shared_ptr<arrow::DataType> normalized_type,
+
ParquetReadTypeAdapter::NormalizeFileType(arrow::struct_(new_schema->fields())));
auto c_schema = std::make_unique<::ArrowSchema>();
- PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*new_type,
c_schema.get()));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*normalized_type,
c_schema.get()));
return c_schema;
}
PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::GetFileSchema")
@@ -378,8 +377,8 @@ Status ParquetFileBatchReader::SetReadSchema(
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> file_schema,
GetLogicalFileSchema());
- // Recursively match read_schema against file_schema by field names.
- // STRUCT supports sub-field projection; LIST/MAP require exact type
match.
+ // Recursively match read_schema against file_schema by field names.
STRUCT supports
+ // sub-field projection; LIST, MAP, and FIXED_SIZE_LIST require
matching nested shapes.
PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> column_indices,
ComputeNestedColumnIndices(read_schema,
file_schema));
@@ -773,20 +772,8 @@ Result<BatchReader::ReadBatch>
ParquetFileBatchReader::NextBatch() {
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
batch->ToStructArray());
PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate());
- PAIMON_ASSIGN_OR_RAISE(bool need_cast,
ParquetTimestampConverter::NeedCastArrayForTimestamp(
- array->type(),
read_data_type_));
- if (need_cast) {
- PAIMON_ASSIGN_OR_RAISE(array,
ParquetTimestampConverter::CastArrayForTimestamp(
- array, read_data_type_,
arrow_pool_));
- }
- PAIMON_ASSIGN_OR_RAISE(need_cast,
ParquetTimestampConverter::NeedCastArrayForTimestamp(
- array->type(), read_data_type_));
- if (need_cast) {
- return Status::Invalid(fmt::format(
- "unexpected: in parquet, after CastArrayForTimestamp, output
type {} not "
- "equal with read schema {}",
- array->type()->ToString(), read_data_type_->ToString()));
- }
+ PAIMON_ASSIGN_OR_RAISE(
+ array, ParquetReadTypeAdapter::AdaptArray(array, read_data_type_,
arrow_pool_));
PAIMON_RETURN_NOT_OK(GenerateRowMapping(array->length()));
std::unique_ptr<ArrowArray> c_array = std::make_unique<ArrowArray>();
std::unique_ptr<ArrowSchema> c_schema =
std::make_unique<ArrowSchema>();
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h
b/src/paimon/format/parquet/parquet_file_batch_reader.h
index 95fcf2be..565dec9b 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.h
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.h
@@ -201,8 +201,8 @@ class ParquetFileBatchReader : public
PrefetchFileBatchReader {
/// Recursively collect leaf column indices for the sub-fields in read_type
/// that match file_type by paimon field ID. Unmatched sub-fields in
file_type
- /// have their leaf indices skipped. Partial projection inside LIST/MAP is
- /// not supported and will return Invalid.
+ /// have their leaf indices skipped. Partial projection inside LIST, MAP,
or
+ /// FIXED_SIZE_LIST is not supported and will return Invalid.
static Status CollectLeafIndices(const std::shared_ptr<arrow::DataType>&
read_type,
const std::shared_ptr<arrow::DataType>&
file_type,
int32_t* leaf_index,
std::vector<int32_t>* indices);
@@ -213,7 +213,8 @@ class ParquetFileBatchReader : public
PrefetchFileBatchReader {
/// Compute leaf column indices by recursively matching read_schema against
/// file_schema using paimon field IDs. STRUCT supports sub-field
projection
- /// (unmatched sub-fields are skipped). LIST/MAP require exact type match.
+ /// (unmatched sub-fields are skipped). LIST, MAP, and FIXED_SIZE_LIST
require
+ /// matching nested shapes.
static Result<std::vector<int32_t>> ComputeNestedColumnIndices(
const std::shared_ptr<arrow::Schema>& read_schema,
const std::shared_ptr<arrow::Schema>& file_schema);
diff --git a/src/paimon/format/parquet/parquet_timestamp_converter.cpp
b/src/paimon/format/parquet/parquet_read_type_adapter.cpp
similarity index 68%
rename from src/paimon/format/parquet/parquet_timestamp_converter.cpp
rename to src/paimon/format/parquet/parquet_read_type_adapter.cpp
index 01521762..8ba47dc0 100644
--- a/src/paimon/format/parquet/parquet_timestamp_converter.cpp
+++ b/src/paimon/format/parquet/parquet_read_type_adapter.cpp
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "paimon/format/parquet/parquet_timestamp_converter.h"
+#include "paimon/format/parquet/parquet_read_type_adapter.h"
#include <memory>
#include <string>
@@ -31,10 +31,9 @@
#include "paimon/core/casting/timestamp_to_timestamp_cast_executor.h"
namespace paimon::parquet {
-namespace {
-bool IsAlreadyLogicalBlob(const std::shared_ptr<arrow::Field>& src_field,
- const std::shared_ptr<arrow::Field>& target_field) {
+bool ParquetReadTypeAdapter::IsCompatibleBlob(const
std::shared_ptr<arrow::Field>& src_field,
+ const
std::shared_ptr<arrow::Field>& target_field) {
// ARROW:schema may restore an inline BLOB's physical binary column to its
logical
// large_binary representation before this compatibility check runs.
return src_field->type()->id() == arrow::Type::LARGE_BINARY &&
@@ -42,9 +41,7 @@ bool IsAlreadyLogicalBlob(const
std::shared_ptr<arrow::Field>& src_field,
BlobUtils::IsBlobMetadata(target_field->metadata());
}
-} // namespace
-
-Result<std::shared_ptr<arrow::DataType>>
ParquetTimestampConverter::AdjustTimezone(
+Result<std::shared_ptr<arrow::DataType>>
ParquetReadTypeAdapter::NormalizeFileType(
const std::shared_ptr<arrow::DataType>& src_data_type) {
arrow::Type::type type = src_data_type->id();
switch (type) {
@@ -54,7 +51,7 @@ Result<std::shared_ptr<arrow::DataType>>
ParquetTimestampConverter::AdjustTimezo
new_fields.reserve(src_struct_type->num_fields());
for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) {
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::DataType>
sub_type,
-
AdjustTimezone(src_struct_type->field(i)->type()));
+
NormalizeFileType(src_struct_type->field(i)->type()));
new_fields.push_back(src_struct_type->field(i)->WithType(sub_type));
}
return arrow::struct_(new_fields);
@@ -62,9 +59,9 @@ Result<std::shared_ptr<arrow::DataType>>
ParquetTimestampConverter::AdjustTimezo
case arrow::Type::type::MAP: {
auto* src_map_type =
checked_cast<arrow::MapType*>(src_data_type.get());
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::DataType> key_type,
- AdjustTimezone(src_map_type->key_type()));
+
NormalizeFileType(src_map_type->key_type()));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::DataType> item_type,
- AdjustTimezone(src_map_type->item_type()));
+
NormalizeFileType(src_map_type->item_type()));
return std::make_shared<arrow::MapType>(
src_map_type->key_field()->WithType(key_type),
src_map_type->item_field()->WithType(item_type));
@@ -72,7 +69,7 @@ Result<std::shared_ptr<arrow::DataType>>
ParquetTimestampConverter::AdjustTimezo
case arrow::Type::type::LIST: {
auto* src_list_type =
checked_cast<arrow::ListType*>(src_data_type.get());
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::DataType> value_type,
-
AdjustTimezone(src_list_type->value_type()));
+
NormalizeFileType(src_list_type->value_type()));
return
arrow::list(src_list_type->value_field()->WithType(value_type));
}
case arrow::Type::type::TIMESTAMP: {
@@ -86,12 +83,25 @@ Result<std::shared_ptr<arrow::DataType>>
ParquetTimestampConverter::AdjustTimezo
}
}
-Result<bool> ParquetTimestampConverter::NeedCastArrayForTimestamp(
+Result<bool> ParquetReadTypeAdapter::NeedsArrayConversion(
const std::shared_ptr<arrow::DataType>& src_data_type,
const std::shared_ptr<arrow::DataType>& target_data_type) {
+ return NeedsArrayConversionImpl(arrow::field("", src_data_type),
+ arrow::field("", target_data_type));
+}
+
+Result<bool> ParquetReadTypeAdapter::NeedsArrayConversionImpl(
+ const std::shared_ptr<arrow::Field>& src_field,
+ const std::shared_ptr<arrow::Field>& target_field) {
+ if (IsCompatibleBlob(src_field, target_field)) {
+ return false;
+ }
+
+ const std::shared_ptr<arrow::DataType>& src_data_type = src_field->type();
+ const std::shared_ptr<arrow::DataType>& target_data_type =
target_field->type();
arrow::Type::type type = src_data_type->id();
if (type != target_data_type->id()) {
- return Status::Invalid(fmt::format("src type {} and target type {}
mismatch",
+ return Status::Invalid(fmt::format("source type {} and target type {}
mismatch",
src_data_type->ToString(),
target_data_type->ToString()));
}
@@ -101,18 +111,13 @@ Result<bool>
ParquetTimestampConverter::NeedCastArrayForTimestamp(
auto* target_struct_type =
checked_cast<arrow::StructType*>(target_data_type.get());
if (src_struct_type->num_fields() !=
target_struct_type->num_fields()) {
return Status::Invalid(
- fmt::format("src type {} and target type {} number of
fields mismatch",
+ fmt::format("source type {} and target type {} number of
fields mismatch",
src_data_type->ToString(),
target_data_type->ToString()));
}
for (int32_t i = 0; i < src_struct_type->num_fields(); ++i) {
- const std::shared_ptr<arrow::Field>& src_field =
src_struct_type->field(i);
- const std::shared_ptr<arrow::Field>& target_field =
target_struct_type->field(i);
- if (IsAlreadyLogicalBlob(src_field, target_field)) {
- continue;
- }
- PAIMON_ASSIGN_OR_RAISE(
- bool need_cast,
- NeedCastArrayForTimestamp(src_field->type(),
target_field->type()));
+ PAIMON_ASSIGN_OR_RAISE(bool need_cast,
+
NeedsArrayConversionImpl(src_struct_type->field(i),
+
target_struct_type->field(i)));
if (need_cast) {
return true;
}
@@ -124,38 +129,54 @@ Result<bool>
ParquetTimestampConverter::NeedCastArrayForTimestamp(
auto* target_map_type =
checked_cast<arrow::MapType*>(target_data_type.get());
PAIMON_ASSIGN_OR_RAISE(
bool need_cast,
- NeedCastArrayForTimestamp(src_map_type->key_type(),
target_map_type->key_type()));
+ NeedsArrayConversionImpl(src_map_type->key_field(),
target_map_type->key_field()));
if (need_cast) {
return true;
}
- PAIMON_ASSIGN_OR_RAISE(
- need_cast,
- NeedCastArrayForTimestamp(src_map_type->item_type(),
target_map_type->item_type()));
+ PAIMON_ASSIGN_OR_RAISE(need_cast,
+
NeedsArrayConversionImpl(src_map_type->item_field(),
+
target_map_type->item_field()));
return need_cast;
}
case arrow::Type::type::LIST: {
auto* src_list_type =
checked_cast<arrow::ListType*>(src_data_type.get());
auto* target_list_type =
checked_cast<arrow::ListType*>(target_data_type.get());
- PAIMON_ASSIGN_OR_RAISE(bool need_cast,
-
NeedCastArrayForTimestamp(src_list_type->value_type(),
-
target_list_type->value_type()));
- return need_cast;
+ return NeedsArrayConversionImpl(src_list_type->value_field(),
+ target_list_type->value_field());
}
case arrow::Type::type::TIMESTAMP: {
auto* src_ts_type =
checked_cast<arrow::TimestampType*>(src_data_type.get());
auto* target_ts_type =
checked_cast<arrow::TimestampType*>(target_data_type.get());
- if (src_ts_type->unit() != target_ts_type->unit() ||
- src_ts_type->timezone() != target_ts_type->timezone()) {
- return true;
- }
- return false;
+ return src_ts_type->unit() != target_ts_type->unit() ||
+ src_ts_type->timezone() != target_ts_type->timezone();
}
default:
return false;
}
}
-Result<std::shared_ptr<arrow::Array>>
ParquetTimestampConverter::CastArrayForTimestamp(
+Result<std::shared_ptr<arrow::Array>> ParquetReadTypeAdapter::AdaptArray(
+ const std::shared_ptr<arrow::Array>& array,
+ const std::shared_ptr<arrow::DataType>& target_data_type,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
+ PAIMON_ASSIGN_OR_RAISE(bool need_cast, NeedsArrayConversion(array->type(),
target_data_type));
+ if (!need_cast) {
+ return array;
+ }
+
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> result,
+ AdaptArrayImpl(array, target_data_type,
arrow_pool));
+ PAIMON_ASSIGN_OR_RAISE(need_cast, NeedsArrayConversion(result->type(),
target_data_type));
+ if (need_cast) {
+ return Status::Invalid(fmt::format(
+ "unexpected: in parquet, after AdaptArray, output type {} does not
match target "
+ "type {}",
+ result->type()->ToString(), target_data_type->ToString()));
+ }
+ return result;
+}
+
+Result<std::shared_ptr<arrow::Array>> ParquetReadTypeAdapter::AdaptArrayImpl(
const std::shared_ptr<arrow::Array>& array,
const std::shared_ptr<arrow::DataType>& target_data_type,
const std::shared_ptr<arrow::MemoryPool>& arrow_pool) {
@@ -171,7 +192,7 @@ Result<std::shared_ptr<arrow::Array>>
ParquetTimestampConverter::CastArrayForTim
const auto& field = struct_array->field(i);
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<arrow::Array> sub_array,
- CastArrayForTimestamp(field,
target_data_type->field(i)->type(), arrow_pool));
+ AdaptArrayImpl(field, target_data_type->field(i)->type(),
arrow_pool));
target_sub_arrays.push_back(sub_array);
target_names.push_back(target_data_type->field(i)->name());
}
@@ -187,10 +208,10 @@ Result<std::shared_ptr<arrow::Array>>
ParquetTimestampConverter::CastArrayForTim
auto* map_type =
checked_cast<arrow::MapType*>(target_data_type.get());
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<arrow::Array> key_array,
- CastArrayForTimestamp(map_array->keys(), map_type->key_type(),
arrow_pool));
+ AdaptArrayImpl(map_array->keys(), map_type->key_type(),
arrow_pool));
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<arrow::Array> item_array,
- CastArrayForTimestamp(map_array->items(),
map_type->item_type(), arrow_pool));
+ AdaptArrayImpl(map_array->items(), map_type->item_type(),
arrow_pool));
return std::make_shared<arrow::MapArray>(
arrow::map(key_array->type(), item_array->type()),
map_array->length(),
map_array->value_offsets(), key_array, item_array,
map_array->null_bitmap(),
@@ -201,7 +222,7 @@ Result<std::shared_ptr<arrow::Array>>
ParquetTimestampConverter::CastArrayForTim
auto* list_type =
checked_cast<arrow::ListType*>(target_data_type.get());
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<arrow::Array> value_array,
- CastArrayForTimestamp(list_array->values(),
list_type->value_type(), arrow_pool));
+ AdaptArrayImpl(list_array->values(), list_type->value_type(),
arrow_pool));
return std::make_shared<arrow::ListArray>(
arrow::list(value_array->type()), list_array->length(),
list_array->value_offsets(),
value_array, list_array->null_bitmap(),
list_array->null_count(),
@@ -213,8 +234,8 @@ Result<std::shared_ptr<arrow::Array>>
ParquetTimestampConverter::CastArrayForTim
auto* ts_target_type =
checked_cast<arrow::TimestampType*>(target_data_type.get());
if (src_type->unit() == arrow::TimeUnit::type::MILLI &&
ts_target_type->unit() == arrow::TimeUnit::type::SECOND) {
- // parquet writer do not support second, and it cast second to
milli.
- // Therefore, in paimon file reader, we cast from milli to
second.
+ // Parquet does not support second-precision timestamps, so
the writer stores
+ // them as milliseconds. Convert them back to seconds for the
Paimon read type.
auto cast_executor =
std::make_shared<TimestampToTimestampCastExecutor>();
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<arrow::Array> target_array,
@@ -222,10 +243,9 @@ Result<std::shared_ptr<arrow::Array>>
ParquetTimestampConverter::CastArrayForTim
return target_array;
}
if (src_type->timezone() != ts_target_type->timezone()) {
- // 1. For nano type, parquet writer will write nano into int96
type, which does
- // not contain any stats or zone info. Therefore in paimon
file reader, we add
- // zone info according to target type 2. For other precision,
parquet reader
- // will return UTC tz. Therefore, in paimon file reader, we
add local zone info.
+ // INT96 carries no timezone, while other Parquet timestamps
may be exposed as
+ // UTC. Attach the timezone required by the Paimon read type
without changing
+ // the underlying values.
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> target_array,
ts_array->View(target_data_type));
return target_array;
@@ -235,4 +255,5 @@ Result<std::shared_ptr<arrow::Array>>
ParquetTimestampConverter::CastArrayForTim
return array;
}
}
+
} // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_timestamp_converter.h
b/src/paimon/format/parquet/parquet_read_type_adapter.h
similarity index 59%
rename from src/paimon/format/parquet/parquet_timestamp_converter.h
rename to src/paimon/format/parquet/parquet_read_type_adapter.h
index 2954bfe4..2f296045 100644
--- a/src/paimon/format/parquet/parquet_timestamp_converter.h
+++ b/src/paimon/format/parquet/parquet_read_type_adapter.h
@@ -25,22 +25,34 @@
namespace paimon::parquet {
-class ParquetTimestampConverter {
+class ParquetReadTypeAdapter {
public:
- ParquetTimestampConverter() = delete;
- ~ParquetTimestampConverter() = delete;
+ ParquetReadTypeAdapter() = delete;
+ ~ParquetReadTypeAdapter() = delete;
- static Result<std::shared_ptr<arrow::DataType>> AdjustTimezone(
+ static Result<std::shared_ptr<arrow::DataType>> NormalizeFileType(
const std::shared_ptr<arrow::DataType>& src_data_type);
- static Result<bool> NeedCastArrayForTimestamp(
+ static Result<bool> NeedsArrayConversion(
const std::shared_ptr<arrow::DataType>& src_data_type,
const std::shared_ptr<arrow::DataType>& target_data_type);
- static Result<std::shared_ptr<arrow::Array>> CastArrayForTimestamp(
+ static Result<std::shared_ptr<arrow::Array>> AdaptArray(
const std::shared_ptr<arrow::Array>& array,
const std::shared_ptr<arrow::DataType>& target_data_type,
const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
+ private:
+ static Result<std::shared_ptr<arrow::Array>> AdaptArrayImpl(
+ const std::shared_ptr<arrow::Array>& array,
+ const std::shared_ptr<arrow::DataType>& target_data_type,
+ const std::shared_ptr<arrow::MemoryPool>& arrow_pool);
+
+ static Result<bool> NeedsArrayConversionImpl(const
std::shared_ptr<arrow::Field>& src_field,
+ const
std::shared_ptr<arrow::Field>& target_field);
+
+ static bool IsCompatibleBlob(const std::shared_ptr<arrow::Field>&
src_field,
+ const std::shared_ptr<arrow::Field>&
target_field);
};
} // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp
b/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp
similarity index 76%
rename from src/paimon/format/parquet/parquet_timestamp_converter_test.cpp
rename to src/paimon/format/parquet/parquet_read_type_adapter_test.cpp
index 784eecd0..17933cb8 100644
--- a/src/paimon/format/parquet/parquet_timestamp_converter_test.cpp
+++ b/src/paimon/format/parquet/parquet_read_type_adapter_test.cpp
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-#include "paimon/format/parquet/parquet_timestamp_converter.h"
+#include "paimon/format/parquet/parquet_read_type_adapter.h"
#include <memory>
@@ -31,7 +31,7 @@
namespace paimon::parquet::test {
-TEST(ParquetTimestampConverterTest, TestNeedCastArrayForTimestamp) {
+TEST(ParquetReadTypeAdapterTest, TestNeedsArrayConversion) {
{
// single field need cast
arrow::FieldVector fields = {
@@ -40,10 +40,10 @@ TEST(ParquetTimestampConverterTest,
TestNeedCastArrayForTimestamp) {
arrow::FieldVector target_fields = {
arrow::field("f0", arrow::timestamp(arrow::TimeUnit::NANO, "UTC")),
};
- ASSERT_OK_AND_ASSIGN(bool need_cast,
-
ParquetTimestampConverter::NeedCastArrayForTimestamp(
+ ASSERT_OK_AND_ASSIGN(bool needs_conversion,
+ ParquetReadTypeAdapter::NeedsArrayConversion(
arrow::struct_(fields),
arrow::struct_(target_fields)));
- ASSERT_TRUE(need_cast);
+ ASSERT_TRUE(needs_conversion);
}
{
// field in list need cast
@@ -52,10 +52,10 @@ TEST(ParquetTimestampConverterTest,
TestNeedCastArrayForTimestamp) {
};
arrow::FieldVector target_fields = {
arrow::field("f2",
arrow::list(arrow::timestamp(arrow::TimeUnit::SECOND)))};
- ASSERT_OK_AND_ASSIGN(bool need_cast,
-
ParquetTimestampConverter::NeedCastArrayForTimestamp(
+ ASSERT_OK_AND_ASSIGN(bool needs_conversion,
+ ParquetReadTypeAdapter::NeedsArrayConversion(
arrow::struct_(fields),
arrow::struct_(target_fields)));
- ASSERT_TRUE(need_cast);
+ ASSERT_TRUE(needs_conversion);
}
{
// field in map need cast
@@ -66,10 +66,10 @@ TEST(ParquetTimestampConverterTest,
TestNeedCastArrayForTimestamp) {
arrow::FieldVector target_fields = {
arrow::field("f1",
arrow::map(arrow::timestamp(arrow::TimeUnit::SECOND),
arrow::timestamp(arrow::TimeUnit::NANO, "UTC")))};
- ASSERT_OK_AND_ASSIGN(bool need_cast,
-
ParquetTimestampConverter::NeedCastArrayForTimestamp(
+ ASSERT_OK_AND_ASSIGN(bool needs_conversion,
+ ParquetReadTypeAdapter::NeedsArrayConversion(
arrow::struct_(fields),
arrow::struct_(target_fields)));
- ASSERT_TRUE(need_cast);
+ ASSERT_TRUE(needs_conversion);
}
{
// field in struct need cast
@@ -84,32 +84,32 @@ TEST(ParquetTimestampConverterTest,
TestNeedCastArrayForTimestamp) {
{arrow::field("f0",
arrow::timestamp(arrow::TimeUnit::MILLI)),
arrow::field("f1",
arrow::timestamp(arrow::TimeUnit::NANO, "UTC"))})),
};
- ASSERT_OK_AND_ASSIGN(bool need_cast,
-
ParquetTimestampConverter::NeedCastArrayForTimestamp(
+ ASSERT_OK_AND_ASSIGN(bool needs_conversion,
+ ParquetReadTypeAdapter::NeedsArrayConversion(
arrow::struct_(fields),
arrow::struct_(target_fields)));
- ASSERT_TRUE(need_cast);
+ ASSERT_TRUE(needs_conversion);
}
}
-TEST(ParquetTimestampConverterTest, TestAlreadyLogicalBlobCompatibility) {
+TEST(ParquetReadTypeAdapterTest, TestAlreadyLogicalBlobCompatibility) {
std::shared_ptr<arrow::Field> restored_blob = arrow::field("blob",
arrow::large_binary());
std::shared_ptr<arrow::Field> expected_physical_blob =
BlobUtils::ToArrowField("blob")->WithType(arrow::binary());
- ASSERT_OK_AND_ASSIGN(bool need_cast,
ParquetTimestampConverter::NeedCastArrayForTimestamp(
- arrow::struct_({restored_blob}),
-
arrow::struct_({expected_physical_blob})));
- ASSERT_FALSE(need_cast);
+ ASSERT_OK_AND_ASSIGN(bool needs_conversion,
ParquetReadTypeAdapter::NeedsArrayConversion(
+
arrow::struct_({restored_blob}),
+
arrow::struct_({expected_physical_blob})));
+ ASSERT_FALSE(needs_conversion);
// Only a binary field carrying Paimon BLOB metadata is compatible with a
large_binary field
// already restored from ARROW:schema. A plain binary field must still be
rejected.
- ASSERT_NOK_WITH_MSG(ParquetTimestampConverter::NeedCastArrayForTimestamp(
+ ASSERT_NOK_WITH_MSG(ParquetReadTypeAdapter::NeedsArrayConversion(
arrow::struct_({restored_blob}),
arrow::struct_({arrow::field("blob",
arrow::binary())})),
- "src type large_binary and target type binary
mismatch");
+ "source type large_binary and target type binary
mismatch");
}
-TEST(ParquetTimestampConverterTest, TestCastArrayForTimestamp) {
+TEST(ParquetReadTypeAdapterTest, TestAdaptArray) {
auto timezone = DateTimeUtils::GetLocalTimezoneName();
arrow::FieldVector fields = {
arrow::field("f1", arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI),
@@ -141,9 +141,9 @@ TEST(ParquetTimestampConverterTest,
TestCastArrayForTimestamp) {
.ValueOrDie());
std::shared_ptr<arrow::MemoryPool> pool = GetArrowPool(GetDefaultPool());
- ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> result_array,
- ParquetTimestampConverter::CastArrayForTimestamp(
- array, arrow::struct_(target_fields), pool));
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<arrow::Array> result_array,
+ ParquetReadTypeAdapter::AdaptArray(array,
arrow::struct_(target_fields), pool));
auto expected_array = std::dynamic_pointer_cast<arrow::StructArray>(
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(target_fields), R"([
@@ -155,7 +155,20 @@ TEST(ParquetTimestampConverterTest,
TestCastArrayForTimestamp) {
ASSERT_TRUE(result_array->Equals(expected_array)) <<
result_array->ToString();
}
-TEST(ParquetTimestampConverterTest, TestAdjustTimezone) {
+TEST(ParquetReadTypeAdapterTest, TestAdaptArrayValidatesResultType) {
+ std::shared_ptr<arrow::DataType> src_type =
arrow::timestamp(arrow::TimeUnit::MICRO);
+ std::shared_ptr<arrow::DataType> target_type =
arrow::timestamp(arrow::TimeUnit::MILLI);
+ std::shared_ptr<arrow::Array> array =
+ arrow::ipc::internal::json::ArrayFromJSON(src_type, R"(["1970-01-01
00:00:01"])")
+ .ValueOrDie();
+ std::shared_ptr<arrow::MemoryPool> pool = GetArrowPool(GetDefaultPool());
+
+ ASSERT_NOK_WITH_MSG(ParquetReadTypeAdapter::AdaptArray(array, target_type,
pool),
+ "after AdaptArray, output type timestamp[us] does not
match target type "
+ "timestamp[ms]");
+}
+
+TEST(ParquetReadTypeAdapterTest, TestNormalizeFileType) {
auto timezone = DateTimeUtils::GetLocalTimezoneName();
arrow::FieldVector fields = {
arrow::field("f1", arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI),
@@ -179,8 +192,8 @@ TEST(ParquetTimestampConverterTest, TestAdjustTimezone) {
arrow::field("f4", arrow::timestamp(arrow::TimeUnit::NANO)),
};
- ASSERT_OK_AND_ASSIGN(auto result_type,
-
ParquetTimestampConverter::AdjustTimezone(arrow::struct_(fields)));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> result_type,
+
ParquetReadTypeAdapter::NormalizeFileType(arrow::struct_(fields)));
ASSERT_TRUE(result_type->Equals(arrow::struct_(target_fields)));
}
} // namespace paimon::parquet::test
diff --git a/test/inte/paimon_read_compat_inte_test.cpp
b/test/inte/paimon_read_compat_inte_test.cpp
index 94d6c99f..a5595150 100644
--- a/test/inte/paimon_read_compat_inte_test.cpp
+++ b/test/inte/paimon_read_compat_inte_test.cpp
@@ -419,6 +419,29 @@ TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) {
ASSERT_EQ(inline_descriptor->Uri(),
"file:///nonexistent/pypaimon-all-types-external-blob.bin");
ASSERT_EQ(inline_descriptor->Offset(), 7);
ASSERT_EQ(inline_descriptor->Length(), 11);
+
+ // Python and Java store MAP<STRING, BLOB> values in standard separate
BLOB files. Validate
+ // resolved payloads, null values, and an empty map. Rust stores raw
values inline in Parquet,
+ // which is not a compatible Paimon BLOB representation and is asserted
separately below.
+ if (writer_prefix == "rust") {
+ return;
+ }
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<arrow::ChunkedArray> map_blob_result,
+ ReadTable(writer_prefix + "_map_blob_types", {"id", "f_map_blob"},
blob_value_options));
+ std::shared_ptr<arrow::StructArray> map_blob_rows =
GetOnlyStructChunk(map_blob_result);
+ ASSERT_TRUE(map_blob_rows);
+ ASSERT_EQ(map_blob_rows->length(), 3);
+ AssertFieldEqualsJson(map_blob_rows, "id", arrow::int32(), "[1, 2, 3]");
+ AssertFieldEqualsJson(map_blob_rows, "f_map_blob",
+ arrow::map(arrow::utf8(), arrow::large_binary()),
+ R"([[ ["left", "blob-map-left"], ["right", null] ],
null, []])");
+}
+
+TEST(PaimonReadCompatInteStandaloneTest, RejectsNonStandardRustMapBlob) {
+ ASSERT_NOK_WITH_MSG(
+ ReadTable("rust_map_blob_types", {"f_map_blob"}),
+ "Parquet does not support partial projection inside list/map: src
map<string, binary");
}
TEST_P(PaimonReadCompatInteTest, ReadsVectorValues) {
@@ -484,7 +507,8 @@ TEST_P(PaimonUnsupportedTypeInteTest, ReportsExpectedError)
{
std::vector<UnsupportedReadParam> UnsupportedReadParams() {
const std::vector<UnsupportedReadCase> read_cases = {
- {"NestedBlob", "nested_blob_types", "f_array_blob", "Blob field must
be a top-level field"},
+ {"ArrayBlob", "array_blob_types", "f_array_blob",
+ "BLOB field must be a top-level field or the direct value of a
top-level MAP field"},
{"TimePrecision0", "time_types", "f_time_0", "Unsupported type: TIME"},
{"TimePrecision3", "time_types", "f_time_3", "Unsupported type: TIME"},
{"TimePrecision6", "time_types", "f_time_6", "Unsupported type: TIME"},
diff --git a/test/test_data/parquet/append_types_compatibility.db/README.md
b/test/test_data/parquet/append_types_compatibility.db/README.md
index 5e4e330b..a1ac373a 100644
--- a/test/test_data/parquet/append_types_compatibility.db/README.md
+++ b/test/test_data/parquet/append_types_compatibility.db/README.md
@@ -19,7 +19,9 @@
# Paimon writer compatibility fixtures
-The `append_types_compatibility.db` database contains equivalent append-only
tables written by Python, Rust, and Java Paimon. Table names use the writer as
a prefix and describe the group of types in that table; no individual table is
claimed to contain every Paimon type.
+The `append_types_compatibility.db` database contains append-only
compatibility tables written by
+Python, Rust, and Java Paimon. Table names use the writer as a prefix and
describe the group of
+types in that table; no individual table is claimed to contain every Paimon
type.
The writers and Parquet implementations are:
@@ -29,11 +31,16 @@ The writers and Parquet implementations are:
## Table groups
-Each writer prefix has four tables with the same schema and logical values:
+Each writer prefix has five tables with the same declared schema and intended
logical values:
- `<writer>_types`: 43 columns and 3 rows covering integral and floating
types, BOOLEAN, CHAR/VARCHAR/STRING, BINARY/VARBINARY/BYTES, BLOB, DATE,
TIMESTAMP and TIMESTAMP_LTZ at precision 0/3/6/9, multiple DECIMAL precisions,
VARIANT, ARRAY, MAP, ROW, and deeply nested combinations. Row 2 is null in
every nullable column and row 3 exercises empty values.
- `<writer>_vector_types`: VECTOR length 3 for BOOLEAN, TINYINT, SMALLINT,
INT, BIGINT, FLOAT, and DOUBLE.
-- `<writer>_nested_blob_types`: ARRAY<BLOB> and MAP<STRING, BLOB>.
Paimon C++ rejects the table schema because ARRAY<BLOB> is unsupported.
Schema validation happens before column projection, so the compatibility test
explicitly checks this error instead of reading either field.
+- `<writer>_array_blob_types`: Paimon C++ does not currently support
ARRAY<BLOB>.
+- `<writer>_map_blob_types`: MAP<STRING, BLOB>, including non-null,
null, and empty map
+ values. Python and Java store these values in standard separate BLOB files,
which Paimon C++
+ can read. Rust stores the raw values inline as Parquet `binary`; this is not
the Paimon BLOB
+ descriptor representation, cannot be read as BLOB by Java, and is
intentionally rejected by
+ Paimon C++.
- `<writer>_time_types`: TIME declarations at precision 0/3/6/9. Paimon C++
currently rejects `TIME` while parsing the table schema.
The tables are separated because Paimon C++ does not allow VECTOR in a
data-evolution table, BLOB requires data evolution, and a schema-level
incompatibility must not prevent compatible columns from being tested.
@@ -42,4 +49,7 @@ The tables are separated because Paimon C++ does not allow
VECTOR in a data-evol
The `f_blob_descriptor` value intentionally points at a nonexistent external
URI. Read it with `blob-as-descriptor=true`; it exercises the inline descriptor
and Parquet `ARROW:schema` path rather than external blob fetching.
-Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`,
but the stored types differ in these fixtures. The Python fixture records the
physical Parquet `binary` column as Arrow `large_binary`, while the Rust
fixture records it as Arrow `binary`. The compatibility test verifies that
Paimon C++ can read both representations and preserve the descriptor value.
+Both PyArrow and parquet-rs persist the original Arrow type in `ARROW:schema`,
but the stored types
+differ for `f_blob_descriptor`. The Python fixture records the physical
Parquet `binary` column as
+Arrow `large_binary`, while the Rust fixture records it as Arrow `binary`. The
compatibility test
+verifies that Paimon C++ can read both representations and preserve the
descriptor value.
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-0.parquet
similarity index 100%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet
copy to
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-0.parquet
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-1.blob
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-1.blob
rename to
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/bucket-0/data-8a49858f-c0a7-468e-8d71-45b65142d422-1.blob
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0
similarity index 88%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0
copy to
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0
index f12e5f84..f6ca091d 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0
and
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-4db45a0c-f933-4043-80de-f1d5a5107966-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0
similarity index 95%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
copy to
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0
index 034d68e1..4b7ac1f0 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
and
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1
similarity index 86%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
copy to
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1
index 034d68e1..b7ae3ef2 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
and
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/manifest/manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
similarity index 65%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0
copy to
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
index 78945df1..0bce94c9 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/schema/schema-0
@@ -12,25 +12,17 @@
"type" : "ARRAY",
"element" : "BLOB"
}
- }, {
- "id" : 2,
- "name" : "f_map_blob",
- "type" : {
- "type" : "MAP",
- "key" : "STRING NOT NULL",
- "value" : "BLOB"
- }
} ],
- "highestFieldId" : 2,
+ "highestFieldId" : 1,
"partitionKeys" : [ ],
"primaryKeys" : [ ],
"options" : {
"bucket" : "-1",
- "blob-field" : "f_array_blob,f_map_blob",
+ "blob-field" : "f_array_blob",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
"manifest.format" : "avro",
"file.format" : "parquet"
},
- "timeMillis" : 1788853436235
+ "timeMillis" : 1788859707631
}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST
similarity index 100%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST
copy to
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/EARLIEST
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST
similarity index 100%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST
copy to
test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/LATEST
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..29a83ff0
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/java_array_blob_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "829d0558-0c81-4701-ba81-108a84c1de2e",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-0",
+ "baseManifestListSize" : 1006,
+ "deltaManifestList" : "manifest-list-38d776fc-d73a-4d8b-a166-9fc47b88c3bb-1",
+ "deltaManifestListSize" : 1115,
+ "commitUser" : "389ad12e-4421-4e4f-ad11-14284923cc57",
+ "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1788859708818,
+ "totalRecordCount" : 6,
+ "deltaRecordCount" : 6,
+ "nextRowId" : 3
+}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-0.parquet
rename to
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-0.parquet
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-1.blob
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/bucket-0/data-08f384c4-ca25-4b99-a024-07dc9526b5dc-2.blob
rename to
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/bucket-0/data-d2b0d467-f565-4453-b400-468f9ec22a54-1.blob
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0
similarity index 88%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0
rename to
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0
index f12e5f84..c803b1c9 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-7de5f36f-9ba7-4853-b344-cedc5ba17995-0
and
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-62c45eba-0a54-4a63-b89d-0ab09a768412-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0
similarity index 86%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1
rename to
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0
index 07144721..655f831b 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1
and
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1
similarity index 86%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
rename to
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1
index 034d68e1..9040aaea 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/manifest/manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0
and
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/manifest/manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
similarity index 68%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0
rename to
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
index 78945df1..340c0ade 100644
---
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/schema/schema-0
@@ -7,13 +7,6 @@
"type" : "INT NOT NULL"
}, {
"id" : 1,
- "name" : "f_array_blob",
- "type" : {
- "type" : "ARRAY",
- "element" : "BLOB"
- }
- }, {
- "id" : 2,
"name" : "f_map_blob",
"type" : {
"type" : "MAP",
@@ -21,16 +14,16 @@
"value" : "BLOB"
}
} ],
- "highestFieldId" : 2,
+ "highestFieldId" : 1,
"partitionKeys" : [ ],
"primaryKeys" : [ ],
"options" : {
"bucket" : "-1",
- "blob-field" : "f_array_blob,f_map_blob",
+ "blob-field" : "f_map_blob",
"row-tracking.enabled" : "true",
"data-evolution.enabled" : "true",
"manifest.format" : "avro",
"file.format" : "parquet"
},
- "timeMillis" : 1788853436235
+ "timeMillis" : 1788859708882
}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/EARLIEST
rename to
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/EARLIEST
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/LATEST
rename to
test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/LATEST
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..edcebd32
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/java_map_blob_types/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "10cfbd5d-65f3-4ef3-b8b0-78c34e831369",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-0",
+ "baseManifestListSize" : 1006,
+ "deltaManifestList" : "manifest-list-d3262408-2da8-408b-9c67-b487ecf8565b-1",
+ "deltaManifestListSize" : 1115,
+ "commitUser" : "df37a7f9-0e78-4d8c-aa5c-13c284be6a37",
+ "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1788859708907,
+ "totalRecordCount" : 6,
+ "deltaRecordCount" : 6,
+ "nextRowId" : 3
+}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1
deleted file mode 100644
index b98c7d14..00000000
---
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/snapshot-1
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "version" : 3,
- "uuid" : "89919eb0-8f08-4098-9002-d19680ca3a6d",
- "id" : 1,
- "schemaId" : 0,
- "baseManifestList" : "manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-0",
- "baseManifestListSize" : 1006,
- "deltaManifestList" : "manifest-list-2b993fd4-7b01-4cc4-925f-aa69eccf1a2e-1",
- "deltaManifestListSize" : 1111,
- "commitUser" : "1192f15e-31f1-42a4-b498-248a36a42985",
- "writerVersion" : "java-2-ali-2.6-94eb84aee341e9ad221c027ed97c8766583a7abe",
- "commitIdentifier" : 9223372036854775807,
- "commitKind" : "APPEND",
- "timeMillis" : 1788853436256,
- "totalRecordCount" : 9,
- "deltaRecordCount" : 9,
- "nextRowId" : 3
-}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/bucket-0/data-043fd246-38df-40f7-9147-c1c416282ecc-0.blob
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-7b1104e4-657e-40ea-8ac5-ecac0ecbc3bf-0.blob
rename to
test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/bucket-0/data-043fd246-38df-40f7-9147-c1c416282ecc-0.blob
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/bucket-0/data-3cb0cb1a-5db9-478e-a727-6f34d6e800e5-0.parquet
similarity index 100%
copy from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet
copy to
test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/bucket-0/data-3cb0cb1a-5db9-478e-a727-6f34d6e800e5-0.parquet
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0
similarity index 87%
copy from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0
copy to
test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0
index eb9d186f..3c001fe9 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0
and
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-8e8abfa0-a7fe-4669-a7be-cd095dee5fac-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0
similarity index 96%
copy from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
copy to
test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0
index 8608500a..105bce89 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
and
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1
similarity index 85%
rename from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1
rename to
test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1
index c85847cb..cf8d9f9c 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1
and
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/manifest/manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0
similarity index 56%
copy from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0
copy to
test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0
index e311af94..9eff531e 100644
---
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/schema/schema-0
@@ -15,19 +15,9 @@
"element": "BLOB",
"nullable": true
}
- },
- {
- "id": 2,
- "name": "f_map_blob",
- "type": {
- "type": "MAP<STRING NOT NULL, BLOB>",
- "key": "STRING NOT NULL",
- "value": "BLOB",
- "nullable": true
- }
}
],
- "highestFieldId": 2,
+ "highestFieldId": 1,
"partitionKeys": [],
"primaryKeys": [],
"options": {
@@ -36,8 +26,8 @@
"manifest.format": "avro",
"row-tracking.enabled": "true",
"data-evolution.enabled": "true",
- "blob-field": "f_array_blob,f_map_blob"
+ "blob-field": "f_array_blob"
},
- "comment": "Generated by pypaimon 2.0.0 with ARRAY<BLOB> and MAP<STRING,
BLOB>.",
- "timeMillis": 1788513128455
+ "comment": "Generated by pypaimon 2.0.0 with ARRAY<BLOB>.",
+ "timeMillis": 1788859561835
}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/LATEST
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/LATEST
rename to
test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/LATEST
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..ac1f6f1a
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/python_array_blob_types/snapshot/snapshot-1
@@ -0,0 +1,15 @@
+{
+ "version": 3,
+ "id": 1,
+ "schemaId": 0,
+ "baseManifestList": "manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-0",
+ "deltaManifestList": "manifest-list-2f17c472-fea6-42d0-8300-12da4874d53a-1",
+ "totalRecordCount": 6,
+ "deltaRecordCount": 6,
+ "commitUser": "657b7800-ea12-4a26-8075-4b2fd4e20d51",
+ "commitIdentifier": 9223372036854775807,
+ "commitKind": "APPEND",
+ "timeMillis": 1788859561849,
+ "nextRowId": 3,
+ "uuid": "43460238-f728-4ee1-8600-1f3d0c13234a"
+}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-2f2a8c74-bd24-4106-9ed5-91d20d711270-0.parquet
rename to
test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-4575b349-773b-4e2f-b364-ad98ee38fa72-0.parquet
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-6e33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/bucket-0/data-3850273b-000b-4248-add7-b30a6c7bb5e1-0.blob
rename to
test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/bucket-0/data-6e33a029-ca69-4453-a05b-2f7ed1a93f63-0.blob
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0
similarity index 87%
rename from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0
rename to
test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0
index eb9d186f..f238c7da 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-0ef6e952-7e56-4135-9ab0-d6e79820dfff-0
and
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-59ba3981-44aa-4af6-960e-9048ce6e883b-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0
similarity index 96%
copy from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
copy to
test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0
index 8608500a..97bbfb6e 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
and
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1
similarity index 85%
rename from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
rename to
test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1
index 8608500a..ad23ae47 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/manifest/manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0
and
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/manifest/manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0
similarity index 61%
rename from
test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0
rename to
test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0
index e311af94..08ed496f 100644
---
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/schema/schema-0
+++
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/schema/schema-0
@@ -9,15 +9,6 @@
},
{
"id": 1,
- "name": "f_array_blob",
- "type": {
- "type": "ARRAY",
- "element": "BLOB",
- "nullable": true
- }
- },
- {
- "id": 2,
"name": "f_map_blob",
"type": {
"type": "MAP<STRING NOT NULL, BLOB>",
@@ -27,7 +18,7 @@
}
}
],
- "highestFieldId": 2,
+ "highestFieldId": 1,
"partitionKeys": [],
"primaryKeys": [],
"options": {
@@ -36,8 +27,8 @@
"manifest.format": "avro",
"row-tracking.enabled": "true",
"data-evolution.enabled": "true",
- "blob-field": "f_array_blob,f_map_blob"
+ "blob-field": "f_map_blob"
},
- "comment": "Generated by pypaimon 2.0.0 with ARRAY<BLOB> and MAP<STRING,
BLOB>.",
- "timeMillis": 1788513128455
+ "comment": "Generated by pypaimon 2.0.0 with MAP<STRING, BLOB>.",
+ "timeMillis": 1788859561850
}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST
similarity index 100%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST
copy to
test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/LATEST
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..abae40fe
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/python_map_blob_types/snapshot/snapshot-1
@@ -0,0 +1,15 @@
+{
+ "version": 3,
+ "id": 1,
+ "schemaId": 0,
+ "baseManifestList": "manifest-list-c762398d-e22c-4676-b43d-4564909a0859-0",
+ "deltaManifestList": "manifest-list-c762398d-e22c-4676-b43d-4564909a0859-1",
+ "totalRecordCount": 6,
+ "deltaRecordCount": 6,
+ "commitUser": "cadd033b-11dd-40e2-a759-0306ab9df091",
+ "commitIdentifier": 9223372036854775807,
+ "commitKind": "APPEND",
+ "timeMillis": 1788859561854,
+ "nextRowId": 3,
+ "uuid": "0279865a-f61e-41bf-8218-69adf804978f"
+}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1
deleted file mode 100644
index 8e955da7..00000000
---
a/test/test_data/parquet/append_types_compatibility.db/python_nested_blob_types/snapshot/snapshot-1
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "version": 3,
- "id": 1,
- "schemaId": 0,
- "baseManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-0",
- "deltaManifestList": "manifest-list-19134fd4-d60f-460e-b880-2633ef1b5a8c-1",
- "totalRecordCount": 9,
- "deltaRecordCount": 9,
- "commitUser": "297eeaeb-5c62-4b3c-8f9e-950b09612bc3",
- "commitIdentifier": 9223372036854775807,
- "commitKind": "APPEND",
- "timeMillis": 1788513128459,
- "nextRowId": 3,
- "uuid": "d2837b32-27c3-453e-8470-ccb9d176d3f0"
-}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet
new file mode 100644
index 00000000..f31d5ecc
Binary files /dev/null and
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/bucket-0/data-334e3ff1-32c3-4f59-98c1-7e366b0f4b0b-0.parquet
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0
new file mode 100644
index 00000000..0d557ef9
Binary files /dev/null and
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-37d9c141-c626-434c-bdc0-506cc6f3d04e-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0
new file mode 100644
index 00000000..760835ce
Binary files /dev/null and
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1
similarity index 89%
rename from
test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0
rename to
test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1
index a01a7456..c025a6df 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0
and
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/manifest/manifest-list-edec5851-5eca-466b-9bde-780df761f932-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0
new file mode 100644
index 00000000..c090dea0
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/schema/schema-0
@@ -0,0 +1 @@
+{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_array_blob","type":{"type":"ARRAY","element":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"file.format":"parquet","manifest.format":"avro","data-evolution.enabled":"true","row-tracking.enabled":"true","bucket":"-1","blob-field":"f_array_blob"},"comment":null,"timeMillis":1788860192967}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST
similarity index 100%
copy from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST
copy to
test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/LATEST
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..51c4b5d9
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_array_blob_types/snapshot/snapshot-1
@@ -0,0 +1 @@
+{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-edec5851-5eca-466b-9bde-780df761f932-0","deltaManifestList":"manifest-list-edec5851-5eca-466b-9bde-780df761f932-1","commitUser":"a8381718-0f79-46d1-92ec-2879a4cca6f8","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788860192978,"totalRecordCount":3,"deltaRecordCount":3,"nextRowId":3}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet
new file mode 100644
index 00000000..08aadfe5
Binary files /dev/null and
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/bucket-0/data-27af7a11-39ca-4131-b12d-81f824f57978-0.parquet
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0
new file mode 100644
index 00000000..72f09905
Binary files /dev/null and
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-30836888-cc9d-4ad0-b200-6da6e101ebd4-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0
new file mode 100644
index 00000000..68de8b16
Binary files /dev/null and
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1
similarity index 89%
rename from
test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1
rename to
test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1
index 5478f74d..7050b7cf 100644
Binary files
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1
and
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/manifest/manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1
differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0
new file mode 100644
index 00000000..b6cd8894
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/schema/schema-0
@@ -0,0 +1 @@
+{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_map_blob","type":{"type":"MAP","key":"STRING NOT
NULL","value":"BLOB"}}],"highestFieldId":1,"partitionKeys":[],"primaryKeys":[],"options":{"blob-field":"f_map_blob","row-tracking.enabled":"true","file.format":"parquet","manifest.format":"avro","data-evolution.enabled":"true","bucket":"-1"},"comment":null,"timeMillis":1788860192979}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/LATEST
similarity index 100%
rename from
test/test_data/parquet/append_types_compatibility.db/java_nested_blob_types/snapshot/LATEST
rename to
test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/LATEST
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/snapshot-1
new file mode 100644
index 00000000..d2926bd6
--- /dev/null
+++
b/test/test_data/parquet/append_types_compatibility.db/rust_map_blob_types/snapshot/snapshot-1
@@ -0,0 +1 @@
+{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-0","deltaManifestList":"manifest-list-ea2b8a07-d439-44e6-86f2-451aa2dc1db6-1","commitUser":"cb443c72-2132-4223-9af0-f39360413a6a","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788860192988,"totalRecordCount":3,"deltaRecordCount":3,"nextRowId":3}
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet
b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet
deleted file mode 100644
index 0383ee56..00000000
Binary files
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/bucket-0/data-4749ab27-cf5e-48e5-8449-3f674480e077-0.parquet
and /dev/null differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0
b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0
deleted file mode 100644
index 286b07f3..00000000
Binary files
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/manifest/manifest-0d1ac1c3-300f-45fe-ad80-45445dec4b45-0
and /dev/null differ
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0
b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0
deleted file mode 100644
index 66ceca0a..00000000
---
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/schema/schema-0
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"id":0,"fields":[{"id":0,"name":"id","type":"INT NOT
NULL"},{"id":1,"name":"f_array_blob","type":{"type":"ARRAY","element":"BLOB"}},{"id":2,"name":"f_map_blob","type":{"type":"MAP","key":"STRING
NOT
NULL","value":"BLOB"}}],"highestFieldId":2,"partitionKeys":[],"primaryKeys":[],"options":{"manifest.format":"avro","file.format":"parquet","data-evolution.enabled":"true","bucket":"-1","row-tracking.enabled":"true","blob-field":"f_array_blob,f_map_blob"},"comment":null,"timeMilli
[...]
\ No newline at end of file
diff --git
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1
b/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1
deleted file mode 100644
index c24110af..00000000
---
a/test/test_data/parquet/append_types_compatibility.db/rust_nested_blob_types/snapshot/snapshot-1
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"id":1,"schemaId":0,"baseManifestList":"manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-0","deltaManifestList":"manifest-list-6e560f60-1b27-45ff-b676-0bb081ff3261-1","commitUser":"ce789997-7558-44de-8068-c5d7e744dba9","commitIdentifier":9223372036854775807,"commitKind":"APPEND","timeMillis":1788858155032,"totalRecordCount":3,"deltaRecordCount":3,"nextRowId":3}
\ No newline at end of file
