HonahX commented on code in PR #8622:
URL: https://github.com/apache/iceberg/pull/8622#discussion_r1338199541
##########
python/pyiceberg/manifest.py:
##########
@@ -79,88 +95,107 @@ def __repr__(self) -> str:
return f"FileFormat.{self.name}"
-DATA_FILE_TYPE = StructType(
- NestedField(
- field_id=134,
- name="content",
- field_type=IntegerType(),
- required=False,
- doc="Contents of the file: 0=data, 1=position deletes, 2=equality
deletes",
- initial_default=DataFileContent.DATA,
- ),
- NestedField(field_id=100, name="file_path", field_type=StringType(),
required=True, doc="Location URI with FS scheme"),
- NestedField(
- field_id=101, name="file_format", field_type=StringType(),
required=True, doc="File format name: avro, orc, or parquet"
- ),
- NestedField(
- field_id=102,
- name="partition",
- field_type=StructType(),
- required=True,
- doc="Partition data tuple, schema based on the partition spec",
- ),
- NestedField(field_id=103, name="record_count", field_type=LongType(),
required=True, doc="Number of records in the file"),
- NestedField(field_id=104, name="file_size_in_bytes",
field_type=LongType(), required=True, doc="Total file size in bytes"),
- NestedField(
- field_id=108,
- name="column_sizes",
- field_type=MapType(key_id=117, key_type=IntegerType(), value_id=118,
value_type=LongType()),
- required=False,
- doc="Map of column id to total size on disk",
- ),
- NestedField(
- field_id=109,
- name="value_counts",
- field_type=MapType(key_id=119, key_type=IntegerType(), value_id=120,
value_type=LongType()),
- required=False,
- doc="Map of column id to total count, including null and NaN",
- ),
- NestedField(
- field_id=110,
- name="null_value_counts",
- field_type=MapType(key_id=121, key_type=IntegerType(), value_id=122,
value_type=LongType()),
- required=False,
- doc="Map of column id to null value count",
- ),
- NestedField(
- field_id=137,
- name="nan_value_counts",
- field_type=MapType(key_id=138, key_type=IntegerType(), value_id=139,
value_type=LongType()),
- required=False,
- doc="Map of column id to number of NaN values in the column",
- ),
- NestedField(
- field_id=125,
- name="lower_bounds",
- field_type=MapType(key_id=126, key_type=IntegerType(), value_id=127,
value_type=BinaryType()),
- required=False,
- doc="Map of column id to lower bound",
- ),
- NestedField(
- field_id=128,
- name="upper_bounds",
- field_type=MapType(key_id=129, key_type=IntegerType(), value_id=130,
value_type=BinaryType()),
- required=False,
- doc="Map of column id to upper bound",
- ),
- NestedField(field_id=131, name="key_metadata", field_type=BinaryType(),
required=False, doc="Encryption key metadata blob"),
- NestedField(
- field_id=132,
- name="split_offsets",
- field_type=ListType(element_id=133, element_type=LongType(),
element_required=True),
- required=False,
- doc="Splittable offsets",
- ),
- NestedField(
- field_id=135,
- name="equality_ids",
- field_type=ListType(element_id=136, element_type=LongType(),
element_required=True),
- required=False,
- doc="Equality comparison field IDs",
- ),
- NestedField(field_id=140, name="sort_order_id", field_type=IntegerType(),
required=False, doc="Sort order ID"),
- NestedField(field_id=141, name="spec_id", field_type=IntegerType(),
required=False, doc="Partition spec ID"),
-)
+def data_file_type(partition_type: StructType) -> StructType:
+ return StructType(
+ NestedField(
+ field_id=134,
+ name="content",
+ field_type=IntegerType(),
+ required=False,
+ doc="Contents of the file: 0=data, 1=position deletes, 2=equality
deletes",
+ initial_default=DataFileContent.DATA,
+ ),
+ NestedField(field_id=100, name="file_path", field_type=StringType(),
required=True, doc="Location URI with FS scheme"),
+ NestedField(
+ field_id=101,
+ name="file_format",
+ field_type=StringType(),
+ required=True,
+ doc="File format name: avro, orc, or parquet",
+ ),
+ NestedField(
+ field_id=102,
+ name="partition",
+ field_type=partition_type,
+ required=True,
+ doc="Partition data tuple, schema based on the partition spec",
+ ),
+ NestedField(field_id=103, name="record_count", field_type=LongType(),
required=True, doc="Number of records in the file"),
+ NestedField(
+ field_id=104, name="file_size_in_bytes", field_type=LongType(),
required=True, doc="Total file size in bytes"
+ ),
+ NestedField(
+ field_id=105,
+ name="block_size_in_bytes",
Review Comment:
Thanks for the suggestion. Based on my understanding not only do we need two
version of `data_file_type`, but we may also need two "versions" of
`DataFile(Record)` such that the field `block_size_in_bytes` is skipped when
written to the v2 manifest file. Specifically, under current Avro write
framework, when handling v2 case, we need to ensure that `block_size_in_bytes`
does not exist in `self._position_to_field_name` of the Record class. I've come
up with a possible solution which I will explain below.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
