rdblue commented on code in PR #40:
URL: https://github.com/apache/iceberg-python/pull/40#discussion_r1349764876
##########
pyiceberg/manifest.py:
##########
@@ -99,101 +100,185 @@ def __repr__(self) -> str:
return f"FileFormat.{self.name}"
-DATA_FILE_TYPE_V1 = StructType(
- NestedField(
- field_id=134,
- name="content",
- field_type=IntegerType(),
- required=False,
- doc="Contents of the file: 0=data, 1=position deletes, 2=equality
deletes",
- initial_default=DataFileContent.DATA,
+DATA_FILE_TYPE: Dict[int, StructType] = {
+ 1: StructType(
+ NestedField(field_id=100, name="file_path", field_type=StringType(),
required=True, doc="Location URI with FS scheme"),
+ NestedField(
+ field_id=101,
+ name="file_format",
+ field_type=StringType(),
+ required=True,
+ doc="File format name: avro, orc, or parquet",
+ ),
+ NestedField(
+ field_id=102,
+ name="partition",
+ field_type=StructType(),
+ required=True,
+ doc="Partition data tuple, schema based on the partition spec",
+ ),
+ NestedField(field_id=103, name="record_count", field_type=LongType(),
required=True, doc="Number of records in the file"),
+ NestedField(
+ field_id=104, name="file_size_in_bytes", field_type=LongType(),
required=True, doc="Total file size in bytes"
+ ),
+ NestedField(
+ field_id=105,
+ name="block_size_in_bytes",
+ field_type=LongType(),
+ required=True,
+ doc="Deprecated. Always write a default in v1. Do not write in
v2.",
+ write_default=DEFAULT_BLOCK_SIZE,
+ ),
+ NestedField(
+ field_id=108,
+ name="column_sizes",
+ field_type=MapType(key_id=117, key_type=IntegerType(),
value_id=118, value_type=LongType()),
+ required=False,
+ doc="Map of column id to total size on disk",
+ ),
+ NestedField(
+ field_id=109,
+ name="value_counts",
+ field_type=MapType(key_id=119, key_type=IntegerType(),
value_id=120, value_type=LongType()),
+ required=False,
+ doc="Map of column id to total count, including null and NaN",
+ ),
+ NestedField(
+ field_id=110,
+ name="null_value_counts",
+ field_type=MapType(key_id=121, key_type=IntegerType(),
value_id=122, value_type=LongType()),
+ required=False,
+ doc="Map of column id to null value count",
+ ),
+ NestedField(
+ field_id=137,
+ name="nan_value_counts",
+ field_type=MapType(key_id=138, key_type=IntegerType(),
value_id=139, value_type=LongType()),
+ required=False,
+ doc="Map of column id to number of NaN values in the column",
+ ),
+ NestedField(
+ field_id=125,
+ name="lower_bounds",
+ field_type=MapType(key_id=126, key_type=IntegerType(),
value_id=127, value_type=BinaryType()),
+ required=False,
+ doc="Map of column id to lower bound",
+ ),
+ NestedField(
+ field_id=128,
+ name="upper_bounds",
+ field_type=MapType(key_id=129, key_type=IntegerType(),
value_id=130, value_type=BinaryType()),
+ required=False,
+ doc="Map of column id to upper bound",
+ ),
+ NestedField(
+ field_id=131, name="key_metadata", field_type=BinaryType(),
required=False, doc="Encryption key metadata blob"
+ ),
+ NestedField(
+ field_id=132,
+ name="split_offsets",
+ field_type=ListType(element_id=133, element_type=LongType(),
element_required=True),
+ required=False,
+ doc="Splittable offsets",
+ ),
+ NestedField(field_id=140, name="sort_order_id",
field_type=IntegerType(), required=False, doc="Sort order ID"),
),
- NestedField(field_id=100, name="file_path", field_type=StringType(),
required=True, doc="Location URI with FS scheme"),
- NestedField(
- field_id=101,
- name="file_format",
- field_type=StringType(),
- required=True,
- doc="File format name: avro, orc, or parquet",
+ 2: StructType(
+ NestedField(
+ field_id=134,
+ name="content",
+ field_type=IntegerType(),
+ required=True,
+ doc="File format name: avro, orc, or parquet",
+ initial_default=DataFileContent.DATA,
+ ),
+ NestedField(field_id=100, name="file_path", field_type=StringType(),
required=True, doc="Location URI with FS scheme"),
+ NestedField(
+ field_id=101,
+ name="file_format",
+ field_type=StringType(),
+ required=True,
+ doc="File format name: avro, orc, or parquet",
+ ),
+ NestedField(
+ field_id=102,
+ name="partition",
+ field_type=StructType(),
+ required=True,
+ doc="Partition data tuple, schema based on the partition spec",
+ ),
+ NestedField(field_id=103, name="record_count", field_type=LongType(),
required=True, doc="Number of records in the file"),
+ NestedField(
+ field_id=104, name="file_size_in_bytes", field_type=LongType(),
required=True, doc="Total file size in bytes"
+ ),
+ NestedField(
+ field_id=108,
+ name="column_sizes",
+ field_type=MapType(key_id=117, key_type=IntegerType(),
value_id=118, value_type=LongType()),
+ required=False,
+ doc="Map of column id to total size on disk",
+ ),
+ NestedField(
+ field_id=109,
+ name="value_counts",
+ field_type=MapType(key_id=119, key_type=IntegerType(),
value_id=120, value_type=LongType()),
+ required=False,
+ doc="Map of column id to total count, including null and NaN",
+ ),
+ NestedField(
+ field_id=110,
+ name="null_value_counts",
+ field_type=MapType(key_id=121, key_type=IntegerType(),
value_id=122, value_type=LongType()),
+ required=False,
+ doc="Map of column id to null value count",
+ ),
+ NestedField(
+ field_id=137,
+ name="nan_value_counts",
+ field_type=MapType(key_id=138, key_type=IntegerType(),
value_id=139, value_type=LongType()),
+ required=False,
+ doc="Map of column id to number of NaN values in the column",
+ ),
+ NestedField(
+ field_id=125,
+ name="lower_bounds",
+ field_type=MapType(key_id=126, key_type=IntegerType(),
value_id=127, value_type=BinaryType()),
+ required=False,
+ doc="Map of column id to lower bound",
+ ),
+ NestedField(
+ field_id=128,
+ name="upper_bounds",
+ field_type=MapType(key_id=129, key_type=IntegerType(),
value_id=130, value_type=BinaryType()),
+ required=False,
+ doc="Map of column id to upper bound",
+ ),
+ NestedField(
+ field_id=131, name="key_metadata", field_type=BinaryType(),
required=False, doc="Encryption key metadata blob"
+ ),
+ NestedField(
+ field_id=132,
+ name="split_offsets",
+ field_type=ListType(element_id=133, element_type=LongType(),
element_required=True),
+ required=False,
+ doc="Splittable offsets",
+ ),
+ NestedField(
+ field_id=135,
+ name="equality_ids",
+ field_type=ListType(element_id=136, element_type=LongType(),
element_required=True),
+ required=False,
+ doc="Field ids used to determine row equality in equality delete
files.",
+ ),
+ NestedField(
+ field_id=140,
+ name="sort_order_id",
+ field_type=IntegerType(),
+ required=False,
+ doc=" ID representing sort order for this file",
Review Comment:
Nit: unnecessary whitespace in the doc.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
