ZENOTME commented on code in PR #76:
URL: https://github.com/apache/iceberg-rust/pull/76#discussion_r1356073854
##########
crates/iceberg/src/spec/manifest_list.rs:
##########
@@ -90,24 +113,72 @@ impl ManifestList {
];
Schema::builder().with_fields(fields).build().unwrap()
}
+}
- /// Get the v1 schema of the manifest list entry.
- pub(crate) fn v1_schema() -> Schema {
- let fields = vec![
- _const_fields::MANIFEST_PATH.clone(),
- _const_fields::MANIFEST_LENGTH.clone(),
- _const_fields::PARTITION_SPEC_ID.clone(),
- _const_fields::ADDED_SNAPSHOT_ID.clone(),
- _const_fields::ADDED_FILES_COUNT_V1.clone().to_owned(),
- _const_fields::EXISTING_FILES_COUNT_V1.clone(),
- _const_fields::DELETED_FILES_COUNT_V1.clone(),
- _const_fields::ADDED_ROWS_COUNT_V1.clone(),
- _const_fields::EXISTING_ROWS_COUNT_V1.clone(),
- _const_fields::DELETED_ROWS_COUNT_V1.clone(),
- _const_fields::PARTITIONS.clone(),
- _const_fields::KEY_METADATA.clone(),
- ];
- Schema::builder().with_fields(fields).build().unwrap()
+pub(crate) static MANIFEST_LIST_AVRO_SCHEMA_V1: Lazy<AvroSchema> =
+ Lazy::new(|| schema_to_avro_schema("manifest_list",
&ManifestList::v1_schema()).unwrap());
+
+pub(crate) static MANIFEST_LIST_AVRO_SCHEMA_V2: Lazy<AvroSchema> =
+ Lazy::new(|| schema_to_avro_schema("manfiest_list",
&ManifestList::v2_schema()).unwrap());
+
+/// A manifest list writer.
+pub struct ManifestListWriter {
+ output_file: OutputFile,
+ format_version: FormatVersion,
+ avro_writer: Writer<'static, Vec<u8>>,
+}
+
+impl ManifestListWriter {
+ /// Construct a new [`ManifestListWriter`] that writes to a provided
[`OutputFile`].
+ pub fn new(
+ output_file: OutputFile,
+ format_version: FormatVersion,
+ metadata: HashMap<String, String>,
+ ) -> Self {
+ let avro_schema = match format_version {
+ FormatVersion::V1 => &MANIFEST_LIST_AVRO_SCHEMA_V1,
+ FormatVersion::V2 => &MANIFEST_LIST_AVRO_SCHEMA_V2,
+ };
+ let mut avro_writer = Writer::new(avro_schema, Vec::new());
+ for (key, value) in metadata {
+ avro_writer.add_user_metadata(key, value).unwrap();
+ }
+ Self {
+ output_file,
+ format_version,
+ avro_writer,
+ }
+ }
+
+ /// Append manifests to be written.
+ pub fn add_manifests(
Review Comment:
nit: How about add_manifest_lists?🤔 It can be distinguished with Manifest.
Also same for following var name. (`manifests`,`manifest`)
##########
crates/iceberg/src/spec/manifest_list.rs:
##########
@@ -90,24 +113,72 @@ impl ManifestList {
];
Schema::builder().with_fields(fields).build().unwrap()
}
+}
- /// Get the v1 schema of the manifest list entry.
- pub(crate) fn v1_schema() -> Schema {
- let fields = vec![
- _const_fields::MANIFEST_PATH.clone(),
- _const_fields::MANIFEST_LENGTH.clone(),
- _const_fields::PARTITION_SPEC_ID.clone(),
- _const_fields::ADDED_SNAPSHOT_ID.clone(),
- _const_fields::ADDED_FILES_COUNT_V1.clone().to_owned(),
- _const_fields::EXISTING_FILES_COUNT_V1.clone(),
- _const_fields::DELETED_FILES_COUNT_V1.clone(),
- _const_fields::ADDED_ROWS_COUNT_V1.clone(),
- _const_fields::EXISTING_ROWS_COUNT_V1.clone(),
- _const_fields::DELETED_ROWS_COUNT_V1.clone(),
- _const_fields::PARTITIONS.clone(),
- _const_fields::KEY_METADATA.clone(),
- ];
- Schema::builder().with_fields(fields).build().unwrap()
+pub(crate) static MANIFEST_LIST_AVRO_SCHEMA_V1: Lazy<AvroSchema> =
+ Lazy::new(|| schema_to_avro_schema("manifest_list",
&ManifestList::v1_schema()).unwrap());
+
+pub(crate) static MANIFEST_LIST_AVRO_SCHEMA_V2: Lazy<AvroSchema> =
+ Lazy::new(|| schema_to_avro_schema("manfiest_list",
&ManifestList::v2_schema()).unwrap());
+
+/// A manifest list writer.
+pub struct ManifestListWriter {
+ output_file: OutputFile,
+ format_version: FormatVersion,
+ avro_writer: Writer<'static, Vec<u8>>,
+}
+
+impl ManifestListWriter {
+ /// Construct a new [`ManifestListWriter`] that writes to a provided
[`OutputFile`].
+ pub fn new(
+ output_file: OutputFile,
+ format_version: FormatVersion,
+ metadata: HashMap<String, String>,
+ ) -> Self {
+ let avro_schema = match format_version {
+ FormatVersion::V1 => &MANIFEST_LIST_AVRO_SCHEMA_V1,
+ FormatVersion::V2 => &MANIFEST_LIST_AVRO_SCHEMA_V2,
+ };
+ let mut avro_writer = Writer::new(avro_schema, Vec::new());
+ for (key, value) in metadata {
+ avro_writer.add_user_metadata(key, value).unwrap();
+ }
+ Self {
+ output_file,
+ format_version,
+ avro_writer,
+ }
+ }
+
+ /// Append manifests to be written.
+ pub fn add_manifests(
Review Comment:
nit: How about add_manifest_lists?🤔 It can be distinguished with Manifest.
Also same for following var name. (`manifests`,`manifest`)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
