ZENOTME commented on code in PR #135: URL: https://github.com/apache/iceberg-rust/pull/135#discussion_r1445893518
########## crates/iceberg/src/writer/mod.rs: ########## @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Iceberg writer module. +//! +//! The writer API is designed to be extensible and flexible. Each writer is decoupled and can be create and config independently. User can: +//! 1.Combine different writer builder to build a writer which have complex write logic. Such as FanoutPartition + DataFileWrite or FanoutPartition + PosititionDeleteFileWrite. +//! 2.Customize the writer and combine it with original writer builder to build a writer which +//! can process the data in a specific way. +//! +//! There are two kinds of writer and related builder: +//! 1. `IcebergWriter` and `IcebergWriterBuilder`, they are focus on the data process logical. +//! If you want to support a new data process logical, you need to implement a new `IcebergWriter` and `IcebergWriterBuilder`. +//! 2. `FileWriter` and `FileWriterBuilder`, they are focus on the physical file write. +//! If you want to support a new physical file format, you need to implement a new `FileWriter` and `FileWriterBuilder`. +//! +//! The create process of iceberg writer is: +//! 1. Create a `FileWriterBuilder`. +//! 1a. Combine it with other `FileWriterBuilder` to get a new `FileWriterBuilder`. +//! 2. Use FileWriterBuilder to create a `IcebergWriterBuilder`. +//! 2a. Combine it with other `IcebergWriterBuilder` to get a new `IcebergWriterBuilder`. +//! 3. Use `build` function in `IcebergWriterBuilder` to create a `IcebergWriter`. +//! +//! # Simple Case 1: Create a data file writer using parquet file format. +//! # TODO(Implement this example) +//! ```ignore +//! // 1. Create a parquet file writer builder. +//! let parquet_writer_builder = ParquetFileWriterBuilder::new(parquet_file_writer_config); +//! // 2. Create a data file writer builder. +//! let DataFileWriterBuilder = DataFileWriterBuilder::new(parquet_writer_builder,data_file_writer_config); +//! // 3. Create a iceberg writer. +//! let iceberg_writer = DataFileWriterBuilder.build(schema).await?; +//! +//! iceberg_writer.write(input).await?; +//! +//! let write_result = iceberg_writer.flush().await?; +//! ``` +//! +//! # Complex Case 2: Create a fanout partition data file writer using parquet file format. +//! # TODO (Implement this example) +//! ```ignore +//! // 1. Create a parquet file writer builder. +//! let parquet_writer_builder = ParquetFileWriterBuilder::new(parquet_file_writer_config); +//! // 2. Create a data file writer builder. +//! let DataFileWriterBuilder = DataFileWriterBuilder::new(parquet_writer_builder,data_file_writer_config); +//! // 3. Create a fanout partition writer builder. +//! let fanout_partition_writer_builder = FanoutPartitionWriterBuilder::new(DataFileWriterBuilder, partition_config); +//! // 4. Create a iceberg writer. +//! let iceberg_writer = fanout_partition_writer_builder.build(schema).await?; +//! +//! iceberg_writer.write(input).await?; +//! +//! let write_result = iceberg_writer.flush().await?; +//! ``` + +use crate::{ + spec::{DataContentType, Struct}, + Result, +}; +use arrow_array::RecordBatch; +use arrow_schema::SchemaRef; + +pub mod file_writer; + +type DefaultInput = RecordBatch; + +/// The builder for iceberg writer. +#[async_trait::async_trait] +pub trait IcebergWriterBuilder<I = DefaultInput>: Send + Clone + 'static { + /// The associated writer type. + type R: IcebergWriter<I>; + /// Build the iceberg writer. + async fn build(self, schema: &SchemaRef) -> Result<Self::R>; +} + +/// The iceberg writer used to write data to iceberg table. +#[async_trait::async_trait] +pub trait IcebergWriter<I = DefaultInput>: Send + 'static { + /// The associated write result type. + type R: IcebergWriteResult; + /// Write data to iceberg table. + async fn write(&mut self, input: I) -> Result<()>; + /// Flush the writer and return the write result. + async fn flush(&mut self) -> Result<Vec<Self::R>>; +} + +/// The write result of iceberg writer. +pub trait IcebergWriteResult: Send + Sync + 'static { Review Comment: In this case, user can see the type of write_result directly and they can call write_result.build() to get the DataFile. We also can expose a build interface to make it clear like: ``` trait IcebergWriteResult { fn build(self) -> Vec<DataFile> } ``` But I'm rethinking do we really need this abstractš¤ We use generic param here because we can have DeltaResult for delta writer like ``` struct DeltaResult { data_file: Vec<DataFile>, pos_delete_file: Vec<DataFile>, eq_delete_file: Vec<DataFile> } ``` But I find that it actually can [read the type from DataFile](https://iceberg.apache.org/spec/#schemas-and-data-types:~:text=int%20with%20meaning%3A%200%3A%20DATA%2C%201%3A%20POSITION%20DELETES%2C%202%3A%20EQUALITY%20DELETES) directly. So return `Vec<DataFile>` also can work. So I think may be we can also discard the IcebergWriteResult, and directly use `write(&mut self, input: I) -> Vec<DataFileBuilder>` to make interface more simple. Because at less for now, in iceberg the return result of writer is always DataFile. I'm not sure should we preserve the extendable for it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For additional commands, e-mail: issues-h...@iceberg.apache.org
