wgtmac commented on code in PR #198: URL: https://github.com/apache/iceberg-cpp/pull/198#discussion_r2317667583
########## src/iceberg/parquet/parquet_writer.cc: ########## @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/parquet/parquet_writer.h" + +#include <memory> + +#include <arrow/c/bridge.h> +#include <arrow/record_batch.h> +#include <arrow/util/key_value_metadata.h> +#include <parquet/arrow/schema.h> +#include <parquet/arrow/writer.h> +#include <parquet/file_writer.h> +#include <parquet/properties.h> + +#include "iceberg/arrow/arrow_error_transform_internal.h" +#include "iceberg/arrow/arrow_fs_file_io_internal.h" +#include "iceberg/schema_internal.h" +#include "iceberg/util/checked_cast.h" +#include "iceberg/util/macros.h" + +namespace iceberg::parquet { + +namespace { + +Result<std::shared_ptr<::arrow::io::OutputStream>> OpenOutputStream( + const WriterOptions& options) { + auto io = internal::checked_pointer_cast<arrow::ArrowFileSystemFileIO>(options.io); + ICEBERG_ARROW_ASSIGN_OR_RETURN(auto output, io->fs()->OpenOutputStream(options.path)); + return output; +} + +} // namespace + +class ParquetWriter::Impl { + public: + Status Open(const WriterOptions& options) { + auto writer_properties = + ::parquet::WriterProperties::Builder().memory_pool(pool_)->build(); + auto arrow_writer_properties = ::parquet::default_arrow_writer_properties(); + + ArrowSchema c_schema; + ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*options.schema, &c_schema)); + ICEBERG_ARROW_ASSIGN_OR_RETURN(arrow_schema_, ::arrow::ImportSchema(&c_schema)); + + std::shared_ptr<::parquet::SchemaDescriptor> schema_descriptor; + ICEBERG_ARROW_RETURN_NOT_OK( + ::parquet::arrow::ToParquetSchema(arrow_schema_.get(), *writer_properties, + *arrow_writer_properties, &schema_descriptor)); + auto schema_node = std::static_pointer_cast<::parquet::schema::GroupNode>( + schema_descriptor->schema_root()); + + ICEBERG_ASSIGN_OR_RAISE(output_stream_, OpenOutputStream(options)); + auto file_writer = ::parquet::ParquetFileWriter::Open( + output_stream_, std::move(schema_node), std::move(writer_properties)); + ICEBERG_ARROW_RETURN_NOT_OK( + ::parquet::arrow::FileWriter::Make(pool_, std::move(file_writer), arrow_schema_, + std::move(arrow_writer_properties), &writer_)); + + return {}; + } + + Status Write(ArrowArray array) { + ICEBERG_ARROW_ASSIGN_OR_RETURN(auto batch, + ::arrow::ImportRecordBatch(&array, arrow_schema_)); + + ICEBERG_ARROW_RETURN_NOT_OK(writer_->WriteRecordBatch(*batch)); + + return {}; + } + + // Close the writer and release resources + Status Close() { + if (writer_ == nullptr) { + return {}; // Already closed + } + + ICEBERG_ARROW_RETURN_NOT_OK(writer_->Close()); + auto& metadata = writer_->metadata(); + split_offsets_.reserve(metadata->num_row_groups()); + for (int i = 0; i < metadata->num_row_groups(); ++i) { + split_offsets_.push_back(metadata->RowGroup(i)->file_offset()); + } + writer_.reset(); + + ICEBERG_ARROW_ASSIGN_OR_RETURN(total_bytes_, output_stream_->Tell()); + ICEBERG_ARROW_RETURN_NOT_OK(output_stream_->Close()); + return {}; + } + + bool Closed() const { return writer_ == nullptr; } + + int64_t length() const { return total_bytes_; } + + std::vector<int64_t> split_offsets() const { return split_offsets_; } + + private: + // TODO(gangwu): make memory pool configurable + ::arrow::MemoryPool* pool_ = ::arrow::default_memory_pool(); + // Schema to write from the Parquet file. + std::shared_ptr<::arrow::Schema> arrow_schema_; + // The output stream to write Parquet file. + std::shared_ptr<::arrow::io::OutputStream> output_stream_; + // Parquet file writer to write ArrowArray. + std::unique_ptr<::parquet::arrow::FileWriter> writer_; + // Total length of the written Parquet file. + int64_t total_bytes_; Review Comment: `total_bytes_` should be initialized to 0 or -1. ########## test/parquet_test.cc: ########## @@ -17,38 +17,121 @@ * under the License. */ +#include <optional> + #include <arrow/array.h> #include <arrow/c/bridge.h> #include <arrow/json/from_string.h> #include <arrow/record_batch.h> #include <arrow/table.h> +#include <arrow/type.h> #include <arrow/util/key_value_metadata.h> #include <parquet/arrow/reader.h> #include <parquet/arrow/writer.h> #include <parquet/metadata.h> +#include "iceberg/arrow/arrow_error_transform_internal.h" #include "iceberg/arrow/arrow_fs_file_io_internal.h" -#include "iceberg/parquet/parquet_reader.h" +#include "iceberg/file_reader.h" +#include "iceberg/file_writer.h" #include "iceberg/parquet/parquet_register.h" +#include "iceberg/result.h" #include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/schema_internal.h" #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" +#include "iceberg/util/macros.h" #include "matchers.h" -#include "temp_file_test_base.h" namespace iceberg::parquet { -class ParquetReaderTest : public TempFileTestBase { +namespace { + +Status WriteTableInner(Writer& writer, std::shared_ptr<::arrow::Array> data) { + ArrowArray arr; + ICEBERG_ARROW_RETURN_NOT_OK(::arrow::ExportArray(*data, &arr)); + ICEBERG_RETURN_UNEXPECTED(writer.Write(arr)); + return writer.Close(); +} + +Status WriteTable(std::shared_ptr<::arrow::Array> data, + const WriterOptions& writer_options) { + ICEBERG_ASSIGN_OR_RAISE( + auto writer, WriterFactoryRegistry::Open(FileFormatType::kParquet, writer_options)); + return WriteTableInner(*writer, data); +} + +Status ReadTable(std::shared_ptr<::arrow::Array>& out, + const ReaderOptions& reader_options) { + ICEBERG_ASSIGN_OR_RAISE( + auto reader, ReaderFactoryRegistry::Open(FileFormatType::kParquet, reader_options)); + ICEBERG_ASSIGN_OR_RAISE(auto read_data, reader->Next()); + + if (!read_data.has_value()) { + out = nullptr; + return {}; + } + auto arrow_c_array = read_data.value(); + + ArrowSchema arrow_schema; + ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*reader_options.projection, &arrow_schema)); + ICEBERG_ARROW_ASSIGN_OR_RETURN(out, + ::arrow::ImportArray(&arrow_c_array, &arrow_schema)); + return {}; +} + +void DoRoundtrip(std::shared_ptr<::arrow::Array> data, std::shared_ptr<Schema> schema, + std::shared_ptr<::arrow::Array>& out) { + std::shared_ptr<FileIO> file_io = arrow::ArrowFileSystemFileIO::MakeMockFileIO(); + const std::string basePath = "base.parquet"; + + auto writer_data = WriterFactoryRegistry::Open( + FileFormatType::kParquet, {.path = basePath, .schema = schema, .io = file_io}); + ASSERT_THAT(writer_data, IsOk()) + << "Failed to create writer: " << writer_data.error().message; + auto writer = std::move(writer_data.value()); + ASSERT_THAT(WriteTableInner(*writer, data), IsOk()); + + ASSERT_THAT(ReadTable(out, {.path = basePath, + .length = writer->length(), + .io = file_io, + .projection = schema}), + IsOk()); + + ASSERT_TRUE(out != nullptr) << "Reader.Next() returned no data"; Review Comment: Fair enough. ########## test/parquet_test.cc: ########## @@ -17,38 +17,120 @@ * under the License. */ +#include <optional> + #include <arrow/array.h> #include <arrow/c/bridge.h> #include <arrow/json/from_string.h> #include <arrow/record_batch.h> #include <arrow/table.h> +#include <arrow/type.h> #include <arrow/util/key_value_metadata.h> #include <parquet/arrow/reader.h> #include <parquet/arrow/writer.h> #include <parquet/metadata.h> +#include "iceberg/arrow/arrow_error_transform_internal.h" #include "iceberg/arrow/arrow_fs_file_io_internal.h" -#include "iceberg/parquet/parquet_reader.h" +#include "iceberg/file_reader.h" +#include "iceberg/file_writer.h" #include "iceberg/parquet/parquet_register.h" +#include "iceberg/result.h" #include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/schema_internal.h" #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" +#include "iceberg/util/macros.h" #include "matchers.h" -#include "temp_file_test_base.h" namespace iceberg::parquet { -class ParquetReaderTest : public TempFileTestBase { +namespace { + +Status WriteArrayInner(Writer& writer, std::shared_ptr<::arrow::Array> data) { Review Comment: Can we rename it to `WriteArray`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
