wgtmac commented on code in PR #176: URL: https://github.com/apache/iceberg-cpp/pull/176#discussion_r2287628313
########## src/iceberg/metadata_adapter.h: ########## @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/metadata_adapter.h Review Comment: Should we rename it to `manifest_adapter.h`? ########## src/iceberg/manifest_writer.cc: ########## @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/manifest_writer.h" + +#include "iceberg/manifest_entry.h" +#include "iceberg/manifest_list.h" +#include "iceberg/schema.h" +#include "iceberg/util/macros.h" +#include "iceberg/v1_metadata.h" +#include "iceberg/v2_metadata.h" +#include "iceberg/v3_metadata.h" +#include "iceberg/v4_metadata.h" + +namespace iceberg { + +/// \brief Write manifest files to a manifest list file. +class ManifestWriterImpl : public ManifestWriter { + public: + ManifestWriterImpl(std::unique_ptr<Writer> writer, + std::unique_ptr<ManifestEntryAdapter> adapter) + : writer_(std::move(writer)), adapter_(std::move(adapter)) {} + + Status Add(const ManifestEntry& entry) override { + if (adapter_->size() >= kBatchSize) { + ICEBERG_ASSIGN_OR_RAISE(auto array, adapter_->FinishAppending()); + ICEBERG_RETURN_UNEXPECTED(writer_->Write(array)); + ICEBERG_RETURN_UNEXPECTED(adapter_->StartAppending()); + } + return adapter_->Append(entry); + } + + Status AddAll(const std::vector<ManifestEntry>& entries) override { + for (const auto& entry : entries) { + ICEBERG_RETURN_UNEXPECTED(Add(entry)); + } + return {}; + } + + Status Close() override { + if (adapter_->size() > 0) { + ICEBERG_ASSIGN_OR_RAISE(auto array, adapter_->FinishAppending()); + ICEBERG_RETURN_UNEXPECTED(writer_->Write(array)); + } + return {}; + } + + private: + static constexpr int64_t kBatchSize = 1024; + std::unique_ptr<Writer> writer_; + std::unique_ptr<ManifestEntryAdapter> adapter_; +}; + +std::shared_ptr<Schema> ParseSchema(std::shared_ptr<Schema> partition_schema) { + auto manifest_entry_schema = + ManifestEntry::TypeFromPartitionType(std::move(partition_schema)); + auto fields_span = manifest_entry_schema->fields(); + std::vector<SchemaField> fields(fields_span.begin(), fields_span.end()); + return std::make_shared<Schema>(fields); +} + +Result<std::unique_ptr<Writer>> OpenFileWriter(std::string_view location, + const std::shared_ptr<Schema> schema, + std::shared_ptr<FileIO> file_io) { + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + WriterFactoryRegistry::Open( + FileFormatType::kAvro, + {.path = std::string(location), .schema = schema, .io = std::move(file_io)})); + return writer; +} + +Result<std::unique_ptr<ManifestWriter>> ManifestWriter::MakeV1Writer( + std::optional<int64_t> snapshot_id, std::string_view manifest_location, + std::shared_ptr<FileIO> file_io, std::shared_ptr<Schema> partition_schema) { + auto schema = ParseSchema(partition_schema); Review Comment: Schemas may vary among versions so we cannot use a common function to create it. ########## src/iceberg/manifest_writer.h: ########## @@ -35,14 +35,57 @@ namespace iceberg { class ICEBERG_EXPORT ManifestWriter { public: virtual ~ManifestWriter() = default; - virtual Status WriteManifestEntries( - const std::vector<ManifestEntry>& entries) const = 0; + + /// \brief Write manifest entry to file. + /// \param entry Manifest entry to write. + /// \return Status::OK() if entry was written successfully + virtual Status Add(const ManifestEntry& entry) = 0; Review Comment: Do we really need to make `ManifestWriter` and `ManifestListWriter` pure virtual if they only have a single `XXXImpl` subclass? ########## src/iceberg/v4_metadata.h: ########## @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/v4_metadata.h + +#include "iceberg/manifest_entry.h" +#include "iceberg/manifest_list.h" +#include "iceberg/metadata_adapter.h" + +namespace iceberg { + +/// \brief Adapter to convert V4 ManifestEntry to `ArrowArray`. +class ManifestEntryAdapterV4 : public ManifestEntryAdapter { Review Comment: Remove this since V4 is far from being finalized. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
