wgtmac commented on code in PR #198:
URL: https://github.com/apache/iceberg-cpp/pull/198#discussion_r2313316596
##########
src/iceberg/parquet/parquet_reader.cc:
##########
@@ -240,6 +241,8 @@ class ParquetReader::Impl {
std::unique_ptr<::parquet::arrow::FileReader> reader_;
// The context to keep track of the reading progress.
std::unique_ptr<ReadContext> context_;
+ // The input stream to read Parquet file.
+ std::shared_ptr<::arrow::io::RandomAccessFile> input_stream_;
Review Comment:
nit: move it before `reader_`?
##########
src/iceberg/schema_internal.h:
##########
@@ -33,7 +34,7 @@ namespace iceberg {
// Apache Arrow C++ uses "PARQUET:field_id" to store field IDs for Parquet.
// Here we follow a similar convention for Iceberg but we might also add
// "PARQUET:field_id" in the future once we implement a Parquet writer.
-constexpr std::string_view kFieldIdKey = "ICEBERG:field_id";
+constexpr std::string_view kFieldIdKey = kParquetFieldIdKey;
Review Comment:
What about removing this line and directly use `kParquetFieldIdKey` in the
source file?
##########
src/iceberg/parquet/parquet_writer.cc:
##########
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/parquet/parquet_writer.h"
+
+#include <memory>
+
+#include <arrow/c/bridge.h>
+#include <arrow/record_batch.h>
+#include <arrow/util/key_value_metadata.h>
+#include <parquet/arrow/schema.h>
+#include <parquet/arrow/writer.h>
+#include <parquet/file_writer.h>
+#include <parquet/properties.h>
+
+#include "iceberg/arrow/arrow_error_transform_internal.h"
+#include "iceberg/arrow/arrow_fs_file_io_internal.h"
+#include "iceberg/schema_internal.h"
+#include "iceberg/util/checked_cast.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg::parquet {
+
+namespace {
+
+Result<std::shared_ptr<::arrow::io::OutputStream>> OpenOutputStream(
+ const WriterOptions& options) {
+ auto io =
internal::checked_pointer_cast<arrow::ArrowFileSystemFileIO>(options.io);
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(auto output,
io->fs()->OpenOutputStream(options.path));
+ return output;
+}
+
+} // namespace
+
+class ParquetWriter::Impl {
+ public:
+ Status Open(const WriterOptions& options) {
+ auto writer_properties =
+ ::parquet::WriterProperties::Builder().memory_pool(pool_)->build();
+ auto arrow_writer_properties =
::parquet::default_arrow_writer_properties();
+
+ ArrowSchema c_schema;
+ ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*options.schema, &c_schema));
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(arrow_schema_,
::arrow::ImportSchema(&c_schema));
+
+ std::shared_ptr<::parquet::SchemaDescriptor> schema_descriptor;
+ ICEBERG_ARROW_RETURN_NOT_OK(
+ ::parquet::arrow::ToParquetSchema(arrow_schema_.get(),
*writer_properties,
+ *arrow_writer_properties,
&schema_descriptor));
+ auto schema_node = std::static_pointer_cast<::parquet::schema::GroupNode>(
+ schema_descriptor->schema_root());
+
+ std::shared_ptr<::arrow::KeyValueMetadata> metadata =
+ ::arrow::key_value_metadata(options.properties);
+
+ ICEBERG_ASSIGN_OR_RAISE(output_stream_, OpenOutputStream(options));
+ auto file_writer = ::parquet::ParquetFileWriter::Open(output_stream_,
schema_node,
+ writer_properties,
metadata);
+ ICEBERG_ARROW_RETURN_NOT_OK(::parquet::arrow::FileWriter::Make(
+ pool_, std::move(file_writer), arrow_schema_, arrow_writer_properties,
&writer_));
+
+ return {};
+ }
+
+ Status Write(ArrowArray array) {
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(auto batch,
+ ::arrow::ImportRecordBatch(&array,
arrow_schema_));
+
+ ICEBERG_ARROW_RETURN_NOT_OK(writer_->WriteRecordBatch(*batch));
+
+ return {};
+ }
+
+ // Close the writer and release resources
+ Status Close() {
+ if (writer_ == nullptr) {
+ return {}; // Already closed
+ }
+
+ ICEBERG_ARROW_RETURN_NOT_OK(writer_->Close());
+ writer_.reset();
+ ICEBERG_ARROW_RETURN_NOT_OK(output_stream_->Close());
+ return {};
+ }
+
+ bool Closed() const { return writer_ == nullptr; }
+
+ private:
+ // TODO(gangwu): make memory pool configurable
+ ::arrow::MemoryPool* pool_ = ::arrow::default_memory_pool();
+ // Schema to write from the Parquet file.
+ std::shared_ptr<::arrow::Schema> arrow_schema_;
+ // Parquet file writer to write ArrowArray.
+ std::unique_ptr<::parquet::arrow::FileWriter> writer_;
+ // The output stream to write Parquet file.
+ std::shared_ptr<::arrow::io::OutputStream> output_stream_;
+};
+
+ParquetWriter::~ParquetWriter() = default;
+
+Status ParquetWriter::Open(const WriterOptions& options) {
+ impl_ = std::make_unique<Impl>();
+ return impl_->Open(options);
+}
+
+Status ParquetWriter::Write(ArrowArray array) { return impl_->Write(array); }
+
+Status ParquetWriter::Close() { return impl_->Close(); }
+
+std::optional<Metrics> ParquetWriter::metrics() {
+ if (!impl_->Closed()) {
+ return std::nullopt;
+ }
+ return {};
+}
+
+std::optional<int64_t> ParquetWriter::length() {
+ if (!impl_->Closed()) {
+ return std::nullopt;
+ }
+ return {};
+}
+
+std::vector<int64_t> ParquetWriter::split_offsets() {
+ if (!impl_->Closed()) {
+ return {};
+ }
+ return {};
Review Comment:
ditto
##########
src/iceberg/parquet/parquet_writer.cc:
##########
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/parquet/parquet_writer.h"
+
+#include <memory>
+
+#include <arrow/c/bridge.h>
+#include <arrow/record_batch.h>
+#include <arrow/util/key_value_metadata.h>
+#include <parquet/arrow/schema.h>
+#include <parquet/arrow/writer.h>
+#include <parquet/file_writer.h>
+#include <parquet/properties.h>
+
+#include "iceberg/arrow/arrow_error_transform_internal.h"
+#include "iceberg/arrow/arrow_fs_file_io_internal.h"
+#include "iceberg/schema_internal.h"
+#include "iceberg/util/checked_cast.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg::parquet {
+
+namespace {
+
+Result<std::shared_ptr<::arrow::io::OutputStream>> OpenOutputStream(
+ const WriterOptions& options) {
+ auto io =
internal::checked_pointer_cast<arrow::ArrowFileSystemFileIO>(options.io);
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(auto output,
io->fs()->OpenOutputStream(options.path));
+ return output;
+}
+
+} // namespace
+
+class ParquetWriter::Impl {
+ public:
+ Status Open(const WriterOptions& options) {
+ auto writer_properties =
+ ::parquet::WriterProperties::Builder().memory_pool(pool_)->build();
+ auto arrow_writer_properties =
::parquet::default_arrow_writer_properties();
+
+ ArrowSchema c_schema;
+ ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*options.schema, &c_schema));
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(arrow_schema_,
::arrow::ImportSchema(&c_schema));
+
+ std::shared_ptr<::parquet::SchemaDescriptor> schema_descriptor;
+ ICEBERG_ARROW_RETURN_NOT_OK(
+ ::parquet::arrow::ToParquetSchema(arrow_schema_.get(),
*writer_properties,
+ *arrow_writer_properties,
&schema_descriptor));
+ auto schema_node = std::static_pointer_cast<::parquet::schema::GroupNode>(
+ schema_descriptor->schema_root());
+
+ std::shared_ptr<::arrow::KeyValueMetadata> metadata =
+ ::arrow::key_value_metadata(options.properties);
Review Comment:
Let's remove this first. These properties are not supposed for metadata.
They are writer configs instead.
##########
test/parquet_test.cc:
##########
@@ -204,4 +280,85 @@ TEST_F(ParquetReaderTest, ReadSplit) {
}
}
+class ParquetReadWrite : public ::testing::Test {
+ protected:
+ static void SetUpTestSuite() { parquet::RegisterAll(); }
+};
+
+TEST_F(ParquetReadWrite, EmptyStruct) {
+ auto schema =
+
std::make_shared<Schema>(std::vector<SchemaField>{SchemaField::MakeRequired(
+ 1, "empty_struct",
std::make_shared<StructType>(std::vector<SchemaField>{}))});
+
+ ArrowSchema arrow_c_schema;
+ ASSERT_THAT(ToArrowSchema(*schema, &arrow_c_schema), IsOk());
+ auto arrow_schema = ::arrow::ImportType(&arrow_c_schema).ValueOrDie();
+
+ auto array = ::arrow::json::ArrayFromJSONString(
+ ::arrow::struct_(arrow_schema->fields()), R"([null, {}])")
+ .ValueOrDie();
+
+ std::shared_ptr<FileIO> file_io =
arrow::ArrowFileSystemFileIO::MakeMockFileIO();
+ const std::string basePath = "base.parquet";
+
+ ASSERT_THAT(WriteTable(array, {.path = basePath, .schema = schema, .io =
file_io}),
+ IsError(ErrorKind::kUnknownError));
Review Comment:
Why this is a kUnknownError?
##########
test/parquet_test.cc:
##########
@@ -17,38 +17,113 @@
* under the License.
*/
+#include <optional>
+
#include <arrow/array.h>
#include <arrow/c/bridge.h>
#include <arrow/json/from_string.h>
#include <arrow/record_batch.h>
#include <arrow/table.h>
+#include <arrow/type.h>
#include <arrow/util/key_value_metadata.h>
+#include <iceberg/file_reader.h>
+#include <iceberg/file_writer.h>
+#include <iceberg/result.h>
+#include <iceberg/schema_field.h>
+#include <iceberg/schema_internal.h>
Review Comment:
```suggestion
```
They are redundant, right? Or if really required, use `""` to include them.
##########
src/iceberg/parquet/parquet_writer.cc:
##########
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/parquet/parquet_writer.h"
+
+#include <memory>
+
+#include <arrow/c/bridge.h>
+#include <arrow/record_batch.h>
+#include <arrow/util/key_value_metadata.h>
+#include <parquet/arrow/schema.h>
+#include <parquet/arrow/writer.h>
+#include <parquet/file_writer.h>
+#include <parquet/properties.h>
+
+#include "iceberg/arrow/arrow_error_transform_internal.h"
+#include "iceberg/arrow/arrow_fs_file_io_internal.h"
+#include "iceberg/schema_internal.h"
+#include "iceberg/util/checked_cast.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg::parquet {
+
+namespace {
+
+Result<std::shared_ptr<::arrow::io::OutputStream>> OpenOutputStream(
+ const WriterOptions& options) {
+ auto io =
internal::checked_pointer_cast<arrow::ArrowFileSystemFileIO>(options.io);
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(auto output,
io->fs()->OpenOutputStream(options.path));
+ return output;
+}
+
+} // namespace
+
+class ParquetWriter::Impl {
+ public:
+ Status Open(const WriterOptions& options) {
+ auto writer_properties =
+ ::parquet::WriterProperties::Builder().memory_pool(pool_)->build();
+ auto arrow_writer_properties =
::parquet::default_arrow_writer_properties();
+
+ ArrowSchema c_schema;
+ ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*options.schema, &c_schema));
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(arrow_schema_,
::arrow::ImportSchema(&c_schema));
+
+ std::shared_ptr<::parquet::SchemaDescriptor> schema_descriptor;
+ ICEBERG_ARROW_RETURN_NOT_OK(
+ ::parquet::arrow::ToParquetSchema(arrow_schema_.get(),
*writer_properties,
+ *arrow_writer_properties,
&schema_descriptor));
+ auto schema_node = std::static_pointer_cast<::parquet::schema::GroupNode>(
+ schema_descriptor->schema_root());
+
+ std::shared_ptr<::arrow::KeyValueMetadata> metadata =
+ ::arrow::key_value_metadata(options.properties);
+
+ ICEBERG_ASSIGN_OR_RAISE(output_stream_, OpenOutputStream(options));
+ auto file_writer = ::parquet::ParquetFileWriter::Open(output_stream_,
schema_node,
+ writer_properties,
metadata);
+ ICEBERG_ARROW_RETURN_NOT_OK(::parquet::arrow::FileWriter::Make(
+ pool_, std::move(file_writer), arrow_schema_, arrow_writer_properties,
&writer_));
+
+ return {};
+ }
+
+ Status Write(ArrowArray array) {
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(auto batch,
+ ::arrow::ImportRecordBatch(&array,
arrow_schema_));
+
+ ICEBERG_ARROW_RETURN_NOT_OK(writer_->WriteRecordBatch(*batch));
+
+ return {};
+ }
+
+ // Close the writer and release resources
+ Status Close() {
+ if (writer_ == nullptr) {
+ return {}; // Already closed
+ }
+
+ ICEBERG_ARROW_RETURN_NOT_OK(writer_->Close());
+ writer_.reset();
+ ICEBERG_ARROW_RETURN_NOT_OK(output_stream_->Close());
+ return {};
+ }
+
+ bool Closed() const { return writer_ == nullptr; }
+
+ private:
+ // TODO(gangwu): make memory pool configurable
+ ::arrow::MemoryPool* pool_ = ::arrow::default_memory_pool();
+ // Schema to write from the Parquet file.
+ std::shared_ptr<::arrow::Schema> arrow_schema_;
+ // Parquet file writer to write ArrowArray.
+ std::unique_ptr<::parquet::arrow::FileWriter> writer_;
+ // The output stream to write Parquet file.
+ std::shared_ptr<::arrow::io::OutputStream> output_stream_;
Review Comment:
nit: define them in the creation order?
##########
test/parquet_test.cc:
##########
@@ -17,38 +17,113 @@
* under the License.
*/
+#include <optional>
+
#include <arrow/array.h>
#include <arrow/c/bridge.h>
#include <arrow/json/from_string.h>
#include <arrow/record_batch.h>
#include <arrow/table.h>
+#include <arrow/type.h>
#include <arrow/util/key_value_metadata.h>
+#include <iceberg/file_reader.h>
+#include <iceberg/file_writer.h>
+#include <iceberg/result.h>
+#include <iceberg/schema_field.h>
+#include <iceberg/schema_internal.h>
#include <parquet/arrow/reader.h>
#include <parquet/arrow/writer.h>
#include <parquet/metadata.h>
+#include "iceberg/arrow/arrow_error_transform_internal.h"
#include "iceberg/arrow/arrow_fs_file_io_internal.h"
-#include "iceberg/parquet/parquet_reader.h"
#include "iceberg/parquet/parquet_register.h"
#include "iceberg/schema.h"
#include "iceberg/type.h"
#include "iceberg/util/checked_cast.h"
+#include "iceberg/util/macros.h"
#include "matchers.h"
-#include "temp_file_test_base.h"
namespace iceberg::parquet {
-class ParquetReaderTest : public TempFileTestBase {
+namespace {
+
+Status WriteTable(std::shared_ptr<::arrow::Array> data,
+ const WriterOptions& writer_options) {
+ ICEBERG_ASSIGN_OR_RAISE(
+ auto writer, WriterFactoryRegistry::Open(FileFormatType::kParquet,
writer_options));
+ ArrowArray arr;
+ ICEBERG_ARROW_RETURN_NOT_OK(::arrow::ExportArray(*data, &arr));
+ ICEBERG_RETURN_UNEXPECTED(writer->Write(arr));
+ return writer->Close();
+}
+
+Status ReadTable(std::optional<std::shared_ptr<::arrow::Array>>* out,
Review Comment:
```suggestion
Status ReadTable(std::shared_ptr<::arrow::Array>* out,
```
It seems we can get rid of optional.
##########
src/iceberg/parquet/parquet_writer.cc:
##########
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/parquet/parquet_writer.h"
+
+#include <memory>
+
+#include <arrow/c/bridge.h>
+#include <arrow/record_batch.h>
+#include <arrow/util/key_value_metadata.h>
+#include <parquet/arrow/schema.h>
+#include <parquet/arrow/writer.h>
+#include <parquet/file_writer.h>
+#include <parquet/properties.h>
+
+#include "iceberg/arrow/arrow_error_transform_internal.h"
+#include "iceberg/arrow/arrow_fs_file_io_internal.h"
+#include "iceberg/schema_internal.h"
+#include "iceberg/util/checked_cast.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg::parquet {
+
+namespace {
+
+Result<std::shared_ptr<::arrow::io::OutputStream>> OpenOutputStream(
+ const WriterOptions& options) {
+ auto io =
internal::checked_pointer_cast<arrow::ArrowFileSystemFileIO>(options.io);
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(auto output,
io->fs()->OpenOutputStream(options.path));
+ return output;
+}
+
+} // namespace
+
+class ParquetWriter::Impl {
+ public:
+ Status Open(const WriterOptions& options) {
+ auto writer_properties =
+ ::parquet::WriterProperties::Builder().memory_pool(pool_)->build();
+ auto arrow_writer_properties =
::parquet::default_arrow_writer_properties();
+
+ ArrowSchema c_schema;
+ ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*options.schema, &c_schema));
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(arrow_schema_,
::arrow::ImportSchema(&c_schema));
+
+ std::shared_ptr<::parquet::SchemaDescriptor> schema_descriptor;
+ ICEBERG_ARROW_RETURN_NOT_OK(
+ ::parquet::arrow::ToParquetSchema(arrow_schema_.get(),
*writer_properties,
+ *arrow_writer_properties,
&schema_descriptor));
+ auto schema_node = std::static_pointer_cast<::parquet::schema::GroupNode>(
+ schema_descriptor->schema_root());
+
+ std::shared_ptr<::arrow::KeyValueMetadata> metadata =
+ ::arrow::key_value_metadata(options.properties);
+
+ ICEBERG_ASSIGN_OR_RAISE(output_stream_, OpenOutputStream(options));
+ auto file_writer = ::parquet::ParquetFileWriter::Open(output_stream_,
schema_node,
+ writer_properties,
metadata);
+ ICEBERG_ARROW_RETURN_NOT_OK(::parquet::arrow::FileWriter::Make(
+ pool_, std::move(file_writer), arrow_schema_, arrow_writer_properties,
&writer_));
+
+ return {};
+ }
+
+ Status Write(ArrowArray array) {
+ ICEBERG_ARROW_ASSIGN_OR_RETURN(auto batch,
+ ::arrow::ImportRecordBatch(&array,
arrow_schema_));
+
+ ICEBERG_ARROW_RETURN_NOT_OK(writer_->WriteRecordBatch(*batch));
+
+ return {};
+ }
+
+ // Close the writer and release resources
+ Status Close() {
+ if (writer_ == nullptr) {
+ return {}; // Already closed
+ }
+
+ ICEBERG_ARROW_RETURN_NOT_OK(writer_->Close());
+ writer_.reset();
+ ICEBERG_ARROW_RETURN_NOT_OK(output_stream_->Close());
+ return {};
+ }
+
+ bool Closed() const { return writer_ == nullptr; }
+
+ private:
+ // TODO(gangwu): make memory pool configurable
+ ::arrow::MemoryPool* pool_ = ::arrow::default_memory_pool();
+ // Schema to write from the Parquet file.
+ std::shared_ptr<::arrow::Schema> arrow_schema_;
+ // Parquet file writer to write ArrowArray.
+ std::unique_ptr<::parquet::arrow::FileWriter> writer_;
+ // The output stream to write Parquet file.
+ std::shared_ptr<::arrow::io::OutputStream> output_stream_;
+};
+
+ParquetWriter::~ParquetWriter() = default;
+
+Status ParquetWriter::Open(const WriterOptions& options) {
+ impl_ = std::make_unique<Impl>();
+ return impl_->Open(options);
+}
+
+Status ParquetWriter::Write(ArrowArray array) { return impl_->Write(array); }
+
+Status ParquetWriter::Close() { return impl_->Close(); }
+
+std::optional<Metrics> ParquetWriter::metrics() {
+ if (!impl_->Closed()) {
+ return std::nullopt;
+ }
+ return {};
+}
+
+std::optional<int64_t> ParquetWriter::length() {
Review Comment:
Why not implement this?
##########
test/parquet_test.cc:
##########
@@ -204,4 +280,85 @@ TEST_F(ParquetReaderTest, ReadSplit) {
}
}
+class ParquetReadWrite : public ::testing::Test {
+ protected:
+ static void SetUpTestSuite() { parquet::RegisterAll(); }
+};
+
+TEST_F(ParquetReadWrite, EmptyStruct) {
+ auto schema =
+
std::make_shared<Schema>(std::vector<SchemaField>{SchemaField::MakeRequired(
+ 1, "empty_struct",
std::make_shared<StructType>(std::vector<SchemaField>{}))});
+
+ ArrowSchema arrow_c_schema;
+ ASSERT_THAT(ToArrowSchema(*schema, &arrow_c_schema), IsOk());
+ auto arrow_schema = ::arrow::ImportType(&arrow_c_schema).ValueOrDie();
+
+ auto array = ::arrow::json::ArrayFromJSONString(
+ ::arrow::struct_(arrow_schema->fields()), R"([null, {}])")
+ .ValueOrDie();
+
+ std::shared_ptr<FileIO> file_io =
arrow::ArrowFileSystemFileIO::MakeMockFileIO();
+ const std::string basePath = "base.parquet";
+
+ ASSERT_THAT(WriteTable(array, {.path = basePath, .schema = schema, .io =
file_io}),
+ IsError(ErrorKind::kUnknownError));
+}
+
+TEST_F(ParquetReadWrite, SimpleStructRoundTrip) {
+ auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
+ SchemaField::MakeOptional(1, "a",
+ struct_({
+ SchemaField::MakeOptional(2, "a1",
int64()),
+ SchemaField::MakeOptional(3, "a2",
string()),
+ })),
+ });
+
+ ArrowSchema arrow_c_schema;
+ ASSERT_THAT(ToArrowSchema(*schema, &arrow_c_schema), IsOk());
+ auto arrow_schema = ::arrow::ImportType(&arrow_c_schema).ValueOrDie();
+
+ auto array =
+
::arrow::json::ArrayFromJSONString(::arrow::struct_(arrow_schema->fields()),
+ R"([[{"a1": 1, "a2": "abc"}],
+ [{"a1": 0}],
+ [{"a2": "edf"}],
+ [{}]])")
+ .ValueOrDie();
+
+ std::shared_ptr<::arrow::Array> out;
+ DoRoundtrip(array, schema, &out);
+
+ ASSERT_TRUE(out->Equals(*array));
+}
+
+TEST_F(ParquetReadWrite, SimpleTypeRoundTrip) {
+ auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
+ SchemaField::MakeOptional(1, "a", boolean()),
+ SchemaField::MakeOptional(2, "b", int32()),
+ SchemaField::MakeOptional(3, "c", int64()),
+ SchemaField::MakeOptional(4, "d", float32()),
+ SchemaField::MakeOptional(5, "e", float64()),
+ SchemaField::MakeOptional(6, "f", string()),
+ SchemaField::MakeOptional(7, "g", time()),
+ SchemaField::MakeOptional(8, "h", timestamp()),
+ });
+
+ ArrowSchema arrow_c_schema;
+ ASSERT_THAT(ToArrowSchema(*schema, &arrow_c_schema), IsOk());
+ auto arrow_schema = ::arrow::ImportType(&arrow_c_schema).ValueOrDie();
+
+ auto array = ::arrow::json::ArrayFromJSONString(
+ ::arrow::struct_(arrow_schema->fields()),
+ R"([[true, 1, 2, 1.1, 1.2, "abc", 44614000,
1756696503000000],
+ [false, 0, 0, 0, 0, "", 0, 0],
Review Comment:
The alignment is a little bit strange...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
