This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new dbdce8fd perf(parquet): reuse leaf column index set across fields in
page-filtered reads (#207)
dbdce8fd is described below
commit dbdce8fd646394833586a324599811a135116f6e
Author: Zhou Hongfeng <[email protected]>
AuthorDate: Thu Aug 20 16:28:44 2026 +0800
perf(parquet): reuse leaf column index set across fields in page-filtered
reads (#207)
---
cmake_modules/arrow.diff | 8 ++++----
src/paimon/format/parquet/page_filtered_row_group_reader.cpp | 10 ++++++----
src/paimon/format/parquet/page_filtered_row_group_reader.h | 9 ++++++---
3 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff
index ce63af35..75e3bb51 100644
--- a/cmake_modules/arrow.diff
+++ b/cmake_modules/arrow.diff
@@ -57,7 +57,7 @@ index 285e2a5973..db919d7ef8 100644
}
+ ::arrow::Status GetColumn(
-+ int i, const std::vector<int>& column_indices,
++ int i, const std::shared_ptr<std::unordered_set<int>>& column_indices,
+ FileColumnIteratorFactory iterator_factory,
+ std::unique_ptr<ColumnReader>* out) override;
+
@@ -235,7 +235,7 @@ index 285e2a5973..db919d7ef8 100644
}
+::arrow::Status FileReaderImpl::GetColumn(
-+ int i, const std::vector<int>& column_indices,
++ int i, const std::shared_ptr<std::unordered_set<int>>& column_indices,
+ FileColumnIteratorFactory iterator_factory,
+ std::unique_ptr<ColumnReader>* out) {
+ RETURN_NOT_OK(BoundsCheckColumn(i));
@@ -244,7 +244,7 @@ index 285e2a5973..db919d7ef8 100644
+ ctx->pool = pool_;
+ ctx->iterator_factory = iterator_factory;
+ ctx->filter_leaves = true;
-+ ctx->included_leaves = VectorToSharedSet(column_indices);
++ ctx->included_leaves = column_indices;
+ std::unique_ptr<ColumnReaderImpl> result;
+ RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result));
+ *out = std::move(result);
@@ -298,7 +298,7 @@ index 6e46ca43f7..e86ff0ef52 100644
+ /// \param iterator_factory factory to create FileColumnIterator per leaf
+ /// \param[out] out the ColumnReader (may be nullptr if all leaves are
pruned)
+ virtual ::arrow::Status GetColumn(
-+ int i, const std::vector<int>& column_indices,
++ int i, const std::shared_ptr<std::unordered_set<int>>& column_indices,
+ FileColumnIteratorFactory iterator_factory,
+ std::unique_ptr<ColumnReader>* out) {
+ return ::arrow::Status::NotImplemented(
diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
index 1b4bdd30..f20f224f 100644
--- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
+++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
@@ -284,9 +284,9 @@ Status PageFilteredRowGroupReader::WaitForPreBuffer(
Result<std::shared_ptr<arrow::ChunkedArray>>
PageFilteredRowGroupReader::ReadFilteredField(
const std::shared_ptr<::parquet::RowGroupPageIndexReader>&
rg_page_index_reader,
- int32_t row_group_index, int32_t field_index, const std::vector<int32_t>&
column_indices,
- const RowRanges& row_ranges, int64_t row_group_row_count,
- ::parquet::arrow::FileReader* arrow_file_reader) {
+ int32_t row_group_index, int32_t field_index,
+ std::shared_ptr<std::unordered_set<int>> column_indices, const RowRanges&
row_ranges,
+ int64_t row_group_row_count, ::parquet::arrow::FileReader*
arrow_file_reader) {
// Factory: set a direct data page read plan on every leaf (per-leaf
OffsetIndex).
// The plan lets Arrow jump over unselected page headers as well as page
bodies.
auto factory =
@@ -397,12 +397,14 @@ Result<std::unique_ptr<arrow::RecordBatchReader>>
PageFilteredRowGroupReader::Re
std::vector<std::shared_ptr<arrow::ChunkedArray>> result_arrays;
result_arrays.reserve(field_indices.size());
+ std::shared_ptr<std::unordered_set<int>> col_indices_set =
+ std::make_shared<std::unordered_set<int>>(column_indices.begin(),
column_indices.end());
// TODO(zhouhongfeng.zhf): This loop could be parallelized.
for (int field_idx : field_indices) {
PAIMON_ASSIGN_OR_RAISE(
std::shared_ptr<arrow::ChunkedArray> chunked_array,
ReadFilteredField(row_group_page_index_reader, row_group_index,
field_idx,
- column_indices, row_ranges, row_group_row_count,
arrow_file_reader));
+ col_indices_set, row_ranges,
row_group_row_count, arrow_file_reader));
if (chunked_array->length() != expected_rows) {
return Status::Invalid(
diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h
b/src/paimon/format/parquet/page_filtered_row_group_reader.h
index 683bde71..a143ae5a 100644
--- a/src/paimon/format/parquet/page_filtered_row_group_reader.h
+++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h
@@ -23,6 +23,7 @@
#include <limits>
#include <memory>
#include <optional>
+#include <unordered_set>
#include <utility>
#include <vector>
@@ -117,11 +118,13 @@ class PageFilteredRowGroupReader {
/// Sets a direct page read plan on all leaves via factory, then drives
each leaf
/// independently via ResetLeaf/SkipRecords/ReadRecords using its own
/// compressed_ranges.
+ /// `column_indices` holds `int` rather than `int32_t` because the set is
+ /// handed straight to Arrow's `FileReader::GetColumn` (to avoid
reconstruction and deep copy)
static Result<std::shared_ptr<arrow::ChunkedArray>> ReadFilteredField(
const std::shared_ptr<::parquet::RowGroupPageIndexReader>&
rg_page_index_reader,
- int32_t row_group_index, int32_t field_index, const
std::vector<int32_t>& column_indices,
- const RowRanges& row_ranges, int64_t row_group_row_count,
- ::parquet::arrow::FileReader* arrow_file_reader);
+ int32_t row_group_index, int32_t field_index,
+ std::shared_ptr<std::unordered_set<int>> column_indices, const
RowRanges& row_ranges,
+ int64_t row_group_row_count, ::parquet::arrow::FileReader*
arrow_file_reader);
};
} // namespace paimon::parquet
