liurenjie1024 commented on code in PR #1602:
URL: https://github.com/apache/iceberg-rust/pull/1602#discussion_r2422430526
##########
crates/iceberg/src/transform/mod.rs:
##########
@@ -29,7 +29,7 @@ mod truncate;
mod void;
/// TransformFunction is a trait that defines the interface for all transform
functions.
-pub trait TransformFunction: Send + Sync {
+pub trait TransformFunction: Send + Sync + std::fmt::Debug {
Review Comment:
```suggestion
pub trait TransformFunction: Send + Sync + Debug {
```
It's better to keep style consistent with others.
##########
crates/integrations/datafusion/src/physical_plan/project.rs:
##########
@@ -0,0 +1,499 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Partition value projection for Iceberg tables.
+
+use std::sync::{Arc, Mutex};
+
+use datafusion::arrow::array::{ArrayRef, RecordBatch, StructArray};
+use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema};
+use datafusion::common::Result as DFResult;
+use datafusion::error::DataFusionError;
+use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_expr::expressions::Column;
+use datafusion::physical_plan::projection::ProjectionExec;
+use datafusion::physical_plan::{ColumnarValue, ExecutionPlan};
+use iceberg::arrow::record_batch_projector::RecordBatchProjector;
+use iceberg::spec::{PartitionSpec, Schema};
+use iceberg::table::Table;
+use iceberg::transform::BoxedTransformFunction;
+
+use crate::to_datafusion_error;
+
+/// Column name for the combined partition values struct
+const PARTITION_VALUES_COLUMN: &str = "_partition";
+
+/// Extends an ExecutionPlan with partition value calculations for Iceberg
tables.
+///
+/// This function takes an input ExecutionPlan and extends it with an
additional column
+/// containing calculated partition values based on the table's partition
specification.
+/// For unpartitioned tables, returns the original plan unchanged.
+///
+/// # Arguments
+/// * `input` - The input ExecutionPlan to extend
+/// * `table` - The Iceberg table with partition specification
+///
+/// # Returns
+/// * `Ok(Arc<dyn ExecutionPlan>)` - Extended plan with partition values column
+/// * `Err` - If partition spec is not found or transformation fails
+pub fn project_with_partition(
+ input: Arc<dyn ExecutionPlan>,
+ table: &Table,
+) -> DFResult<Arc<dyn ExecutionPlan>> {
+ let metadata = table.metadata();
+ let partition_spec = metadata.default_partition_spec();
+ let table_schema = metadata.current_schema();
+
+ if partition_spec.is_unpartitioned() {
+ return Ok(input);
+ }
+
+ let input_schema = input.schema();
+ let partition_type = build_partition_type(partition_spec,
table_schema.as_ref())?;
+ let calculator = PartitionValueCalculator::new(
+ partition_spec.as_ref().clone(),
+ table_schema.as_ref().clone(),
+ partition_type,
+ )?;
+
+ let mut projection_exprs: Vec<(Arc<dyn PhysicalExpr>, String)> =
+ Vec::with_capacity(input_schema.fields().len() + 1);
+
+ for (index, field) in input_schema.fields().iter().enumerate() {
+ let column_expr = Arc::new(Column::new(field.name(), index));
+ projection_exprs.push((column_expr, field.name().clone()));
+ }
+
+ let partition_expr = Arc::new(PartitionExpr::new(calculator));
+ projection_exprs.push((partition_expr,
PARTITION_VALUES_COLUMN.to_string()));
+
+ let projection = ProjectionExec::try_new(projection_exprs, input)?;
+ Ok(Arc::new(projection))
+}
+
+/// PhysicalExpr implementation for partition value calculation
+#[derive(Debug, Clone)]
+struct PartitionExpr {
+ calculator: Arc<Mutex<PartitionValueCalculator>>,
+}
+
+impl PartitionExpr {
+ fn new(calculator: PartitionValueCalculator) -> Self {
+ Self {
+ calculator: Arc::new(Mutex::new(calculator)),
+ }
+ }
+}
+
+// Manual PartialEq/Eq implementations for pointer-based equality
+// (two PartitionExpr are equal if they share the same calculator instance)
+impl PartialEq for PartitionExpr {
+ fn eq(&self, other: &Self) -> bool {
+ Arc::ptr_eq(&self.calculator, &other.calculator)
+ }
+}
+
+impl Eq for PartitionExpr {}
+
+impl PhysicalExpr for PartitionExpr {
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+
+ fn data_type(&self, _input_schema: &ArrowSchema) -> DFResult<DataType> {
+ let calculator = self
+ .calculator
+ .lock()
+ .map_err(|e| DataFusionError::Internal(format!("Failed to lock
calculator: {}", e)))?;
+ Ok(calculator.partition_type.clone())
+ }
+
+ fn nullable(&self, _input_schema: &ArrowSchema) -> DFResult<bool> {
+ Ok(false)
+ }
+
+ fn evaluate(&self, batch: &RecordBatch) -> DFResult<ColumnarValue> {
+ let calculator = self
+ .calculator
+ .lock()
+ .map_err(|e| DataFusionError::Internal(format!("Failed to lock
calculator: {}", e)))?;
+ let array = calculator.calculate(batch)?;
+ Ok(ColumnarValue::Array(array))
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
+ vec![]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ _children: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> DFResult<Arc<dyn PhysicalExpr>> {
+ Ok(self)
+ }
+
+ fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ if let Ok(calculator) = self.calculator.lock() {
+ let field_names: Vec<String> = calculator
+ .partition_spec
+ .fields()
+ .iter()
+ .map(|pf| format!("{}({})", pf.transform, pf.name))
+ .collect();
+ write!(f, "iceberg_partition_values[{}]", field_names.join(", "))
+ } else {
+ write!(f, "iceberg_partition_values")
+ }
+ }
+}
+
+impl std::fmt::Display for PartitionExpr {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ if let Ok(calculator) = self.calculator.lock() {
+ let field_names: Vec<&str> = calculator
+ .partition_spec
+ .fields()
+ .iter()
+ .map(|pf| pf.name.as_str())
+ .collect();
+ write!(f, "iceberg_partition_values({})", field_names.join(", "))
+ } else {
+ write!(f, "iceberg_partition_values")
+ }
+ }
+}
+
+impl std::hash::Hash for PartitionExpr {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ std::any::TypeId::of::<Self>().hash(state);
Review Comment:
This is a little odd, why not derive `Hash` for `PartitionValueCalculator`?
##########
crates/iceberg/src/arrow/mod.rs:
##########
@@ -28,7 +28,8 @@ pub mod delete_file_loader;
pub(crate) mod delete_filter;
mod reader;
-pub(crate) mod record_batch_projector;
+/// RecordBatch projection utilities
+pub mod record_batch_projector;
Review Comment:
Why do we need to make this pub?
##########
crates/integrations/datafusion/src/physical_plan/project.rs:
##########
@@ -0,0 +1,499 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Partition value projection for Iceberg tables.
+
+use std::sync::{Arc, Mutex};
+
+use datafusion::arrow::array::{ArrayRef, RecordBatch, StructArray};
+use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema};
+use datafusion::common::Result as DFResult;
+use datafusion::error::DataFusionError;
+use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_expr::expressions::Column;
+use datafusion::physical_plan::projection::ProjectionExec;
+use datafusion::physical_plan::{ColumnarValue, ExecutionPlan};
+use iceberg::arrow::record_batch_projector::RecordBatchProjector;
+use iceberg::spec::{PartitionSpec, Schema};
+use iceberg::table::Table;
+use iceberg::transform::BoxedTransformFunction;
+
+use crate::to_datafusion_error;
+
+/// Column name for the combined partition values struct
+const PARTITION_VALUES_COLUMN: &str = "_partition";
+
+/// Extends an ExecutionPlan with partition value calculations for Iceberg
tables.
+///
+/// This function takes an input ExecutionPlan and extends it with an
additional column
+/// containing calculated partition values based on the table's partition
specification.
+/// For unpartitioned tables, returns the original plan unchanged.
+///
+/// # Arguments
+/// * `input` - The input ExecutionPlan to extend
+/// * `table` - The Iceberg table with partition specification
+///
+/// # Returns
+/// * `Ok(Arc<dyn ExecutionPlan>)` - Extended plan with partition values column
+/// * `Err` - If partition spec is not found or transformation fails
+pub fn project_with_partition(
+ input: Arc<dyn ExecutionPlan>,
+ table: &Table,
+) -> DFResult<Arc<dyn ExecutionPlan>> {
+ let metadata = table.metadata();
+ let partition_spec = metadata.default_partition_spec();
+ let table_schema = metadata.current_schema();
+
+ if partition_spec.is_unpartitioned() {
+ return Ok(input);
+ }
+
+ let input_schema = input.schema();
+ let partition_type = build_partition_type(partition_spec,
table_schema.as_ref())?;
+ let calculator = PartitionValueCalculator::new(
Review Comment:
This is implicit assume that the `input_schema` exactly matches iceberg
table schema. I think this assumption is valid for now, but we should add a
check here to ensure that.
##########
crates/integrations/datafusion/src/physical_plan/project.rs:
##########
@@ -0,0 +1,499 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Partition value projection for Iceberg tables.
+
+use std::sync::{Arc, Mutex};
+
+use datafusion::arrow::array::{ArrayRef, RecordBatch, StructArray};
+use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema};
+use datafusion::common::Result as DFResult;
+use datafusion::error::DataFusionError;
+use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_expr::expressions::Column;
+use datafusion::physical_plan::projection::ProjectionExec;
+use datafusion::physical_plan::{ColumnarValue, ExecutionPlan};
+use iceberg::arrow::record_batch_projector::RecordBatchProjector;
+use iceberg::spec::{PartitionSpec, Schema};
+use iceberg::table::Table;
+use iceberg::transform::BoxedTransformFunction;
+
+use crate::to_datafusion_error;
+
+/// Column name for the combined partition values struct
+const PARTITION_VALUES_COLUMN: &str = "_partition";
+
+/// Extends an ExecutionPlan with partition value calculations for Iceberg
tables.
+///
+/// This function takes an input ExecutionPlan and extends it with an
additional column
+/// containing calculated partition values based on the table's partition
specification.
+/// For unpartitioned tables, returns the original plan unchanged.
+///
+/// # Arguments
+/// * `input` - The input ExecutionPlan to extend
+/// * `table` - The Iceberg table with partition specification
+///
+/// # Returns
+/// * `Ok(Arc<dyn ExecutionPlan>)` - Extended plan with partition values column
+/// * `Err` - If partition spec is not found or transformation fails
+pub fn project_with_partition(
+ input: Arc<dyn ExecutionPlan>,
+ table: &Table,
+) -> DFResult<Arc<dyn ExecutionPlan>> {
+ let metadata = table.metadata();
+ let partition_spec = metadata.default_partition_spec();
+ let table_schema = metadata.current_schema();
+
+ if partition_spec.is_unpartitioned() {
+ return Ok(input);
+ }
+
+ let input_schema = input.schema();
+ let partition_type = build_partition_type(partition_spec,
table_schema.as_ref())?;
+ let calculator = PartitionValueCalculator::new(
+ partition_spec.as_ref().clone(),
+ table_schema.as_ref().clone(),
+ partition_type,
+ )?;
+
+ let mut projection_exprs: Vec<(Arc<dyn PhysicalExpr>, String)> =
+ Vec::with_capacity(input_schema.fields().len() + 1);
+
+ for (index, field) in input_schema.fields().iter().enumerate() {
+ let column_expr = Arc::new(Column::new(field.name(), index));
+ projection_exprs.push((column_expr, field.name().clone()));
+ }
+
+ let partition_expr = Arc::new(PartitionExpr::new(calculator));
+ projection_exprs.push((partition_expr,
PARTITION_VALUES_COLUMN.to_string()));
+
+ let projection = ProjectionExec::try_new(projection_exprs, input)?;
+ Ok(Arc::new(projection))
+}
+
+/// PhysicalExpr implementation for partition value calculation
+#[derive(Debug, Clone)]
+struct PartitionExpr {
+ calculator: Arc<Mutex<PartitionValueCalculator>>,
Review Comment:
We no longer need this lock?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
