liurenjie1024 commented on code in PR #1602: URL: https://github.com/apache/iceberg-rust/pull/1602#discussion_r2431995151
########## crates/integrations/datafusion/src/physical_plan/project.rs: ########## @@ -0,0 +1,557 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Partition value projection for Iceberg tables. + +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, RecordBatch, StructArray}; +use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema}; +use datafusion::common::Result as DFResult; +use datafusion::error::DataFusionError; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr::expressions::Column; +use datafusion::physical_plan::projection::ProjectionExec; +use datafusion::physical_plan::{ColumnarValue, ExecutionPlan}; +use iceberg::arrow::record_batch_projector::RecordBatchProjector; +use iceberg::arrow::schema_to_arrow_schema; +use iceberg::spec::{PartitionSpec, Schema}; +use iceberg::table::Table; +use iceberg::transform::BoxedTransformFunction; + +use crate::to_datafusion_error; + +/// Column name for the combined partition values struct +const PARTITION_VALUES_COLUMN: &str = "_partition"; + +/// Extends an ExecutionPlan with partition value calculations for Iceberg tables. +/// +/// This function takes an input ExecutionPlan and extends it with an additional column +/// containing calculated partition values based on the table's partition specification. +/// For unpartitioned tables, returns the original plan unchanged. +/// +/// # Arguments +/// * `input` - The input ExecutionPlan to extend +/// * `table` - The Iceberg table with partition specification +/// +/// # Returns +/// * `Ok(Arc<dyn ExecutionPlan>)` - Extended plan with partition values column +/// * `Err` - If partition spec is not found or transformation fails +pub fn project_with_partition( + input: Arc<dyn ExecutionPlan>, + table: &Table, +) -> DFResult<Arc<dyn ExecutionPlan>> { + let metadata = table.metadata(); + let partition_spec = metadata.default_partition_spec(); + let table_schema = metadata.current_schema(); + + if partition_spec.is_unpartitioned() { + return Ok(input); + } + + let input_schema = input.schema(); + + // Validate that input schema matches the table schema + validate_schema_compatibility(&input_schema, table_schema.as_ref())?; + + let partition_type = build_partition_type(partition_spec, table_schema.as_ref())?; + let calculator = PartitionValueCalculator::new( + partition_spec.as_ref().clone(), + table_schema.as_ref().clone(), + partition_type, + )?; + + let mut projection_exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = + Vec::with_capacity(input_schema.fields().len() + 1); + + for (index, field) in input_schema.fields().iter().enumerate() { + let column_expr = Arc::new(Column::new(field.name(), index)); + projection_exprs.push((column_expr, field.name().clone())); + } + + let partition_expr = Arc::new(PartitionExpr::new(calculator)); + projection_exprs.push((partition_expr, PARTITION_VALUES_COLUMN.to_string())); + + let projection = ProjectionExec::try_new(projection_exprs, input)?; + Ok(Arc::new(projection)) +} + +/// PhysicalExpr implementation for partition value calculation +#[derive(Debug, Clone)] +struct PartitionExpr { + calculator: Arc<PartitionValueCalculator>, +} + +impl PartitionExpr { + fn new(calculator: PartitionValueCalculator) -> Self { + Self { + calculator: Arc::new(calculator), + } + } +} + +// Manual PartialEq/Eq implementations for pointer-based equality +// (two PartitionExpr are equal if they share the same calculator instance) +impl PartialEq for PartitionExpr { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.calculator, &other.calculator) + } +} + +impl Eq for PartitionExpr {} + +impl PhysicalExpr for PartitionExpr { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn data_type(&self, _input_schema: &ArrowSchema) -> DFResult<DataType> { + Ok(self.calculator.partition_type.clone()) + } + + fn nullable(&self, _input_schema: &ArrowSchema) -> DFResult<bool> { + Ok(false) + } + + fn evaluate(&self, batch: &RecordBatch) -> DFResult<ColumnarValue> { + let array = self.calculator.calculate(batch)?; + Ok(ColumnarValue::Array(array)) + } + + fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> { + vec![] + } + + fn with_new_children( + self: Arc<Self>, + _children: Vec<Arc<dyn PhysicalExpr>>, + ) -> DFResult<Arc<dyn PhysicalExpr>> { + Ok(self) + } + + fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let field_names: Vec<String> = self + .calculator + .partition_spec + .fields() + .iter() + .map(|pf| format!("{}({})", pf.transform, pf.name)) + .collect(); + write!(f, "iceberg_partition_values[{}]", field_names.join(", ")) + } +} + +impl std::fmt::Display for PartitionExpr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let field_names: Vec<&str> = self + .calculator + .partition_spec + .fields() + .iter() + .map(|pf| pf.name.as_str()) + .collect(); + write!(f, "iceberg_partition_values({})", field_names.join(", ")) + } +} + +impl std::hash::Hash for PartitionExpr { + fn hash<H: std::hash::Hasher>(&self, state: &mut H) { + // Two PartitionExpr are equal if they share the same calculator Arc + Arc::as_ptr(&self.calculator).hash(state); + } +} + +/// Calculator for partition values in Iceberg tables +#[derive(Debug)] +struct PartitionValueCalculator { + partition_spec: PartitionSpec, + partition_type: DataType, + projector: RecordBatchProjector, + transform_functions: Vec<BoxedTransformFunction>, +} + +impl PartitionValueCalculator { + fn new( + partition_spec: PartitionSpec, + table_schema: Schema, + partition_type: DataType, + ) -> DFResult<Self> { + if partition_spec.is_unpartitioned() { + return Err(DataFusionError::Internal( + "Cannot create partition calculator for unpartitioned table".to_string(), + )); + } + + let transform_functions: Result<Vec<BoxedTransformFunction>, _> = partition_spec + .fields() + .iter() + .map(|pf| iceberg::transform::create_transform_function(&pf.transform)) + .collect(); + + let transform_functions = transform_functions.map_err(to_datafusion_error)?; + + let source_field_ids: Vec<i32> = partition_spec + .fields() + .iter() + .map(|pf| pf.source_id) + .collect(); + + let projector = RecordBatchProjector::from_iceberg_schema( + Arc::new(table_schema.clone()), + &source_field_ids, + ) + .map_err(to_datafusion_error)?; + + Ok(Self { + partition_spec, + partition_type, + projector, + transform_functions, + }) + } + + fn calculate(&self, batch: &RecordBatch) -> DFResult<ArrayRef> { + let source_columns = self + .projector + .project_column(batch.columns()) + .map_err(to_datafusion_error)?; + + let expected_struct_fields = match &self.partition_type { + DataType::Struct(fields) => fields.clone(), + _ => { + return Err(DataFusionError::Internal( + "Expected partition type must be a struct".to_string(), + )); + } + }; + + let mut partition_values = Vec::with_capacity(self.partition_spec.fields().len()); + + for (source_column, transform_fn) in source_columns.iter().zip(&self.transform_functions) { + let partition_value = transform_fn + .transform(source_column.clone()) + .map_err(to_datafusion_error)?; + + partition_values.push(partition_value); + } + + let struct_array = StructArray::try_new(expected_struct_fields, partition_values, None) + .map_err(|e| DataFusionError::ArrowError(e, None))?; + + Ok(Arc::new(struct_array)) + } +} + +/// Validates that the input Arrow schema is compatible with the Iceberg table schema. +/// +/// This ensures that: +/// - All fields in the input schema have matching names in the table schema +/// - The Arrow data types are compatible with the corresponding Iceberg types +fn validate_schema_compatibility( Review Comment: There are several problems with this implementation: 1. It only checks that arrow schema is a subset of iceberg schema, but we need to ensure that they are exactly same. 2. It didn't check order of fields. 3. It didn't take the metadata of nested data type into account. I would suggest another implementation: 1. Create a function to visit arrow schema recursively to remove all metadata of all fields, including nested fields. 2. Remove metadata of input arrow schema and converted arrow schema, compare them using built in `==` of arrow fields. ########## crates/integrations/datafusion/src/physical_plan/project.rs: ########## @@ -0,0 +1,557 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Partition value projection for Iceberg tables. + +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, RecordBatch, StructArray}; +use datafusion::arrow::datatypes::{DataType, Schema as ArrowSchema}; +use datafusion::common::Result as DFResult; +use datafusion::error::DataFusionError; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr::expressions::Column; +use datafusion::physical_plan::projection::ProjectionExec; +use datafusion::physical_plan::{ColumnarValue, ExecutionPlan}; +use iceberg::arrow::record_batch_projector::RecordBatchProjector; +use iceberg::arrow::schema_to_arrow_schema; +use iceberg::spec::{PartitionSpec, Schema}; +use iceberg::table::Table; +use iceberg::transform::BoxedTransformFunction; + +use crate::to_datafusion_error; + +/// Column name for the combined partition values struct +const PARTITION_VALUES_COLUMN: &str = "_partition"; + +/// Extends an ExecutionPlan with partition value calculations for Iceberg tables. +/// +/// This function takes an input ExecutionPlan and extends it with an additional column +/// containing calculated partition values based on the table's partition specification. +/// For unpartitioned tables, returns the original plan unchanged. +/// +/// # Arguments +/// * `input` - The input ExecutionPlan to extend +/// * `table` - The Iceberg table with partition specification +/// +/// # Returns +/// * `Ok(Arc<dyn ExecutionPlan>)` - Extended plan with partition values column +/// * `Err` - If partition spec is not found or transformation fails +pub fn project_with_partition( + input: Arc<dyn ExecutionPlan>, + table: &Table, +) -> DFResult<Arc<dyn ExecutionPlan>> { + let metadata = table.metadata(); + let partition_spec = metadata.default_partition_spec(); + let table_schema = metadata.current_schema(); + + if partition_spec.is_unpartitioned() { + return Ok(input); + } + + let input_schema = input.schema(); + + // Validate that input schema matches the table schema + validate_schema_compatibility(&input_schema, table_schema.as_ref())?; + + let partition_type = build_partition_type(partition_spec, table_schema.as_ref())?; + let calculator = PartitionValueCalculator::new( + partition_spec.as_ref().clone(), + table_schema.as_ref().clone(), + partition_type, + )?; + + let mut projection_exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = + Vec::with_capacity(input_schema.fields().len() + 1); + + for (index, field) in input_schema.fields().iter().enumerate() { + let column_expr = Arc::new(Column::new(field.name(), index)); + projection_exprs.push((column_expr, field.name().clone())); + } + + let partition_expr = Arc::new(PartitionExpr::new(calculator)); + projection_exprs.push((partition_expr, PARTITION_VALUES_COLUMN.to_string())); + + let projection = ProjectionExec::try_new(projection_exprs, input)?; + Ok(Arc::new(projection)) +} + +/// PhysicalExpr implementation for partition value calculation +#[derive(Debug, Clone)] +struct PartitionExpr { + calculator: Arc<PartitionValueCalculator>, Review Comment: 1. Do we still need to `PartitionValueCalculator` struct? 2. Why we need this `Arc`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
