gstvg commented on code in PR #21193:
URL: https://github.com/apache/datafusion/pull/21193#discussion_r3208846802
##########
datafusion/substrait/src/logical_plan/producer/substrait_producer.rs:
##########
@@ -471,4 +551,109 @@ impl SubstraitProducer for DefaultSubstraitProducer<'_> {
rel_type: Some(rel_type),
}))
}
+
+ fn push_lambda_parameters(
+ &mut self,
+ lambda_parameters: Vec<FieldRef>,
+ ) -> datafusion::common::Result<()> {
+ let lambda_parameters_map = lambda_parameters_map(self,
lambda_parameters)?;
+
+ self.lambda_producer
+ .push_lambda_parameters(lambda_parameters_map);
+
+ Ok(())
+ }
+
+ fn pop_lambda_parameters(&mut self) -> datafusion::common::Result<()> {
+ self.lambda_producer.pop_lambda_parameters()
+ }
+
+ fn lambda_variable(&self, name: &str) -> datafusion::common::Result<(u32,
i32)> {
+ self.lambda_producer.lambda_variable(name)
+ }
+
+ fn lambda_parameter_type(
+ &self,
+ name: &str,
+ ) -> datafusion::common::Result<substrait::proto::Type> {
+ self.lambda_producer.lambda_parameter_type(name)
+ }
+}
+
+/// Default implementation of lambda related methods of the
[SubstraitProducer] trait
+///
+/// Can be embedded into a custom [SubstraitProducer] to implement them
+pub struct DefaultSubstraitLambdaProducer {
Review Comment:
Same reason for the consumer:
```rust
struct CustomSubstraitProducer {
extensions: Extensions,
state: Arc<SessionState>,
// You can reuse existing producer code related to lambdas
lambda_producer: DefaultSubstraitLambdaProducer,
}
impl SubstraitProducer for CustomSubstraitProducer {
fn register_function(&mut self, signature: String) -> u32 {
self.extensions.register_function(&signature)
}
fn register_type(&mut self, type_name: String) -> u32 {
self.extensions.register_type(&type_name)
}
fn get_extensions(self) -> Extensions {
self.extensions
}
fn push_lambda_parameters(
&mut self,
lambda_parameters: Vec<FieldRef>,
) -> datafusion::common::Result<()> {
let lambda_parameters_map = lambda_parameters_map(self,
lambda_parameters)?;
self.lambda_producer
.push_lambda_parameters(lambda_parameters_map);
Ok(())
}
fn pop_lambda_parameters(&mut self) -> datafusion::common::Result<()> {
self.lambda_producer.pop_lambda_parameters()
}
fn lambda_variable(&self, name: &str) ->
datafusion::common::Result<(u32, i32)> {
self.lambda_producer.lambda_variable(name)
}
fn lambda_parameter_type(
&self,
name: &str,
) -> datafusion::common::Result<substrait::proto::Type> {
self.lambda_producer.lambda_parameter_type(name)
}
}
```
##########
datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs:
##########
@@ -35,7 +38,68 @@ pub fn from_higher_order_function(
fun: &expr::HigherOrderFunction,
schema: &DFSchemaRef,
) -> datafusion::common::Result<Expression> {
- from_function(producer, fun.name(), &fun.args, schema)
+ let mut lambda_parameters = fun.lambda_parameters(schema)?.into_iter();
+
+ let num_lambdas = fun
+ .args
+ .iter()
+ .filter(|arg| matches!(arg, Expr::Lambda(_)))
+ .count();
+
+ if lambda_parameters.len() != num_lambdas {
+ return substrait_err!(
+ "{} returned {} lambdas but {num_lambdas} expected",
+ fun.name(),
+ lambda_parameters.len()
+ );
+ }
+
+ let arguments = fun
+ .args
+ .iter()
+ .map(|arg| {
+ let arg = match arg {
+ Expr::Lambda(l) => {
+ let lambda_parameters =
+ lambda_parameters.next().ok_or_else(|| {
+ internal_datafusion_err!(
+ "lambda_parameters len should have been
checked above"
+ )
+ })?;
+
+ let named_lambda_parameters =
+ std::iter::zip(&l.params, lambda_parameters)
+ .map(|(name, parameter)| parameter.renamed(name))
+ .collect();
+
+ producer.push_lambda_parameters(named_lambda_parameters)?;
+
+ let arg = producer.handle_expr(arg, schema)?;
Review Comment:
also at
https://github.com/apache/datafusion/pull/21193/changes/9b6a8e1f18aadd976aaae9af40d7c7ca12095c96
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
