milenkovicm commented on code in PR #2588: URL: https://github.com/apache/datafusion-ballista/pull/2588#discussion_r4304460152
########## python/src/ffi_planner_workaround.rs: ########## @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Workaround for missing file-format support in `datafusion-ffi` 55.2.0. +//! +//! datafusion-python replaces the codecs on an installed [`FFI_QueryPlanner`] +//! with its own, so Ballista decodes every logical plan through a +//! `ForeignLogicalExtensionCodec`. That type's `try_decode_file_format` is a +//! hard-coded `not_impl_err!`, which breaks any plan carrying a +//! `FileFormatFactory` (`LogicalPlan::Copy`, i.e. `DataFrame.write_*`). +//! +//! The file formats in those plans are written by the session's terminal +//! `DefaultLogicalExtensionCodec` without a chain envelope, so they can be +//! decoded locally. We replace the planner's `create_physical_plan` entry +//! with one that builds a temporary planner handle whose logical codec is a +//! local [`FileFormatFallbackCodec`] around the session's codec, and forwards +//! the call to `datafusion-ffi`'s own entry on that handle. The `clone` entry +//! is replaced too, so the override survives the clone datafusion-python +//! makes when it swaps codecs. +//! +//! This relies on the private field layouts of `FFI_QueryPlanner` and +//! `FFI_SessionRef`, mirrored below. It is only valid for the pinned +//! `datafusion-ffi = "=55.2.0"`. +//! +//! TODO: remove once `datafusion-ffi` delegates file formats to +//! `DefaultLogicalExtensionCodec` in `ForeignLogicalExtensionCodec`. + +use async_ffi::FfiFuture; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::TableProvider; +use datafusion::common::{Result, TableReference}; +use datafusion::datasource::file_format::FileFormatFactory; +use datafusion::execution::context::{QueryPlanner, SessionContext}; +use datafusion::execution::{TaskContext, TaskContextProvider}; +use datafusion::logical_expr::{ + AggregateUDF, Extension, HigherOrderUDF, LogicalPlan, ScalarUDF, WindowUDF, +}; +use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; +use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; +use datafusion_ffi::query_planner::FFI_QueryPlanner; +use datafusion_ffi::util::FFI_Result; +use datafusion_proto::logical_plan::{ + DefaultLogicalExtensionCodec, LogicalExtensionCodec, +}; +use stabby::vec::Vec as SVec; +use std::ffi::c_void; +use std::mem::MaybeUninit; +use std::sync::{Arc, OnceLock}; + +/// Same size and alignment as `datafusion_ffi::session::FFI_SessionRef` +/// (55.2.0), which is `pub(crate)`. It is only passed through by value, so +/// ownership moves to the callee and nothing here drops it. The struct is far +/// larger than two registers, so every supported C ABI passes it in memory +/// regardless of field types. +#[repr(C)] +struct FfiSessionRefBytes { Review Comment: there is no issue filled in. i have added comment in cargo.toml with some details explaining situation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
