milenkovicm commented on code in PR #1995: URL: https://github.com/apache/datafusion-ballista/pull/1995#discussion_r3564776260
########## ballista/executor/src/runtime_cache.rs: ########## @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Session-scoped reuse of executor runtime state. + +use std::num::NonZeroUsize; +use std::sync::Arc; + +use ballista_core::RuntimeProducer; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::prelude::SessionConfig; +use lru::LruCache; +use parking_lot::Mutex; + +/// Derives a task's runtime from a shared base [`RuntimeEnv`] by installing a +/// fresh per-task memory pool. The base's object-store registry and the cache +/// manager's underlying file-metadata (footer) cache are preserved via +/// [`RuntimeEnvBuilder::from_runtime_env`](datafusion::execution::runtime_env::RuntimeEnvBuilder::from_runtime_env) +/// — the outer `CacheManager` is rebuilt around that same inner cache — so +/// read-side state is shared while the memory pool stays per task. +pub type MemoryPoolPolicy = Arc< + dyn Fn(Arc<RuntimeEnv>, &SessionConfig) -> datafusion::error::Result<Arc<RuntimeEnv>> + + Send + + Sync, +>; + +/// A bounded, session-keyed cache of shared *base* [`RuntimeEnv`]s. +/// +/// A base env carries the read-side state safe to share across all tasks of a +/// session: the object-store registry and the cache manager (whose Parquet +/// footer cache is thereby reused across the session's tasks and queries). +/// `RuntimeEnvBuilder::from_runtime_env` also carries over the disk manager +/// (rooted at the executor's `work_dir`), so a session's tasks share one +/// `DiskManager` too; this is safe because spill temp files are uniquely +/// named, matching the standard one-`RuntimeEnv`-per-`SessionContext` model. +/// Each task's real runtime is produced by applying [`MemoryPoolPolicy`] to +/// the shared base, which installs a fresh per-task memory pool — so memory +/// isolation is unchanged. +/// +/// The cache is bounded by an LRU of `capacity` sessions. A capacity of `0` +/// disables caching entirely: every call builds a fresh base env, matching the +/// behavior of building a runtime per task. +pub struct SessionRuntimeCache { Review Comment: would it make sense to make `SessionRuntimeCache` a trait and provide `DefaultSessionRuntimeCache`? would users ever want to change this implementation ? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
