When a cpuset isolated partition first requests kernel-noise isolation,
it needs to offload RCU callbacks from the affected CPUs. The existing
rcu_nocb_cpu_offload() requires rcu_nocbs= or nohz_full= at boot so
that rcu_organize_nocb_kthreads() has already set rdp->nocb_gp_rdp for
each CPU. Without a boot parameter, nocb_gp_rdp is NULL and offload
fails immediately.
Introduce lazy nocb initialization so that the first call into the
isolation path triggers the one-time setup automatically:
rcu_nocb_lazy_init() - allocates rcu_nocb_mask and calls
rcu_organize_nocb_kthreads() (now without __init) to set
nocb_gp_rdp for every possible CPU. Uses a dedicated mutex
for serialization with a fast-path read of nocb_is_setup.
rcu_nocb_cpu_isolate() - exported entry point called per-CPU
while the CPU is offline. Calls rcu_nocb_lazy_init() for the
one-time setup, spawns the GP and CB kthreads via the existing
rcu_spawn_cpu_nocb_kthread(), then finalizes offload through
rcu_nocb_cpu_offload(). Adding the CPU to the GP kthread's
nocb_head_rdp list is handled by nocb_gp_toggle_rdp() in the
GP kthread, so rcu_organize_nocb_kthreads() can run with an
empty mask.
Remove __init from rcu_organize_nocb_kthreads() to allow this
runtime call path; the function itself has no __initdata dependencies.
Co-developed-by: Qiliang Yuan <[email protected]>
Signed-off-by: Qiliang Yuan <[email protected]>
Signed-off-by: Jing Wu <[email protected]>
---
include/linux/rcupdate.h | 2 ++
kernel/rcu/tree.h | 2 +-
kernel/rcu/tree_nocb.h | 43 ++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index bfa765132de85..5937485d59118 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -149,6 +149,7 @@ static __always_inline void rcu_irq_work_resched(void) { }
void rcu_init_nohz(void);
int rcu_nocb_cpu_offload(int cpu);
int rcu_nocb_cpu_deoffload(int cpu);
+int rcu_nocb_cpu_isolate(int cpu);
void rcu_nocb_flush_deferred_wakeup(void);
#define RCU_NOCB_LOCKDEP_WARN(c, s) RCU_LOCKDEP_WARN(c, s)
@@ -158,6 +159,7 @@ void rcu_nocb_flush_deferred_wakeup(void);
static inline void rcu_init_nohz(void) { }
static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
+static inline int rcu_nocb_cpu_isolate(int cpu) { return -EINVAL; }
static inline void rcu_nocb_flush_deferred_wakeup(void) { }
#define RCU_NOCB_LOCKDEP_WARN(c, s)
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 7dfc57e9adb18..f3d31918ea322 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -517,7 +517,7 @@ static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
unsigned long flags);
static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp);
#ifdef CONFIG_RCU_NOCB_CPU
-static void __init rcu_organize_nocb_kthreads(void);
+static void rcu_organize_nocb_kthreads(void);
/*
* Disable IRQs before checking offloaded state so that local
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index 1047b30cd46b7..694fd615f1809 100644
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -1347,6 +1347,47 @@ void __init rcu_init_nohz(void)
rcu_organize_nocb_kthreads();
}
+static DEFINE_MUTEX(rcu_nocb_lazy_mutex);
+
+/*
+ * Lazily initialize nocb infrastructure on the first call. Allocates
+ * rcu_nocb_mask and sets nocb_gp_rdp for every possible CPU so that
+ * rcu_nocb_cpu_isolate() can offload callbacks without rcu_nocbs= at boot.
+ */
+static noinline int rcu_nocb_lazy_init(void)
+{
+ if (rcu_state.nocb_is_setup)
+ return 0;
+
+ mutex_lock(&rcu_nocb_lazy_mutex);
+ if (!rcu_state.nocb_is_setup) {
+ if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
+ mutex_unlock(&rcu_nocb_lazy_mutex);
+ return -ENOMEM;
+ }
+ rcu_organize_nocb_kthreads();
+ rcu_state.nocb_is_setup = true;
+ }
+ mutex_unlock(&rcu_nocb_lazy_mutex);
+ return 0;
+}
+
+/*
+ * Offload RCU callbacks for a CPU entering a kernel-noise isolated partition.
+ * @cpu must be offline. Lazily initializes nocb infrastructure on first use.
+ */
+int rcu_nocb_cpu_isolate(int cpu)
+{
+ int ret;
+
+ ret = rcu_nocb_lazy_init();
+ if (ret)
+ return ret;
+ rcu_spawn_cpu_nocb_kthread(cpu);
+ return rcu_nocb_cpu_offload(cpu);
+}
+EXPORT_SYMBOL_GPL(rcu_nocb_cpu_isolate);
+
/* Initialize per-rcu_data variables for no-CBs CPUs. */
static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
{
@@ -1439,7 +1480,7 @@ module_param(rcu_nocb_gp_stride, int, 0444);
/*
* Initialize GP-CB relationships for all no-CBs CPU.
*/
-static void __init rcu_organize_nocb_kthreads(void)
+static void rcu_organize_nocb_kthreads(void)
{
int cpu;
bool firsttime = true;
--
2.43.0
