When kernel-noise isolation is requested for a CPU at runtime via a
cpuset isolated partition, tick suppression must be activated on the
affected CPU without requiring nohz_full= at boot.
tick_nohz_full_mask and tick_nohz_full_running are currently set only
during boot by tick_nohz_full_setup(). There is no runtime path to add
or remove CPUs from the full-dynticks set.
Add tick_nohz_cpu_isolate() and tick_nohz_cpu_deisolate(), both called
with the target CPU offline (between remove_cpu and add_cpu in the
hotplug cycling path):
tick_nohz_cpu_isolate(cpu) - lazily allocates tick_nohz_full_mask
if it was never set up at boot, sets the CPU's bit in the mask,
sets tick_nohz_full_running, and calls ct_cpu_track_user() to
activate per-CPU context tracking so kernel/user transitions
suppress the scheduler tick.
tick_nohz_cpu_deisolate(cpu) - reverses the above: deactivates
context tracking via ct_cpu_untrack_user(), clears the CPU's bit,
and clears tick_nohz_full_running when the mask becomes empty.
A per-function mutex guards the lazy allocation and running flag
updates against concurrent isolation requests.
Co-developed-by: Qiliang Yuan <[email protected]>
Signed-off-by: Qiliang Yuan <[email protected]>
Signed-off-by: Jing Wu <[email protected]>
---
include/linux/tick.h | 4 ++++
kernel/time/tick-sched.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 49 insertions(+)
diff --git a/include/linux/tick.h b/include/linux/tick.h
index 738007d6f577f..c4038e62533ba 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -210,6 +210,8 @@ extern void tick_nohz_dep_set_signal(struct task_struct
*tsk,
extern void tick_nohz_dep_clear_signal(struct signal_struct *signal,
enum tick_dep_bits bit);
extern bool tick_nohz_cpu_hotpluggable(unsigned int cpu);
+extern int tick_nohz_cpu_isolate(int cpu);
+extern void tick_nohz_cpu_deisolate(int cpu);
/*
* The below are tick_nohz_[set,clear]_dep() wrappers that optimize off-cases
@@ -281,6 +283,8 @@ static inline bool tick_nohz_full_cpu(int cpu) { return
false; }
static inline void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit) { }
static inline void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit) { }
static inline bool tick_nohz_cpu_hotpluggable(unsigned int cpu) { return true;
}
+static inline int tick_nohz_cpu_isolate(int cpu) { return -EINVAL; }
+static inline void tick_nohz_cpu_deisolate(int cpu) { }
static inline void tick_dep_set(enum tick_dep_bits bit) { }
static inline void tick_dep_clear(enum tick_dep_bits bit) { }
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index ba7adc671c580..1bc01e2ab525b 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -621,6 +621,51 @@ void __tick_nohz_task_switch(void)
}
}
+static DEFINE_MUTEX(tick_nohz_cpu_isolate_mutex);
+
+/*
+ * tick_nohz_cpu_isolate - Add a CPU to the full-dynticks set at runtime.
+ * @cpu: the CPU to isolate; must be offline.
+ *
+ * Lazily allocates tick_nohz_full_mask on the first call so that no
+ * nohz_full= boot parameter is required. Activates per-CPU context
+ * tracking so that kernel/user transitions suppress the scheduler tick.
+ */
+int tick_nohz_cpu_isolate(int cpu)
+{
+ mutex_lock(&tick_nohz_cpu_isolate_mutex);
+ if (!cpumask_available(tick_nohz_full_mask)) {
+ if (!zalloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) {
+ mutex_unlock(&tick_nohz_cpu_isolate_mutex);
+ return -ENOMEM;
+ }
+ }
+ cpumask_set_cpu(cpu, tick_nohz_full_mask);
+ if (!tick_nohz_full_running)
+ tick_nohz_full_running = true;
+ mutex_unlock(&tick_nohz_cpu_isolate_mutex);
+
+ ct_cpu_track_user(cpu);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(tick_nohz_cpu_isolate);
+
+/*
+ * tick_nohz_cpu_deisolate - Remove a CPU from the full-dynticks set.
+ * @cpu: the CPU to de-isolate; must be offline.
+ */
+void tick_nohz_cpu_deisolate(int cpu)
+{
+ ct_cpu_untrack_user(cpu);
+
+ mutex_lock(&tick_nohz_cpu_isolate_mutex);
+ cpumask_clear_cpu(cpu, tick_nohz_full_mask);
+ if (cpumask_empty(tick_nohz_full_mask))
+ tick_nohz_full_running = false;
+ mutex_unlock(&tick_nohz_cpu_isolate_mutex);
+}
+EXPORT_SYMBOL_GPL(tick_nohz_cpu_deisolate);
+
/* Get the boot-time nohz CPU list from the kernel parameters. */
void __init tick_nohz_full_setup(cpumask_var_t cpumask)
{
--
2.43.0
