Adds the thread_set_affinity and thread_get_affinity RPCs.
The user facing API uses the processor_name_array_t data type for
consistency with the rest of GnuMach. The server side stores
affinities as bitmasks to allow for performant code in the
scheduler.
thread_set_affinity refuses an affinity mask that would end up
being entirely disabled.
The code touches CPU shutdown and reassignment: CPU removal is
refused if there is still some thread with an affinity mask that
would be entirely disabled.
---
include/mach/gnumach.defs | 8 ++
kern/machine.c | 77 +++++++++++++++++
kern/thread.c | 168 ++++++++++++++++++++++++++++++++++++++
kern/thread.h | 56 +++++++++++++
4 files changed, 309 insertions(+)
diff --git a/include/mach/gnumach.defs b/include/mach/gnumach.defs
index f5b2f7f2..9f98b47b 100644
--- a/include/mach/gnumach.defs
+++ b/include/mach/gnumach.defs
@@ -257,3 +257,11 @@ routine vm_get_size_limit(
map : vm_task_t;
out current_limit : vm_size_t;
out max_limit : vm_size_t);
+
+routine thread_set_affinity(
+ thread : thread_t;
+ processors_list : processor_name_array_t);
+
+routine thread_get_affinity(
+ thread : thread_t;
+ out processors_list : processor_name_array_t);
diff --git a/kern/machine.c b/kern/machine.c
index f1380367..e730434d 100644
--- a/kern/machine.c
+++ b/kern/machine.c
@@ -233,6 +233,64 @@ processor_request_action(
thread_wakeup((event_t)&action_queue);
}
+
+/*
+ * thread_would_hang() helper function for thread_shutdown and
+ * thread_assign operations.
+ *
+ * Refuse shutdown if it would strand any affinity-constrained thread in
+ * this processor set.
+ *
+ * Walk the pset thread queue and, for each thread whose affinity mask
+ * currently admits the processor being shut down, verify that some other
+ * processor in the same pset also satisfies that mask.
+ *
+ * If no alternate eligible processor exists, shutting this processor down
+ * would leave that thread without any runnable CPU in the pset, so abort
+ * the operation.
+ */
+static
+thread_t
+thread_would_hang(
+ processor_t processor)
+{
+ processor_set_t pset;
+ thread_t thread;
+ processor_t p;
+
+ pset = processor->processor_set;
+ pset_lock(pset);
+
+ queue_iterate(&pset->threads, thread, thread_t, pset_threads) {
+ thread_lock(thread);
+
+ if (thread->has_affinity &&
+ cpu_affinity_test(&thread->affinity, processor->slot_num)) {
+
+ queue_iterate(&pset->processors, p, processor_t, processors) {
+ if (p == processor)
+ continue;
+
+ if (cpu_affinity_test(&thread->affinity, p->slot_num)) {
+ goto thread_can_run;
+ }
+ }
+
+ thread_unlock(thread);
+ pset_unlock(pset);
+
+ return thread;
+ }
+
+thread_can_run:
+ thread_unlock(thread);
+ }
+
+ pset_unlock(pset);
+
+ return THREAD_NULL;
+}
+
#if MACH_HOST
/*
* processor_assign() changes the processor set that a processor is
@@ -256,6 +314,14 @@ processor_assign(
return(KERN_INVALID_ARGUMENT);
}
+ /*
+ * Refuse assignment if it would strand any affinity-constrained thread in
+ * this processor set.
+ *
+ */
+ if(thread_would_hang(processor) != THREAD_NULL)
+ return(KERN_FAILURE);
+
/*
* Get pset reference to donate to processor_request_action.
*/
@@ -362,6 +428,17 @@ processor_shutdown(processor_t processor)
return(KERN_SUCCESS);
}
+ /*
+ * Refuse shutdown if it would strand any affinity-constrained thread in
+ * this processor set.
+ *
+ */
+ if(thread_would_hang(processor) != THREAD_NULL) {
+ processor_unlock(processor);
+ splx(s);
+ return(KERN_FAILURE);
+ }
+
processor_request_action(processor, PROCESSOR_SET_NULL);
processor_unlock(processor);
splx(s);
diff --git a/kern/thread.c b/kern/thread.c
index deb9688d..f00304b2 100644
--- a/kern/thread.c
+++ b/kern/thread.c
@@ -367,6 +367,8 @@ void thread_init(void)
/* thread_template.processor_set (later) */
thread_template.bound_processor = PROCESSOR_NULL;
+ /* thread_template.has_affinity = FALSE; */
+ /* cpu_affinity_zero(&thread_template.affinity); */
#if MACH_HOST
thread_template.may_assign = TRUE;
thread_template.assign_active = FALSE;
@@ -451,6 +453,11 @@ kern_return_t thread_create(
pset_reference(pset);
task_unlock(parent_task);
+ /*
+ * Reset affinity
+ */
+ new_thread->has_affinity = FALSE;
+
/*
* This thread will mosty probably start working, assume it
* will take its share of CPU, to avoid having to find it out
@@ -2672,3 +2679,164 @@ thread_get_name(
return KERN_SUCCESS;
}
+
+/*
+ * thread_enforce_affinity
+ *
+ * Enforce the current thread affinity constraint.
+ *
+ * If the thread is active on a processor outside its allowed affinity set,
+ * trigger an AST on that processor so the thread is preempted and
+ * rescheduled onto a permitted CPU.
+ *
+ * Assumes the thread is locked and splsched is active.
+ */
+void
+thread_enforce_affinity(
+ thread_t thread)
+{
+ int i;
+ processor_t p;
+#if NCPUS > 1
+ if (thread->has_affinity == FALSE)
+ return;
+
+ for (i = 0; i < smp_get_numcpus(); i++) {
+ p = cpu_to_processor(i);
+
+ if (p == PROCESSOR_NULL || p->state == PROCESSOR_OFF_LINE)
+ continue;
+
+ if (percpu_array[i].active_thread == thread) {
+ if (!cpu_affinity_test(&thread->affinity, i))
+ cause_ast_check(p);
+ break;
+ }
+ }
+#endif
+}
+
+
+/*
+ *
+ * thread_set_affinity
+ *
+ * Set thread affinity, the procs array contains the target CPUs.
+ * A NULL array indicates to clear affinity and set default.
+ * Returns an error if any CPU is outside of the thread processor set.
+ *
+ * If the thread is currently executing, it may wait until its time slice
+ * is up before switching onto the specified processor.
+ *
+ */
+kern_return_t
+thread_set_affinity(
+ thread_t thread,
+ processor_name_array_t processor_list,
+ natural_t countp)
+{
+ cpu_affinity_t newmask;
+ processor_t p;
+ int i;
+ boolean_t any = FALSE;
+ spl_t s;
+
+ if (thread == THREAD_NULL)
+ return KERN_INVALID_ARGUMENT;
+
+ if (processor_list == NULL || countp == 0) {
+ thread->has_affinity = FALSE;
+ return KERN_SUCCESS;
+ }
+
+ cpu_affinity_zero(&newmask);
+
+ thread_lock(thread);
+ pset_lock(thread->processor_set);
+
+ for (i = 0; i < countp; i++) {
+ p = convert_port_to_processor_name(
+ (ipc_port_t)processor_list[i]);
+
+ if (p == PROCESSOR_NULL)
+ continue;
+
+ cpu_affinity_set(&newmask, (unsigned)p->slot_num);
+
+ /* reaquire at least one processor in the correct pset */
+ if (p->processor_set != thread->processor_set)
+ continue;
+
+ any = TRUE;
+ }
+
+ if (!any) {
+ pset_unlock(thread->processor_set);
+ thread_unlock(thread);
+ return KERN_INVALID_ARGUMENT; /* empty effective set */
+ }
+
+ thread->affinity = newmask;
+ thread->has_affinity = TRUE;
+
+ s = splsched();
+ thread_enforce_affinity(thread);
+ splx(s);
+
+ pset_unlock(thread->processor_set);
+ thread_unlock(thread);
+
+ return KERN_SUCCESS;
+}
+
+/*
+ * thread_get_affinity
+ *
+ * Return the thread affinity set. If thread doesn't have any affinity set,
+ * returns a void processor_list.
+ *
+ */
+kern_return_t
+thread_get_affinity(
+ thread_t thread,
+ processor_name_array_t *processor_list,
+ natural_t *countp)
+{
+ int i, n = 0;
+ natural_t count;
+ ipc_port_t *tp;
+
+ if (thread == THREAD_NULL)
+ return KERN_INVALID_ARGUMENT;
+
+ if (thread->has_affinity == FALSE) {
+ *processor_list = NULL;
+ *countp = 0;
+
+ return KERN_SUCCESS;
+ }
+
+ thread_lock(thread);
+
+ count = cpu_affinity_count(&thread->affinity);
+ assert(count <= NCPUS);
+
+ tp = (ipc_port_t *)kalloc(count * sizeof(ipc_port_t));
+ if (tp == NULL) {
+ thread_unlock(thread);
+ return KERN_RESOURCE_SHORTAGE;
+ }
+
+ for (i = 0; i < NCPUS; i++) {
+ if (cpu_affinity_test(&thread->affinity, i))
+ tp[n++] = convert_processor_name_to_port(
+ cpu_to_processor(i));
+ }
+
+ thread_unlock(thread);
+
+ *countp = count;
+ *processor_list = (mach_port_t *)tp;
+
+ return KERN_SUCCESS;
+}
diff --git a/kern/thread.h b/kern/thread.h
index e5128dad..2e5ceb47 100644
--- a/kern/thread.h
+++ b/kern/thread.h
@@ -34,6 +34,7 @@
#ifndef _KERN_THREAD_H_
#define _KERN_THREAD_H_
+#include <string.h>
#include <mach/boolean.h>
#include <mach/thread_info.h>
#include <mach/thread_status.h>
@@ -54,6 +55,48 @@
#include <machine/thread.h>
#include <ipc/ipc_kmsg_queue.h>
+typedef struct cpu_affinity {
+ uint32_t bits[(NCPUS + 31) / 32];
+} cpu_affinity_t;
+
+static inline void
+cpu_affinity_zero(cpu_affinity_t *m) {
+ memset(m->bits, 0, sizeof(m->bits));
+}
+
+static inline void
+cpu_affinity_set(cpu_affinity_t *m, unsigned cpu) {
+ m->bits[cpu >> 5] |= (1u << (cpu & 31));
+}
+
+static inline boolean_t
+cpu_affinity_test(const cpu_affinity_t *m, unsigned cpu) {
+ return (m->bits[cpu >> 5] & (1u << (cpu & 31))) != 0;
+}
+
+static inline unsigned int
+popcnt_uint32(uint32_t n) {
+ unsigned int count = 0;
+
+ while (n) {
+ n &= n - 1;
+ count++;
+ }
+
+ return count;
+}
+
+static inline natural_t
+cpu_affinity_count(const cpu_affinity_t *m) {
+ natural_t total = 0;
+ int i;
+
+ for (i = 0; i < sizeof(m->bits) / sizeof(m->bits[0]); i++)
+ total += popcnt_uint32(m->bits[i]);
+
+ return total;
+}
+
/*
* Thread name buffer size. Use the same size as the task so
* the thread can inherit the task's name.
@@ -223,6 +266,8 @@ struct thread {
/* Processor data structures */
processor_set_t processor_set; /* assigned processor set */
processor_t bound_processor; /* bound to processor ?*/
+ cpu_affinity_t affinity; /* cpu affinity mask */
+ boolean_t has_affinity; /* if FALSE, use default mask */
sample_control_t pc_sample;
@@ -360,6 +405,14 @@ extern kern_return_t thread_set_name(
extern kern_return_t thread_get_name(
thread_t thread,
kernel_debug_name_t name);
+extern kern_return_t thread_set_affinity(
+ thread_t thread,
+ processor_name_array_t processor_list,
+ natural_t countp);
+extern kern_return_t thread_get_affinity(
+ thread_t thread,
+ processor_name_array_t *processor_list,
+ natural_t *countp);
#endif
/*
@@ -387,6 +440,9 @@ extern thread_t kernel_thread(
extern void reaper_thread(void) __attribute__((noreturn));
+extern void thread_enforce_affinity(thread_t);
+
+
#if MACH_HOST
extern void thread_freeze(
thread_t thread);
--
2.53.0
