Do not overload the resctrl tasks file with architecture-specific non-PID tokens. The tasks ABI remains a list of task IDs, while the new devices file carries external requesters assigned to a resctrl group.
Add architecture hooks for assigning and showing those external objects, and reject group removal, reparenting, or pseudo-lock setup while devices are still assigned. The teardown path performs a best-effort reset to the default group. The devices file is an assignment interface only. It does not describe a new resource schema or domain; resource allocation and monitoring policy remain described by the existing schemata and info files. Signed-off-by: Zhanpeng Zhang <[email protected]> --- Documentation/filesystems/resctrl.rst | 26 ++++ arch/Kconfig | 6 + fs/resctrl/rdtgroup.c | 206 +++++++++++++++++++++++++- include/linux/resctrl.h | 45 ++++++ 4 files changed, 280 insertions(+), 3 deletions(-) diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst index e4b66af55ffb..fce61d019114 100644 --- a/Documentation/filesystems/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -581,6 +581,32 @@ All groups contain the following files: idle tasks. Instead, a CPU's idle task is always considered as a member of the group owning the CPU. +"devices": + On architectures that support external requester assignment through + resctrl, reading this file shows the devices or device groups assigned + to this resource group. Writing an architecture-specific device token + moves that external requester to the group. Multiple tokens can be + separated by commas and are processed sequentially. A failure aborts the + write, but requesters moved before the failure remain in their new groups. + + On RISC-V, an IOMMU group is identified by the following token:: + + iommu_group:<group-id> + + Each assigned IOMMU group is reported on a separate line when the file is + read. Writing the token to the root control group's devices file restores + the IOMMU group to the reserved default QoS IDs. Default assignments are + not listed in the root devices file. + + This file only controls external requester membership. Resource + allocation and monitoring policy remains described by schemata and + info files. + + Resource groups with assigned devices cannot be removed or reparented. + Move devices to another group first. + + Failures will be logged to /sys/fs/resctrl/info/last_cmd_status. + "cpus": Reading this file shows a bitmask of the logical CPUs owned by this group. Writing a mask to this file will add and remove diff --git a/arch/Kconfig b/arch/Kconfig index fa7507ac8e13..36a4f4fbb164 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1615,6 +1615,12 @@ config ARCH_HAS_CPU_RESCTRL monitoring and control interfaces provided by the 'resctrl' filesystem (see RESCTRL_FS). +config ARCH_HAS_RESCTRL_DEVICES + bool + help + An architecture selects this option to indicate that external + device objects can be attached to resctrl resource groups. + config HAVE_ARCH_COMPILER_H bool help diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index af2cbab14497..2e424c911049 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -99,13 +99,32 @@ void rdt_last_cmd_puts(const char *s) seq_buf_puts(&last_cmd_status, s); } +void resctrl_last_cmd_puts(const char *s) +{ + rdt_last_cmd_puts(s); +} + +static void rdt_last_cmd_vprintf(const char *fmt, va_list ap) +{ + lockdep_assert_held(&rdtgroup_mutex); + seq_buf_vprintf(&last_cmd_status, fmt, ap); +} + void rdt_last_cmd_printf(const char *fmt, ...) { va_list ap; va_start(ap, fmt); - lockdep_assert_held(&rdtgroup_mutex); - seq_buf_vprintf(&last_cmd_status, fmt, ap); + rdt_last_cmd_vprintf(fmt, ap); + va_end(ap); +} + +void resctrl_last_cmd_printf(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + rdt_last_cmd_vprintf(fmt, ap); va_end(ap); } @@ -766,6 +785,151 @@ static int rdtgroup_move_task(pid_t pid, struct rdtgroup *rdtgrp, return ret; } +static bool rdtgroup_effective_ids(struct rdtgroup *r, + struct resctrl_group_ids *ids) +{ + if (!r || !ids) + return false; + + if (r->type == RDTMON_GROUP) + ids->closid = r->mon.parent->closid; + else if (r->type == RDTCTRL_GROUP) + ids->closid = r->closid; + else + return false; + + ids->rmid = r->mon.rmid; + return true; +} + +#ifdef CONFIG_ARCH_HAS_RESCTRL_DEVICES +static void show_rdt_devices(struct rdtgroup *r, struct seq_file *s) +{ + struct resctrl_group_ids ids; + + if (rdtgroup_effective_ids(r, &ids)) + resctrl_arch_devices_show(s, ids); +} + +static int rdtgroup_reset_all_devices(struct rdtgroup *to) +{ + struct resctrl_group_ids default_ids; + + if (!to || !rdtgroup_effective_ids(to, &default_ids)) + return 0; + + return resctrl_arch_devices_reset_all(default_ids); +} + +static bool rdtgroup_arch_devices_assigned(struct rdtgroup *rdtgrp) +{ + struct resctrl_group_ids ids; + + if (!rdtgroup_effective_ids(rdtgrp, &ids)) + return false; + + return resctrl_arch_devices_assigned(ids); +} + +static bool rdtgroup_child_arch_devices_assigned(struct rdtgroup *rdtgrp) +{ + struct rdtgroup *crgrp; + + list_for_each_entry(crgrp, &rdtgrp->mon.crdtgrp_list, mon.crdtgrp_list) { + if (rdtgroup_arch_devices_assigned(crgrp)) + return true; + } + + return false; +} + +static int rdtgroup_reject_assigned_devices(struct rdtgroup *rdtgrp, + bool include_children, + const char *operation) +{ + if (!rdtgroup_arch_devices_assigned(rdtgrp) && + (!include_children || !rdtgroup_child_arch_devices_assigned(rdtgrp))) + return 0; + + rdt_last_cmd_printf("Move devices out before %s group\n", operation); + return -EBUSY; +} + +static ssize_t rdtgroup_devices_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct resctrl_group_ids ids; + struct rdtgroup *rdtgrp; + char *tok; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + rdt_last_cmd_clear(); + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = -EINVAL; + rdt_last_cmd_puts("Pseudo-locking in progress\n"); + goto unlock; + } + + if (!rdtgroup_effective_ids(rdtgrp, &ids)) { + ret = -EINVAL; + goto unlock; + } + + while ((tok = strsep(&buf, ","))) { + tok = strim(tok); + if (!*tok) { + rdt_last_cmd_puts("Device list parsing error\n"); + ret = -EINVAL; + break; + } + + ret = resctrl_arch_devices_write(tok, ids); + if (ret) + break; + } + +unlock: + rdtgroup_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +static int rdtgroup_devices_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (rdtgrp) + show_rdt_devices(rdtgrp, s); + else + ret = -ENOENT; + rdtgroup_kn_unlock(of->kn); + + return ret; +} +#else +static inline int rdtgroup_reset_all_devices(struct rdtgroup *to) +{ + return 0; +} + +static inline int rdtgroup_reject_assigned_devices(struct rdtgroup *rdtgrp, + bool include_children, + const char *operation) +{ + return 0; +} +#endif + static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { @@ -1491,6 +1655,11 @@ static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of, rdtgrp->mode = RDT_MODE_EXCLUSIVE; } else if (IS_ENABLED(CONFIG_RESCTRL_FS_PSEUDO_LOCK) && !strcmp(buf, "pseudo-locksetup")) { + ret = rdtgroup_reject_assigned_devices(rdtgrp, true, + "entering pseudo-locksetup"); + if (ret) + goto out; + ret = rdtgroup_locksetup_enter(rdtgrp); if (ret) goto out; @@ -2067,6 +2236,16 @@ static struct rftype res_common_files[] = { .seq_show = rdtgroup_tasks_show, .fflags = RFTYPE_BASE, }, +#ifdef CONFIG_ARCH_HAS_RESCTRL_DEVICES + { + .name = "devices", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_devices_write, + .seq_show = rdtgroup_devices_show, + .fflags = RFTYPE_BASE, + }, +#endif { .name = "mon_hw_id", .mode = 0444, @@ -3061,6 +3240,8 @@ static void rmdir_all_sub(void) /* Move all tasks to the default resource group */ rdt_move_group_tasks(NULL, &rdtgroup_default, NULL); + WARN_ON_ONCE(rdtgroup_reset_all_devices(&rdtgroup_default)); + list_for_each_entry_safe(rdtgrp, tmp, &rdt_all_groups, rdtgroup_list) { /* Free any child rmids */ free_all_child_rdtgrp(rdtgrp); @@ -3970,6 +4151,11 @@ static int rdtgroup_rmdir_mon(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) struct rdtgroup *prdtgrp = rdtgrp->mon.parent; u32 closid, rmid; int cpu; + int ret; + + ret = rdtgroup_reject_assigned_devices(rdtgrp, false, "removing"); + if (ret) + return ret; /* Give any tasks back to the parent group */ rdt_move_group_tasks(rdtgrp, prdtgrp, tmpmask); @@ -4020,6 +4206,11 @@ static int rdtgroup_rmdir_ctrl(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) { u32 closid, rmid; int cpu; + int ret; + + ret = rdtgroup_reject_assigned_devices(rdtgrp, true, "removing"); + if (ret) + return ret; /* Give any tasks back to the default group */ rdt_move_group_tasks(rdtgrp, &rdtgroup_default, tmpmask); @@ -4093,7 +4284,10 @@ static int rdtgroup_rmdir(struct kernfs_node *kn) rdtgrp != &rdtgroup_default) { if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { - ret = rdtgroup_ctrl_remove(rdtgrp); + ret = rdtgroup_reject_assigned_devices(rdtgrp, true, + "removing"); + if (!ret) + ret = rdtgroup_ctrl_remove(rdtgrp); } else { ret = rdtgroup_rmdir_ctrl(rdtgrp, tmpmask); } @@ -4210,6 +4404,12 @@ static int rdtgroup_rename(struct kernfs_node *kn, goto out; } + if (rdtgrp->mon.parent != new_prdtgrp) { + ret = rdtgroup_reject_assigned_devices(rdtgrp, false, "reparenting"); + if (ret) + goto out; + } + /* * Allocate the cpumask for use in mongrp_reparent() to avoid the * possibility of failing to allocate it after kernfs_rename() has diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 73ff522448a0..a4b5c2d5e814 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -8,6 +8,8 @@ #include <linux/pid.h> #include <linux/resctrl_types.h> +struct seq_file; + #ifdef CONFIG_ARCH_HAS_CPU_RESCTRL #include <asm/resctrl.h> #endif @@ -18,6 +20,49 @@ #define RESCTRL_PICK_ANY_CPU -1 +/** + * struct resctrl_group_ids - resctrl control and monitoring IDs + * @closid: resource control class ID + * @rmid: resource monitoring ID + */ +struct resctrl_group_ids { + u32 closid; + u32 rmid; +}; + +void resctrl_last_cmd_puts(const char *s); +void resctrl_last_cmd_printf(const char *fmt, ...) __printf(1, 2); + +#ifdef CONFIG_ARCH_HAS_RESCTRL_DEVICES +int resctrl_arch_devices_write(char *tok, struct resctrl_group_ids ids); +void resctrl_arch_devices_show(struct seq_file *s, + struct resctrl_group_ids ids); +bool resctrl_arch_devices_assigned(struct resctrl_group_ids ids); +int resctrl_arch_devices_reset_all(struct resctrl_group_ids default_ids); +#else +static inline int resctrl_arch_devices_write(char *tok, + struct resctrl_group_ids ids) +{ + return -EOPNOTSUPP; +} + +static inline void resctrl_arch_devices_show(struct seq_file *s, + struct resctrl_group_ids ids) +{ +} + +static inline bool resctrl_arch_devices_assigned(struct resctrl_group_ids ids) +{ + return false; +} + +static inline int +resctrl_arch_devices_reset_all(struct resctrl_group_ids default_ids) +{ + return 0; +} +#endif + #ifdef CONFIG_PROC_CPU_RESCTRL int proc_resctrl_show(struct seq_file *m, -- 2.50.1 (Apple Git-155)
