From: Hui Zhu <[email protected]> BPF programs can observe memory pressure on a cgroup, e.g. refault stats via bpf_mem_cgroup_page_state(), but cannot act on it: triggering reclaim requires writing to memory.reclaim, which BPF cannot do.
Add bpf_proactive_reclaim(), a sleepable kfunc performing one proactive reclaim pass on a memcg, like a write to memory.reclaim but without retrying until the target is reached, so that when and how hard to reclaim is BPF policy rather than hard-coded thresholds. The reclaim target of a single call is capped at MEMCG_CHARGE_BATCH, as high_work_func() does for memory.high; reclaiming more is left to the program, which can call the kfunc once per bpf_wq callback and stop at any point. It is limited to BPF_PROG_TYPE_SYSCALL, because other sleepable programs may run with filesystem locks held, on which the reclaim path could deadlock via filesystem shrinkers. Convert MIN_SWAPPINESS, MAX_SWAPPINESS and SWAPPINESS_ANON_ONLY from macros to an enum so that they are emitted into BTF and usable from BPF programs via vmlinux.h. Signed-off-by: Hui Zhu <[email protected]> --- mm/bpf_memcontrol.c | 61 ++++++++++++++++++++++++++++++++++++++++++++- mm/internal.h | 10 +++++--- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c index 716df49d7647..d6b6f4f8359a 100644 --- a/mm/bpf_memcontrol.c +++ b/mm/bpf_memcontrol.c @@ -8,6 +8,8 @@ #include <linux/memcontrol.h> #include <linux/bpf.h> +#include "internal.h" + __bpf_kfunc_start_defs(); /** @@ -159,6 +161,47 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg) mem_cgroup_flush_stats(memcg); } +/** + * bpf_proactive_reclaim - proactively reclaim memory from a memory cgroup + * @memcg: the target memory cgroup to reclaim from + * @size: the amount of memory to reclaim, in bytes, clamped to + * MEMCG_CHARGE_BATCH + * @swappiness: the reclaim swappiness, in the range [MIN_SWAPPINESS, + * SWAPPINESS_ANON_ONLY], or -1 to use the memcg's own + * + * Performs one proactive reclaim pass on @memcg, like a write to + * memory.reclaim but without retrying until @size is reached. Call it + * repeatedly to reclaim more than one batch. + * + * Only available to BPF_PROG_TYPE_SYSCALL, because other sleepable programs + * may run with filesystem locks held, which the reclaim path can deadlock + * on via filesystem shrinkers. + * + * Return: The amount of memory reclaimed, in bytes, or a negative error. + */ +__bpf_kfunc long bpf_proactive_reclaim(struct mem_cgroup *memcg, + unsigned long size, + int swappiness) +{ + unsigned long nr_reclaimed; + unsigned long nr_pages; + + if (swappiness < -1 || swappiness > SWAPPINESS_ANON_ONLY) + return -EINVAL; + + if (size < PAGE_SIZE) + return -EINVAL; + + nr_pages = min(size / PAGE_SIZE, (unsigned long)MEMCG_CHARGE_BATCH); + + nr_reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages, GFP_KERNEL, + MEMCG_RECLAIM_MAY_SWAP | + MEMCG_RECLAIM_PROACTIVE, + swappiness == -1 ? NULL : &swappiness); + + return nr_reclaimed * PAGE_SIZE; +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(bpf_memcontrol_kfuncs) @@ -174,19 +217,35 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE) BTF_KFUNCS_END(bpf_memcontrol_kfuncs) +BTF_KFUNCS_START(bpf_memcontrol_reclaim_kfuncs) +BTF_ID_FLAGS(func, bpf_proactive_reclaim, KF_SLEEPABLE) +BTF_KFUNCS_END(bpf_memcontrol_reclaim_kfuncs) + static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = { .owner = THIS_MODULE, .set = &bpf_memcontrol_kfuncs, }; +static const struct btf_kfunc_id_set bpf_memcontrol_reclaim_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_memcontrol_reclaim_kfuncs, +}; + static int __init bpf_memcontrol_init(void) { int err; err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_memcontrol_kfunc_set); - if (err) + if (err) { pr_warn("error while registering bpf memcontrol kfuncs: %d", err); + return err; + } + + err = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &bpf_memcontrol_reclaim_kfunc_set); + if (err) + pr_warn("error registering bpf reclaim kfuncs: %d\n", err); return err; } diff --git a/mm/internal.h b/mm/internal.h index 38b1165212c9..44e95e87e3cd 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -74,11 +74,13 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) #define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 +enum { + MIN_SWAPPINESS = 0, + MAX_SWAPPINESS = 200, -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + /* Just reclaim from anon folios in proactive memory reclaim */ + SWAPPINESS_ANON_ONLY = MAX_SWAPPINESS + 1, +}; unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages, -- 2.43.0
