diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c index 716df49d7647..d8f579c28560 100644 --- a/mm/bpf_memcontrol.c +++ b/mm/bpf_memcontrol.c @@ -8,6 +8,8 @@ #include #include +#include "internal.h" + __bpf_kfunc_start_defs(); /** @@ -159,6 +161,51 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg) mem_cgroup_flush_stats(memcg); } +/** + * bpf_proactive_reclaim - proactively reclaim memory from a memory cgroup + * @memcg: the target memory cgroup to reclaim from. + * @size: the amount of memory to reclaim, in bytes, clamped to + * MEMCG_CHARGE_BATCH. + * @swappiness: the reclaim swappiness, in the range [MIN_SWAPPINESS, + * MAX_SWAPPINESS], or one of the special modes: -1 to use + * the memcg's own swappiness, or SWAPPINESS_ANON_ONLY to + * reclaim only anon folios. + * + * Performs one proactive reclaim pass on @memcg, like a write to + * memory.reclaim but without retrying until @size is reached. Call it + * repeatedly to reclaim more than one batch. + * + * Only available to BPF_PROG_TYPE_SYSCALL, because other sleepable programs + * may run with filesystem locks held, which the reclaim path can deadlock + * on via filesystem shrinkers. + * + * Return: The amount of memory reclaimed, in bytes, or a negative error. + */ +__bpf_kfunc long bpf_proactive_reclaim(struct mem_cgroup *memcg, + unsigned long size, + int swappiness) +{ + unsigned long nr_reclaimed; + unsigned long nr_pages; + + if (swappiness != -1 && swappiness != SWAPPINESS_ANON_ONLY) { + if (swappiness < MIN_SWAPPINESS || swappiness > MAX_SWAPPINESS) + return -EINVAL; + } + + if (size < PAGE_SIZE) + return -EINVAL; + + nr_pages = min(size / PAGE_SIZE, (unsigned long)MEMCG_CHARGE_BATCH); + + nr_reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages, GFP_KERNEL, + MEMCG_RECLAIM_MAY_SWAP | + MEMCG_RECLAIM_PROACTIVE, + swappiness == -1 ? NULL : &swappiness); + + return nr_reclaimed * PAGE_SIZE; +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(bpf_memcontrol_kfuncs) @@ -174,19 +221,35 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE) BTF_KFUNCS_END(bpf_memcontrol_kfuncs) +BTF_KFUNCS_START(bpf_memcontrol_reclaim_kfuncs) +BTF_ID_FLAGS(func, bpf_proactive_reclaim, KF_SLEEPABLE) +BTF_KFUNCS_END(bpf_memcontrol_reclaim_kfuncs) + static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = { .owner = THIS_MODULE, .set = &bpf_memcontrol_kfuncs, }; +static const struct btf_kfunc_id_set bpf_memcontrol_reclaim_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_memcontrol_reclaim_kfuncs, +}; + static int __init bpf_memcontrol_init(void) { int err; err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_memcontrol_kfunc_set); - if (err) + if (err) { pr_warn("error while registering bpf memcontrol kfuncs: %d", err); + return err; + } + + err = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &bpf_memcontrol_reclaim_kfunc_set); + if (err) + pr_warn("error registering bpf reclaim kfuncs: %d\n", err); return err; } diff --git a/mm/internal.h b/mm/internal.h index 7c326965a94f..9e3be236f18c 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -77,11 +77,13 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) #define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 +enum { + MIN_SWAPPINESS = 0, + MAX_SWAPPINESS = 200, -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + /* Just reclaim from anon folios in proactive memory reclaim */ + SWAPPINESS_ANON_ONLY = MAX_SWAPPINESS + 1, +}; unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages,