github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/bpf: Add bpf_proactive_reclaim kfunc

Browse Source 
BPF programs can observe memory pressure on a cgroup, e.g. refault
stats via bpf_mem_cgroup_page_state(), but cannot act on it:
triggering reclaim requires writing to memory.reclaim, which BPF
cannot do.

Add bpf_proactive_reclaim(), a sleepable kfunc performing one
proactive reclaim pass on a memcg, like a write to memory.reclaim
but without retrying until the target is reached, so that when and
how hard to reclaim is BPF policy rather than hard-coded thresholds.
The reclaim target of a single call is capped at MEMCG_CHARGE_BATCH,
as high_work_func() does for memory.high; reclaiming more is left to
the program, which can call the kfunc once per bpf_wq callback and
stop at any point. It is limited to BPF_PROG_TYPE_SYSCALL, because
other sleepable programs may run with filesystem locks held, on
which the reclaim path could deadlock via filesystem shrinkers.

Convert MIN_SWAPPINESS, MAX_SWAPPINESS and SWAPPINESS_ANON_ONLY from
macros to an enum so that they are emitted into BTF and usable from
BPF programs via vmlinux.h.

Signed-off-by: Hui Zhu <zhuhui@kylinos.cn>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: JP Kobryn <jp.kobryn@linux.dev>
Reviewed-by: Barry Song <baohua@kernel.org>
Link: https://lore.kernel.org/r/20260921030854.17858-1-hui.zhu@linux.dev
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
This commit is contained in:
Hui Zhu 2026-09-21 11:08:54 +08:00 committed by Kumar Kartikeya Dwivedi
parent b99f71407c
commit 48ea1ba704
No known key found for this signature in database
GPG Key ID: 472D377B63542F83
2 changed files with 70 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
mm/bpf_memcontrol.c
View File

@ -8,6 +8,8 @@
#include <linux/memcontrol.h>
#include <linux/bpf.h>
#include "internal.h"
__bpf_kfunc_start_defs();
/**
@ -159,6 +161,51 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg)
mem_cgroup_flush_stats(memcg);
}
/**
* bpf_proactive_reclaim - proactively reclaim memory from a memory cgroup
* @memcg: the target memory cgroup to reclaim from.
* @size: the amount of memory to reclaim, in bytes, clamped to
* MEMCG_CHARGE_BATCH.
* @swappiness: the reclaim swappiness, in the range [MIN_SWAPPINESS,
* MAX_SWAPPINESS], or one of the special modes: -1 to use
* the memcg's own swappiness, or SWAPPINESS_ANON_ONLY to
* reclaim only anon folios.
*
* Performs one proactive reclaim pass on @memcg, like a write to
* memory.reclaim but without retrying until @size is reached. Call it
* repeatedly to reclaim more than one batch.
*
* Only available to BPF_PROG_TYPE_SYSCALL, because other sleepable programs
* may run with filesystem locks held, which the reclaim path can deadlock
* on via filesystem shrinkers.
*
* Return: The amount of memory reclaimed, in bytes, or a negative error.
*/
__bpf_kfunc long bpf_proactive_reclaim(struct mem_cgroup *memcg,
unsigned long size,
int swappiness)
{
unsigned long nr_reclaimed;
unsigned long nr_pages;
if (swappiness != -1 && swappiness != SWAPPINESS_ANON_ONLY) {
if (swappiness < MIN_SWAPPINESS || swappiness > MAX_SWAPPINESS)
return -EINVAL;
}
if (size < PAGE_SIZE)
return -EINVAL;
nr_pages = min(size / PAGE_SIZE, (unsigned long)MEMCG_CHARGE_BATCH);
nr_reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages, GFP_KERNEL,
MEMCG_RECLAIM_MAY_SWAP |
MEMCG_RECLAIM_PROACTIVE,
swappiness == -1 ? NULL : &swappiness);
return nr_reclaimed * PAGE_SIZE;
}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(bpf_memcontrol_kfuncs)
@ -174,19 +221,35 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE)
BTF_KFUNCS_END(bpf_memcontrol_kfuncs)
BTF_KFUNCS_START(bpf_memcontrol_reclaim_kfuncs)
BTF_ID_FLAGS(func, bpf_proactive_reclaim, KF_SLEEPABLE)
BTF_KFUNCS_END(bpf_memcontrol_reclaim_kfuncs)
static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = {
.owner = THIS_MODULE,
.set = &bpf_memcontrol_kfuncs,
};
static const struct btf_kfunc_id_set bpf_memcontrol_reclaim_kfunc_set = {
.owner = THIS_MODULE,
.set = &bpf_memcontrol_reclaim_kfuncs,
};
static int __init bpf_memcontrol_init(void)
{
int err;
err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC,
&bpf_memcontrol_kfunc_set);
if (err)
if (err) {
pr_warn("error while registering bpf memcontrol kfuncs: %d", err);
return err;
}
err = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
&bpf_memcontrol_reclaim_kfunc_set);
if (err)
pr_warn("error registering bpf reclaim kfuncs: %d\n", err);
return err;
}

10
mm/internal.h
View File

@ -77,11 +77,13 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
#define MEMCG_RECLAIM_MAY_SWAP (1 << 1)
#define MEMCG_RECLAIM_PROACTIVE (1 << 2)
#define MIN_SWAPPINESS 0
#define MAX_SWAPPINESS 200
enum {
MIN_SWAPPINESS = 0,
MAX_SWAPPINESS = 200,
/* Just reclaim from anon folios in proactive memory reclaim */
#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1)
/* Just reclaim from anon folios in proactive memory reclaim */
SWAPPINESS_ANON_ONLY = MAX_SWAPPINESS + 1,
};
unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
unsigned long nr_pages,