github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/vmalloc: fix vmalloc_dump_obj cross-zone VA lookup

Browse Source 
vmalloc_dump_obj() searches only one vmap node (addr_to_node(addr)), but a
vmalloc allocation may span multiple vmap zones.  The VA is stored in only
one node's rb-tree (addr_to_node(va_start)), so an object pointer in a
different zone than va_start maps to a different node and the search
misses.  This affects any allocation larger than vmap_zone_size (64 KiB)
on multi-CPU systems.

Extract find_vmap_area_lock() from find_vmap_area() to share the
cross-node iteration logic.  The helper supports both spin_lock and
spin_trylock, the latter for atomic dump contexts (OOM, KASAN, RCU).

Link: https://lore.kernel.org/20260928-vmalloc_dump_obj-v4-2-6f288a431edc@linux.dev
Signed-off-by: Ye Liu <liuye@kylinos.cn>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reviewed-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
Cc: Paul Walmsley <pjw@kernel.org>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexandre Ghiti <alex@ghiti.fr>
This commit is contained in:
Ye Liu 2026-09-28 16:15:55 +08:00 committed by Andrew Morton
parent f4399c9927
commit 4b52da0120
1 changed files with 66 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
mm/vmalloc.c
View File

@ -2503,67 +2503,94 @@ static void free_unmap_vmap_area(struct vmap_area *va)
free_vmap_area_noflush(va);
}
struct vmap_area *find_vmap_area(unsigned long addr)
static inline int next_vmap_node_id(int i)
{
return (i + nr_vmap_nodes - 1) % nr_vmap_nodes;
}
enum vmap_lock_mode {
VMAP_LOCK,
VMAP_TRYLOCK,
};
/*
* Search for a vmap_area at @addr across all vmap nodes. An
* addr_to_node_id(addr) converts an address to a node index where
* a VA is located. If VA spans several zones and passed addr is not
* the same as va->va_start, what is not common, we may need to scan
* extra nodes. See an example:
*
* <----va---->
* -|-----|-----|-----|-----|-
* 1 2 0 1
*
* VA resides in node 1 whereas it spans 1, 2 an 0. If passed addr
* is within 2 or 0 nodes we should do extra work.
*
* Returns the VA with @locked_vn->busy.lock held; the caller must
* release it. If @mode is VMAP_TRYLOCK, nodes that cannot be locked
* are skipped.
*/
static struct vmap_area *
find_vmap_area_lock(unsigned long addr, struct vmap_node **locked_vn,
enum vmap_lock_mode mode)
{
struct vmap_node *vn;
struct vmap_area *va;
int i, j;
*locked_vn = NULL;
if (unlikely(!vmap_initialized))
return NULL;
/*
* An addr_to_node_id(addr) converts an address to a node index
* where a VA is located. If VA spans several zones and passed
* addr is not the same as va->va_start, what is not common, we
* may need to scan extra nodes. See an example:
*
* <----va---->
* -|-----|-----|-----|-----|-
* 1 2 0 1
*
* VA resides in node 1 whereas it spans 1, 2 an 0. If passed
* addr is within 2 or 0 nodes we should do extra work.
*/
i = j = addr_to_node_id(addr);
do {
vn = &vmap_nodes[i];
spin_lock(&vn->busy.lock);
va = __find_vmap_area(addr, &vn->busy.root);
spin_unlock(&vn->busy.lock);
if (mode == VMAP_LOCK) {
spin_lock(&vn->busy.lock);
} else {
if (!spin_trylock(&vn->busy.lock))
continue;
}
if (va)
va = __find_vmap_area(addr, &vn->busy.root);
if (va) {
*locked_vn = vn;
return va;
} while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j);
}
spin_unlock(&vn->busy.lock);
} while ((i = next_vmap_node_id(i)) != j);
return NULL;
}
struct vmap_area *find_vmap_area(unsigned long addr)
{
struct vmap_node *vn;
struct vmap_area *va;
va = find_vmap_area_lock(addr, &vn, VMAP_LOCK);
if (va)
spin_unlock(&vn->busy.lock);
return va;
}
static struct vmap_area *find_unlink_vmap_area(unsigned long addr)
{
struct vmap_node *vn;
struct vmap_area *va;
int i, j;
/*
* Check the comment in the find_vmap_area() about the loop.
*/
i = j = addr_to_node_id(addr);
do {
vn = &vmap_nodes[i];
spin_lock(&vn->busy.lock);
va = __find_vmap_area(addr, &vn->busy.root);
if (va)
unlink_va(va, &vn->busy.root);
va = find_vmap_area_lock(addr, &vn, VMAP_LOCK);
if (va) {
unlink_va(va, &vn->busy.root);
spin_unlock(&vn->busy.lock);
}
if (va)
return va;
} while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j);
return NULL;
return va;
}
/*** Per cpu kva allocator ***/
@ -5269,14 +5296,11 @@ bool vmalloc_dump_obj(void *object)
unsigned long nr_pages;
addr = PAGE_ALIGN_DOWN((unsigned long) object);
vn = addr_to_node(addr);
if (!spin_trylock(&vn->busy.lock))
return false;
va = __find_vmap_area(addr, &vn->busy.root);
va = find_vmap_area_lock(addr, &vn, VMAP_TRYLOCK);
if (!va || !va->vm) {
spin_unlock(&vn->busy.lock);
if (va)
spin_unlock(&vn->busy.lock);
return false;
}