github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm: list_lru: introduce caller locking for additions and deletions

Browse Source 
Locking is currently internal to the list_lru API.  However, a caller
might want to keep auxiliary state synchronized with the LRU state.

For example, the THP shrinker uses the lock of its custom LRU to keep
PG_partially_mapped and vmstats consistent.

To allow the THP shrinker to switch to list_lru, provide normal and
irqsafe locking primitives as well as caller-locked variants of the
addition and deletion functions.

Link: https://lore.kernel.org/20260527204757.2544958-7-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: David Hildenbrand (Arm) <david@kernel.org>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Reviewed-by: Liam R. Howlett (Oracle) <liam@infradead.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Kairui Song <ryncsn@gmail.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Mikhail Zaslonko <zaslonko@linux.ibm.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nico Pache <npache@redhat.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Usama Arif <usama.arif@linux.dev>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Johannes Weiner 2026-05-27 16:45:13 -04:00 committed by Andrew Morton
parent bc7adb3b3f
commit 1479b44c72
2 changed files with 145 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
include/linux/list_lru.h
View File

@ -83,6 +83,46 @@ int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru,
gfp_t gfp);
void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *parent);
/**
* list_lru_lock: lock the sublist for the given node and memcg
* @lru: the lru pointer
* @nid: the node id of the sublist to lock.
* @memcg: pointer to the cgroup of the sublist to lock. On return,
* updated to the cgroup whose sublist was actually locked,
* which may be an ancestor if the original memcg was dying.
*
* Returns the locked list_lru_one sublist. The caller must call
* list_lru_unlock() when done.
*
* You must ensure that the memcg is not freed during this call (e.g., with
* rcu or by taking a css refcnt).
*
* Return: the locked list_lru_one, or NULL on failure
*/
struct list_lru_one *list_lru_lock(struct list_lru *lru, int nid,
struct mem_cgroup **memcg);
/**
* list_lru_unlock: unlock a sublist locked by list_lru_lock()
* @l: the list_lru_one to unlock
*/
void list_lru_unlock(struct list_lru_one *l);
struct list_lru_one *list_lru_lock_irq(struct list_lru *lru, int nid,
struct mem_cgroup **memcg);
void list_lru_unlock_irq(struct list_lru_one *l);
struct list_lru_one *list_lru_lock_irqsave(struct list_lru *lru, int nid,
struct mem_cgroup **memcg, unsigned long *irq_flags);
void list_lru_unlock_irqrestore(struct list_lru_one *l,
unsigned long *irq_flags);
/* Caller-locked variants, see list_lru_add() etc for documentation */
bool __list_lru_add(struct list_lru *lru, struct list_lru_one *l,
struct list_head *item, int nid, struct mem_cgroup *memcg);
bool __list_lru_del(struct list_lru *lru, struct list_lru_one *l,
struct list_head *item, int nid);
/**
* list_lru_add: add an element to the lru list's tail
* @lru: the lru pointer
@ -115,6 +155,9 @@ void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *paren
bool list_lru_add(struct list_lru *lru, struct list_head *item, int nid,
struct mem_cgroup *memcg);
bool list_lru_add_irq(struct list_lru *lru, struct list_head *item, int nid,
struct mem_cgroup *memcg);
/**
* list_lru_add_obj: add an element to the lru list's tail
* @lru: the lru pointer

133
mm/list_lru.c
View File

@ -15,17 +15,23 @@
#include "slab.h"
#include "internal.h"
static inline void lock_list_lru(struct list_lru_one *l, bool irq)
static inline void lock_list_lru(struct list_lru_one *l, bool irq,
unsigned long *irq_flags)
{
if (irq)
if (irq_flags)
spin_lock_irqsave(&l->lock, *irq_flags);
else if (irq)
spin_lock_irq(&l->lock);
else
spin_lock(&l->lock);
}
static inline void unlock_list_lru(struct list_lru_one *l, bool irq_off)
static inline void unlock_list_lru(struct list_lru_one *l, bool irq_off,
unsigned long *irq_flags)
{
if (irq_off)
if (irq_flags)
spin_unlock_irqrestore(&l->lock, *irq_flags);
else if (irq_off)
spin_unlock_irq(&l->lock);
else
spin_unlock(&l->lock);
@ -78,7 +84,8 @@ list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx)
static inline struct list_lru_one *
lock_list_lru_of_memcg(struct list_lru *lru, int nid,
struct mem_cgroup **memcg, bool irq, bool skip_empty)
struct mem_cgroup **memcg, bool irq,
unsigned long *irq_flags, bool skip_empty)
{
struct list_lru_one *l;
@ -86,12 +93,12 @@ lock_list_lru_of_memcg(struct list_lru *lru, int nid,
again:
l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(*memcg));
if (likely(l)) {
lock_list_lru(l, irq);
lock_list_lru(l, irq, irq_flags);
if (likely(READ_ONCE(l->nr_items) != LONG_MIN)) {
rcu_read_unlock();
return l;
}
unlock_list_lru(l, irq);
unlock_list_lru(l, irq, irq_flags);
}
/*
* Caller may simply bail out if raced with reparenting or
@ -132,24 +139,58 @@ list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx)
static inline struct list_lru_one *
lock_list_lru_of_memcg(struct list_lru *lru, int nid,
struct mem_cgroup **memcg, bool irq, bool skip_empty)
struct mem_cgroup **memcg, bool irq,
unsigned long *irq_flags, bool skip_empty)
{
struct list_lru_one *l = &lru->node[nid].lru;
lock_list_lru(l, irq);
lock_list_lru(l, irq, irq_flags);
return l;
}
#endif /* CONFIG_MEMCG */
/* The caller must ensure the memcg lifetime. */
bool list_lru_add(struct list_lru *lru, struct list_head *item, int nid,
struct mem_cgroup *memcg)
struct list_lru_one *list_lru_lock(struct list_lru *lru, int nid,
struct mem_cgroup **memcg)
{
struct list_lru_node *nlru = &lru->node[nid];
struct list_lru_one *l;
return lock_list_lru_of_memcg(lru, nid, memcg, /*irq=*/false,
/*irq_flags=*/NULL, /*skip_empty=*/false);
}
l = lock_list_lru_of_memcg(lru, nid, &memcg, false, false);
void list_lru_unlock(struct list_lru_one *l)
{
unlock_list_lru(l, /*irq_off=*/false, /*irq_flags=*/NULL);
}
struct list_lru_one *list_lru_lock_irq(struct list_lru *lru, int nid,
struct mem_cgroup **memcg)
{
return lock_list_lru_of_memcg(lru, nid, memcg, /*irq=*/true,
/*irq_flags=*/NULL, /*skip_empty=*/false);
}
void list_lru_unlock_irq(struct list_lru_one *l)
{
unlock_list_lru(l, /*irq_off=*/true, /*irq_flags=*/NULL);
}
struct list_lru_one *list_lru_lock_irqsave(struct list_lru *lru, int nid,
struct mem_cgroup **memcg,
unsigned long *flags)
{
return lock_list_lru_of_memcg(lru, nid, memcg, /*irq=*/true,
/*irq_flags=*/flags, /*skip_empty=*/false);
}
void list_lru_unlock_irqrestore(struct list_lru_one *l, unsigned long *flags)
{
unlock_list_lru(l, /*irq_off=*/true, /*irq_flags=*/flags);
}
bool __list_lru_add(struct list_lru *lru, struct list_lru_one *l,
struct list_head *item, int nid,
struct mem_cgroup *memcg)
{
if (list_empty(item)) {
list_add_tail(item, &l->list);
/*
@ -159,15 +200,50 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item, int nid,
*/
if (!l->nr_items++)
set_shrinker_bit(memcg, nid, lru_shrinker_id(lru));
unlock_list_lru(l, false);
atomic_long_inc(&nlru->nr_items);
atomic_long_inc(&lru->node[nid].nr_items);
return true;
}
unlock_list_lru(l, false);
return false;
}
EXPORT_SYMBOL_GPL(list_lru_add);
bool __list_lru_del(struct list_lru *lru, struct list_lru_one *l,
struct list_head *item, int nid)
{
if (!list_empty(item)) {
list_del_init(item);
l->nr_items--;
atomic_long_dec(&lru->node[nid].nr_items);
return true;
}
return false;
}
/* The caller must ensure the memcg lifetime. */
bool list_lru_add(struct list_lru *lru, struct list_head *item, int nid,
struct mem_cgroup *memcg)
{
struct list_lru_one *l;
bool ret;
l = list_lru_lock(lru, nid, &memcg);
ret = __list_lru_add(lru, l, item, nid, memcg);
list_lru_unlock(l);
return ret;
}
bool list_lru_add_irq(struct list_lru *lru, struct list_head *item,
int nid, struct mem_cgroup *memcg)
{
struct list_lru_one *l;
bool ret;
l = list_lru_lock_irq(lru, nid, &memcg);
ret = __list_lru_add(lru, l, item, nid, memcg);
list_lru_unlock_irq(l);
return ret;
}
bool list_lru_add_obj(struct list_lru *lru, struct list_head *item)
{
bool ret;
@ -189,19 +265,13 @@ EXPORT_SYMBOL_GPL(list_lru_add_obj);
bool list_lru_del(struct list_lru *lru, struct list_head *item, int nid,
struct mem_cgroup *memcg)
{
struct list_lru_node *nlru = &lru->node[nid];
struct list_lru_one *l;
bool ret;
l = lock_list_lru_of_memcg(lru, nid, &memcg, false, false);
if (!list_empty(item)) {
list_del_init(item);
l->nr_items--;
unlock_list_lru(l, false);
atomic_long_dec(&nlru->nr_items);
return true;
}
unlock_list_lru(l, false);
return false;
l = list_lru_lock(lru, nid, &memcg);
ret = __list_lru_del(lru, l, item, nid);
list_lru_unlock(l);
return ret;
}
bool list_lru_del_obj(struct list_lru *lru, struct list_head *item)
@ -274,7 +344,8 @@ __list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
unsigned long isolated = 0;
restart:
l = lock_list_lru_of_memcg(lru, nid, &memcg, irq_off, true);
l = lock_list_lru_of_memcg(lru, nid, &memcg, /*irq=*/irq_off,
/*irq_flags=*/NULL, /*skip_empty=*/true);
if (!l)
return isolated;
list_for_each_safe(item, n, &l->list) {
@ -315,7 +386,7 @@ __list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
BUG();
}
}
unlock_list_lru(l, irq_off);
unlock_list_lru(l, irq_off, NULL);
out:
return isolated;
}