github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm, swap: consolidate cluster allocation helpers

Browse Source 
Swap cluster table management is spread across several narrow helpers.  As
a result, the allocation and fallback sequences are open-coded in multiple
places.

A few more per-cluster tables will be added soon, so avoid duplicating
these sequences per table type.  Fold the existing pairs into
cluster-oriented helpers, and rename for consistency.

No functional change, only a few sanity checks are slightly adjusted.

Link: https://lore.kernel.org/20260517-swap-table-p4-v5-9-88ae43e064c7@tencent.com
Signed-off-by: Kairui Song <kasong@tencent.com>
Acked-by: Chris Li <chrisl@kernel.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: David Hildenbrand <david@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Youngjun Park <youngjun.park@lge.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Kairui Song 2026-05-17 23:39:48 +08:00 committed by Andrew Morton
parent bc34e87a51
commit cdd77f84d9
1 changed files with 49 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
mm/swapfile.c
View File

@ -411,20 +411,7 @@ static inline unsigned int cluster_offset(struct swap_info_struct *si,
return cluster_index(si, ci) * SWAPFILE_CLUSTER;
}
static struct swap_table *swap_table_alloc(gfp_t gfp)
{
struct folio *folio;
if (!SWP_TABLE_USE_PAGE)
return kmem_cache_zalloc(swap_table_cachep, gfp);
folio = folio_alloc(gfp | __GFP_ZERO, 0);
if (folio)
return folio_address(folio);
return NULL;
}
static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
static void swap_cluster_free_table_folio_rcu_cb(struct rcu_head *head)
{
struct folio *folio;
@ -432,15 +419,46 @@ static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
folio_put(folio);
}
static void swap_table_free(struct swap_table *table)
static void swap_cluster_free_table(struct swap_cluster_info *ci)
{
struct swap_table *table;
table = (struct swap_table *)rcu_dereference_protected(ci->table, true);
if (!table)
return;
rcu_assign_pointer(ci->table, NULL);
if (!SWP_TABLE_USE_PAGE) {
kmem_cache_free(swap_table_cachep, table);
return;
}
call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head),
swap_table_free_folio_rcu_cb);
swap_cluster_free_table_folio_rcu_cb);
}
static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
{
struct swap_table *table = NULL;
struct folio *folio;
/* The cluster must be empty and not on any list during allocation. */
VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
if (rcu_access_pointer(ci->table))
return 0;
if (SWP_TABLE_USE_PAGE) {
folio = folio_alloc(gfp | __GFP_ZERO, 0);
if (folio)
table = folio_address(folio);
} else {
table = kmem_cache_zalloc(swap_table_cachep, gfp);
}
if (!table)
return -ENOMEM;
rcu_assign_pointer(ci->table, table);
return 0;
}
/*
@ -471,27 +489,15 @@ static void swap_cluster_assert_empty(struct swap_cluster_info *ci,
WARN_ON_ONCE(nr == SWAPFILE_CLUSTER && ci->extend_table);
}
static void swap_cluster_free_table(struct swap_cluster_info *ci)
{
struct swap_table *table;
/* Only empty cluster's table is allow to be freed */
lockdep_assert_held(&ci->lock);
table = (void *)rcu_dereference_protected(ci->table, true);
rcu_assign_pointer(ci->table, NULL);
swap_table_free(table);
}
/*
* Allocate swap table for one cluster. Attempt an atomic allocation first,
* then fallback to sleeping allocation.
*/
static struct swap_cluster_info *
swap_cluster_alloc_table(struct swap_info_struct *si,
swap_cluster_populate(struct swap_info_struct *si,
struct swap_cluster_info *ci)
{
struct swap_table *table;
int ret;
/*
* Only cluster isolation from the allocator does table allocation.
@ -502,14 +508,9 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
lockdep_assert_held(&si->global_cluster_lock);
lockdep_assert_held(&ci->lock);
/* The cluster must be free and was just isolated from the free list. */
VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
if (table) {
rcu_assign_pointer(ci->table, table);
if (!swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
__GFP_NOWARN))
return ci;
}
/*
* Try a sleep allocation. Each isolated free cluster may cause
@ -521,7 +522,8 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
spin_unlock(&si->global_cluster_lock);
local_unlock(&percpu_swap_cluster.lock);
table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | GFP_KERNEL);
ret = swap_cluster_alloc_table(ci, __GFP_HIGH | __GFP_NOMEMALLOC |
GFP_KERNEL);
/*
* Back to atomic context. We might have migrated to a new CPU with a
@ -536,20 +538,11 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
spin_lock(&si->global_cluster_lock);
spin_lock(&ci->lock);
/* Nothing except this helper should touch a dangling empty cluster. */
if (WARN_ON_ONCE(cluster_table_is_alloced(ci))) {
if (table)
swap_table_free(table);
return ci;
}
if (!table) {
if (ret) {
move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
spin_unlock(&ci->lock);
return NULL;
}
rcu_assign_pointer(ci->table, table);
return ci;
}
@ -621,12 +614,11 @@ static struct swap_cluster_info *isolate_lock_cluster(
}
spin_unlock(&si->lock);
if (found && !cluster_table_is_alloced(found)) {
/* Only an empty free cluster's swap table can be freed. */
VM_WARN_ON_ONCE(flags != CLUSTER_FLAG_FREE);
/* Cluster's table is freed when and only when it's on the free list. */
if (found && flags == CLUSTER_FLAG_FREE) {
VM_WARN_ON_ONCE(list != &si->free_clusters);
VM_WARN_ON_ONCE(!cluster_is_empty(found));
return swap_cluster_alloc_table(si, found);
VM_WARN_ON_ONCE(cluster_table_is_alloced(found));
return swap_cluster_populate(si, found);
}
return found;
@ -769,7 +761,6 @@ static int swap_cluster_setup_bad_slot(struct swap_info_struct *si,
unsigned int ci_off = offset % SWAPFILE_CLUSTER;
unsigned long idx = offset / SWAPFILE_CLUSTER;
struct swap_cluster_info *ci;
struct swap_table *table;
int ret = 0;
/* si->max may got shrunk by swap swap_activate() */
@ -790,12 +781,9 @@ static int swap_cluster_setup_bad_slot(struct swap_info_struct *si,
}
ci = cluster_info + idx;
if (!ci->table) {
table = swap_table_alloc(GFP_KERNEL);
if (!table)
return -ENOMEM;
rcu_assign_pointer(ci->table, table);
}
/* Need to allocate swap table first for initial bad slot marking. */
if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL))
return -ENOMEM;
spin_lock(&ci->lock);
/* Check for duplicated bad swap slots. */
if (__swap_table_xchg(ci, ci_off, SWP_TB_BAD) != SWP_TB_NULL) {
@ -2920,7 +2908,7 @@ static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
ci = cluster_info + i;
/* Cluster with bad marks count will have a remaining table */
spin_lock(&ci->lock);
if (rcu_dereference_protected(ci->table, true)) {
if (cluster_table_is_alloced(ci)) {
swap_cluster_assert_empty(ci, 0, SWAPFILE_CLUSTER, true);
swap_cluster_free_table(ci);
}