github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/collapse: add collapse.h for the collapse interface

Browse Source 
khugepaged.c holds both the users of collapse and the machinery that
performs it.  The daemon's scan loop, the sysfs tunables, MADV_COLLAPSE's
entry point and the collapse itself all sit in one file and reach into
each other freely.  Nothing marks where a user ends and the engine
begins.

Start drawing that line.

Add mm/collapse.h for what the two sides have to agree on:

  - enum scan_result - what the engine hands back;
  - struct collapse_control - the state a request carries.

And two constants move with them:

  - KHUGEPAGED_MAX_PTES_LIMIT -> COLLAPSE_MAX_PTES_LIMIT;
  - KHUGEPAGED_MIN_MTHP_ORDER -> COLLAPSE_MIN_MTHP_ORDER.

Neither is a fact about the daemon, so both lose the KHUGEPAGED_ prefix.

No functional change.

Link: https://lore.kernel.org/20261014171145.3292736-5-kas@kernel.org
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Assisted-by: LLM
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Nico Pache <nico.pache@linux.dev>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Usama Arif <usama.arif@linux.dev>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Jann Horn <jannh@google.com>
This commit is contained in:
Kiryl Shutsemau (Meta) 2026-10-14 18:11:37 +01:00 committed by Andrew Morton
parent d950ec79fc
commit 81f8bfda35
3 changed files with 78 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MAINTAINERS
View File

@ -17452,6 +17452,7 @@ F: Documentation/admin-guide/mm/transhuge.rst
F: include/linux/huge_mm.h
F: include/linux/khugepaged.h
F: include/trace/events/huge_memory.h
F: mm/collapse.h
F: mm/huge_memory.c
F: mm/khugepaged.c
F: mm/mm_slot.h

64
mm/collapse.h Normal file
View File

@ -0,0 +1,64 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __MM_COLLAPSE_H
#define __MM_COLLAPSE_H
#include <linux/mm.h>
#include <linux/nodemask.h>
#include <linux/pgtable.h>
#include <linux/types.h>
#define COLLAPSE_MAX_PTES_LIMIT (HPAGE_PMD_NR - 1)
#define COLLAPSE_MIN_MTHP_ORDER 2
enum scan_result {
SCAN_FAIL,
SCAN_SUCCEED,
SCAN_NO_PTE_TABLE,
SCAN_PMD_MAPPED,
SCAN_EXCEED_NONE_PTE,
SCAN_EXCEED_SWAP_PTE,
SCAN_EXCEED_SHARED_PTE,
SCAN_PTE_NON_PRESENT,
SCAN_PTE_UFFD,
SCAN_PTE_MAPPED_HUGEPAGE,
SCAN_LACK_REFERENCED_PAGE,
SCAN_PAGE_NULL,
SCAN_SCAN_ABORT,
SCAN_PAGE_COUNT,
SCAN_PAGE_LRU,
SCAN_PAGE_LOCK,
SCAN_PAGE_ANON,
SCAN_PAGE_LAZYFREE,
SCAN_PAGE_COMPOUND,
SCAN_ANY_PROCESS,
SCAN_VMA_NULL,
SCAN_VMA_CHECK,
SCAN_ADDRESS_RANGE,
SCAN_DEL_PAGE_LRU,
SCAN_ALLOC_HUGE_PAGE_FAIL,
SCAN_CGROUP_CHARGE_FAIL,
SCAN_TRUNCATED,
SCAN_PAGE_HAS_PRIVATE,
SCAN_STORE_FAILED,
SCAN_COPY_MC,
SCAN_PAGE_FILLED,
SCAN_PAGE_DIRTY_OR_WRITEBACK,
};
struct collapse_control {
bool is_khugepaged;
/* Num pages scanned per node */
u32 node_load[MAX_NUMNODES];
/* Num pages scanned (see khugepaged_pages_to_scan) */
unsigned int progress;
/* nodemask for allocation fallback */
nodemask_t alloc_nmask;
/* Each bit marks a PTE the scan accepted as a collapse source */
DECLARE_BITMAP(eligible_ptes, MAX_PTRS_PER_PTE);
};
#endif /* __MM_COLLAPSE_H */

79
mm/khugepaged.c
View File

@ -26,44 +26,10 @@
#include <linux/cleanup.h>
#include <asm/tlb.h>
#include "collapse.h"
#include "internal.h"
#include "page_alloc.h"
#include "mm_slot.h"
enum scan_result {
SCAN_FAIL,
SCAN_SUCCEED,
SCAN_NO_PTE_TABLE,
SCAN_PMD_MAPPED,
SCAN_EXCEED_NONE_PTE,
SCAN_EXCEED_SWAP_PTE,
SCAN_EXCEED_SHARED_PTE,
SCAN_PTE_NON_PRESENT,
SCAN_PTE_UFFD,
SCAN_PTE_MAPPED_HUGEPAGE,
SCAN_LACK_REFERENCED_PAGE,
SCAN_PAGE_NULL,
SCAN_SCAN_ABORT,
SCAN_PAGE_COUNT,
SCAN_PAGE_LRU,
SCAN_PAGE_LOCK,
SCAN_PAGE_ANON,
SCAN_PAGE_LAZYFREE,
SCAN_PAGE_COMPOUND,
SCAN_ANY_PROCESS,
SCAN_VMA_NULL,
SCAN_VMA_CHECK,
SCAN_ADDRESS_RANGE,
SCAN_DEL_PAGE_LRU,
SCAN_ALLOC_HUGE_PAGE_FAIL,
SCAN_CGROUP_CHARGE_FAIL,
SCAN_TRUNCATED,
SCAN_PAGE_HAS_PRIVATE,
SCAN_STORE_FAILED,
SCAN_COPY_MC,
SCAN_PAGE_FILLED,
SCAN_PAGE_DIRTY_OR_WRITEBACK,
};
#include "page_alloc.h"
#define CREATE_TRACE_POINTS
#include <trace/events/huge_memory.h>
@ -91,7 +57,6 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
*
* Note that these are only respected if collapse was initiated by khugepaged.
*/
#define KHUGEPAGED_MAX_PTES_LIMIT (HPAGE_PMD_NR - 1)
unsigned int khugepaged_max_ptes_none __read_mostly;
static unsigned int khugepaged_max_ptes_swap __read_mostly;
static unsigned int khugepaged_max_ptes_shared __read_mostly;
@ -101,24 +66,6 @@ static DEFINE_READ_MOSTLY_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
static struct kmem_cache *mm_slot_cache __ro_after_init;
#define KHUGEPAGED_MIN_MTHP_ORDER 2
struct collapse_control {
bool is_khugepaged;
/* Num pages scanned per node */
u32 node_load[MAX_NUMNODES];
/* Num pages scanned (see khugepaged_pages_to_scan) */
unsigned int progress;
/* nodemask for allocation fallback */
nodemask_t alloc_nmask;
/* Each bit marks a PTE the scan accepted as a collapse source */
DECLARE_BITMAP(eligible_ptes, MAX_PTRS_PER_PTE);
};
/**
* struct khugepaged_scan - cursor for scanning
* @mm_head: the head of the mm list to scan
@ -267,7 +214,7 @@ static ssize_t max_ptes_none_store(struct kobject *kobj,
unsigned long max_ptes_none;
err = kstrtoul(buf, 10, &max_ptes_none);
if (err || max_ptes_none > KHUGEPAGED_MAX_PTES_LIMIT)
if (err || max_ptes_none > COLLAPSE_MAX_PTES_LIMIT)
return -EINVAL;
khugepaged_max_ptes_none = max_ptes_none;
@ -292,7 +239,7 @@ static ssize_t max_ptes_swap_store(struct kobject *kobj,
unsigned long max_ptes_swap;
err = kstrtoul(buf, 10, &max_ptes_swap);
if (err || max_ptes_swap > KHUGEPAGED_MAX_PTES_LIMIT)
if (err || max_ptes_swap > COLLAPSE_MAX_PTES_LIMIT)
return -EINVAL;
khugepaged_max_ptes_swap = max_ptes_swap;
@ -318,7 +265,7 @@ static ssize_t max_ptes_shared_store(struct kobject *kobj,
unsigned long max_ptes_shared;
err = kstrtoul(buf, 10, &max_ptes_shared);
if (err || max_ptes_shared > KHUGEPAGED_MAX_PTES_LIMIT)
if (err || max_ptes_shared > COLLAPSE_MAX_PTES_LIMIT)
return -EINVAL;
khugepaged_max_ptes_shared = max_ptes_shared;
@ -378,19 +325,19 @@ static unsigned int collapse_max_ptes_none(struct collapse_control *cc,
if (is_pmd_order(order))
return max_ptes_none;
/*
* for mTHP collapse with the sysctl value set to KHUGEPAGED_MAX_PTES_LIMIT,
* for mTHP collapse with the sysctl value set to COLLAPSE_MAX_PTES_LIMIT,
* scale the maximum number of PTEs to the order of the collapse.
*/
if (max_ptes_none == KHUGEPAGED_MAX_PTES_LIMIT)
if (max_ptes_none == COLLAPSE_MAX_PTES_LIMIT)
return (1 << order) - 1;
/*
* For mTHP collapse of values other than 0 or KHUGEPAGED_MAX_PTES_LIMIT,
* For mTHP collapse of values other than 0 or COLLAPSE_MAX_PTES_LIMIT,
* emit a warning and return 0.
*/
if (max_ptes_none)
pr_warn_once("mTHP collapse does not support max_ptes_none"
" values other than 0 or %u, defaulting to 0.\n",
KHUGEPAGED_MAX_PTES_LIMIT);
COLLAPSE_MAX_PTES_LIMIT);
return 0;
}
@ -476,7 +423,7 @@ int __init khugepaged_init(void)
return -ENOMEM;
khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
khugepaged_max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
khugepaged_max_ptes_none = COLLAPSE_MAX_PTES_LIMIT;
khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;
khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2;
@ -1601,8 +1548,8 @@ static enum scan_result mthp_collapse(struct mm_struct *mm,
* any smaller order enabled. When at the smallest order
* we must always move to the next offset.
*/
if (order > KHUGEPAGED_MIN_MTHP_ORDER &&
(enabled_orders & GENMASK(order - 1, 0))) {
if (order > COLLAPSE_MIN_MTHP_ORDER &&
(enabled_orders & GENMASK(order - 1, 0))) {
order--;
continue;
}
@ -1666,7 +1613,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
* is then checked again in mthp_collapse() for each attempted order.
*/
if (enabled_orders != BIT(HPAGE_PMD_ORDER))
max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
max_ptes_none = COLLAPSE_MAX_PTES_LIMIT;
pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl);
if (!pte) {