mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
6d06041c83
A collapse is two jobs. One reads a PTE table under mmap_lock and decides
whether the range is worth collapsing. The other allocates, isolates,
copies and flushes, and wants the lock given up first.
collapse_single_pmd() did both, so the boundary between them was somewhere
in the middle of a function.
Give each half its own function:
- collapse_scan_pmd() scans one table. It picks the side:
collapse_scan_anon(), the anonymous scan that used to carry the
collapse_scan_pmd() name, or collapse_scan_file().
- collapse_pmd() does the collapse the scan asked for, and is handed
what the scan returned. SCAN_SUCCEED means there is something to
collapse. SCAN_PTE_MAPPED_HUGEPAGE means the page cache already
holds the PMD folio and only the PTE table is left to retract.
Anything else is why there is nothing to do. It calls
collapse_anon(), formerly mthp_collapse(), or collapse_file().
For now collapse_single_pmd() stays, as the two of them with the
mmap_lock drop in between, so its callers see what they saw before.
collapse_control_init() sets a control up before its first scan.
What the scan found and the collapse needs travels in collapse_control.
For an anonymous table that is the orders and the referenced and
swapped-out counts. collapse_anon() and collapse_huge_page() now read
them from there instead of taking them as arguments.
For a file it is the file itself and the offset in it. A file collapse
works on the page cache and never sees a VMA, so the scan takes the
reference while it still has one and the collapse gives it back.
The file scan moves under mmap_lock with the anonymous one, where before
the lock was given up first. The lock is now held over the page cache
walk, an RCU walk over one table's worth of slots with no PTL, and taken
fewer times. collapse_scan_mm_slot() ends its walk whenever the lock was
dropped, so a refused file table used to cost khugepaged an unlock, a
trip back through khugepaged_do_scan(), a relock and a VMA lookup. Now
only a table that goes on to be collapsed does.
Link: https://lore.kernel.org/20261014171145.3292736-10-kas@kernel.org
Signed-off-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Lance Yang <lance.yang@linux.dev>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Assisted-by: LLM
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Nico Pache <nico.pache@linux.dev>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Usama Arif <usama.arif@linux.dev>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Jann Horn <jannh@google.com>
122 lines
3.0 KiB
C
122 lines
3.0 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef __MM_COLLAPSE_H
|
|
#define __MM_COLLAPSE_H
|
|
|
|
#include <linux/mm.h>
|
|
#include <linux/nodemask.h>
|
|
#include <linux/pgtable.h>
|
|
#include <linux/types.h>
|
|
|
|
#define COLLAPSE_MAX_PTES_LIMIT (HPAGE_PMD_NR - 1)
|
|
#define COLLAPSE_MIN_MTHP_ORDER 2
|
|
|
|
enum scan_result {
|
|
SCAN_FAIL,
|
|
SCAN_SUCCEED,
|
|
SCAN_NO_PTE_TABLE,
|
|
SCAN_PMD_MAPPED,
|
|
SCAN_EXCEED_NONE_PTE,
|
|
SCAN_EXCEED_SWAP_PTE,
|
|
SCAN_EXCEED_SHARED_PTE,
|
|
SCAN_PTE_NON_PRESENT,
|
|
SCAN_PTE_UFFD,
|
|
SCAN_PTE_MAPPED_HUGEPAGE,
|
|
SCAN_LACK_REFERENCED_PAGE,
|
|
SCAN_PAGE_NULL,
|
|
SCAN_SCAN_ABORT,
|
|
SCAN_PAGE_COUNT,
|
|
SCAN_PAGE_LRU,
|
|
SCAN_PAGE_LOCK,
|
|
SCAN_PAGE_ANON,
|
|
SCAN_PAGE_LAZYFREE,
|
|
SCAN_PAGE_COMPOUND,
|
|
SCAN_ANY_PROCESS,
|
|
SCAN_VMA_NULL,
|
|
SCAN_VMA_CHECK,
|
|
SCAN_ADDRESS_RANGE,
|
|
SCAN_DEL_PAGE_LRU,
|
|
SCAN_ALLOC_HUGE_PAGE_FAIL,
|
|
SCAN_CGROUP_CHARGE_FAIL,
|
|
SCAN_TRUNCATED,
|
|
SCAN_PAGE_HAS_PRIVATE,
|
|
SCAN_STORE_FAILED,
|
|
SCAN_COPY_MC,
|
|
SCAN_PAGE_FILLED,
|
|
SCAN_PAGE_DIRTY_OR_WRITEBACK,
|
|
};
|
|
|
|
/* How many PTEs of a window may be missing, swapped out or shared */
|
|
struct collapse_limits {
|
|
/* Counted over a PMD-sized window; HPAGE_PMD_NR means "no limit" */
|
|
unsigned int max_ptes_none;
|
|
unsigned int max_ptes_swap;
|
|
unsigned int max_ptes_shared;
|
|
};
|
|
|
|
/* What a collapse is allowed to do, decided by the caller that asks for it */
|
|
struct collapse_policy {
|
|
/* Limits for a PMD-sized window */
|
|
struct collapse_limits pmd;
|
|
|
|
/*
|
|
* Limits for a smaller window. Its max_ptes_none is either 0 or
|
|
* COLLAPSE_MAX_PTES_LIMIT, the latter meaning all but one PTE of the
|
|
* window whatever its order; any other value counts as 0.
|
|
*/
|
|
struct collapse_limits sub_pmd;
|
|
|
|
/* Leave clean lazyfree folios to reclaim rather than collapse them */
|
|
bool anon_skip_lazyfree;
|
|
|
|
/* Refuse an anonymous range with no sign of use */
|
|
bool anon_require_referenced;
|
|
|
|
/* Map the PMD over a file collapse instead of leaving it to a fault */
|
|
bool file_install_pmd;
|
|
|
|
/* Write dirty pages back and retry once instead of refusing them */
|
|
bool file_writeback_dirty;
|
|
|
|
/* How hard to try for a destination folio */
|
|
gfp_t gfp;
|
|
|
|
/* Which VMAs are eligible, as thp_vma_allowable_orders() spells it */
|
|
enum tva_type tva_type;
|
|
};
|
|
|
|
struct collapse_control {
|
|
struct collapse_policy policy;
|
|
|
|
/* Num pages scanned per node */
|
|
u32 node_load[MAX_NUMNODES];
|
|
|
|
/* Num pages scanned (see khugepaged_pages_to_scan) */
|
|
unsigned int progress;
|
|
|
|
/* nodemask for allocation fallback */
|
|
nodemask_t alloc_nmask;
|
|
|
|
/* Each bit marks a PTE the scan accepted as a collapse source */
|
|
DECLARE_BITMAP(eligible_ptes, MAX_PTRS_PER_PTE);
|
|
|
|
/*
|
|
* What a scan found and the collapse after it needs. Live only between
|
|
* the two, and read by nobody else.
|
|
*/
|
|
|
|
/* Anonymous only: orders to try, referenced and swap PTE counts */
|
|
unsigned long scan_orders;
|
|
int scan_referenced;
|
|
int scan_unmapped;
|
|
|
|
/*
|
|
* File only. A file collapse works on the page cache and never sees a
|
|
* VMA, so the scan takes a reference to the file while it still has
|
|
* one, and the collapse gives it back.
|
|
*/
|
|
struct file *scan_file;
|
|
pgoff_t scan_pgoff;
|
|
};
|
|
|
|
#endif /* __MM_COLLAPSE_H */
|