github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/hugetlb: switch HugeTLB to section-based vmemmap optimization

Browse Source 
HugeTLB bootmem vmemmap optimization still carries its own early setup
path, including pre-populating optimized mappings before the generic
sparse-vmemmap code runs.

Now that the section-based vmemmap optimization can derive HugeTLB vmemmap
deduplication from section metadata, HugeTLB only needs to mark the
bootmem huge page range with the appropriate order.  The generic
sparse-vmemmap population path can then allocate and map the shared tail
vmemmap pages without any HugeTLB-specific early population code.

Do that by recording the compound page order when a bootmem huge page is
allocated and dropping the dedicated pre-HVO helpers and related
special-casing.

This removes duplicate early setup logic and switches HugeTLB to the
section-based vmemmap optimization path.

Link: https://lore.kernel.org/20260910063256.64386-11-songmuchun@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Acked-by: Qi Zheng <qi.zheng@linux.dev>
Cc: David Hildenbrand (Arm) <david@kernel.org>
Cc: David Laight <david.laight.linux@gmail.com>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
This commit is contained in:
Muchun Song 2026-09-10 14:32:49 +08:00 committed by Andrew Morton
parent 74d2a4e89f
commit b8abb1c0c4
7 changed files with 48 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/linux/hugetlb.h
View File

@ -171,7 +171,6 @@ struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio);
extern int movable_gigantic_pages __read_mostly;
extern int sysctl_hugetlb_shm_group __read_mostly;
extern struct list_head huge_boot_pages[MAX_NUMNODES];
void hugetlb_bootmem_struct_page_init(void);
void hugetlb_bootmem_alloc(void);

3
include/linux/mm.h
View File

@ -5159,9 +5159,6 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
int node, struct vmem_altmap *altmap);
int vmemmap_populate(unsigned long start, unsigned long end, int node,
struct vmem_altmap *altmap);
int vmemmap_populate_hvo(unsigned long start, unsigned long end,
unsigned int order, struct zone *zone,
unsigned long headsize);
void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node,
unsigned long headsize);
void vmemmap_populate_print_last(void);

31
mm/hugetlb.c
View File

@ -52,6 +52,7 @@
#include "hugetlb_cma.h"
#include "hugetlb_internal.h"
#include "mm_init.h"
#include "sparse.h"
#include <linux/page-isolation.h>
int hugetlb_max_hstate __read_mostly;
@ -59,7 +60,7 @@ unsigned int default_hstate_idx;
struct hstate hstates[HUGE_MAX_HSTATE];
__initdata nodemask_t hugetlb_bootmem_nodes;
__initdata struct list_head huge_boot_pages[MAX_NUMNODES];
static struct list_head huge_boot_pages[MAX_NUMNODES] __initdata;
/*
* Due to ordering constraints across the init code for various
@ -3150,6 +3151,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
} else {
list_add_tail(&m->list, &huge_boot_pages[nid]);
m->flags |= HUGE_BOOTMEM_ZONES_VALID;
hugetlb_vmemmap_optimize_bootmem_page(m);
/*
* Only initialize the head struct page in memmap_init_reserved_pages,
* rest of the struct pages will be initialized by the HugeTLB
@ -3310,6 +3312,8 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
* this folio.
*/
folio_set_hugetlb_vmemmap_optimized(folio);
section_set_compound_order_range(folio_pfn(folio),
folio_nr_pages(folio), 0);
if (hugetlb_bootmem_page_earlycma(m))
folio_set_hugetlb_cma(folio);
@ -3353,31 +3357,6 @@ void __init hugetlb_bootmem_struct_page_init(void)
.max_threads = num_node_state(N_MEMORY),
.numa_aware = true,
};
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
struct zone *zone;
for_each_zone(zone) {
for (int i = 0; i < VMEMMAP_OPTIMIZATION_NR_ORDERS; i++) {
struct page *tail, *p;
unsigned int order;
tail = zone->vmemmap_tails[i];
if (!tail)
continue;
order = i + VMEMMAP_OPTIMIZATION_MIN_ORDER;
p = page_to_virt(tail);
/*
* prep_and_add_bootmem_folios() can access pageblock
* flags on bootmem HugeTLB pages, so initialize the
* shared tail struct pages here before bootmem folios
* start using them.
*/
for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
init_compound_tail(p + j, NULL, order, zone);
}
}
#endif
padata_do_multithreaded(&job);
}

91
mm/hugetlb_vmemmap.c
View File

@ -18,8 +18,7 @@
#include <asm/tlbflush.h>
#include "hugetlb_vmemmap.h"
#include "internal.h"
#include "mm_init.h"
#include "sparse.h"
/**
* struct vmemmap_remap_walk - walk vmemmap page table
@ -706,95 +705,19 @@ void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head
__hugetlb_vmemmap_optimize_folios(h, folio_list, true);
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
/* Return true of a bootmem allocated HugeTLB page should be pre-HVO-ed */
static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m)
void __init hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m)
{
unsigned long section_size, psize, pmd_vmemmap_size;
phys_addr_t paddr;
if (!READ_ONCE(vmemmap_optimize_enabled))
return false;
if (!hugetlb_vmemmap_optimizable(m->hstate))
return false;
psize = huge_page_size(m->hstate);
paddr = virt_to_phys(m);
/*
* Pre-HVO only works if the bootmem huge page
* is aligned to the section size.
*/
section_size = (1UL << PA_SECTION_SHIFT);
if (!IS_ALIGNED(paddr, section_size) ||
!IS_ALIGNED(psize, section_size))
return false;
/*
* The pre-HVO code does not deal with splitting PMDS,
* so the bootmem page must be aligned to the number
* of base pages that can be mapped with one vmemmap PMD.
*/
pmd_vmemmap_size = (PMD_SIZE / (sizeof(struct page))) << PAGE_SHIFT;
if (!IS_ALIGNED(paddr, pmd_vmemmap_size) ||
!IS_ALIGNED(psize, pmd_vmemmap_size))
return false;
return true;
}
/*
* Initialize memmap section for a gigantic page, HVO-style.
*/
void __init hugetlb_vmemmap_init_early(int nid)
{
unsigned long psize, paddr, section_size;
unsigned long ns, i, pnum, pfn, nr_pages;
unsigned long start, end;
struct huge_bootmem_page *m = NULL;
void *map;
struct hstate *h = m->hstate;
unsigned long pfn = PHYS_PFN(__pa(m));
if (!READ_ONCE(vmemmap_optimize_enabled))
return;
section_size = (1UL << PA_SECTION_SHIFT);
list_for_each_entry(m, &huge_boot_pages[nid], list) {
struct zone *zone;
if (!vmemmap_should_optimize_bootmem_page(m))
continue;
nr_pages = pages_per_huge_page(m->hstate);
psize = nr_pages << PAGE_SHIFT;
paddr = virt_to_phys(m);
pfn = PHYS_PFN(paddr);
map = pfn_to_page(pfn);
start = (unsigned long)map;
end = start + hugetlb_vmemmap_size(m->hstate);
zone = pfn_to_zone(pfn, nid);
if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate),
zone, HUGETLB_VMEMMAP_RESERVE_SIZE))
panic("Failed to allocate memmap for HugeTLB page\n");
memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE));
pnum = pfn_to_section_nr(pfn);
ns = psize / section_size;
for (i = 0; i < ns; i++) {
sparse_init_early_section(nid, map, pnum,
SECTION_IS_VMEMMAP_PREINIT);
map += section_map_size();
pnum++;
}
section_set_compound_order_range(pfn, pages_per_huge_page(h),
huge_page_order(h));
if (vmemmap_optimizable_order(pfn_to_section_compound_order(pfn)))
m->flags |= HUGE_BOOTMEM_HVO;
}
}
#endif
static const struct ctl_table hugetlb_vmemmap_sysctls[] = {
{

16
mm/hugetlb_vmemmap.h
View File

@ -9,8 +9,7 @@
#ifndef _LINUX_HUGETLB_VMEMMAP_H
#define _LINUX_HUGETLB_VMEMMAP_H
#include <linux/hugetlb.h>
#include <linux/io.h>
#include <linux/memblock.h>
#include "internal.h"
/*
* Reserve one vmemmap page, all vmemmap addresses are mapped to it. See
@ -27,10 +26,7 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio);
void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list);
void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list);
#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
void hugetlb_vmemmap_init_early(int nid);
#endif
void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m);
static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h)
{
@ -76,14 +72,14 @@ static inline void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h,
{
}
static inline void hugetlb_vmemmap_init_early(int nid)
{
}
static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
{
return 0;
}
static inline void hugetlb_vmemmap_optimize_bootmem_page(struct huge_bootmem_page *m)
{
}
#endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */
static inline bool hugetlb_vmemmap_optimizable(const struct hstate *h)

31
mm/sparse-vmemmap.c
View File

@ -32,8 +32,6 @@
#include <asm/dma.h>
#include <asm/tlbflush.h>
#include "hugetlb_vmemmap.h"
/*
* Flags for vmemmap_populate_range and friends.
*/
@ -404,34 +402,6 @@ void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end,
}
}
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end,
unsigned int order, struct zone *zone,
unsigned long headsize)
{
unsigned long maddr;
struct page *tail;
pte_t *pte;
int node = zone_to_nid(zone);
tail = vmemmap_get_tail(order, zone);
if (!tail)
return -ENOMEM;
for (maddr = addr; maddr < addr + headsize; maddr += PAGE_SIZE) {
pte = vmemmap_populate_address(maddr, node, NULL, -1, 0);
if (!pte)
return -ENOMEM;
}
/*
* Reuse the last page struct page mapped above for the rest.
*/
return vmemmap_populate_range(maddr, end, node, NULL,
page_to_pfn(tail), 0);
}
#endif
void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
unsigned long addr, unsigned long next)
{
@ -634,7 +604,6 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
*/
void __init sparse_vmemmap_init_nid_early(int nid)
{
hugetlb_vmemmap_init_early(nid);
}
#endif

30
mm/sparse.h
View File

@ -16,6 +16,26 @@ static inline unsigned int section_compound_order(const struct mem_section *sect
return section->compound_page_order;
}
static inline void section_set_compound_order(struct mem_section *section,
unsigned int order)
{
VM_WARN_ON(section_compound_order(section) && order &&
section_compound_order(section) != order);
section->compound_page_order = order;
}
static inline void section_set_compound_order_range(unsigned long pfn,
unsigned long nr_pages, unsigned int order)
{
unsigned long section_nr = pfn_to_section_nr(pfn);
if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION))
return;
for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++)
section_set_compound_order(__nr_to_section(section_nr + i), order);
}
static inline unsigned int pfn_to_section_compound_order(unsigned long pfn)
{
return section_compound_order(__pfn_to_section(pfn));
@ -26,6 +46,16 @@ static inline unsigned int section_compound_order(const struct mem_section *sect
return 0;
}
static inline void section_set_compound_order(struct mem_section *section,
unsigned int order)
{
}
static inline void section_set_compound_order_range(unsigned long pfn,
unsigned long nr_pages, unsigned int order)
{
}
static inline unsigned int pfn_to_section_compound_order(unsigned long pfn)
{
return 0;