github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-13 06:23:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/hugetlb_vmemmap: move bootmem HVO setup to early init

Browse Source 
Bootmem HugeTLB pages currently defer HVO setup to
hugetlb_vmemmap_init_late(), because the optimization needs zone
information.

Now that zone initialization is available earlier, the bootmem HVO setup
can be done directly from hugetlb_vmemmap_init_early().  This lets
gigantic HugeTLB pages apply HVO as soon as they are allocated.

Bootmem gigantic pages that span multiple zones are now filtered out when
they are allocated, so the remaining bootmem gigantic pages seen by later
hugetlb initialization are already zone-valid.  As a result,
hugetlb_vmemmap_init_late() no longer needs to handle bootmem HVO setup.

Link: https://lore.kernel.org/20260612035903.2468601-16-songmuchun@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Acked-by: Usama Arif <usama.arif@linux.dev>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Frank van der Linden <fvdl@google.com>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Oscar Salvador (SUSE) <osalvador@kernel.org>
Cc: "Ritesh Harjani (IBM)" <ritesh.list@gmail.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Muchun Song 2026-06-12 11:58:59 +08:00 committed by Andrew Morton
parent 20549ccf4d
commit 51d73513dd
1 changed files with 25 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
mm/hugetlb_vmemmap.c
View File

@ -745,6 +745,20 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m)
return true;
}
static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
{
struct zone *zone;
enum zone_type zone_type;
for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
zone = &NODE_DATA(nid)->node_zones[zone_type];
if (zone_spans_pfn(zone, pfn))
return zone;
}
return NULL;
}
/*
* Initialize memmap section for a gigantic page, HVO-style.
*/
@ -752,6 +766,7 @@ void __init hugetlb_vmemmap_init_early(int nid)
{
unsigned long psize, paddr, section_size;
unsigned long ns, i, pnum, pfn, nr_pages;
unsigned long start, end;
struct huge_bootmem_page *m = NULL;
void *map;
@ -761,6 +776,8 @@ void __init hugetlb_vmemmap_init_early(int nid)
section_size = (1UL << PA_SECTION_SHIFT);
list_for_each_entry(m, &huge_boot_pages[nid], list) {
struct zone *zone;
if (!vmemmap_should_optimize_bootmem_page(m))
continue;
@ -769,6 +786,14 @@ void __init hugetlb_vmemmap_init_early(int nid)
paddr = virt_to_phys(m);
pfn = PHYS_PFN(paddr);
map = pfn_to_page(pfn);
start = (unsigned long)map;
end = start + hugetlb_vmemmap_size(m->hstate);
zone = pfn_to_zone(nid, pfn);
if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate),
zone, HUGETLB_VMEMMAP_RESERVE_SIZE))
panic("Failed to allocate memmap for HugeTLB page\n");
memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE));
pnum = pfn_to_section_nr(pfn);
ns = psize / section_size;
@ -784,76 +809,8 @@ void __init hugetlb_vmemmap_init_early(int nid)
}
}
static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
{
struct zone *zone;
enum zone_type zone_type;
for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
zone = &NODE_DATA(nid)->node_zones[zone_type];
if (zone_spans_pfn(zone, pfn))
return zone;
}
return NULL;
}
void __init hugetlb_vmemmap_init_late(int nid)
{
struct huge_bootmem_page *m, *tm;
unsigned long phys, nr_pages, start, end;
unsigned long pfn, nr_mmap;
struct zone *zone = NULL;
struct hstate *h;
void *map;
if (!READ_ONCE(vmemmap_optimize_enabled))
return;
list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) {
if (!(m->flags & HUGE_BOOTMEM_HVO))
continue;
phys = virt_to_phys(m);
h = m->hstate;
pfn = PHYS_PFN(phys);
nr_pages = pages_per_huge_page(h);
map = pfn_to_page(pfn);
start = (unsigned long)map;
end = start + nr_pages * sizeof(struct page);
if (!hugetlb_bootmem_page_zones_valid(nid, m)) {
/*
* Oops, the hugetlb page spans multiple zones.
* Remove it from the list, and populate it normally.
*/
list_del(&m->list);
vmemmap_populate(start, end, nid, NULL);
nr_mmap = end - start;
memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
memblock_phys_free(phys, huge_page_size(h));
continue;
}
if (!zone || !zone_spans_pfn(zone, pfn))
zone = pfn_to_zone(nid, pfn);
if (WARN_ON_ONCE(!zone))
continue;
if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone,
HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) {
/* Fallback if HVO population fails */
vmemmap_populate(start, end, nid, NULL);
nr_mmap = end - start;
} else {
m->flags |= HUGE_BOOTMEM_ZONES_VALID;
nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE;
}
memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
}
}
#endif