github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/sparse-vmemmap: support section-based vmemmap optimization

Browse Source 
Teach sparse-vmemmap population code to use the compound page order when
deciding whether a vmemmap page can be optimized.

With this information, the common sparse-vmemmap population path can
allocate or reuse shared tail vmemmap pages directly instead of relying on
HugeTLB-specific handling.

This centralizes vmemmap optimization logic in the sparse-vmemmap code,
based on section metadata, and prepares for sharing the same mechanism
across different users of vmemmap optimization, including HugeTLB and DAX.

Link: https://lore.kernel.org/20260910063256.64386-9-songmuchun@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Acked-by: Qi Zheng <qi.zheng@linux.dev>
Cc: David Hildenbrand (Arm) <david@kernel.org>
Cc: David Laight <david.laight.linux@gmail.com>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
This commit is contained in:
Muchun Song 2026-09-10 14:32:47 +08:00 committed by Andrew Morton
parent 38fe068a2d
commit a64b8c60c6
3 changed files with 47 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
mm/sparse-vmemmap.c
View File

@ -148,8 +148,7 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
start, end - 1);
}
#ifdef CONFIG_MEMORY_HOTPLUG
static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
const struct mem_section *ms = __pfn_to_section(pfn);
@ -175,7 +174,6 @@ static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long n
return 0;
}
#endif
static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
{
@ -215,19 +213,44 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *
return tail;
}
#else
static inline struct page *vmemmap_get_tail(unsigned int order, struct zone *zone)
{
return NULL;
}
#endif
static __meminit void *vmemmap_alloc_pte(unsigned long pfn, int node,
struct vmem_altmap *altmap)
{
struct zone *zone;
struct page *page;
const unsigned int order = pfn_to_section_compound_order(pfn);
if (!vmemmap_optimizable_pfn(pfn))
return vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
zone = pfn_to_zone(pfn, node);
page = vmemmap_get_tail(order, zone);
if (!page)
return NULL;
return page_address(page);
}
static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
struct vmem_altmap *altmap,
unsigned long ptpfn, unsigned long flags)
{
pte_t *pte = pte_offset_kernel(pmd, addr);
unsigned long pfn = page_to_pfn((struct page *)addr);
if (pte_none(ptep_get(pte))) {
pte_t entry;
void *p;
if (ptpfn == (unsigned long)-1) {
p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
void *p = vmemmap_alloc_pte(pfn, node, altmap);
if (!p)
return NULL;
ptpfn = PHYS_PFN(__pa(p));
@ -246,7 +269,8 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in
}
entry = pfn_pte(ptpfn, PAGE_KERNEL);
set_pte_at(&init_mm, addr, pte, entry);
}
} else if (WARN_ON_ONCE(vmemmap_optimizable_pfn(pfn)))
return NULL;
return pte;
}
@ -435,6 +459,9 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
pmd_t *pmd;
for (addr = start; addr < end; addr = next) {
unsigned long pfn = page_to_pfn((struct page *)addr);
const struct mem_section *ms = __pfn_to_section(pfn);
next = pmd_addr_end(addr, end);
pgd = vmemmap_pgd_populate(addr, node);
@ -450,7 +477,7 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
return -ENOMEM;
pmd = pmd_offset(pud, addr);
if (pmd_none(pmdp_get(pmd))) {
if (pmd_none(pmdp_get(pmd)) && !section_vmemmap_optimizable(ms)) {
void *p;
p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
@ -468,8 +495,11 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
*/
return -ENOMEM;
}
} else if (vmemmap_check_pmd(pmd, node, addr, next))
} else if (vmemmap_check_pmd(pmd, node, addr, next)) {
if (WARN_ON_ONCE(section_vmemmap_optimizable(ms)))
return -EOPNOTSUPP;
continue;
}
if (vmemmap_populate_basepages(addr, next, node, altmap))
return -ENOMEM;
}

4
mm/sparse.c
View File

@ -305,8 +305,8 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
nid, NULL, NULL);
if (!map)
panic("Failed to allocate memmap for section %lu\n", pnum);
memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
PAGE_SIZE));
memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION,
NULL, NULL));
sparse_init_early_section(nid, map, pnum, 0);
}
}

7
mm/sparse.h
View File

@ -111,8 +111,15 @@ static inline void sparse_init(void) {}
*/
#ifdef CONFIG_SPARSEMEM_VMEMMAP
void sparse_init_subsection_map(void);
int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
#else
static inline void sparse_init_subsection_map(void) {}
static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
{
return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
}
#endif /* CONFIG_SPARSEMEM_VMEMMAP */
#endif /* __MM_SPARSE_H */