diff --git a/include/linux/memblock.h b/include/linux/memblock.h index f142c8a8cd5b..678fe466529a 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -51,6 +51,7 @@ extern unsigned long long max_possible_pfn; * memory reservations yet, so we get scratch memory from the previous * kernel that we know is good to use. It is the only memory that * allocations may happen from in this phase. + * @MEMBLOCK_RSRV_HUGETLB: memory is reserved for hugetlb pages */ enum memblock_flags { MEMBLOCK_NONE = 0x0, /* No special request */ @@ -61,6 +62,7 @@ enum memblock_flags { MEMBLOCK_RSRV_NOINIT = 0x10, /* don't initialize struct pages */ MEMBLOCK_RSRV_KERN = 0x20, /* memory reserved for kernel use */ MEMBLOCK_KHO_SCRATCH = 0x40, /* scratch memory for kexec handover */ + MEMBLOCK_RSRV_HUGETLB = 0x80, /* memory reserved for hugetlb pages */ }; /** @@ -420,6 +422,7 @@ void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align, void *memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, phys_addr_t max_addr, int nid); +void *memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid); static __always_inline void *memblock_alloc(phys_addr_t size, phys_addr_t align) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5bfbfc7e463e..65da7e6b2812 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3029,14 +3029,7 @@ static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) if (hugetlb_early_cma(h)) return hugetlb_cma_alloc_bootmem(h, nid, node_exact); - if (node_exact) - return memblock_alloc_exact_nid_raw(huge_page_size(h), - huge_page_size(h), 0, - MEMBLOCK_ALLOC_ACCESSIBLE, nid); - - return memblock_alloc_try_nid_raw(huge_page_size(h), - huge_page_size(h), 0, - MEMBLOCK_ALLOC_ACCESSIBLE, nid); + return memblock_alloc_hugetlb(huge_page_size(h), nid, node_exact); } void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) diff --git a/mm/memblock.c b/mm/memblock.c index ea735bcd4dd1..77c6540e7528 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1504,6 +1504,32 @@ int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, return 0; } +static void memblock_prep_allocation(phys_addr_t start, phys_addr_t size, + bool kmemleak_trace) +{ + /* + * Skip kmemleak for those places like kasan_init() and + * early_pgtable_alloc() due to high volume. + */ + if (kmemleak_trace) + /* + * Memblock allocated blocks are never reported as + * leaks. This is because many of these blocks are + * only referred via the physical address which is + * not looked up by kmemleak. + */ + kmemleak_alloc_phys(start, size, 0); + + /* + * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, + * require memory to be accepted before it can be used by the + * guest. + * + * Accept the memory of the allocated buffer. + */ + accept_memory(start, size); +} + /** * memblock_alloc_range_nid - allocate boot memory block * @size: size of memory block to be allocated in bytes @@ -1578,28 +1604,7 @@ phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, return 0; done: - /* - * Skip kmemleak for those places like kasan_init() and - * early_pgtable_alloc() due to high volume. - */ - if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) - /* - * Memblock allocated blocks are never reported as - * leaks. This is because many of these blocks are - * only referred via the physical address which is - * not looked up by kmemleak. - */ - kmemleak_alloc_phys(found, size, 0); - - /* - * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, - * require memory to be accepted before it can be used by the - * guest. - * - * Accept the memory of the allocated buffer. - */ - accept_memory(found, size); - + memblock_prep_allocation(found, size, end != MEMBLOCK_ALLOC_NOLEAKTRACE); return found; } @@ -1754,6 +1759,77 @@ void * __init memblock_alloc_try_nid_raw( false); } +/** + * memblock_alloc_hugetlb - allocate boot memory for HugeTLB pages + * @size: size of the memory to be allocated in bytes + * @nid: nid of the free memory to find, %NUMA_NO_NODE for any node + * @exact_nid: only allocate from the specified nid. If %false, the specified + * nid is tried first, and then all nodes are tried as fallback. + * + * HugeTLB pages are always aligned by their size, so the alignment matches + * @size. Since the memory is for userspace, mirrored memory is not used. The + * memory is not zeroed. Does not panic if request cannot be satisfied. + * + * Return: + * Virtual address of allocated memory block on success, %NULL on failure. + */ +void * __init memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid) +{ + enum memblock_flags flags = choose_memblock_flags(); + phys_addr_t addr, start = 0, end = MEMBLOCK_ALLOC_ACCESSIBLE; + + memblock_dbg("%s: %llu bytes, nid=%d, exact_nid=%d %pS\n", __func__, + (u64)size, nid, exact_nid, (void *)_RET_IP_); + + /* Don't waste mirrored memory on HugeTLB pages. */ + flags &= ~MEMBLOCK_MIRROR; +retry: + /* HugeTLB pages are always aligned by their size. */ + addr = memblock_find_in_range_node(size, size, start, end, nid, flags); + if (addr) + goto found; + + /* Try all nodes if allowed. */ + if (numa_valid_node(nid) && !exact_nid) { + nid = NUMA_NO_NODE; + /* + * If a previous candidate overlapped with KHO scratch, it would + * update start or end. Now that the search is opening to all + * nodes, reset them. + */ + start = 0; + end = MEMBLOCK_ALLOC_ACCESSIBLE; + + goto retry; + } + + /* Found nothing... :-( */ + return NULL; + +found: + /* + * HugeTLB pages can be preserved with KHO and no preserved memory can + * be in scratch. So retry if found address overlaps with scratch. + * + * Scratch areas are normally not very large, so this shouldn't take too + * many retries. + */ + if (kho_scratch_overlap(addr, size)) { + if (memblock_bottom_up()) + start = addr + size; + else + end = addr; + + goto retry; + } + + if (__memblock_reserve(addr, size, nid, MEMBLOCK_RSRV_KERN | MEMBLOCK_RSRV_HUGETLB)) + return NULL; + + memblock_prep_allocation(addr, size, true); + return phys_to_virt(addr); +} + /** * memblock_alloc_try_nid - allocate boot memory block * @size: size of memory block to be allocated in bytes