mirror of
https://github.com/torvalds/linux.git
synced 2026-09-11 20:13:02 +02:00
kho: make boot time huge page allocation work nicely with KHO
Today allocation of gigantic pages in HugeTLB cannot work reliably with KHO:
* HugeTLB allocates gigantic pages using memblock and autoscaling of KHO
scratch accounts for these allocations. When gigantic pages occupy half
of the memory of more, KHO fails to allocate its scratch memory.
* After kexec handover, memblock allocations exclusively use KHO scratch
that is not supposed to contain preserved memory. This essentially blocks
preservation of HugeTLB with gigantic pages.
Extend early memory pools available for KHO kernel with areas that are
guaranteed not to contain preserved memory.
-----BEGIN PGP SIGNATURE-----
iQEzBAABCgAdFiEEeOVYVaWZL5900a/pOQOGJssO/ZEFAmqK9NwACgkQOQOGJssO
/ZGN3wgAqX/mXawYnhwDW2J931VsT54RuEctSNTCZ4Va8CWfeVjSV2bD2BlM+ibi
VtsvEAIdKb8tyx3t+3JLR3jrANE5XcxeDiS7sJG7QWaek6G++GdAmrm7q98rU7Pc
rqX8kMf65AZpHuV5wzKgF1fuYYur5Y4sKK00GVq+hPyWshmeYhaa+nGtJNe67D1a
CFw38r5WAPs/DwyvWg/3yfupbgTG6OShHPnKxqR7aaOJE4YnD3snsBM7hot/ZI7e
kz4TqixkxKn1RXq0XDcj8w11LxhxxsI67x02Fnnc1ClgMynCgDOvRXW6B93qRIRM
ZUK8fbzIxFDQHnfzWRXcIIsN7r54Ow==
=JHVN
-----END PGP SIGNATURE-----
Merge tag 'liveupdate-v7.3-rc1-20260823' of git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux
Pull more liveupdate updates from Mike Rapoport:
"Make boot time huge page allocation work nicely with kexec handover.
Today allocation of gigantic pages in HugeTLB cannot work reliably
with kexec handover (KHO):
- HugeTLB allocates gigantic pages using memblock and autoscaling of
KHO scratch accounts for these allocations. When gigantic pages
occupy half of the memory of more, KHO fails to allocate its
scratch memory.
- After kexec handover, memblock allocations exclusively use KHO
scratch that is not supposed to contain preserved memory. This
essentially blocks preservation of HugeTLB with gigantic pages.
Extend early memory pools available for KHO kernel with areas that are
guaranteed not to contain preserved memory"
* tag 'liveupdate-v7.3-rc1-20260823' of git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux: (21 commits)
kho: exclude hugetlb memory from scratch size calculation
memblock: add memblock_reserved_hugetlb_size()
memblock: make HugeTLB bootmem allocation work with KHO
memblock: always include KHO headers
kho: extend scratch
mm/mm_init: don't rely on memblock to get KHO scratch migratetype
kho: initialize preserved memory map radix tree earlier
kho: initialize kho_scratch pointer earlier in boot
kho: expose kho_scratch_overlap() to kexec_handover.h
kho: add kho_radix_init_tree()
kho: allow destroying KHO radix tree
kho: allow early-boot usage of the KHO radix tree
kho: add data argument to radix walk callback
kho: add callback for table pages
kho: add a struct for radix callbacks
kho: move all memory retrieval logic to kho_mem_retrieve()
kho: store incoming radix tree in kho_in
kho: disallow wide keys in radix tree
kho: make radix max key width more obvious
kho: generalize radix tree APIs
...
This commit is contained in:
Linus Torvalds
commit
91959a31a3
|
|
@ -14381,6 +14381,7 @@ S: Maintained
|
|||
T: git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
|
||||
F: Documentation/admin-guide/mm/kho.rst
|
||||
F: Documentation/core-api/kho/*
|
||||
F: include/asm-generic/kexec_handover.h
|
||||
F: include/linux/kexec_handover.h
|
||||
F: include/linux/kho/
|
||||
F: include/linux/kho_block.h
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@
|
|||
#include <asm/setup.h> /* For COMMAND_LINE_SIZE */
|
||||
#undef _SETUP
|
||||
|
||||
#include <asm/kexec_handover.h>
|
||||
|
||||
extern unsigned long get_cmd_line_ptr(void);
|
||||
|
||||
/* Simplified build-specific string for starting entropy. */
|
||||
|
|
|
|||
|
|
@ -15,3 +15,4 @@ generic-y += fprobe.h
|
|||
generic-y += mcs_spinlock.h
|
||||
generic-y += mmzone.h
|
||||
generic-y += ring_buffer.h
|
||||
generic-y += kexec_handover.h
|
||||
|
|
|
|||
|
|
@ -69,8 +69,6 @@ extern void x86_ce4100_early_setup(void);
|
|||
static inline void x86_ce4100_early_setup(void) { }
|
||||
#endif
|
||||
|
||||
#include <linux/kexec_handover.h>
|
||||
|
||||
#ifndef _SETUP
|
||||
|
||||
#include <asm/espfix.h>
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
#include <linux/init_ohci1394_dma.h>
|
||||
#include <linux/initrd.h>
|
||||
#include <linux/iscsi_ibft.h>
|
||||
#include <linux/kexec_handover.h>
|
||||
#include <linux/memblock.h>
|
||||
#include <linux/panic_notifier.h>
|
||||
#include <linux/pci.h>
|
||||
|
|
|
|||
12
include/asm-generic/kexec_handover.h
Normal file
12
include/asm-generic/kexec_handover.h
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef __ASM_GENERIC_KEXEC_HANDOVER_H
|
||||
#define __ASM_GENERIC_KEXEC_HANDOVER_H
|
||||
|
||||
#include <linux/types.h>
|
||||
|
||||
struct kho_scratch {
|
||||
phys_addr_t addr;
|
||||
phys_addr_t size;
|
||||
};
|
||||
|
||||
#endif /* __ASM_GENERIC_KEXEC_HANDOVER_H */
|
||||
|
|
@ -5,11 +5,8 @@
|
|||
#include <linux/err.h>
|
||||
#include <linux/errno.h>
|
||||
#include <linux/types.h>
|
||||
|
||||
struct kho_scratch {
|
||||
phys_addr_t addr;
|
||||
phys_addr_t size;
|
||||
};
|
||||
#include <linux/mm.h>
|
||||
#include <asm-generic/kexec_handover.h>
|
||||
|
||||
struct kho_vmalloc;
|
||||
|
||||
|
|
@ -37,9 +34,20 @@ void kho_remove_subtree(void *blob);
|
|||
int kho_retrieve_subtree(const char *name, phys_addr_t *phys, size_t *size);
|
||||
|
||||
void kho_memory_init(void);
|
||||
void kho_memory_init_early(void);
|
||||
|
||||
void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys,
|
||||
u64 scratch_len);
|
||||
|
||||
bool kho_scratch_overlap(phys_addr_t phys, size_t size);
|
||||
|
||||
static inline enum migratetype kho_scratch_migratetype(unsigned long pfn,
|
||||
enum migratetype mt)
|
||||
{
|
||||
if (kho_scratch_overlap(PFN_PHYS(pfn), pageblock_nr_pages << PAGE_SHIFT))
|
||||
return MIGRATE_CMA;
|
||||
return mt;
|
||||
}
|
||||
#else
|
||||
static inline bool kho_is_enabled(void)
|
||||
{
|
||||
|
|
@ -112,10 +120,23 @@ static inline int kho_retrieve_subtree(const char *name, phys_addr_t *phys,
|
|||
|
||||
static inline void kho_memory_init(void) { }
|
||||
|
||||
static inline void kho_memory_init_early(void) { }
|
||||
|
||||
static inline void kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
|
||||
phys_addr_t scratch_phys, u64 scratch_len)
|
||||
{
|
||||
}
|
||||
|
||||
static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline enum migratetype kho_scratch_migratetype(unsigned long pfn,
|
||||
enum migratetype mt)
|
||||
{
|
||||
return mt;
|
||||
}
|
||||
#endif /* CONFIG_KEXEC_HANDOVER */
|
||||
|
||||
#endif /* LINUX_KEXEC_HANDOVER_H */
|
||||
|
|
|
|||
|
|
@ -257,11 +257,8 @@ struct kho_vmalloc {
|
|||
* memory. These constants govern the indexing, sizing, and depth of the tree.
|
||||
*/
|
||||
enum kho_radix_consts {
|
||||
/*
|
||||
* The bit position of the order bit (and also the length of the
|
||||
* shifted physical address) for an order-0 page.
|
||||
*/
|
||||
KHO_ORDER_0_LOG2 = 64 - PAGE_SHIFT,
|
||||
/* Need to store the PFN, plus one bit for order. */
|
||||
KHO_RADIX_KEY_WIDTH = 64 - PAGE_SHIFT + 1,
|
||||
|
||||
/* Size of the table in kho_radix_node, in log2 */
|
||||
KHO_TABLE_SIZE_LOG2 = const_ilog2(PAGE_SIZE / sizeof(phys_addr_t)),
|
||||
|
|
@ -274,7 +271,7 @@ enum kho_radix_consts {
|
|||
* and 1 bitmap level.
|
||||
*/
|
||||
KHO_TREE_MAX_DEPTH =
|
||||
DIV_ROUND_UP(KHO_ORDER_0_LOG2 - KHO_BITMAP_SIZE_LOG2 + 1,
|
||||
DIV_ROUND_UP(KHO_RADIX_KEY_WIDTH - KHO_BITMAP_SIZE_LOG2,
|
||||
KHO_TABLE_SIZE_LOG2) + 1,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -11,15 +11,19 @@
|
|||
/**
|
||||
* DOC: Kexec Handover Radix Tree
|
||||
*
|
||||
* This is a radix tree implementation for tracking physical memory pages
|
||||
* across kexec transitions. It was developed for the KHO mechanism but is
|
||||
* designed for broader use by any subsystem that needs to preserve pages.
|
||||
* This is a radix tree implementation for tracking numeric keys across kexec
|
||||
* transitions. It was developed for the KHO preserved memory map but is
|
||||
* designed for broader use by any subsystem that needs to track keys.
|
||||
* Conceptually speaking, the data structure is similar to a set. It tracks the
|
||||
* presence or absence of numeric keys.
|
||||
*
|
||||
* The radix tree is a multi-level tree where leaf nodes are bitmaps
|
||||
* representing individual pages. To allow pages of different sizes (orders)
|
||||
* representing individual keys.
|
||||
*
|
||||
* For the KHO preserved memory map, to allow pages of different sizes (orders)
|
||||
* to be stored efficiently in a single tree, it uses a unique key encoding
|
||||
* scheme. Each key is an unsigned long that combines a page's physical
|
||||
* address and its order.
|
||||
* scheme. Each key is an unsigned long that combines a page's physical address
|
||||
* and its order.
|
||||
*
|
||||
* Client code is responsible for allocating the root node of the tree,
|
||||
* initializing the mutex lock, and managing its lifecycle. It must use the
|
||||
|
|
@ -34,37 +38,53 @@ struct kho_radix_tree {
|
|||
struct mutex lock; /* protects the tree's structure and root pointer */
|
||||
};
|
||||
|
||||
typedef int (*kho_radix_tree_walk_callback_t)(phys_addr_t phys,
|
||||
unsigned int order);
|
||||
/**
|
||||
* struct kho_radix_walk_cb - Callbacks for KHO radix tree walk.
|
||||
* @leaf: Called on each present key in the radix tree.
|
||||
* @node: Called on each node of the radix tree itself. Receives the
|
||||
* physical address of the page containing the node.
|
||||
*
|
||||
* For each callback, a return value of 0 continues the walk and a non-zero
|
||||
* return value is directly returned to the caller.
|
||||
*/
|
||||
struct kho_radix_walk_cb {
|
||||
int (*leaf)(unsigned long key, void *data);
|
||||
int (*node)(phys_addr_t phys, void *data);
|
||||
};
|
||||
|
||||
#ifdef CONFIG_KEXEC_HANDOVER
|
||||
|
||||
int kho_radix_add_page(struct kho_radix_tree *tree, unsigned long pfn,
|
||||
unsigned int order);
|
||||
|
||||
void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn,
|
||||
unsigned int order);
|
||||
|
||||
int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key);
|
||||
void kho_radix_del_key(struct kho_radix_tree *tree, unsigned long key);
|
||||
int kho_radix_walk_tree(struct kho_radix_tree *tree,
|
||||
kho_radix_tree_walk_callback_t cb);
|
||||
const struct kho_radix_walk_cb *cb, void *data);
|
||||
int kho_radix_init_tree(struct kho_radix_tree *tree, struct kho_radix_node *root);
|
||||
void kho_radix_destroy_tree(struct kho_radix_tree *tree);
|
||||
|
||||
#else /* #ifdef CONFIG_KEXEC_HANDOVER */
|
||||
|
||||
static inline int kho_radix_add_page(struct kho_radix_tree *tree, long pfn,
|
||||
unsigned int order)
|
||||
static inline int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
|
||||
static inline void kho_radix_del_page(struct kho_radix_tree *tree,
|
||||
unsigned long pfn, unsigned int order) { }
|
||||
static inline void kho_radix_del_key(struct kho_radix_tree *tree,
|
||||
unsigned long key) { }
|
||||
|
||||
static inline int kho_radix_walk_tree(struct kho_radix_tree *tree,
|
||||
kho_radix_tree_walk_callback_t cb)
|
||||
const struct kho_radix_walk_cb *cb, void *data)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
|
||||
static inline int kho_radix_init_tree(struct kho_radix_tree *tree,
|
||||
struct kho_radix_node *root)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void kho_radix_destroy_tree(struct kho_radix_tree *tree) { }
|
||||
|
||||
#endif /* #ifdef CONFIG_KEXEC_HANDOVER */
|
||||
|
||||
#endif /* _LINUX_KHO_RADIX_TREE_H */
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ extern unsigned long long max_possible_pfn;
|
|||
* memory reservations yet, so we get scratch memory from the previous
|
||||
* kernel that we know is good to use. It is the only memory that
|
||||
* allocations may happen from in this phase.
|
||||
* @MEMBLOCK_RSRV_HUGETLB: memory is reserved for hugetlb pages
|
||||
*/
|
||||
enum memblock_flags {
|
||||
MEMBLOCK_NONE = 0x0, /* No special request */
|
||||
|
|
@ -61,6 +62,7 @@ enum memblock_flags {
|
|||
MEMBLOCK_RSRV_NOINIT = 0x10, /* don't initialize struct pages */
|
||||
MEMBLOCK_RSRV_KERN = 0x20, /* memory reserved for kernel use */
|
||||
MEMBLOCK_KHO_SCRATCH = 0x40, /* scratch memory for kexec handover */
|
||||
MEMBLOCK_RSRV_HUGETLB = 0x80, /* memory reserved for hugetlb pages */
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -420,6 +422,7 @@ void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align,
|
|||
void *memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align,
|
||||
phys_addr_t min_addr, phys_addr_t max_addr,
|
||||
int nid);
|
||||
void *memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid);
|
||||
|
||||
static __always_inline void *memblock_alloc(phys_addr_t size, phys_addr_t align)
|
||||
{
|
||||
|
|
@ -484,6 +487,7 @@ static inline __init_memblock bool memblock_bottom_up(void)
|
|||
phys_addr_t memblock_phys_mem_size(void);
|
||||
phys_addr_t memblock_reserved_size(void);
|
||||
phys_addr_t memblock_reserved_kern_size(phys_addr_t limit, int nid);
|
||||
phys_addr_t memblock_reserved_hugetlb_size(phys_addr_t limit, int nid);
|
||||
unsigned long memblock_estimated_nr_free_pages(void);
|
||||
phys_addr_t memblock_start_of_DRAM(void);
|
||||
phys_addr_t memblock_end_of_DRAM(void);
|
||||
|
|
@ -613,28 +617,9 @@ static inline void memtest_report_meminfo(struct seq_file *m) { }
|
|||
#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH
|
||||
void memblock_set_kho_scratch_only(void);
|
||||
void memblock_clear_kho_scratch_only(void);
|
||||
bool memblock_is_kho_scratch_memory(phys_addr_t addr);
|
||||
|
||||
static inline enum migratetype kho_scratch_migratetype(unsigned long pfn,
|
||||
enum migratetype mt)
|
||||
{
|
||||
if (memblock_is_kho_scratch_memory(PFN_PHYS(pfn)))
|
||||
return MIGRATE_CMA;
|
||||
return mt;
|
||||
}
|
||||
#else
|
||||
static inline void memblock_set_kho_scratch_only(void) { }
|
||||
static inline void memblock_clear_kho_scratch_only(void) { }
|
||||
static inline bool memblock_is_kho_scratch_memory(phys_addr_t addr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline enum migratetype kho_scratch_migratetype(unsigned long pfn,
|
||||
enum migratetype mt)
|
||||
{
|
||||
return mt;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _LINUX_MEMBLOCK_H */
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ luo-y := \
|
|||
luo_session.o
|
||||
|
||||
obj-$(CONFIG_KEXEC_HANDOVER) += kexec_handover.o
|
||||
obj-$(CONFIG_KEXEC_HANDOVER_DEBUG) += kexec_handover_debug.o
|
||||
obj-$(CONFIG_KEXEC_HANDOVER_DEBUGFS) += kexec_handover_debugfs.o
|
||||
|
||||
obj-$(CONFIG_LIVEUPDATE) += luo.o
|
||||
|
|
|
|||
|
|
@ -95,8 +95,25 @@ static struct kho_out kho_out = {
|
|||
},
|
||||
};
|
||||
|
||||
struct kho_in {
|
||||
phys_addr_t fdt_phys;
|
||||
phys_addr_t scratch_phys;
|
||||
char previous_release[__NEW_UTS_LEN + 1];
|
||||
u32 kexec_count;
|
||||
struct kho_debugfs dbg;
|
||||
struct kho_radix_tree radix_tree;
|
||||
};
|
||||
|
||||
static struct kho_in kho_in = {
|
||||
};
|
||||
|
||||
static const void *kho_get_fdt(void)
|
||||
{
|
||||
return kho_in.fdt_phys ? phys_to_virt(kho_in.fdt_phys) : NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* kho_radix_encode_key - Encodes a physical address and order into a radix key.
|
||||
* kho_encode_radix_key - Encodes a physical address and order into a radix key.
|
||||
* @phys: The physical address of the page.
|
||||
* @order: The order of the page.
|
||||
*
|
||||
|
|
@ -106,35 +123,38 @@ static struct kho_out kho_out = {
|
|||
*
|
||||
* Return: The encoded unsigned long radix key.
|
||||
*/
|
||||
static unsigned long kho_radix_encode_key(phys_addr_t phys, unsigned int order)
|
||||
static unsigned long kho_encode_radix_key(phys_addr_t phys, unsigned int order)
|
||||
{
|
||||
/* Order bits part */
|
||||
unsigned long h = 1UL << (KHO_ORDER_0_LOG2 - order);
|
||||
/* Shifted physical address part */
|
||||
unsigned long l = phys >> (PAGE_SHIFT + order);
|
||||
/* The physical address is encoded by shifting the PFN by its order. */
|
||||
unsigned long shift = PAGE_SHIFT + order;
|
||||
/* Order bit goes right before the shifted PFN. */
|
||||
unsigned long h = 1UL << (64 - shift);
|
||||
/* Shifted PFN. */
|
||||
unsigned long l = phys >> shift;
|
||||
|
||||
return h | l;
|
||||
}
|
||||
|
||||
/**
|
||||
* kho_radix_decode_key - Decodes a radix key back into a physical address and order.
|
||||
* kho_decode_radix_key - Decodes a radix key back into a physical address and order.
|
||||
* @key: The unsigned long key to decode.
|
||||
* @order: An output parameter, a pointer to an unsigned int where the decoded
|
||||
* page order will be stored.
|
||||
*
|
||||
* This function reverses the encoding performed by kho_radix_encode_key(),
|
||||
* This function reverses the encoding performed by kho_encode_radix_key(),
|
||||
* extracting the original physical address and page order from a given key.
|
||||
*
|
||||
* Return: The decoded physical address.
|
||||
*/
|
||||
static phys_addr_t kho_radix_decode_key(unsigned long key, unsigned int *order)
|
||||
static phys_addr_t kho_decode_radix_key(unsigned long key, unsigned int *order)
|
||||
{
|
||||
unsigned int order_bit = fls64(key);
|
||||
/* fls64() indexes starting from 1. */
|
||||
unsigned int order_bit = fls64(key) - 1;
|
||||
phys_addr_t phys;
|
||||
|
||||
/* order_bit is numbered starting at 1 from fls64 */
|
||||
*order = KHO_ORDER_0_LOG2 - order_bit + 1;
|
||||
/* The order is discarded by the shift */
|
||||
/* order bit goes right before the shifted PFN. */
|
||||
*order = 64 - (PAGE_SHIFT + order_bit);
|
||||
/* The order bit is discarded by the shift */
|
||||
phys = key << (PAGE_SHIFT + *order);
|
||||
|
||||
return phys;
|
||||
|
|
@ -154,25 +174,47 @@ static unsigned long kho_radix_get_table_index(unsigned long key,
|
|||
return (key >> s) % (1 << KHO_TABLE_SIZE_LOG2);
|
||||
}
|
||||
|
||||
static void __ref *kho_radix_alloc_node(void)
|
||||
{
|
||||
struct kho_radix_node *node;
|
||||
|
||||
if (slab_is_available())
|
||||
node = (struct kho_radix_node *)get_zeroed_page(GFP_KERNEL);
|
||||
else
|
||||
node = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
static void __ref kho_radix_free_node(struct kho_radix_node *node)
|
||||
{
|
||||
if (slab_is_available())
|
||||
free_page((unsigned long)node);
|
||||
else
|
||||
memblock_free(node, PAGE_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* kho_radix_add_page - Marks a page as preserved in the radix tree.
|
||||
* kho_radix_add_key - Add a key to the radix tree.
|
||||
* @tree: The KHO radix tree.
|
||||
* @pfn: The page frame number of the page to preserve.
|
||||
* @order: The order of the page.
|
||||
* @key: The key to add.
|
||||
*
|
||||
* This function traverses the radix tree based on the key derived from @pfn
|
||||
* and @order. It sets the corresponding bit in the leaf bitmap to mark the
|
||||
* page for preservation. If intermediate nodes do not exist along the path,
|
||||
* they are allocated and added to the tree.
|
||||
* This function traverses the radix tree based on the @key provided. It sets the
|
||||
* corresponding bit in the leaf bitmap to mark the @key as present. If
|
||||
* intermediate nodes do not exist along the path, they are allocated and added
|
||||
* to the tree.
|
||||
*
|
||||
* NOTE: Currently only keys of width up to %KHO_RADIX_KEY_WIDTH are supported.
|
||||
* This limit only exists because current users of the radix tree don't use more
|
||||
* than that. Changing the maximum width requires changing the tree depth, which
|
||||
* needs bumping the ABI version.
|
||||
*
|
||||
* Return: 0 on success, or a negative error code on failure.
|
||||
*/
|
||||
int kho_radix_add_page(struct kho_radix_tree *tree,
|
||||
unsigned long pfn, unsigned int order)
|
||||
int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key)
|
||||
{
|
||||
/* Newly allocated nodes for error cleanup */
|
||||
struct kho_radix_node *intermediate_nodes[KHO_TREE_MAX_DEPTH] = { 0 };
|
||||
unsigned long key = kho_radix_encode_key(PFN_PHYS(pfn), order);
|
||||
struct kho_radix_node *anchor_node = NULL;
|
||||
struct kho_radix_node *node = tree->root;
|
||||
struct kho_radix_node *new_node;
|
||||
|
|
@ -183,6 +225,9 @@ int kho_radix_add_page(struct kho_radix_tree *tree,
|
|||
if (WARN_ON_ONCE(!tree->root))
|
||||
return -EINVAL;
|
||||
|
||||
if (unlikely(fls64(key) > KHO_RADIX_KEY_WIDTH))
|
||||
return -ERANGE;
|
||||
|
||||
might_sleep();
|
||||
|
||||
guard(mutex)(&tree->lock);
|
||||
|
|
@ -197,7 +242,7 @@ int kho_radix_add_page(struct kho_radix_tree *tree,
|
|||
}
|
||||
|
||||
/* Next node is empty, create a new node for it */
|
||||
new_node = (struct kho_radix_node *)get_zeroed_page(GFP_KERNEL);
|
||||
new_node = kho_radix_alloc_node();
|
||||
if (!new_node) {
|
||||
err = -ENOMEM;
|
||||
goto err_free_nodes;
|
||||
|
|
@ -228,29 +273,26 @@ int kho_radix_add_page(struct kho_radix_tree *tree,
|
|||
err_free_nodes:
|
||||
for (i = KHO_TREE_MAX_DEPTH - 1; i > 0; i--) {
|
||||
if (intermediate_nodes[i])
|
||||
free_page((unsigned long)intermediate_nodes[i]);
|
||||
kho_radix_free_node(intermediate_nodes[i]);
|
||||
}
|
||||
if (anchor_node)
|
||||
anchor_node->table[anchor_idx] = 0;
|
||||
|
||||
return err;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kho_radix_add_page);
|
||||
EXPORT_SYMBOL_GPL(kho_radix_add_key);
|
||||
|
||||
/**
|
||||
* kho_radix_del_page - Removes a page's preservation status from the radix tree.
|
||||
* kho_radix_del_key - Removes the key from the radix tree.
|
||||
* @tree: The KHO radix tree.
|
||||
* @pfn: The page frame number of the page to unpreserve.
|
||||
* @order: The order of the page.
|
||||
* @key: The key to remove.
|
||||
*
|
||||
* This function traverses the radix tree and clears the bit corresponding to
|
||||
* the page, effectively removing its "preserved" status. It does not free
|
||||
* the tree's intermediate nodes, even if they become empty.
|
||||
* the @key, effectively removing it from the tree. It does not free the tree's
|
||||
* intermediate nodes, even if they become empty.
|
||||
*/
|
||||
void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn,
|
||||
unsigned int order)
|
||||
void kho_radix_del_key(struct kho_radix_tree *tree, unsigned long key)
|
||||
{
|
||||
unsigned long key = kho_radix_encode_key(PFN_PHYS(pfn), order);
|
||||
struct kho_radix_node *node = tree->root;
|
||||
struct kho_radix_leaf *leaf;
|
||||
unsigned int i, idx;
|
||||
|
|
@ -258,6 +300,10 @@ void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn,
|
|||
if (WARN_ON_ONCE(!tree->root))
|
||||
return;
|
||||
|
||||
/* Keys wider than KHO_RADIX_KEY_WIDTH are not allowed to be added. */
|
||||
if (unlikely(fls64(key) > KHO_RADIX_KEY_WIDTH))
|
||||
return;
|
||||
|
||||
might_sleep();
|
||||
|
||||
guard(mutex)(&tree->lock);
|
||||
|
|
@ -281,21 +327,85 @@ void kho_radix_del_page(struct kho_radix_tree *tree, unsigned long pfn,
|
|||
idx = kho_radix_get_bitmap_index(key);
|
||||
__clear_bit(idx, leaf->bitmap);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kho_radix_del_page);
|
||||
EXPORT_SYMBOL_GPL(kho_radix_del_key);
|
||||
|
||||
static int kho_radix_walk_leaf(struct kho_radix_leaf *leaf,
|
||||
unsigned long key,
|
||||
kho_radix_tree_walk_callback_t cb)
|
||||
static void __kho_radix_destroy_tree(struct kho_radix_node *root,
|
||||
unsigned int level)
|
||||
{
|
||||
unsigned long i;
|
||||
|
||||
if (level == 0) {
|
||||
kho_radix_free_node(root);
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = 0; i < PAGE_SIZE / sizeof(phys_addr_t); i++) {
|
||||
if (root->table[i])
|
||||
__kho_radix_destroy_tree(phys_to_virt(root->table[i]),
|
||||
level - 1);
|
||||
}
|
||||
|
||||
kho_radix_free_node(root);
|
||||
}
|
||||
|
||||
/**
|
||||
* kho_radix_init_tree - initialize the radix tree.
|
||||
* @tree: the tree to initialize.
|
||||
* @root: root table of the radix tree.
|
||||
*
|
||||
* Initialize the radix tree with the given root node. If root is %NULL, an
|
||||
* empty root table is allocated. If root is not %NULL, it is the caller's
|
||||
* responsibility to make sure the root is valid and in the correct format.
|
||||
*
|
||||
* Return: 0 on success, -errno on failure.
|
||||
*/
|
||||
int kho_radix_init_tree(struct kho_radix_tree *tree, struct kho_radix_node *root)
|
||||
{
|
||||
if (!root)
|
||||
root = kho_radix_alloc_node();
|
||||
if (!root)
|
||||
return -ENOMEM;
|
||||
|
||||
tree->root = root;
|
||||
mutex_init(&tree->lock);
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kho_radix_init_tree);
|
||||
|
||||
/**
|
||||
* kho_radix_destroy_tree - Destroy the radix tree
|
||||
* @tree: The radix tree to destroy
|
||||
*
|
||||
* Walk @tree and free all its nodes.
|
||||
*/
|
||||
void kho_radix_destroy_tree(struct kho_radix_tree *tree)
|
||||
{
|
||||
if (!tree->root)
|
||||
return;
|
||||
|
||||
__kho_radix_destroy_tree(tree->root, KHO_TREE_MAX_DEPTH - 1);
|
||||
tree->root = NULL;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kho_radix_destroy_tree);
|
||||
|
||||
static int kho_radix_walk_leaf(struct kho_radix_leaf *leaf, unsigned long key,
|
||||
const struct kho_radix_walk_cb *cb, void *data)
|
||||
{
|
||||
unsigned long *bitmap = (unsigned long *)leaf;
|
||||
unsigned int order;
|
||||
phys_addr_t phys;
|
||||
unsigned int i;
|
||||
int err;
|
||||
|
||||
if (cb->node) {
|
||||
err = cb->node(virt_to_phys(leaf), data);
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
|
||||
if (!cb->leaf)
|
||||
return 0;
|
||||
|
||||
for_each_set_bit(i, bitmap, PAGE_SIZE * BITS_PER_BYTE) {
|
||||
phys = kho_radix_decode_key(key | i, &order);
|
||||
err = cb(phys, order);
|
||||
err = cb->leaf(key | i, data);
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
|
|
@ -305,7 +415,7 @@ static int kho_radix_walk_leaf(struct kho_radix_leaf *leaf,
|
|||
|
||||
static int __kho_radix_walk_tree(struct kho_radix_node *root,
|
||||
unsigned int level, unsigned long start,
|
||||
kho_radix_tree_walk_callback_t cb)
|
||||
const struct kho_radix_walk_cb *cb, void *data)
|
||||
{
|
||||
struct kho_radix_node *node;
|
||||
struct kho_radix_leaf *leaf;
|
||||
|
|
@ -313,6 +423,12 @@ static int __kho_radix_walk_tree(struct kho_radix_node *root,
|
|||
unsigned int shift;
|
||||
int err;
|
||||
|
||||
if (cb->node) {
|
||||
err = cb->node(virt_to_phys(root), data);
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
|
||||
for (i = 0; i < PAGE_SIZE / sizeof(phys_addr_t); i++) {
|
||||
if (!root->table[i])
|
||||
continue;
|
||||
|
|
@ -329,10 +445,10 @@ static int __kho_radix_walk_tree(struct kho_radix_node *root,
|
|||
* node is pointing to the level 0 bitmap.
|
||||
*/
|
||||
leaf = (struct kho_radix_leaf *)node;
|
||||
err = kho_radix_walk_leaf(leaf, key, cb);
|
||||
err = kho_radix_walk_leaf(leaf, key, cb, data);
|
||||
} else {
|
||||
err = __kho_radix_walk_tree(node, level - 1,
|
||||
key, cb);
|
||||
key, cb, data);
|
||||
}
|
||||
|
||||
if (err)
|
||||
|
|
@ -343,28 +459,27 @@ static int __kho_radix_walk_tree(struct kho_radix_node *root,
|
|||
}
|
||||
|
||||
/**
|
||||
* kho_radix_walk_tree - Traverses the radix tree and calls a callback for each preserved page.
|
||||
* kho_radix_walk_tree - Traverses the radix tree and calls a callback for each key.
|
||||
* @tree: A pointer to the KHO radix tree to walk.
|
||||
* @cb: A callback function of type kho_radix_tree_walk_callback_t that will be
|
||||
* invoked for each preserved page found in the tree. The callback receives
|
||||
* the physical address and order of the preserved page.
|
||||
* @cb: Set of callbacks to be invoked during the tree walk.
|
||||
* @data: Opaque data pointer passed to each callback in @cb.
|
||||
*
|
||||
* This function walks the radix tree, searching from the specified top level
|
||||
* down to the lowest level (level 0). For each preserved page found, it invokes
|
||||
* the provided callback, passing the page's physical address and order.
|
||||
* This function walks the radix tree, searching from the top level down to the
|
||||
* lowest level (level 0), invoking the appropriate callbacks.
|
||||
*
|
||||
* Return: 0 if the walk completed the specified tree, or the non-zero return
|
||||
* value from the callback that stopped the walk.
|
||||
*/
|
||||
int kho_radix_walk_tree(struct kho_radix_tree *tree,
|
||||
kho_radix_tree_walk_callback_t cb)
|
||||
const struct kho_radix_walk_cb *cb, void *data)
|
||||
{
|
||||
if (WARN_ON_ONCE(!tree->root))
|
||||
return -EINVAL;
|
||||
|
||||
guard(mutex)(&tree->lock);
|
||||
|
||||
return __kho_radix_walk_tree(tree->root, KHO_TREE_MAX_DEPTH - 1, 0, cb);
|
||||
return __kho_radix_walk_tree(tree->root, KHO_TREE_MAX_DEPTH - 1, 0, cb,
|
||||
data);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kho_radix_walk_tree);
|
||||
|
||||
|
|
@ -495,13 +610,16 @@ static struct page *__init kho_get_preserved_page(phys_addr_t phys,
|
|||
return pfn_to_page(pfn);
|
||||
}
|
||||
|
||||
static int __init kho_preserved_memory_reserve(phys_addr_t phys,
|
||||
unsigned int order)
|
||||
static int __init kho_preserved_memory_reserve(unsigned long key, void *data)
|
||||
{
|
||||
union kho_page_info info;
|
||||
struct page *page;
|
||||
unsigned int order;
|
||||
phys_addr_t phys;
|
||||
u64 sz;
|
||||
|
||||
phys = kho_decode_radix_key(key, &order);
|
||||
|
||||
sz = 1UL << (order + PAGE_SHIFT);
|
||||
page = kho_get_preserved_page(phys, order);
|
||||
|
||||
|
|
@ -530,6 +648,13 @@ static phys_addr_t __init kho_get_mem_map_phys(const void *fdt)
|
|||
return get_unaligned((const u64 *)mem_ptr);
|
||||
}
|
||||
|
||||
static void __init *kho_get_mem_map(const void *fdt)
|
||||
{
|
||||
phys_addr_t phys = kho_get_mem_map_phys(fdt);
|
||||
|
||||
return phys ? phys_to_virt(phys) : NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* With KHO enabled, memory can become fragmented because KHO regions may
|
||||
* be anywhere in physical address space. The scratch regions give us a
|
||||
|
|
@ -630,19 +755,24 @@ early_param("kho_scratch", kho_parse_scratch_size);
|
|||
static void __init scratch_size_update(void)
|
||||
{
|
||||
/*
|
||||
* If fixed sizes are not provided via command line, calculate them
|
||||
* now.
|
||||
* If fixed sizes are not provided via command line, calculate them now.
|
||||
* Remove HugeTLB allocations from it because they never get allocated
|
||||
* from scratch.
|
||||
*/
|
||||
if (scratch_scale) {
|
||||
phys_addr_t size;
|
||||
|
||||
size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT,
|
||||
NUMA_NO_NODE);
|
||||
size -= memblock_reserved_hugetlb_size(ARCH_LOW_ADDRESS_LIMIT,
|
||||
NUMA_NO_NODE);
|
||||
size = size * scratch_scale / 100;
|
||||
scratch_size_lowmem = size;
|
||||
|
||||
size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
|
||||
NUMA_NO_NODE);
|
||||
size -= memblock_reserved_hugetlb_size(MEMBLOCK_ALLOC_ANYWHERE,
|
||||
NUMA_NO_NODE);
|
||||
size = size * scratch_scale / 100 - scratch_size_lowmem;
|
||||
scratch_size_global = size;
|
||||
}
|
||||
|
|
@ -662,6 +792,9 @@ static phys_addr_t __init scratch_size_node(int nid)
|
|||
if (scratch_scale) {
|
||||
size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
|
||||
nid);
|
||||
/* Do not count HugeTLB pages. */
|
||||
size -= memblock_reserved_hugetlb_size(MEMBLOCK_ALLOC_ANYWHERE,
|
||||
nid);
|
||||
size = size * scratch_scale / 100;
|
||||
} else {
|
||||
size = scratch_size_pernode;
|
||||
|
|
@ -670,6 +803,22 @@ static phys_addr_t __init scratch_size_node(int nid)
|
|||
return round_up(size, SCRATCH_ALIGNMENT_BYTES);
|
||||
}
|
||||
|
||||
bool kho_scratch_overlap(phys_addr_t phys, size_t size)
|
||||
{
|
||||
phys_addr_t scratch_start, scratch_end;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < kho_scratch_cnt; i++) {
|
||||
scratch_start = kho_scratch[i].addr;
|
||||
scratch_end = kho_scratch[i].addr + kho_scratch[i].size;
|
||||
|
||||
if (phys < scratch_end && (phys + size) > scratch_start)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* kho_reserve_scratch - Reserve a contiguous chunk of memory for kexec
|
||||
*
|
||||
|
|
@ -757,6 +906,140 @@ static void __init kho_reserve_scratch(void)
|
|||
kho_enable = false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Look for free blocks of 1G. This is a heuristic chosen to work efficiently
|
||||
* with large systems with hundreds of gigabytes of memory. It will work poorly
|
||||
* on smaller systems. The algorithm itself doesn't depend on the actual value,
|
||||
* so it can be changed to a different heuristic later if needed.
|
||||
*/
|
||||
#define KHO_SCRATCH_EXT_BLKSIZE SZ_1G
|
||||
#define KHO_SCRATCH_EXT_BLKSHIFT const_ilog2(KHO_SCRATCH_EXT_BLKSIZE)
|
||||
|
||||
/* Called for the KHO preserved memory radix tree. */
|
||||
static int __init kho_ext_walk_leaf(unsigned long key, void *data)
|
||||
{
|
||||
struct kho_radix_tree *busy_blocks = data;
|
||||
phys_addr_t start, end;
|
||||
unsigned int order;
|
||||
int err;
|
||||
|
||||
/*
|
||||
* The key is from the KHO preserved memory radix tree. It is decoded to
|
||||
* a physical address of a preservation and its order.
|
||||
*/
|
||||
start = kho_decode_radix_key(key, &order);
|
||||
end = start + (1UL << (order + PAGE_SHIFT));
|
||||
|
||||
while (start < end) {
|
||||
err = kho_radix_add_key(busy_blocks, start >> KHO_SCRATCH_EXT_BLKSHIFT);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
start += (1UL << KHO_SCRATCH_EXT_BLKSHIFT);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Called for the KHO preserved memory radix tree. */
|
||||
static int __init kho_ext_walk_node(phys_addr_t phys, void *data)
|
||||
{
|
||||
struct kho_radix_tree *busy_blocks = data;
|
||||
|
||||
return kho_radix_add_key(busy_blocks, phys >> KHO_SCRATCH_EXT_BLKSHIFT);
|
||||
}
|
||||
|
||||
/* Called for the busy block radix tree. */
|
||||
static int __init kho_ext_mark_scratch(unsigned long key, void *data)
|
||||
{
|
||||
phys_addr_t *prev_end = data;
|
||||
phys_addr_t start = key << KHO_SCRATCH_EXT_BLKSHIFT;
|
||||
int err;
|
||||
|
||||
if (start > *prev_end) {
|
||||
err = memblock_mark_kho_scratch(*prev_end, start - *prev_end);
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
|
||||
*prev_end = start + (1UL << KHO_SCRATCH_EXT_BLKSHIFT);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* kho_extend_scratch - Extend the scratch regions
|
||||
*
|
||||
* The KHO preserved memory radix tree mixes both physical address and order
|
||||
* into a single key. This makes it hard to look for free ranges directly. This
|
||||
* function first walks the radix tree and digests it down into another radix
|
||||
* tree, whose keys identify blocks of size KHO_SCRATCH_EXT_BLKSIZE which
|
||||
* contain preserved memory.
|
||||
*
|
||||
* Then it walks the digested radix tree and marks everything that doesn't have
|
||||
* preserved memory as scratch.
|
||||
*
|
||||
* NOTE: This function allocates memory so it should be called when scratch has
|
||||
* available space.
|
||||
*
|
||||
* NOTE: The pages of the KHO preserved memory radix tree tables are not marked
|
||||
* as preserved in the preserved memory tree. But they are expected to remain
|
||||
* untouched until the tree is fully parsed. So this function also considers
|
||||
* them to be "preserved memory" and marks their blocks as busy.
|
||||
*
|
||||
* NOTE: efi_init()::reserve_regions() removes all regions except
|
||||
* MEMBLOCK_KHO_SCRATCH. This function adds such regions but they are not KHO
|
||||
* scratch memory, so they should not be removed. This function should always be
|
||||
* called after reserve_regions().
|
||||
*/
|
||||
static void __init kho_extend_scratch(void)
|
||||
{
|
||||
const struct kho_radix_walk_cb kho_cb = {
|
||||
.leaf = kho_ext_walk_leaf,
|
||||
.node = kho_ext_walk_node,
|
||||
};
|
||||
const struct kho_radix_walk_cb ext_cb = {
|
||||
.leaf = kho_ext_mark_scratch,
|
||||
};
|
||||
static struct lock_class_key busy_radix_class;
|
||||
struct kho_radix_tree busy_blocks;
|
||||
phys_addr_t prev_end = 0;
|
||||
int err = 0;
|
||||
|
||||
err = kho_radix_init_tree(&busy_blocks, NULL);
|
||||
if (err)
|
||||
goto print;
|
||||
|
||||
/*
|
||||
* The walk of kho_in.radix_tree adds keys to busy_blocks. The walk
|
||||
* takes the kho_in radix tree lock and adding the key takes busy_blocks
|
||||
* lock. Since both are struct kho_radix_tree and share the same lock
|
||||
* class, lockdep gets confused. Set a different class for
|
||||
* busy_blocks.lock to make lockdep happy.
|
||||
*/
|
||||
lockdep_set_class(&busy_blocks.lock, &busy_radix_class);
|
||||
|
||||
/* Walk the KHO radix tree to find busy blocks. */
|
||||
err = kho_radix_walk_tree(&kho_in.radix_tree, &kho_cb, &busy_blocks);
|
||||
if (err)
|
||||
goto out;
|
||||
|
||||
/* Walk the busy blocks and mark everything between keys as scratch. */
|
||||
err = kho_radix_walk_tree(&busy_blocks, &ext_cb, &prev_end);
|
||||
if (err)
|
||||
goto out;
|
||||
|
||||
/* Mark everything from last busy block to end of DRAM. */
|
||||
if (prev_end < memblock_end_of_DRAM())
|
||||
err = memblock_mark_kho_scratch(prev_end, memblock_end_of_DRAM() - prev_end);
|
||||
|
||||
/* fallthrough */
|
||||
out:
|
||||
kho_radix_destroy_tree(&busy_blocks);
|
||||
print:
|
||||
if (err)
|
||||
pr_err("Failed to extend scratch: %pe\n", ERR_PTR(err));
|
||||
}
|
||||
|
||||
/**
|
||||
* kho_add_subtree - record the physical address of a sub blob in KHO root tree.
|
||||
* @name: name of the sub tree.
|
||||
|
|
@ -867,10 +1150,12 @@ int kho_preserve_folio(struct folio *folio)
|
|||
const unsigned long pfn = folio_pfn(folio);
|
||||
const unsigned int order = folio_order(folio);
|
||||
|
||||
if (WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order)))
|
||||
if (IS_ENABLED(CONFIG_KEXEC_HANDOVER_DEBUG) &&
|
||||
WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order)))
|
||||
return -EINVAL;
|
||||
|
||||
return kho_radix_add_page(tree, pfn, order);
|
||||
return kho_radix_add_key(tree, kho_encode_radix_key(PFN_PHYS(pfn),
|
||||
order));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kho_preserve_folio);
|
||||
|
||||
|
|
@ -888,7 +1173,7 @@ void kho_unpreserve_folio(struct folio *folio)
|
|||
const unsigned long pfn = folio_pfn(folio);
|
||||
const unsigned int order = folio_order(folio);
|
||||
|
||||
kho_radix_del_page(tree, pfn, order);
|
||||
kho_radix_del_key(tree, kho_encode_radix_key(PFN_PHYS(pfn), order));
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(kho_unpreserve_folio);
|
||||
|
||||
|
|
@ -917,7 +1202,8 @@ static void __kho_unpreserve(struct kho_radix_tree *tree,
|
|||
while (pfn < end_pfn) {
|
||||
order = __kho_preserve_pages_order(pfn, end_pfn);
|
||||
|
||||
kho_radix_del_page(tree, pfn, order);
|
||||
kho_radix_del_key(tree, kho_encode_radix_key(PFN_PHYS(pfn),
|
||||
order));
|
||||
|
||||
pfn += 1 << order;
|
||||
}
|
||||
|
|
@ -942,7 +1228,8 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages)
|
|||
unsigned long failed_pfn = 0;
|
||||
int err = 0;
|
||||
|
||||
if (WARN_ON(kho_scratch_overlap(start_pfn << PAGE_SHIFT,
|
||||
if (IS_ENABLED(CONFIG_KEXEC_HANDOVER_DEBUG) &&
|
||||
WARN_ON(kho_scratch_overlap(start_pfn << PAGE_SHIFT,
|
||||
nr_pages << PAGE_SHIFT))) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
|
@ -950,7 +1237,8 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages)
|
|||
while (pfn < end_pfn) {
|
||||
unsigned int order = __kho_preserve_pages_order(pfn, end_pfn);
|
||||
|
||||
err = kho_radix_add_page(tree, pfn, order);
|
||||
err = kho_radix_add_key(tree, kho_encode_radix_key(PFN_PHYS(pfn),
|
||||
order));
|
||||
if (err) {
|
||||
failed_pfn = pfn;
|
||||
break;
|
||||
|
|
@ -1326,22 +1614,6 @@ void kho_restore_free(void *mem)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(kho_restore_free);
|
||||
|
||||
struct kho_in {
|
||||
phys_addr_t fdt_phys;
|
||||
phys_addr_t scratch_phys;
|
||||
char previous_release[__NEW_UTS_LEN + 1];
|
||||
u32 kexec_count;
|
||||
struct kho_debugfs dbg;
|
||||
};
|
||||
|
||||
static struct kho_in kho_in = {
|
||||
};
|
||||
|
||||
static const void *kho_get_fdt(void)
|
||||
{
|
||||
return kho_in.fdt_phys ? phys_to_virt(kho_in.fdt_phys) : NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* is_kho_boot - check if current kernel was booted via KHO-enabled
|
||||
* kexec
|
||||
|
|
@ -1409,26 +1681,24 @@ int kho_retrieve_subtree(const char *name, phys_addr_t *phys, size_t *size)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(kho_retrieve_subtree);
|
||||
|
||||
static int __init kho_mem_retrieve(const void *fdt)
|
||||
static void __init kho_mem_retrieve(void)
|
||||
{
|
||||
struct kho_radix_tree tree;
|
||||
const phys_addr_t *mem;
|
||||
int len;
|
||||
const struct kho_radix_walk_cb cb = {
|
||||
.leaf = kho_preserved_memory_reserve,
|
||||
};
|
||||
|
||||
/* Retrieve the KHO radix tree from passed-in FDT. */
|
||||
mem = fdt_getprop(fdt, 0, KHO_FDT_MEMORY_MAP_PROP_NAME, &len);
|
||||
if (kho_radix_walk_tree(&kho_in.radix_tree, &cb, NULL))
|
||||
goto err;
|
||||
|
||||
if (!mem || len != sizeof(*mem)) {
|
||||
pr_err("failed to get preserved KHO memory tree\n");
|
||||
return -ENOENT;
|
||||
}
|
||||
return;
|
||||
|
||||
if (!*mem)
|
||||
return -EINVAL;
|
||||
|
||||
tree.root = phys_to_virt(*mem);
|
||||
mutex_init(&tree.lock);
|
||||
return kho_radix_walk_tree(&tree, kho_preserved_memory_reserve);
|
||||
err:
|
||||
/*
|
||||
* Failed to initialize preserved memory. Clear FDT and radix so KHO
|
||||
* users don't treat it as a KHO boot.
|
||||
*/
|
||||
kho_in.fdt_phys = 0;
|
||||
kho_in.radix_tree.root = NULL;
|
||||
}
|
||||
|
||||
static __init int kho_out_fdt_setup(void)
|
||||
|
|
@ -1554,16 +1824,14 @@ static __init int kho_init(void)
|
|||
if (!kho_enable)
|
||||
return 0;
|
||||
|
||||
tree->root = kzalloc(PAGE_SIZE, GFP_KERNEL);
|
||||
if (!tree->root) {
|
||||
err = -ENOMEM;
|
||||
err = kho_radix_init_tree(tree, NULL);
|
||||
if (err)
|
||||
goto err_free_scratch;
|
||||
}
|
||||
|
||||
kho_out.fdt = kho_alloc_preserve(PAGE_SIZE);
|
||||
if (IS_ERR(kho_out.fdt)) {
|
||||
err = PTR_ERR(kho_out.fdt);
|
||||
goto err_free_kho_radix_tree_root;
|
||||
goto err_free_kho_radix_tree;
|
||||
}
|
||||
|
||||
err = kho_debugfs_init();
|
||||
|
|
@ -1614,9 +1882,8 @@ static __init int kho_init(void)
|
|||
|
||||
err_free_fdt:
|
||||
kho_unpreserve_free(kho_out.fdt);
|
||||
err_free_kho_radix_tree_root:
|
||||
kfree(tree->root);
|
||||
tree->root = NULL;
|
||||
err_free_kho_radix_tree:
|
||||
kho_radix_destroy_tree(tree);
|
||||
err_free_scratch:
|
||||
kho_out.fdt = NULL;
|
||||
for (int i = 0; i < kho_scratch_cnt; i++) {
|
||||
|
|
@ -1630,16 +1897,52 @@ static __init int kho_init(void)
|
|||
}
|
||||
fs_initcall(kho_init);
|
||||
|
||||
void __init kho_memory_init_early(void)
|
||||
{
|
||||
const void *fdt = kho_get_fdt();
|
||||
void *mem_map;
|
||||
|
||||
if (!is_kho_boot())
|
||||
return;
|
||||
|
||||
/*
|
||||
* kho_get_mem_map() should always succeed. If it fails, kho_populate()
|
||||
* catches that and never sets kho_in.scratch_phys, which stops memory
|
||||
* retrieval.
|
||||
*/
|
||||
mem_map = kho_get_mem_map(fdt);
|
||||
if (WARN_ON(!mem_map))
|
||||
goto err;
|
||||
|
||||
/*
|
||||
* kho_scratch_overlap() needs kho_scratch to be initialized. It
|
||||
* is used by free_area_init() on KHO boots, so initialize it
|
||||
* early.
|
||||
*/
|
||||
kho_scratch = phys_to_virt(kho_in.scratch_phys);
|
||||
|
||||
if (kho_radix_init_tree(&kho_in.radix_tree, mem_map))
|
||||
goto err;
|
||||
|
||||
kho_extend_scratch();
|
||||
|
||||
return;
|
||||
|
||||
err:
|
||||
/*
|
||||
* Failed to initialize preserved memory radix tree. Clear FDT
|
||||
* and scratch so KHO users don't treat it as a KHO boot.
|
||||
*/
|
||||
kho_in.fdt_phys = 0;
|
||||
kho_in.scratch_phys = 0;
|
||||
}
|
||||
|
||||
void __init kho_memory_init(void)
|
||||
{
|
||||
if (kho_in.scratch_phys) {
|
||||
kho_scratch = phys_to_virt(kho_in.scratch_phys);
|
||||
|
||||
if (kho_mem_retrieve(kho_get_fdt()))
|
||||
kho_in.fdt_phys = 0;
|
||||
} else {
|
||||
if (kho_in.scratch_phys)
|
||||
kho_mem_retrieve();
|
||||
else
|
||||
kho_reserve_scratch();
|
||||
}
|
||||
}
|
||||
|
||||
void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* kexec_handover_debug.c - kexec handover optional debug functionality
|
||||
* Copyright (C) 2025 Google LLC, Pasha Tatashin <pasha.tatashin@soleen.com>
|
||||
*/
|
||||
|
||||
#define pr_fmt(fmt) "KHO: " fmt
|
||||
|
||||
#include "kexec_handover_internal.h"
|
||||
|
||||
bool kho_scratch_overlap(phys_addr_t phys, size_t size)
|
||||
{
|
||||
phys_addr_t scratch_start, scratch_end;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < kho_scratch_cnt; i++) {
|
||||
scratch_start = kho_scratch[i].addr;
|
||||
scratch_end = kho_scratch[i].addr + kho_scratch[i].size;
|
||||
|
||||
if (phys < scratch_end && (phys + size) > scratch_start)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
|
@ -41,13 +41,4 @@ static inline void kho_debugfs_blob_remove(struct kho_debugfs *dbg,
|
|||
void *blob) { }
|
||||
#endif /* CONFIG_KEXEC_HANDOVER_DEBUGFS */
|
||||
|
||||
#ifdef CONFIG_KEXEC_HANDOVER_DEBUG
|
||||
bool kho_scratch_overlap(phys_addr_t phys, size_t size);
|
||||
#else
|
||||
static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endif /* CONFIG_KEXEC_HANDOVER_DEBUG */
|
||||
|
||||
#endif /* LINUX_KEXEC_HANDOVER_INTERNAL_H */
|
||||
|
|
|
|||
|
|
@ -3079,14 +3079,7 @@ static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact)
|
|||
if (hugetlb_early_cma(h))
|
||||
return hugetlb_cma_alloc_bootmem(h, nid, node_exact);
|
||||
|
||||
if (node_exact)
|
||||
return memblock_alloc_exact_nid_raw(huge_page_size(h),
|
||||
huge_page_size(h), 0,
|
||||
MEMBLOCK_ALLOC_ACCESSIBLE, nid);
|
||||
|
||||
return memblock_alloc_try_nid_raw(huge_page_size(h),
|
||||
huge_page_size(h), 0,
|
||||
MEMBLOCK_ALLOC_ACCESSIBLE, nid);
|
||||
return memblock_alloc_hugetlb(huge_page_size(h), nid, node_exact);
|
||||
}
|
||||
|
||||
void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid)
|
||||
|
|
|
|||
154
mm/memblock.c
154
mm/memblock.c
|
|
@ -19,11 +19,9 @@
|
|||
#include <linux/mutex.h>
|
||||
#include <linux/string_helpers.h>
|
||||
|
||||
#ifdef CONFIG_KEXEC_HANDOVER
|
||||
#include <linux/libfdt.h>
|
||||
#include <linux/kexec_handover.h>
|
||||
#include <linux/kho/abi/memblock.h>
|
||||
#endif /* CONFIG_KEXEC_HANDOVER */
|
||||
|
||||
#include <asm/sections.h>
|
||||
#include <linux/io.h>
|
||||
|
|
@ -1507,6 +1505,32 @@ int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void memblock_prep_allocation(phys_addr_t start, phys_addr_t size,
|
||||
bool kmemleak_trace)
|
||||
{
|
||||
/*
|
||||
* Skip kmemleak for those places like kasan_init() and
|
||||
* early_pgtable_alloc() due to high volume.
|
||||
*/
|
||||
if (kmemleak_trace)
|
||||
/*
|
||||
* Memblock allocated blocks are never reported as
|
||||
* leaks. This is because many of these blocks are
|
||||
* only referred via the physical address which is
|
||||
* not looked up by kmemleak.
|
||||
*/
|
||||
kmemleak_alloc_phys(start, size, 0);
|
||||
|
||||
/*
|
||||
* Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP,
|
||||
* require memory to be accepted before it can be used by the
|
||||
* guest.
|
||||
*
|
||||
* Accept the memory of the allocated buffer.
|
||||
*/
|
||||
accept_memory(start, size);
|
||||
}
|
||||
|
||||
/**
|
||||
* memblock_alloc_range_nid - allocate boot memory block
|
||||
* @size: size of memory block to be allocated in bytes
|
||||
|
|
@ -1581,28 +1605,7 @@ phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
|
|||
return 0;
|
||||
|
||||
done:
|
||||
/*
|
||||
* Skip kmemleak for those places like kasan_init() and
|
||||
* early_pgtable_alloc() due to high volume.
|
||||
*/
|
||||
if (end != MEMBLOCK_ALLOC_NOLEAKTRACE)
|
||||
/*
|
||||
* Memblock allocated blocks are never reported as
|
||||
* leaks. This is because many of these blocks are
|
||||
* only referred via the physical address which is
|
||||
* not looked up by kmemleak.
|
||||
*/
|
||||
kmemleak_alloc_phys(found, size, 0);
|
||||
|
||||
/*
|
||||
* Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP,
|
||||
* require memory to be accepted before it can be used by the
|
||||
* guest.
|
||||
*
|
||||
* Accept the memory of the allocated buffer.
|
||||
*/
|
||||
accept_memory(found, size);
|
||||
|
||||
memblock_prep_allocation(found, size, end != MEMBLOCK_ALLOC_NOLEAKTRACE);
|
||||
return found;
|
||||
}
|
||||
|
||||
|
|
@ -1757,6 +1760,77 @@ void * __init memblock_alloc_try_nid_raw(
|
|||
false);
|
||||
}
|
||||
|
||||
/**
|
||||
* memblock_alloc_hugetlb - allocate boot memory for HugeTLB pages
|
||||
* @size: size of the memory to be allocated in bytes
|
||||
* @nid: nid of the free memory to find, %NUMA_NO_NODE for any node
|
||||
* @exact_nid: only allocate from the specified nid. If %false, the specified
|
||||
* nid is tried first, and then all nodes are tried as fallback.
|
||||
*
|
||||
* HugeTLB pages are always aligned by their size, so the alignment matches
|
||||
* @size. Since the memory is for userspace, mirrored memory is not used. The
|
||||
* memory is not zeroed. Does not panic if request cannot be satisfied.
|
||||
*
|
||||
* Return:
|
||||
* Virtual address of allocated memory block on success, %NULL on failure.
|
||||
*/
|
||||
void * __init memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid)
|
||||
{
|
||||
enum memblock_flags flags = choose_memblock_flags();
|
||||
phys_addr_t addr, start = 0, end = MEMBLOCK_ALLOC_ACCESSIBLE;
|
||||
|
||||
memblock_dbg("%s: %llu bytes, nid=%d, exact_nid=%d %pS\n", __func__,
|
||||
(u64)size, nid, exact_nid, (void *)_RET_IP_);
|
||||
|
||||
/* Don't waste mirrored memory on HugeTLB pages. */
|
||||
flags &= ~MEMBLOCK_MIRROR;
|
||||
retry:
|
||||
/* HugeTLB pages are always aligned by their size. */
|
||||
addr = memblock_find_in_range_node(size, size, start, end, nid, flags);
|
||||
if (addr)
|
||||
goto found;
|
||||
|
||||
/* Try all nodes if allowed. */
|
||||
if (numa_valid_node(nid) && !exact_nid) {
|
||||
nid = NUMA_NO_NODE;
|
||||
/*
|
||||
* If a previous candidate overlapped with KHO scratch, it would
|
||||
* update start or end. Now that the search is opening to all
|
||||
* nodes, reset them.
|
||||
*/
|
||||
start = 0;
|
||||
end = MEMBLOCK_ALLOC_ACCESSIBLE;
|
||||
|
||||
goto retry;
|
||||
}
|
||||
|
||||
/* Found nothing... :-( */
|
||||
return NULL;
|
||||
|
||||
found:
|
||||
/*
|
||||
* HugeTLB pages can be preserved with KHO and no preserved memory can
|
||||
* be in scratch. So retry if found address overlaps with scratch.
|
||||
*
|
||||
* Scratch areas are normally not very large, so this shouldn't take too
|
||||
* many retries.
|
||||
*/
|
||||
if (kho_scratch_overlap(addr, size)) {
|
||||
if (memblock_bottom_up())
|
||||
start = addr + size;
|
||||
else
|
||||
end = addr;
|
||||
|
||||
goto retry;
|
||||
}
|
||||
|
||||
if (__memblock_reserve(addr, size, nid, MEMBLOCK_RSRV_KERN | MEMBLOCK_RSRV_HUGETLB))
|
||||
return NULL;
|
||||
|
||||
memblock_prep_allocation(addr, size, true);
|
||||
return phys_to_virt(addr);
|
||||
}
|
||||
|
||||
/**
|
||||
* memblock_alloc_try_nid - allocate boot memory block
|
||||
* @size: size of memory block to be allocated in bytes
|
||||
|
|
@ -1826,6 +1900,28 @@ phys_addr_t __init_memblock memblock_reserved_size(void)
|
|||
return memblock.reserved.total_size;
|
||||
}
|
||||
|
||||
phys_addr_t __init_memblock memblock_reserved_hugetlb_size(phys_addr_t limit, int nid)
|
||||
{
|
||||
struct memblock_region *r;
|
||||
phys_addr_t total = 0;
|
||||
|
||||
for_each_reserved_mem_region(r) {
|
||||
phys_addr_t size = r->size;
|
||||
|
||||
if (r->base > limit)
|
||||
break;
|
||||
|
||||
if (r->base + r->size > limit)
|
||||
size = limit - r->base;
|
||||
|
||||
if (nid == memblock_get_region_node(r) || !numa_valid_node(nid))
|
||||
if (r->flags & MEMBLOCK_RSRV_HUGETLB)
|
||||
total += size;
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
phys_addr_t __init_memblock memblock_reserved_kern_size(phys_addr_t limit, int nid)
|
||||
{
|
||||
struct memblock_region *r;
|
||||
|
|
@ -2510,16 +2606,6 @@ __init void memblock_clear_kho_scratch_only(void)
|
|||
{
|
||||
kho_scratch_only = false;
|
||||
}
|
||||
|
||||
bool __init_memblock memblock_is_kho_scratch_memory(phys_addr_t addr)
|
||||
{
|
||||
int i = memblock_search(&memblock.memory, addr);
|
||||
|
||||
if (i == -1)
|
||||
return false;
|
||||
|
||||
return memblock_is_kho_scratch(&memblock.memory.regions[i]);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_KEXEC_HANDOVER
|
||||
|
|
|
|||
29
mm/mm_init.c
29
mm/mm_init.c
|
|
@ -684,7 +684,9 @@ static __meminit void pageblock_migratetype_init_range(unsigned long pfn,
|
|||
const unsigned long end = pfn + nr_pages;
|
||||
|
||||
for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) {
|
||||
init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false);
|
||||
enum migratetype mt = kho_scratch_migratetype(pfn, migratetype);
|
||||
|
||||
init_pageblock_migratetype(pfn_to_page(pfn), mt, false);
|
||||
if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION))
|
||||
cond_resched();
|
||||
}
|
||||
|
|
@ -901,8 +903,9 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
|
|||
* over the place during system boot.
|
||||
*/
|
||||
if (pageblock_aligned(pfn)) {
|
||||
init_pageblock_migratetype(page, migratetype,
|
||||
isolate_pageblock);
|
||||
enum migratetype mt = kho_scratch_migratetype(pfn, migratetype);
|
||||
|
||||
init_pageblock_migratetype(page, mt, isolate_pageblock);
|
||||
cond_resched();
|
||||
}
|
||||
pfn++;
|
||||
|
|
@ -912,8 +915,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
|
|||
static void __init memmap_init_zone_range(struct zone *zone,
|
||||
unsigned long start_pfn,
|
||||
unsigned long end_pfn,
|
||||
unsigned long *hole_pfn,
|
||||
enum migratetype mt)
|
||||
unsigned long *hole_pfn)
|
||||
{
|
||||
unsigned long zone_start_pfn = zone->zone_start_pfn;
|
||||
unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
|
||||
|
|
@ -926,7 +928,8 @@ static void __init memmap_init_zone_range(struct zone *zone,
|
|||
return;
|
||||
|
||||
memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
|
||||
zone_end_pfn, MEMINIT_EARLY, NULL, mt, false);
|
||||
zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE,
|
||||
false);
|
||||
|
||||
if (*hole_pfn < start_pfn)
|
||||
init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
|
||||
|
|
@ -942,8 +945,6 @@ static void __init memmap_init(void)
|
|||
|
||||
for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
|
||||
struct pglist_data *node = NODE_DATA(nid);
|
||||
enum migratetype mt =
|
||||
kho_scratch_migratetype(start_pfn, MIGRATE_MOVABLE);
|
||||
|
||||
for (j = 0; j < MAX_NR_ZONES; j++) {
|
||||
struct zone *zone = node->node_zones + j;
|
||||
|
|
@ -952,7 +953,7 @@ static void __init memmap_init(void)
|
|||
continue;
|
||||
|
||||
memmap_init_zone_range(zone, start_pfn, end_pfn,
|
||||
&hole_pfn, mt);
|
||||
&hole_pfn);
|
||||
zone_id = j;
|
||||
}
|
||||
}
|
||||
|
|
@ -1913,7 +1914,7 @@ unsigned long __init node_map_pfn_alignment(void)
|
|||
|
||||
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
|
||||
static void __init deferred_free_pages(unsigned long pfn,
|
||||
unsigned long nr_pages, enum migratetype mt)
|
||||
unsigned long nr_pages)
|
||||
{
|
||||
struct page *page;
|
||||
unsigned long i;
|
||||
|
|
@ -1921,7 +1922,7 @@ static void __init deferred_free_pages(unsigned long pfn,
|
|||
if (!nr_pages)
|
||||
return;
|
||||
|
||||
pageblock_migratetype_init_range(pfn, nr_pages, mt, true);
|
||||
pageblock_migratetype_init_range(pfn, nr_pages, MIGRATE_MOVABLE, true);
|
||||
|
||||
page = pfn_to_page(pfn);
|
||||
|
||||
|
|
@ -1991,8 +1992,6 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
|
|||
for_each_free_mem_range(i, nid, 0, &start, &end, NULL) {
|
||||
unsigned long spfn = PFN_UP(start);
|
||||
unsigned long epfn = PFN_DOWN(end);
|
||||
enum migratetype mt =
|
||||
kho_scratch_migratetype(spfn, MIGRATE_MOVABLE);
|
||||
|
||||
if (spfn >= end_pfn)
|
||||
break;
|
||||
|
|
@ -2005,7 +2004,7 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
|
|||
unsigned long chunk_end = min(mo_pfn, epfn);
|
||||
|
||||
nr_pages += deferred_init_pages(zone, spfn, chunk_end);
|
||||
deferred_free_pages(spfn, chunk_end - spfn, mt);
|
||||
deferred_free_pages(spfn, chunk_end - spfn);
|
||||
|
||||
spfn = chunk_end;
|
||||
|
||||
|
|
@ -2628,6 +2627,8 @@ void __init __weak mem_init(void)
|
|||
|
||||
void __init mm_core_init_early(void)
|
||||
{
|
||||
kho_memory_init_early();
|
||||
|
||||
free_area_init();
|
||||
|
||||
hugetlb_cma_reserve();
|
||||
|
|
|
|||
|
|
@ -46,4 +46,9 @@ static inline bool __is_kernel(unsigned long addr)
|
|||
|
||||
#define __SetPageReserved(p) ((void)(p))
|
||||
|
||||
static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
|||
9
tools/testing/memblock/linux/kexec_handover.h
Normal file
9
tools/testing/memblock/linux/kexec_handover.h
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef LINUX_KEXEC_HANDOVER_H
|
||||
#define LINUX_KEXEC_HANDOVER_H
|
||||
|
||||
/*
|
||||
* Header stub to avoid test build breakage; we don't need to actually implement
|
||||
* any KHO functions as they are not used in the tests.
|
||||
*/
|
||||
#endif /* LINUX_KEXEC_HANDOVER_H */
|
||||
10
tools/testing/memblock/linux/kho/abi/memblock.h
Normal file
10
tools/testing/memblock/linux/kho/abi/memblock.h
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
|
||||
#ifndef _LINUX_KHO_ABI_MEMBLOCK_H
|
||||
#define _LINUX_KHO_ABI_MEMBLOCK_H
|
||||
|
||||
/*
|
||||
* Header stub to avoid test build breakage; we don't need to actually define
|
||||
* any ABI as they are not used in the tests.
|
||||
*/
|
||||
#endif /* _LINUX_KHO_ABI_MEMBLOCK_H */
|
||||
9
tools/testing/memblock/linux/libfdt.h
Normal file
9
tools/testing/memblock/linux/libfdt.h
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef _INCLUDE_LIBFDT_H_
|
||||
#define _INCLUDE_LIBFDT_H_
|
||||
|
||||
/*
|
||||
* Header stub to avoid test build breakage; we don't need to actually implement
|
||||
* any FDT functions as they are not used in the tests.
|
||||
*/
|
||||
#endif /* _INCLUDE_LIBFDT_H_ */
|
||||
Loading…
Reference in New Issue
Block a user