diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 147e19376505..8449fa75a64d 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -84,6 +84,23 @@ static struct kho_out kho_out = { }, }; +struct kho_in { + phys_addr_t fdt_phys; + phys_addr_t scratch_phys; + char previous_release[__NEW_UTS_LEN + 1]; + u32 kexec_count; + struct kho_debugfs dbg; + struct kho_radix_tree radix_tree; +}; + +static struct kho_in kho_in = { +}; + +static const void *kho_get_fdt(void) +{ + return kho_in.fdt_phys ? phys_to_virt(kho_in.fdt_phys) : NULL; +} + /** * kho_encode_radix_key - Encodes a physical address and order into a radix key. * @phys: The physical address of the page. @@ -870,6 +887,140 @@ static void __init kho_reserve_scratch(void) kho_enable = false; } +/* + * Look for free blocks of 1G. This is a heuristic chosen to work efficiently + * with large systems with hundreds of gigabytes of memory. It will work poorly + * on smaller systems. The algorithm itself doesn't depend on the actual value, + * so it can be changed to a different heuristic later if needed. + */ +#define KHO_SCRATCH_EXT_BLKSIZE SZ_1G +#define KHO_SCRATCH_EXT_BLKSHIFT const_ilog2(KHO_SCRATCH_EXT_BLKSIZE) + +/* Called for the KHO preserved memory radix tree. */ +static int __init kho_ext_walk_leaf(unsigned long key, void *data) +{ + struct kho_radix_tree *busy_blocks = data; + phys_addr_t start, end; + unsigned int order; + int err; + + /* + * The key is from the KHO preserved memory radix tree. It is decoded to + * a physical address of a preservation and its order. + */ + start = kho_decode_radix_key(key, &order); + end = start + (1UL << (order + PAGE_SHIFT)); + + while (start < end) { + err = kho_radix_add_key(busy_blocks, start >> KHO_SCRATCH_EXT_BLKSHIFT); + if (err) + return err; + + start += (1UL << KHO_SCRATCH_EXT_BLKSHIFT); + } + + return 0; +} + +/* Called for the KHO preserved memory radix tree. */ +static int __init kho_ext_walk_node(phys_addr_t phys, void *data) +{ + struct kho_radix_tree *busy_blocks = data; + + return kho_radix_add_key(busy_blocks, phys >> KHO_SCRATCH_EXT_BLKSHIFT); +} + +/* Called for the busy block radix tree. */ +static int __init kho_ext_mark_scratch(unsigned long key, void *data) +{ + phys_addr_t *prev_end = data; + phys_addr_t start = key << KHO_SCRATCH_EXT_BLKSHIFT; + int err; + + if (start > *prev_end) { + err = memblock_mark_kho_scratch(*prev_end, start - *prev_end); + if (err) + return err; + } + + *prev_end = start + (1UL << KHO_SCRATCH_EXT_BLKSHIFT); + return 0; +} + +/* + * kho_extend_scratch - Extend the scratch regions + * + * The KHO preserved memory radix tree mixes both physical address and order + * into a single key. This makes it hard to look for free ranges directly. This + * function first walks the radix tree and digests it down into another radix + * tree, whose keys identify blocks of size KHO_SCRATCH_EXT_BLKSIZE which + * contain preserved memory. + * + * Then it walks the digested radix tree and marks everything that doesn't have + * preserved memory as scratch. + * + * NOTE: This function allocates memory so it should be called when scratch has + * available space. + * + * NOTE: The pages of the KHO preserved memory radix tree tables are not marked + * as preserved in the preserved memory tree. But they are expected to remain + * untouched until the tree is fully parsed. So this function also considers + * them to be "preserved memory" and marks their blocks as busy. + * + * NOTE: efi_init()::reserve_regions() removes all regions except + * MEMBLOCK_KHO_SCRATCH. This function adds such regions but they are not KHO + * scratch memory, so they should not be removed. This function should always be + * called after reserve_regions(). + */ +static void __init kho_extend_scratch(void) +{ + const struct kho_radix_walk_cb kho_cb = { + .leaf = kho_ext_walk_leaf, + .node = kho_ext_walk_node, + }; + const struct kho_radix_walk_cb ext_cb = { + .leaf = kho_ext_mark_scratch, + }; + static struct lock_class_key busy_radix_class; + struct kho_radix_tree busy_blocks; + phys_addr_t prev_end = 0; + int err = 0; + + err = kho_radix_init_tree(&busy_blocks, NULL); + if (err) + goto print; + + /* + * The walk of kho_in.radix_tree adds keys to busy_blocks. The walk + * takes the kho_in radix tree lock and adding the key takes busy_blocks + * lock. Since both are struct kho_radix_tree and share the same lock + * class, lockdep gets confused. Set a different class for + * busy_blocks.lock to make lockdep happy. + */ + lockdep_set_class(&busy_blocks.lock, &busy_radix_class); + + /* Walk the KHO radix tree to find busy blocks. */ + err = kho_radix_walk_tree(&kho_in.radix_tree, &kho_cb, &busy_blocks); + if (err) + goto out; + + /* Walk the busy blocks and mark everything between keys as scratch. */ + err = kho_radix_walk_tree(&busy_blocks, &ext_cb, &prev_end); + if (err) + goto out; + + /* Mark everything from last busy block to end of DRAM. */ + if (prev_end < memblock_end_of_DRAM()) + err = memblock_mark_kho_scratch(prev_end, memblock_end_of_DRAM() - prev_end); + + /* fallthrough */ +out: + kho_radix_destroy_tree(&busy_blocks); +print: + if (err) + pr_err("Failed to extend scratch: %pe\n", ERR_PTR(err)); +} + /** * kho_add_subtree - record the physical address of a sub blob in KHO root tree. * @name: name of the sub tree. @@ -1444,23 +1595,6 @@ void kho_restore_free(void *mem) } EXPORT_SYMBOL_GPL(kho_restore_free); -struct kho_in { - phys_addr_t fdt_phys; - phys_addr_t scratch_phys; - char previous_release[__NEW_UTS_LEN + 1]; - u32 kexec_count; - struct kho_debugfs dbg; - struct kho_radix_tree radix_tree; -}; - -static struct kho_in kho_in = { -}; - -static const void *kho_get_fdt(void) -{ - return kho_in.fdt_phys ? phys_to_virt(kho_in.fdt_phys) : NULL; -} - /** * is_kho_boot - check if current kernel was booted via KHO-enabled * kexec @@ -1771,6 +1905,8 @@ void __init kho_memory_init_early(void) if (kho_radix_init_tree(&kho_in.radix_tree, mem_map)) goto err; + kho_extend_scratch(); + return; err: