mirror of
https://github.com/torvalds/linux.git
synced 2026-05-30 18:13:41 +02:00
book3s64/hash: Add kfence functionality
Now that linear map functionality of debug_pagealloc is made generic, enable kfence to use this generic infrastructure. 1. Define kfence related linear map variables. - u8 *linear_map_kf_hash_slots; - unsigned long linear_map_kf_hash_count; - DEFINE_RAW_SPINLOCK(linear_map_kf_hash_lock); 2. The linear map size allocated in RMA region is quite small (KFENCE_POOL_SIZE >> PAGE_SHIFT) which is 512 bytes by default. 3. kfence pool memory is reserved using memblock_phys_alloc() which has can come from anywhere. (default 255 objects => ((1+255) * 2) << PAGE_SHIFT = 32MB) 4. The hash slot information for kfence memory gets added in linear map in hash_linear_map_add_slot() (which also adds for debug_pagealloc). Reported-by: Pavithra Prakash <pavrampu@linux.vnet.ibm.com> Signed-off-by: Ritesh Harjani (IBM) <ritesh.list@gmail.com> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://patch.msgid.link/5c2b61941b344077a2b8654dab46efa0322af3af.1729271995.git.ritesh.list@gmail.com
This commit is contained in:
Ritesh Harjani (IBM)
Michael Ellerman
parent
47dd2e63d4
commit
8fec58f503
|
|
@ -10,7 +10,6 @@
|
|||
|
||||
#include <linux/mm.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/mmu.h>
|
||||
|
||||
#ifdef CONFIG_PPC64_ELF_ABI_V1
|
||||
#define ARCH_FUNC_PREFIX "."
|
||||
|
|
@ -26,10 +25,6 @@ static inline void disable_kfence(void)
|
|||
|
||||
static inline bool arch_kfence_init_pool(void)
|
||||
{
|
||||
#ifdef CONFIG_PPC64
|
||||
if (!radix_enabled())
|
||||
return false;
|
||||
#endif
|
||||
return !kfence_disabled;
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@
|
|||
#include <linux/random.h>
|
||||
#include <linux/elf-randomize.h>
|
||||
#include <linux/of_fdt.h>
|
||||
#include <linux/kfence.h>
|
||||
|
||||
#include <asm/interrupt.h>
|
||||
#include <asm/processor.h>
|
||||
|
|
@ -66,6 +67,7 @@
|
|||
#include <asm/pte-walk.h>
|
||||
#include <asm/asm-prototypes.h>
|
||||
#include <asm/ultravisor.h>
|
||||
#include <asm/kfence.h>
|
||||
|
||||
#include <mm/mmu_decl.h>
|
||||
|
||||
|
|
@ -271,7 +273,7 @@ void hash__tlbiel_all(unsigned int action)
|
|||
WARN(1, "%s called on pre-POWER7 CPU\n", __func__);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_DEBUG_PAGEALLOC
|
||||
#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
|
||||
static void kernel_map_linear_page(unsigned long vaddr, unsigned long idx,
|
||||
u8 *slots, raw_spinlock_t *lock)
|
||||
{
|
||||
|
|
@ -325,11 +327,13 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long idx,
|
|||
mmu_linear_psize,
|
||||
mmu_kernel_ssize, 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_DEBUG_PAGEALLOC
|
||||
static u8 *linear_map_hash_slots;
|
||||
static unsigned long linear_map_hash_count;
|
||||
static DEFINE_RAW_SPINLOCK(linear_map_hash_lock);
|
||||
static inline void hash_debug_pagealloc_alloc_slots(void)
|
||||
static void hash_debug_pagealloc_alloc_slots(void)
|
||||
{
|
||||
unsigned long max_hash_count = ppc64_rma_size / 4;
|
||||
|
||||
|
|
@ -352,7 +356,8 @@ static inline void hash_debug_pagealloc_alloc_slots(void)
|
|||
__func__, linear_map_hash_count, &ppc64_rma_size);
|
||||
}
|
||||
|
||||
static inline void hash_debug_pagealloc_add_slot(phys_addr_t paddr, int slot)
|
||||
static inline void hash_debug_pagealloc_add_slot(phys_addr_t paddr,
|
||||
int slot)
|
||||
{
|
||||
if (!debug_pagealloc_enabled() || !linear_map_hash_count)
|
||||
return;
|
||||
|
|
@ -386,21 +391,149 @@ static int hash_debug_pagealloc_map_pages(struct page *page, int numpages,
|
|||
return 0;
|
||||
}
|
||||
|
||||
int hash__kernel_map_pages(struct page *page, int numpages, int enable)
|
||||
{
|
||||
return hash_debug_pagealloc_map_pages(page, numpages, enable);
|
||||
}
|
||||
|
||||
#else /* CONFIG_DEBUG_PAGEALLOC */
|
||||
int hash__kernel_map_pages(struct page *page, int numpages,
|
||||
int enable)
|
||||
static inline void hash_debug_pagealloc_alloc_slots(void) {}
|
||||
static inline void hash_debug_pagealloc_add_slot(phys_addr_t paddr, int slot) {}
|
||||
static int __maybe_unused
|
||||
hash_debug_pagealloc_map_pages(struct page *page, int numpages, int enable)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
static inline void hash_debug_pagealloc_alloc_slots(void) {}
|
||||
static inline void hash_debug_pagealloc_add_slot(phys_addr_t paddr, int slot) {}
|
||||
#endif /* CONFIG_DEBUG_PAGEALLOC */
|
||||
|
||||
#ifdef CONFIG_KFENCE
|
||||
static u8 *linear_map_kf_hash_slots;
|
||||
static unsigned long linear_map_kf_hash_count;
|
||||
static DEFINE_RAW_SPINLOCK(linear_map_kf_hash_lock);
|
||||
|
||||
static phys_addr_t kfence_pool;
|
||||
|
||||
static inline void hash_kfence_alloc_pool(void)
|
||||
{
|
||||
|
||||
/* allocate linear map for kfence within RMA region */
|
||||
linear_map_kf_hash_count = KFENCE_POOL_SIZE >> PAGE_SHIFT;
|
||||
linear_map_kf_hash_slots = memblock_alloc_try_nid(
|
||||
linear_map_kf_hash_count, 1,
|
||||
MEMBLOCK_LOW_LIMIT, ppc64_rma_size,
|
||||
NUMA_NO_NODE);
|
||||
if (!linear_map_kf_hash_slots) {
|
||||
pr_err("%s: memblock for linear map (%lu) failed\n", __func__,
|
||||
linear_map_kf_hash_count);
|
||||
goto err;
|
||||
}
|
||||
|
||||
/* allocate kfence pool early */
|
||||
kfence_pool = memblock_phys_alloc_range(KFENCE_POOL_SIZE, PAGE_SIZE,
|
||||
MEMBLOCK_LOW_LIMIT, MEMBLOCK_ALLOC_ANYWHERE);
|
||||
if (!kfence_pool) {
|
||||
pr_err("%s: memblock for kfence pool (%lu) failed\n", __func__,
|
||||
KFENCE_POOL_SIZE);
|
||||
memblock_free(linear_map_kf_hash_slots,
|
||||
linear_map_kf_hash_count);
|
||||
linear_map_kf_hash_count = 0;
|
||||
goto err;
|
||||
}
|
||||
memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE);
|
||||
|
||||
return;
|
||||
err:
|
||||
pr_info("Disabling kfence\n");
|
||||
disable_kfence();
|
||||
}
|
||||
|
||||
static inline void hash_kfence_map_pool(void)
|
||||
{
|
||||
unsigned long kfence_pool_start, kfence_pool_end;
|
||||
unsigned long prot = pgprot_val(PAGE_KERNEL);
|
||||
|
||||
if (!kfence_pool)
|
||||
return;
|
||||
|
||||
kfence_pool_start = (unsigned long) __va(kfence_pool);
|
||||
kfence_pool_end = kfence_pool_start + KFENCE_POOL_SIZE;
|
||||
__kfence_pool = (char *) kfence_pool_start;
|
||||
BUG_ON(htab_bolt_mapping(kfence_pool_start, kfence_pool_end,
|
||||
kfence_pool, prot, mmu_linear_psize,
|
||||
mmu_kernel_ssize));
|
||||
memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE);
|
||||
}
|
||||
|
||||
static inline void hash_kfence_add_slot(phys_addr_t paddr, int slot)
|
||||
{
|
||||
unsigned long vaddr = (unsigned long) __va(paddr);
|
||||
unsigned long lmi = (vaddr - (unsigned long)__kfence_pool)
|
||||
>> PAGE_SHIFT;
|
||||
|
||||
if (!kfence_pool)
|
||||
return;
|
||||
BUG_ON(!is_kfence_address((void *)vaddr));
|
||||
BUG_ON(lmi >= linear_map_kf_hash_count);
|
||||
linear_map_kf_hash_slots[lmi] = slot | 0x80;
|
||||
}
|
||||
|
||||
static int hash_kfence_map_pages(struct page *page, int numpages, int enable)
|
||||
{
|
||||
unsigned long flags, vaddr, lmi;
|
||||
int i;
|
||||
|
||||
WARN_ON_ONCE(!linear_map_kf_hash_count);
|
||||
local_irq_save(flags);
|
||||
for (i = 0; i < numpages; i++, page++) {
|
||||
vaddr = (unsigned long)page_address(page);
|
||||
lmi = (vaddr - (unsigned long)__kfence_pool) >> PAGE_SHIFT;
|
||||
|
||||
/* Ideally this should never happen */
|
||||
if (lmi >= linear_map_kf_hash_count) {
|
||||
WARN_ON_ONCE(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (enable)
|
||||
kernel_map_linear_page(vaddr, lmi,
|
||||
linear_map_kf_hash_slots,
|
||||
&linear_map_kf_hash_lock);
|
||||
else
|
||||
kernel_unmap_linear_page(vaddr, lmi,
|
||||
linear_map_kf_hash_slots,
|
||||
&linear_map_kf_hash_lock);
|
||||
}
|
||||
local_irq_restore(flags);
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
static inline void hash_kfence_alloc_pool(void) {}
|
||||
static inline void hash_kfence_map_pool(void) {}
|
||||
static inline void hash_kfence_add_slot(phys_addr_t paddr, int slot) {}
|
||||
static int __maybe_unused
|
||||
hash_kfence_map_pages(struct page *page, int numpages, int enable)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
|
||||
int hash__kernel_map_pages(struct page *page, int numpages, int enable)
|
||||
{
|
||||
void *vaddr = page_address(page);
|
||||
|
||||
if (is_kfence_address(vaddr))
|
||||
return hash_kfence_map_pages(page, numpages, enable);
|
||||
else
|
||||
return hash_debug_pagealloc_map_pages(page, numpages, enable);
|
||||
}
|
||||
|
||||
static void hash_linear_map_add_slot(phys_addr_t paddr, int slot)
|
||||
{
|
||||
if (is_kfence_address(__va(paddr)))
|
||||
hash_kfence_add_slot(paddr, slot);
|
||||
else
|
||||
hash_debug_pagealloc_add_slot(paddr, slot);
|
||||
}
|
||||
#else
|
||||
static void hash_linear_map_add_slot(phys_addr_t paddr, int slot) {}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* 'R' and 'C' update notes:
|
||||
* - Under pHyp or KVM, the updatepp path will not set C, thus it *will*
|
||||
|
|
@ -559,7 +692,8 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
|
|||
break;
|
||||
|
||||
cond_resched();
|
||||
hash_debug_pagealloc_add_slot(paddr, ret);
|
||||
/* add slot info in debug_pagealloc / kfence linear map */
|
||||
hash_linear_map_add_slot(paddr, ret);
|
||||
}
|
||||
return ret < 0 ? ret : 0;
|
||||
}
|
||||
|
|
@ -940,7 +1074,7 @@ static void __init htab_init_page_sizes(void)
|
|||
bool aligned = true;
|
||||
init_hpte_page_sizes();
|
||||
|
||||
if (!debug_pagealloc_enabled()) {
|
||||
if (!debug_pagealloc_enabled_or_kfence()) {
|
||||
/*
|
||||
* Pick a size for the linear mapping. Currently, we only
|
||||
* support 16M, 1M and 4K which is the default
|
||||
|
|
@ -1261,6 +1395,7 @@ static void __init htab_initialize(void)
|
|||
prot = pgprot_val(PAGE_KERNEL);
|
||||
|
||||
hash_debug_pagealloc_alloc_slots();
|
||||
hash_kfence_alloc_pool();
|
||||
/* create bolted the linear mapping in the hash table */
|
||||
for_each_mem_range(i, &base, &end) {
|
||||
size = end - base;
|
||||
|
|
@ -1277,6 +1412,7 @@ static void __init htab_initialize(void)
|
|||
BUG_ON(htab_bolt_mapping(base, base + size, __pa(base),
|
||||
prot, mmu_linear_psize, mmu_kernel_ssize));
|
||||
}
|
||||
hash_kfence_map_pool();
|
||||
memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
|
||||
|
||||
/*
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user