diff --git a/include/linux/kfence.h b/include/linux/kfence.h index e1f984697d06..ed4affc4f6ac 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -4,7 +4,6 @@ #define _LINUX_KFENCE_H #include -#include #include #ifdef CONFIG_KFENCE @@ -17,7 +16,13 @@ #define KFENCE_POOL_SIZE ((CONFIG_KFENCE_NUM_OBJECTS + 1) * 2 * PAGE_SIZE) extern char *__kfence_pool; +#ifdef CONFIG_KFENCE_STATIC_KEYS +#include DECLARE_STATIC_KEY_FALSE(kfence_allocation_key); +#else +#include +extern atomic_t kfence_allocation_gate; +#endif /** * is_kfence_address() - check if an address belongs to KFENCE pool @@ -104,7 +109,11 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags); */ static __always_inline void *kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags) { +#ifdef CONFIG_KFENCE_STATIC_KEYS if (static_branch_unlikely(&kfence_allocation_key)) +#else + if (unlikely(!atomic_read(&kfence_allocation_gate))) +#endif return __kfence_alloc(s, size, flags); return NULL; } diff --git a/lib/Kconfig.kfence b/lib/Kconfig.kfence index b209cd02042b..b88ac9d6b2e6 100644 --- a/lib/Kconfig.kfence +++ b/lib/Kconfig.kfence @@ -6,7 +6,6 @@ config HAVE_ARCH_KFENCE menuconfig KFENCE bool "KFENCE: low-overhead sampling-based memory safety error detector" depends on HAVE_ARCH_KFENCE && !KASAN && (SLAB || SLUB) - depends on JUMP_LABEL # To ensure performance, require jump labels select STACKTRACE help KFENCE is a low-overhead sampling-based detector of heap out-of-bounds @@ -23,6 +22,17 @@ menuconfig KFENCE if KFENCE +config KFENCE_STATIC_KEYS + bool "Use static keys to set up allocations" + default y + depends on JUMP_LABEL # To ensure performance, require jump labels + help + Use static keys (static branches) to set up KFENCE allocations. Using + static keys is normally recommended, because it avoids a dynamic + branch in the allocator's fast path. However, with very low sample + intervals, or on systems that do not support jump labels, a dynamic + branch may still be an acceptable performance trade-off. + config KFENCE_SAMPLE_INTERVAL int "Default sample interval in milliseconds" default 100 diff --git a/mm/kfence/core.c b/mm/kfence/core.c index ae49868dae0a..d011cd9f8634 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -88,11 +88,13 @@ struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS]; static struct list_head kfence_freelist = LIST_HEAD_INIT(kfence_freelist); static DEFINE_RAW_SPINLOCK(kfence_freelist_lock); /* Lock protecting freelist. */ +#ifdef CONFIG_KFENCE_STATIC_KEYS /* The static key to set up a KFENCE allocation. */ DEFINE_STATIC_KEY_FALSE(kfence_allocation_key); +#endif /* Gates the allocation, ensuring only one succeeds in a given period. */ -static atomic_t allocation_gate = ATOMIC_INIT(1); +atomic_t kfence_allocation_gate = ATOMIC_INIT(1); /* Statistics counters for debugfs. */ enum kfence_counter_id { @@ -579,29 +581,31 @@ late_initcall(kfence_debugfs_init); static struct delayed_work kfence_timer; static void toggle_allocation_gate(struct work_struct *work) { - unsigned long end_wait; - if (!READ_ONCE(kfence_enabled)) return; /* Enable static key, and await allocation to happen. */ - atomic_set(&allocation_gate, 0); + atomic_set(&kfence_allocation_gate, 0); +#ifdef CONFIG_KFENCE_STATIC_KEYS static_branch_enable(&kfence_allocation_key); /* * Await an allocation. Timeout after 1 second, in case the kernel stops * doing allocations, to avoid stalling this worker task for too long. */ - end_wait = jiffies + HZ; - do { - set_current_state(TASK_UNINTERRUPTIBLE); - if (atomic_read(&allocation_gate) != 0) - break; - schedule_timeout(1); - } while (time_before(jiffies, end_wait)); - __set_current_state(TASK_RUNNING); + { + unsigned long end_wait = jiffies + HZ; + do { + set_current_state(TASK_UNINTERRUPTIBLE); + if (atomic_read(&kfence_allocation_gate) != 0) + break; + schedule_timeout(1); + } while (time_before(jiffies, end_wait)); + __set_current_state(TASK_RUNNING); + } /* Disable static key and reset timer. */ static_branch_disable(&kfence_allocation_key); +#endif schedule_delayed_work(&kfence_timer, msecs_to_jiffies(kfence_sample_interval)); } static DECLARE_DELAYED_WORK(kfence_timer, toggle_allocation_gate); @@ -707,7 +711,7 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags) * sense to continue writing to it and pay the associated contention * cost, in case we have a large number of concurrent allocations. */ - if (atomic_read(&allocation_gate) || atomic_inc_return(&allocation_gate) > 1) + if (atomic_read(&kfence_allocation_gate) || atomic_inc_return(&kfence_allocation_gate) > 1) return NULL; if (!READ_ONCE(kfence_enabled))