github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-07 05:55:44 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

FROMGIT: kfence: add option to use KFENCE without static keys

Browse Source 
For certain usecases, specifically where the sample interval is always
set to a very low value such as 1ms, it can make sense to use a dynamic
branch instead of static branches due to the overhead of toggling a
static branch.

Therefore, add a new Kconfig option to remove the static branches and
instead check kfence_allocation_gate if a KFENCE allocation should be
set up.

Link: https://lkml.kernel.org/r/20210111091544.3287013-1-elver@google.com
Signed-off-by: Marco Elver <elver@google.com>
Suggested-by: Jörn Engel <joern@purestorage.com>
Reviewed-by: Jörn Engel <joern@purestorage.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Jann Horn <jannh@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

Bug: 177201466
(cherry picked from commit c01761611b325c1e4ec7d3e236cc9db003cb82fd
    https://github.com/hnaz/linux-mm v5.11-rc4-mmots-2021-01-21-20-10)
Test: CONFIG_KFENCE_KUNIT_TEST=y passes on Cuttlefish
Signed-off-by: Alexander Potapenko <glider@google.com>
Change-Id: I68a112a8ff68fa24742b198e036f130a9757c27f
This commit is contained in:
Marco Elver 2021-01-22 09:18:33 +00:00 committed by Alistair Delva
parent 97d9142889
commit 33ad66179a
3 changed files with 38 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
include/linux/kfence.h
View File

@ -4,7 +4,6 @@
#define _LINUX_KFENCE_H
#include <linux/mm.h>
#include <linux/static_key.h>
#include <linux/types.h>
#ifdef CONFIG_KFENCE
@ -17,7 +16,13 @@
#define KFENCE_POOL_SIZE ((CONFIG_KFENCE_NUM_OBJECTS + 1) * 2 * PAGE_SIZE)
extern char *__kfence_pool;
#ifdef CONFIG_KFENCE_STATIC_KEYS
#include <linux/static_key.h>
DECLARE_STATIC_KEY_FALSE(kfence_allocation_key);
#else
#include <linux/atomic.h>
extern atomic_t kfence_allocation_gate;
#endif
/**
* is_kfence_address() - check if an address belongs to KFENCE pool
@ -104,7 +109,11 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags);
*/
static __always_inline void *kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags)
{
#ifdef CONFIG_KFENCE_STATIC_KEYS
if (static_branch_unlikely(&kfence_allocation_key))
#else
if (unlikely(!atomic_read(&kfence_allocation_gate)))
#endif
return __kfence_alloc(s, size, flags);
return NULL;
}

12
lib/Kconfig.kfence
View File

@ -6,7 +6,6 @@ config HAVE_ARCH_KFENCE
menuconfig KFENCE
bool "KFENCE: low-overhead sampling-based memory safety error detector"
depends on HAVE_ARCH_KFENCE && !KASAN && (SLAB || SLUB)
depends on JUMP_LABEL # To ensure performance, require jump labels
select STACKTRACE
help
KFENCE is a low-overhead sampling-based detector of heap out-of-bounds
@ -23,6 +22,17 @@ menuconfig KFENCE
if KFENCE
config KFENCE_STATIC_KEYS
bool "Use static keys to set up allocations"
default y
depends on JUMP_LABEL # To ensure performance, require jump labels
help
Use static keys (static branches) to set up KFENCE allocations. Using
static keys is normally recommended, because it avoids a dynamic
branch in the allocator's fast path. However, with very low sample
intervals, or on systems that do not support jump labels, a dynamic
branch may still be an acceptable performance trade-off.
config KFENCE_SAMPLE_INTERVAL
int "Default sample interval in milliseconds"
default 100

30
mm/kfence/core.c
View File

@ -88,11 +88,13 @@ struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS];
static struct list_head kfence_freelist = LIST_HEAD_INIT(kfence_freelist);
static DEFINE_RAW_SPINLOCK(kfence_freelist_lock); /* Lock protecting freelist. */
#ifdef CONFIG_KFENCE_STATIC_KEYS
/* The static key to set up a KFENCE allocation. */
DEFINE_STATIC_KEY_FALSE(kfence_allocation_key);
#endif
/* Gates the allocation, ensuring only one succeeds in a given period. */
static atomic_t allocation_gate = ATOMIC_INIT(1);
atomic_t kfence_allocation_gate = ATOMIC_INIT(1);
/* Statistics counters for debugfs. */
enum kfence_counter_id {
@ -579,29 +581,31 @@ late_initcall(kfence_debugfs_init);
static struct delayed_work kfence_timer;
static void toggle_allocation_gate(struct work_struct *work)
{
unsigned long end_wait;
if (!READ_ONCE(kfence_enabled))
return;
/* Enable static key, and await allocation to happen. */
atomic_set(&allocation_gate, 0);
atomic_set(&kfence_allocation_gate, 0);
#ifdef CONFIG_KFENCE_STATIC_KEYS
static_branch_enable(&kfence_allocation_key);
/*
* Await an allocation. Timeout after 1 second, in case the kernel stops
* doing allocations, to avoid stalling this worker task for too long.
*/
end_wait = jiffies + HZ;
do {
set_current_state(TASK_UNINTERRUPTIBLE);
if (atomic_read(&allocation_gate) != 0)
break;
schedule_timeout(1);
} while (time_before(jiffies, end_wait));
__set_current_state(TASK_RUNNING);
{
unsigned long end_wait = jiffies + HZ;
do {
set_current_state(TASK_UNINTERRUPTIBLE);
if (atomic_read(&kfence_allocation_gate) != 0)
break;
schedule_timeout(1);
} while (time_before(jiffies, end_wait));
__set_current_state(TASK_RUNNING);
}
/* Disable static key and reset timer. */
static_branch_disable(&kfence_allocation_key);
#endif
schedule_delayed_work(&kfence_timer, msecs_to_jiffies(kfence_sample_interval));
}
static DECLARE_DELAYED_WORK(kfence_timer, toggle_allocation_gate);
@ -707,7 +711,7 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags)
* sense to continue writing to it and pay the associated contention
* cost, in case we have a large number of concurrent allocations.
*/
if (atomic_read(&allocation_gate) || atomic_inc_return(&allocation_gate) > 1)
if (atomic_read(&kfence_allocation_gate) || atomic_inc_return(&kfence_allocation_gate) > 1)
return NULL;
if (!READ_ONCE(kfence_enabled))