github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-29 12:24:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

kprobes: Use dedicated kthread for kprobe optimizer

Browse Source 
Instead of using generic workqueue, use a dedicated kthread for optimizing
kprobes, because it can wait (sleep) for a long time inside the process
by synchronize_rcu_task(). This means other works can be stopped until it
finishes.

Link: https://lore.kernel.org/all/176970170302.114949.5175231591310436910.stgit@devnote2/

Suggested-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
This commit is contained in:
Masami Hiramatsu (Google) 2026-01-30 00:48:23 +09:00
parent b8121b9cdc
commit 73c12f2094
1 changed files with 86 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
kernel/kprobes.c
View File

@ -32,6 +32,7 @@
#include <linux/debugfs.h>
#include <linux/sysctl.h>
#include <linux/kdebug.h>
#include <linux/kthread.h>
#include <linux/memory.h>
#include <linux/ftrace.h>
#include <linux/cpu.h>
@ -40,6 +41,7 @@
#include <linux/perf_event.h>
#include <linux/execmem.h>
#include <linux/cleanup.h>
#include <linux/wait.h>
#include <asm/sections.h>
#include <asm/cacheflush.h>
@ -514,9 +516,18 @@ static LIST_HEAD(optimizing_list);
static LIST_HEAD(unoptimizing_list);
static LIST_HEAD(freeing_list);
static void kprobe_optimizer(struct work_struct *work);
static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
static void optimize_kprobe(struct kprobe *p);
static struct task_struct *kprobe_optimizer_task;
static wait_queue_head_t kprobe_optimizer_wait;
static atomic_t optimizer_state;
enum {
OPTIMIZER_ST_IDLE = 0,
OPTIMIZER_ST_KICKED = 1,
OPTIMIZER_ST_FLUSHING = 2,
};
static DECLARE_COMPLETION(optimizer_completion);
#define OPTIMIZE_DELAY 5
/*
@ -609,14 +620,10 @@ static void do_free_cleaned_kprobes(void)
}
}
/* Start optimizer after OPTIMIZE_DELAY passed */
static void kick_kprobe_optimizer(void)
{
schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
}
static void kick_kprobe_optimizer(void);
/* Kprobe jump optimizer */
static void kprobe_optimizer(struct work_struct *work)
static void kprobe_optimizer(void)
{
guard(mutex)(&kprobe_mutex);
@ -647,9 +654,53 @@ static void kprobe_optimizer(struct work_struct *work)
do_free_cleaned_kprobes();
}
/* Step 5: Kick optimizer again if needed */
/* Step 5: Kick optimizer again if needed. But if there is a flush requested, */
if (completion_done(&optimizer_completion))
complete(&optimizer_completion);
if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
kick_kprobe_optimizer();
kick_kprobe_optimizer(); /*normal kick*/
}
static int kprobe_optimizer_thread(void *data)
{
while (!kthread_should_stop()) {
/* To avoid hung_task, wait in interruptible state. */
wait_event_interruptible(kprobe_optimizer_wait,
atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE ||
kthread_should_stop());
if (kthread_should_stop())
break;
/*
* If it was a normal kick, wait for OPTIMIZE_DELAY.
* This wait can be interrupted by a flush request.
*/
if (atomic_read(&optimizer_state) == 1)
wait_event_interruptible_timeout(
kprobe_optimizer_wait,
atomic_read(&optimizer_state) == OPTIMIZER_ST_FLUSHING ||
kthread_should_stop(),
OPTIMIZE_DELAY);
if (kthread_should_stop())
break;
atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);
kprobe_optimizer();
}
return 0;
}
/* Start optimizer after OPTIMIZE_DELAY passed */
static void kick_kprobe_optimizer(void)
{
lockdep_assert_held(&kprobe_mutex);
if (atomic_cmpxchg(&optimizer_state,
OPTIMIZER_ST_IDLE, OPTIMIZER_ST_KICKED) == OPTIMIZER_ST_IDLE)
wake_up(&kprobe_optimizer_wait);
}
static void wait_for_kprobe_optimizer_locked(void)
@ -657,13 +708,17 @@ static void wait_for_kprobe_optimizer_locked(void)
lockdep_assert_held(&kprobe_mutex);
while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
init_completion(&optimizer_completion);
/*
* Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's
* idle. If it's already kicked, it will see the state change.
*/
if (atomic_xchg_acquire(&optimizer_state,
OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING)
wake_up(&kprobe_optimizer_wait);
mutex_unlock(&kprobe_mutex);
/* This will also make 'optimizing_work' execute immmediately */
flush_delayed_work(&optimizing_work);
/* 'optimizing_work' might not have been queued yet, relax */
cpu_relax();
wait_for_completion(&optimizer_completion);
mutex_lock(&kprobe_mutex);
}
}
@ -1016,8 +1071,21 @@ static void __disarm_kprobe(struct kprobe *p, bool reopt)
}
}
static void __init init_optprobe(void)
{
#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
/* Init 'kprobe_optinsn_slots' for allocation */
kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
#endif
init_waitqueue_head(&kprobe_optimizer_wait);
atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);
kprobe_optimizer_task = kthread_run(kprobe_optimizer_thread, NULL,
"kprobe-optimizer");
}
#else /* !CONFIG_OPTPROBES */
#define init_optprobe() do {} while (0)
#define optimize_kprobe(p) do {} while (0)
#define unoptimize_kprobe(p, f) do {} while (0)
#define kill_optimized_kprobe(p) do {} while (0)
@ -2700,10 +2768,8 @@ static int __init init_kprobes(void)
/* By default, kprobes are armed */
kprobes_all_disarmed = false;
#if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)
/* Init 'kprobe_optinsn_slots' for allocation */
kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
#endif
/* Initialize the optimization infrastructure */
init_optprobe();
err = arch_init_kprobes();
if (!err)