From b00192d78bb4f8b85d3545c7f86722610f088554 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 30 Apr 2026 16:39:27 -0700 Subject: [PATCH 01/36] locking/barrier: Use correct parameter names Use the correct parameter names in kernel-doc comments to avoid warnings: Warning: include/asm-generic/barrier.h:245 function parameter 'cond_expr' not described in 'smp_cond_load_relaxed' Warning: include/asm-generic/barrier.h:267 function parameter 'cond_expr' not described in 'smp_cond_load_acquire' Signed-off-by: Randy Dunlap Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260430233927.2620050-1-rdunlap@infradead.org --- include/asm-generic/barrier.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h index d4f581c1e21d..b99cb57dfccc 100644 --- a/include/asm-generic/barrier.h +++ b/include/asm-generic/barrier.h @@ -235,7 +235,7 @@ do { \ /** * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees * @ptr: pointer to the variable to wait on - * @cond: boolean expression to wait for + * @cond_expr: boolean expression to wait for * * Equivalent to using READ_ONCE() on the condition variable. * @@ -259,7 +259,7 @@ do { \ /** * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering * @ptr: pointer to the variable to wait on - * @cond: boolean expression to wait for + * @cond_expr: boolean expression to wait for * * Equivalent to using smp_load_acquire() on the condition variable but employs * the control dependency of the wait to reduce the barrier on many platforms. From 89976cd73739dcb73745705a63ccc67a8be26cdf Mon Sep 17 00:00:00 2001 From: Aniket Gattani Date: Sun, 3 May 2026 21:22:03 +0000 Subject: [PATCH 02/36] sched/membarrier: Use per-CPU mutexes for targeted commands Currently, the membarrier system call uses a single global mutex (`membarrier_ipi_mutex`) to serialize expedited commands. This causes significant contention on large systems when multiple threads invoke membarrier concurrently, even if they target different CPUs. This contention becomes critical when combined with CFS bandwidth throttling/unthrottling, during which interrupts can be disabled for relatively long periods on target CPUs. If membarrier is waiting for a response from such a CPU, it holds the global mutex, blocking all other membarrier calls on the system. This cascade effect can lead to hard lockups when thousands of threads stall waiting for the mutex. Optimize `MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ` when a specific CPU is targeted by introducing per-CPU mutexes. Broadcast commands and commands without a specific CPU target continue to use the global mutex. This prevents the cascade lockup scenario. As measured by the stress test introduced in the subsequent patch, on an AMD Turin machine with 384 CPUs (2 NUMA nodes with SMT=2), this optimization yields 200x more throughput. Signed-off-by: Aniket Gattani Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260503212205.3714217-2-aniketgattani@google.com --- kernel/sched/membarrier.c | 77 ++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c index 623445603725..3d88e900a17f 100644 --- a/kernel/sched/membarrier.c +++ b/kernel/sched/membarrier.c @@ -164,8 +164,26 @@ | MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK \ | MEMBARRIER_CMD_GET_REGISTRATIONS) +/* + * Scoped guard for memory barriers on entry and exit. + * Matches memory barriers before & after rq->curr modification in scheduler. + */ +DEFINE_LOCK_GUARD_0(mb, smp_mb(), smp_mb()) static DEFINE_MUTEX(membarrier_ipi_mutex); +static DEFINE_PER_CPU(struct mutex, membarrier_cpu_mutexes); + #define SERIALIZE_IPI() guard(mutex)(&membarrier_ipi_mutex) +#define SERIALIZE_IPI_CPU(cpu_id) guard(mutex)(&per_cpu(membarrier_cpu_mutexes, cpu_id)) + +static int __init membarrier_init(void) +{ + int i; + + for_each_possible_cpu(i) + mutex_init(&per_cpu(membarrier_cpu_mutexes, i)); + return 0; +} +core_initcall(membarrier_init); static void ipi_mb(void *info) { @@ -315,7 +333,6 @@ static int membarrier_global_expedited(void) static int membarrier_private_expedited(int flags, int cpu_id) { - cpumask_var_t tmpmask; struct mm_struct *mm = current->mm; smp_call_func_t ipi_func = ipi_mb; @@ -352,30 +369,45 @@ static int membarrier_private_expedited(int flags, int cpu_id) * On RISC-V, this barrier pairing is also needed for the * SYNC_CORE command when switching between processes, cf. * the inline comments in membarrier_arch_switch_mm(). + * + * Memory barrier on the caller thread _after_ we finished + * waiting for the last IPI. Matches memory barriers before + * rq->curr modification in scheduler. */ - smp_mb(); /* system call entry is not a mb. */ - - if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) - return -ENOMEM; - - SERIALIZE_IPI(); - cpus_read_lock(); - + guard(mb)(); if (cpu_id >= 0) { + if (cpu_id >= nr_cpu_ids || !cpu_possible(cpu_id)) + return 0; + + SERIALIZE_IPI_CPU(cpu_id); + guard(cpus_read_lock)(); struct task_struct *p; - if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id)) - goto out; + if (!cpu_online(cpu_id)) + return 0; + rcu_read_lock(); p = rcu_dereference(cpu_rq(cpu_id)->curr); if (!p || p->mm != mm) { rcu_read_unlock(); - goto out; + return 0; } rcu_read_unlock(); + /* + * smp_call_function_single() will call ipi_func() if cpu_id + * is the calling CPU. + */ + smp_call_function_single(cpu_id, ipi_func, NULL, 1); } else { + cpumask_var_t __free(free_cpumask_var) tmpmask = CPUMASK_VAR_NULL; int cpu; + if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) + return -ENOMEM; + + SERIALIZE_IPI(); + guard(cpus_read_lock)(); + rcu_read_lock(); for_each_online_cpu(cpu) { struct task_struct *p; @@ -385,15 +417,6 @@ static int membarrier_private_expedited(int flags, int cpu_id) __cpumask_set_cpu(cpu, tmpmask); } rcu_read_unlock(); - } - - if (cpu_id >= 0) { - /* - * smp_call_function_single() will call ipi_func() if cpu_id - * is the calling CPU. - */ - smp_call_function_single(cpu_id, ipi_func, NULL, 1); - } else { /* * For regular membarrier, we can save a few cycles by * skipping the current cpu -- we're about to do smp_mb() @@ -420,18 +443,6 @@ static int membarrier_private_expedited(int flags, int cpu_id) } } -out: - if (cpu_id < 0) - free_cpumask_var(tmpmask); - cpus_read_unlock(); - - /* - * Memory barrier on the caller thread _after_ we finished - * waiting for the last IPI. Matches memory barriers before - * rq->curr modification in scheduler. - */ - smp_mb(); /* exit from system call is not a mb */ - return 0; } From a5959728548caf0aa17879342ee6571a29d54574 Mon Sep 17 00:00:00 2001 From: Aniket Gattani Date: Sun, 3 May 2026 21:22:04 +0000 Subject: [PATCH 03/36] sched/membarrier: Modernize membarrier_global_expedited with cleanup guards Replace explicit lock/unlock and free calls with scoped guards and automatic cleanup constructs. Signed-off-by: Aniket Gattani Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260503212205.3714217-3-aniketgattani@google.com --- kernel/sched/membarrier.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c index 3d88e900a17f..12b68a77630a 100644 --- a/kernel/sched/membarrier.c +++ b/kernel/sched/membarrier.c @@ -267,23 +267,19 @@ void membarrier_update_current_mm(struct mm_struct *next_mm) static int membarrier_global_expedited(void) { + cpumask_var_t __free(free_cpumask_var) tmpmask = CPUMASK_VAR_NULL; int cpu; - cpumask_var_t tmpmask; if (num_online_cpus() == 1) return 0; - /* - * Matches memory barriers after rq->curr modification in - * scheduler. - */ - smp_mb(); /* system call entry is not a mb. */ - if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) return -ENOMEM; + guard(mb)(); SERIALIZE_IPI(); - cpus_read_lock(); + guard(cpus_read_lock)(); + rcu_read_lock(); for_each_online_cpu(cpu) { struct task_struct *p; @@ -319,15 +315,6 @@ static int membarrier_global_expedited(void) smp_call_function_many(tmpmask, ipi_mb, NULL, 1); preempt_enable(); - free_cpumask_var(tmpmask); - cpus_read_unlock(); - - /* - * Memory barrier on the caller thread _after_ we finished - * waiting for the last IPI. Matches memory barriers before - * rq->curr modification in scheduler. - */ - smp_mb(); /* exit from system call is not a mb */ return 0; } From 03240f5de2dd312f388f3e493f194d54d43e2924 Mon Sep 17 00:00:00 2001 From: Aniket Gattani Date: Sun, 3 May 2026 21:22:05 +0000 Subject: [PATCH 04/36] selftests/membarrier: Add rseq stress test for CFS throttle interactions Add a new stress test to exercise the interaction between targeted expedited membarrier commands and CFS bandwidth throttling. The test creates a deep cgroup hierarchy and aggressively hammers the membarrier syscall to expose lock contention and latency issues. This serves as a reliable reproducer for the `membarrier_ipi_mutex` cascade lockup, ensuring future changes to membarrier locking do not regress targeted command latency. Closes: https://lore.kernel.org/r/202604151516.Vc7Ro4LP-lkp@intel.com/ Reported-by: kernel test robot Signed-off-by: Aniket Gattani Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260503212205.3714217-4-aniketgattani@google.com --- tools/testing/selftests/membarrier/Makefile | 5 +- .../membarrier/membarrier_rseq_stress.c | 951 ++++++++++++++++++ 2 files changed, 954 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/membarrier/membarrier_rseq_stress.c diff --git a/tools/testing/selftests/membarrier/Makefile b/tools/testing/selftests/membarrier/Makefile index fc840e06ff56..829f95c83515 100644 --- a/tools/testing/selftests/membarrier/Makefile +++ b/tools/testing/selftests/membarrier/Makefile @@ -1,8 +1,9 @@ # SPDX-License-Identifier: GPL-2.0-only -CFLAGS += -g $(KHDR_INCLUDES) +CFLAGS += -g $(KHDR_INCLUDES) -pthread -I../../../../tools/include LDLIBS += -lpthread TEST_GEN_PROGS := membarrier_test_single_thread \ - membarrier_test_multi_thread + membarrier_test_multi_thread \ + membarrier_rseq_stress include ../lib.mk diff --git a/tools/testing/selftests/membarrier/membarrier_rseq_stress.c b/tools/testing/selftests/membarrier/membarrier_rseq_stress.c new file mode 100644 index 000000000000..c188d7498610 --- /dev/null +++ b/tools/testing/selftests/membarrier/membarrier_rseq_stress.c @@ -0,0 +1,951 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Membarrier stress test for CFS throttle interactions. + * + * Reproducer for the interaction between CFS throttle and expedited membarrier. + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../kselftest.h" + +/* -- Architecture-specific rseq signature -- */ +#if defined(__x86_64__) || defined(__i386__) +# define RSEQ_SIG 0x53053053U +#elif defined(__aarch64__) +# define RSEQ_SIG 0xd428bc00U +#elif defined(__powerpc__) || defined(__powerpc64__) +# define RSEQ_SIG 0x0f000000U +#elif defined(__s390__) || defined(__s390x__) +# define RSEQ_SIG 0x0c000000U +#else +# define RSEQ_SIG 0 +# define UNSUPPORTED_ARCH 1 +#endif + +/* -- rseq ABI (kernel uapi; define locally for portability) -- */ +#define RSEQ_CPU_ID_UNINITIALIZED ((__u32)-1) + +#include + +struct rseq_abi { + __u32 cpu_id_start; + __u32 cpu_id; + __u64 rseq_cs; + __u32 flags; + __u32 node_id; + __u32 mm_cid; + char end[0]; +} __aligned(32); + +/* -- membarrier constants (not in all distro headers) -- */ +#ifndef MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ +# define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ (1 << 7) +#endif +#ifndef MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ +# define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ (1 << 8) +#endif +#ifndef MEMBARRIER_CMD_FLAG_CPU +# define MEMBARRIER_CMD_FLAG_CPU (1 << 0) +#endif + +/* -- Test parameters -- */ +#define N_SIBLINGS 2000 +#define NEST_DEPTH 5 +static char g_cgroup_path[4096]; +static int use_cgroup_v2; + +#define CFS_QUOTA_US 1000 +#define CFS_PERIOD_US 5000 +#define N_HAMMER_PER_CPU 25 +#define N_BURNER_PER_CPU 50 +#define MAX_STRESS_CPUS 1024 +#define TEST_DURATION_SEC 20 + +/* Latency thresholds for the sentinel */ +#define LATENCY_WARN_MS 50 +#define LATENCY_CRITICAL_MS 200 + +/* Sentinel sampling interval */ +#define SENTINEL_INTERVAL_US 500 + +/* -- Shared globals -- */ +static atomic_int g_stop; +static atomic_int g_stop_sentinel; +static atomic_long g_max_latency_us; +static atomic_long g_interval_max_latency_us; +static atomic_long g_mb_ok; +static atomic_long g_mb_err; +static int g_ncpus_stress; +static int *g_stress_cpus; + +static atomic_int g_test_ready; + +/* Per-thread rseq ABI block registered with the kernel */ +static __thread struct rseq_abi tls_rseq + __attribute__((tls_model("initial-exec"))) __aligned(32) = { + .cpu_id = RSEQ_CPU_ID_UNINITIALIZED, +}; + +/* -- Utility -- */ +static int write_file(const char *path, const char *val) +{ + int fd = open(path, O_WRONLY | O_CLOEXEC); + + if (fd < 0) + return -errno; + + size_t len = strlen(val); + ssize_t r = write(fd, val, len); + + close(fd); + if (r < 0) + return -errno; + if ((size_t)r != len) + return -EIO; + return 0; +} + +static uint64_t monotonic_us(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000ULL + ts.tv_nsec / 1000ULL; +} + +static void update_max_latency(long lat) +{ + long old = atomic_load_explicit(&g_max_latency_us, memory_order_relaxed); + + while (lat > old) { + if (atomic_compare_exchange_weak_explicit(&g_max_latency_us, &old, lat, + memory_order_relaxed, memory_order_relaxed)) + break; + } + + old = atomic_load_explicit(&g_interval_max_latency_us, memory_order_relaxed); + while (lat > old) { + if (atomic_compare_exchange_weak_explicit(&g_interval_max_latency_us, &old, lat, + memory_order_relaxed, memory_order_relaxed)) + break; + } +} + +static void init_stress_cpus(void) +{ + cpu_set_t set; + int capacity = MAX_STRESS_CPUS; + + g_stress_cpus = malloc(capacity * sizeof(int)); + if (!g_stress_cpus) + ksft_exit_fail_msg("malloc failed for g_stress_cpus\n"); + + if (sched_getaffinity(0, sizeof(set), &set) < 0) + ksft_exit_fail_msg("sched_getaffinity failed\n"); + + for (int i = 0; i < CPU_SETSIZE && g_ncpus_stress < capacity; i++) { + if (CPU_ISSET(i, &set)) + g_stress_cpus[g_ncpus_stress++] = i; + } + + if (g_ncpus_stress == 0) + ksft_exit_skip("No CPUs available for stress test\n"); + + ksft_print_msg("Stressing %d CPUs discovered via affinity\n", g_ncpus_stress); +} + +/* -- rseq / membarrier helpers -- */ +static int rseq_register_thread(void) +{ + int r = syscall(SYS_rseq, &tls_rseq, sizeof(tls_rseq), 0, RSEQ_SIG); + + return (r == 0 || errno == EBUSY || errno == EINVAL) ? 0 : -1; +} + +static int rseq_register_thread_at(struct rseq_abi *rseq) +{ + int r = syscall(SYS_rseq, rseq, sizeof(*rseq), 0, RSEQ_SIG); + + return (r == 0 || errno == EBUSY || errno == EINVAL) ? 0 : -1; +} + +static int membarrier_register_rseq_mm(void) +{ + return syscall(SYS_membarrier, + MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0); +} + +/* -- cgroup helpers -- */ +static void rm_cgroup_recursive(const char *path) +{ + DIR *dir = opendir(path); + + if (!dir) + return; + struct dirent *entry; + + while ((entry = readdir(dir)) != NULL) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + continue; + if (entry->d_type == DT_DIR) { + char sub_path[4096]; + + snprintf(sub_path, sizeof(sub_path), "%s/%s", path, entry->d_name); + rm_cgroup_recursive(sub_path); + } + } + closedir(dir); + rmdir(path); +} + +static void cgroup_teardown(void); + +static int cgroup_setup(void) +{ + struct stat st; + + if (stat("/sys/fs/cgroup/cpu", &st) == 0) { + use_cgroup_v2 = 0; + snprintf(g_cgroup_path, sizeof(g_cgroup_path), + "/sys/fs/cgroup/cpu/membarrier_stress_test"); + } else if (stat("/dev/cgroup/cpu", &st) == 0) { + use_cgroup_v2 = 0; + snprintf(g_cgroup_path, sizeof(g_cgroup_path), + "/dev/cgroup/cpu/membarrier_stress_test"); + } else if (stat("/cgroup/cpu", &st) == 0) { + use_cgroup_v2 = 0; + snprintf(g_cgroup_path, sizeof(g_cgroup_path), + "/cgroup/cpu/membarrier_stress_test"); + } else if (stat("/sys/fs/cgroup/cgroup.controllers", &st) == 0) { + use_cgroup_v2 = 1; + snprintf(g_cgroup_path, sizeof(g_cgroup_path), + "/sys/fs/cgroup/membarrier_stress_test"); + } else { + ksft_print_msg("WARN: cgroup mount not found. Using v2 at /sys/fs/cgroup\n"); + use_cgroup_v2 = 1; + snprintf(g_cgroup_path, sizeof(g_cgroup_path), + "/sys/fs/cgroup/membarrier_stress_test"); + } + + /* Robust cleanup before setup */ + cgroup_teardown(); + + if (use_cgroup_v2) { + /* Enable cpu controller in root cgroup */ + if (write_file("/sys/fs/cgroup/cgroup.subtree_control", "+cpu") < 0) + ksft_print_msg("WARN: failed to enable cpu controller in /sys/fs/cgroup\n"); + } + + if (mkdir(g_cgroup_path, 0755) < 0 && errno != EEXIST) { + ksft_print_msg("mkdir base %s failed: %s\n", g_cgroup_path, strerror(errno)); + return -1; + } + + if (use_cgroup_v2) { + char ctrl_path[4096]; + + snprintf(ctrl_path, sizeof(ctrl_path), "%s/cgroup.subtree_control", g_cgroup_path); + if (write_file(ctrl_path, "+cpu") < 0) + ksft_print_msg("WARN: failed to enable cpu controller in %s\n", + g_cgroup_path); + } + + for (int i = 0; i < N_SIBLINGS; i++) { + char sibling_path[4096]; + + snprintf(sibling_path, sizeof(sibling_path), "%s/n%d", g_cgroup_path, i); + if (mkdir(sibling_path, 0755) < 0 && errno != EEXIST) { + ksft_print_msg("mkdir wide %s failed: %s\n", sibling_path, strerror(errno)); + return -1; + } + + if (use_cgroup_v2) { + char ctrl_path[4096]; + + snprintf(ctrl_path, sizeof(ctrl_path), + "%s/cgroup.subtree_control", sibling_path); + if (write_file(ctrl_path, "+cpu") < 0) + ksft_print_msg("WARN: failed to enable cpu controller in %s\n", + sibling_path); + } + + char current_path[4096]; + + snprintf(current_path, sizeof(current_path), "%s", sibling_path); + for (int j = 0; j < NEST_DEPTH; j++) { + snprintf(current_path + strlen(current_path), + sizeof(current_path) - strlen(current_path), "/d%d", j); + if (mkdir(current_path, 0755) < 0 && errno != EEXIST) { + ksft_print_msg("mkdir deep %s failed: %s\n", + current_path, strerror(errno)); + return -1; + } + + /* Enable for all but the leaf */ + if (use_cgroup_v2 && j < NEST_DEPTH - 1) { + char ctrl_path[4096]; + + snprintf(ctrl_path, sizeof(ctrl_path), "%s/cgroup.subtree_control", + current_path); + if (write_file(ctrl_path, "+cpu") < 0) + ksft_print_msg("WARN: cannot enable cpu controller in %s\n", + current_path); + } + } + } + + char quota[64], period[64], max_str[128]; + + snprintf(quota, sizeof(quota), "%d", CFS_QUOTA_US); + snprintf(period, sizeof(period), "%d", CFS_PERIOD_US); + snprintf(max_str, sizeof(max_str), "%d %d", CFS_QUOTA_US, CFS_PERIOD_US); + + if (use_cgroup_v2) { + char max_path[4096]; + + snprintf(max_path, sizeof(max_path), "%s/cpu.max", g_cgroup_path); + if (write_file(max_path, max_str) < 0) { + ksft_print_msg("ERROR: cannot write cpu.max at %s\n", max_path); + return -1; + } + ksft_print_msg("cgroup (v2) %s: cpu.max=%s\n", g_cgroup_path, max_str); + } else { + char quota_path[4096], period_path[4096]; + + snprintf(quota_path, sizeof(quota_path), "%s/cpu.cfs_quota_us", g_cgroup_path); + snprintf(period_path, sizeof(period_path), "%s/cpu.cfs_period_us", g_cgroup_path); + + if (write_file(period_path, period) < 0) { + ksft_print_msg("ERROR: cannot write cpu.cfs_period_us at %s\n", + period_path); + return -1; + } + if (write_file(quota_path, quota) < 0) { + ksft_print_msg("ERROR: cannot write cpu.cfs_quota_us at %s\n", quota_path); + return -1; + } + ksft_print_msg("cgroup (v1) %s: cpu.cfs_quota_us=%d cpu.cfs_period_us=%d\n", + g_cgroup_path, CFS_QUOTA_US, CFS_PERIOD_US); + } + + return 0; +} + +static int cgroup_add_pid_to_path(pid_t pid, const char *path) +{ + char buf[32], file_path[4096]; + + snprintf(buf, sizeof(buf), "%d", (int)pid); + if (use_cgroup_v2) { + snprintf(file_path, sizeof(file_path), "%s/cgroup.procs", path); + return write_file(file_path, buf); + } + /* In v1, try tasks first, fallback to cgroup.procs */ + snprintf(file_path, sizeof(file_path), "%s/tasks", path); + int r = write_file(file_path, buf); + + if (r < 0) { + snprintf(file_path, sizeof(file_path), "%s/cgroup.procs", path); + r = write_file(file_path, buf); + } + return r; +} + +static void cgroup_teardown(void) +{ + rm_cgroup_recursive(g_cgroup_path); +} + +static void cgroup_unthrottle(void) +{ + if (use_cgroup_v2) { + char max_path[4096]; + + snprintf(max_path, sizeof(max_path), "%s/cpu.max", g_cgroup_path); + write_file(max_path, "max"); + } else { + char quota_path[4096]; + + snprintf(quota_path, sizeof(quota_path), "%s/cpu.cfs_quota_us", g_cgroup_path); + write_file(quota_path, "-1"); + } +} + +/* -- CPU burner (inside throttled child process) -- */ +static void *burner_thread_fn(void *arg) +{ + struct rseq_abi my_rseq; + int cpu = (int)(uintptr_t)arg; + + memset(&my_rseq, 0, sizeof(my_rseq)); + my_rseq.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; + + if (rseq_register_thread_at(&my_rseq) < 0) { + perror("rseq_register (burner)"); + return NULL; + } + + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + if (sched_setaffinity(0, sizeof(set), &set) < 0) + perror("sched_setaffinity (burner)"); + + unsigned long sink = 0; + + while (!atomic_load_explicit(&g_stop, memory_order_relaxed)) { + sink++; + /* Prevent compiler from optimizing the loop away */ + asm volatile("" : "+g"(sink)); + } + + return NULL; +} + +static int burner_thread_fn_wrapper(void *arg) +{ + burner_thread_fn(arg); + return 0; +} + +static int leaf_child_fn(void *arg) +{ + int i = (int)(uintptr_t)arg; + int total_burners = g_ncpus_stress * N_BURNER_PER_CPU; + int n_threads_per_leaf = total_burners / N_SIBLINGS; + + if (i < (total_burners % N_SIBLINGS)) + n_threads_per_leaf++; + + prctl(PR_SET_PDEATHSIG, SIGTERM); + if (getppid() == 1) + _exit(1); + + char leaf_path[4096]; + + snprintf(leaf_path, sizeof(leaf_path), "%s/n%d", g_cgroup_path, i); + for (int j = 0; j < NEST_DEPTH; j++) + snprintf(leaf_path + strlen(leaf_path), + sizeof(leaf_path) - strlen(leaf_path), "/d%d", j); + + int r = cgroup_add_pid_to_path(getpid(), leaf_path); + + if (r < 0) { + char buf[512]; + int len = snprintf(buf, sizeof(buf), + "[leaf child %d] failed to join cgroup %s: err %d\n", + i, leaf_path, -r); + (void)!write(2, buf, len); + _exit(1); + } + + for (int j = 0; j < n_threads_per_leaf; j++) { + int cpu = g_stress_cpus[(i * n_threads_per_leaf + j) % g_ncpus_stress]; + + /* Allocate stack via mmap (bypasses heap) */ + size_t stack_size = 64 * 1024; + void *stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (stack == MAP_FAILED) { + const char *msg = "mmap stack failed\n"; + (void)!write(2, msg, strlen(msg)); + _exit(1); + } + + /* Use raw clone to create a thread sharing the VM and thread group */ + pid_t pid = clone(burner_thread_fn_wrapper, stack + stack_size, + CLONE_VM | CLONE_THREAD | CLONE_SIGHAND, + (void *)(uintptr_t)cpu); + if (pid < 0) { + const char *msg = "clone burner failed\n"; + (void)!write(2, msg, strlen(msg)); + _exit(1); + } + } + + // Wait for SIGTERM + sigset_t mask; + + sigemptyset(&mask); + sigaddset(&mask, SIGTERM); + int sig; + + sigwait(&mask, &sig); + + _exit(0); +} + +struct leaf_info { + pid_t pid; + void *stack; +}; + +static int run_throttle_child(void *arg) +{ + (void)arg; + prctl(PR_SET_PDEATHSIG, SIGTERM); + if (getppid() == 1) + _exit(1); + + int n_leafs = N_SIBLINGS; + + /* Block signals before spawning to avoid missing early failures */ + sigset_t mask; + + sigemptyset(&mask); + sigaddset(&mask, SIGTERM); + sigaddset(&mask, SIGCHLD); + sigprocmask(SIG_BLOCK, &mask, NULL); + + /* Use mmap for tracking structures to avoid glibc heap usage */ + struct leaf_info *leaves = mmap(NULL, n_leafs * sizeof(struct leaf_info), + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (leaves == MAP_FAILED) { + const char *msg = "mmap leaves array failed\n"; + (void)!write(2, msg, strlen(msg)); + _exit(1); + } + + for (int i = 0; i < n_leafs; i++) { + size_t stack_size = 64 * 1024; + void *stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (stack == MAP_FAILED) { + const char *msg = "mmap leaf stack failed\n"; + (void)!write(2, msg, strlen(msg)); + _exit(1); + } + + leaves[i].stack = stack; + + pid_t pid = clone(leaf_child_fn, stack + stack_size, + CLONE_VM | SIGCHLD, (void *)(uintptr_t)i); + + if (pid < 0) { + const char *msg = "clone (leaf child) failed\n"; + (void)!write(2, msg, strlen(msg)); + + /* Clean up successfully spawned children */ + for (int j = 0; j < i; j++) { + kill(leaves[j].pid, SIGTERM); + waitpid(leaves[j].pid, NULL, 0); + munmap(leaves[j].stack, stack_size); + } + munmap(leaves, n_leafs * sizeof(struct leaf_info)); + + if (errno == EAGAIN) + _exit(4); + else + _exit(1); + } + leaves[i].pid = pid; + } + + int failed = 0; + + while (1) { + int sig; + + sigwait(&mask, &sig); + + if (sig == SIGTERM) { + break; + } else if (sig == SIGCHLD) { + int status; + pid_t pid; + + // Reap all dead children + while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { + for (int i = 0; i < n_leafs; i++) { + if (leaves[i].pid == pid) { + leaves[i].pid = 0; + break; + } + } + if ((WIFEXITED(status) && WEXITSTATUS(status) != 0) || + WIFSIGNALED(status)) { + char buf[128]; + int len = snprintf(buf, sizeof(buf), + "[manager] child %d died unexpectedly (status %d)\n", + pid, WEXITSTATUS(status)); + (void)!write(2, buf, len); + failed = 1; + } + } + if (failed) + break; + } + } + + // Terminate all leaf kids + for (int i = 0; i < n_leafs; i++) { + if (leaves[i].pid > 0) + kill(leaves[i].pid, SIGTERM); + } + + for (int i = 0; i < n_leafs; i++) { + if (leaves[i].pid > 0) + waitpid(leaves[i].pid, NULL, 0); + munmap(leaves[i].stack, 64 * 1024); + } + + munmap(leaves, n_leafs * sizeof(struct leaf_info)); + + _exit(failed ? 1 : 0); +} + +/* -- Membarrier hammer thread -- */ +static void *hammer_thread_fn(void *arg) +{ + int target_cpu = *(int *)arg; + long local_ok = 0; + long local_err = 0; + int count = 0; + const int batch_size = 1024; + + if (rseq_register_thread() < 0) { + ksft_print_msg("[hammer] rseq_register failed: %s\n", strerror(errno)); + return NULL; + } + + membarrier_register_rseq_mm(); + + while (!atomic_load_explicit(&g_stop, memory_order_relaxed)) { + int r = syscall(SYS_membarrier, + MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, + MEMBARRIER_CMD_FLAG_CPU, + target_cpu); + if (__builtin_expect(r == 0, 1)) + local_ok++; + else + local_err++; + + count++; + if (__builtin_expect(count >= batch_size, 0)) { + atomic_fetch_add_explicit(&g_mb_ok, local_ok, memory_order_relaxed); + atomic_fetch_add_explicit(&g_mb_err, local_err, memory_order_relaxed); + local_ok = 0; + local_err = 0; + count = 0; + } + } + + /* Flush any remaining counts on exit */ + if (local_ok > 0) + atomic_fetch_add_explicit(&g_mb_ok, local_ok, memory_order_relaxed); + if (local_err > 0) + atomic_fetch_add_explicit(&g_mb_err, local_err, memory_order_relaxed); + + return NULL; +} + +/* -- Latency sentinel -- */ +static void *sentinel_thread_fn(void *arg) +{ + (void)arg; + struct sched_param sp = { .sched_priority = 20 }; + + if (sched_setscheduler(0, SCHED_FIFO, &sp) < 0) + ksft_print_msg("WARN: no SCHED_FIFO for sentinel (less precise)\n"); + + while (!atomic_load_explicit(&g_test_ready, memory_order_relaxed) && + !atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed)) { + struct timespec ts = {0, 1000 * 1000}; /* 1ms */ + + clock_nanosleep(CLOCK_MONOTONIC, 0, &ts, NULL); + } + + uint64_t prev = monotonic_us(); + + while (!atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed)) { + struct timespec ts = { + .tv_sec = 0, + .tv_nsec = SENTINEL_INTERVAL_US * 1000L, + }; + clock_nanosleep(CLOCK_MONOTONIC, 0, &ts, NULL); + + uint64_t now = monotonic_us(); + long latency_us = (long)(now - prev) - SENTINEL_INTERVAL_US; + + prev = now; + + if (latency_us <= 0) + continue; + + update_max_latency(latency_us); + + if (latency_us > LATENCY_CRITICAL_MS * 1000L) { + ksft_print_msg("\n[SENTINEL] CRITICAL: %ld ms delay (lockup precursor!)\n", + latency_us / 1000); + } else if (latency_us > LATENCY_WARN_MS * 1000L) { + ksft_print_msg("\n[SENTINEL] WARN: %ld ms latency spike\n", + latency_us / 1000); + } + } + return NULL; +} + +/* -- Progress reporter -- */ +static void *reporter_thread_fn(void *arg) +{ + (void)arg; + int elapsed = 0; + + while (!atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed)) { + for (int i = 0; i < 5; i++) { + sleep(1); + if (atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed)) + break; + } + if (atomic_load_explicit(&g_stop_sentinel, memory_order_relaxed)) + break; + elapsed += 5; + long interval_max = atomic_exchange_explicit(&g_interval_max_latency_us, + 0, memory_order_relaxed); + + ksft_print_msg("[%3ds] mb: ok=%-10ld err=%-8ld | max_lat=%ld us\n", + elapsed, + atomic_load(&g_mb_ok), + atomic_load(&g_mb_err), + interval_max); + } + return NULL; +} + +/* -- Main -- */ +int main(void) +{ + ksft_print_header(); +#ifdef UNSUPPORTED_ARCH + ksft_exit_skip("Unsupported architecture\n"); +#endif + ksft_set_plan(1); + + if (geteuid() != 0) + ksft_exit_skip("Must run as root (cgroup + SCHED_FIFO)\n"); + + init_stress_cpus(); + + ksft_print_msg("=== membarrier rseq + CFS unthrottle stress ===\n"); + ksft_print_msg("Stressing CPUs: %d\n", g_ncpus_stress); + ksft_print_msg("Quota: %d/%d us (~%d unthrottles/sec/CPU)\n", + CFS_QUOTA_US, CFS_PERIOD_US, + 1000000 / CFS_PERIOD_US); + ksft_print_msg("Hammer threads: %d per CPU (%d total)\n", + N_HAMMER_PER_CPU, g_ncpus_stress * N_HAMMER_PER_CPU); + ksft_print_msg("Duration: %d seconds\n\n", TEST_DURATION_SEC); + + if (cgroup_setup() < 0) { + cgroup_teardown(); + ksft_exit_skip("cgroup_setup failed (missing permissions or v2 ctrls?)\n"); + } + + if (rseq_register_thread() < 0) { + ksft_print_msg("rseq_register (%s) failed: %s\n", __func__, strerror(errno)); + cgroup_teardown(); + ksft_exit_skip("rseq syscall failed or not available\n"); + } + if (membarrier_register_rseq_mm() < 0) { + ksft_print_msg("MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ: %s\n" + "Kernel >= 5.10 with CONFIG_RSEQ required.\n", + strerror(errno)); + cgroup_teardown(); + ksft_exit_skip("membarrier register failed\n"); + } + ksft_print_msg("rseq membarrier registered OK\n"); + + sigset_t sigmask; + + sigemptyset(&sigmask); + sigaddset(&sigmask, SIGTERM); + sigprocmask(SIG_BLOCK, &sigmask, NULL); + + void *stack = malloc(1024 * 1024); + + if (!stack) { + perror("malloc stack"); + cgroup_teardown(); + ksft_exit_fail_msg("Malloc stack failed\n"); + } + pid_t child = clone(run_throttle_child, stack + 1024 * 1024, CLONE_VM | SIGCHLD, NULL); + + if (child < 0) { + perror("clone"); + cgroup_teardown(); + ksft_exit_fail_msg("Clone failed\n"); + } + + sigprocmask(SIG_UNBLOCK, &sigmask, NULL); + ksft_print_msg("Throttle child PID %d started\n", child); + + int n_threads = g_ncpus_stress * N_HAMMER_PER_CPU + 2; + pthread_t *threads = (pthread_t *)calloc(n_threads, sizeof(pthread_t)); + int *cpuargs = (int *)calloc(g_ncpus_stress * N_HAMMER_PER_CPU, sizeof(int)); + + if (!threads || !cpuargs) { + perror("calloc"); + kill(child, SIGTERM); + waitpid(child, NULL, 0); + cgroup_teardown(); + ksft_exit_fail_msg("Thread allocation failed\n"); + } + + int ti = 0, ai = 0; + int r; + + ksft_print_msg("Creating sentinel thread...\n"); + r = pthread_create(&threads[ti], NULL, sentinel_thread_fn, NULL); + if (r != 0) { + kill(child, SIGTERM); + waitpid(child, NULL, 0); + cgroup_teardown(); + free(threads); + free(cpuargs); + free(g_stress_cpus); + ksft_exit_fail_msg("pthread_create (sentinel) failed: %s\n", strerror(r)); + } + ti++; + + ksft_print_msg("Creating reporter thread...\n"); + r = pthread_create(&threads[ti], NULL, reporter_thread_fn, NULL); + if (r != 0) { + atomic_store(&g_stop_sentinel, 1); + pthread_join(threads[0], NULL); + kill(child, SIGTERM); + waitpid(child, NULL, 0); + cgroup_teardown(); + free(threads); + free(cpuargs); + free(g_stress_cpus); + ksft_exit_fail_msg("pthread_create (reporter) failed: %s\n", strerror(r)); + } + ti++; + + ksft_print_msg("Creating %d hammer threads...\n", g_ncpus_stress * N_HAMMER_PER_CPU); + for (int i = 0; i < g_ncpus_stress; i++) { + int cpu = g_stress_cpus[i]; + + for (int j = 0; j < N_HAMMER_PER_CPU; j++) { + cpuargs[ai] = cpu; + r = pthread_create(&threads[ti], NULL, hammer_thread_fn, &cpuargs[ai]); + if (r != 0) { + ksft_print_msg("pthread_create failed at thread %d: %s\n", + ti, strerror(r)); + + atomic_store(&g_stop_sentinel, 1); + pthread_join(threads[0], NULL); + pthread_join(threads[1], NULL); + + atomic_store(&g_stop, 1); + for (int k = 2; k < ti; k++) + pthread_join(threads[k], NULL); + + kill(child, SIGTERM); + waitpid(child, NULL, 0); + cgroup_teardown(); + + free(threads); + free(cpuargs); + free(g_stress_cpus); + + if (r == EAGAIN) + ksft_exit_skip("Resource limits prevent threads\n"); + else + ksft_exit_fail_msg("Failed to create hammer thread\n"); + } + ti++; + ai++; + } + } + + ksft_print_msg("All threads running. Tip: monitor dmesg for lockups\n\n"); + + atomic_store_explicit(&g_test_ready, 1, memory_order_relaxed); + int child_failed = 0; + int child_status = 0; + + for (int i = 0; i < TEST_DURATION_SEC; i++) { + sleep(1); + int r = waitpid(child, &child_status, WNOHANG); + + if (r == child) { + child_failed = 1; + break; + } + } + + atomic_store(&g_stop_sentinel, 1); + pthread_join(threads[0], NULL); + pthread_join(threads[1], NULL); + + atomic_store(&g_stop, 1); + + /* Unthrottle to allow children to exit quickly */ + cgroup_unthrottle(); + + if (!child_failed) { + kill(child, SIGTERM); + waitpid(child, NULL, 0); + } + for (int i = 2; i < ti; i++) + pthread_join(threads[i], NULL); + + long max_lat = atomic_load(&g_max_latency_us); + long total_ok = atomic_load(&g_mb_ok); + long total_err = atomic_load(&g_mb_err); + + ksft_print_msg("\n=== RESULTS ===\n"); + ksft_print_msg("membarrier syscalls : %ld ok %ld errors\n", total_ok, total_err); + ksft_print_msg("Max scheduler latency: %ld us (%ld ms)\n", max_lat, max_lat / 1000); + cgroup_teardown(); + free(threads); + free(cpuargs); + free(g_stress_cpus); + + if (child_failed) { + if (WIFEXITED(child_status) && WEXITSTATUS(child_status) == 4) + ksft_exit_skip("Manager child skipped (resource limits?)\n"); + ksft_test_result_fail("membarrier_rseq_stress: Manager child died early\n"); + ksft_exit_fail(); + } else if (total_ok == 0) { + ksft_test_result_fail("membarrier_rseq_stress: No successful membarrier calls\n"); + ksft_exit_fail(); + } else if (total_err > 0) { + ksft_test_result_fail("membarrier_rseq_stress: syscall errors\n"); + ksft_exit_fail(); + } else if (max_lat > LATENCY_CRITICAL_MS * 1000L) { + ksft_test_result_fail("membarrier_rseq_stress: LOCKUP PRECURSOR\n"); + ksft_exit_fail(); + } else if (max_lat > LATENCY_WARN_MS * 1000L) { + ksft_test_result_fail("membarrier_rseq_stress: significant latency spike\n"); + ksft_exit_fail(); + } else { + ksft_test_result_pass("membarrier_rseq_stress\n"); + ksft_exit_pass(); + } + + return 0; +} From a9e4e50519e99e5be2d69fa4eebec6a4848ae201 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 8 May 2026 10:45:16 -0700 Subject: [PATCH 05/36] locking/rtmutex: Annotate API and implementation Enable context analysis for struct rt_mutex and annotate all functions that accept a struct rt_mutex pointer. In the __rt_mutex_lock_common() callers, instead of adding the __no_context_analysis annotation, emit a runtime warning if the __rt_mutex_lock_common() return value is not zero and add an __acquire() statement. Signed-off-by: Bart Van Assche Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260508174520.1416285-1-bvanassche@acm.org --- include/linux/rtmutex.h | 22 +++++++++++++++------- kernel/locking/rtmutex.c | 4 ++++ kernel/locking/rtmutex_api.c | 31 ++++++++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h index 78e7e588817c..9e1f012f89db 100644 --- a/include/linux/rtmutex.h +++ b/include/linux/rtmutex.h @@ -56,6 +56,8 @@ static inline struct task_struct *rt_mutex_owner(struct rt_mutex_base *lock) #endif extern void rt_mutex_base_init(struct rt_mutex_base *rtb); +context_lock_struct(rt_mutex); + /** * The rt_mutex structure * @@ -108,8 +110,10 @@ do { \ extern void __rt_mutex_init(struct rt_mutex *lock, const char *name, struct lock_class_key *key); #ifdef CONFIG_DEBUG_LOCK_ALLOC -extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass); -extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock); +extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) + __acquires(lock); +extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) + __acquires(lock); #define rt_mutex_lock(lock) rt_mutex_lock_nested(lock, 0) #define rt_mutex_lock_nest_lock(lock, nest_lock) \ do { \ @@ -118,15 +122,19 @@ extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map * } while (0) #else -extern void rt_mutex_lock(struct rt_mutex *lock); +extern void rt_mutex_lock(struct rt_mutex *lock) __acquires(lock); #define rt_mutex_lock_nested(lock, subclass) rt_mutex_lock(lock) #define rt_mutex_lock_nest_lock(lock, nest_lock) rt_mutex_lock(lock) #endif -extern int rt_mutex_lock_interruptible(struct rt_mutex *lock); -extern int rt_mutex_lock_killable(struct rt_mutex *lock); -extern int rt_mutex_trylock(struct rt_mutex *lock); +extern int rt_mutex_lock_interruptible(struct rt_mutex *lock) + __cond_acquires(0, lock); +extern int rt_mutex_lock_killable(struct rt_mutex *lock) + __cond_acquires(0, lock); +extern int rt_mutex_trylock(struct rt_mutex *lock) + __cond_acquires(true, lock); -extern void rt_mutex_unlock(struct rt_mutex *lock); +extern void rt_mutex_unlock(struct rt_mutex *lock) + __releases(lock); #endif diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 4f386ea6c792..9147d6a31b78 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -484,6 +484,7 @@ static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_nod static __always_inline void rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) + __must_hold(&lock->wait_lock) { lockdep_assert_held(&lock->wait_lock); @@ -492,6 +493,7 @@ rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) static __always_inline void rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) + __must_hold(&lock->wait_lock) { lockdep_assert_held(&lock->wait_lock); @@ -1092,6 +1094,7 @@ static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task, static int __sched try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task, struct rt_mutex_waiter *waiter) + __must_hold(&lock->wait_lock) { lockdep_assert_held(&lock->wait_lock); @@ -1319,6 +1322,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock, */ static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh, struct rt_mutex_base *lock) + __must_hold(&lock->wait_lock) { struct rt_mutex_waiter *waiter; diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 124219aea46e..7c40b91422a0 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -41,6 +41,7 @@ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock, unsigned int state, struct lockdep_map *nest_lock, unsigned int subclass) + __cond_acquires(0, lock) { int ret; @@ -67,13 +68,27 @@ EXPORT_SYMBOL(rt_mutex_base_init); */ void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) { - __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass); + if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass) == 0) + return; + /* + * The code below is never reached because __rt_mutex_lock_common() only + * returns an error code if interrupted by a signal or upon a timeout. + */ + WARN_ON_ONCE(true); + __acquire(lock); } EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) { - __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0); + if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0) == 0) + return; + /* + * The code below is never reached because __rt_mutex_lock_common() only + * returns an error code if interrupted by a signal or upon a timeout. + */ + WARN_ON_ONCE(true); + __acquire(lock); } EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock); @@ -86,7 +101,14 @@ EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock); */ void __sched rt_mutex_lock(struct rt_mutex *lock) { - __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0); + if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0) == 0) + return; + /* + * The code below is never reached because __rt_mutex_lock_common() only + * returns an error code if interrupted by a signal or upon a timeout. + */ + WARN_ON_ONCE(true); + __acquire(lock); } EXPORT_SYMBOL_GPL(rt_mutex_lock); #endif @@ -157,6 +179,7 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock) { mutex_release(&lock->dep_map, _RET_IP_); __rt_mutex_unlock(&lock->rtmutex); + __release(lock); } EXPORT_SYMBOL_GPL(rt_mutex_unlock); @@ -182,6 +205,7 @@ int __sched __rt_mutex_futex_trylock(struct rt_mutex_base *lock) */ bool __sched __rt_mutex_futex_unlock(struct rt_mutex_base *lock, struct rt_wake_q_head *wqh) + __must_hold(&lock->wait_lock) { lockdep_assert_held(&lock->wait_lock); @@ -312,6 +336,7 @@ int __sched __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter, struct task_struct *task, struct wake_q_head *wake_q) + __must_hold(&lock->wait_lock) { int ret; From f45c5c4adb27540d43302155e2fbaeca6c3c6d03 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 15 May 2026 14:43:31 +0200 Subject: [PATCH 06/36] compiler-context-analysis: Bump required Clang version to 23 Clang 23 introduces several major improvements: 1. Support for multiple arguments in the `guarded_by` and `pt_guarded_by` attributes [1]. This allows defining variables protected by multiple context locks, where read access requires holding at least one lock (shared or exclusive), and write access requires holding all of them exclusively. 2. Function pointer support [2]. We can now add attributes to function pointers just like we do on normal functions. 3. A fix to use arrays of locks [3]. Each index is now correctly treated as a separate lock instance. 4. A fix for implicit member access in attributes [4]. This allows to use __guarded_by(&foo->lock) correctly. Overall that makes it worthwhile bumping the compiler version instead of trying to make both Clang 22 and later work while supporting these new features. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Nathan Chancellor Reviewed-by: Bart Van Assche Link: https://github.com/llvm/llvm-project/pull/186838 [1] Link: https://github.com/llvm/llvm-project/pull/191187 [2] Link: https://github.com/llvm/llvm-project/pull/148551 [3] Link: https://github.com/llvm/llvm-project/pull/194457 [4] Link: https://patch.msgid.link/20260515124426.2227783-1-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 +- include/linux/compiler-context-analysis.h | 30 ++++++++++++++------ lib/Kconfig.debug | 4 +-- lib/test_context-analysis.c | 24 ++++++++++++++++ 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 54d9ee28de98..8e71e1e75b5b 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -17,7 +17,7 @@ features. To enable for Clang, configure the kernel with:: CONFIG_WARN_CONTEXT_ANALYSIS=y -The feature requires Clang 22 or later. +The feature requires Clang 23 or later. The analysis is *opt-in by default*, and requires declaring which modules and subsystems should be analyzed in the respective `Makefile`:: diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index a9317571e6af..8302ebc2ea8c 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -39,12 +39,14 @@ # define __assumes_shared_ctx_lock(...) __attribute__((assert_shared_capability(__VA_ARGS__))) /** - * __guarded_by - struct member and globals attribute, declares variable - * only accessible within active context + * __guarded_by() - struct member and globals attribute, declares variable + * only accessible within active context + * @...: context lock instance pointer(s) * * Declares that the struct member or global variable is only accessible within - * the context entered by the given context lock. Read operations on the data - * require shared access, while write operations require exclusive access. + * the context entered by the given context lock(s). Read operations on the data + * require shared access to at least one of the context locks, while write + * operations require exclusive access to all listed context locks. * * .. code-block:: c * @@ -52,17 +54,24 @@ * spinlock_t lock; * long counter __guarded_by(&lock); * }; + * + * struct some_state { + * spinlock_t lock1, lock2; + * long counter __guarded_by(&lock1, &lock2); + * }; */ # define __guarded_by(...) __attribute__((guarded_by(__VA_ARGS__))) /** - * __pt_guarded_by - struct member and globals attribute, declares pointed-to - * data only accessible within active context + * __pt_guarded_by() - struct member and globals attribute, declares pointed-to + * data only accessible within active context + * @...: context lock instance pointer(s) * * Declares that the data pointed to by the struct member pointer or global * pointer is only accessible within the context entered by the given context - * lock. Read operations on the data require shared access, while write - * operations require exclusive access. + * lock(s). Read operations on the data require shared access to at least one + * of the context locks, while write operations require exclusive access to all + * listed context locks. * * .. code-block:: c * @@ -70,6 +79,11 @@ * spinlock_t lock; * long *counter __pt_guarded_by(&lock); * }; + * + * struct some_state { + * spinlock_t lock1, lock2; + * long *counter __pt_guarded_by(&lock1, &lock2); + * }; */ # define __pt_guarded_by(...) __attribute__((pt_guarded_by(__VA_ARGS__))) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8ff5adcfe1e0..b0f3028a213d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -630,7 +630,7 @@ config DEBUG_FORCE_WEAK_PER_CPU config WARN_CONTEXT_ANALYSIS bool "Compiler context-analysis warnings" - depends on CC_IS_CLANG && CLANG_VERSION >= 220100 + depends on CC_IS_CLANG && CLANG_VERSION >= 230000 # Branch profiling re-defines "if", which messes with the compiler's # ability to analyze __cond_acquires(..), resulting in false positives. depends on !TRACE_BRANCH_PROFILING @@ -641,7 +641,7 @@ config WARN_CONTEXT_ANALYSIS and releasing user-definable "context locks". Clang's name of the feature is "Thread Safety Analysis". Requires - Clang 22.1.0 or later. + Clang 23 or later. Produces warnings by default. Select CONFIG_WERROR if you wish to turn these warnings into errors. diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 06b4a6a028e0..316f4dfcda65 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -159,6 +159,10 @@ TEST_SPINLOCK_COMMON(read_lock, struct test_mutex_data { struct mutex mtx; int counter __guarded_by(&mtx); + + struct mutex mtx2; + int anyread __guarded_by(&mtx, &mtx2); + int *anyptr __pt_guarded_by(&mtx, &mtx2); }; static void __used test_mutex_init(struct test_mutex_data *d) @@ -219,6 +223,26 @@ static void __used test_mutex_cond_guard(struct test_mutex_data *d) } } +static void __used test_mutex_multiguard(struct test_mutex_data *d) +{ + mutex_lock(&d->mtx); + (void)d->anyread; + (void)*d->anyptr; + mutex_unlock(&d->mtx); + + mutex_lock(&d->mtx2); + (void)d->anyread; + (void)*d->anyptr; + mutex_unlock(&d->mtx2); + + mutex_lock(&d->mtx); + mutex_lock(&d->mtx2); + d->anyread++; + (*d->anyptr)++; + mutex_unlock(&d->mtx2); + mutex_unlock(&d->mtx); +} + struct test_seqlock_data { seqlock_t sl; int counter __guarded_by(&sl); From 88331c4ec23a28c1006ec532fa64763d4c695e90 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 19 May 2026 13:03:15 +0200 Subject: [PATCH 07/36] seqlock: Allow UBSAN_ALIGNMENT to fail optimizing With gcc-15 and gcc-16 with UBSAN_ALIGNMENT enabled the compiler fails to inline and optimize __scoped_seqlock_bug() away on s390: s390x-16.1.0-ld: kernel/sched/build_policy.o: in function `__scoped_seqlock_next': /.../seqlock.h:1286:(.text+0x22030): undefined reference to `__scoped_seqlock_bug' Fix this by adding UBSAN_ALIGNMENT to the list of config options where a not inlined empty __scoped_seqlock_bug() is allowed. Closes: https://lore.kernel.org/r/20260515092057.810542-1-arnd@kernel.org/ Reported-by: Arnd Bergmann Signed-off-by: Heiko Carstens Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260519110315.1385307-1-hca@linux.ibm.com --- include/linux/seqlock.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 5a40252b8334..f865491c4f2c 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1259,14 +1259,15 @@ static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) extern void __scoped_seqlock_invalid_target(void); -#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || defined(CONFIG_KASAN) +#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || \ + defined(CONFIG_KASAN) || defined(CONFIG_UBSAN_ALIGNMENT) /* * For some reason some GCC-8 architectures (nios2, alpha) have trouble * determining that the ss_done state is impossible in __scoped_seqlock_next() * below. * - * Similarly KASAN is known to confuse compilers enough to break this. But we - * don't care about code quality for KASAN builds anyway. + * Similarly KASAN and UBSAN_ALIGNMENT are known to confuse compilers enough + * to break this. But we don't care about code quality for such builds anyway. */ static inline void __scoped_seqlock_bug(void) { } #else From cb19b8a9add2ed3774c1276e0a3906e52e8cf39a Mon Sep 17 00:00:00 2001 From: "WEI-HONG, YE" <1234567weewee457@gmail.com> Date: Mon, 25 May 2026 13:04:50 +0000 Subject: [PATCH 08/36] locking/qspinlock: Clarify pending field layout For CONFIG_NR_CPUS < 16K, _Q_PENDING_BITS is 8 and the pending field occupies bits 8-15 of the lock word. The current comment documents bit 8 as pending and bits 9-15 as unused, which describes the pending flag value rather than the field layout. Describe bits 8-15 as the pending byte so the layout description is consistent with the lock byte. Signed-off-by: WEI-HONG, YE <1234567weewee457@gmail.com> Signed-off-by: Peter Zijlstra (Intel) Acked-by: Waiman Long Link: https://patch.msgid.link/20260525130450.723937-1-1234567weewee457@gmail.com --- include/asm-generic/qspinlock_types.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h index 2fd1fb89ec36..7d17b9bb069c 100644 --- a/include/asm-generic/qspinlock_types.h +++ b/include/asm-generic/qspinlock_types.h @@ -53,8 +53,7 @@ typedef struct qspinlock { * * When NR_CPUS < 16K * 0- 7: locked byte - * 8: pending - * 9-15: not used + * 8-15: pending byte * 16-17: tail index * 18-31: tail cpu (+1) * From d8c897b20bf4d4cbb1e935a8ceb666bcc0f82580 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 23 May 2026 20:51:22 +0200 Subject: [PATCH 09/36] lockdep/selftests: Restore migrate_disable() state on PREEMPT_RT The lockdep selftests deliberately run unbalanced locking patterns. dotest() restores the task state they leave behind before running the next testcase. On PREEMPT_RT, spin_lock() uses migrate_disable() instead of disabling preemption. dotest() cleans up the resulting migration-disabled state, but that cleanup is still guarded by CONFIG_SMP. That used to match the scheduler data model, where migration_disabled was also CONFIG_SMP-only. The commit referenced below made SMP scheduler state unconditional, so CONFIG_SMP=n PREEMPT_RT kernels with CONFIG_DEBUG_LOCKING_API_SELFTESTS=y report success from the selftests and then trip over stale current->migration_disabled state: releasing a pinned lock bad: scheduling from the idle thread! Kernel panic - not syncing: Fatal exception Save and restore current->migration_disabled for every PREEMPT_RT build. Fixes: cac5cefbade9 ("sched/smp: Make SMP unconditional") Assisted-by: Codex:gpt-5 Signed-off-by: Karl Mehltretter Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260523185123.17482-2-kmehltretter@gmail.com --- lib/locking-selftest.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index d939403331b5..cb806b628352 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1431,9 +1431,7 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { int saved_preempt_count = preempt_count(); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP int saved_mgd_count = current->migration_disabled; -#endif int saved_rcu_count = current->rcu_read_lock_nesting; #endif @@ -1471,10 +1469,8 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) preempt_count_set(saved_preempt_count); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP while (current->migration_disabled > saved_mgd_count) migrate_enable(); -#endif while (current->rcu_read_lock_nesting > saved_rcu_count) rcu_read_unlock(); From 06961d60a0e410bf8df69ccff7eb1bd824912b8f Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 23 May 2026 20:51:23 +0200 Subject: [PATCH 10/36] lockdep/selftests: Restore sched_rt_mutex state on PREEMPT_RT The WW-mutex selftests deliberately exercise failing lock paths. On PREEMPT_RT, some of those paths enter the RT-mutex scheduler helpers. The change referenced by the Fixes tag made those helpers track RT-mutex scheduling state in current->sched_rt_mutex. The bit is normally cleared by the matching post-schedule helper, but some WW-mutex selftests disable the runtime debug_locks flag before that happens. With debug_locks cleared, lockdep_assert() does not evaluate the expression that clears the bit, leaving stale state for the next testcase. With CONFIG_PREEMPT_RT=y and CONFIG_DEBUG_LOCKING_API_SELFTESTS=y, that stale state produces warnings such as: WARNING: kernel/sched/core.c:7557 at rt_mutex_pre_schedule+0x26/0x2d RIP: 0010:rt_mutex_pre_schedule+0x26/0x2d Save and restore current->sched_rt_mutex around each testcase, matching the existing PREEMPT_RT cleanup for task-local migration and RCU state. Fixes: d14f9e930b90 ("locking/rtmutex: Use rt_mutex specific scheduler helpers") Assisted-by: Codex:gpt-5 Signed-off-by: Karl Mehltretter Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sebastian Andrzej Siewior Link: https://patch.msgid.link/20260523185123.17482-3-kmehltretter@gmail.com --- lib/locking-selftest.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index cb806b628352..bfafe1204c7b 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1433,6 +1433,7 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) #ifdef CONFIG_PREEMPT_RT int saved_mgd_count = current->migration_disabled; int saved_rcu_count = current->rcu_read_lock_nesting; + int saved_sched_rt_mutex = current->sched_rt_mutex; #endif WARN_ON(irqs_disabled()); @@ -1469,6 +1470,8 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) preempt_count_set(saved_preempt_count); #ifdef CONFIG_PREEMPT_RT + current->sched_rt_mutex = saved_sched_rt_mutex; + while (current->migration_disabled > saved_mgd_count) migrate_enable(); From 813e5598e5b551a1fb82b516428ce2f135921122 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 2 Jun 2026 07:12:50 +0000 Subject: [PATCH 11/36] nvdimm: Convert nvdimm_bus guard to class The nvdimm_bus guard accepts NULL and skips locking when NULL is passed. Convert from DEFINE_GUARD() to DEFINE_CLASS() + DEFINE_CLASS_IS_GUARD(). This is a preparatory change for making DEFINE_GUARD() constructors __nonnull_args(). nvdimm_bus legitimately passes NULL, so it must be adjusted to avoid a compile error. No functional change. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dave Jiang Link: https://patch.msgid.link/8c0417904d280896ecf2e9923ffa9f20076f59b8.1780064327.git.d@ilvokhin.com --- drivers/nvdimm/nd.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index b199eea3260e..18b64559664b 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -632,8 +632,11 @@ u64 nd_region_interleave_set_cookie(struct nd_region *nd_region, u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region); void nvdimm_bus_lock(struct device *dev); void nvdimm_bus_unlock(struct device *dev); -DEFINE_GUARD(nvdimm_bus, struct device *, - if (_T) nvdimm_bus_lock(_T), if (_T) nvdimm_bus_unlock(_T)); +DEFINE_CLASS(nvdimm_bus, struct device *, + if (_T) nvdimm_bus_unlock(_T), + ({ if (_T) nvdimm_bus_lock(_T); _T; }), + struct device *_T); +DEFINE_CLASS_IS_GUARD(nvdimm_bus); bool is_nvdimm_bus_locked(struct device *dev); void nvdimm_check_and_set_ro(struct gendisk *disk); From 08d4a7837f008ea6031c1292aa839ad881d7b3f1 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 2 Jun 2026 07:12:51 +0000 Subject: [PATCH 12/36] genirq: Move NULL check into irqdesc_lock guard unlock expression irqdesc_lock uses __DEFINE_UNLOCK_GUARD() directly with a custom constructor that can set .lock to NULL. In preparation for removing the NULL check from __DEFINE_UNLOCK_GUARD(), move the NULL check into the irqdesc_lock unlock expression, making the NULL handling explicit at the call site. No functional change. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Thomas Gleixner Link: https://patch.msgid.link/ab457810653e4356e29b2d74ba616478bd9328ad.1780064327.git.d@ilvokhin.com --- kernel/irq/internals.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 9412e57056f5..347cb333b9fe 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -171,7 +171,7 @@ void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus) __DEFINE_CLASS_IS_CONDITIONAL(irqdesc_lock, true); __DEFINE_UNLOCK_GUARD(irqdesc_lock, struct irq_desc, - __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus), + if (_T->lock) __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus), unsigned long flags; bool bus); static inline class_irqdesc_lock_t class_irqdesc_lock_constructor(unsigned int irq, bool bus, From 22302af28d3f7c3ca38536978c80db51d2a9e283 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 2 Jun 2026 07:12:52 +0000 Subject: [PATCH 13/36] cleanup: Annotate guard constructors with nonnull Add __nonnull_args() to unconditional guard constructors so the compiler warns when NULL is statically known to be passed: - DEFINE_GUARD(): re-declare the constructor with __nonnull_args(). - __DEFINE_LOCK_GUARD_1(): annotate the constructor directly. DEFINE_LOCK_GUARD_0() needs no annotation: its constructor takes no pointer arguments (.lock is hardcoded to (void *)1). Define the __nonnull_args() macro in compiler_attributes.h, following the existing convention for attribute wrappers. Deliberately not named '__nonnull', to avoid clashing with glibc's __nonnull() when kernel and userspace headers are combined (User Mode Linux for example). Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/85fee12eec20abfcf711443518e8f0caec982a86.1780064327.git.d@ilvokhin.com --- include/linux/cleanup.h | 4 +++- include/linux/compiler_attributes.h | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index ea95ca4bc11c..4e60d519713c 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -397,6 +397,7 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond __DEFINE_GUARD_LOCK_PTR(_name, _T) #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ + static __always_inline __nonnull_args() _type class_##_name##_constructor(_type _T); \ DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \ DEFINE_CLASS_IS_GUARD(_name) @@ -497,7 +498,8 @@ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) #define __DEFINE_LOCK_GUARD_1(_name, _type, ...) \ -static __always_inline class_##_name##_t class_##_name##_constructor(_type *l) \ +static __always_inline __nonnull_args() \ +class_##_name##_t class_##_name##_constructor(_type *l) \ __no_context_analysis \ { \ class_##_name##_t _t = { .lock = l }, *_T = &_t; \ diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index c16d4199bf92..cffe09387ea6 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -230,6 +230,15 @@ */ #define noinline __attribute__((__noinline__)) +/* + * Note: deliberately not named '__nonnull', to avoid clashing with glibc's + * __nonnull() when kernel and userspace headers are combined. + * + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-nonnull + * clang: https://clang.llvm.org/docs/AttributeReference.html#nonnull + */ +#define __nonnull_args(x...) __attribute__((__nonnull__(x))) + /* * Optional: only supported since gcc >= 8 * Optional: not supported by clang From a13ab9dd6eb2a89f14b466c3884730ea7969253f Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Tue, 2 Jun 2026 07:12:53 +0000 Subject: [PATCH 14/36] cleanup: Remove NULL check from unconditional guards The unconditional guard destructors check whether the lock pointer is NULL before unlocking. This check is dead code because unconditional guards guarantee a non-NULL lock pointer at destructor time. DEFINE_GUARD() runs the lock operation unconditionally in the constructor. If the pointer were NULL, the lock operation (e.g. mutex_lock(NULL)) would crash before the constructor returns. The destructor never runs with a NULL pointer. All DEFINE_GUARD() users dereference the pointer in their lock. Verified by auditing every instance found by: git grep -n -A 1 'DEFINE_GUARD('. The only exception is xe_pm_runtime_release_only, whose constructor is a noop, but it has no callers. __DEFINE_UNLOCK_GUARD() has only a few usages outside of include/linux/cleanup.h: tty_port_tty (NULL-checks in its tty_kref_put() call), irqdesc_lock (fixed earlier) and two guards in kernel/sched/sched.h (dereference the pointer unconditionally in their lock constructors). DEFINE_LOCK_GUARD_1() sets .lock from its argument and runs the lock operation in the constructor. Same reasoning applies. All DEFINE_LOCK_GUARD_1() users dereference the pointer in their lock. Also, verified by auditing every match of: git grep -n 'DEFINE_LOCK_GUARD_1('. DEFINE_LOCK_GUARD_0() hardcodes .lock = (void *)1 in the constructor, so it is never NULL by construction. Conditional (_try) variants: DEFINE_GUARD_COND() and DEFINE_LOCK_GUARD_1_COND() use EXTEND_CLASS_COND(), whose wrapper destructor returns early when the lock was not acquired, before reaching the base destructor since commit 2deccd5c862a ("cleanup: Optimize guards"): if (_cond) return; class_##_name##_destructor(_T); As compiled by GCC-11 with defconfig on top of the locking/core: Total: Before=23889980, After=23834334, chg -0.23% Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/0503a089389b2270c478a873e095cf0a4ff26d24.1780064327.git.d@ilvokhin.com --- include/linux/cleanup.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index 4e60d519713c..65416938e318 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -398,7 +398,7 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ static __always_inline __nonnull_args() _type class_##_name##_constructor(_type _T); \ - DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \ + DEFINE_CLASS(_name, _type, _unlock, ({ _lock; _T; }), _type _T); \ DEFINE_CLASS_IS_GUARD(_name) #define DEFINE_GUARD_COND_4(_name, _ext, _lock, _cond) \ @@ -492,7 +492,7 @@ typedef struct { \ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ __no_context_analysis \ { \ - if (_T->lock) { _unlock; } \ + _unlock; \ } \ \ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) From c06cd66387da92e6cdac44e16c7b5ef9219c53ac Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:21 +0200 Subject: [PATCH 15/36] percpu: Sanitize __percpu_qual include hell Slapping __percpu_qual into the next available header is sloppy at best. It's required by __percpu which is defined in compiler_types.h and that is meant to be included without requiring a boatload of other headers so that a struct or function declaration can contain a __percpu qualifier w/o further prerequisites. This implicit dependency on linux/percpu.h makes that impossible and causes a major problem when trying to separate headers. Create asm/percpu_types.h and move it there. Include that from compiler_types.h and the whole recursion problem goes away. Fix up UM so it uses the generic header and includes it in the UM_HOST build, which pulls in compiler_types.h. The USER_CFLAGS fix was suggested by Richard. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260602090535.254874125@kernel.org --- arch/um/Makefile | 3 ++- arch/um/include/asm/Kbuild | 1 + arch/x86/include/asm/percpu.h | 5 ----- arch/x86/include/asm/percpu_types.h | 17 +++++++++++++++++ include/asm-generic/Kbuild | 1 + include/asm-generic/percpu_types.h | 19 +++++++++++++++++++ include/linux/compiler_types.h | 3 +++ include/linux/percpu.h | 11 ++++++----- 8 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 arch/x86/include/asm/percpu_types.h create mode 100644 include/asm-generic/percpu_types.h diff --git a/arch/um/Makefile b/arch/um/Makefile index 721b652ffb65..937639edc295 100644 --- a/arch/um/Makefile +++ b/arch/um/Makefile @@ -71,7 +71,8 @@ USER_CFLAGS = $(patsubst $(KERNEL_DEFINES),,$(patsubst -I%,,$(KBUILD_CFLAGS))) \ -D_FILE_OFFSET_BITS=64 -idirafter $(srctree)/include \ -idirafter $(objtree)/include -D__KERNEL__ -D__UM_HOST__ \ -include $(srctree)/include/linux/compiler-version.h \ - -include $(srctree)/include/linux/kconfig.h + -include $(srctree)/include/linux/kconfig.h \ + -idirafter $(ARCH_DIR)/include/generated #This will adjust *FLAGS accordingly to the platform. include $(srctree)/$(ARCH_DIR)/Makefile-os-Linux diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 1b9b82bbe322..e91ba12b7ffc 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -16,6 +16,7 @@ generic-y += module.h generic-y += module.lds.h generic-y += parport.h generic-y += percpu.h +generic-y += percpu_types.h generic-y += preempt.h generic-y += runtime-const.h generic-y += softirq_stack.h diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 409981468cba..cef9a4ca9841 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -40,12 +40,10 @@ #endif #define __percpu_prefix -#define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg) #else /* !CONFIG_CC_HAS_NAMED_AS: */ #define __percpu_prefix __force_percpu_prefix -#define __percpu_seg_override #endif /* CONFIG_CC_HAS_NAMED_AS */ @@ -82,7 +80,6 @@ #define __force_percpu_prefix #define __percpu_prefix -#define __percpu_seg_override #define PER_CPU_VAR(var) (var)__percpu_rel @@ -92,8 +89,6 @@ # define __my_cpu_type(var) typeof(var) # define __my_cpu_ptr(ptr) (ptr) # define __my_cpu_var(var) (var) - -# define __percpu_qual __percpu_seg_override #else # define __my_cpu_type(var) typeof(var) __percpu_seg_override # define __my_cpu_ptr(ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr) diff --git a/arch/x86/include/asm/percpu_types.h b/arch/x86/include/asm/percpu_types.h new file mode 100644 index 000000000000..0aa3e47a3643 --- /dev/null +++ b/arch/x86/include/asm/percpu_types.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_PERCPU_TYPES_H +#define _ASM_X86_PERCPU_TYPES_H + +#if defined(CONFIG_SMP) && defined(CONFIG_CC_HAS_NAMED_AS) +#define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg) +#else /* !CONFIG_CC_HAS_NAMED_AS: */ +#define __percpu_seg_override +#endif + +#if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(USE_TYPEOF_UNQUAL) +#define __percpu_qual __percpu_seg_override +#endif + +#include + +#endif diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 2c53a1e0b760..15df9dcb42a5 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -44,6 +44,7 @@ mandatory-y += module.lds.h mandatory-y += msi.h mandatory-y += pci.h mandatory-y += percpu.h +mandatory-y += percpu_types.h mandatory-y += pgalloc.h mandatory-y += preempt.h mandatory-y += rqspinlock.h diff --git a/include/asm-generic/percpu_types.h b/include/asm-generic/percpu_types.h new file mode 100644 index 000000000000..a095cea7fa20 --- /dev/null +++ b/include/asm-generic/percpu_types.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_PERCPU_TYPES_H_ +#define _ASM_GENERIC_PERCPU_TYPES_H_ + +#ifndef __ASSEMBLER__ +/* + * __percpu_qual is the qualifier for the percpu named address space. + * + * Most architectures use generic named address space for percpu variables but + * some architectures define percpu variables in different named address space. + * E.g. on x86, percpu variable may be declared as being relative to the %fs or + * %gs segments using __seg_fs or __seg_gs named address space qualifier. + */ +#ifndef __percpu_qual +# define __percpu_qual +#endif + +#endif /* __ASSEMBLER__ */ +#endif /* _ASM_GENERIC_PERCPU_TYPES_H_ */ diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index e8fd77593b68..7ad37adda1dd 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -634,6 +634,9 @@ struct ftrace_likely_data { #else #define __unqual_scalar_typeof(x) __typeof_unqual__(x) #endif + +#include + #endif /* !__ASSEMBLY__ */ /* diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 85bf8dd9f087..2f5a889aa50d 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -3,13 +3,14 @@ #define __LINUX_PERCPU_H #include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include #include +#include #include From c1ffc9c6e4f8a13dd68e97920c9a24d095c6e41a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:25 +0200 Subject: [PATCH 16/36] futex: Move futex task related data into a struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Having all these members in task_struct along with the required #ifdeffery is annoying, does not allow efficient initializing of the data with memset() and makes extending it tedious. Move it into a data structure and fix up all usage sites. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Mathieu Desnoyers Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.308220888@kernel.org --- Documentation/locking/robust-futexes.rst | 8 ++-- include/linux/futex.h | 12 ++--- include/linux/futex_types.h | 36 ++++++++++++++ include/linux/sched.h | 16 ++----- kernel/exit.c | 4 +- kernel/futex/core.c | 61 ++++++++++++------------ kernel/futex/pi.c | 26 +++++----- kernel/futex/syscalls.c | 23 ++++----- 8 files changed, 102 insertions(+), 84 deletions(-) create mode 100644 include/linux/futex_types.h diff --git a/Documentation/locking/robust-futexes.rst b/Documentation/locking/robust-futexes.rst index 6361fb01c9c1..1423f53ea2f4 100644 --- a/Documentation/locking/robust-futexes.rst +++ b/Documentation/locking/robust-futexes.rst @@ -94,7 +94,7 @@ time, the kernel checks this user-space list: are there any robust futex locks to be cleaned up? In the common case, at do_exit() time, there is no list registered, so -the cost of robust futexes is just a simple current->robust_list != NULL +the cost of robust futexes is just a current->futex.robust_list != NULL comparison. If the thread has registered a list, then normally the list is empty. If the thread/process crashed or terminated in some incorrect way then the list might be non-empty: in this case the kernel carefully @@ -178,9 +178,9 @@ one to query the registered list pointer:: size_t __user *len_ptr); List registration is very fast: the pointer is simply stored in -current->robust_list. [Note that in the future, if robust futexes become -widespread, we could extend sys_clone() to register a robust-list head -for new threads, without the need of another syscall.] +current->futex.robust_list. [Note that in the future, if robust futexes +become widespread, we could extend sys_clone() to register a robust-list +head for new threads, without the need of another syscall.] So there is virtually zero overhead for tasks not using robust futexes, and even for robust futex users, there is only one extra syscall per diff --git a/include/linux/futex.h b/include/linux/futex.h index 9e9750f04980..563e8dd67179 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -64,14 +64,10 @@ enum { static inline void futex_init_task(struct task_struct *tsk) { - tsk->robust_list = NULL; -#ifdef CONFIG_COMPAT - tsk->compat_robust_list = NULL; -#endif - INIT_LIST_HEAD(&tsk->pi_state_list); - tsk->pi_state_cache = NULL; - tsk->futex_state = FUTEX_STATE_OK; - mutex_init(&tsk->futex_exit_mutex); + memset(&tsk->futex, 0, sizeof(tsk->futex)); + INIT_LIST_HEAD(&tsk->futex.pi_state_list); + tsk->futex.state = FUTEX_STATE_OK; + mutex_init(&tsk->futex.exit_mutex); } void futex_exit_recursive(struct task_struct *tsk); diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h new file mode 100644 index 000000000000..9c6c0dc4148d --- /dev/null +++ b/include/linux/futex_types.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_FUTEX_TYPES_H +#define _LINUX_FUTEX_TYPES_H + +#ifdef CONFIG_FUTEX +#include +#include + +struct compat_robust_list_head; +struct futex_pi_state; +struct robust_list_head; + +/** + * struct futex_sched_data - Futex related per task data + * @robust_list: User space registered robust list pointer + * @compat_robust_list: User space registered robust list pointer for compat tasks + * @pi_state_list: List head for Priority Inheritance (PI) state management + * @pi_state_cache: Pointer to cache one PI state object per task + * @exit_mutex: Mutex for serializing exit + * @state: Futex handling state to handle exit races correctly + */ +struct futex_sched_data { + struct robust_list_head __user *robust_list; +#ifdef CONFIG_COMPAT + struct compat_robust_list_head __user *compat_robust_list; +#endif + struct list_head pi_state_list; + struct futex_pi_state *pi_state_cache; + struct mutex exit_mutex; + unsigned int state; +}; +#else +struct futex_sched_data { }; +#endif /* !CONFIG_FUTEX */ + +#endif /* _LINUX_FUTEX_TYPES_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 368c7b4d7cb5..c88fc10e9c38 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -64,7 +65,6 @@ struct bpf_net_context; struct capture_control; struct cfs_rq; struct fs_struct; -struct futex_pi_state; struct io_context; struct io_uring_task; struct mempolicy; @@ -76,7 +76,6 @@ struct pid_namespace; struct pipe_inode_info; struct rcu_node; struct reclaim_state; -struct robust_list_head; struct root_domain; struct rq; struct sched_attr; @@ -1331,16 +1330,9 @@ struct task_struct { u32 closid; u32 rmid; #endif -#ifdef CONFIG_FUTEX - struct robust_list_head __user *robust_list; -#ifdef CONFIG_COMPAT - struct compat_robust_list_head __user *compat_robust_list; -#endif - struct list_head pi_state_list; - struct futex_pi_state *pi_state_cache; - struct mutex futex_exit_mutex; - unsigned int futex_state; -#endif + + struct futex_sched_data futex; + #ifdef CONFIG_PERF_EVENTS u8 perf_recursion[PERF_NR_CONTEXTS]; struct perf_event_context *perf_event_ctxp; diff --git a/kernel/exit.c b/kernel/exit.c index 25e9cb6de7e7..1b4e55b60bc2 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -988,8 +988,8 @@ void __noreturn do_exit(long code) proc_exit_connector(tsk); mpol_put_task_policy(tsk); #ifdef CONFIG_FUTEX - if (unlikely(current->pi_state_cache)) - kfree(current->pi_state_cache); + if (unlikely(current->futex.pi_state_cache)) + kfree(current->futex.pi_state_cache); #endif /* * Make sure we are holding no locks: diff --git a/kernel/futex/core.c b/kernel/futex/core.c index ff2a4fb2993f..e7d33d2771ec 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -32,18 +32,19 @@ * "But they come in a choice of three flavours!" */ #include -#include -#include #include -#include -#include -#include -#include #include -#include -#include +#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include #include "futex.h" #include "../locking/rtmutex_common.h" @@ -829,7 +830,7 @@ void wait_for_owner_exiting(int ret, struct task_struct *exiting) if (WARN_ON_ONCE(ret == -EBUSY && !exiting)) return; - mutex_lock(&exiting->futex_exit_mutex); + mutex_lock(&exiting->futex.exit_mutex); /* * No point in doing state checking here. If the waiter got here * while the task was in exec()->exec_futex_release() then it can @@ -838,7 +839,7 @@ void wait_for_owner_exiting(int ret, struct task_struct *exiting) * already. Highly unlikely and not a problem. Just one more round * through the futex maze. */ - mutex_unlock(&exiting->futex_exit_mutex); + mutex_unlock(&exiting->futex.exit_mutex); put_task_struct(exiting); } @@ -1047,7 +1048,7 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, * * In both cases the following conditions are met: * - * 1) task->robust_list->list_op_pending != NULL + * 1) task->futex.robust_list->list_op_pending != NULL * @pending_op == true * 2) The owner part of user space futex value == 0 * 3) Regular futex: @pi == false @@ -1152,7 +1153,7 @@ static inline int fetch_robust_entry(struct robust_list __user **entry, */ static void exit_robust_list(struct task_struct *curr) { - struct robust_list_head __user *head = curr->robust_list; + struct robust_list_head __user *head = curr->futex.robust_list; struct robust_list __user *entry, *next_entry, *pending; unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; unsigned int next_pi; @@ -1246,7 +1247,7 @@ compat_fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **ent */ static void compat_exit_robust_list(struct task_struct *curr) { - struct compat_robust_list_head __user *head = curr->compat_robust_list; + struct compat_robust_list_head __user *head = curr->futex.compat_robust_list; struct robust_list __user *entry, *next_entry, *pending; unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; unsigned int next_pi; @@ -1322,7 +1323,7 @@ static void compat_exit_robust_list(struct task_struct *curr) */ static void exit_pi_state_list(struct task_struct *curr) { - struct list_head *next, *head = &curr->pi_state_list; + struct list_head *next, *head = &curr->futex.pi_state_list; struct futex_pi_state *pi_state; union futex_key key = FUTEX_KEY_INIT; @@ -1406,19 +1407,19 @@ static inline void exit_pi_state_list(struct task_struct *curr) { } static void futex_cleanup(struct task_struct *tsk) { - if (unlikely(tsk->robust_list)) { + if (unlikely(tsk->futex.robust_list)) { exit_robust_list(tsk); - tsk->robust_list = NULL; + tsk->futex.robust_list = NULL; } #ifdef CONFIG_COMPAT - if (unlikely(tsk->compat_robust_list)) { + if (unlikely(tsk->futex.compat_robust_list)) { compat_exit_robust_list(tsk); - tsk->compat_robust_list = NULL; + tsk->futex.compat_robust_list = NULL; } #endif - if (unlikely(!list_empty(&tsk->pi_state_list))) + if (unlikely(!list_empty(&tsk->futex.pi_state_list))) exit_pi_state_list(tsk); } @@ -1442,23 +1443,23 @@ static void futex_cleanup(struct task_struct *tsk) void futex_exit_recursive(struct task_struct *tsk) { /* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */ - if (tsk->futex_state == FUTEX_STATE_EXITING) { - __assume_ctx_lock(&tsk->futex_exit_mutex); - mutex_unlock(&tsk->futex_exit_mutex); + if (tsk->futex.state == FUTEX_STATE_EXITING) { + __assume_ctx_lock(&tsk->futex.exit_mutex); + mutex_unlock(&tsk->futex.exit_mutex); } - tsk->futex_state = FUTEX_STATE_DEAD; + tsk->futex.state = FUTEX_STATE_DEAD; } static void futex_cleanup_begin(struct task_struct *tsk) - __acquires(&tsk->futex_exit_mutex) + __acquires(&tsk->futex.exit_mutex) { /* * Prevent various race issues against a concurrent incoming waiter * including live locks by forcing the waiter to block on - * tsk->futex_exit_mutex when it observes FUTEX_STATE_EXITING in + * tsk->futex.exit_mutex when it observes FUTEX_STATE_EXITING in * attach_to_pi_owner(). */ - mutex_lock(&tsk->futex_exit_mutex); + mutex_lock(&tsk->futex.exit_mutex); /* * Switch the state to FUTEX_STATE_EXITING under tsk->pi_lock. @@ -1472,23 +1473,23 @@ static void futex_cleanup_begin(struct task_struct *tsk) * be observed in exit_pi_state_list(). */ raw_spin_lock_irq(&tsk->pi_lock); - tsk->futex_state = FUTEX_STATE_EXITING; + tsk->futex.state = FUTEX_STATE_EXITING; raw_spin_unlock_irq(&tsk->pi_lock); } static void futex_cleanup_end(struct task_struct *tsk, int state) - __releases(&tsk->futex_exit_mutex) + __releases(&tsk->futex.exit_mutex) { /* * Lockless store. The only side effect is that an observer might * take another loop until it becomes visible. */ - tsk->futex_state = state; + tsk->futex.state = state; /* * Drop the exit protection. This unblocks waiters which observed * FUTEX_STATE_EXITING to reevaluate the state. */ - mutex_unlock(&tsk->futex_exit_mutex); + mutex_unlock(&tsk->futex.exit_mutex); } void futex_exec_release(struct task_struct *tsk) diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 643199fdbe62..e037a97fe277 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -14,7 +14,7 @@ int refill_pi_state_cache(void) { struct futex_pi_state *pi_state; - if (likely(current->pi_state_cache)) + if (likely(current->futex.pi_state_cache)) return 0; pi_state = kzalloc_obj(*pi_state); @@ -28,17 +28,17 @@ int refill_pi_state_cache(void) refcount_set(&pi_state->refcount, 1); pi_state->key = FUTEX_KEY_INIT; - current->pi_state_cache = pi_state; + current->futex.pi_state_cache = pi_state; return 0; } static struct futex_pi_state *alloc_pi_state(void) { - struct futex_pi_state *pi_state = current->pi_state_cache; + struct futex_pi_state *pi_state = current->futex.pi_state_cache; WARN_ON(!pi_state); - current->pi_state_cache = NULL; + current->futex.pi_state_cache = NULL; return pi_state; } @@ -60,7 +60,7 @@ static void pi_state_update_owner(struct futex_pi_state *pi_state, if (new_owner) { raw_spin_lock(&new_owner->pi_lock); WARN_ON(!list_empty(&pi_state->list)); - list_add(&pi_state->list, &new_owner->pi_state_list); + list_add(&pi_state->list, &new_owner->futex.pi_state_list); pi_state->owner = new_owner; raw_spin_unlock(&new_owner->pi_lock); } @@ -96,7 +96,7 @@ void put_pi_state(struct futex_pi_state *pi_state) raw_spin_unlock_irqrestore(&pi_state->pi_mutex.wait_lock, flags); } - if (current->pi_state_cache) { + if (current->futex.pi_state_cache) { kfree(pi_state); } else { /* @@ -106,7 +106,7 @@ void put_pi_state(struct futex_pi_state *pi_state) */ pi_state->owner = NULL; refcount_set(&pi_state->refcount, 1); - current->pi_state_cache = pi_state; + current->futex.pi_state_cache = pi_state; } } @@ -179,7 +179,7 @@ void put_pi_state(struct futex_pi_state *pi_state) * * p->pi_lock: * - * p->pi_state_list -> pi_state->list, relation + * p->futex.pi_state_list -> pi_state->list, relation * pi_mutex->owner -> pi_state->owner, relation * * pi_state->refcount: @@ -327,7 +327,7 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval, * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the * caller that the alleged owner is busy. */ - if (tsk && tsk->futex_state != FUTEX_STATE_DEAD) + if (tsk && tsk->futex.state != FUTEX_STATE_DEAD) return -EBUSY; /* @@ -346,8 +346,8 @@ static int handle_exit_race(u32 __user *uaddr, u32 uval, * *uaddr = 0xC0000000; tsk = get_task(PID); * } if (!tsk->flags & PF_EXITING) { * ... attach(); - * tsk->futex_state = } else { - * FUTEX_STATE_DEAD; if (tsk->futex_state != + * tsk->futex.state = } else { + * FUTEX_STATE_DEAD; if (tsk->futex.state != * FUTEX_STATE_DEAD) * return -EAGAIN; * return -ESRCH; <--- FAIL @@ -396,7 +396,7 @@ static void __attach_to_pi_owner(struct task_struct *p, union futex_key *key, pi_state->key = *key; WARN_ON(!list_empty(&pi_state->list)); - list_add(&pi_state->list, &p->pi_state_list); + list_add(&pi_state->list, &p->futex.pi_state_list); /* * Assignment without holding pi_state->pi_mutex.wait_lock is safe * because there is no concurrency as the object is not published yet. @@ -440,7 +440,7 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, * in futex_exit_release(), we do this protected by p->pi_lock: */ raw_spin_lock_irq(&p->pi_lock); - if (unlikely(p->futex_state != FUTEX_STATE_OK)) { + if (unlikely(p->futex.state != FUTEX_STATE_OK)) { /* * The task is on the way out. When the futex state is * FUTEX_STATE_DEAD, we know that the task has finished diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 77ad9691f6a6..8944ff4930d7 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -25,17 +25,13 @@ * @head: pointer to the list-head * @len: length of the list-head, as userspace expects */ -SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, - size_t, len) +SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, size_t, len) { - /* - * The kernel knows only one size for now: - */ + /* The kernel knows only one size for now. */ if (unlikely(len != sizeof(*head))) return -EINVAL; - current->robust_list = head; - + current->futex.robust_list = head; return 0; } @@ -43,9 +39,9 @@ static inline void __user *futex_task_robust_list(struct task_struct *p, bool co { #ifdef CONFIG_COMPAT if (compat) - return p->compat_robust_list; + return p->futex.compat_robust_list; #endif - return p->robust_list; + return p->futex.robust_list; } static void __user *futex_get_robust_list_common(int pid, bool compat) @@ -475,15 +471,13 @@ SYSCALL_DEFINE4(futex_requeue, } #ifdef CONFIG_COMPAT -COMPAT_SYSCALL_DEFINE2(set_robust_list, - struct compat_robust_list_head __user *, head, - compat_size_t, len) +COMPAT_SYSCALL_DEFINE2(set_robust_list, struct compat_robust_list_head __user *, head, + compat_size_t, len) { if (unlikely(len != sizeof(*head))) return -EINVAL; - current->compat_robust_list = head; - + current->futex.compat_robust_list = head; return 0; } @@ -523,4 +517,3 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); } #endif /* CONFIG_COMPAT_32BIT_TIME */ - From d7b3f52c861f54ba2fff15696d3798277fb4c19f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:29 +0200 Subject: [PATCH 17/36] futex: Make futex_mm_init() void Nothing fails there. Mop up the leftovers of the early version of this, which did an allocation. While at it clean up the stubs and the #ifdef comments to make the header file readable. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260602090535.356789395@kernel.org --- include/linux/futex.h | 28 +++++++++++----------------- kernel/fork.c | 8 ++------ kernel/futex/core.c | 3 +-- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/include/linux/futex.h b/include/linux/futex.h index 563e8dd67179..9e6218c2be66 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -81,22 +81,20 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) #ifdef CONFIG_FUTEX_PRIVATE_HASH int futex_hash_allocate_default(void); void futex_hash_free(struct mm_struct *mm); -int futex_mm_init(struct mm_struct *mm); - -#else /* !CONFIG_FUTEX_PRIVATE_HASH */ +void futex_mm_init(struct mm_struct *mm); +#else /* CONFIG_FUTEX_PRIVATE_HASH */ static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline int futex_mm_init(struct mm_struct *mm) { return 0; } -#endif /* CONFIG_FUTEX_PRIVATE_HASH */ +static inline void futex_mm_init(struct mm_struct *mm) { } +#endif /* !CONFIG_FUTEX_PRIVATE_HASH */ -#else /* !CONFIG_FUTEX */ +#else /* CONFIG_FUTEX */ static inline void futex_init_task(struct task_struct *tsk) { } static inline void futex_exit_recursive(struct task_struct *tsk) { } static inline void futex_exit_release(struct task_struct *tsk) { } static inline void futex_exec_release(struct task_struct *tsk) { } -static inline long do_futex(u32 __user *uaddr, int op, u32 val, - ktime_t *timeout, u32 __user *uaddr2, - u32 val2, u32 val3) +static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, + u32 __user *uaddr2, u32 val2, u32 val3) { return -EINVAL; } @@ -104,13 +102,9 @@ static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsig { return -EINVAL; } -static inline int futex_hash_allocate_default(void) -{ - return 0; -} +static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline int futex_mm_init(struct mm_struct *mm) { return 0; } +static inline void futex_mm_init(struct mm_struct *mm) { } +#endif /* !CONFIG_FUTEX */ -#endif - -#endif +#endif /* _LINUX_FUTEX_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 5f3fdfdb14c7..bb490d97c222 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1101,6 +1101,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, #endif mm_init_uprobes_state(mm); hugetlb_count_init(mm); + futex_mm_init(mm); mm_flags_clear_all(mm); if (current->mm) { @@ -1113,11 +1114,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->def_flags = 0; } - if (futex_mm_init(mm)) - goto fail_mm_init; - if (mm_alloc_pgd(mm)) - goto fail_nopgd; + goto fail_mm_init; if (mm_alloc_id(mm)) goto fail_noid; @@ -1144,8 +1142,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm_free_id(mm); fail_noid: mm_free_pgd(mm); -fail_nopgd: - futex_hash_free(mm); fail_mm_init: free_mm(mm); return NULL; diff --git a/kernel/futex/core.c b/kernel/futex/core.c index e7d33d2771ec..ec23de4912b3 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1720,7 +1720,7 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) return atomic_long_read(&mm->futex_atomic) == 0; } -int futex_mm_init(struct mm_struct *mm) +void futex_mm_init(struct mm_struct *mm) { mutex_init(&mm->futex_hash_lock); RCU_INIT_POINTER(mm->futex_phash, NULL); @@ -1729,7 +1729,6 @@ int futex_mm_init(struct mm_struct *mm) mm->futex_ref = NULL; atomic_long_set(&mm->futex_atomic, 0); mm->futex_batches = get_state_synchronize_rcu(); - return 0; } void futex_hash_free(struct mm_struct *mm) From 1f7f4816b9b05e5110bc1c8a05c3c478e2dae11b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:34 +0200 Subject: [PATCH 18/36] futex: Move futex related mm_struct data into a struct Having all these members in mm_struct along with the required #ifdeffery is annoying, does not allow efficient initializing of the data with memset() and makes extending it tedious. Move it into a data structure and fix up all usage sites. The extra struct for the private hash is intentional to make integration of other conditional mechanisms easier in terms of initialization and separation. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260602090535.407756793@kernel.org --- include/linux/futex_types.h | 36 +++++++++- include/linux/mm_types.h | 12 +--- kernel/futex/core.c | 133 +++++++++++++++++------------------- 3 files changed, 98 insertions(+), 83 deletions(-) diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h index 9c6c0dc4148d..d41557d2790d 100644 --- a/include/linux/futex_types.h +++ b/include/linux/futex_types.h @@ -3,6 +3,7 @@ #define _LINUX_FUTEX_TYPES_H #ifdef CONFIG_FUTEX +#include #include #include @@ -29,8 +30,41 @@ struct futex_sched_data { struct mutex exit_mutex; unsigned int state; }; -#else + +#ifdef CONFIG_FUTEX_PRIVATE_HASH +/** + * struct futex_mm_phash - Futex private hash related per MM data + * @lock: Mutex to protect the private hash operations + * @hash: RCU managed pointer to the private hash + * @hash_new: Pointer to a newly allocated private hash + * @batches: Batch state for RCU synchronization + * @rcu: RCU head for call_rcu() + * @atomic: Aggregate value for @hash_ref + * @ref: Per CPU reference counter for a private hash + */ +struct futex_mm_phash { + struct mutex lock; + struct futex_private_hash __rcu *hash; + struct futex_private_hash *hash_new; + unsigned long batches; + struct rcu_head rcu; + atomic_long_t atomic; + unsigned int __percpu *ref; +}; +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +struct futex_mm_phash { }; +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + +/** + * struct futex_mm_data - Futex related per MM data + * @phash: Futex private hash related data + */ +struct futex_mm_data { + struct futex_mm_phash phash; +}; +#else /* CONFIG_FUTEX */ struct futex_sched_data { }; +struct futex_mm_data { }; #endif /* !CONFIG_FUTEX */ #endif /* _LINUX_FUTEX_TYPES_H */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a308e2c23b82..1d0c8d8875aa 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -1270,16 +1271,7 @@ struct mm_struct { */ seqcount_t mm_lock_seq; #endif -#ifdef CONFIG_FUTEX_PRIVATE_HASH - struct mutex futex_hash_lock; - struct futex_private_hash __rcu *futex_phash; - struct futex_private_hash *futex_phash_new; - /* futex-ref */ - unsigned long futex_batches; - struct rcu_head futex_rcu; - atomic_long_t futex_atomic; - unsigned int __percpu *futex_ref; -#endif + struct futex_mm_data futex; unsigned long hiwater_rss; /* High-watermark of RSS usage */ unsigned long hiwater_vm; /* High-water virtual memory usage */ diff --git a/kernel/futex/core.c b/kernel/futex/core.c index ec23de4912b3..79456b0ac1c6 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -188,13 +188,13 @@ __futex_hash_private(union futex_key *key, struct futex_private_hash *fph) return NULL; if (!fph) - fph = rcu_dereference(key->private.mm->futex_phash); + fph = rcu_dereference(key->private.mm->futex.phash.hash); if (!fph || !fph->hash_mask) return NULL; - hash = jhash2((void *)&key->private.address, - sizeof(key->private.address) / 4, + hash = jhash2((void *)&key->private.address, sizeof(key->private.address) / 4, key->both.offset); + return &fph->queues[hash & fph->hash_mask]; } @@ -233,18 +233,17 @@ static void futex_rehash_private(struct futex_private_hash *old, } } -static bool __futex_pivot_hash(struct mm_struct *mm, - struct futex_private_hash *new) +static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash *new) { + struct futex_mm_phash *mmph = &mm->futex.phash; struct futex_private_hash *fph; - WARN_ON_ONCE(mm->futex_phash_new); + WARN_ON_ONCE(mmph->hash_new); - fph = rcu_dereference_protected(mm->futex_phash, - lockdep_is_held(&mm->futex_hash_lock)); + fph = rcu_dereference_protected(mmph->hash, lockdep_is_held(&mmph->lock)); if (fph) { if (!futex_ref_is_dead(fph)) { - mm->futex_phash_new = new; + mmph->hash_new = new; return false; } @@ -252,8 +251,8 @@ static bool __futex_pivot_hash(struct mm_struct *mm, } new->state = FR_PERCPU; scoped_guard(rcu) { - mm->futex_batches = get_state_synchronize_rcu(); - rcu_assign_pointer(mm->futex_phash, new); + mmph->batches = get_state_synchronize_rcu(); + rcu_assign_pointer(mmph->hash, new); } kvfree_rcu(fph, rcu); return true; @@ -261,12 +260,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm, static void futex_pivot_hash(struct mm_struct *mm) { - scoped_guard(mutex, &mm->futex_hash_lock) { + scoped_guard(mutex, &mm->futex.phash.lock) { struct futex_private_hash *fph; - fph = mm->futex_phash_new; + fph = mm->futex.phash.hash_new; if (fph) { - mm->futex_phash_new = NULL; + mm->futex.phash.hash_new = NULL; __futex_pivot_hash(mm, fph); } } @@ -289,7 +288,7 @@ struct futex_private_hash *futex_private_hash(void) scoped_guard(rcu) { struct futex_private_hash *fph; - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); if (!fph) return NULL; @@ -412,8 +411,7 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * private hash) is returned if existing. Otherwise a hash bucket from the * global hash is returned. */ -static struct futex_hash_bucket * -__futex_hash(union futex_key *key, struct futex_private_hash *fph) +static struct futex_hash_bucket *__futex_hash(union futex_key *key, struct futex_private_hash *fph) { int node = key->both.node; u32 hash; @@ -426,8 +424,7 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) return hb; } - hash = jhash2((u32 *)key, - offsetof(typeof(*key), both.offset) / sizeof(u32), + hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(u32), key->both.offset); if (node == FUTEX_NO_NODE) { @@ -442,8 +439,7 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) */ node = (hash >> futex_hashshift) % nr_node_ids; if (!node_possible(node)) { - node = find_next_bit_wrap(node_possible_map.bits, - nr_node_ids, node); + node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node); } } @@ -460,9 +456,8 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) * Return: Initialized hrtimer_sleeper structure or NULL if no timeout * value given */ -struct hrtimer_sleeper * -futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, - int flags, u64 range_ns) +struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, + int flags, u64 range_ns) { if (!time) return NULL; @@ -1554,17 +1549,17 @@ static void __futex_ref_atomic_begin(struct futex_private_hash *fph) * otherwise it would be impossible for it to have reported success * from futex_ref_is_dead(). */ - WARN_ON_ONCE(atomic_long_read(&mm->futex_atomic) != 0); + WARN_ON_ONCE(atomic_long_read(&mm->futex.phash.atomic) != 0); /* * Set the atomic to the bias value such that futex_ref_{get,put}() * will never observe 0. Will be fixed up in __futex_ref_atomic_end() * when folding in the percpu count. */ - atomic_long_set(&mm->futex_atomic, LONG_MAX); + atomic_long_set(&mm->futex.phash.atomic, LONG_MAX); smp_store_release(&fph->state, FR_ATOMIC); - call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); + call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu); } static void __futex_ref_atomic_end(struct futex_private_hash *fph) @@ -1585,7 +1580,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) * Therefore the per-cpu counter is now stable, sum and reset. */ for_each_possible_cpu(cpu) { - unsigned int *ptr = per_cpu_ptr(mm->futex_ref, cpu); + unsigned int *ptr = per_cpu_ptr(mm->futex.phash.ref, cpu); count += *ptr; *ptr = 0; } @@ -1593,7 +1588,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) /* * Re-init for the next cycle. */ - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ /* * Add actual count, subtract bias and initial refcount. @@ -1601,7 +1596,7 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) * The moment this atomic operation happens, futex_ref_is_dead() can * become true. */ - ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex_atomic); + ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex.phash.atomic); if (!ret) wake_up_var(mm); @@ -1611,8 +1606,8 @@ static void __futex_ref_atomic_end(struct futex_private_hash *fph) static void futex_ref_rcu(struct rcu_head *head) { - struct mm_struct *mm = container_of(head, struct mm_struct, futex_rcu); - struct futex_private_hash *fph = rcu_dereference_raw(mm->futex_phash); + struct mm_struct *mm = container_of(head, struct mm_struct, futex.phash.rcu); + struct futex_private_hash *fph = rcu_dereference_raw(mm->futex.phash.hash); if (fph->state == FR_PERCPU) { /* @@ -1641,7 +1636,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) /* * Can only transition the current fph; */ - WARN_ON_ONCE(rcu_dereference_raw(mm->futex_phash) != fph); + WARN_ON_ONCE(rcu_dereference_raw(mm->futex.phash.hash) != fph); /* * We enqueue at least one RCU callback. Ensure mm stays if the task * exits before the transition is completed. @@ -1652,9 +1647,9 @@ static void futex_ref_drop(struct futex_private_hash *fph) * In order to avoid the following scenario: * * futex_hash() __futex_pivot_hash() - * guard(rcu); guard(mm->futex_hash_lock); - * fph = mm->futex_phash; - * rcu_assign_pointer(&mm->futex_phash, new); + * guard(rcu); guard(mm->futex.phash.lock); + * fph = mm->futex.phash.hash; + * rcu_assign_pointer(&mm->futex.phash.hash, new); * futex_hash_allocate() * futex_ref_drop() * fph->state = FR_ATOMIC; @@ -1669,7 +1664,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) * There must be at least one full grace-period between publishing a * new fph and trying to replace it. */ - if (poll_state_synchronize_rcu(mm->futex_batches)) { + if (poll_state_synchronize_rcu(mm->futex.phash.batches)) { /* * There was a grace-period, we can begin now. */ @@ -1677,7 +1672,7 @@ static void futex_ref_drop(struct futex_private_hash *fph) return; } - call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); + call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu); } static bool futex_ref_get(struct futex_private_hash *fph) @@ -1687,11 +1682,11 @@ static bool futex_ref_get(struct futex_private_hash *fph) guard(preempt)(); if (READ_ONCE(fph->state) == FR_PERCPU) { - __this_cpu_inc(*mm->futex_ref); + __this_cpu_inc(*mm->futex.phash.ref); return true; } - return atomic_long_inc_not_zero(&mm->futex_atomic); + return atomic_long_inc_not_zero(&mm->futex.phash.atomic); } static bool futex_ref_put(struct futex_private_hash *fph) @@ -1701,11 +1696,11 @@ static bool futex_ref_put(struct futex_private_hash *fph) guard(preempt)(); if (READ_ONCE(fph->state) == FR_PERCPU) { - __this_cpu_dec(*mm->futex_ref); + __this_cpu_dec(*mm->futex.phash.ref); return false; } - return atomic_long_dec_and_test(&mm->futex_atomic); + return atomic_long_dec_and_test(&mm->futex.phash.atomic); } static bool futex_ref_is_dead(struct futex_private_hash *fph) @@ -1717,27 +1712,23 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) if (smp_load_acquire(&fph->state) == FR_PERCPU) return false; - return atomic_long_read(&mm->futex_atomic) == 0; + return atomic_long_read(&mm->futex.phash.atomic) == 0; } void futex_mm_init(struct mm_struct *mm) { - mutex_init(&mm->futex_hash_lock); - RCU_INIT_POINTER(mm->futex_phash, NULL); - mm->futex_phash_new = NULL; - /* futex-ref */ - mm->futex_ref = NULL; - atomic_long_set(&mm->futex_atomic, 0); - mm->futex_batches = get_state_synchronize_rcu(); + memset(&mm->futex, 0, sizeof(mm->futex)); + mutex_init(&mm->futex.phash.lock); + mm->futex.phash.batches = get_state_synchronize_rcu(); } void futex_hash_free(struct mm_struct *mm) { struct futex_private_hash *fph; - free_percpu(mm->futex_ref); - kvfree(mm->futex_phash_new); - fph = rcu_dereference_raw(mm->futex_phash); + free_percpu(mm->futex.phash.ref); + kvfree(mm->futex.phash.hash_new); + fph = rcu_dereference_raw(mm->futex.phash.hash); if (fph) kvfree(fph); } @@ -1748,10 +1739,10 @@ static bool futex_pivot_pending(struct mm_struct *mm) guard(rcu)(); - if (!mm->futex_phash_new) + if (!mm->futex.phash.hash_new) return true; - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); return futex_ref_is_dead(fph); } @@ -1793,7 +1784,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) * Once we've disabled the global hash there is no way back. */ scoped_guard(rcu) { - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference(mm->futex.phash.hash); if (fph && !fph->hash_mask) { if (custom) return -EBUSY; @@ -1801,15 +1792,15 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) } } - if (!mm->futex_ref) { + if (!mm->futex.phash.ref) { /* * This will always be allocated by the first thread and * therefore requires no locking. */ - mm->futex_ref = alloc_percpu(unsigned int); - if (!mm->futex_ref) + mm->futex.phash.ref = alloc_percpu(unsigned int); + if (!mm->futex.phash.ref) return -ENOMEM; - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ } fph = kvzalloc(struct_size(fph, queues, hash_slots), @@ -1832,14 +1823,14 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) wait_var_event(mm, futex_pivot_pending(mm)); } - scoped_guard(mutex, &mm->futex_hash_lock) { + scoped_guard(mutex, &mm->futex.phash.lock) { struct futex_private_hash *free __free(kvfree) = NULL; struct futex_private_hash *cur, *new; - cur = rcu_dereference_protected(mm->futex_phash, - lockdep_is_held(&mm->futex_hash_lock)); - new = mm->futex_phash_new; - mm->futex_phash_new = NULL; + cur = rcu_dereference_protected(mm->futex.phash.hash, + lockdep_is_held(&mm->futex.phash.lock)); + new = mm->futex.phash.hash_new; + mm->futex.phash.hash_new = NULL; if (fph) { if (cur && !cur->hash_mask) { @@ -1849,7 +1840,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) * the second one returns here. */ free = fph; - mm->futex_phash_new = new; + mm->futex.phash.hash_new = new; return -EBUSY; } if (cur && !new) { @@ -1879,7 +1870,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) if (new) { /* - * Will set mm->futex_phash_new on failure; + * Will set mm->futex.phash.new_hash on failure; * futex_private_hash_get() will try again. */ if (!__futex_pivot_hash(mm, new) && custom) @@ -1898,11 +1889,9 @@ int futex_hash_allocate_default(void) return 0; scoped_guard(rcu) { - threads = min_t(unsigned int, - get_nr_threads(current), - num_online_cpus()); + threads = min_t(unsigned int, get_nr_threads(current), num_online_cpus()); - fph = rcu_dereference(current->mm->futex_phash); + fph = rcu_dereference(current->mm->futex.phash.hash); if (fph) { if (fph->custom) return 0; @@ -1929,7 +1918,7 @@ static int futex_hash_get_slots(void) struct futex_private_hash *fph; guard(rcu)(); - fph = rcu_dereference(current->mm->futex_phash); + fph = rcu_dereference(current->mm->futex.phash.hash); if (fph && fph->hash_mask) return fph->hash_mask + 1; return 0; From 2cb5251d3d64d57c172185b9b608f704b3015f26 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:38 +0200 Subject: [PATCH 19/36] futex: Provide UABI defines for robust list entry modifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The marker for PI futexes in the robust list is a hardcoded 0x1 which lacks any sensible form of documentation. Provide proper defines for the bit and the mask and fix up the usage sites. Thereby convert the boolean pi argument into a modifier argument, which allows new modifier bits to be trivially added and conveyed. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Mathieu Desnoyers Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.458758556@kernel.org --- include/uapi/linux/futex.h | 4 +++ kernel/futex/core.c | 53 ++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index 7e2744ec8933..29bf2f65406a 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -177,6 +177,10 @@ struct robust_list_head { */ #define ROBUST_LIST_LIMIT 2048 +/* Modifiers for robust_list_head::list_op_pending */ +#define FUTEX_ROBUST_MOD_PI (0x1UL) +#define FUTEX_ROBUST_MOD_MASK (FUTEX_ROBUST_MOD_PI) + /* * bitset with all bits set for the FUTEX_xxx_BITSET OPs to request a * match of any bit. diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 79456b0ac1c6..61f4f559afa1 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1008,8 +1008,9 @@ void futex_unqueue_pi(struct futex_q *q) * dying task, and do notification if so: */ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, - bool pi, bool pending_op) + unsigned int mod, bool pending_op) { + bool pi = !!(mod & FUTEX_ROBUST_MOD_PI); u32 uval, nval, mval; pid_t owner; int err; @@ -1127,21 +1128,21 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, */ static inline int fetch_robust_entry(struct robust_list __user **entry, struct robust_list __user * __user *head, - unsigned int *pi) + unsigned int *mod) { unsigned long uentry; if (get_user(uentry, (unsigned long __user *)head)) return -EFAULT; - *entry = (void __user *)(uentry & ~1UL); - *pi = uentry & 1; + *entry = (void __user *)(uentry & ~FUTEX_ROBUST_MOD_MASK); + *mod = uentry & FUTEX_ROBUST_MOD_MASK; return 0; } /* - * Walk curr->robust_list (very carefully, it's a userspace list!) + * Walk curr->futex.robust_list (very carefully, it's a userspace list!) * and mark any locks found there dead, and notify any waiters. * * We silently return on any sign of list-walking problem. @@ -1149,9 +1150,8 @@ static inline int fetch_robust_entry(struct robust_list __user **entry, static void exit_robust_list(struct task_struct *curr) { struct robust_list_head __user *head = curr->futex.robust_list; + unsigned int limit = ROBUST_LIST_LIMIT, cur_mod, next_mod, pend_mod; struct robust_list __user *entry, *next_entry, *pending; - unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; - unsigned int next_pi; unsigned long futex_offset; int rc; @@ -1159,7 +1159,7 @@ static void exit_robust_list(struct task_struct *curr) * Fetch the list head (which was registered earlier, via * sys_set_robust_list()): */ - if (fetch_robust_entry(&entry, &head->list.next, &pi)) + if (fetch_robust_entry(&entry, &head->list.next, &cur_mod)) return; /* * Fetch the relative futex offset: @@ -1170,7 +1170,7 @@ static void exit_robust_list(struct task_struct *curr) * Fetch any possibly pending lock-add first, and handle it * if it exists: */ - if (fetch_robust_entry(&pending, &head->list_op_pending, &pip)) + if (fetch_robust_entry(&pending, &head->list_op_pending, &pend_mod)) return; next_entry = NULL; /* avoid warning with gcc */ @@ -1179,20 +1179,20 @@ static void exit_robust_list(struct task_struct *curr) * Fetch the next entry in the list before calling * handle_futex_death: */ - rc = fetch_robust_entry(&next_entry, &entry->next, &next_pi); + rc = fetch_robust_entry(&next_entry, &entry->next, &next_mod); /* * A pending lock might already be on the list, so * don't process it twice: */ if (entry != pending) { if (handle_futex_death((void __user *)entry + futex_offset, - curr, pi, HANDLE_DEATH_LIST)) + curr, cur_mod, HANDLE_DEATH_LIST)) return; } if (rc) return; entry = next_entry; - pi = next_pi; + cur_mod = next_mod; /* * Avoid excessively long or circular lists: */ @@ -1204,7 +1204,7 @@ static void exit_robust_list(struct task_struct *curr) if (pending) { handle_futex_death((void __user *)pending + futex_offset, - curr, pip, HANDLE_DEATH_PENDING); + curr, pend_mod, HANDLE_DEATH_PENDING); } } @@ -1223,29 +1223,28 @@ static void __user *futex_uaddr(struct robust_list __user *entry, */ static inline int compat_fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **entry, - compat_uptr_t __user *head, unsigned int *pi) + compat_uptr_t __user *head, unsigned int *pflags) { if (get_user(*uentry, head)) return -EFAULT; - *entry = compat_ptr((*uentry) & ~1); - *pi = (unsigned int)(*uentry) & 1; + *entry = compat_ptr((*uentry) & ~FUTEX_ROBUST_MOD_MASK); + *pflags = (unsigned int)(*uentry) & FUTEX_ROBUST_MOD_MASK; return 0; } /* - * Walk curr->robust_list (very carefully, it's a userspace list!) + * Walk curr->futex.robust_list (very carefully, it's a userspace list!) * and mark any locks found there dead, and notify any waiters. * * We silently return on any sign of list-walking problem. */ static void compat_exit_robust_list(struct task_struct *curr) { - struct compat_robust_list_head __user *head = curr->futex.compat_robust_list; + struct compat_robust_list_head __user *head = current->futex.compat_robust_list; + unsigned int limit = ROBUST_LIST_LIMIT, cur_mod, next_mod, pend_mod; struct robust_list __user *entry, *next_entry, *pending; - unsigned int limit = ROBUST_LIST_LIMIT, pi, pip; - unsigned int next_pi; compat_uptr_t uentry, next_uentry, upending; compat_long_t futex_offset; int rc; @@ -1254,7 +1253,7 @@ static void compat_exit_robust_list(struct task_struct *curr) * Fetch the list head (which was registered earlier, via * sys_set_robust_list()): */ - if (compat_fetch_robust_entry(&uentry, &entry, &head->list.next, &pi)) + if (compat_fetch_robust_entry(&uentry, &entry, &head->list.next, &cur_mod)) return; /* * Fetch the relative futex offset: @@ -1265,8 +1264,7 @@ static void compat_exit_robust_list(struct task_struct *curr) * Fetch any possibly pending lock-add first, and handle it * if it exists: */ - if (compat_fetch_robust_entry(&upending, &pending, - &head->list_op_pending, &pip)) + if (compat_fetch_robust_entry(&upending, &pending, &head->list_op_pending, &pend_mod)) return; next_entry = NULL; /* avoid warning with gcc */ @@ -1276,7 +1274,7 @@ static void compat_exit_robust_list(struct task_struct *curr) * handle_futex_death: */ rc = compat_fetch_robust_entry(&next_uentry, &next_entry, - (compat_uptr_t __user *)&entry->next, &next_pi); + (compat_uptr_t __user *)&entry->next, &next_mod); /* * A pending lock might already be on the list, so * dont process it twice: @@ -1284,15 +1282,14 @@ static void compat_exit_robust_list(struct task_struct *curr) if (entry != pending) { void __user *uaddr = futex_uaddr(entry, futex_offset); - if (handle_futex_death(uaddr, curr, pi, - HANDLE_DEATH_LIST)) + if (handle_futex_death(uaddr, curr, cur_mod, HANDLE_DEATH_LIST)) return; } if (rc) return; uentry = next_uentry; entry = next_entry; - pi = next_pi; + cur_mod = next_mod; /* * Avoid excessively long or circular lists: */ @@ -1304,7 +1301,7 @@ static void compat_exit_robust_list(struct task_struct *curr) if (pending) { void __user *uaddr = futex_uaddr(pending, futex_offset); - handle_futex_death(uaddr, curr, pip, HANDLE_DEATH_PENDING); + handle_futex_death(uaddr, curr, pend_mod, HANDLE_DEATH_PENDING); } } #endif From 6149fc36c09b91050b62e8e68a91027df8df7345 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:42 +0200 Subject: [PATCH 20/36] uaccess: Provide unsafe_atomic_store_release_user() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The upcoming support for unlocking robust futexes in the kernel requires store release semantics. Syscalls do not imply memory ordering on all architectures so the unlock operation requires a barrier. This barrier can be avoided when stores imply release like on x86. Provide a generic version with a smp_mb() before the unsafe_put_user(), which can be overridden by architectures. Provide also a ARCH_MEMORY_ORDER_TSO Kconfig option, which can be selected by architectures with Total Store Order (TSO), where store implies release, so that the smp_mb() in the generic implementation can be avoided. If that is set a barrier() is used instead of smp_mb(), which is not required for the use case at hand, but makes it future proof for other usage to prevent the compiler from reordering. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.513181528@kernel.org --- arch/Kconfig | 4 ++++ include/linux/uaccess.h | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/arch/Kconfig b/arch/Kconfig index e86880045158..83d362fca38c 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -403,6 +403,10 @@ config ARCH_32BIT_OFF_T config ARCH_32BIT_USTAT_F_TINODE bool +# Selected by architectures with Total Store Order (TSO) +config ARCH_MEMORY_ORDER_TSO + bool + config HAVE_ASM_MODVERSIONS bool help diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 56328601218c..c6bd20065a5d 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -649,6 +649,17 @@ static inline void user_access_restore(unsigned long flags) { } #define user_read_access_end user_access_end #endif +#ifndef unsafe_atomic_store_release_user +# define unsafe_atomic_store_release_user(val, uptr, elbl) \ + do { \ + if (!IS_ENABLED(CONFIG_ARCH_MEMORY_ORDER_TSO)) \ + smp_mb(); \ + else \ + barrier(); \ + unsafe_put_user(val, uptr, elbl); \ + } while (0) +#endif + /* Define RW variant so the below _mode macro expansion works */ #define masked_user_rw_access_begin(u) masked_user_access_begin(u) #define user_rw_access_begin(u, s) user_access_begin(u, s) From 7b125c44d0b7f617ee81dffd14ce116149d03cb6 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:47 +0200 Subject: [PATCH 21/36] x86: Select ARCH_MEMORY_ORDER_TSO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generic unsafe_atomic_store_release_user() implementation does: if (!IS_ENABLED(CONFIG_ARCH_MEMORY_ORDER_TSO)) smp_mb(); unsafe_put_user(); As x86 implements Total Store Order (TSO) which means stores imply release, select ARCH_MEMORY_ORDER_TSO to avoid the unnecessary smp_mb(). Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.564499644@kernel.org --- arch/x86/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index f3f7cb01d69d..1ce62a996192 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -113,6 +113,7 @@ config X86 select ARCH_HAS_ZONE_DMA_SET if EXPERT select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_HAVE_EXTRA_ELF_NOTES + select ARCH_MEMORY_ORDER_TSO select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI select ARCH_MIGHT_HAVE_PC_PARPORT From 1fd053d26f0333485cdbaa9d6e7b8cb53f54de95 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:51 +0200 Subject: [PATCH 22/36] futex: Cleanup UAPI defines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the operand defines tabular for readability sake. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.615600933@kernel.org --- include/uapi/linux/futex.h | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index 29bf2f65406a..75df1eaff75d 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -25,23 +25,22 @@ #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 -#define FUTEX_CMD_MASK ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME) -#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG) -#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG) -#define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG) -#define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG) -#define FUTEX_WAKE_OP_PRIVATE (FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG) -#define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG) -#define FUTEX_LOCK_PI2_PRIVATE (FUTEX_LOCK_PI2 | FUTEX_PRIVATE_FLAG) -#define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG) -#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG) +#define FUTEX_CMD_MASK ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME) + +#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG) +#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG) +#define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG) +#define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG) +#define FUTEX_WAKE_OP_PRIVATE (FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG) +#define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG) +#define FUTEX_LOCK_PI2_PRIVATE (FUTEX_LOCK_PI2 | FUTEX_PRIVATE_FLAG) +#define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG) +#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG) #define FUTEX_WAIT_BITSET_PRIVATE (FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG) #define FUTEX_WAKE_BITSET_PRIVATE (FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG) -#define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | \ - FUTEX_PRIVATE_FLAG) -#define FUTEX_CMP_REQUEUE_PI_PRIVATE (FUTEX_CMP_REQUEUE_PI | \ - FUTEX_PRIVATE_FLAG) +#define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | FUTEX_PRIVATE_FLAG) +#define FUTEX_CMP_REQUEUE_PI_PRIVATE (FUTEX_CMP_REQUEUE_PI | FUTEX_PRIVATE_FLAG) /* * Flags for futex2 syscalls. From 3ca9595d9fb6cce6633a5b03d98c2aecb5499838 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:55 +0200 Subject: [PATCH 23/36] futex: Add support for unlocking robust futexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unlocking robust non-PI futexes happens in user space with the following sequence: 1) robust_list_set_op_pending(mutex); 2) robust_list_remove(mutex); lval = 0; 3) lval = atomic_xchg(lock, lval); 4) if (lval & WAITERS) 5) sys_futex(WAKE,....); 6) robust_list_clear_op_pending(); That opens a window between #3 and #6 where the mutex could be acquired by some other task which observes that it is the last user and: A) unmaps the mutex memory B) maps a different file, which ends up covering the same address When the original task exits before reaching #6 then the kernel robust list handling observes the pending op entry and tries to fix up user space. In case that the newly mapped data contains the TID of the exiting thread at the address of the mutex/futex the kernel will set the owner died bit in that memory and therefore corrupting unrelated data. PI futexes have a similar problem both for the non-contented user space unlock and the in kernel unlock: 1) robust_list_set_op_pending(mutex); 2) robust_list_remove(mutex); lval = gettid(); 3) if (!atomic_try_cmpxchg(lock, lval, 0)) 4) sys_futex(UNLOCK_PI,....); 5) robust_list_clear_op_pending(); Address the first part of the problem where the futexes have waiters and need to enter the kernel anyway. Add a new FUTEX_ROBUST_UNLOCK flag, which is valid for the sys_futex() FUTEX_UNLOCK_PI, FUTEX_WAKE, FUTEX_WAKE_BITSET operations. This deliberately omits FUTEX_WAKE_OP from this treatment as it's unclear whether this is needed and there is no usage of it in glibc either to investigate. For the futex2 syscall family this needs to be implemented with a new syscall. The sys_futex() case [ab]uses the @uaddr2 argument to hand the pointer to robust_list_head::list_pending_op into the kernel. This argument is only evaluated when the FUTEX_ROBUST_UNLOCK bit is set and is therefore backward compatible. This is an explicit argument to avoid the lookup of the robust list pointer and retrieving the pending op pointer from there. User space has the pointer already available so it can just put it into the @uaddr2 argument. Aside of that this allows the usage of multiple robust lists in the future without any changes to the internal functions as they just operate on the provided pointer. This requires a second flag FUTEX_ROBUST_LIST32 which indicates that the robust list pointer points to an u32 and not to an u64. This is required for two reasons: 1) sys_futex() has no compat variant 2) The gaming emulators use both both 64-bit and compat 32-bit robust lists in the same 64-bit application As a consequence 32-bit applications have to set this flag unconditionally so they can run on a 64-bit kernel in compat mode unmodified. 32-bit kernels return an error code when the flag is not set. 64-bit kernels will happily clear the full 64 bits if user space fails to set it. In case of FUTEX_UNLOCK_PI this clears the robust list pending op when the unlock succeeded. In case of errors, the user space value is still locked by the caller and therefore the above cannot happen. In case of FUTEX_WAKE* this does the unlock of the futex in the kernel and clears the robust list pending op when the unlock was successful. If not, the user space value is still locked and user space has to deal with the returned error. That means that the unlocking of non-PI robust futexes has to use the same try_cmpxchg() unlock scheme as PI futexes. If the clearing of the pending list op fails (fault) then the kernel clears the registered robust list pointer if it matches to prevent that exit() will try to handle invalid data. That's a valid paranoid decision because the robust list head sits usually in the TLS and if the TLS is not longer accessible then the chance for fixing up the resulting mess is very close to zero. The problem of non-contended unlocks still exists and will be addressed separately. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.670514505@kernel.org --- include/uapi/linux/futex.h | 29 ++++++++++++++++++++- io_uring/futex.c | 2 +- kernel/futex/core.c | 53 ++++++++++++++++++++++++++++++++++++-- kernel/futex/futex.h | 15 +++++++++-- kernel/futex/pi.c | 15 +++++++++-- kernel/futex/syscalls.c | 13 +++++++--- kernel/futex/waitwake.c | 30 +++++++++++++++++++-- 7 files changed, 144 insertions(+), 13 deletions(-) diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index 75df1eaff75d..10a36c551675 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -25,8 +25,11 @@ #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 +#define FUTEX_ROBUST_UNLOCK 512 +#define FUTEX_ROBUST_LIST32 1024 -#define FUTEX_CMD_MASK ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME) +#define FUTEX_CMD_MASK ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME | \ + FUTEX_ROBUST_UNLOCK | FUTEX_ROBUST_LIST32) #define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG) #define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG) @@ -42,6 +45,30 @@ #define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | FUTEX_PRIVATE_FLAG) #define FUTEX_CMP_REQUEUE_PI_PRIVATE (FUTEX_CMP_REQUEUE_PI | FUTEX_PRIVATE_FLAG) +/* + * Operations to unlock a futex, clear the robust list pending op pointer and + * wake waiters. + */ +#define FUTEX_UNLOCK_PI_LIST64 (FUTEX_UNLOCK_PI | FUTEX_ROBUST_UNLOCK) +#define FUTEX_UNLOCK_PI_LIST64_PRIVATE (FUTEX_UNLOCK_PI_LIST64 | FUTEX_PRIVATE_FLAG) +#define FUTEX_UNLOCK_PI_LIST32 (FUTEX_UNLOCK_PI | FUTEX_ROBUST_UNLOCK | \ + FUTEX_ROBUST_LIST32) +#define FUTEX_UNLOCK_PI_LIST32_PRIVATE (FUTEX_UNLOCK_PI_LIST32 | FUTEX_PRIVATE_FLAG) + +#define FUTEX_UNLOCK_WAKE_LIST64 (FUTEX_WAKE | FUTEX_ROBUST_UNLOCK) +#define FUTEX_UNLOCK_WAKE_LIST64_PRIVATE (FUTEX_UNLOCK_WAKE_LIST64 | FUTEX_PRIVATE_FLAG) + +#define FUTEX_UNLOCK_WAKE_LIST32 (FUTEX_WAKE | FUTEX_ROBUST_UNLOCK | \ + FUTEX_ROBUST_LIST32) +#define FUTEX_UNLOCK_WAKE_LIST32_PRIVATE (FUTEX_UNLOCK_WAKE_LIST32 | FUTEX_PRIVATE_FLAG) + +#define FUTEX_UNLOCK_BITSET_LIST64 (FUTEX_WAKE_BITSET | FUTEX_ROBUST_UNLOCK) +#define FUTEX_UNLOCK_BITSET_LIST64_PRIVATE (FUTEX_UNLOCK_BITSET_LIST64 | FUTEX_PRIVATE_FLAG) + +#define FUTEX_UNLOCK_BITSET_LIST32 (FUTEX_WAKE_BITSET | FUTEX_ROBUST_UNLOCK | \ + FUTEX_ROBUST_LIST32) +#define FUTEX_UNLOCK_BITSET_LIST32_PRIVATE (FUTEX_UNLOCK_BITSET_LIST32 | FUTEX_PRIVATE_FLAG) + /* * Flags for futex2 syscalls. * diff --git a/io_uring/futex.c b/io_uring/futex.c index 9cc1788ef4c6..906701b3c5c6 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -327,7 +327,7 @@ int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags) * Strict flags - ensure that waking 0 futexes yields a 0 result. * See commit 43adf8449510 ("futex: FLAGS_STRICT") for details. */ - ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags, + ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags, NULL, iof->futex_val, iof->futex_mask); if (ret < 0) req_set_fail(req); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 61f4f559afa1..77ccb7787c34 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1062,7 +1062,7 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, owner = uval & FUTEX_TID_MASK; if (pending_op && !pi && !owner) { - futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, 1, + futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1, FUTEX_BITSET_MATCH_ANY); return 0; } @@ -1116,7 +1116,7 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, * PI futexes happens in exit_pi_state(): */ if (!pi && (uval & FUTEX_WAITERS)) { - futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, 1, + futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1, FUTEX_BITSET_MATCH_ANY); } @@ -1208,6 +1208,27 @@ static void exit_robust_list(struct task_struct *curr) } } +static bool robust_list_clear_pending(unsigned long __user *pop) +{ + struct robust_list_head __user *head = current->futex.robust_list; + + if (!put_user(0UL, pop)) + return true; + + /* + * Just give up. The robust list head is usually part of TLS, so the + * chance that this gets resolved is close to zero. + * + * If @pop_addr is the robust_list_head::list_op_pending pointer then + * clear the robust list head pointer to prevent further damage when the + * task exits. Better a few stale futexes than corrupted memory. But + * that's mostly an academic exercise. + */ + if (pop == (unsigned long __user *)&head->list_op_pending) + current->futex.robust_list = NULL; + return false; +} + #ifdef CONFIG_COMPAT static void __user *futex_uaddr(struct robust_list __user *entry, compat_long_t futex_offset) @@ -1304,6 +1325,21 @@ static void compat_exit_robust_list(struct task_struct *curr) handle_futex_death(uaddr, curr, pend_mod, HANDLE_DEATH_PENDING); } } + +static bool compat_robust_list_clear_pending(u32 __user *pop) +{ + struct compat_robust_list_head __user *head = current->futex.compat_robust_list; + + if (!put_user(0U, pop)) + return true; + + /* See comment in robust_list_clear_pending(). */ + if (pop == &head->list_op_pending) + current->futex.compat_robust_list = NULL; + return false; +} +#else +static bool compat_robust_list_clear_pending(u32 __user *pop_addr) { return false; } #endif #ifdef CONFIG_FUTEX_PI @@ -1397,6 +1433,19 @@ static void exit_pi_state_list(struct task_struct *curr) static inline void exit_pi_state_list(struct task_struct *curr) { } #endif +bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags) +{ + bool size32bit = !!(flags & FLAGS_ROBUST_LIST32); + + if (!IS_ENABLED(CONFIG_64BIT) && !size32bit) + return false; + + if (IS_ENABLED(CONFIG_64BIT) && size32bit) + return compat_robust_list_clear_pending(pop); + + return robust_list_clear_pending(pop); +} + static void futex_cleanup(struct task_struct *tsk) { if (unlikely(tsk->futex.robust_list)) { diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 9f6bf6f585fc..79ef2c709c81 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -40,6 +40,8 @@ #define FLAGS_NUMA 0x0080 #define FLAGS_STRICT 0x0100 #define FLAGS_MPOL 0x0200 +#define FLAGS_ROBUST_UNLOCK 0x0400 +#define FLAGS_ROBUST_LIST32 0x0800 /* FUTEX_ to FLAGS_ */ static inline unsigned int futex_to_flags(unsigned int op) @@ -52,6 +54,12 @@ static inline unsigned int futex_to_flags(unsigned int op) if (op & FUTEX_CLOCK_REALTIME) flags |= FLAGS_CLOCKRT; + if (op & FUTEX_ROBUST_UNLOCK) + flags |= FLAGS_ROBUST_UNLOCK; + + if (op & FUTEX_ROBUST_LIST32) + flags |= FLAGS_ROBUST_LIST32; + return flags; } @@ -449,13 +457,16 @@ extern int futex_unqueue_multiple(struct futex_vector *v, int count); extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count, struct hrtimer_sleeper *to); -extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); +extern int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop, + int nr_wake, u32 bitset); extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, int nr_wake, int nr_wake2, int op); -extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags); +extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags, void __user *pop); extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock); +bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags); + #endif /* _FUTEX_H */ diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index e037a97fe277..9dd5c0b1ac6a 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -1139,7 +1139,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl * This is the in-kernel slowpath: we look up the PI state (if any), * and do the rt-mutex unlock. */ -int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) +static int __futex_unlock_pi(u32 __user *uaddr, unsigned int flags) { u32 curval, uval, vpid = task_pid_vnr(current); union futex_key key = FUTEX_KEY_INIT; @@ -1148,7 +1148,6 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) if (!IS_ENABLED(CONFIG_FUTEX_PI)) return -ENOSYS; - retry: if (get_user(uval, uaddr)) return -EFAULT; @@ -1302,3 +1301,15 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) return ret; } +int futex_unlock_pi(u32 __user *uaddr, unsigned int flags, void __user *pop) +{ + int ret = __futex_unlock_pi(uaddr, flags); + + if (ret || !(flags & FLAGS_ROBUST_UNLOCK)) + return ret; + + if (!futex_robust_list_clear_pending(pop, flags)) + return -EFAULT; + + return 0; +} diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 8944ff4930d7..2fa19d9d008d 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -118,6 +118,13 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, return -ENOSYS; } + if (flags & FLAGS_ROBUST_UNLOCK) { + if (cmd != FUTEX_WAKE && + cmd != FUTEX_WAKE_BITSET && + cmd != FUTEX_UNLOCK_PI) + return -ENOSYS; + } + switch (cmd) { case FUTEX_WAIT: val3 = FUTEX_BITSET_MATCH_ANY; @@ -128,7 +135,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, val3 = FUTEX_BITSET_MATCH_ANY; fallthrough; case FUTEX_WAKE_BITSET: - return futex_wake(uaddr, flags, val, val3); + return futex_wake(uaddr, flags, uaddr2, val, val3); case FUTEX_REQUEUE: return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, NULL, 0); case FUTEX_CMP_REQUEUE: @@ -141,7 +148,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, case FUTEX_LOCK_PI2: return futex_lock_pi(uaddr, flags, timeout, 0); case FUTEX_UNLOCK_PI: - return futex_unlock_pi(uaddr, flags); + return futex_unlock_pi(uaddr, flags, uaddr2); case FUTEX_TRYLOCK_PI: return futex_lock_pi(uaddr, flags, NULL, 1); case FUTEX_WAIT_REQUEUE_PI: @@ -375,7 +382,7 @@ SYSCALL_DEFINE4(futex_wake, if (!futex_validate_input(flags, mask)) return -EINVAL; - return futex_wake(uaddr, FLAGS_STRICT | flags, nr, mask); + return futex_wake(uaddr, FLAGS_STRICT | flags, NULL, nr, mask); } /* diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index ceed9d879059..8f5e5d302356 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -149,13 +149,36 @@ void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q) wake_q_add_safe(wake_q, p); } +/* + * If requested, clear the robust list pending op and unlock the futex + */ +static bool futex_robust_unlock(u32 __user *uaddr, unsigned int flags, void __user *pop) +{ + if (!(flags & FLAGS_ROBUST_UNLOCK)) + return true; + + /* First unlock the futex, which requires release semantics. */ + scoped_user_write_access(uaddr, efault) + unsafe_atomic_store_release_user(0, uaddr, efault); + + /* + * Clear the pending list op now. If that fails, then the task is in + * deeper trouble as the robust list head is usually part of the TLS. + * The chance of survival is close to zero. + */ + return futex_robust_list_clear_pending(pop, flags); + +efault: + return false; +} + /* * Wake up waiters matching bitset queued on this futex (uaddr). */ -int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) +int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop, int nr_wake, u32 bitset) { - struct futex_q *this, *next; union futex_key key = FUTEX_KEY_INIT; + struct futex_q *this, *next; DEFINE_WAKE_Q(wake_q); int ret; @@ -166,6 +189,9 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) if (unlikely(ret != 0)) return ret; + if (!futex_robust_unlock(uaddr, flags, pop)) + return -EFAULT; + if ((flags & FLAGS_STRICT) && !nr_wake) return 0; From 042df0c1d48609a85580dcbaff498c95ced20a5f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:09:59 +0200 Subject: [PATCH 24/36] futex: Add robust futex unlock IP range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There will be a VDSO function to unlock robust futexes in user space. The unlock sequence is racy vs. clearing the list_pending_op pointer in the tasks robust list head. To plug this race the kernel needs to know the instruction window. As the VDSO is per MM the addresses are stored in mm_struct::futex. Architectures which implement support for this have to update these addresses when the VDSO is (re)mapped and indicate the pending op pointer size which is matching the IP. Arguably this could be resolved by chasing mm->context->vdso->image, but that's architecture specific and requires to touch quite some cache lines. Having it in mm::futex reduces the cache line impact and avoids having yet another set of architecture specific functionality. To support multi size robust list applications (gaming) this provides two ranges when COMPAT is enabled. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.718926819@kernel.org --- include/linux/futex.h | 21 +++++++++++++--- include/linux/futex_types.h | 28 ++++++++++++++++++++++ init/Kconfig | 6 +++++ kernel/futex/core.c | 48 +++++++++++++++++++++++++++++-------- 4 files changed, 90 insertions(+), 13 deletions(-) diff --git a/include/linux/futex.h b/include/linux/futex.h index 9e6218c2be66..cb2a182547dd 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -81,11 +81,9 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) #ifdef CONFIG_FUTEX_PRIVATE_HASH int futex_hash_allocate_default(void); void futex_hash_free(struct mm_struct *mm); -void futex_mm_init(struct mm_struct *mm); #else /* CONFIG_FUTEX_PRIVATE_HASH */ static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline void futex_mm_init(struct mm_struct *mm) { } #endif /* !CONFIG_FUTEX_PRIVATE_HASH */ #else /* CONFIG_FUTEX */ @@ -104,7 +102,24 @@ static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsig } static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline void futex_mm_init(struct mm_struct *mm) { } #endif /* !CONFIG_FUTEX */ +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +void futex_reset_cs_ranges(struct futex_mm_data *fd); + +static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx, + unsigned long start, unsigned long end, bool sz32) +{ + fd->unlock.cs_ranges[idx].start_ip = start; + fd->unlock.cs_ranges[idx].len = end - start; + fd->unlock.cs_ranges[idx].pop_size32 = sz32; +} +#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ + +#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) +void futex_mm_init(struct mm_struct *mm); +#else +static inline void futex_mm_init(struct mm_struct *mm) { } +#endif + #endif /* _LINUX_FUTEX_H */ diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h index d41557d2790d..d320c0571f0c 100644 --- a/include/linux/futex_types.h +++ b/include/linux/futex_types.h @@ -55,12 +55,40 @@ struct futex_mm_phash { struct futex_mm_phash { }; #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +/** + * struct futex_unlock_cs_range - Range for the VDSO unlock critical section + * @start_ip: The start IP of the robust futex unlock critical section (inclusive) + * @len: The length of the robust futex unlock critical section + * @pop_size32: Pending OP pointer size indicator. 0 == 64-bit, 1 == 32-bit + */ +struct futex_unlock_cs_range { + unsigned long start_ip; + unsigned int len; + unsigned int pop_size32; +}; + +#define FUTEX_ROBUST_MAX_CS_RANGES (1 + IS_ENABLED(CONFIG_COMPAT)) + +/** + * struct futex_unlock_cs_ranges - Futex unlock VSDO critical sections + * @cs_ranges: Array of critical section ranges + */ +struct futex_unlock_cs_ranges { + struct futex_unlock_cs_range cs_ranges[FUTEX_ROBUST_MAX_CS_RANGES]; +}; +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +struct futex_unlock_cs_ranges { }; +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + /** * struct futex_mm_data - Futex related per MM data * @phash: Futex private hash related data + * @unlock: Futex unlock VDSO critical sections */ struct futex_mm_data { struct futex_mm_phash phash; + struct futex_unlock_cs_ranges unlock; }; #else /* CONFIG_FUTEX */ struct futex_sched_data { }; diff --git a/init/Kconfig b/init/Kconfig index 2937c4d308ae..165b08e336a8 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1842,6 +1842,12 @@ config FUTEX_MPOL depends on FUTEX && NUMA default y +config HAVE_FUTEX_ROBUST_UNLOCK + bool + +config FUTEX_ROBUST_UNLOCK + def_bool FUTEX && HAVE_GENERIC_VDSO && GENERIC_IRQ_ENTRY && RSEQ && HAVE_FUTEX_ROBUST_UNLOCK + config EPOLL bool "Enable eventpoll support" if EXPERT default y diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 77ccb7787c34..aad6e501fabd 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1761,11 +1761,11 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph) return atomic_long_read(&mm->futex.phash.atomic) == 0; } -void futex_mm_init(struct mm_struct *mm) +static void futex_hash_init_mm(struct futex_mm_data *fd) { - memset(&mm->futex, 0, sizeof(mm->futex)); - mutex_init(&mm->futex.phash.lock); - mm->futex.phash.batches = get_state_synchronize_rcu(); + memset(&fd->phash, 0, sizeof(fd->phash)); + mutex_init(&fd->phash.lock); + fd->phash.batches = get_state_synchronize_rcu(); } void futex_hash_free(struct mm_struct *mm) @@ -1969,19 +1969,47 @@ static int futex_hash_get_slots(void) return fph->hash_mask + 1; return 0; } +#else /* CONFIG_FUTEX_PRIVATE_HASH */ +static inline int futex_hash_allocate(unsigned int hslots, unsigned int flags) { return -EINVAL; } +static inline int futex_hash_get_slots(void) { return 0; } +static inline void futex_hash_init_mm(struct futex_mm_data *fd) { } +#endif /* !CONFIG_FUTEX_PRIVATE_HASH */ -#else - -static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +static void futex_invalidate_cs_ranges(struct futex_mm_data *fd) { - return -EINVAL; + /* + * Invalidate start_ip so that the quick check fails for ip >= start_ip + * if VDSO is not mapped or the second slot is not available for compat + * tasks as they use VDSO32 which does not provide the 64-bit pointer + * variant. + */ + for (int i = 0; i < FUTEX_ROBUST_MAX_CS_RANGES; i++) + fd->unlock.cs_ranges[i].start_ip = ~0UL; } -static int futex_hash_get_slots(void) +void futex_reset_cs_ranges(struct futex_mm_data *fd) { - return 0; + memset(fd->unlock.cs_ranges, 0, sizeof(fd->unlock.cs_ranges)); + futex_invalidate_cs_ranges(fd); } +static void futex_robust_unlock_init_mm(struct futex_mm_data *fd) +{ + /* mm_dup() preserves the range, mm_alloc() clears it */ + if (!fd->unlock.cs_ranges[0].start_ip) + futex_invalidate_cs_ranges(fd); +} +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +static inline void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { } +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + +#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) +void futex_mm_init(struct mm_struct *mm) +{ + futex_hash_init_mm(&mm->futex); + futex_robust_unlock_init_mm(&mm->futex); +} #endif int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) From 7010c39d8fc5063af69ee63f905e592e046f8e5d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:10:04 +0200 Subject: [PATCH 25/36] futex: Provide infrastructure to plug the non contended robust futex unlock race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the FUTEX_ROBUST_UNLOCK mechanism is used for unlocking (PI-)futexes, then the unlock sequence in user space looks like this: 1) robust_list_set_op_pending(mutex); 2) robust_list_remove(mutex); lval = gettid(); 3) if (atomic_try_cmpxchg(&mutex->lock, lval, 0)) 4) robust_list_clear_op_pending(); else 5) sys_futex(OP | FUTEX_ROBUST_UNLOCK, ....); That still leaves a minimal race window between #3 and #4 where the mutex could be acquired by some other task, which observes that it is the last user and: 1) unmaps the mutex memory 2) maps a different file, which ends up covering the same address When then the original task exits before reaching #5 then the kernel robust list handling observes the pending op entry and tries to fix up user space. In case that the newly mapped data contains the TID of the exiting thread at the address of the mutex/futex the kernel will set the owner died bit in that memory and therefore corrupt unrelated data. On X86 this boils down to this simplified assembly sequence: mov %esi,%eax // Load TID into EAX xor %ecx,%ecx // Set ECX to 0 #3 lock cmpxchg %ecx,(%rdi) // Try the TID -> 0 transition .Lstart: jnz .Lend #4 movq %rcx,(%rdx) // Clear list_op_pending .Lend: If the cmpxchg() succeeds and the task is interrupted before it can clear list_op_pending in the robust list head (#4) and the task crashes in a signal handler or gets killed then it ends up in do_exit() and subsequently in the robust list handling, which then might run into the unmap/map issue described above. This is only relevant when user space was interrupted and a signal is pending. The fix-up has to be done before signal delivery is attempted because: 1) The signal might be fatal so get_signal() ends up in do_exit() 2) The signal handler might crash or the task is killed before returning from the handler. At that point the instruction pointer in pt_regs is not longer the instruction pointer of the initially interrupted unlock sequence. The right place to handle this is in __exit_to_user_mode_loop() before invoking arch_do_signal_or_restart() as this covers obviously both scenarios. As this is only relevant when the task was interrupted in user space, this is tied to RSEQ and the generic entry code as RSEQ keeps track of user space interrupts unconditionally even if the task does not have a RSEQ region installed. That makes the decision very lightweight: if (current->rseq.user_irq && within(regs, csr->unlock_ip_range)) futex_fixup_robust_unlock(regs, csr); futex_fixup_robust_unlock() then invokes a architecture specific function to return the pending op pointer or NULL. The function evaluates the register content to decide whether the pending ops pointer in the robust list head needs to be cleared. Assuming the above unlock sequence, then on x86 this decision is the trivial evaluation of the zero flag: return regs->eflags & X86_EFLAGS_ZF ? regs->dx : NULL; Other architectures might need to do more complex evaluations due to LLSC, but the approach is valid in general. The size of the pointer is determined from the matching range struct, which covers both 32-bit and 64-bit builds including COMPAT. The unlock sequence is going to be placed in the VDSO so that the kernel can keep everything synchronized, especially the register usage. The resulting code sequence for user space is: if (__vdso_futex_robust_list$SZ_try_unlock(lock, tid, &pending_op) != tid) err = sys_futex($OP | FUTEX_ROBUST_UNLOCK,....); Both the VDSO unlock and the kernel side unlock ensure that the pending_op pointer is always cleared when the lock becomes unlocked. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.773669210@kernel.org --- include/linux/futex.h | 39 +++++++++++++++++++++++++++++++- include/vdso/futex.h | 52 +++++++++++++++++++++++++++++++++++++++++++ kernel/entry/common.c | 9 +++++--- kernel/futex/core.c | 18 +++++++++++++++ 4 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 include/vdso/futex.h diff --git a/include/linux/futex.h b/include/linux/futex.h index cb2a182547dd..51f4ccdc9092 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -105,7 +105,41 @@ static inline int futex_hash_free(struct mm_struct *mm) { return 0; } #endif /* !CONFIG_FUTEX */ #ifdef CONFIG_FUTEX_ROBUST_UNLOCK +#include + void futex_reset_cs_ranges(struct futex_mm_data *fd); +void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr); + +static inline bool futex_within_robust_unlock(struct pt_regs *regs, + struct futex_unlock_cs_range *csr) +{ + unsigned long ip = instruction_pointer(regs); + + return ip >= csr->start_ip && ip < csr->start_ip + csr->len; +} + +static inline void futex_fixup_robust_unlock(struct pt_regs *regs) +{ + struct futex_unlock_cs_range *csr; + + /* + * Avoid dereferencing current->mm if not returning from interrupt. + * current->rseq.event is going to be used subsequently, so bringing the + * cache line in is not a big deal. + */ + if (!current->rseq.event.user_irq) + return; + + csr = current->mm->futex.unlock.cs_ranges; + + /* The loop is optimized out for !COMPAT */ + for (int r = 0; r < FUTEX_ROBUST_MAX_CS_RANGES; r++, csr++) { + if (unlikely(futex_within_robust_unlock(regs, csr))) { + __futex_fixup_robust_unlock(regs, csr); + return; + } + } +} static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx, unsigned long start, unsigned long end, bool sz32) @@ -114,7 +148,10 @@ static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned in fd->unlock.cs_ranges[idx].len = end - start; fd->unlock.cs_ranges[idx].pop_size32 = sz32; } -#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { } +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + #if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) void futex_mm_init(struct mm_struct *mm); diff --git a/include/vdso/futex.h b/include/vdso/futex.h new file mode 100644 index 000000000000..3cd175eefe64 --- /dev/null +++ b/include/vdso/futex.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _VDSO_FUTEX_H +#define _VDSO_FUTEX_H + +#include + +/** + * __vdso_futex_robust_list64_try_unlock - Try to unlock an uncontended robust futex + * with a 64-bit pending op pointer + * @lock: Pointer to the futex lock object + * @tid: The TID of the calling task + * @pop: Pointer to the task's robust_list_head::list_pending_op + * + * Return: The content of *@lock. On success this is the same as @tid. + * + * The function implements: + * if (atomic_try_cmpxchg(lock, &tid, 0)) + * *op = NULL; + * return tid; + * + * There is a race between a successful unlock and clearing the pending op + * pointer in the robust list head. If the calling task is interrupted in the + * race window and has to handle a (fatal) signal on return to user space then + * the kernel handles the clearing of @pending_op before attempting to deliver + * the signal. That ensures that a task cannot exit with a potentially invalid + * pending op pointer. + * + * User space uses it in the following way: + * + * if (__vdso_futex_robust_list64_try_unlock(lock, tid, &pending_op) != tid) + * err = sys_futex($OP | FUTEX_ROBUST_UNLOCK,....); + * + * If the unlock attempt fails due to the FUTEX_WAITERS bit set in the lock, + * then the syscall does the unlock, clears the pending op pointer and wakes the + * requested number of waiters. + */ +__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop); + +/** + * __vdso_futex_robust_list32_try_unlock - Try to unlock an uncontended robust futex + * with a 32-bit pending op pointer + * @lock: Pointer to the futex lock object + * @tid: The TID of the calling task + * @pop: Pointer to the task's robust_list_head::list_pending_op + * + * Return: The content of *@lock. On success this is the same as @tid. + * + * Same as __vdso_futex_robust_list64_try_unlock() just with a 32-bit @pop pointer. + */ +__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop); + +#endif diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 19d2244a9fef..e3d381fd3d25 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include +#include #include +#include #include #include #include +#include #include /* Workaround to allow gradual conversion of architecture code */ @@ -60,8 +61,10 @@ static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *re if (ti_work & _TIF_PATCH_PENDING) klp_update_patch_state(current); - if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) { + futex_fixup_robust_unlock(regs); arch_do_signal_or_restart(regs); + } if (ti_work & _TIF_NOTIFY_RESUME) resume_user_mode_work(regs); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index aad6e501fabd..6ea4a97796a1 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -46,6 +46,8 @@ #include #include +#include + #include "futex.h" #include "../locking/rtmutex_common.h" @@ -1446,6 +1448,22 @@ bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags) return robust_list_clear_pending(pop); } +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr) +{ + /* + * arch_futex_robust_unlock_get_pop() returns the list pending op pointer from + * @regs if the try_cmpxchg() succeeded. + */ + void __user *pop = arch_futex_robust_unlock_get_pop(regs); + + if (!pop) + return; + + futex_robust_list_clear_pending(pop, csr->pop_size32 ? FLAGS_ROBUST_LIST32 : 0); +} +#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */ + static void futex_cleanup(struct task_struct *tsk) { if (unlikely(tsk->futex.robust_list)) { From 61cfc8e372d1971e0a96d3f1f8b5ee29916b3385 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:10:08 +0200 Subject: [PATCH 26/36] x86/vdso: Prepare for robust futex unlock support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There will be a VDSO function to unlock non-contended robust futexes in user space. The unlock sequence is racy vs. clearing the list_pending_op pointer in the task's robust list head. To plug this race the kernel needs to know the critical section window so it can clear the pointer when the task is interrupted within that race window. The window is determined by labels in the inline assembly. Add these symbols to the vdso2c generator and use them in the VDSO VMA code to update the critical section addresses in mm_struct::futex on (re)map(). The symbols are not exported to user space, but available in the debug version of the vDSO. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Link: https://patch.msgid.link/20260602090535.828312645@kernel.org --- arch/x86/entry/vdso/vma.c | 29 +++++++++++++++++++++++++++++ arch/x86/include/asm/vdso.h | 4 ++++ arch/x86/tools/vdso2c.c | 16 ++++++++++------ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index a6bfcc8243cd..9a953e7c76db 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -6,6 +6,7 @@ */ #include #include +#include #include #include #include @@ -73,6 +74,31 @@ static void vdso_fix_landing(const struct vdso_image *image, regs->ip = new_vma->vm_start + ipoffset; } +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +static void vdso_futex_robust_unlock_update_ips(void) +{ + const struct vdso_image *image = current->mm->context.vdso_image; + unsigned long vdso = (unsigned long) current->mm->context.vdso; + struct futex_mm_data *fd = ¤t->mm->futex; + unsigned int idx = 0; + + futex_reset_cs_ranges(fd); + +#ifdef CONFIG_X86_64 + futex_set_vdso_cs_range(fd, idx, vdso + image->sym___futex_list64_try_unlock_cs_start, + vdso + image->sym___futex_list64_try_unlock_cs_end, false); + idx++; +#endif /* CONFIG_X86_64 */ + +#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT) + futex_set_vdso_cs_range(fd, idx, vdso + image->sym___futex_list32_try_unlock_cs_start, + vdso + image->sym___futex_list32_try_unlock_cs_end, true); +#endif /* CONFIG_X86_32 || CONFIG_COMPAT */ +} +#else +static inline void vdso_futex_robust_unlock_update_ips(void) { } +#endif + static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma) { @@ -80,6 +106,7 @@ static int vdso_mremap(const struct vm_special_mapping *sm, vdso_fix_landing(image, new_vma); current->mm->context.vdso = (void __user *)new_vma->vm_start; + vdso_futex_robust_unlock_update_ips(); return 0; } @@ -185,6 +212,8 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr) current->mm->context.vdso = (void __user *)text_start; current->mm->context.vdso_image = image; + vdso_futex_robust_unlock_update_ips(); + up_fail: mmap_write_unlock(mm); return ret; diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h index f2d49212ae90..4e735158c1b2 100644 --- a/arch/x86/include/asm/vdso.h +++ b/arch/x86/include/asm/vdso.h @@ -24,6 +24,10 @@ struct vdso_image { long sym_int80_landing_pad; long sym_vdso32_sigreturn_landing_pad; long sym_vdso32_rt_sigreturn_landing_pad; + long sym___futex_list64_try_unlock_cs_start; + long sym___futex_list64_try_unlock_cs_end; + long sym___futex_list32_try_unlock_cs_start; + long sym___futex_list32_try_unlock_cs_end; }; extern const struct vdso_image vdso64_image; diff --git a/arch/x86/tools/vdso2c.c b/arch/x86/tools/vdso2c.c index b8a555763f43..64a636b9fc37 100644 --- a/arch/x86/tools/vdso2c.c +++ b/arch/x86/tools/vdso2c.c @@ -75,12 +75,16 @@ struct vdso_sym { }; struct vdso_sym required_syms[] = { - {"__kernel_vsyscall", true}, - {"__kernel_sigreturn", true}, - {"__kernel_rt_sigreturn", true}, - {"int80_landing_pad", true}, - {"vdso32_rt_sigreturn_landing_pad", true}, - {"vdso32_sigreturn_landing_pad", true}, + {"__kernel_vsyscall", true}, + {"__kernel_sigreturn", true}, + {"__kernel_rt_sigreturn", true}, + {"int80_landing_pad", true}, + {"vdso32_rt_sigreturn_landing_pad", true}, + {"vdso32_sigreturn_landing_pad", true}, + {"__futex_list64_try_unlock_cs_start", true}, + {"__futex_list64_try_unlock_cs_end", true}, + {"__futex_list32_try_unlock_cs_start", true}, + {"__futex_list32_try_unlock_cs_end", true}, }; __attribute__((format(printf, 1, 2))) __attribute__((noreturn)) From a2274cc0091ed4fdce10fad68d08c529b8d3e7dd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 2 Jun 2026 11:10:12 +0200 Subject: [PATCH 27/36] x86/vdso: Implement __vdso_futex_robust_try_unlock() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the FUTEX_ROBUST_UNLOCK mechanism is used for unlocking (PI-)futexes, then the unlock sequence in userspace looks like this: 1) robust_list_set_op_pending(mutex); 2) robust_list_remove(mutex); lval = gettid(); 3) if (atomic_try_cmpxchg(&mutex->lock, lval, 0)) 4) robust_list_clear_op_pending(); else 5) sys_futex(OP,...FUTEX_ROBUST_UNLOCK); That still leaves a minimal race window between #3 and #4 where the mutex could be acquired by some other task which observes that it is the last user and: 1) unmaps the mutex memory 2) maps a different file, which ends up covering the same address When then the original task exits before reaching #5 then the kernel robust list handling observes the pending op entry and tries to fix up user space. In case that the newly mapped data contains the TID of the exiting thread at the address of the mutex/futex the kernel will set the owner died bit in that memory and therefore corrupt unrelated data. Provide a VDSO function which exposes the critical section window in the VDSO symbol table. The resulting addresses are updated in the task's mm when the VDSO is (re)map()'ed. The core code detects when a task was interrupted within the critical section and is about to deliver a signal. It then invokes an architecture specific function which determines whether the pending op pointer has to be cleared or not. The unlock assembly sequence on 64-bit is: mov %esi,%eax // Load TID into EAX xor %ecx,%ecx // Set ECX to 0 lock cmpxchg %ecx,(%rdi) // Try the TID -> 0 transition .Lstart: jnz .Lend movq %rcx,(%rdx) // Clear list_op_pending .Lend: ret So the decision can be simply based on the ZF state in regs->flags. The pending op pointer is always in DX independent of the build mode (32/64-bit) to make the pending op pointer retrieval uniform. The size of the pointer is stored in the matching criticial section range struct and the core code retrieves it from there. So the pointer retrieval function does not have to care. It is bit-size independent: return regs->flags & X86_EFLAGS_ZF ? regs->dx : NULL; There are two entry points to handle the different robust list pending op pointer size: __vdso_futex_robust_list64_try_unlock() __vdso_futex_robust_list32_try_unlock() The 32-bit VDSO provides only __vdso_futex_robust_list32_try_unlock(). The 64-bit VDSO provides always __vdso_futex_robust_list64_try_unlock() and when COMPAT is enabled also the list32 variant, which is required to support multi-size robust list pointers used by gaming emulators. The unlock function is inspired by an idea from Mathieu Desnoyers. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: André Almeida Acked-by: Uros Bizjak Link: https://lore.kernel.org/20260311185409.1988269-1-mathieu.desnoyers@efficios.com Link: https://patch.msgid.link/20260602090535.883796247@kernel.org --- arch/x86/Kconfig | 1 + arch/x86/entry/vdso/common/vfutex.c | 71 ++++++++++++++++++++++++ arch/x86/entry/vdso/vdso32/Makefile | 5 +- arch/x86/entry/vdso/vdso32/vdso32.lds.S | 3 + arch/x86/entry/vdso/vdso32/vfutex.c | 1 + arch/x86/entry/vdso/vdso64/Makefile | 7 ++- arch/x86/entry/vdso/vdso64/vdso64.lds.S | 7 +++ arch/x86/entry/vdso/vdso64/vdsox32.lds.S | 7 +++ arch/x86/entry/vdso/vdso64/vfutex.c | 1 + arch/x86/include/asm/futex_robust.h | 19 +++++++ 10 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 arch/x86/entry/vdso/common/vfutex.c create mode 100644 arch/x86/entry/vdso/vdso32/vfutex.c create mode 100644 arch/x86/entry/vdso/vdso64/vfutex.c create mode 100644 arch/x86/include/asm/futex_robust.h diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 1ce62a996192..fdaef60b46d6 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -239,6 +239,7 @@ config X86 select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA if X86_32 select HAVE_EXIT_THREAD + select HAVE_FUTEX_ROBUST_UNLOCK select HAVE_GENERIC_TIF_BITS select HAVE_GUP_FAST select HAVE_FENTRY if X86_64 || DYNAMIC_FTRACE diff --git a/arch/x86/entry/vdso/common/vfutex.c b/arch/x86/entry/vdso/common/vfutex.c new file mode 100644 index 000000000000..454f059278e4 --- /dev/null +++ b/arch/x86/entry/vdso/common/vfutex.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +/* + * Assembly template for the try unlock functions. The basic functionality is: + * + * mov esi, %eax Move the TID into EAX + * xor %ecx, %ecx Clear ECX + * lock_cmpxchgl %ecx, (%rdi) Attempt the TID -> 0 transition + * .Lcs_start: Start of the critical section + * jnz .Lcs_end If cmpxchl failed jump to the end + * .Lcs_success: Start of the success section + * movq %rcx, (%rdx) Set the pending op pointer to 0 + * .Lcs_end: End of the critical section + * + * .Lcs_start and .Lcs_end establish the critical section range. .Lcs_success is + * technically not required, but there for illustration, debugging and testing. + * + * When CONFIG_COMPAT is enabled then the 64-bit VDSO provides two functions. + * One for the regular 64-bit sized pending operation pointer and one for a + * 32-bit sized pointer to support gaming emulators. + * + * The 32-bit VDSO provides only the one for 32-bit sized pointers. + */ +#define __stringify_1(x...) #x +#define __stringify(x...) __stringify_1(x) + +#define LABEL(prefix, which) __stringify(prefix##_try_unlock_cs_##which:) + +#define JNZ_END(prefix) "jnz " __stringify(prefix) "_try_unlock_cs_end\n" + +#define CLEAR_POPQ "movq %[zero], %a[pop]\n" +#define CLEAR_POPL "movl %k[zero], %a[pop]\n" + +#define futex_robust_try_unlock(prefix, clear_pop, __lock, __tid, __pop)\ +({ \ + asm volatile ( \ + " \n" \ + " lock cmpxchgl %k[zero], %a[lock] \n" \ + " \n" \ + LABEL(prefix, start) \ + " \n" \ + JNZ_END(prefix) \ + " \n" \ + LABEL(prefix, success) \ + " \n" \ + clear_pop \ + " \n" \ + LABEL(prefix, end) \ + : [tid] "+&a" (__tid) \ + : [lock] "D" (__lock), \ + [pop] "d" (__pop), \ + [zero] "r" (0UL) \ + : "memory" \ + ); \ + __tid; \ +}) + +#ifdef __x86_64__ +__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop) +{ + return futex_robust_try_unlock(__futex_list64, CLEAR_POPQ, lock, tid, pop); +} +#endif /* __x86_64__ */ + +#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT) +__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop) +{ + return futex_robust_try_unlock(__futex_list32, CLEAR_POPL, lock, tid, pop); +} +#endif /* CONFIG_X86_32 || CONFIG_COMPAT */ diff --git a/arch/x86/entry/vdso/vdso32/Makefile b/arch/x86/entry/vdso/vdso32/Makefile index ded4fc6a48cd..ab4b1f635f66 100644 --- a/arch/x86/entry/vdso/vdso32/Makefile +++ b/arch/x86/entry/vdso/vdso32/Makefile @@ -7,8 +7,9 @@ vdsos-y := 32 # Files to link into the vDSO: -vobjs-y := note.o vclock_gettime.o vgetcpu.o -vobjs-y += system_call.o sigreturn.o +vobjs-y := note.o vclock_gettime.o vgetcpu.o +vobjs-y += system_call.o sigreturn.o +vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o # Compilation flags flags-y := -DBUILD_VDSO32 -m32 -mregparm=0 diff --git a/arch/x86/entry/vdso/vdso32/vdso32.lds.S b/arch/x86/entry/vdso/vdso32/vdso32.lds.S index 55554f80d930..cee8f7f9fe80 100644 --- a/arch/x86/entry/vdso/vdso32/vdso32.lds.S +++ b/arch/x86/entry/vdso/vdso32/vdso32.lds.S @@ -30,6 +30,9 @@ VERSION __vdso_clock_gettime64; __vdso_clock_getres_time64; __vdso_getcpu; +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK + __vdso_futex_robust_list32_try_unlock; +#endif }; LINUX_2.5 { diff --git a/arch/x86/entry/vdso/vdso32/vfutex.c b/arch/x86/entry/vdso/vdso32/vfutex.c new file mode 100644 index 000000000000..940a6ee30026 --- /dev/null +++ b/arch/x86/entry/vdso/vdso32/vfutex.c @@ -0,0 +1 @@ +#include "common/vfutex.c" diff --git a/arch/x86/entry/vdso/vdso64/Makefile b/arch/x86/entry/vdso/vdso64/Makefile index bfffaf1aeecc..7c0790065b5e 100644 --- a/arch/x86/entry/vdso/vdso64/Makefile +++ b/arch/x86/entry/vdso/vdso64/Makefile @@ -8,9 +8,10 @@ vdsos-y := 64 vdsos-$(CONFIG_X86_X32_ABI) += x32 # Files to link into the vDSO: -vobjs-y := note.o vclock_gettime.o vgetcpu.o -vobjs-y += vgetrandom.o vgetrandom-chacha.o -vobjs-$(CONFIG_X86_SGX) += vsgx.o +vobjs-y := note.o vclock_gettime.o vgetcpu.o +vobjs-y += vgetrandom.o vgetrandom-chacha.o +vobjs-$(CONFIG_X86_SGX) += vsgx.o +vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o # Compilation flags flags-y := -DBUILD_VDSO64 -m64 -mcmodel=small diff --git a/arch/x86/entry/vdso/vdso64/vdso64.lds.S b/arch/x86/entry/vdso/vdso64/vdso64.lds.S index 5ce3f2b6373a..4a72122da81b 100644 --- a/arch/x86/entry/vdso/vdso64/vdso64.lds.S +++ b/arch/x86/entry/vdso/vdso64/vdso64.lds.S @@ -32,6 +32,13 @@ VERSION { #endif getrandom; __vdso_getrandom; + +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK + __vdso_futex_robust_list64_try_unlock; +#ifdef CONFIG_COMPAT + __vdso_futex_robust_list32_try_unlock; +#endif +#endif local: *; }; } diff --git a/arch/x86/entry/vdso/vdso64/vdsox32.lds.S b/arch/x86/entry/vdso/vdso64/vdsox32.lds.S index 3dbd20c8dacc..b917dc69f62f 100644 --- a/arch/x86/entry/vdso/vdso64/vdsox32.lds.S +++ b/arch/x86/entry/vdso/vdso64/vdsox32.lds.S @@ -22,6 +22,13 @@ VERSION { __vdso_getcpu; __vdso_time; __vdso_clock_getres; + +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK + __vdso_futex_robust_list64_try_unlock; +#ifdef CONFIG_COMPAT + __vdso_futex_robust_list32_try_unlock; +#endif +#endif local: *; }; } diff --git a/arch/x86/entry/vdso/vdso64/vfutex.c b/arch/x86/entry/vdso/vdso64/vfutex.c new file mode 100644 index 000000000000..940a6ee30026 --- /dev/null +++ b/arch/x86/entry/vdso/vdso64/vfutex.c @@ -0,0 +1 @@ +#include "common/vfutex.c" diff --git a/arch/x86/include/asm/futex_robust.h b/arch/x86/include/asm/futex_robust.h new file mode 100644 index 000000000000..e87954703ae2 --- /dev/null +++ b/arch/x86/include/asm/futex_robust.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_FUTEX_ROBUST_H +#define _ASM_X86_FUTEX_ROBUST_H + +#include + +static __always_inline void __user *x86_futex_robust_unlock_get_pop(struct pt_regs *regs) +{ + /* + * If ZF is set then the cmpxchg succeeded and the pending op pointer + * needs to be cleared. + */ + return regs->flags & X86_EFLAGS_ZF ? (void __user *)regs->dx : NULL; +} + +#define arch_futex_robust_unlock_get_pop(regs) \ + x86_futex_robust_unlock_get_pop(regs) + +#endif /* _ASM_X86_FUTEX_ROBUST_H */ From 3f63e2545978abda58f2cf7ff0d7a2942965e8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Tue, 2 Jun 2026 11:10:16 +0200 Subject: [PATCH 28/36] Documentation: futex: Add a note about robust list race condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a note to the documentation giving a brief explanation why doing a robust futex release in userspace is racy, what should be done to avoid it and provide links to read more. [ tglx: Fixed a few typos ] Signed-off-by: André Almeida Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260329-tonyk-vdso_test-v2-1-b7db810e44a1@igalia.com Link: https://patch.msgid.link/20260602090535.936286833@kernel.org --- Documentation/locking/robust-futex-ABI.rst | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Documentation/locking/robust-futex-ABI.rst b/Documentation/locking/robust-futex-ABI.rst index f24904f1c16f..5e6a0665b8ba 100644 --- a/Documentation/locking/robust-futex-ABI.rst +++ b/Documentation/locking/robust-futex-ABI.rst @@ -153,6 +153,9 @@ On removal: 3) release the futex lock, and 4) clear the 'lock_op_pending' word. +Please note that the removal of a robust futex purely in userspace is +racy. Refer to the next chapter to learn more and how to avoid this. + On exit, the kernel will consider the address stored in 'list_op_pending' and the address of each 'lock word' found by walking the list starting at 'head'. For each such address, if the bottom 30 @@ -182,3 +185,44 @@ any point: When the kernel sees a list entry whose 'lock word' doesn't have the current threads TID in the lower 30 bits, it does nothing with that entry, and goes on to the next entry. + +Robust release is racy +---------------------- + +The removal of a robust futex from the list is racy when doing it solely in +userspace. Quoting Thomas Gleixner for the explanation: + + The robust futex unlock mechanism is racy in respect to the clearing of the + robust_list_head::list_op_pending pointer because unlock and clearing the + pointer are not atomic. The race window is between the unlock and clearing + the pending op pointer. If the task is forced to exit in this window, exit + will access a potentially invalid pending op pointer when cleaning up the + robust list. That happens if another task manages to unmap the object + containing the lock before the cleanup, which results in an UAF. In the + worst case this UAF can lead to memory corruption when unrelated content + has been mapped to the same address by the time the access happens. + +A full in-depth analysis can be read at +https://lore.kernel.org/lkml/20260316162316.356674433@kernel.org/ + +To overcome that, the kernel needs to participate in the lock release operation. +This ensures that the release happens "atomically" with regard to releasing +the lock and removing the address from ``list_op_pending``. If the release is +interrupted by a signal, the kernel will also verify if it interrupted the +release operation. + +For the contended unlock case, where other threads are waiting for the lock +release, there's the ``FUTEX_ROBUST_UNLOCK`` operation feature flag for the +``futex()`` system call, which must be used with one of the following +operations: ``FUTEX_WAKE``, ``FUTEX_WAKE_BITSET`` or ``FUTEX_UNLOCK_PI``. +The kernel will release the lock (set the futex word to zero), clean the +``list_op_pending`` field. Then, it will proceed with the normal wake path. + +For the non-contended path, there's still a race between checking the futex word +and clearing the ``list_op_pending`` field. To solve this without the need of a +complete system call, userspace should call the virtual syscall +``__vdso_futex_robust_listXX_try_unlock()`` (where XX is either 32 or 64, +depending on the size of the pointer). If the vDSO call succeeds, it means that +it released the lock and cleared ``list_op_pending``. If it fails, that means +that there are waiters for this lock and a call to ``futex()`` syscall with +``FUTEX_ROBUST_UNLOCK`` is needed. From 608323bf7bb85bbb647eca4373acef247f105e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Almeida?= Date: Tue, 2 Jun 2026 11:10:21 +0200 Subject: [PATCH 29/36] selftests: futex: Add tests for robust release operations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests for __vdso_futex_robust_listXX_try_unlock() and for the futex() op FUTEX_ROBUST_UNLOCK. Test the contended and uncontended cases for the vDSO functions and all ops combinations for FUTEX_ROBUST_UNLOCK. [ tglx: Replace the VDSO function lookup ] Signed-off-by: André Almeida Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260329-tonyk-vdso_test-v2-2-b7db810e44a1@igalia.com Link: https://patch.msgid.link/20260602090535.988101541@kernel.org --- .../selftests/futex/functional/robust_list.c | 239 ++++++++++++++++++ .../selftests/futex/include/futextest.h | 6 + 2 files changed, 245 insertions(+) diff --git a/tools/testing/selftests/futex/functional/robust_list.c b/tools/testing/selftests/futex/functional/robust_list.c index e7d1254e18ca..b3fab60181d5 100644 --- a/tools/testing/selftests/futex/functional/robust_list.c +++ b/tools/testing/selftests/futex/functional/robust_list.c @@ -27,12 +27,15 @@ #include "futextest.h" #include "../../kselftest_harness.h" +#include #include #include #include +#include #include #include #include +#include #include #include @@ -42,6 +45,10 @@ #define SLEEP_US 100 +#if __SIZEOF_LONG__ == 8 +# define BUILD_64 +#endif + static pthread_barrier_t barrier, barrier2; static int set_robust_list(struct robust_list_head *head, size_t len) @@ -54,6 +61,12 @@ static int get_robust_list(int pid, struct robust_list_head **head, size_t *len_ return syscall(SYS_get_robust_list, pid, head, len_ptr); } +static int sys_futex_robust_unlock(_Atomic(uint32_t) *uaddr, unsigned int op, int val, + void *list_op_pending, unsigned int val3) +{ + return syscall(SYS_futex, uaddr, op, val, NULL, list_op_pending, val3, 0); +} + /* * Basic lock struct, contains just the futex word and the robust list element * Real implementations have also a *prev to easily walk in the list @@ -549,4 +562,230 @@ TEST(test_circular_list) ksft_test_result_pass("%s\n", __func__); } +/* + * Below are tests for the fix of robust release race condition. Please read the following + * thread to learn more about the issue in the first place and why the following functions fix it: + * https://lore.kernel.org/lkml/20260316162316.356674433@kernel.org/ + */ + +/* + * Auxiliary code for binding the vDSO functions + */ +static void *get_vdso_func_addr(const char *function) +{ + const char *vdso_names[] = { + "linux-vdso.so.1", "linux-gate.so.1", "linux-vdso32.so.1", "linux-vdso64.so.1", + }; + + for (int i = 0; i < ARRAY_SIZE(vdso_names); i++) { + void *vdso = dlopen(vdso_names[i], RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD); + + if (vdso) + return dlsym(vdso, function); + } + return NULL; +} + +/* + * These are the real vDSO function signatures: + * + * __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop) + * __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop) + * + * So for the generic entry point we need to use a void pointer as the last argument + */ +FIXTURE(vdso_unlock) +{ + uint32_t (*vdso)(_Atomic(uint32_t) *lock, uint32_t tid, void *pop); +}; + +FIXTURE_VARIANT(vdso_unlock) +{ + bool is_32; + char func_name[]; +}; + +FIXTURE_SETUP(vdso_unlock) +{ + self->vdso = get_vdso_func_addr(variant->func_name); +} + +FIXTURE_TEARDOWN(vdso_unlock) {} + +FIXTURE_VARIANT_ADD(vdso_unlock, 32) +{ + .func_name = "__vdso_futex_robust_list32_try_unlock", + .is_32 = true, +}; + +FIXTURE_VARIANT_ADD(vdso_unlock, 64) +{ + .func_name = "__vdso_futex_robust_list64_try_unlock", + .is_32 = false, +}; + +/* + * Test the vDSO robust_listXX_try_unlock() for the uncontended case. The virtual syscall should + * return the thread ID of the lock owner, the lock word must be 0 and the list_op_pending should + * be NULL. + */ +TEST_F(vdso_unlock, test_robust_try_unlock_uncontended) +{ + struct lock_struct lock = { .futex = 0 }; + _Atomic(unsigned int) *futex = &lock.futex; + struct robust_list_head head; + uintptr_t exp = (uintptr_t) NULL; + pid_t tid = gettid(); + int ret; + + if (!self->vdso) { + ksft_test_result_skip("%s not found\n", variant->func_name); + return; + } + + *futex = tid; + + ret = set_list(&head); + if (ret) + ksft_test_result_fail("set_robust_list error\n"); + + head.list_op_pending = &lock.list; + + ret = self->vdso(futex, tid, &head.list_op_pending); + + ASSERT_EQ(ret, tid); + ASSERT_EQ(*futex, 0); + + /* Check only the lower 32 bits for the 32-bit entry point */ + if (variant->is_32) { + exp = (uintptr_t)(unsigned long)&lock.list; + exp &= ~0xFFFFFFFFULL; + } + + ASSERT_EQ((uintptr_t)(unsigned long)head.list_op_pending, exp); +} + +/* + * If the lock is contended, the operation fails. The return value is the value found at the + * futex word (tid | FUTEX_WAITERS), the futex word is not modified and the list_op_pending is_32 + * not cleared. + */ +TEST_F(vdso_unlock, test_robust_try_unlock_contended) +{ + struct lock_struct lock = { .futex = 0 }; + _Atomic(unsigned int) *futex = &lock.futex; + struct robust_list_head head; + pid_t tid = gettid(); + int ret; + + if (!self->vdso) { + ksft_test_result_skip("%s not found\n", variant->func_name); + return; + } + + *futex = tid | FUTEX_WAITERS; + + ret = set_list(&head); + if (ret) + ksft_test_result_fail("set_robust_list error\n"); + + head.list_op_pending = &lock.list; + + ret = self->vdso(futex, tid, &head.list_op_pending); + + ASSERT_EQ(ret, tid | FUTEX_WAITERS); + ASSERT_EQ(*futex, tid | FUTEX_WAITERS); + ASSERT_EQ(head.list_op_pending, &lock.list); +} + +FIXTURE(futex_op) {}; + +FIXTURE_VARIANT(futex_op) +{ + unsigned int op; + unsigned int val3; +}; + +FIXTURE_SETUP(futex_op) {} + +FIXTURE_TEARDOWN(futex_op) {} + +FIXTURE_VARIANT_ADD(futex_op, wake) +{ + .op = FUTEX_WAKE, + .val3 = 0, +}; + +FIXTURE_VARIANT_ADD(futex_op, wake_bitset) +{ + .op = FUTEX_WAKE_BITSET, + .val3 = FUTEX_BITSET_MATCH_ANY, +}; + +FIXTURE_VARIANT_ADD(futex_op, unlock_pi) +{ + .op = FUTEX_UNLOCK_PI, + .val3 = 0, +}; + +FIXTURE_VARIANT_ADD(futex_op, wake32) +{ + .op = FUTEX_WAKE | FUTEX_ROBUST_LIST32, + .val3 = 0, +}; + +FIXTURE_VARIANT_ADD(futex_op, wake_bitset32) +{ + .op = FUTEX_WAKE_BITSET | FUTEX_ROBUST_LIST32, + .val3 = FUTEX_BITSET_MATCH_ANY, +}; + +FIXTURE_VARIANT_ADD(futex_op, unlock_pi32) +{ + .op = FUTEX_UNLOCK_PI | FUTEX_ROBUST_LIST32, + .val3 = 0, +}; + +/* + * The syscall should return the number of tasks waken (for this test, 0), clear the futex word and + * clear list_op_pending + */ +TEST_F(futex_op, test_futex_robust_unlock) +{ + struct lock_struct lock = { .futex = 0 }; + _Atomic(unsigned int) *futex = &lock.futex; + uintptr_t exp = (uintptr_t) NULL; + struct robust_list_head head; + pid_t tid = gettid(); + int ret; + +#ifndef BUILD_64 + if (!(variant->op & FUTEX_ROBUST_LIST32)) { + ksft_test_result_skip("Not supported for 32 bit build\n"); + return; + } +#endif + + *futex = tid | FUTEX_WAITERS; + + ret = set_list(&head); + if (ret) + ksft_test_result_fail("set_robust_list error\n"); + + head.list_op_pending = &lock.list; + + ret = sys_futex_robust_unlock(futex, FUTEX_ROBUST_UNLOCK | variant->op, tid, + &head.list_op_pending, variant->val3); + + ASSERT_EQ(ret, 0); + ASSERT_EQ(*futex, 0); + + if (variant->op & FUTEX_ROBUST_LIST32) { + exp = (uint64_t)(unsigned long)&lock.list; + exp &= ~0xFFFFFFFFULL; + } + + ASSERT_EQ((uintptr_t)(unsigned long)head.list_op_pending, exp); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/futex/include/futextest.h b/tools/testing/selftests/futex/include/futextest.h index 3d48e9789d9f..df33f31d6994 100644 --- a/tools/testing/selftests/futex/include/futextest.h +++ b/tools/testing/selftests/futex/include/futextest.h @@ -38,6 +38,12 @@ typedef volatile u_int32_t futex_t; #ifndef FUTEX_CMP_REQUEUE_PI #define FUTEX_CMP_REQUEUE_PI 12 #endif +#ifndef FUTEX_ROBUST_UNLOCK +#define FUTEX_ROBUST_UNLOCK 512 +#endif +#ifndef FUTEX_ROBUST_LIST32 +#define FUTEX_ROBUST_LIST32 1024 +#endif #ifndef FUTEX_WAIT_REQUEUE_PI_PRIVATE #define FUTEX_WAIT_REQUEUE_PI_PRIVATE (FUTEX_WAIT_REQUEUE_PI | \ FUTEX_PRIVATE_FLAG) From a40e0f8eadd44d7b0f856b54c876aea1b93415f4 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Fri, 5 Jun 2026 03:06:22 -0700 Subject: [PATCH 30/36] cleanup: Specify nonnull argument index The guard constructors were annotated with an empty __nonnull_args(), relying on __nonnull__() marking every pointer parameter as non-NULL. Sparse cannot parse the empty argument list. Both constructors take the lock pointer as their first parameter, so specify the index explicitly: __nonnull_args(1). Reported-by: Dan Carpenter Closes: https://lore.kernel.org/all/aiJi0WcYE8FZt-jO@stanley.mountain/ Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/aiKpH3cLBEj3TF2Q@shell.ilvokhin.com --- include/linux/cleanup.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index 65416938e318..b1b5698cbf1b 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -397,7 +397,7 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond __DEFINE_GUARD_LOCK_PTR(_name, _T) #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ - static __always_inline __nonnull_args() _type class_##_name##_constructor(_type _T); \ + static __always_inline __nonnull_args(1) _type class_##_name##_constructor(_type _T); \ DEFINE_CLASS(_name, _type, _unlock, ({ _lock; _T; }), _type _T); \ DEFINE_CLASS_IS_GUARD(_name) @@ -498,7 +498,7 @@ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) #define __DEFINE_LOCK_GUARD_1(_name, _type, ...) \ -static __always_inline __nonnull_args() \ +static __always_inline __nonnull_args(1) \ class_##_name##_t class_##_name##_constructor(_type *l) \ __no_context_analysis \ { \ From 4770880855359b345314ca1d0b28f9be8886eba8 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Thu, 4 Jun 2026 22:23:29 -0700 Subject: [PATCH 31/36] MAINTAINERS: Add RUST [SYNC] entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have two pull requests on Rust synchronization primitives with 10+ patches in a row for recent cycles, so it makes sense to start the effort of handling this area as a group. Luckily for me, Gary Guo and Alice Ryhl agreed to help as co-maintainers, and we also have a talented group of reviewers: Lyude Paul started the SpinLockIrq work [1] and did an amazing job at improving the design and implementation. Daniel Almeida resolved the Lock issue [2] and he did a fair amount of reviews in areas related to synchronization primitives already. Onur Özkan started the ww_mutex work [3] and did an amazing job at consolidating various design requirements and decisions. Of course, this only reflects my own knowledge, and I believe they did way more outside what I'm aware of ;-) Note that having this MAINTAINERS entry is meant to bring more people to help on the synchronization primitives in Rust, which means for patch submissions and design discussion, please still involve the corresponding maintainers (e.g. LOCKING and ATOMIC), scripts/get_maintainers.pl should have this covered. Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Acked-by: Daniel Almeida Acked-by: Danilo Krummrich Acked-by: Onur Özkan Acked-by: Gary Guo Acked-by: Alice Ryhl Link: https://lore.kernel.org/rust-for-linux/20260302232154.861916-1-lyude@redhat.com/ [1] Link: https://lore.kernel.org/all/20250828-lock-t-when-t-is-pinned-v2-0-b067c4b93fd6@collabora.com/ [2] Link: https://lore.kernel.org/rust-for-linux/20260103073554.34855-1-work@onurozkan.dev/ [3] Link: https://patch.msgid.link/20260415232830.8128-1-boqun@kernel.org Link: https://patch.msgid.link/20260605052331.1628-2-boqun@kernel.org --- MAINTAINERS | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 882214b0e7db..7c0f853b881b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23450,6 +23450,20 @@ S: Maintained T: git https://github.com/Rust-for-Linux/linux.git rust-analyzer-next F: scripts/generate_rust_analyzer.py +RUST [SYNC] +M: Boqun Feng +M: Gary Guo +M: Alice Ryhl +R: Lyude Paul +R: Daniel Almeida +R: "Onur Özkan" +L: linux-kernel@vger.kernel.org +L: rust-for-linux@vger.kernel.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/boqun/linux.git rust-sync +F: rust/kernel/sync.rs +F: rust/kernel/sync/ + RXRPC SOCKETS (AF_RXRPC) M: David Howells M: Marc Dionne From a837dd95e841586c3a6bbe41c41843b392a1b725 Mon Sep 17 00:00:00 2001 From: Fabricio Parra Date: Thu, 4 Jun 2026 22:23:31 -0700 Subject: [PATCH 32/36] rust: sync: completion: Mark inline complete_all and wait_for_completion When building the kernel using the llvm-22.1.0-rust-1.93.1-x86_64 toolchain provided by kernel.org with ARCH=x86_64, the following symbols are generated: $ nm vmlinux | grep ' _R'.*Completion | rustfilt ffffffff81827930 T ::complete_all ffffffff81827950 T ::wait_for_completion These Rust methods are thin wrappers around the C completion helpers `complete_all` and `wait_for_completion`. Mark them `#[inline]` to keep the wrapper pattern consistent with other small Rust helper methods. After applying this patch, the above command will produce no output. Suggested-by: Alice Ryhl Signed-off-by: Fabricio Parra Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Gary Guo Link: https://github.com/Rust-for-Linux/linux/issues/1145 Link: https://patch.msgid.link/20260316151056.287-1-a@alice0.com Link: https://patch.msgid.link/20260605052331.1628-4-boqun@kernel.org --- rust/kernel/sync/completion.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs index c50012a940a3..35ff049ff078 100644 --- a/rust/kernel/sync/completion.rs +++ b/rust/kernel/sync/completion.rs @@ -94,6 +94,7 @@ fn as_raw(&self) -> *mut bindings::completion { /// /// This method wakes up all tasks waiting on this completion; after this operation the /// completion is permanently done, i.e. signals all current and future waiters. + #[inline] pub fn complete_all(&self) { // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. unsafe { bindings::complete_all(self.as_raw()) }; @@ -105,6 +106,7 @@ pub fn complete_all(&self) { /// timeout. /// /// See also [`Completion::complete_all`]. + #[inline] pub fn wait_for_completion(&self) { // SAFETY: `self.as_raw()` is a pointer to a valid `struct completion`. unsafe { bindings::wait_for_completion(self.as_raw()) }; From a734d9fca84e1d4fa0cb442ef5f84c88f8212d32 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 10 Jun 2026 23:20:09 +0200 Subject: [PATCH 33/36] futex: Optimize futex hash bucket access patterns Breno reported significant c2c HITM in a futex hash heavy workload. It turns out that the hash bucket to private hash table reverse pointer (futex_hash_bucket::priv) was to blame. Notably when the hash buckets are heavily contended, the: 'fph = bh->priv;' load in futex_hash() will typically miss and consequently become quite expensive. Since this load in particular is quite superfluous, removing it is fairly straight forward. However, removing it does not in fact achieve anything much. The pain moves to the next user, notably: futex_hash_put(). Therefore rework the whole private hash refcounting to avoid needing this back pointer (and removing it). Instead of passing around 'struct futex_hash_bucket *hb', pass around a new structure that contains it and the related 'struct futex_private_hash *fph' pointer in tandem. Funnily this turns out to remove more code than it adds and significantly improves futex hash performance (as measured by 'perf bench futex hash'): SKL dual socket 112 threads: Baseline Patched shared (16k) 1571857 1641435 + 4.4% autosize (512) 646390 903371 +39.7% -b 256 464395 587014 +26.4% -b 512 715687 995943 +39.2% -b 1024 995085 1396328 +40.3% -b 2048 1293114 1668395 +29.0% -b 4096 2124438 2240228 + 5.5% Zen3 dual socket 256 threads: Baseline Patched shared (16k) 1275840 1381279 + 8.2% autosize (512) 1252745 1482179 +18.3% -b 256 856274 955455 +11.5% -b 512 1267490 1544010 +21.8% -b 1024 1424013 1625424 +14.1% -b 2048 1505181 1669342 +10.9% -b 4096 1465993 1688932 +15.2% AMD EPYC 9D64 (Zen4, single socket) 176 threads: Baseline Patched Delta shared (16k) 1,230,599 1,368,655 +11.2% autosize (1024) 1,285,440 1,556,946 +21.1% -b 256 1,341,471 1,520,303 +13.3% -b 512 1,438,330 1,599,319 +11.2% -b 1024 1,443,772 1,622,493 +12.4% -b 2048 1,472,108 1,643,975 +11.7% -b 4096 1,333,098 1,570,897 +17.8% Reported-by: Breno Leitao Signed-off-by: Peter Zijlstra (Intel) Tested-by: Breno Leitao Tested-by: Thomas Gleixner Link: https://patch.msgid.link/20260610135510.GB1430057@noisy.programming.kicks-ass.net --- kernel/futex/core.c | 111 ++++++++++++---------------------------- kernel/futex/futex.h | 37 +++++++++----- kernel/futex/pi.c | 21 ++++---- kernel/futex/requeue.c | 20 ++++---- kernel/futex/waitwake.c | 17 +++--- 5 files changed, 87 insertions(+), 119 deletions(-) diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 6ea4a97796a1..179b26e9c934 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -127,7 +127,7 @@ late_initcall(fail_futex_debugfs); #endif /* CONFIG_FAIL_FUTEX */ static struct futex_hash_bucket * -__futex_hash(union futex_key *key, struct futex_private_hash *fph); +__futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_private_hash **fph_p); #ifdef CONFIG_FUTEX_PRIVATE_HASH static bool futex_ref_get(struct futex_private_hash *fph); @@ -136,15 +136,6 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph); enum { FR_PERCPU = 0, FR_ATOMIC }; -static inline bool futex_key_is_private(union futex_key *key) -{ - /* - * Relies on get_futex_key() to set either bit for shared - * futexes -- see comment with union futex_key. - */ - return !(key->both.offset & (FUT_OFF_INODE | FUT_OFF_MMSHARED)); -} - static bool futex_private_hash_get(struct futex_private_hash *fph) { return futex_ref_get(fph); @@ -152,48 +143,15 @@ static bool futex_private_hash_get(struct futex_private_hash *fph) void futex_private_hash_put(struct futex_private_hash *fph) { - if (futex_ref_put(fph)) + if (fph && futex_ref_put(fph)) wake_up_var(fph->mm); } -/** - * futex_hash_get - Get an additional reference for the local hash. - * @hb: ptr to the private local hash. - * - * Obtain an additional reference for the already obtained hash bucket. The - * caller must already own an reference. - */ -void futex_hash_get(struct futex_hash_bucket *hb) -{ - struct futex_private_hash *fph = hb->priv; - - if (!fph) - return; - WARN_ON_ONCE(!futex_private_hash_get(fph)); -} - -void futex_hash_put(struct futex_hash_bucket *hb) -{ - struct futex_private_hash *fph = hb->priv; - - if (!fph) - return; - futex_private_hash_put(fph); -} - static struct futex_hash_bucket * __futex_hash_private(union futex_key *key, struct futex_private_hash *fph) { u32 hash; - if (!futex_key_is_private(key)) - return NULL; - - if (!fph) - fph = rcu_dereference(key->private.mm->futex.phash.hash); - if (!fph || !fph->hash_mask) - return NULL; - hash = jhash2((void *)&key->private.address, sizeof(key->private.address) / 4, key->both.offset); @@ -214,13 +172,12 @@ static void futex_rehash_private(struct futex_private_hash *old, spin_lock(&hb_old->lock); plist_for_each_entry_safe(this, tmp, &hb_old->chain, list) { - plist_del(&this->list, &hb_old->chain); futex_hb_waiters_dec(hb_old); WARN_ON_ONCE(this->lock_ptr != &hb_old->lock); - hb_new = __futex_hash(&this->key, new); + hb_new = __futex_hash(&this->key, new, NULL); futex_hb_waiters_inc(hb_new); /* * The new pointer isn't published yet but an already @@ -273,9 +230,8 @@ static void futex_pivot_hash(struct mm_struct *mm) } } -struct futex_private_hash *futex_private_hash(void) +struct futex_private_hash *futex_private_hash(struct mm_struct *mm) { - struct mm_struct *mm = current->mm; /* * Ideally we don't loop. If there is a replacement in progress * then a new private hash is already prepared and a reference can't be @@ -301,18 +257,17 @@ struct futex_private_hash *futex_private_hash(void) goto again; } -struct futex_hash_bucket *futex_hash(union futex_key *key) +struct futex_bucket_ref futex_hash(union futex_key *key) { - struct futex_private_hash *fph; - struct futex_hash_bucket *hb; - again: scoped_guard(rcu) { - hb = __futex_hash(key, NULL); - fph = hb->priv; + struct futex_private_hash *fph = NULL; + struct futex_hash_bucket *hb; + + hb = __futex_hash(key, NULL, &fph); if (!fph || futex_private_hash_get(fph)) - return hb; + return (struct futex_bucket_ref){ .hb = hb, .fph = fph }; } futex_pivot_hash(key->private.mm); goto again; @@ -320,15 +275,9 @@ struct futex_hash_bucket *futex_hash(union futex_key *key) #else /* !CONFIG_FUTEX_PRIVATE_HASH */ -static struct futex_hash_bucket * -__futex_hash_private(union futex_key *key, struct futex_private_hash *fph) +struct futex_bucket_ref futex_hash(union futex_key *key) { - return NULL; -} - -struct futex_hash_bucket *futex_hash(union futex_key *key) -{ - return __futex_hash(key, NULL); + return (struct futex_bucket_ref){ .hb = __futex_hash(key, NULL, NULL), .fph = NULL }; } #endif /* CONFIG_FUTEX_PRIVATE_HASH */ @@ -406,6 +355,8 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * __futex_hash - Return the hash bucket * @key: Pointer to the futex key for which the hash is calculated * @fph: Pointer to private hash if known + * @fph_p: Pointer to a private hash pointer; output for the private hash + * used when set. * * We hash on the keys returned from get_futex_key (see below) and return the * corresponding hash bucket. @@ -413,18 +364,23 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * private hash) is returned if existing. Otherwise a hash bucket from the * global hash is returned. */ -static struct futex_hash_bucket *__futex_hash(union futex_key *key, struct futex_private_hash *fph) +static struct futex_hash_bucket * +__futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_private_hash **fph_p) { int node = key->both.node; u32 hash; - if (node == FUTEX_NO_NODE) { - struct futex_hash_bucket *hb; - - hb = __futex_hash_private(key, fph); - if (hb) - return hb; +#ifdef CONFIG_FUTEX_PRIVATE_HASH + if (node == FUTEX_NO_NODE && futex_key_is_private(key)) { + if (!fph) + fph = rcu_dereference(key->private.mm->futex.phash.hash); + if (fph && fph->hash_mask) { + if (fph_p) + *fph_p = fph; + return __futex_hash_private(key, fph); + } } +#endif hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(u32), key->both.offset); @@ -1367,7 +1323,7 @@ static void exit_pi_state_list(struct task_struct *curr) * on the mutex. */ WARN_ON(curr != current); - guard(private_hash)(); + guard(private_hash)(current->mm); /* * We are a ZOMBIE and nobody can enqueue itself on * pi_state_list anymore, but we have to be careful @@ -1379,7 +1335,8 @@ static void exit_pi_state_list(struct task_struct *curr) pi_state = list_entry(next, struct futex_pi_state, list); key = pi_state->key; if (1) { - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; /* * We can race against put_pi_state() removing itself from the @@ -1576,12 +1533,8 @@ void futex_exit_release(struct task_struct *tsk) futex_cleanup_end(tsk, FUTEX_STATE_DEAD); } -static void futex_hash_bucket_init(struct futex_hash_bucket *fhb, - struct futex_private_hash *fph) +static void futex_hash_bucket_init(struct futex_hash_bucket *fhb) { -#ifdef CONFIG_FUTEX_PRIVATE_HASH - fhb->priv = fph; -#endif atomic_set(&fhb->waiters, 0); plist_head_init(&fhb->chain); spin_lock_init(&fhb->lock); @@ -1877,7 +1830,7 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) fph->mm = mm; for (i = 0; i < hash_slots; i++) - futex_hash_bucket_init(&fph->queues[i], fph); + futex_hash_bucket_init(&fph->queues[i]); if (custom) { /* @@ -2082,7 +2035,7 @@ static int __init futex_init(void) BUG_ON(!table); for (i = 0; i < hashsize; i++) - futex_hash_bucket_init(&table[i], NULL); + futex_hash_bucket_init(&table[i]); futex_queues[n] = table; } diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 79ef2c709c81..f00f0863ed44 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -134,6 +134,15 @@ static inline bool should_fail_futex(bool fshared) } #endif +static inline bool futex_key_is_private(union futex_key *key) +{ + /* + * Relies on get_futex_key() to set either bit for shared + * futexes -- see comment with union futex_key. + */ + return !(key->both.offset & (FUT_OFF_INODE | FUT_OFF_MMSHARED)); +} + /* * Hash buckets are shared by all the futex_keys that hash to the same * location. Each key may have multiple futex_q structures, one for each task @@ -143,7 +152,6 @@ struct futex_hash_bucket { atomic_t waiters; spinlock_t lock; struct plist_head chain; - struct futex_private_hash *priv; } ____cacheline_aligned_in_smp; /* @@ -183,7 +191,7 @@ typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q); * @requeue_pi_key: the requeue_pi target futex key * @bitset: bitset for the optional bitmasked wakeup * @requeue_state: State field for futex_requeue_pi() - * @drop_hb_ref: Waiter should drop the extra hash bucket reference if true + * @drop_fph: Waiter should drop the extra private hash reference when set * @requeue_wait: RCU wait for futex_requeue_pi() (RT only) * * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so @@ -210,7 +218,7 @@ struct futex_q { union futex_key *requeue_pi_key; u32 bitset; atomic_t requeue_state; - bool drop_hb_ref; + struct futex_private_hash *drop_fph; #ifdef CONFIG_PREEMPT_RT struct rcuwait requeue_wait; #endif @@ -230,28 +238,29 @@ extern struct hrtimer_sleeper * futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, int flags, u64 range_ns); -extern struct futex_hash_bucket *futex_hash(union futex_key *key); -#ifdef CONFIG_FUTEX_PRIVATE_HASH -extern void futex_hash_get(struct futex_hash_bucket *hb); -extern void futex_hash_put(struct futex_hash_bucket *hb); +struct futex_bucket_ref { + struct futex_hash_bucket *hb; + struct futex_private_hash *fph; +}; -extern struct futex_private_hash *futex_private_hash(void); +#ifdef CONFIG_FUTEX_PRIVATE_HASH +extern struct futex_private_hash *futex_private_hash(struct mm_struct *mm); extern void futex_private_hash_put(struct futex_private_hash *fph); #else /* !CONFIG_FUTEX_PRIVATE_HASH */ -static inline void futex_hash_get(struct futex_hash_bucket *hb) { } -static inline void futex_hash_put(struct futex_hash_bucket *hb) { } -static inline struct futex_private_hash *futex_private_hash(void) { return NULL; } +static inline struct futex_private_hash *futex_private_hash(struct mm_struct *mm) { return NULL; } static inline void futex_private_hash_put(struct futex_private_hash *fph) { } #endif -DEFINE_CLASS(hb, struct futex_hash_bucket *, - if (_T) futex_hash_put(_T), +extern struct futex_bucket_ref futex_hash(union futex_key *key); + +DEFINE_CLASS(hbr, struct futex_bucket_ref, + if (_T.fph) futex_private_hash_put(_T.fph), futex_hash(key), union futex_key *key); DEFINE_CLASS(private_hash, struct futex_private_hash *, if (_T) futex_private_hash_put(_T), - futex_private_hash(), void); + futex_private_hash(mm), struct mm_struct *mm); /** * futex_match - Check whether two futex keys are equal diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 9dd5c0b1ac6a..795011ea1202 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -945,7 +945,8 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl retry_private: if (1) { - CLASS(hb, hb)(&q.key); + CLASS(hbr, hbr)(&q.key); + auto hb = hbr.hb; futex_q_lock(&q, hb); @@ -1009,7 +1010,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl * the thread, performing resize, will block on hb->lock during * the requeue. */ - futex_hash_put(no_free_ptr(hb)); + futex_private_hash_put(no_free_ptr(hbr.fph)); /* * Must be done before we enqueue the waiter, here is unfortunately * under the hb lock, but that *should* work because it does nothing. @@ -1100,11 +1101,9 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl __release(&hb->lock); futex_unqueue_pi(&q); spin_unlock(q.lock_ptr); - if (q.drop_hb_ref) { - CLASS(hb, hb)(&q.key); - /* Additional reference from futex_unlock_pi() */ - futex_hash_put(hb); - } + + /* Additional reference from futex_unlock_pi() */ + futex_private_hash_put(q.drop_fph); goto out; out_unlock_put_key: @@ -1161,7 +1160,8 @@ static int __futex_unlock_pi(u32 __user *uaddr, unsigned int flags) if (ret) return ret; - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; spin_lock(&hb->lock); retry_hb: @@ -1218,8 +1218,9 @@ static int __futex_unlock_pi(u32 __user *uaddr, unsigned int flags) * Acquire a reference for the leaving waiter to ensure * valid futex_q::lock_ptr. */ - futex_hash_get(hb); - top_waiter->drop_hb_ref = true; + if (futex_key_is_private(&key)) + top_waiter->drop_fph = futex_private_hash(key.private.mm); + __futex_unqueue(top_waiter); raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); goto retry_hb; diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index b597cb3d17fc..79823ad13683 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -241,8 +241,8 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key, * Acquire a reference for the waiter to ensure valid * futex_q::lock_ptr. */ - futex_hash_get(hb); - q->drop_hb_ref = true; + if (futex_key_is_private(key)) + q->drop_fph = futex_private_hash(key->private.mm); q->lock_ptr = &hb->lock; task = READ_ONCE(q->task); @@ -459,8 +459,10 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1, retry_private: if (1) { - CLASS(hb, hb1)(&key1); - CLASS(hb, hb2)(&key2); + CLASS(hbr, hbr1)(&key1); + CLASS(hbr, hbr2)(&key2); + auto hb1 = hbr1.hb; + auto hb2 = hbr2.hb; futex_hb_waiters_inc(hb2); double_lock_hb(hb1, hb2); @@ -832,7 +834,8 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, switch (futex_requeue_pi_wakeup_sync(&q)) { case Q_REQUEUE_PI_IGNORE: { - CLASS(hb, hb)(&q.key); + CLASS(hbr, hbr)(&q.key); + auto hb = hbr.hb; /* The waiter is still on uaddr1 */ spin_lock(&hb->lock); ret = handle_early_requeue_pi_wakeup(hb, &q, to); @@ -902,11 +905,8 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, default: BUG(); } - if (q.drop_hb_ref) { - CLASS(hb, hb)(&q.key); - /* Additional reference from requeue_pi_wake_futex() */ - futex_hash_put(hb); - } + /* Additional reference from requeue_pi_wake_futex() */ + futex_private_hash_put(q.drop_fph); out: if (to) { diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index 8f5e5d302356..d4483d15d30a 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -195,7 +195,8 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop, int nr_w if ((flags & FLAGS_STRICT) && !nr_wake) return 0; - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; /* Make sure we really have tasks to wakeup */ if (!futex_hb_waiters_pending(hb)) @@ -292,8 +293,10 @@ int futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, retry_private: if (1) { - CLASS(hb, hb1)(&key1); - CLASS(hb, hb2)(&key2); + CLASS(hbr, hbr1)(&key1); + CLASS(hbr, hbr2)(&key2); + auto hb1 = hbr1.hb; + auto hb2 = hbr2.hb; double_lock_hb(hb1, hb2); op_ret = futex_atomic_op_inuser(op, uaddr2); @@ -435,7 +438,7 @@ int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken) * Make sure to have a reference on the private_hash such that we * don't block on rehash after changing the task state below. */ - guard(private_hash)(); + guard(private_hash)(current->mm); /* * Enqueuing multiple futexes is tricky, because we need to enqueue @@ -472,7 +475,8 @@ int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken) u32 val = vs[i].w.val; if (1) { - CLASS(hb, hb)(&q->key); + CLASS(hbr, hbr)(&q->key); + auto hb = hbr.hb; futex_q_lock(q, hb); ret = futex_get_value_locked(&uval, uaddr); @@ -647,7 +651,8 @@ int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, retry_private: if (1) { - CLASS(hb, hb)(&q->key); + CLASS(hbr, hbr)(&q->key); + auto hb = hbr.hb; futex_q_lock(q, hb); From 76080d81e511f66eeb094b204e1e217bff674922 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Thu, 4 Jun 2026 07:15:05 +0000 Subject: [PATCH 34/36] tracing/lock: Remove unnecessary linux/sched.h include None of the trace events in lock.h reference anything from linux/sched.h. Remove the unnecessary include. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Usama Arif Link: https://patch.msgid.link/85e14a0685c1d46c3e80de2dc01358a48169b263.1780506267.git.d@ilvokhin.com --- include/trace/events/lock.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/trace/events/lock.h b/include/trace/events/lock.h index 8e89baa3775f..da978f2afb45 100644 --- a/include/trace/events/lock.h +++ b/include/trace/events/lock.h @@ -5,7 +5,6 @@ #if !defined(_TRACE_LOCK_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_LOCK_H -#include #include /* flags for lock:contention_begin */ From 7cfa62cf9432df67b1c95a59984a03a3bc023f98 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Thu, 4 Jun 2026 07:15:06 +0000 Subject: [PATCH 35/36] locking/percpu-rwsem: Extract __percpu_up_read() Move the percpu_up_read() slowpath out of the inline function into a new __percpu_up_read() to avoid binary size increase from adding a tracepoint to an inlined function. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Usama Arif Link: https://patch.msgid.link/3dd2a1b9ab4f469e1892766cb63f41d6b0f53d29.1780506267.git.d@ilvokhin.com --- include/linux/percpu-rwsem.h | 15 +++------------ kernel/locking/percpu-rwsem.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index c8cb010d655e..39d5bf8e6562 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -107,6 +107,8 @@ static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) return ret; } +extern void __percpu_up_read(struct percpu_rw_semaphore *sem); + static inline void percpu_up_read(struct percpu_rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); @@ -118,18 +120,7 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) if (likely(rcu_sync_is_idle(&sem->rss))) { this_cpu_dec(*sem->read_count); } else { - /* - * slowpath; reader will only ever wake a single blocked - * writer. - */ - smp_mb(); /* B matches C */ - /* - * In other words, if they see our decrement (presumably to - * aggregate zero, as that is the only time it matters) they - * will also see our critical section. - */ - this_cpu_dec(*sem->read_count); - rcuwait_wake_up(&sem->writer); + __percpu_up_read(sem); } preempt_enable(); } diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index ef234469baac..f3ee7a0d6047 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -288,3 +288,21 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) rcu_sync_exit(&sem->rss); } EXPORT_SYMBOL_GPL(percpu_up_write); + +void __percpu_up_read(struct percpu_rw_semaphore *sem) +{ + lockdep_assert_preemption_disabled(); + /* + * slowpath; reader will only ever wake a single blocked + * writer. + */ + smp_mb(); /* B matches C */ + /* + * In other words, if they see our decrement (presumably to + * aggregate zero, as that is the only time it matters) they + * will also see our critical section. + */ + this_cpu_dec(*sem->read_count); + rcuwait_wake_up(&sem->writer); +} +EXPORT_SYMBOL_GPL(__percpu_up_read); From 4f070ccb4dc4692e3b6757819fb80655f58b4f12 Mon Sep 17 00:00:00 2001 From: Dmitry Ilvokhin Date: Thu, 4 Jun 2026 07:15:07 +0000 Subject: [PATCH 36/36] locking: Add contended_release tracepoint to sleepable locks Add the contended_release trace event. This tracepoint fires on the holder side when a contended lock is released, complementing the existing contention_begin/contention_end tracepoints which fire on the waiter side. This enables correlating lock hold time under contention with waiter events by lock address. Add trace_contended_release()/trace_call__contended_release() calls to the slowpath unlock paths of sleepable locks: mutex, rtmutex, semaphore, rwsem, percpu-rwsem, and RT-specific rwbase locks. Where possible, trace_contended_release() fires before the lock is released and before the waiter is woken. For some lock types, the tracepoint fires after the release but before the wake. Making the placement consistent across all lock types is not worth the added complexity. For reader/writer locks, the tracepoint fires for every reader releasing while a writer is waiting, not only for the last reader. Signed-off-by: Dmitry Ilvokhin Signed-off-by: Peter Zijlstra (Intel) Acked-by: Paul E. McKenney Acked-by: Usama Arif Link: https://patch.msgid.link/02f4f6c5ce6761e7f6587cf0ff2289d962ecddd4.1780506267.git.d@ilvokhin.com --- include/trace/events/lock.h | 17 +++++++++++++++++ kernel/locking/mutex.c | 4 ++++ kernel/locking/percpu-rwsem.c | 11 +++++++++++ kernel/locking/rtmutex.c | 1 + kernel/locking/rwbase_rt.c | 6 ++++++ kernel/locking/rwsem.c | 10 ++++++++-- kernel/locking/semaphore.c | 4 ++++ 7 files changed, 51 insertions(+), 2 deletions(-) diff --git a/include/trace/events/lock.h b/include/trace/events/lock.h index da978f2afb45..1ded869cd619 100644 --- a/include/trace/events/lock.h +++ b/include/trace/events/lock.h @@ -137,6 +137,23 @@ TRACE_EVENT(contention_end, TP_printk("%p (ret=%d)", __entry->lock_addr, __entry->ret) ); +TRACE_EVENT(contended_release, + + TP_PROTO(void *lock), + + TP_ARGS(lock), + + TP_STRUCT__entry( + __field(void *, lock_addr) + ), + + TP_fast_assign( + __entry->lock_addr = lock; + ), + + TP_printk("%p", __entry->lock_addr) +); + #endif /* _TRACE_LOCK_H */ /* This part must be outside protection */ diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 09534628dc01..43b7f7e281a0 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -1023,6 +1023,9 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne wake_q_add(&wake_q, next); } + if (trace_contended_release_enabled() && waiter) + trace_call__contended_release(lock); + if (owner & MUTEX_FLAG_HANDOFF) __mutex_handoff(lock, next); @@ -1220,6 +1223,7 @@ EXPORT_SYMBOL(ww_mutex_lock_interruptible); EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(contended_release); /** * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index f3ee7a0d6047..f7e152c40d6d 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -263,6 +263,9 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); + if (trace_contended_release_enabled() && wq_has_sleeper(&sem->waiters)) + trace_call__contended_release(sem); + /* * Signal the writer is done, no fast path yet. * @@ -292,6 +295,14 @@ EXPORT_SYMBOL_GPL(percpu_up_write); void __percpu_up_read(struct percpu_rw_semaphore *sem) { lockdep_assert_preemption_disabled(); + /* + * After percpu_up_write() completes, rcu_sync_is_idle() can still + * return false during the grace period, forcing readers into this + * slowpath. Only trace when a writer is actually waiting for + * readers to drain. + */ + if (trace_contended_release_enabled() && rcuwait_active(&sem->writer)) + trace_call__contended_release(sem); /* * slowpath; reader will only ever wake a single blocked * writer. diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 9147d6a31b78..28beae7d21fe 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -1470,6 +1470,7 @@ static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock) raw_spin_lock_irqsave(&lock->wait_lock, flags); } + trace_contended_release(lock); /* * The wakeup next waiter path does not suffer from the above * race. See the comments there. diff --git a/kernel/locking/rwbase_rt.c b/kernel/locking/rwbase_rt.c index 82e078c0665a..2835c9ef9b3f 100644 --- a/kernel/locking/rwbase_rt.c +++ b/kernel/locking/rwbase_rt.c @@ -174,6 +174,8 @@ static void __sched __rwbase_read_unlock(struct rwbase_rt *rwb, static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb, unsigned int state) { + if (trace_contended_release_enabled() && rt_mutex_owner(&rwb->rtmutex)) + trace_call__contended_release(rwb); /* * rwb->readers can only hit 0 when a writer is waiting for the * active readers to leave the critical section. @@ -205,6 +207,8 @@ static inline void rwbase_write_unlock(struct rwbase_rt *rwb) unsigned long flags; raw_spin_lock_irqsave(&rtm->wait_lock, flags); + if (trace_contended_release_enabled() && rt_mutex_has_waiters(rtm)) + trace_call__contended_release(rwb); __rwbase_write_unlock(rwb, WRITER_BIAS, flags); } @@ -214,6 +218,8 @@ static inline void rwbase_write_downgrade(struct rwbase_rt *rwb) unsigned long flags; raw_spin_lock_irqsave(&rtm->wait_lock, flags); + if (trace_contended_release_enabled() && rt_mutex_has_waiters(rtm)) + trace_call__contended_release(rwb); /* Release it and account current as reader */ __rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags); } diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index bf647097369c..b9c180ac1eee 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -1387,6 +1387,8 @@ static inline void __up_read(struct rw_semaphore *sem) rwsem_clear_reader_owned(sem); tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count); DEBUG_RWSEMS_WARN_ON(tmp < 0, sem); + if (trace_contended_release_enabled() && (tmp & RWSEM_FLAG_WAITERS)) + trace_call__contended_release(sem); if (unlikely((tmp & (RWSEM_LOCK_MASK|RWSEM_FLAG_WAITERS)) == RWSEM_FLAG_WAITERS)) { clear_nonspinnable(sem); @@ -1413,8 +1415,10 @@ static inline void __up_write(struct rw_semaphore *sem) preempt_disable(); rwsem_clear_owner(sem); tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); - if (unlikely(tmp & RWSEM_FLAG_WAITERS)) + if (unlikely(tmp & RWSEM_FLAG_WAITERS)) { + trace_contended_release(sem); rwsem_wake(sem); + } preempt_enable(); } @@ -1437,8 +1441,10 @@ static inline void __downgrade_write(struct rw_semaphore *sem) tmp = atomic_long_fetch_add_release( -RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count); rwsem_set_reader_owned(sem); - if (tmp & RWSEM_FLAG_WAITERS) + if (tmp & RWSEM_FLAG_WAITERS) { + trace_contended_release(sem); rwsem_downgrade_wake(sem); + } preempt_enable(); } diff --git a/kernel/locking/semaphore.c b/kernel/locking/semaphore.c index 74d41433ba13..233730c25933 100644 --- a/kernel/locking/semaphore.c +++ b/kernel/locking/semaphore.c @@ -230,6 +230,10 @@ void __sched up(struct semaphore *sem) sem->count++; else __up(sem, &wake_q); + + if (trace_contended_release_enabled() && !wake_q_empty(&wake_q)) + trace_call__contended_release(sem); + raw_spin_unlock_irqrestore(&sem->lock, flags); if (!wake_q_empty(&wake_q)) wake_up_q(&wake_q);