diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 1ce62a996192..fdaef60b46d6 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -239,6 +239,7 @@ config X86 select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA if X86_32 select HAVE_EXIT_THREAD + select HAVE_FUTEX_ROBUST_UNLOCK select HAVE_GENERIC_TIF_BITS select HAVE_GUP_FAST select HAVE_FENTRY if X86_64 || DYNAMIC_FTRACE diff --git a/arch/x86/entry/vdso/common/vfutex.c b/arch/x86/entry/vdso/common/vfutex.c new file mode 100644 index 000000000000..454f059278e4 --- /dev/null +++ b/arch/x86/entry/vdso/common/vfutex.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +/* + * Assembly template for the try unlock functions. The basic functionality is: + * + * mov esi, %eax Move the TID into EAX + * xor %ecx, %ecx Clear ECX + * lock_cmpxchgl %ecx, (%rdi) Attempt the TID -> 0 transition + * .Lcs_start: Start of the critical section + * jnz .Lcs_end If cmpxchl failed jump to the end + * .Lcs_success: Start of the success section + * movq %rcx, (%rdx) Set the pending op pointer to 0 + * .Lcs_end: End of the critical section + * + * .Lcs_start and .Lcs_end establish the critical section range. .Lcs_success is + * technically not required, but there for illustration, debugging and testing. + * + * When CONFIG_COMPAT is enabled then the 64-bit VDSO provides two functions. + * One for the regular 64-bit sized pending operation pointer and one for a + * 32-bit sized pointer to support gaming emulators. + * + * The 32-bit VDSO provides only the one for 32-bit sized pointers. + */ +#define __stringify_1(x...) #x +#define __stringify(x...) __stringify_1(x) + +#define LABEL(prefix, which) __stringify(prefix##_try_unlock_cs_##which:) + +#define JNZ_END(prefix) "jnz " __stringify(prefix) "_try_unlock_cs_end\n" + +#define CLEAR_POPQ "movq %[zero], %a[pop]\n" +#define CLEAR_POPL "movl %k[zero], %a[pop]\n" + +#define futex_robust_try_unlock(prefix, clear_pop, __lock, __tid, __pop)\ +({ \ + asm volatile ( \ + " \n" \ + " lock cmpxchgl %k[zero], %a[lock] \n" \ + " \n" \ + LABEL(prefix, start) \ + " \n" \ + JNZ_END(prefix) \ + " \n" \ + LABEL(prefix, success) \ + " \n" \ + clear_pop \ + " \n" \ + LABEL(prefix, end) \ + : [tid] "+&a" (__tid) \ + : [lock] "D" (__lock), \ + [pop] "d" (__pop), \ + [zero] "r" (0UL) \ + : "memory" \ + ); \ + __tid; \ +}) + +#ifdef __x86_64__ +__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop) +{ + return futex_robust_try_unlock(__futex_list64, CLEAR_POPQ, lock, tid, pop); +} +#endif /* __x86_64__ */ + +#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT) +__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop) +{ + return futex_robust_try_unlock(__futex_list32, CLEAR_POPL, lock, tid, pop); +} +#endif /* CONFIG_X86_32 || CONFIG_COMPAT */ diff --git a/arch/x86/entry/vdso/vdso32/Makefile b/arch/x86/entry/vdso/vdso32/Makefile index ded4fc6a48cd..ab4b1f635f66 100644 --- a/arch/x86/entry/vdso/vdso32/Makefile +++ b/arch/x86/entry/vdso/vdso32/Makefile @@ -7,8 +7,9 @@ vdsos-y := 32 # Files to link into the vDSO: -vobjs-y := note.o vclock_gettime.o vgetcpu.o -vobjs-y += system_call.o sigreturn.o +vobjs-y := note.o vclock_gettime.o vgetcpu.o +vobjs-y += system_call.o sigreturn.o +vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o # Compilation flags flags-y := -DBUILD_VDSO32 -m32 -mregparm=0 diff --git a/arch/x86/entry/vdso/vdso32/vdso32.lds.S b/arch/x86/entry/vdso/vdso32/vdso32.lds.S index 55554f80d930..cee8f7f9fe80 100644 --- a/arch/x86/entry/vdso/vdso32/vdso32.lds.S +++ b/arch/x86/entry/vdso/vdso32/vdso32.lds.S @@ -30,6 +30,9 @@ VERSION __vdso_clock_gettime64; __vdso_clock_getres_time64; __vdso_getcpu; +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK + __vdso_futex_robust_list32_try_unlock; +#endif }; LINUX_2.5 { diff --git a/arch/x86/entry/vdso/vdso32/vfutex.c b/arch/x86/entry/vdso/vdso32/vfutex.c new file mode 100644 index 000000000000..940a6ee30026 --- /dev/null +++ b/arch/x86/entry/vdso/vdso32/vfutex.c @@ -0,0 +1 @@ +#include "common/vfutex.c" diff --git a/arch/x86/entry/vdso/vdso64/Makefile b/arch/x86/entry/vdso/vdso64/Makefile index bfffaf1aeecc..7c0790065b5e 100644 --- a/arch/x86/entry/vdso/vdso64/Makefile +++ b/arch/x86/entry/vdso/vdso64/Makefile @@ -8,9 +8,10 @@ vdsos-y := 64 vdsos-$(CONFIG_X86_X32_ABI) += x32 # Files to link into the vDSO: -vobjs-y := note.o vclock_gettime.o vgetcpu.o -vobjs-y += vgetrandom.o vgetrandom-chacha.o -vobjs-$(CONFIG_X86_SGX) += vsgx.o +vobjs-y := note.o vclock_gettime.o vgetcpu.o +vobjs-y += vgetrandom.o vgetrandom-chacha.o +vobjs-$(CONFIG_X86_SGX) += vsgx.o +vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o # Compilation flags flags-y := -DBUILD_VDSO64 -m64 -mcmodel=small diff --git a/arch/x86/entry/vdso/vdso64/vdso64.lds.S b/arch/x86/entry/vdso/vdso64/vdso64.lds.S index 5ce3f2b6373a..4a72122da81b 100644 --- a/arch/x86/entry/vdso/vdso64/vdso64.lds.S +++ b/arch/x86/entry/vdso/vdso64/vdso64.lds.S @@ -32,6 +32,13 @@ VERSION { #endif getrandom; __vdso_getrandom; + +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK + __vdso_futex_robust_list64_try_unlock; +#ifdef CONFIG_COMPAT + __vdso_futex_robust_list32_try_unlock; +#endif +#endif local: *; }; } diff --git a/arch/x86/entry/vdso/vdso64/vdsox32.lds.S b/arch/x86/entry/vdso/vdso64/vdsox32.lds.S index 3dbd20c8dacc..b917dc69f62f 100644 --- a/arch/x86/entry/vdso/vdso64/vdsox32.lds.S +++ b/arch/x86/entry/vdso/vdso64/vdsox32.lds.S @@ -22,6 +22,13 @@ VERSION { __vdso_getcpu; __vdso_time; __vdso_clock_getres; + +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK + __vdso_futex_robust_list64_try_unlock; +#ifdef CONFIG_COMPAT + __vdso_futex_robust_list32_try_unlock; +#endif +#endif local: *; }; } diff --git a/arch/x86/entry/vdso/vdso64/vfutex.c b/arch/x86/entry/vdso/vdso64/vfutex.c new file mode 100644 index 000000000000..940a6ee30026 --- /dev/null +++ b/arch/x86/entry/vdso/vdso64/vfutex.c @@ -0,0 +1 @@ +#include "common/vfutex.c" diff --git a/arch/x86/include/asm/futex_robust.h b/arch/x86/include/asm/futex_robust.h new file mode 100644 index 000000000000..e87954703ae2 --- /dev/null +++ b/arch/x86/include/asm/futex_robust.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_FUTEX_ROBUST_H +#define _ASM_X86_FUTEX_ROBUST_H + +#include + +static __always_inline void __user *x86_futex_robust_unlock_get_pop(struct pt_regs *regs) +{ + /* + * If ZF is set then the cmpxchg succeeded and the pending op pointer + * needs to be cleared. + */ + return regs->flags & X86_EFLAGS_ZF ? (void __user *)regs->dx : NULL; +} + +#define arch_futex_robust_unlock_get_pop(regs) \ + x86_futex_robust_unlock_get_pop(regs) + +#endif /* _ASM_X86_FUTEX_ROBUST_H */