From 59b3732f95dda1fbd2234514d35f4fb6b5bb6d85 Mon Sep 17 00:00:00 2001 From: Kyle Zeng Date: Fri, 7 Aug 2026 17:07:03 +0200 Subject: [PATCH 1/6] futex/pi: Reject cross-mm private futex owners A private futex key borrows the waiter's mm without taking an mm_users reference. Nevertheless, attach_to_pi_owner() currently accepts an owner from a different address space and copies the private key into the owner's PI state. When that owner exits, exit_pi_state_list() uses the saved key to find the hash bucket and acquires a reference to the waiter's private hash. If the last user of the waiter's mm exits concurrently, futex_hash_free() frees the hash while the owner still uses its bucket and reference. Prevent this by validating in attach_to_pi_owner() that, for private futexes, the owner mm and waiter mm are the same. Perform the check with the owner's pi_lock held and after validating owner::futex::state to serialize against a concurrent PI-state exit cleanup. [ tglx: Amended comment ] Fixes: 80367ad01d93 ("futex: Add basic infrastructure for local task local hash") Signed-off-by: Kyle Zeng Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra Assisted-by: Codex:gpt-5.6-sol Cc: stable@vger.kernel.org --- kernel/futex/pi.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 795011ea1202..3e277ef155cd 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -465,6 +465,26 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, return ret; } + /* + * If the owner is about to exit() or exec() and tries to modify + * p::futex::exit_state it is serialized against this code by + * p::pi_lock. + */ + if (IS_ENABLED(CONFIG_MMU) && futex_key_is_private(key)) { + /* + * A private futex key holds a pointer to the waiter's mm + * without holding a reference on it. So it must not be attached + * to an owner in a different address space. Otherwise that + * owner's exit cleanup could access the private hash after the + * key's mm is freed. + */ + if (unlikely(p->mm != key->private.mm)) { + raw_spin_unlock_irq(&p->pi_lock); + put_task_struct(p); + return -EPERM; + } + } + __attach_to_pi_owner(p, key, ps); raw_spin_unlock_irq(&p->pi_lock); From f9ece060cc43eae8a1f148737d193ba0d07b8f88 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Aug 2026 17:07:08 +0200 Subject: [PATCH 2/6] futex: Sanitize and document task_struct::futex::state transitions The futex state is used to prevent a waiter from attaching to the lock owner while the owner runs the futex cleanup in exit() or exec(). Only the state transition from FUTEX_STATE_OK to FUTEX_STATE_EXITING must be done with the task's pi_lock held, the transition away from FUTEX_STATE_EXITING has no serialization requirements on the writer side, but it's completely non obvious why. It's magically protected by exit_pi_state(), which operates under tsk::pi_lock, as that's the state which has to be correct when the waiter observes the new state. OTOH, taking the pi_lock in futex_cleanup_end() is not a performance issue because at that point the lock should be uncontended in the vast majority of cases. Aside of that the handling of FUTEX_STATE_EXITING in attach_to_pi_owner() and handle_exit_race() is confusing at best. Protect the store in futex_cleanup_end() with tsk::pi_lock, handle FUTEX_STATE_EXITING in attach_to_pi_owner() explicitly and document how this is supposed to work. Reported-by: Peter Zijlstra Signed-off-by: Thomas Gleixner Reviewed-by: Kyle Zeng Acked-by: Peter Zijlstra Cc: stable@vger.kernel.org --- kernel/futex/core.c | 8 ++-- kernel/futex/pi.c | 105 +++++++++++++++++++++++++++++--------------- 2 files changed, 73 insertions(+), 40 deletions(-) diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 128c5752f225..0ea2c1a89f6c 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1527,11 +1527,9 @@ static void futex_cleanup_begin(struct task_struct *tsk) static void futex_cleanup_end(struct task_struct *tsk, int state) __releases(&tsk->futex.exit_mutex) { - /* - * Lockless store. The only side effect is that an observer might - * take another loop until it becomes visible. - */ - tsk->futex.state = state; + scoped_guard(raw_spinlock_irq, &tsk->pi_lock) + tsk->futex.state = state; + /* * Drop the exit protection. This unblocks waiters which observed * FUTEX_STATE_EXITING to reevaluate the state. diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 3e277ef155cd..2731e55f99e5 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -193,6 +193,48 @@ void put_pi_state(struct futex_pi_state *pi_state) * pi_mutex->wait_lock * p->pi_lock * + * Futex kernel state: + * + * The kernel tracks the task state in p::futex::state to protect against exit() + * and exec(). The states are: + * + * - FUTEX_STATE_OK when the task is alive and waiters can be attached + * + * - FUTEX_STATE_EXITING when the task cleans up the robust list and pi + * state. Concurrent waiters cannot attach anymore and have to wait until the + * cleanup is finished to re-evaluate the potential changes of robust list and + * pi state cleanups. + * + * - FUTEX_STATE_DEAD when the task has cleaned up the robust list and + * is about to fully exit. + * + * exec() switches back to FUTEX_STATE_OK after the cleanup. + * + * The state has two related locks: + * + * 1) p::pi_lock + * + * p::pi_lock has to be taken by the waiter when evaluating the state to + * protect against a concurrent exit/exec cleanup by the owner. If the state + * is OK then the waiter can be attached to the owner while still holding + * pi_lock. + * + * The cleanup code has to hold it for all state transitions to ensure that + * the stores to the state cannot be reordered against previous stores on + * which the waiter correctness depends on. + * + * 2) p::futex::exit_mutex + * + * The mutex is acquired when the cleanup starts and released at the end. It + * obviously is not serializing the owner's cleanup against itself. It is + * used to avoid a live lock caused by a waiter preempting the owner's + * cleanup. Such a waiter would busy loop forever waiting for the owner to + * finish the cleanup. + * + * To prevent this, waiters have to drop all locks when observing + * FUTEX_STATE_EXITING and block on the mutex. When the owner releases the + * mutex after finishing the cleanup the waiters make progress and + * re-evaluate the situation. */ /* @@ -318,18 +360,10 @@ static int attach_to_pi_state(u32 __user *uaddr, u32 uval, return ret; } -static int handle_exit_race(u32 __user *uaddr, u32 uval, - struct task_struct *tsk) +static int handle_exit_race(u32 __user *uaddr, u32 uval) { u32 uval2; - /* - * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the - * caller that the alleged owner is busy. - */ - if (tsk && tsk->futex.state != FUTEX_STATE_DEAD) - return -EBUSY; - /* * Reread the user space value to handle the following situation: * @@ -427,7 +461,7 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, return -EAGAIN; p = find_get_task_by_vpid(pid); if (!p) - return handle_exit_race(uaddr, uval, NULL); + return handle_exit_race(uaddr, uval); if (unlikely(p->flags & PF_KTHREAD)) { put_task_struct(p); @@ -435,41 +469,42 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, } /* - * We need to look at the task state to figure out, whether the - * task is exiting. To protect against the change of the task state - * in futex_exit_release(), we do this protected by p->pi_lock: + * We need to look at the task state to figure out whether the task is + * exiting. To protect against the change of the task state from + * FUTEX_STATE_OK to FUTEX_STATE_EXISTING in futex_cleanup_begin() it is + * required to do this protected by p->pi_lock, which prevents the owner + * from concurrently starting the exit cleanup. + * + * If the state is FUTEX_STATE_OK pi_lock must be held until the waiter + * is attached to protect against a concurrent exit()/exec(). */ raw_spin_lock_irq(&p->pi_lock); + + /* Validate that the task is ready for futex operations. */ if (unlikely(p->futex.state != FUTEX_STATE_OK)) { /* - * The task is on the way out. When the futex state is - * FUTEX_STATE_DEAD, we know that the task has finished - * the cleanup: + * The task is on the way out. When state is FUTEX_STATE_EXITING + * the cleanup is in progress. To avoid a live lock when the + * waiter preempted the owner, store the task pointer in + * @exiting and keep the reference on the task. The calling code + * will drop all locks, block on @p::futex::exit_mutex and wait + * for the owner to finish the cleanup. Once the owner released + * the mutex the waiter drops the reference count and + * re-evaluates the situation. */ - int ret = handle_exit_race(uaddr, uval, p); + if (p->futex.state == FUTEX_STATE_EXITING) { + raw_spin_unlock_irq(&p->pi_lock); + *exiting = p; + return -EBUSY; + } + + int ret = handle_exit_race(uaddr, uval); raw_spin_unlock_irq(&p->pi_lock); - /* - * If the owner task is between FUTEX_STATE_EXITING and - * FUTEX_STATE_DEAD then store the task pointer and keep - * the reference on the task struct. The calling code will - * drop all locks, wait for the task to reach - * FUTEX_STATE_DEAD and then drop the refcount. This is - * required to prevent a live lock when the current task - * preempted the exiting task between the two states. - */ - if (ret == -EBUSY) - *exiting = p; - else - put_task_struct(p); + put_task_struct(p); return ret; } - /* - * If the owner is about to exit() or exec() and tries to modify - * p::futex::exit_state it is serialized against this code by - * p::pi_lock. - */ if (IS_ENABLED(CONFIG_MMU) && futex_key_is_private(key)) { /* * A private futex key holds a pointer to the waiter's mm From c5f0bc9fd1cec4a00400cc727fcde03e0fde17cc Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Aug 2026 17:07:13 +0200 Subject: [PATCH 3/6] futex/pi: Plug private futex exec() race The check for private futexes whether the waiter's mm, which is stored in the futex_key and copied into the pi_state, is the same as the owner's mm is not sufficient for exec(). exec() has a gap where the mm check fails to give the correct answer: exec() ... exec_release_mm() futex_exec_release() tsk::futex::exit_state = EXITING; cleanup_robust_list(); 1) tsk::futex::exit_state = OK; ... old_mm = tsk::mm; 2) tsk::mm = ->mm; Between #1 and #2 the check for the mm is wrong as that mm is about to be swapped out and eventually freed. Plug this gap by: 1) Setting tsk::futex::exit_state to FUTEX_STATE_DEAD in futex_exec_release() 2) Setting tsk::futex::exit_state to FUTEX_STATE_OK after the mm has been switched. From a futex point of view the task is dead after it finished the robust list cleanup up to the point where it sets the state to OK again. Fixes: 80367ad01d93 ("futex: Add basic infrastructure for local task local hash") Signed-off-by: Thomas Gleixner Reviewed-by: Kyle Zeng Acked-by: Peter Zijlstra Cc: stable@vger.kernel.org --- fs/exec.c | 7 +++-- include/linux/futex.h | 2 ++ kernel/futex/core.c | 62 ++++++++++++++++++++++++++++--------------- kernel/futex/pi.c | 22 ++++++++++----- 4 files changed, 64 insertions(+), 29 deletions(-) diff --git a/fs/exec.c b/fs/exec.c index c7b8f2d6366c..d01523d0d8b4 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -854,6 +855,7 @@ static int exec_mmap(struct linux_binprm *bprm) /* Notify parent that we're no longer interested in the old VM */ tsk = current; old_mm = current->mm; + /* Clean up futexes and release the mm */ exec_mm_release(tsk, old_mm); ret = down_write_killable(&tsk->signal->exec_update_lock); @@ -902,9 +904,10 @@ static int exec_mmap(struct linux_binprm *bprm) BUG_ON(active_mm != old_mm); /* Defer teardown to setup_new_exec(), outside the exec locks. */ bprm->old_mm = old_mm; - return 0; + } else { + mmdrop_lazy_tlb(active_mm); } - mmdrop_lazy_tlb(active_mm); + futex_exec_done(tsk); return 0; } diff --git a/include/linux/futex.h b/include/linux/futex.h index 51f4ccdc9092..51d5faa1266f 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -73,6 +73,7 @@ static inline void futex_init_task(struct task_struct *tsk) void futex_exit_recursive(struct task_struct *tsk); void futex_exit_release(struct task_struct *tsk); void futex_exec_release(struct task_struct *tsk); +void futex_exec_done(struct task_struct *tsk); long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3); @@ -91,6 +92,7 @@ static inline void futex_init_task(struct task_struct *tsk) { } static inline void futex_exit_recursive(struct task_struct *tsk) { } static inline void futex_exit_release(struct task_struct *tsk) { } static inline void futex_exec_release(struct task_struct *tsk) { } +static inline void futex_exec_done(struct task_struct *tsk) { } static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3) { diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 0ea2c1a89f6c..3c1562df0370 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1524,11 +1524,11 @@ static void futex_cleanup_begin(struct task_struct *tsk) raw_spin_unlock_irq(&tsk->pi_lock); } -static void futex_cleanup_end(struct task_struct *tsk, int state) +static void futex_cleanup_end(struct task_struct *tsk) __releases(&tsk->futex.exit_mutex) { scoped_guard(raw_spinlock_irq, &tsk->pi_lock) - tsk->futex.state = state; + tsk->futex.state = FUTEX_STATE_DEAD; /* * Drop the exit protection. This unblocks waiters which observed @@ -1537,29 +1537,49 @@ static void futex_cleanup_end(struct task_struct *tsk, int state) mutex_unlock(&tsk->futex.exit_mutex); } -void futex_exec_release(struct task_struct *tsk) -{ - /* - * The state handling is done for consistency, but in the case of - * exec() there is no way to prevent further damage as the PID stays - * the same. But for the unlikely and arguably buggy case that a - * futex is held on exec(), this provides at least as much state - * consistency protection which is possible. - */ - futex_cleanup_begin(tsk); - futex_cleanup(tsk); - /* - * Reset the state to FUTEX_STATE_OK. The task is alive and about - * exec a new binary. - */ - futex_cleanup_end(tsk, FUTEX_STATE_OK); -} - void futex_exit_release(struct task_struct *tsk) { futex_cleanup_begin(tsk); futex_cleanup(tsk); - futex_cleanup_end(tsk, FUTEX_STATE_DEAD); + futex_cleanup_end(tsk); +} + +void futex_exec_release(struct task_struct *tsk) +{ + /* + * exec() makes it interesting for futexes because the TID of the task + * stays the same, but from a futex perspective the task has to be + * treated like an exiting task. This is especially important for the + * sanity check for private futexes in attach_to_pi_owner() which + * compares the owner's mm with the waiter's mm. + * + * That check would give the wrong answer if futex_cleanup_end() would + * set the state to FUTEX_STATE_OK as long as the task still has the old + * mm. + * + * After the task has switched to the new mm it sets it to + * FUTEX_STATE_OK again in futex_exec_done(). + */ + futex_exit_release(tsk); +} + +/* + * exec() has switched to the new mm. Futex operations are safe again. + */ +void futex_exec_done(struct task_struct *tsk) +{ + /* + * This store does not have to take tsk::futex::exit_mutex because the + * phase where waiters block on it during state FUTEX_STATE_EXITING has + * been finished when futex_cleanup_end() set the state to + * FUTEX_STATE_DEAD. + * + * This transitions back from FUTEX_STATE_DEAD to FUTEX_STATE_OK. The + * ordering guarantee required here is that the previous store to + * tsk::mm in the calling code cannot be reordered against this store. + */ + guard(raw_spinlock_irq)(&tsk->pi_lock); + tsk->futex.state = FUTEX_STATE_OK; } static void futex_hash_bucket_init(struct futex_hash_bucket *fhb) diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 2731e55f99e5..88788e584ec8 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -200,15 +200,25 @@ void put_pi_state(struct futex_pi_state *pi_state) * * - FUTEX_STATE_OK when the task is alive and waiters can be attached * - * - FUTEX_STATE_EXITING when the task cleans up the robust list and pi + * - FUTEX_STATE_EXITING when the task cleans up the robust list and PI * state. Concurrent waiters cannot attach anymore and have to wait until the - * cleanup is finished to re-evaluate the potential changes of robust list and - * pi state cleanups. + * cleanup is finished to re-evaluate the potential changes caused by the + * robust list and PI state cleanups. * - * - FUTEX_STATE_DEAD when the task has cleaned up the robust list and - * is about to fully exit. + * - FUTEX_STATE_DEAD when the task has cleaned up the robust list. This state + * is set independent of exit() or exec(). In the exit() case the task is + * gone. In the exec() case this ensures that nothing can attach to the task + * after cleaning up the robust list and PI state before it has switched to + * the new mm. From a futex point of view the task is dead until it sets the + * state to FUTEX_STATE_OK again after switching to the new mm. * - * exec() switches back to FUTEX_STATE_OK after the cleanup. + * The valid state transitions for exit(): + * + * FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD + * + * The valid state transitions for exec(): + * + * FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD -> FUTEX_STATE_OK * * The state has two related locks: * From 221b62e97811845340c888993ae049467399d2b5 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Aug 2026 17:07:17 +0200 Subject: [PATCH 4/6] futex: Clean up the redundant exit/exec functions futex_exit_release() and futex_exec_release() are identical now. That means also exit_mm_release() and exec_mm_release() are identical. Consolidate the whole lot and remove the redundant copies. Signed-off-by: Thomas Gleixner Reviewed-by: Kyle Zeng Acked-by: Peter Zijlstra --- fs/exec.c | 2 +- include/linux/futex.h | 6 ++---- include/linux/sched/mm.h | 10 ++++++---- kernel/exit.c | 2 +- kernel/fork.c | 10 ++-------- kernel/futex/core.c | 37 +++++++++++++++++-------------------- 6 files changed, 29 insertions(+), 38 deletions(-) diff --git a/fs/exec.c b/fs/exec.c index d01523d0d8b4..22df492e38fe 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -856,7 +856,7 @@ static int exec_mmap(struct linux_binprm *bprm) tsk = current; old_mm = current->mm; /* Clean up futexes and release the mm */ - exec_mm_release(tsk, old_mm); + mm_exit_exec_release(tsk, old_mm); ret = down_write_killable(&tsk->signal->exec_update_lock); if (ret) diff --git a/include/linux/futex.h b/include/linux/futex.h index 51d5faa1266f..18ed18d5cbc1 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -71,8 +71,7 @@ static inline void futex_init_task(struct task_struct *tsk) } void futex_exit_recursive(struct task_struct *tsk); -void futex_exit_release(struct task_struct *tsk); -void futex_exec_release(struct task_struct *tsk); +void futex_exit_exec_release(struct task_struct *tsk); void futex_exec_done(struct task_struct *tsk); long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, @@ -90,8 +89,7 @@ static inline int futex_hash_free(struct mm_struct *mm) { return 0; } #else /* CONFIG_FUTEX */ static inline void futex_init_task(struct task_struct *tsk) { } static inline void futex_exit_recursive(struct task_struct *tsk) { } -static inline void futex_exit_release(struct task_struct *tsk) { } -static inline void futex_exec_release(struct task_struct *tsk) { } +static inline void futex_exit_exec_release(struct task_struct *tsk) { } static inline void futex_exec_done(struct task_struct *tsk) { } static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3) diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df584..7fe7dfd6f8be 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -155,10 +155,12 @@ extern struct mm_struct *get_task_mm(struct task_struct *task); * succeeds. */ extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); -/* Remove the current tasks stale references to the old mm_struct on exit() */ -extern void exit_mm_release(struct task_struct *, struct mm_struct *); -/* Remove the current tasks stale references to the old mm_struct on exec() */ -extern void exec_mm_release(struct task_struct *, struct mm_struct *); + +/* + * Remove the current tasks stale references to the old mm_struct on exit() and + * exec(). Cleans up futexes as well. + */ +extern void mm_exit_exec_release(struct task_struct *, struct mm_struct *); #ifdef CONFIG_MEMCG extern void mm_update_next_owner(struct mm_struct *mm); diff --git a/kernel/exit.c b/kernel/exit.c index 2c0b1c02920f..adf93b90315d 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -582,7 +582,7 @@ static void exit_mm(void) { struct mm_struct *mm = current->mm; - exit_mm_release(current, mm); + mm_exit_exec_release(current, mm); if (!mm) return; diff --git a/kernel/fork.c b/kernel/fork.c index f0e2e131a9a5..7c6918b468b7 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1502,15 +1502,9 @@ static void mm_release(struct task_struct *tsk, struct mm_struct *mm) complete_vfork_done(tsk); } -void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm) +void mm_exit_exec_release(struct task_struct *tsk, struct mm_struct *mm) { - futex_exit_release(tsk); - mm_release(tsk, mm); -} - -void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm) -{ - futex_exec_release(tsk); + futex_exit_exec_release(tsk); mm_release(tsk, mm); } diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 3c1562df0370..2d1dbde54362 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1537,32 +1537,29 @@ static void futex_cleanup_end(struct task_struct *tsk) mutex_unlock(&tsk->futex.exit_mutex); } -void futex_exit_release(struct task_struct *tsk) +/* + * Invoked from mm_exit_exec_release() to cleanup the robust lists and pi state + * of the outgoing task. + * + * exec() makes it interesting for futexes because the TID of the task stays the + * same, but from a futex perspective the task has to be treated like an exiting + * task. This is especially important for the sanity check for private futexes + * in attach_to_pi_owner() which compares the owner's mm with the waiter's mm. + * + * That check would give the wrong answer if futex_cleanup_end() would + * set the state to FUTEX_STATE_OK as long as the task still has the old + * mm. + * + * After the task has switched to the new mm it sets it to + * FUTEX_STATE_OK again in futex_exec_done(). + */ +void futex_exit_exec_release(struct task_struct *tsk) { futex_cleanup_begin(tsk); futex_cleanup(tsk); futex_cleanup_end(tsk); } -void futex_exec_release(struct task_struct *tsk) -{ - /* - * exec() makes it interesting for futexes because the TID of the task - * stays the same, but from a futex perspective the task has to be - * treated like an exiting task. This is especially important for the - * sanity check for private futexes in attach_to_pi_owner() which - * compares the owner's mm with the waiter's mm. - * - * That check would give the wrong answer if futex_cleanup_end() would - * set the state to FUTEX_STATE_OK as long as the task still has the old - * mm. - * - * After the task has switched to the new mm it sets it to - * FUTEX_STATE_OK again in futex_exec_done(). - */ - futex_exit_release(tsk); -} - /* * exec() has switched to the new mm. Futex operations are safe again. */ From bde0238083647381d4747355c5a19115a3422b96 Mon Sep 17 00:00:00 2001 From: Hyunwoo Kim Date: Tue, 11 Aug 2026 23:03:16 +0900 Subject: [PATCH 5/6] futex: Fix race on the initial mm->futex.phash.ref allocation futex_hash_allocate() allocates mm->futex.phash.ref without any locking. Commit d9b05321e21e ("futex: Move futex_hash_free() back to __mmput()") moved the allocation here and assumed that the process has just a single thread at this point. Commit ee9dce44362b ("futex: Drop CLONE_THREAD requirement for private default hash alloc") widened need_futex_hash_allocate_default() to cover any CLONE_VM clone, but left out vfork because the parent is suspended and cannot race. That no longer holds once vfork is nested. If a vfork child calls vfork again and is then killed with SIGKILL, the parent is released from its vfork wait and runs concurrently with the grandchild in the same mm. Neither of them went through futex_hash_allocate_default(). When both call prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS) at the same time, each one sees mm->futex.phash.ref as NULL and stores its own percpu counter. Only the last store survives. The counter stored first is no longer reachable from the mm, so the references on it are not seen by __futex_ref_atomic_end(). A private hash that still has references is then considered dead and freed, and a task that still holds one of its buckets writes into freed memory in futex_q_lock(). Store the counter once with cmpxchg() and let the loser free_percpu() its own. The initial reference has to be taken before the store, otherwise another task can install a private hash while the counter is still 0. Fixes: d9b05321e21e ("futex: Move futex_hash_free() back to __mmput()") Signed-off-by: Hyunwoo Kim Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://patch.msgid.link/ansrpP4ImE1MaBY9@v4bel --- kernel/futex/core.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 2d1dbde54362..b38222e81879 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -1857,14 +1857,18 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) } if (!mm->futex.phash.ref) { - /* - * This will always be allocated by the first thread and - * therefore requires no locking. - */ - mm->futex.phash.ref = alloc_percpu(unsigned int); - if (!mm->futex.phash.ref) + unsigned int __percpu *ref = alloc_percpu(unsigned int); + + if (!ref) return -ENOMEM; - this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ + + /* + * Tasks sharing the mm can run this concurrently, so take the + * initial reference before publishing the counter. + */ + this_cpu_inc(*ref); /* 0 -> 1 */ + if (cmpxchg(&mm->futex.phash.ref, NULL, ref)) + free_percpu(ref); } fph = kvzalloc(struct_size(fph, queues, hash_slots), From d8aa5dd97944a72d4a9e3cc79bb80fcac7d6e829 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 20 Aug 2026 09:49:27 +0200 Subject: [PATCH 6/6] futex: Fix might_sleep() warning in futex_pivot_pending() A younger me put a WARN in might_sleep() to warn about nested sleep loops. This younger me also build a wait-loop variant that can deal with it. This wait-loop variant doesn't have all the fancy wrappers, since it isn't used much. It also lacks wait-bit support. Add the wait-bit support and use it to fix the nested wait issue. Fixes: 8e7ff730dd96 ("futex: Fix race in futex_pivot_pending() during private hash resize") Reported-by: syzbot+350a93852ac854927f45@syzkaller.appspotmail.com Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260820074927.GH1246887@noisy.programming.kicks-ass.net Closes: https://syzkaller.appspot.com/bug?extid=350a93852ac854927f45 --- include/linux/wait.h | 1 + include/linux/wait_bit.h | 1 + kernel/futex/core.c | 27 ++++++++++++++++++++++++++- kernel/sched/wait.c | 15 +++++++++++++++ kernel/sched/wait_bit.c | 14 +++++++++++--- 5 files changed, 54 insertions(+), 4 deletions(-) diff --git a/include/linux/wait.h b/include/linux/wait.h index dce055e6add3..7e215330199c 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -1228,6 +1228,7 @@ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_en void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); #define DEFINE_WAIT_FUNC(name, function) \ diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index ace7379d627d..553d7b23e3ad 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -32,6 +32,7 @@ int out_of_line_wait_on_bit_timeout(unsigned long *word, int, wait_bit_action_f int out_of_line_wait_on_bit_lock(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode); struct wait_queue_head *bit_waitqueue(unsigned long *word, int bit); extern void __init wait_bit_init(void); +extern struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg); int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index b38222e81879..f7af97c57d16 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -45,6 +45,7 @@ #include #include #include +#include #include @@ -1884,11 +1885,35 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) futex_hash_bucket_init(&fph->queues[i]); if (custom) { + struct wait_bit_queue_entry __wbq_entry; + struct wait_queue_head *__wq_head; + /* * Only let prctl() wait / retry; don't unduly delay clone(). */ again: - wait_var_event(mm, futex_pivot_pending(mm)); + __wq_head = __var_waitqueue(mm); + init_wait_var_entry(&__wbq_entry, mm, 0); + __wbq_entry.wq_entry.func = woken_wake_bit_function; + add_wait_queue(__wq_head, &__wbq_entry.wq_entry); + + /* + * add_wait_queue() futex_ref_put() + * MB (this) MB (implied) + * futex_pivot_pending() wake_up_var() + * waitqueue_active() + * + * Notably, it must not be possible to see + * !futex_pivot_pending() && !waitqueue_active(). + */ + smp_mb(); + + while (!futex_pivot_pending(mm) && + wait_woken(&__wbq_entry.wq_entry, TASK_UNINTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT)) + /* empty */; + + remove_wait_queue(__wq_head, &__wbq_entry.wq_entry); } scoped_guard(mutex, &mm->futex.phash.lock) { diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 20f27e2cf7ae..d033f600f48c 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -5,6 +5,7 @@ * (C) 2004 Nadia Yvette Chambers, Oracle */ #include "sched.h" +#include void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key) { @@ -463,3 +464,17 @@ int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sy return default_wake_function(wq_entry, mode, sync, key); } EXPORT_SYMBOL(woken_wake_function); + +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg) +{ + struct wait_bit_key *key = __var_wake_key(wq_entry, arg); + if (!key) + return 0; + + /* Pairs with the smp_store_mb() in wait_woken(). */ + smp_mb(); /* C */ + wq_entry->flags |= WQ_FLAG_WOKEN; + + return default_wake_function(wq_entry, mode, sync, key); +} +EXPORT_SYMBOL(woken_wake_bit_function); diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c index 1088d3b7012c..348f7211b4aa 100644 --- a/kernel/sched/wait_bit.c +++ b/kernel/sched/wait_bit.c @@ -167,9 +167,7 @@ wait_queue_head_t *__var_waitqueue(void *p) } EXPORT_SYMBOL(__var_waitqueue); -static int -var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, - int sync, void *arg) +struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg) { struct wait_bit_key *key = arg; struct wait_bit_queue_entry *wbq_entry = @@ -177,6 +175,16 @@ var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, if (wbq_entry->key.flags != key->flags || wbq_entry->key.bit_nr != key->bit_nr) + return NULL; + + return key; +} + +static int var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, + int sync, void *arg) +{ + struct wait_bit_key *key = __var_wake_key(wq_entry, arg); + if (!key) return 0; return autoremove_wake_function(wq_entry, mode, sync, key);