diff --git a/fs/exec.c b/fs/exec.c index a14f28b15607..745f6eb5279e 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -854,7 +855,8 @@ static int exec_mmap(struct linux_binprm *bprm) /* Notify parent that we're no longer interested in the old VM */ tsk = current; old_mm = current->mm; - exec_mm_release(tsk, old_mm); + /* Clean up futexes and release the mm */ + mm_exit_exec_release(tsk, old_mm); ret = down_write_killable(&tsk->signal->exec_update_lock); if (ret) @@ -902,9 +904,10 @@ static int exec_mmap(struct linux_binprm *bprm) BUG_ON(active_mm != old_mm); /* Defer teardown to setup_new_exec(), outside the exec locks. */ bprm->old_mm = old_mm; - return 0; + } else { + mmdrop_lazy_tlb(active_mm); } - mmdrop_lazy_tlb(active_mm); + futex_exec_done(tsk); return 0; } diff --git a/include/linux/futex.h b/include/linux/futex.h index 51f4ccdc9092..18ed18d5cbc1 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -71,8 +71,8 @@ static inline void futex_init_task(struct task_struct *tsk) } void futex_exit_recursive(struct task_struct *tsk); -void futex_exit_release(struct task_struct *tsk); -void futex_exec_release(struct task_struct *tsk); +void futex_exit_exec_release(struct task_struct *tsk); +void futex_exec_done(struct task_struct *tsk); long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3); @@ -89,8 +89,8 @@ static inline int futex_hash_free(struct mm_struct *mm) { return 0; } #else /* CONFIG_FUTEX */ static inline void futex_init_task(struct task_struct *tsk) { } static inline void futex_exit_recursive(struct task_struct *tsk) { } -static inline void futex_exit_release(struct task_struct *tsk) { } -static inline void futex_exec_release(struct task_struct *tsk) { } +static inline void futex_exit_exec_release(struct task_struct *tsk) { } +static inline void futex_exec_done(struct task_struct *tsk) { } static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3) { diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 10be8a54b416..d7c6a942aa7e 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -155,10 +155,12 @@ extern struct mm_struct *get_task_mm(struct task_struct *task); * succeeds. */ extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); -/* Remove the current tasks stale references to the old mm_struct on exit() */ -extern void exit_mm_release(struct task_struct *, struct mm_struct *); -/* Remove the current tasks stale references to the old mm_struct on exec() */ -extern void exec_mm_release(struct task_struct *, struct mm_struct *); + +/* + * Remove the current tasks stale references to the old mm_struct on exit() and + * exec(). Cleans up futexes as well. + */ +extern void mm_exit_exec_release(struct task_struct *, struct mm_struct *); #ifdef CONFIG_MEMCG extern void mm_update_next_owner(struct mm_struct *mm); diff --git a/include/linux/wait.h b/include/linux/wait.h index dce055e6add3..7e215330199c 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -1228,6 +1228,7 @@ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_en void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); #define DEFINE_WAIT_FUNC(name, function) \ diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index ace7379d627d..553d7b23e3ad 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -32,6 +32,7 @@ int out_of_line_wait_on_bit_timeout(unsigned long *word, int, wait_bit_action_f int out_of_line_wait_on_bit_lock(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode); struct wait_queue_head *bit_waitqueue(unsigned long *word, int bit); extern void __init wait_bit_init(void); +extern struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg); int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); diff --git a/kernel/exit.c b/kernel/exit.c index 182c06671c78..97686af89501 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -581,7 +581,7 @@ static void exit_mm(void) { struct mm_struct *mm = current->mm; - exit_mm_release(current, mm); + mm_exit_exec_release(current, mm); if (!mm) return; diff --git a/kernel/fork.c b/kernel/fork.c index 1e68404bd773..e645675dd727 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1508,15 +1508,9 @@ static void mm_release(struct task_struct *tsk, struct mm_struct *mm) complete_vfork_done(tsk); } -void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm) +void mm_exit_exec_release(struct task_struct *tsk, struct mm_struct *mm) { - futex_exit_release(tsk); - mm_release(tsk, mm); -} - -void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm) -{ - futex_exec_release(tsk); + futex_exit_exec_release(tsk); mm_release(tsk, mm); } diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 7ab106616990..51ba5e1257c0 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -46,6 +46,7 @@ #include #include #include +#include #include @@ -1527,14 +1528,12 @@ static void futex_cleanup_begin(struct task_struct *tsk) raw_spin_unlock_irq(&tsk->pi_lock); } -static void futex_cleanup_end(struct task_struct *tsk, int state) +static void futex_cleanup_end(struct task_struct *tsk) __releases(&tsk->futex.exit_mutex) { - /* - * Lockless store. The only side effect is that an observer might - * take another loop until it becomes visible. - */ - tsk->futex.state = state; + scoped_guard(raw_spinlock_irq, &tsk->pi_lock) + tsk->futex.state = FUTEX_STATE_DEAD; + /* * Drop the exit protection. This unblocks waiters which observed * FUTEX_STATE_EXITING to reevaluate the state. @@ -1542,29 +1541,46 @@ static void futex_cleanup_end(struct task_struct *tsk, int state) mutex_unlock(&tsk->futex.exit_mutex); } -void futex_exec_release(struct task_struct *tsk) +/* + * Invoked from mm_exit_exec_release() to cleanup the robust lists and pi state + * of the outgoing task. + * + * exec() makes it interesting for futexes because the TID of the task stays the + * same, but from a futex perspective the task has to be treated like an exiting + * task. This is especially important for the sanity check for private futexes + * in attach_to_pi_owner() which compares the owner's mm with the waiter's mm. + * + * That check would give the wrong answer if futex_cleanup_end() would + * set the state to FUTEX_STATE_OK as long as the task still has the old + * mm. + * + * After the task has switched to the new mm it sets it to + * FUTEX_STATE_OK again in futex_exec_done(). + */ +void futex_exit_exec_release(struct task_struct *tsk) { - /* - * The state handling is done for consistency, but in the case of - * exec() there is no way to prevent further damage as the PID stays - * the same. But for the unlikely and arguably buggy case that a - * futex is held on exec(), this provides at least as much state - * consistency protection which is possible. - */ futex_cleanup_begin(tsk); futex_cleanup(tsk); - /* - * Reset the state to FUTEX_STATE_OK. The task is alive and about - * exec a new binary. - */ - futex_cleanup_end(tsk, FUTEX_STATE_OK); + futex_cleanup_end(tsk); } -void futex_exit_release(struct task_struct *tsk) +/* + * exec() has switched to the new mm. Futex operations are safe again. + */ +void futex_exec_done(struct task_struct *tsk) { - futex_cleanup_begin(tsk); - futex_cleanup(tsk); - futex_cleanup_end(tsk, FUTEX_STATE_DEAD); + /* + * This store does not have to take tsk::futex::exit_mutex because the + * phase where waiters block on it during state FUTEX_STATE_EXITING has + * been finished when futex_cleanup_end() set the state to + * FUTEX_STATE_DEAD. + * + * This transitions back from FUTEX_STATE_DEAD to FUTEX_STATE_OK. The + * ordering guarantee required here is that the previous store to + * tsk::mm in the calling code cannot be reordered against this store. + */ + guard(raw_spinlock_irq)(&tsk->pi_lock); + tsk->futex.state = FUTEX_STATE_OK; } static void futex_hash_bucket_init(struct futex_hash_bucket *fhb) @@ -1844,14 +1860,18 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) } if (!mm->futex.phash.ref) { - /* - * This will always be allocated by the first thread and - * therefore requires no locking. - */ - mm->futex.phash.ref = alloc_percpu(unsigned int); - if (!mm->futex.phash.ref) + unsigned int __percpu *ref = alloc_percpu(unsigned int); + + if (!ref) return -ENOMEM; - this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */ + + /* + * Tasks sharing the mm can run this concurrently, so take the + * initial reference before publishing the counter. + */ + this_cpu_inc(*ref); /* 0 -> 1 */ + if (cmpxchg(&mm->futex.phash.ref, NULL, ref)) + free_percpu(ref); } fph = kvzalloc(struct_size(fph, queues, hash_slots), @@ -1867,11 +1887,35 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) futex_hash_bucket_init(&fph->queues[i]); if (custom) { + struct wait_bit_queue_entry __wbq_entry; + struct wait_queue_head *__wq_head; + /* * Only let prctl() wait / retry; don't unduly delay clone(). */ again: - wait_var_event(mm, futex_pivot_pending(mm)); + __wq_head = __var_waitqueue(mm); + init_wait_var_entry(&__wbq_entry, mm, 0); + __wbq_entry.wq_entry.func = woken_wake_bit_function; + add_wait_queue(__wq_head, &__wbq_entry.wq_entry); + + /* + * add_wait_queue() futex_ref_put() + * MB (this) MB (implied) + * futex_pivot_pending() wake_up_var() + * waitqueue_active() + * + * Notably, it must not be possible to see + * !futex_pivot_pending() && !waitqueue_active(). + */ + smp_mb(); + + while (!futex_pivot_pending(mm) && + wait_woken(&__wbq_entry.wq_entry, TASK_UNINTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT)) + /* empty */; + + remove_wait_queue(__wq_head, &__wbq_entry.wq_entry); } scoped_guard(mutex, &mm->futex.phash.lock) { diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 795011ea1202..88788e584ec8 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -193,6 +193,58 @@ void put_pi_state(struct futex_pi_state *pi_state) * pi_mutex->wait_lock * p->pi_lock * + * Futex kernel state: + * + * The kernel tracks the task state in p::futex::state to protect against exit() + * and exec(). The states are: + * + * - FUTEX_STATE_OK when the task is alive and waiters can be attached + * + * - FUTEX_STATE_EXITING when the task cleans up the robust list and PI + * state. Concurrent waiters cannot attach anymore and have to wait until the + * cleanup is finished to re-evaluate the potential changes caused by the + * robust list and PI state cleanups. + * + * - FUTEX_STATE_DEAD when the task has cleaned up the robust list. This state + * is set independent of exit() or exec(). In the exit() case the task is + * gone. In the exec() case this ensures that nothing can attach to the task + * after cleaning up the robust list and PI state before it has switched to + * the new mm. From a futex point of view the task is dead until it sets the + * state to FUTEX_STATE_OK again after switching to the new mm. + * + * The valid state transitions for exit(): + * + * FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD + * + * The valid state transitions for exec(): + * + * FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD -> FUTEX_STATE_OK + * + * The state has two related locks: + * + * 1) p::pi_lock + * + * p::pi_lock has to be taken by the waiter when evaluating the state to + * protect against a concurrent exit/exec cleanup by the owner. If the state + * is OK then the waiter can be attached to the owner while still holding + * pi_lock. + * + * The cleanup code has to hold it for all state transitions to ensure that + * the stores to the state cannot be reordered against previous stores on + * which the waiter correctness depends on. + * + * 2) p::futex::exit_mutex + * + * The mutex is acquired when the cleanup starts and released at the end. It + * obviously is not serializing the owner's cleanup against itself. It is + * used to avoid a live lock caused by a waiter preempting the owner's + * cleanup. Such a waiter would busy loop forever waiting for the owner to + * finish the cleanup. + * + * To prevent this, waiters have to drop all locks when observing + * FUTEX_STATE_EXITING and block on the mutex. When the owner releases the + * mutex after finishing the cleanup the waiters make progress and + * re-evaluate the situation. */ /* @@ -318,18 +370,10 @@ static int attach_to_pi_state(u32 __user *uaddr, u32 uval, return ret; } -static int handle_exit_race(u32 __user *uaddr, u32 uval, - struct task_struct *tsk) +static int handle_exit_race(u32 __user *uaddr, u32 uval) { u32 uval2; - /* - * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the - * caller that the alleged owner is busy. - */ - if (tsk && tsk->futex.state != FUTEX_STATE_DEAD) - return -EBUSY; - /* * Reread the user space value to handle the following situation: * @@ -427,7 +471,7 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, return -EAGAIN; p = find_get_task_by_vpid(pid); if (!p) - return handle_exit_race(uaddr, uval, NULL); + return handle_exit_race(uaddr, uval); if (unlikely(p->flags & PF_KTHREAD)) { put_task_struct(p); @@ -435,36 +479,57 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, } /* - * We need to look at the task state to figure out, whether the - * task is exiting. To protect against the change of the task state - * in futex_exit_release(), we do this protected by p->pi_lock: + * We need to look at the task state to figure out whether the task is + * exiting. To protect against the change of the task state from + * FUTEX_STATE_OK to FUTEX_STATE_EXISTING in futex_cleanup_begin() it is + * required to do this protected by p->pi_lock, which prevents the owner + * from concurrently starting the exit cleanup. + * + * If the state is FUTEX_STATE_OK pi_lock must be held until the waiter + * is attached to protect against a concurrent exit()/exec(). */ raw_spin_lock_irq(&p->pi_lock); + + /* Validate that the task is ready for futex operations. */ if (unlikely(p->futex.state != FUTEX_STATE_OK)) { /* - * The task is on the way out. When the futex state is - * FUTEX_STATE_DEAD, we know that the task has finished - * the cleanup: + * The task is on the way out. When state is FUTEX_STATE_EXITING + * the cleanup is in progress. To avoid a live lock when the + * waiter preempted the owner, store the task pointer in + * @exiting and keep the reference on the task. The calling code + * will drop all locks, block on @p::futex::exit_mutex and wait + * for the owner to finish the cleanup. Once the owner released + * the mutex the waiter drops the reference count and + * re-evaluates the situation. */ - int ret = handle_exit_race(uaddr, uval, p); + if (p->futex.state == FUTEX_STATE_EXITING) { + raw_spin_unlock_irq(&p->pi_lock); + *exiting = p; + return -EBUSY; + } + + int ret = handle_exit_race(uaddr, uval); raw_spin_unlock_irq(&p->pi_lock); - /* - * If the owner task is between FUTEX_STATE_EXITING and - * FUTEX_STATE_DEAD then store the task pointer and keep - * the reference on the task struct. The calling code will - * drop all locks, wait for the task to reach - * FUTEX_STATE_DEAD and then drop the refcount. This is - * required to prevent a live lock when the current task - * preempted the exiting task between the two states. - */ - if (ret == -EBUSY) - *exiting = p; - else - put_task_struct(p); + put_task_struct(p); return ret; } + if (IS_ENABLED(CONFIG_MMU) && futex_key_is_private(key)) { + /* + * A private futex key holds a pointer to the waiter's mm + * without holding a reference on it. So it must not be attached + * to an owner in a different address space. Otherwise that + * owner's exit cleanup could access the private hash after the + * key's mm is freed. + */ + if (unlikely(p->mm != key->private.mm)) { + raw_spin_unlock_irq(&p->pi_lock); + put_task_struct(p); + return -EPERM; + } + } + __attach_to_pi_owner(p, key, ps); raw_spin_unlock_irq(&p->pi_lock); diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 20f27e2cf7ae..d033f600f48c 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -5,6 +5,7 @@ * (C) 2004 Nadia Yvette Chambers, Oracle */ #include "sched.h" +#include void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key) { @@ -463,3 +464,17 @@ int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sy return default_wake_function(wq_entry, mode, sync, key); } EXPORT_SYMBOL(woken_wake_function); + +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg) +{ + struct wait_bit_key *key = __var_wake_key(wq_entry, arg); + if (!key) + return 0; + + /* Pairs with the smp_store_mb() in wait_woken(). */ + smp_mb(); /* C */ + wq_entry->flags |= WQ_FLAG_WOKEN; + + return default_wake_function(wq_entry, mode, sync, key); +} +EXPORT_SYMBOL(woken_wake_bit_function); diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c index 1088d3b7012c..348f7211b4aa 100644 --- a/kernel/sched/wait_bit.c +++ b/kernel/sched/wait_bit.c @@ -167,9 +167,7 @@ wait_queue_head_t *__var_waitqueue(void *p) } EXPORT_SYMBOL(__var_waitqueue); -static int -var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, - int sync, void *arg) +struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg) { struct wait_bit_key *key = arg; struct wait_bit_queue_entry *wbq_entry = @@ -177,6 +175,16 @@ var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, if (wbq_entry->key.flags != key->flags || wbq_entry->key.bit_nr != key->bit_nr) + return NULL; + + return key; +} + +static int var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, + int sync, void *arg) +{ + struct wait_bit_key *key = __var_wake_key(wq_entry, arg); + if (!key) return 0; return autoremove_wake_function(wq_entry, mode, sync, key);