diff --git a/include/linux/futex.h b/include/linux/futex.h index 18ed18d5cbc1..a895d8c69d66 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -23,28 +23,32 @@ struct task_struct; * 01 : Shared futex (PTHREAD_PROCESS_SHARED) * mapped on a file (reference on the underlying inode) * 10 : Shared futex (PTHREAD_PROCESS_SHARED) - * (but private mapping on an mm, and reference taken on it) + * (but private mapping on an mm) */ -#define FUT_OFF_INODE 1 /* We set bit 0 if key has a reference on inode */ -#define FUT_OFF_MMSHARED 2 /* We set bit 1 if key has a reference on mm */ +#define FUT_OFF_INODE 1 /* We set bit 0 if shared key identifies an inode */ +#define FUT_OFF_MMSHARED 2 /* We set bit 1 if shared key identifies an mm */ union futex_key { + /* For FUT_OFF_INODE */ struct { u64 i_seq; unsigned long pgoff; unsigned int offset; /* unsigned int node; */ } shared; + + /* For FUT_OFF_MMSHARED or private */ struct { union { - struct mm_struct *mm; - u64 __tmp; + struct mm_struct *mm; /* for private */ + u64 mm_seq; /* for FUT_OFF_MMSHARED */ }; unsigned long address; unsigned int offset; /* unsigned int node; */ } private; + struct { u64 ptr; unsigned long word; @@ -152,11 +156,10 @@ static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned in static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { } #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ - -#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) +#ifdef CONFIG_FUTEX void futex_mm_init(struct mm_struct *mm); #else static inline void futex_mm_init(struct mm_struct *mm) { } -#endif +#endif /* !CONFIG_FUTEX */ #endif /* _LINUX_FUTEX_H */ diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h index d320c0571f0c..34b16f2828e7 100644 --- a/include/linux/futex_types.h +++ b/include/linux/futex_types.h @@ -85,10 +85,12 @@ struct futex_unlock_cs_ranges { }; * struct futex_mm_data - Futex related per MM data * @phash: Futex private hash related data * @unlock: Futex unlock VDSO critical sections + * @unique_id: Unique ID of the MM, for FUT_OFF_MMSHARED futexes */ struct futex_mm_data { struct futex_mm_phash phash; struct futex_unlock_cs_ranges unlock; + atomic64_t unique_id; }; #else /* CONFIG_FUTEX */ struct futex_sched_data { }; diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index 0bb6c03481fa..c74b5c87e10f 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -468,6 +468,25 @@ static inline void irqentry_exit_to_kernel_mode_preempt(struct pt_regs *regs, irqentry_exit_cond_resched(); } +/* + * The deferred hrtimer rearm is the tail of the timer interrupt, moved to the + * interrupt return path so that it runs after a possible reschedule. By now + * irq_exit_rcu() has left lockdep's hardirq context. Re-enter it for the + * rearm, so that the hrtimer base lock and the timekeeping sequence count are + * not chained onto the locks held by the interrupted context, which would add + * new lock chains for every lock stack an interrupt hits. + * + * The rearm on return to user mode needs no such annotation: no locks are + * held there, so it uses the same lock chains as the rearm in hardirq + * context. + */ +static __always_inline void irqentry_hrtimer_rearm_deferred(void) +{ + lockdep_hardirq_enter(); + hrtimer_rearm_deferred(); + lockdep_hardirq_exit(); +} + /** * irqentry_exit_to_kernel_mode_after_preempt - Establish trace, lockdep and RCU state * @regs: Pointer to current's pt_regs @@ -491,7 +510,7 @@ irqentry_exit_to_kernel_mode_after_preempt(struct pt_regs *regs, irqentry_state_ */ if (state.exit_rcu) { instrumentation_begin(); - hrtimer_rearm_deferred(); + irqentry_hrtimer_rearm_deferred(); /* Tell the tracer that IRET will enable interrupts */ trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); @@ -502,7 +521,7 @@ irqentry_exit_to_kernel_mode_after_preempt(struct pt_regs *regs, irqentry_state_ } instrumentation_begin(); - hrtimer_rearm_deferred(); + irqentry_hrtimer_rearm_deferred(); /* Covers both tracing and lockdep */ trace_hardirqs_on(); instrumentation_end(); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 095f9fe440e3..a240638f69e5 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -439,12 +439,35 @@ struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper } /* - * Generate a machine wide unique identifier for this inode. + * Generate a machine wide unique identifier for this object (inode or mm). * * This relies on u64 not wrapping in the life-time of the machine; which with * 1ns resolution means almost 585 years. - * - * This further relies on the fact that a well formed program will not unmap + */ +static u64 get_object_id(atomic64_t *object_seq) +{ + static atomic64_t last_assigned_seq; + u64 old; + + /* Does the object already have a sequence number? */ + old = atomic64_read(object_seq); + if (likely(old)) + return old; + + for (;;) { + u64 new = atomic64_inc_return(&last_assigned_seq); + if (WARN_ON_ONCE(!new)) + continue; + + old = 0; + if (!atomic64_try_cmpxchg_relaxed(object_seq, &old, new)) + return old; + return new; + } +} + +/* + * This relies on the fact that a well formed program will not unmap * the file while it has a (shared) futex waiting on it. This mapping will have * a file reference which pins the mount and inode. * @@ -458,24 +481,7 @@ struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper */ static u64 get_inode_sequence_number(struct inode *inode) { - static atomic64_t i_seq; - u64 old; - - /* Does the inode already have a sequence number? */ - old = atomic64_read(&inode->i_sequence); - if (likely(old)) - return old; - - for (;;) { - u64 new = atomic64_inc_return(&i_seq); - if (WARN_ON_ONCE(!new)) - continue; - - old = 0; - if (!atomic64_try_cmpxchg_relaxed(&inode->i_sequence, &old, new)) - return old; - return new; - } + return get_object_id(&inode->i_sequence); } /** @@ -683,8 +689,8 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, goto out; } - key->both.offset |= FUT_OFF_MMSHARED; /* ref taken on mm */ - key->private.mm = mm; + key->both.offset |= FUT_OFF_MMSHARED; + key->private.mm_seq = get_object_id(&mm->futex.unique_id); key->private.address = address; } else { @@ -2055,13 +2061,12 @@ static void futex_robust_unlock_init_mm(struct futex_mm_data *fd) static inline void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { } #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ -#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) void futex_mm_init(struct mm_struct *mm) { + atomic64_set(&mm->futex.unique_id, 0); futex_hash_init_mm(&mm->futex); futex_robust_unlock_init_mm(&mm->futex); } -#endif int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) {