diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa6d30ce73d1..14faa11ef23c 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -695,4 +695,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re void synchronize_rcu_trivial_preempt(void); #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU +#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +bool rcu_is_task_rcu_boosted(void); +#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +static inline bool rcu_is_task_rcu_boosted(void) { return false; } +#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) + + #endif /* __LINUX_RCU_H */ diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index ffeca1b7e170..1a033ae1f1eb 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -80,6 +80,7 @@ MODULE_AUTHOR("Paul E. McKenney and Josh Triplett is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() && + !in_serving_softirq() && !in_hardirq() && !in_nmi() && + READ_ONCE(firsttime) && xchg(&firsttime, 0)) { + WARN_ON_ONCE(deboost_timeliness_check); + nsegs = rtors.rtrsp - rtors.rtseg; + nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS); + pr_alert("Slow-deboost rcutorture reader segments:\n"); + rcu_torture_dump_read_segs(rtors.rtseg, nsegs); + } return true; } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3..d950722eb8f9 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1320,6 +1320,41 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ } +#ifdef CONFIG_RCU_TORTURE_TEST + +/* + * Is the current task RCU priority boosted? This is used by + * rcutorture to check that tasks are always deboosted once then exit + * an RCU read-side critical section, no matter how many overlapping + * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(), + * or local_irq_disable() made up that reader. + * + * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex) + * are safe because tasks release ->boost_mtx when they own it, they + * cannot be boosted unless current->rcu_blocked_node is non-NULL, + * current->rcu_blocked_node is modified only by the current task, + * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner + * switching among other tasks cannot force an equality comparison. + */ +bool rcu_is_task_rcu_boosted(void) +{ + bool ret; + struct rcu_node *rnp; + struct task_struct *t = current; + + preempt_disable(); // Stabilize ->rcu_blocked_node + rnp = t->rcu_blocked_node; + if (!rnp) + ret = false; + else + ret = (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t); + preempt_enable(); + return ret; +} +EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted); + +#endif // #ifdef CONFIG_RCU_TORTURE_TEST + #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)