diff --git a/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst b/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst index 414f8a2012d6..cf0f9cdca7e8 100644 --- a/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst +++ b/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst @@ -410,7 +410,7 @@ workqueues (see Documentation/core-api/workqueue.rst). The requesting task still does counter snapshotting and funnel-lock processing, but the task reaching the top of the funnel lock does a -``schedule_work()`` (from ``_synchronize_rcu_expedited()`` so that a +``schedule_work()`` (from ``_synchronize_rcu_expedited()``) so that a workqueue kthread does the actual grace-period processing. Because workqueue kthreads do not accept POSIX signals, grace-period-wait processing need not allow for POSIX signals. In addition, this approach diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg index d05bc7b27edb..95a66de40ca5 100644 --- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg +++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg @@ -3933,7 +3933,7 @@ font-style="normal" y="-3914.085" x="3745.7725" - xml:space="preserve">rcu__report_qs_rdp()) + xml:space="preserve">rcu__report_qs_rdp() rcu__report_qs_rdp()) + xml:space="preserve">rcu__report_qs_rdp() a = 42; /* Each field in its own cache line. */ @@ -293,7 +293,7 @@ Then one approach is to use locking, for example, as follows:: { struct foo *p; - p = kmalloc(...); + p = kmalloc_obj(*p); if (p == NULL) deal_with_it(); spin_lock(&p->lock); diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f22..40d2831add4a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6117,6 +6117,11 @@ Kernel parameters the number of CPUs. For example, -2 selects N (the number of CPUs), -3 selects N+1, and so on. + rcutorture.nwriters= [KNL] + Set number of RCU writers, which must be either + zero or one. For additional writers, use instead + the rcutorture.nfakewriters parameter. + rcutorture.object_debug= [KNL] Enable debug-object double-call_rcu() testing. @@ -6206,6 +6211,15 @@ Kernel parameters and stall_gp_kthread are specified, the kthread is starved first, then the CPU. + rcutorture.stall_only= [KNL] + Shut off all rcutorture kthreads other than the + RCU CPU stall-warning test kthreads. The purpose + of this is to test production applictions' + reactions to CPU stalls, and with minimal + additional overhead. Or you can omit the + stall-warning tests as well and get a heavy + no-op, your choice! + rcutorture.stat_interval= [KNL] Time (s) between statistics printk()s. diff --git a/MAINTAINERS b/MAINTAINERS index 806bd2d80d15..b6ef971ee408 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -24933,6 +24933,7 @@ SLEEPABLE READ-COPY UPDATE (SRCU) M: Lai Jiangshan M: "Paul E. McKenney" M: Josh Triplett +M: Onur Özkan (RUST) R: Steven Rostedt R: Mathieu Desnoyers L: rcu@vger.kernel.org @@ -24941,6 +24942,8 @@ W: http://www.rdrop.com/users/paulmck/RCU/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux.git rcu/dev F: include/linux/srcu*.h F: kernel/rcu/srcu*.c +F: rust/helpers/srcu.c +F: rust/kernel/sync/srcu.rs SMACK SECURITY MODULE M: Casey Schaufler diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index ce00f1726e95..9a741ce05885 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -217,15 +217,15 @@ static inline void exit_tasks_rcu_finish(void) { } /** * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU * - * This macro resembles cond_resched(), except that it is defined to + * This function resembles cond_resched(), except that it is defined to * report potential quiescent states to RCU-tasks even if the cond_resched() * machinery were to be shut off, as some advocate for PREEMPTION kernels. */ -#define cond_resched_tasks_rcu_qs() \ -do { \ - rcu_tasks_qs(current, false); \ - cond_resched(); \ -} while (0) +static inline void cond_resched_tasks_rcu_qs(void) +{ + rcu_tasks_qs(current, false); + cond_resched(); +} /** * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states @@ -499,12 +499,12 @@ context_unsafe( \ */ #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, local, space) \ +#define __rcu_access_pointer(p, local, space) context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(local)); \ -}) +}) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..273c59a03251 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -95,10 +95,13 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) */ static inline void rcu_read_lock_trace(void) { + int n; struct task_struct *t = current; rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); - if (t->trc_reader_nesting++) { + n = READ_ONCE(t->trc_reader_nesting); + WRITE_ONCE(t->trc_reader_nesting, n + 1); + if (n) { // In case we interrupted a Tasks Trace RCU reader. return; } @@ -119,12 +122,17 @@ static inline void rcu_read_lock_trace(void) */ static inline void rcu_read_unlock_trace(void) { + int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; - scp = t->trc_reader_scp; - barrier(); // scp before nesting to protect against interrupt handler. - if (!--t->trc_reader_nesting) { + n = READ_ONCE(t->trc_reader_nesting) - 1; + if (n) { + WRITE_ONCE(t->trc_reader_nesting, n); + } else { + scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness. + barrier(); // scp before nesting to protect against interrupt handler. + WRITE_ONCE(t->trc_reader_nesting, n); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); @@ -198,10 +206,13 @@ static inline void rcu_tasks_trace_expedite_current(void) srcu_expedite_current(&rcu_tasks_trace_srcu_struct); } +unsigned long rcu_tasks_trace_batches_completed(void); + // Placeholders to enable stepwise transition. void __init rcu_tasks_trace_suppress_unused(void); #else +static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; } /* * The BPF JIT forms these addresses even when it doesn't call these * functions, so provide definitions that result in runtime errors. diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..c5ab7df6fe5c 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -25,20 +25,19 @@ context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_lockdep(ssp, name, key); +} #ifndef CONFIG_TINY_SRCU int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU -#define init_srcu_struct(ssp) \ -({ \ - static struct lock_class_key __srcu_key; \ - \ - __init_srcu_struct((ssp), #ssp, &__srcu_key); \ -}) - #define init_srcu_struct_fast(ssp) \ ({ \ static struct lock_class_key __srcu_key; \ @@ -56,7 +55,12 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -int init_srcu_struct(struct srcu_struct *ssp); +int init_srcu_struct_generic(struct srcu_struct *ssp); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_generic(ssp); +} #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); @@ -65,6 +69,13 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define __SRCU_DEP_MAP_INIT(srcu_name) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +#define init_srcu_struct(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct((ssp), #ssp, &__srcu_key); \ +}) + /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 905b629e8fa3..fbcf13bc12d1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -154,4 +154,17 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, data_race(READ_ONCE(ssp->srcu_idx_max))); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + return READ_ONCE(ssp->srcu_lock_nesting[0]) || READ_ONCE(ssp->srcu_lock_nesting[1]); +} + #endif diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index fd1a9270cb9a..75e54e4f963f 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -374,4 +374,28 @@ static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flav __srcu_check_read_flavor(ssp, read_flavor); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + int cpu; + unsigned long sum = 0; + + for_each_possible_cpu(cpu) { + struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); + + sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); + sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); + sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); + sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); + } + return sum; +} + #endif diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 762299291e09..332df7a7a634 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -140,7 +140,6 @@ config FORCE_TASKS_TRACE_RCU config TASKS_TRACE_RCU bool default n - select IRQ_WORK config TASKS_TRACE_RCU_NO_MB bool "Override RCU Tasks Trace inclusion of read-side memory barriers" diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 0507bb7dac12..39a9f6fa9a7b 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -704,4 +704,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re void synchronize_rcu_trivial_preempt(void); #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU +#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +bool rcu_is_task_rcu_boosted(void); +#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +static inline bool rcu_is_task_rcu_boosted(void) { return false; } +#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) + + #endif /* __LINUX_RCU_H */ diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index d71911ac911a..794937e13e7c 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -80,6 +80,7 @@ MODULE_AUTHOR("Paul E. McKenney and Josh Triplett get_gp_seq(); ts = rcu_trace_clock_local(); if ((preempt_count() & HARDIRQ_MASK) || softirq_count()) @@ -635,6 +642,7 @@ static struct rcu_torture_ops rcu_ops = { .format_gp_seqs = rcutorture_format_gp_seqs, .set_gpwrap_lag = rcu_set_gpwrap_lag, .get_gpwrap_count = rcu_get_gpwrap_count, + .is_task_rcu_boosted = rcu_is_task_rcu_boosted, .irq_capable = 1, .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), .extendables = RCUTORTURE_MAX_EXTEND, @@ -760,11 +768,16 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) const long uspertick = 1000000 / HZ; const long longdelay = 10; - /* We want there to be long-running readers, but not all the time. */ + // If there is a forward-progress test in flight, don't delay. + if (atomic_read(&rcu_fwd_cb_nodelay)) + return; + + // We want there to be long-running readers, but not all the time. + // The !rcu_preempt_depth() is for RCU Tasks Trace. delay = torture_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); - if (!delay && in_task()) { + if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled()) { schedule_timeout_interruptible(longdelay); rtrsp->rt_delay_jiffies = longdelay; } else { @@ -1213,15 +1226,24 @@ static struct rcu_torture_ops tasks_rude_ops = { * Definitions for tracing RCU-tasks torture testing. */ +// Note that an RCU Tasks Trace GP must imply an RCU GP. static int tasks_tracing_torture_read_lock(void) { - rcu_read_lock_trace(); - return 0; + int use_rcu = !(jiffies & 0xff); + + if (use_rcu) + rcu_read_lock(); + else + rcu_read_lock_trace(); + return use_rcu; } -static void tasks_tracing_torture_read_unlock(int idx) +static void tasks_tracing_torture_read_unlock(int use_rcu) { - rcu_read_unlock_trace(); + if (use_rcu) + rcu_read_unlock(); + else + rcu_read_unlock_trace(); } static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) @@ -1236,7 +1258,8 @@ static struct rcu_torture_ops tasks_tracing_ops = { .read_delay = srcu_read_delay, /* just reuse srcu's version. */ .readunlock = tasks_tracing_torture_read_unlock, .readlock_held = rcu_read_lock_trace_held, - .get_gp_seq = rcu_no_completed, + .get_gp_seq = rcu_tasks_trace_batches_completed, + .gp_diff = rcu_seq_diff, .deferred_free = rcu_tasks_tracing_torture_deferred_free, .sync = synchronize_rcu_tasks_trace, .exp_sync = synchronize_rcu_tasks_trace, @@ -2124,6 +2147,8 @@ static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. } +static DEFINE_PER_CPU(bool, torture_in_scf_handler); + // Verify the specified RCUTORTURE_RDR* state. #define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count() static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) @@ -2133,7 +2158,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE) || in_nmi()) return; - WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq(), ROEC_ARGS); + WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq() && !this_cpu_read(torture_in_scf_handler), ROEC_ARGS); WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS); // If CONFIG_PREEMPT_COUNT=n, further checks are unreliable. @@ -2150,7 +2175,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) // Interrupt handlers have all sorts of stuff disabled, so ignore // unintended disabling. - if (in_serving_softirq() || in_hardirq()) + if (in_serving_softirq() || in_hardirq() || this_cpu_read(torture_in_scf_handler)) return; WARN_ONCE(cur_ops->extendables && @@ -2342,12 +2367,19 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) mask |= RCUTORTURE_RDR_RCU_1; } + /* + * Don't mess with interrupt masking in interrupt handlers. + */ + if (in_hardirq() || this_cpu_read(torture_in_scf_handler)) + mask &= ~(preempts_irq | bhs); + /* * Can't enable bh w/irq disabled. */ if (mask & RCUTORTURE_RDR_IRQ) mask |= oldmask & bhs; + /* * Ideally these sequences would be detected in debug builds * (regardless of RT), but until then don't stop testing @@ -2401,6 +2433,80 @@ struct rcu_torture_one_read_state { unsigned long long ts; }; +static void rcu_torture_dump_read_segs(struct rt_read_seg *rrsp, int nsegs) +{ + bool firsttime; + int i; + int j; + + firsttime = 1; + for (i = 0; i < nsegs; i++) { + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) + pr_alert("\t%lluus ", div64_u64(rrsp[i].rt_ts, 1000ULL)); + else + pr_alert("\t"); + pr_cont("%d: %#4x", i, rrsp[i].rt_readstate); + if (rrsp[i].rt_delay_jiffies != 0) { + pr_cont("%s%ldjiffies", firsttime ? "" : "+", + rrsp[i].rt_delay_jiffies); + firsttime = 0; + } + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { + pr_cont(" CPU %2d", rrsp[i].rt_cpu); + if (rrsp[i].rt_cpu != rrsp[i].rt_end_cpu) + pr_cont("->%-2d", rrsp[i].rt_end_cpu); + else + pr_cont(" ..."); + } + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && + cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { + char buf1[20+1]; + char buf2[20+1]; + char sepchar = '-'; + + cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq, buf1, ARRAY_SIZE(buf1)); + cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq_end, buf2, ARRAY_SIZE(buf2)); + if (rrsp[i].rt_gp_seq == rrsp[i].rt_gp_seq_end) { + if (buf2[0]) { + for (j = 0; buf2[j]; j++) + buf2[j] = '.'; + if (j) + buf2[j - 1] = ' '; + } + sepchar = ' '; + } + pr_cont(" %s%c%s", buf1, sepchar, buf2); + } + if (rrsp[i].rt_delay_ms != 0) { + pr_cont(" %s%ldms", firsttime ? "" : "+", rrsp[i].rt_delay_ms); + firsttime = 0; + } + if (rrsp[i].rt_delay_us != 0) { + pr_cont(" %s%ldus", firsttime ? "" : "+", rrsp[i].rt_delay_us); + firsttime = 0; + } + pr_cont("%s", rrsp[i].rt_preempted ? " preempted" : ""); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_BH) + pr_cont(" BH"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_IRQ) + pr_cont(" IRQ"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) + pr_cont(" PREEMPT"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RBH) + pr_cont(" RBH"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_SCHED) + pr_cont(" SCHED"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_1) + pr_cont(" RCU_1"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_2) + pr_cont(" RCU_2"); + pr_cont("\n"); + + } + if (rt_read_preempted) + pr_alert("\tReader was preempted.\n"); +} + static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp, struct torture_random_state *trsp) { @@ -2465,13 +2571,13 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, rtorsp->ts, rtorsp->started, completed); rcu_ftrace_dump(DUMP_ALL); } - __this_cpu_inc(rcu_torture_count[pipe_count]); + this_cpu_inc(rcu_torture_count[pipe_count]); completed = rcutorture_seq_diff(completed, rtorsp->started); if (completed > RCU_TORTURE_PIPE_LEN) { /* Should not happen, but... */ completed = RCU_TORTURE_PIPE_LEN; } - __this_cpu_inc(rcu_torture_batch[completed]); + this_cpu_inc(rcu_torture_batch[completed]); preempt_enable(); if (rtorsp->checkpolling) { if (cur_ops->get_gp_state && cur_ops->poll_gp_state) @@ -2514,7 +2620,9 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, */ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) { + static int firsttime = 1; int newstate; + unsigned int nsegs; struct rcu_torture_one_read_state rtors; WARN_ON_ONCE(!rcu_is_watching()); @@ -2526,6 +2634,26 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) return false; rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); rcu_torture_one_read_end(&rtors, trsp); + + // This splat will happen on systems built with CONFIG_IRQ_WORK=n + // and on systems where arch_irq_work_has_interrupt() returns false. + // It might also happen on systems using a short-duration clock + // interrupt instead of a self-IPI (powerpc, s390) or that use + // neither a self-IPI nor a short-duration clock interrupts + // (all architectures using the generic implementation + // of arch_irq_work_raise()). On such systems, RCU cannot + // guarantee to immediately deboost RCU readers when the outermost + // rcu_read_unlock() does not end the full segmented RCU read-side + // critical section. + if (cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() && + !in_serving_softirq() && !in_hardirq() && !in_nmi() && + READ_ONCE(firsttime) && xchg(&firsttime, 0)) { + WARN_ON_ONCE(deboost_timeliness_check); + nsegs = rtors.rtrsp - rtors.rtseg; + nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS); + pr_alert("Slow-deboost rcutorture reader segments:\n"); + rcu_torture_dump_read_segs(rtors.rtseg, nsegs); + } return true; } @@ -2545,7 +2673,7 @@ static void rcu_torture_timer(struct timer_list *unused) atomic_long_inc(&n_rcu_torture_timers); (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); - /* Test call_rcu() invocation from interrupt handler. */ + /* Test call_rcu() invocation from softirq handler. */ if (cur_ops->call) { struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); @@ -2554,6 +2682,41 @@ static void rcu_torture_timer(struct timer_list *unused) } } +static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_irq_rand); + +/* + * RCU torture reader from timer handler. Dereferences rcu_torture_current, + * incrementing the corresponding element of the pipeline array. The + * counter in the element should never be greater than 1, otherwise, the + * RCU implementation is broken. + * + * Note that on some systems, "interrupts" from idle are direct calls + * rather than interrupts. The torture_in_scf_handler per-CPU variable + * accounts for this case. + */ +static void rcu_torture_irq(void *unused) +{ + WARN_ON_ONCE(in_nmi()); + lockdep_assert_irqs_disabled(); + atomic_long_inc(&n_rcu_torture_irqs); + this_cpu_write(torture_in_scf_handler, true); + (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_irq_rand), -1); + this_cpu_write(torture_in_scf_handler, false); + + // Test call_rcu() invocation from interrupt handler. Interrupts + // will always be disabled here, even in CONFIG_PREEMPT_RT=y kernels. + // The "right" thing to do would be to create a special-purpose + // lockless or raw-spinlock-protected allocator, but in the meantime, + // skip testing call_rcu() from interrupt handlers in kernels built + // with either CONFIG_PREEMPT_RT=y or CONFIG_PROVE_LOCKING=y. + if (cur_ops->call && !IS_ENABLED(CONFIG_PROVE_LOCKING) && !IS_ENABLED(CONFIG_PREEMPT_RT)) { + struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); + + if (rhp) + cur_ops->call(rhp, rcu_torture_timer_cb); + } +} + /* * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, * incrementing the corresponding element of the pipeline array. The @@ -2563,6 +2726,7 @@ static void rcu_torture_timer(struct timer_list *unused) static int rcu_torture_reader(void *arg) { + unsigned long lastscf = jiffies; unsigned long lastsleep = jiffies; long myid = (long)arg; int mynumonline = myid; @@ -2576,8 +2740,25 @@ rcu_torture_reader(void *arg) tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. do { if (irqreader && cur_ops->irq_capable) { - if (!timer_pending(&t)) + if (!timer_pending(&t)) { + int cpu; + mod_timer(&t, jiffies + 1); + preempt_disable(); + cpu = torture_random(&rand) % nr_cpu_ids; + if (!cpu_online(cpu)) { + cpu = cpumask_next(cpu, cpu_online_mask); + if (cpu >= nr_cpu_ids) + cpu = cpumask_next(-1, cpu_online_mask); + } + // An smp_call_function_single() to self is not an interrupt! + if (cpu != smp_processor_id() && + time_after(jiffies, lastscf + HZ * nrealreaders / 50)) { + smp_call_function_single(cpu, rcu_torture_irq, NULL, 0); + lastscf = jiffies; + } + preempt_enable(); + } } if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) schedule_timeout_interruptible(HZ); @@ -2853,10 +3034,11 @@ rcu_torture_stats_print(void) atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), n_rcu_torture_barrier_error, n_rcu_torture_boost_ktrerror); - pr_cont("rtbf: %ld rtb: %ld nt: %ld ", + pr_cont("rtbf: %ld rtb: %ld nt: %ld ni: %ld ", n_rcu_torture_boost_failure, n_rcu_torture_boosts, - atomic_long_read(&n_rcu_torture_timers)); + atomic_long_read(&n_rcu_torture_timers), + atomic_long_read(&n_rcu_torture_irqs)); if (updownreaders) pr_cont("ndowns: %lu nups: %lu nhrt: %lu nmigrates: %lu ", ndowns, nups, nunexpired, nmigrates); torture_onoff_stats(); @@ -2913,10 +3095,10 @@ rcu_torture_stats_print(void) if (cur_ops->get_gp_data) cur_ops->get_gp_data(&flags, &gp_seq); wtp = READ_ONCE(writer_task); - pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", + pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %c cpu %d\n", rcu_torture_writer_state_getname(), rcu_torture_writer_state, gp_seq, flags, - wtp == NULL ? ~0U : wtp->__state, + wtp == NULL ? '?' : task_state_to_char(wtp), wtp == NULL ? -1 : (int)task_cpu(wtp)); if (!splatted && wtp) { sched_show_task(wtp); @@ -3000,7 +3182,7 @@ static void rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) { pr_alert("%s" TORTURE_FLAG - "--- %s: nreaders=%d nfakewriters=%d " + "--- %s: nreaders=%d nwriters=%d nfakewriters=%d " "stat_interval=%d verbose=%d test_no_idle_hz=%d " "shuffle_interval=%d stutter=%d irqreader=%d " "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " @@ -3015,7 +3197,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) "nocbs_nthreads=%d nocbs_toggle=%d " "test_nmis=%d " "preempt_duration=%d preempt_interval=%d n_up_down=%d\n", - torture_type, tag, nrealreaders, nrealfakewriters, + torture_type, tag, nrealreaders, nwriters, nrealfakewriters, stat_interval, verbose, test_no_idle_hz, shuffle_interval, stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, test_boost, cur_ops->can_boost, @@ -3447,13 +3629,17 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) unsigned long stopat; unsigned long stoppedat; - pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); - if (READ_ONCE(rcu_fwd_emergency_stop)) + if (READ_ONCE(rcu_fwd_emergency_stop)) { + pr_alert("%s: Emergency stop, so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); return; /* Get out of the way quickly, no GP wait! */ - if (!cur_ops->call) + } + if (!cur_ops->call) { + pr_alert("%s: No ->call(), so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); return; /* Can't do call_rcu() fwd prog without ->call. */ + } /* Loop continuously posting RCU callbacks. */ + pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); atomic_inc(&rcu_fwd_cb_nodelay); cur_ops->sync(); /* Later readers see above write. */ WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); @@ -4020,9 +4206,7 @@ static int rcu_torture_preempt(void *unused) // Wait for preempt_interval ms with up to 100us fuzz. torture_hrtimeout_ms(preempt_interval, 100, &rand); // Select online CPU. - cpu = cpumask_next(cpu, cpu_online_mask); - if (cpu >= nr_cpu_ids) - cpu = cpumask_next(-1, cpu_online_mask); + cpu = cpumask_next_wrap(cpu, cpu_online_mask); WARN_ON_ONCE(cpu >= nr_cpu_ids); // Move to that CPU, if can't do so, retry later. if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false)) @@ -4094,11 +4278,9 @@ static void rcu_gpwrap_lag_cleanup(void) static void rcu_torture_cleanup(void) { - int firsttime; int flags = 0; unsigned long gp_seq = 0; int i; - int j; if (torture_cleanup_begin()) { if (cur_ops->cb_barrier != NULL) { @@ -4183,76 +4365,8 @@ rcu_torture_cleanup(void) pr_alert("Failure/close-call rcutorture reader segments:\n"); if (rt_read_nsegs == 0) pr_alert("\t: No segments recorded!!!\n"); - firsttime = 1; - for (i = 0; i < rt_read_nsegs; i++) { - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) - pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL)); - else - pr_alert("\t"); - pr_cont("%d: %#4x", i, err_segs[i].rt_readstate); - if (err_segs[i].rt_delay_jiffies != 0) { - pr_cont("%s%ldjiffies", firsttime ? "" : "+", - err_segs[i].rt_delay_jiffies); - firsttime = 0; - } - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { - pr_cont(" CPU %2d", err_segs[i].rt_cpu); - if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu) - pr_cont("->%-2d", err_segs[i].rt_end_cpu); - else - pr_cont(" ..."); - } - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && - cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { - char buf1[20+1]; - char buf2[20+1]; - char sepchar = '-'; - - cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq, - buf1, ARRAY_SIZE(buf1)); - cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end, - buf2, ARRAY_SIZE(buf2)); - if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) { - if (buf2[0]) { - for (j = 0; buf2[j]; j++) - buf2[j] = '.'; - if (j) - buf2[j - 1] = ' '; - } - sepchar = ' '; - } - pr_cont(" %s%c%s", buf1, sepchar, buf2); - } - if (err_segs[i].rt_delay_ms != 0) { - pr_cont(" %s%ldms", firsttime ? "" : "+", - err_segs[i].rt_delay_ms); - firsttime = 0; - } - if (err_segs[i].rt_delay_us != 0) { - pr_cont(" %s%ldus", firsttime ? "" : "+", - err_segs[i].rt_delay_us); - firsttime = 0; - } - pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : ""); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH) - pr_cont(" BH"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ) - pr_cont(" IRQ"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) - pr_cont(" PREEMPT"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH) - pr_cont(" RBH"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED) - pr_cont(" SCHED"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1) - pr_cont(" RCU_1"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2) - pr_cont(" RCU_2"); - pr_cont("\n"); - - } - if (rt_read_preempted) - pr_alert("\tReader was preempted.\n"); + else + rcu_torture_dump_read_segs(err_segs, rt_read_nsegs); } if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); @@ -4557,6 +4671,23 @@ rcu_torture_init(void) cur_ops = NULL; goto unwind; } + if (stall_only) { + pr_alert("rcu-torture: stall_only specified, suppressing all else.\n"); + fqs_stutter = 0; + fwd_progress = 0; + n_barrier_cbs = 0; + nfakewriters = 0; + nocbs_nthreads = 0; + nreaders = 0; + n_up_down = 0; + nwriters = 0; + onoff_interval = 0; + preempt_duration = 0; + read_exit_burst = 0; + shuffle_interval = 0; + stutter = 0; + test_boost = 0; + } if (cur_ops->fqs == NULL && fqs_duration != 0) { pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); fqs_duration = 0; @@ -4663,10 +4794,11 @@ rcu_torture_init(void) goto unwind; } - firsterr = torture_create_kthread(rcu_torture_writer, NULL, - writer_task); - if (torture_init_error(firsterr)) - goto unwind; + if (nwriters) { + firsterr = torture_create_kthread(rcu_torture_writer, NULL, writer_task); + if (torture_init_error(firsterr)) + goto unwind; + } firsterr = rcu_torture_updown_init(); if (torture_init_error(firsterr)) diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index a2e2d516e51b..558ba8d316db 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -48,31 +48,31 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, - struct lock_class_key *key) +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) { /* Don't re-initialize a lock while it is held. */ debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); lockdep_init_map(&ssp->dep_map, name, key, 0); return init_srcu_struct_fields(ssp); } -EXPORT_SYMBOL_GPL(__init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /* - * init_srcu_struct - initialize a sleep-RCU structure + * init_srcu_struct_generic - initialize a sleep-RCU structure * @ssp: structure to initialize. * * Must invoke this on a given srcu_struct before passing that srcu_struct * to any other function. Each srcu_struct represents a separate domain * of SRCU protection. */ -int init_srcu_struct(struct srcu_struct *ssp) +int init_srcu_struct_generic(struct srcu_struct *ssp) { return init_srcu_struct_fields(ssp); } -EXPORT_SYMBOL_GPL(init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_generic); #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ @@ -85,7 +85,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); */ void cleanup_srcu_struct(struct srcu_struct *ssp) { - WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); + WARN_ON(srcu_readers_active(ssp)); irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 519a35719c89..ed204b3f4b84 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -266,12 +266,13 @@ __init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock return init_srcu_struct_fields(ssp, false); } -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) { ssp->srcu_reader_flavor = 0; return __init_srcu_struct_common(ssp, name, key); } -EXPORT_SYMBOL_GPL(__init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { @@ -291,7 +292,7 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** - * init_srcu_struct - initialize a sleep-RCU structure + * init_srcu_struct_generic - initialize a sleep-RCU structure * @ssp: structure to initialize. * * Use this in place of DEFINE_SRCU() and DEFINE_STATIC_SRCU() @@ -301,12 +302,12 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); * to any other function. Each srcu_struct represents a separate domain * of SRCU protection. */ -int init_srcu_struct(struct srcu_struct *ssp) +int init_srcu_struct_generic(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = 0; return init_srcu_struct_fields(ssp, false); } -EXPORT_SYMBOL_GPL(init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_generic); /** * init_srcu_struct_fast - initialize a fast-reader sleep-RCU structure @@ -598,31 +599,6 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) return srcu_readers_lock_idx(ssp, idx, did_gp, unlocks); } -/** - * srcu_readers_active - returns true if there are readers. and false - * otherwise - * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). - * - * Note that this is not an atomic primitive, and can therefore suffer - * severe errors when invoked on an active srcu_struct. That said, it - * can be useful as an error check at cleanup time. - */ -static bool srcu_readers_active(struct srcu_struct *ssp) -{ - int cpu; - unsigned long sum = 0; - - for_each_possible_cpu(cpu) { - struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - - sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); - sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); - sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); - sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); - } - return sum; -} - /* * We use an adaptive strategy for synchronize_srcu() and especially for * synchronize_srcu_expedited(). We spin for a fixed time period @@ -725,7 +701,12 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) for_each_possible_cpu(cpu) { struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - timer_delete_sync(&sdp->delay_work); + // Call srcu_barrier() before this cleanup_srcu_struct() + // to avoid triggering this WARN_ON(). + if (WARN_ON(timer_delete_sync(&sdp->delay_work) && + rcu_segcblist_n_cbs(&sdp->srcu_cblist)) && + rcu_cpu_beenfullyonline(sdp->cpu)) + queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); flush_work(&sdp->work); if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist))) return; /* Forgot srcu_barrier(), so just leak it! */ diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 92971499a12c..627295396cd9 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -67,7 +67,6 @@ struct rcu_tasks_percpu { * @gp_start: Most recent grace-period start in jiffies. * @tasks_gp_seq: Number of grace periods completed since boot in upper bits. * @n_ipis: Number of IPIs sent to encourage grace periods to end. - * @n_ipis_fails: Number of IPI-send failures. * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @lazy_jiffies: Number of jiffies to allow callbacks to be lazy. * @pregp_func: This flavor's pre-grace-period function (optional). @@ -102,7 +101,6 @@ struct rcu_tasks { unsigned long gp_start; unsigned long tasks_gp_seq; unsigned long n_ipis; - unsigned long n_ipis_fails; struct task_struct *kthread_ptr; unsigned long lazy_jiffies; rcu_tasks_gp_func_t gp_func; @@ -157,8 +155,8 @@ static struct rcu_tasks rt_name = \ #ifdef CONFIG_TASKS_RCU /* Report delay of scan exiting tasklist in rcu_tasks_postscan(). */ -static void tasks_rcu_exit_srcu_stall(struct timer_list *unused); -static DEFINE_TIMER(tasks_rcu_exit_srcu_stall_timer, tasks_rcu_exit_srcu_stall); +static void tasks_rcu_exit_stall(struct timer_list *unused); +static DEFINE_TIMER(tasks_rcu_exit_stall_timer, tasks_rcu_exit_stall); #endif /* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */ @@ -397,7 +395,11 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } rcu_read_unlock(); - /* We can't create the thread unless interrupts are enabled. */ + // We can't create the kthread with interrupts disabled because a + // scheduler spinlock might be held, so kthread creation is deferred + // until core_initcall() time. Similarly, wakeups are deferred using + // irq_work in order to avoid potential scheduler-lock-deadlock + // lockdep splats. if (needwake && READ_ONCE(rtp->kthread_ptr)) irq_work_queue(&rtpcp->rtp_irq_work); } @@ -683,7 +685,6 @@ static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp) t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname); if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __func__, rtp->name)) return; - smp_mb(); /* Ensure others see full kthread. */ } #ifndef CONFIG_TINY_RCU @@ -722,6 +723,7 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) bool havecbs = false; bool haveurgent = false; bool haveurgentcbs = false; + bool havependtimer = false; for_each_possible_cpu(cpu) { struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); @@ -732,19 +734,22 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) haveurgent = true; if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp)) haveurgentcbs = true; - if (havecbs && haveurgent && haveurgentcbs) + if (data_race(timer_pending(&rtpcp->lazy_timer))) + havependtimer = true; + if (havecbs && haveurgent && haveurgentcbs && havependtimer) break; } - pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c%c%c l:%lu %s\n", + pr_info("%s: %s(%d) since %lu g:%lu i:%lu %c%c%c%c%c l:%lu %s\n", rtp->kname, tasks_gp_state_getname(rtp), data_race(rtp->gp_state), jiffies - data_race(rtp->gp_jiffies), data_race(rcu_seq_current(&rtp->tasks_gp_seq)), - data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis), + data_race(rtp->n_ipis), ".k"[!!data_race(rtp->kthread_ptr)], ".C"[havecbs], ".u"[haveurgent], ".U"[haveurgentcbs], + ".P"[havependtimer], rtp->lazy_jiffies, s); } @@ -1027,8 +1032,8 @@ static void rcu_tasks_postscan(struct list_head *hop) int rtsi = READ_ONCE(rcu_task_stall_info); if (!IS_ENABLED(CONFIG_TINY_RCU)) { - tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi; - add_timer(&tasks_rcu_exit_srcu_stall_timer); + tasks_rcu_exit_stall_timer.expires = jiffies + rtsi; + add_timer(&tasks_rcu_exit_stall_timer); } /* @@ -1081,7 +1086,7 @@ static void rcu_tasks_postscan(struct list_head *hop) } if (!IS_ENABLED(CONFIG_TINY_RCU)) - timer_delete_sync(&tasks_rcu_exit_srcu_stall_timer); + timer_delete_sync(&tasks_rcu_exit_stall_timer); } /* See if tasks are still holding out, complain if so. */ @@ -1153,7 +1158,7 @@ static void rcu_tasks_postgp(struct rcu_tasks *rtp) synchronize_rcu(); } -static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) +static void tasks_rcu_exit_stall(struct timer_list *unused) { #ifndef CONFIG_TINY_RCU int rtsi; @@ -1163,8 +1168,8 @@ static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) __func__, rcu_tasks.kname, rcu_tasks.tasks_gp_seq, tasks_gp_state_getname(&rcu_tasks), jiffies - rcu_tasks.gp_jiffies); pr_info("Please check any exiting tasks stuck between calls to exit_tasks_rcu_start() and exit_tasks_rcu_finish()\n"); - tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi; - add_timer(&tasks_rcu_exit_srcu_stall_timer); + tasks_rcu_exit_stall_timer.expires = jiffies + rtsi; + add_timer(&tasks_rcu_exit_stall_timer); #endif // #ifndef CONFIG_TINY_RCU } @@ -1174,7 +1179,7 @@ static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) * @func: actual callback function to be invoked after the grace period * * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU + * period elapses, in other words after all currently executing rcu-tasks * read-side critical sections have completed. call_rcu_tasks() assumes * that the read-side critical sections end at a voluntary context * switch (not a preemption!), cond_resched_tasks_rcu_qs(), entry into idle, @@ -1360,8 +1365,8 @@ DEFINE_RCU_TASKS(rcu_tasks_rude, rcu_tasks_rude_wait_gp, call_rcu_tasks_rude, * @func: actual callback function to be invoked after the grace period * * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU - * read-side critical sections have completed. call_rcu_tasks_rude() + * period elapses, in other words after all currently executing rude + * rcu-tasks read-side critical sections have completed. call_rcu_tasks_rude() * assumes that the read-side critical sections end at context switch, * cond_resched_tasks_rcu_qs(), or transition to usermode execution (as * usermode execution is schedulable). As such, there are no read-side @@ -1385,7 +1390,7 @@ static void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func) * * Control will return to the caller some time after a rude rcu-tasks * grace period has elapsed, in other words after all currently - * executing rcu-tasks read-side critical sections have elapsed. These + * executing rude rcu-tasks read-side critical sections have elapsed. These * read-side critical sections are delimited by calls to schedule(), * cond_resched_tasks_rcu_qs(), userspace execution (which is a schedulable * context), and (in theory, anyway) cond_resched(). @@ -1455,6 +1460,7 @@ struct rcu_tasks_test_desc { const char *name; bool notrun; unsigned long runstart; + void (*gp_dbg)(void); }; static struct rcu_tasks_test_desc tests[] = { @@ -1462,6 +1468,8 @@ static struct rcu_tasks_test_desc tests[] = { .name = "call_rcu_tasks()", /* If not defined, the test is skipped. */ .notrun = IS_ENABLED(CONFIG_TASKS_RCU), + /* Dump rcu tasks status, if test failed. */ + .gp_dbg = show_rcu_tasks_classic_gp_kthread }, { .name = "call_rcu_tasks_trace()", @@ -1521,6 +1529,8 @@ static int rcu_tasks_verify_self_tests(void) while (tests[i].notrun) { // still hanging. if (time_after(jiffies, tests[i].runstart + bst)) { pr_err("%s has failed boot-time tests.\n", tests[i].name); + if (tests[i].gp_dbg) + tests[i].gp_dbg(); ret = -1; break; } @@ -1606,4 +1616,10 @@ static inline void rcu_tasks_bootup_oddness(void) {} DEFINE_SRCU_FAST(rcu_tasks_trace_srcu_struct); EXPORT_SYMBOL_GPL(rcu_tasks_trace_srcu_struct); +unsigned long rcu_tasks_trace_batches_completed(void) +{ + return srcu_batches_completed(&rcu_tasks_trace_srcu_struct); +} +EXPORT_SYMBOL_GPL(rcu_tasks_trace_batches_completed); + #endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index f6809ccedfc1..21b6ce1dffb6 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -357,9 +357,10 @@ bool rcu_watching_zero_in_eqs(int cpu, int *vp) */ notrace void rcu_momentary_eqs(void) { + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; - raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); + WRITE_ONCE(rdp->rcu_need_heavy_qs, false); seq = ct_state_inc(2 * CT_RCU_WATCHING); /* It is illegal to call this from idle state. */ WARN_ON_ONCE(!(seq & CT_RCU_WATCHING)); @@ -2543,7 +2544,7 @@ rcu_check_quiescent_state(struct rcu_data *rdp) * Was there a quiescent state since the beginning of the grace * period? If no, then exit and wait for the next call. */ - if (rdp->cpu_no_qs.b.norm) + if (READ_ONCE(rdp->cpu_no_qs.b.norm)) return; /* @@ -2674,7 +2675,7 @@ static void rcu_do_batch(struct rcu_data *rdp) // reporting, so check time limits for them. if (rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING && rcu_do_batch_check_time(count, tlimit, jlimit_check, jlimit)) { - rdp->rcu_cpu_has_work = 1; + WRITE_ONCE(rdp->rcu_cpu_has_work, 1); break; } } @@ -2910,7 +2911,7 @@ static __latent_entropy void rcu_core(void) rcu_accelerate_cbs_unlocked(rnp, rdp); } - rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); + rcu_check_gp_start_stall(rnp, rcu_jiffies_till_stall_check()); /* If there are callbacks ready, invoke them. */ if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_ready_cbs(&rdp->cblist) && @@ -2951,7 +2952,7 @@ static void invoke_rcu_core_kthread(void) unsigned long flags; local_irq_save(flags); - __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); + this_cpu_write(rcu_data.rcu_cpu_has_work, 1); t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); if (t != NULL && t != current) rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); @@ -2978,7 +2979,7 @@ static void rcu_cpu_kthread_park(unsigned int cpu) static int rcu_cpu_kthread_should_run(unsigned int cpu) { - return __this_cpu_read(rcu_data.rcu_cpu_has_work); + return this_cpu_read(rcu_data.rcu_cpu_has_work); } /* @@ -2999,7 +3000,7 @@ static void rcu_cpu_kthread(unsigned int cpu) local_bh_disable(); *statusp = RCU_KTHREAD_RUNNING; local_irq_disable(); - work = *workp; + work = READ_ONCE(*workp); WRITE_ONCE(*workp, 0); local_irq_enable(); if (work) @@ -3044,7 +3045,7 @@ static int __init rcu_spawn_core_kthreads(void) return 0; } -static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func) +static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head) { rcu_segcblist_enqueue(&rdp->cblist, head); trace_rcu_callback(rcu_state.name, head, @@ -3056,9 +3057,9 @@ static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_cal * Handle any core-RCU processing required by a call_rcu() invocation. */ static void call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags) + unsigned long flags) { - rcutree_enqueue(rdp, head, func); + rcutree_enqueue(rdp, head); /* * If called from an extended quiescent state, invoke the RCU * core in order to force a re-evaluation of RCU's idleness. @@ -3199,9 +3200,9 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) check_cb_ovld(rdp); if (unlikely(rcu_rdp_is_offloaded(rdp))) - call_rcu_nocb(rdp, head, func, flags, lazy); + call_rcu_nocb(rdp, head, flags, lazy); else - call_rcu_core(rdp, head, func, flags); + call_rcu_core(rdp, head, flags); local_irq_restore(flags); } diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 01a1b2985abd..eedfa43059e8 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -296,6 +296,11 @@ struct rcu_data { int cpu; }; +static inline void rcu_defer_qs_clear(struct rcu_data *rdp) +{ + WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); +} + /* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0 #define RCU_NOCB_WAKE_BYPASS 1 @@ -386,7 +391,6 @@ struct rcu_state { struct mutex exp_mutex; /* Serialize expedited GP. */ struct mutex exp_wake_mutex; /* Serialize wakeup. */ unsigned long expedited_sequence; /* Take a ticket. */ - atomic_t expedited_need_qs; /* # CPUs left to check in. */ struct swait_queue_head expedited_wq; /* Wait for check-ins. */ int ncpus_snap; /* # CPUs seen last time. */ u8 cbovld; /* Callback overload now? */ @@ -504,7 +508,7 @@ static bool wake_nocb_gp(struct rcu_data *rdp); static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j, bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy); + unsigned long flags, bool lazy); static void __maybe_unused __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, unsigned long flags); static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level); @@ -541,8 +545,7 @@ static bool rcu_nohz_full_cpu(void); static void record_gp_stall_check_time(void); static void rcu_iw_handler(struct irq_work *iwp); static void check_cpu_stall(struct rcu_data *rdp); -static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, - const unsigned long gpssdelay); +static void rcu_check_gp_start_stall(struct rcu_node *rnp, const unsigned long gpssdelay); /* Forward declarations for tree_exp.h. */ static void sync_rcu_do_polled_gp(struct work_struct *wp); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 1ab83c98af51..682a3fb6087e 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -733,7 +733,7 @@ static void rcu_exp_need_qs(void) { lockdep_assert_irqs_disabled(); ASSERT_EXCLUSIVE_WRITER_SCOPED(*this_cpu_ptr(&rcu_data.cpu_no_qs.b.exp)); - __this_cpu_write(rcu_data.cpu_no_qs.b.exp, true); + this_cpu_write(rcu_data.cpu_no_qs.b.exp, true); /* Store .exp before .rcu_urgent_qs. */ smp_store_release(this_cpu_ptr(&rcu_data.rcu_urgent_qs), true); set_need_resched_current(); @@ -872,7 +872,7 @@ static void rcu_exp_handler(void *unused) ASSERT_EXCLUSIVE_WRITER_SCOPED(rdp->cpu_no_qs.b.exp); if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || - __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + this_cpu_read(rcu_data.cpu_no_qs.b.exp)) return; if (rcu_is_cpu_rrupt_from_idle() || (IS_ENABLED(CONFIG_PREEMPT_COUNT) && preempt_bh_enabled)) { diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index afb35f921b33..19bb42672baf 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -657,13 +657,13 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy) + unsigned long flags, bool lazy) { bool was_alldone; if (!rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) { /* Not enqueued on bypass but locked, do regular enqueue */ - rcutree_enqueue(rdp, head, func); + rcutree_enqueue(rdp, head); __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */ } } @@ -1736,7 +1736,7 @@ static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy) + unsigned long flags, bool lazy) { WARN_ON_ONCE(1); /* Should be dead code! */ } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3..743c16247fc0 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -298,11 +298,11 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n"); - if (__this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { + if (this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { trace_rcu_grace_period(TPS("rcu_preempt"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); - __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); + this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */ WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, false); } @@ -488,7 +488,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) rdp = this_cpu_ptr(&rcu_data); if (rdp->defer_qs_pending == DEFER_QS_PENDING) - rdp->defer_qs_pending = DEFER_QS_IDLE; + rcu_defer_qs_clear(rdp); /* * If RCU core is waiting for this CPU to exit its critical section, @@ -599,7 +599,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) */ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) { - return (__this_cpu_read(rcu_data.cpu_no_qs.b.exp) || + return (this_cpu_read(rcu_data.cpu_no_qs.b.exp) || READ_ONCE(t->rcu_read_unlock_special.s)) && rcu_preempt_depth() == 0; } @@ -614,9 +614,35 @@ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) notrace void rcu_preempt_deferred_qs(struct task_struct *t) { unsigned long flags; + struct rcu_data *rdp; - if (!rcu_preempt_need_deferred_qs(t)) + if (!rcu_preempt_need_deferred_qs(t)) { + /* + * If we got here from a softirq/irq_work that fired while + * rcu_preempt_depth() > 0, the deferred-QS mechanism has been + * consumed without doing any work: rcu_preempt_need_deferred_qs() + * just returned false because the task is still in a reader, so + * the actual QS report has to wait for the next + * rcu_read_unlock(). + * + * Clear ->defer_qs_pending here so the next outer + * rcu_read_unlock_special() can re-arm a fresh mechanism (in + * particular the irq_work path, which the local_irq_enable() + * recovery boundary cannot itself reschedule from). + * + * Recursion safety: rcu_preempt_depth() > 0 means we are inside + * an outer reader, so any inner rcu_read_unlock() reached via + * tracing (bpf programs attached to trace points) brings + * nesting to outer (> 0), never to 0, so no recursive + * raise_softirq_irqoff()/irq_work_queue_on() can be triggered + * by this clear. + */ + if (rcu_preempt_depth() > 0) { + rdp = this_cpu_ptr(&rcu_data); + rcu_defer_qs_clear(rdp); + } return; + } local_irq_save(flags); rcu_preempt_deferred_qs_irqrestore(t, flags); } @@ -645,7 +671,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) * 5. Deferred QS reporting does not happen. */ if (rcu_preempt_depth() > 0) - WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); + rcu_defer_qs_clear(rdp); } /* @@ -923,10 +949,10 @@ void rcu_read_unlock_strict(void) * * The in_atomic_preempt_off() check ensures that we come here holding * the last preempt_count (which will get dropped once we return to - * __rcu_read_unlock(). + * __rcu_read_unlock()). */ rdp = this_cpu_ptr(&rcu_data); - rdp->cpu_no_qs.b.norm = false; + WRITE_ONCE(rdp->cpu_no_qs.b.norm, false); rcu_report_qs_rdp(rdp); udelay(rcu_unlock_delay); } @@ -950,12 +976,12 @@ static void __init rcu_bootup_announce(void) static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!"); - if (!__this_cpu_read(rcu_data.cpu_no_qs.s)) + if (!this_cpu_read(rcu_data.cpu_no_qs.s)) return; trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); - __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); - if (__this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); + if (this_cpu_read(rcu_data.cpu_no_qs.b.exp)) rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); } @@ -970,7 +996,7 @@ void rcu_all_qs(void) { unsigned long flags; - if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) + if (!READ_ONCE(*raw_cpu_ptr(&rcu_data.rcu_urgent_qs))) return; preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ @@ -978,8 +1004,8 @@ void rcu_all_qs(void) preempt_enable(); return; } - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) { local_irq_save(flags); rcu_momentary_eqs(); local_irq_restore(flags); @@ -999,8 +1025,8 @@ void rcu_note_context_switch(bool preempt) /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) goto out; - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) rcu_momentary_eqs(); out: rcu_tasks_qs(current, preempt); @@ -1320,6 +1346,41 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ } +#ifdef CONFIG_RCU_TORTURE_TEST + +/* + * Is the current task RCU priority boosted? This is used by + * rcutorture to check that tasks are always deboosted once then exit + * an RCU read-side critical section, no matter how many overlapping + * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(), + * or local_irq_disable() made up that reader. + * + * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex) + * are safe because tasks release ->boost_mtx when they own it, they + * cannot be boosted unless current->rcu_blocked_node is non-NULL, + * current->rcu_blocked_node is modified only by the current task, + * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner + * switching among other tasks cannot force an equality comparison. + */ +bool rcu_is_task_rcu_boosted(void) +{ + bool ret; + struct rcu_node *rnp; + struct task_struct *t = current; + + preempt_disable(); // Stabilize ->rcu_blocked_node + rnp = t->rcu_blocked_node; + if (!rnp) + ret = false; + else + ret = (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t); + preempt_enable(); + return ret; +} +EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted); + +#endif // #ifdef CONFIG_RCU_TORTURE_TEST + #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index cf7ae51cba40..20634edfb44d 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -573,13 +573,13 @@ static void rcu_check_gp_kthread_starvation(void) if (rcu_is_gp_kthread_starving(&j)) { cpu = gpk ? task_cpu(gpk) : -1; - pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x ->cpu=%d\n", + pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%c ->cpu=%d\n", rcu_state.name, j, (long)rcu_seq_current(&rcu_state.gp_seq), data_race(READ_ONCE(rcu_state.gp_flags)), gp_state_getname(rcu_state.gp_state), data_race(READ_ONCE(rcu_state.gp_state)), - gpk ? data_race(READ_ONCE(gpk->__state)) : ~0, cpu); + gpk ? task_state_to_char(gpk) : '?', cpu); if (gpk) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); @@ -616,12 +616,12 @@ static void rcu_check_gp_kthread_expired_fqs_timer(void) time_after(jiffies, jiffies_fqs + RCU_STALL_MIGHT_MIN) && gpk && !READ_ONCE(gpk->on_rq)) { cpu = task_cpu(gpk); - pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x\n", + pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%c\n", rcu_state.name, (jiffies - jiffies_fqs), (long)rcu_seq_current(&rcu_state.gp_seq), data_race(READ_ONCE(rcu_state.gp_flags)), // Diagnostic read gp_state_getname(RCU_GP_WAIT_FQS), RCU_GP_WAIT_FQS, - data_race(READ_ONCE(gpk->__state))); + task_state_to_char(gpk)); pr_err("\tPossible timer handling issue on cpu=%d timer-softirq=%u\n", cpu, kstat_softirqs_cpu(TIMER_SOFTIRQ, cpu)); } @@ -927,20 +927,13 @@ bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) } EXPORT_SYMBOL_GPL(rcu_check_boost_fail); -/* - * Show the state of the grace-period kthreads. - */ -void show_rcu_gp_kthreads(void) +static noinline_for_stack void show_rcu_state(void) { - unsigned long cbs = 0; - int cpu; unsigned long j; unsigned long ja; unsigned long jr; unsigned long js; unsigned long jw; - struct rcu_data *rdp; - struct rcu_node *rnp; struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); j = jiffies; @@ -948,30 +941,48 @@ void show_rcu_gp_kthreads(void) jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity)); js = j - data_race(READ_ONCE(rcu_state.gp_start)); jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time)); - pr_info("%s: wait state: %s(%d) ->state: %#x ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n", + pr_info("%s: wait state: %s(%d) ->state: %c ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n", rcu_state.name, gp_state_getname(rcu_state.gp_state), data_race(READ_ONCE(rcu_state.gp_state)), - t ? data_race(READ_ONCE(t->__state)) : 0x1ffff, t ? t->rt_priority : 0xffU, + t ? task_state_to_char(t) : '?', t ? t->rt_priority : 0xffU, js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)), (long)data_race(READ_ONCE(rcu_state.gp_seq)), (long)data_race(READ_ONCE(rcu_get_root()->gp_seq_needed)), data_race(READ_ONCE(rcu_state.gp_max)), data_race(READ_ONCE(rcu_state.gp_flags))); +} + +static noinline_for_stack void show_rcu_node(struct rcu_node *rnp) +{ + pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n", + rnp->grplo, rnp->grphi, + (long)data_race(READ_ONCE(rnp->gp_seq)), + (long)data_race(READ_ONCE(rnp->gp_seq_needed)), + data_race(READ_ONCE(rnp->qsmask)), + ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], + ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], + ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], + ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], + data_race(READ_ONCE(rnp->n_boosts))); +} + +/* + * Show the state of the grace-period kthreads. + */ +void show_rcu_gp_kthreads(void) +{ + unsigned long cbs = 0; + int cpu; + struct rcu_data *rdp; + struct rcu_node *rnp; + + show_rcu_state(); rcu_for_each_node_breadth_first(rnp) { if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) && !data_race(READ_ONCE(rnp->qsmask)) && !data_race(READ_ONCE(rnp->boost_tasks)) && !data_race(READ_ONCE(rnp->exp_tasks)) && !data_race(READ_ONCE(rnp->gp_tasks))) continue; - pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n", - rnp->grplo, rnp->grphi, - (long)data_race(READ_ONCE(rnp->gp_seq)), - (long)data_race(READ_ONCE(rnp->gp_seq_needed)), - data_race(READ_ONCE(rnp->qsmask)), - ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], - ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], - ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], - ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], - data_race(READ_ONCE(rnp->n_boosts))); + show_rcu_node(rnp); if (!rcu_is_leaf_node(rnp)) continue; for_each_leaf_node_possible_cpu(rnp, cpu) { @@ -998,8 +1009,7 @@ EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); * This function checks for grace-period requests that fail to motivate * RCU to come out of its idle mode. */ -static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, - const unsigned long gpssdelay) +static void rcu_check_gp_start_stall(struct rcu_node *rnp, const unsigned long gpssdelay) { unsigned long flags; unsigned long j; @@ -1074,7 +1084,7 @@ void rcu_fwd_progress_check(unsigned long j) __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end))); preempt_disable(); rdp = this_cpu_ptr(&rcu_data); - rcu_check_gp_start_stall(rdp->mynode, rdp, j); + rcu_check_gp_start_stall(rdp->mynode, j); preempt_enable(); } for_each_possible_cpu(cpu) { diff --git a/kernel/torture.c b/kernel/torture.c index 77cb3589b19f..8c4e6b2fe8ba 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -577,6 +577,8 @@ static int torture_shuffle(void *arg) */ int torture_shuffle_init(long shuffint) { + int ret; + shuffle_interval = shuffint; shuffle_idle_cpu = -1; @@ -587,7 +589,10 @@ int torture_shuffle_init(long shuffint) } /* Create the shuffler thread */ - return torture_create_kthread(torture_shuffle, NULL, shuffler_task); + ret = torture_create_kthread(torture_shuffle, NULL, shuffler_task); + if (ret) + free_cpumask_var(shuffle_tmp_mask); + return ret; } EXPORT_SYMBOL_GPL(torture_shuffle_init); diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..35954f38ff6a 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -91,6 +91,7 @@ #include "slab.c" #include "spinlock.c" #include "string.c" +#include "srcu.c" #include "sync.c" #include "task.c" #include "time.c" diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c new file mode 100644 index 000000000000..1a2f563640e0 --- /dev/null +++ b/rust/helpers/srcu.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, + const char *name, + struct lock_class_key *key) +{ + return __init_srcu_struct(ssp, name, key); +} + +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + +__rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) +{ + return srcu_read_lock(ssp); +} + +__rust_helper void rust_helper_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ + srcu_read_unlock(ssp, idx); +} + +__rust_helper void rust_helper_srcu_barrier(struct srcu_struct *ssp) +{ + srcu_barrier(ssp); +} + +__rust_helper void rust_helper_synchronize_srcu_expedited(struct srcu_struct *ssp) +{ + synchronize_srcu_expedited(ssp); +} diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..0d6a5f1300c3 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -21,6 +21,7 @@ pub mod rcu; mod refcount; mod set_once; +pub mod srcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use completion::Completion; @@ -31,6 +32,7 @@ pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; +pub use srcu::Srcu; /// Represents a lockdep class. /// diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs new file mode 100644 index 000000000000..723e5e277fd6 --- /dev/null +++ b/rust/kernel/sync/srcu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Sleepable read-copy update (SRCU) support. +//! +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) + +use crate::{ + bindings, + error::to_result, + prelude::*, + sync::LockClassKey, + types::{ + NotThreadSafe, + Opaque, // + }, +}; + +use pin_init::pin_data; + +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. +#[doc(hidden)] +#[macro_export] +macro_rules! new_srcu { + ($($name:literal)?) => { + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_srcu; + +/// Sleepable read-copy update primitive. +/// +/// SRCU readers may sleep while holding the read-side guard. +/// +/// The destructor waits for active readers and callbacks, so it may sleep. +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. +/// +/// # Invariants +/// +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API +/// and it remains pinned and valid until the pinned destructor runs. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct Srcu { + #[pin] + inner: Opaque, +} + +impl Srcu { + /// Creates a new SRCU instance. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + try_pin_init!(Self { + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and + // it remains pinned until `PinnedDrop` runs. + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. + to_result(unsafe { + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) + }) + }), + }) + } + + /// Enters an SRCU read-side critical section. + /// + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical + /// section active and makes `drop` sleep forever. + #[inline] + pub fn read_lock(&self) -> Guard<'_> { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; + + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. + Guard { + srcu: self, + idx, + _not_send: NotThreadSafe, + } + } + + /// Waits until all pre-existing SRCU readers have completed. + #[inline] + pub fn synchronize(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(self.inner.get()) }; + } + + /// Waits until all pre-existing SRCU readers have completed, expedited. + /// + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default + /// and use this variant only when reducing reset or teardown latency is more important + /// than the extra cost. + #[inline] + pub fn synchronize_expedited(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Srcu { + fn drop(self: Pin<&mut Self>) { + let ptr = self.inner.get(); + + if crate::warn_on!( + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` + // and `srcu_readers_active()` only checks the active reader count. + unsafe { bindings::srcu_readers_active(ptr) } + ) { + // `cleanup_srcu_struct()` may return early if there are still active readers. + // This should only happen if a guard was leaked with `mem::forget`, which is + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). + // + // Another consequence of leaking guards is that `call_srcu()` callbacks will + // never run because the grace period can never complete due to permanently + // active readers (i.e. leaked guards). + // + // If this ever happens, that means the guard was leaked by mistake and the + // caller must fix the bug. Sleeping here is intentional and less harmful + // than risking a UAF. + // + // SAFETY: By the type invariants, `self` contains a valid and pinned + // `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(ptr) }; + } + + // Ensure all SRCU callbacks have been finished before freeing. + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::srcu_barrier(ptr) }; + + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::cleanup_srcu_struct(ptr) }; + } +} + +// SAFETY: `srcu_struct` may be shared and used across threads. +unsafe impl Send for Srcu {} +// SAFETY: `srcu_struct` may be shared and used concurrently. +unsafe impl Sync for Srcu {} + +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. +/// +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. +/// +/// # Invariants +/// +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct Guard<'a> { + srcu: &'a Srcu, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Guard<'_> { + /// Explicitly releases the SRCU read-side critical section. + #[inline] + pub fn unlock(self) {} +} + +impl Drop for Guard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, + // which returns a valid index for the SRCU instance. + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; + } +}