diff --git a/kernel/smp.c b/kernel/smp.c index dc6582bb35d0..52dffc86555c 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -137,10 +137,10 @@ csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd) trace_csd_function_exit(func, csd); } -#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG - static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled); +#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG + /* * Parse the csdlock_debug= kernel boot parameter. * @@ -342,6 +342,10 @@ static __always_inline void csd_lock_wait(call_single_data_t *csd) smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); } #else +static __always_inline void __csd_lock_wait(call_single_data_t *csd) +{ +} + static void csd_lock_record(call_single_data_t *csd) { } @@ -354,8 +358,23 @@ static __always_inline void csd_lock_wait(call_single_data_t *csd) static __always_inline void csd_lock(call_single_data_t *csd) { - csd_lock_wait(csd); - csd->node.u_flags |= CSD_FLAG_LOCK; + if (IS_ENABLED(CONFIG_CSD_LOCK_WAIT_DEBUG) && + static_branch_unlikely(&csdlock_debug_enabled)) { + + for (;;) { + unsigned int flags; + + __csd_lock_wait(csd); + flags = READ_ONCE(csd->node.u_flags); + + if (!(flags & CSD_FLAG_LOCK) && + try_cmpxchg_acquire(&csd->node.u_flags, &flags, flags | CSD_FLAG_LOCK)) + break; + } + } else { + csd_lock_wait(csd); + csd->node.u_flags |= CSD_FLAG_LOCK; + } /* * prevent CPU from reordering the above assignment