diff --git a/kernel/sched/core.c b/kernel/sched/core.c index acb5894e7714..162b24c76077 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4917,6 +4917,34 @@ static inline void finish_task(struct task_struct *prev) smp_store_release(&prev->on_cpu, 0); } +/* + * Only called from __schedule context + * + * There are some cases where we are going to re-do the action + * that added the balance callbacks. We may not be in a state + * where we can run them, so just zap them so they can be + * properly re-added on the next time around. This is similar + * handling to running the callbacks, except we just don't call + * them. + */ +static void zap_balance_callbacks(struct rq *rq) +{ + struct balance_callback *next, *head; + bool found = false; + + lockdep_assert_rq_held(rq); + + head = rq->balance_callback; + while (head) { + if (head == &balance_push_callback) + found = true; + next = head->next; + head->next = NULL; + head = next; + } + rq->balance_callback = found ? &balance_push_callback : NULL; +} + static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) { void (*func)(struct rq *rq); @@ -6862,10 +6890,14 @@ static void __sched notrace __schedule(int sched_mode) rq_set_donor(rq, next); if (unlikely(next->blocked_on)) { next = find_proxy_task(rq, next, &rf); - if (!next) + if (!next) { + zap_balance_callbacks(rq); goto pick_again; - if (next == rq->idle) + } + if (next == rq->idle) { + zap_balance_callbacks(rq); goto keep_resched; + } } if (rq->donor == prev_donor && prev != next) { struct task_struct *donor = rq->donor;