diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index b586bc70053e..c7c4ff7c2234 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3625,7 +3625,7 @@ static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; add_new_task_to_grp(new); } @@ -3640,7 +3640,7 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); unsigned long thermal_cap; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /* @@ -3671,7 +3671,7 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) unsigned long flags; struct rq *rq = cpu_rq(cpu); - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; raw_spin_lock_irqsave(&rq->lock, flags); set_window_start(rq); @@ -3682,14 +3682,14 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) static void android_rvh_sched_cpu_dying(void *unused, int cpu) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; clear_walt_request(cpu); } static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsigned int new_cpu) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (new_cpu < 0) return; @@ -3698,14 +3698,14 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign static void android_rvh_sched_fork(void *unused, struct task_struct *p) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; init_new_task_load(p); } static void android_rvh_new_task_stats(void *unused, struct task_struct *p) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; mark_task_starting(p); } @@ -3714,7 +3714,7 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int { struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (!!(curr->flags & PF_IDLE)) { if (hardirq_count() || in_serving_softirq()) @@ -3727,7 +3727,7 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int static void android_rvh_flush_task(void *unused, struct task_struct *p) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; walt_task_dead(p); } @@ -3737,7 +3737,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st u64 wallclock = sched_ktime_clock(); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; wts->last_enqueued_ts = wallclock; sched_update_nr_prod(rq->cpu, true); @@ -3755,7 +3755,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (p == wrq->ed_task) is_ed_task_present(rq, sched_ktime_clock()); @@ -3777,7 +3777,7 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p bool old_misfit, misfit; int change; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *need_update = false; @@ -3815,7 +3815,7 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) unsigned int old_load; struct walt_related_thread_group *grp = NULL; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; rq_lock_irqsave(rq, &rf); old_load = task_load(p); @@ -3837,7 +3837,7 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct unsigned long flags; int cpu = p->cpu; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags); @@ -3852,7 +3852,7 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) u32 old_load; struct walt_related_thread_group *grp; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; set_window_start(rq); wallclock = sched_ktime_clock(); @@ -3876,7 +3876,7 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, u64 wallclock = sched_ktime_clock(); struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (likely(prev != next)) { if (!prev->on_rq) @@ -3894,7 +3894,7 @@ static void android_rvh_resume_cpus(void *unused, struct cpumask *resuming_cpus, struct rq *rq; unsigned long flags; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /* * send a reschedule event on all resumed CPUs @@ -3916,7 +3916,7 @@ static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (cpumask_subset(&wts->cpus_requested, cpus_requested)) *ret = set_cpus_allowed_ptr(p, &wts->cpus_requested); @@ -3926,7 +3926,7 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; wts->last_sleep_ts = 0; wts->wake_up_idle = false; @@ -3939,21 +3939,21 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) static void android_rvh_ttwu_cond(void *unused, bool *cond) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *cond = sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT; } static void android_rvh_sched_exec(void *unused, bool *cond) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *cond = true; } static void android_rvh_build_perf_domains(void *unused, bool *eas_check) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *eas_check = true; } @@ -3986,7 +3986,7 @@ static void register_walt_hooks(void) } atomic64_t walt_irq_work_lastq_ws; -DEFINE_STATIC_KEY_TRUE(walt_disabled); +bool walt_disabled = true; static int walt_init_stop_handler(void *data) { @@ -4026,6 +4026,8 @@ static int walt_init_stop_handler(void *data) walt_update_cluster_topology(); + walt_disabled = false; + for_each_possible_cpu(cpu) { raw_spin_unlock(&cpu_rq(cpu)->lock); } @@ -4056,7 +4058,6 @@ static void walt_init(void) walt_cfs_init(); stop_machine(walt_init_stop_handler, NULL, NULL); - static_branch_disable(&walt_disabled); hdr = register_sysctl_table(walt_base_table); kmemleak_not_leak(hdr); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index aafedd563cfc..1c76ba19e24b 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -32,7 +32,7 @@ /* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */ #define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1) -extern struct static_key_true walt_disabled; +extern bool walt_disabled; enum task_event { PUT_PREV_TASK = 0, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index d7b9cac3fdab..cfd63eb6aa19 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -665,7 +665,7 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, int sync; int sibling_count_hint; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); @@ -684,7 +684,7 @@ static inline struct task_struct *task_of(struct sched_entity *se) static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vruntime) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (entity_is_task(se)) { unsigned long thresh = sysctl_sched_latency; @@ -710,7 +710,7 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task) { struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (task && current->signal && (current->signal->oom_score_adj == 0) && @@ -723,7 +723,7 @@ static void walt_binder_low_latency_clear(void *unused, struct binder_transactio { struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (wts->low_latency & WALT_LOW_LATENCY_BINDER) wts->low_latency &= ~WALT_LOW_LATENCY_BINDER; diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index f6109e466487..30202177a806 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -491,7 +491,7 @@ static void walt_lb_tick(void *unused, struct rq *rq) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (!rq->misfit_task_load || !walt_fair_task(p)) return; @@ -633,7 +633,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); bool help_min_cap = false; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /*Cluster isn't initialized until after WALT is enabled*/ @@ -729,7 +729,7 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, int busiest_cpu = -1; struct cpumask src_mask; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *done = 1; *busiest = NULL; @@ -767,7 +767,7 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, struct task_struct *p, int new_cpu, int *detached) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /* * WALT expects both source and destination rqs to be @@ -796,7 +796,7 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, static void walt_nohz_balancer_kick(void *unused, struct rq *rq, unsigned int *flags, int *done) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *done = 1; @@ -815,7 +815,7 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, { bool to_lower; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(task_cpu(p)); diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index d08e730fcbce..5ae1936151ad 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -24,7 +24,7 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task int cluster; int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (!ret) @@ -119,7 +119,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c int ret, target = -1; struct cpumask *lowest_mask; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /* For anything but wake ups, just return the task_cpu */