From 915ef0bd8b0e3a2fcf5b4490970b85d18d6c4a28 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:39:37 -0700 Subject: [PATCH] sched: Improve the scheduler This change is for general scheduler improvement. Change-Id: I064002ae2c7d8ffd444975ad74524f5fa8de5465 Signed-off-by: Shaleen Agrawal Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 58 ++++++++++++++++++++++++++++++++---- kernel/sched/walt/walt.h | 4 +++ kernel/sched/walt/walt_cfs.c | 10 +++++++ kernel/sched/walt/walt_lb.c | 17 +++++++++-- kernel/sched/walt/walt_rt.c | 2 ++ 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 33822e50ec52..4b0dcd61e4fe 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3723,6 +3723,8 @@ static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new) { + if (static_branch_unlikely(&walt_disabled)) + return; add_new_task_to_grp(new); } @@ -3736,6 +3738,9 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); unsigned long thermal_cap; + if (static_branch_unlikely(&walt_disabled)) + return; + /* * thermal_pressure = max_capacity - curr_cap_as_per_thermal. * so, @@ -3764,6 +3769,8 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) unsigned long flags; struct rq *rq = cpu_rq(cpu); + if (static_branch_unlikely(&walt_disabled)) + return; raw_spin_lock_irqsave(&rq->lock, flags); set_window_start(rq); raw_spin_unlock_irqrestore(&rq->lock, flags); @@ -3773,11 +3780,15 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) static void android_rvh_sched_cpu_dying(void *unused, int cpu) { + if (static_branch_unlikely(&walt_disabled)) + return; clear_walt_request(cpu); } static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsigned int new_cpu) { + if (static_branch_unlikely(&walt_disabled)) + return; if (new_cpu < 0) return; fixup_busy_time(p, (int) new_cpu); @@ -3785,11 +3796,15 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign static void android_rvh_sched_fork(void *unused, struct task_struct *p) { + if (static_branch_unlikely(&walt_disabled)) + return; init_new_task_load(p); } static void android_rvh_new_task_stats(void *unused, struct task_struct *p) { + if (static_branch_unlikely(&walt_disabled)) + return; mark_task_starting(p); } @@ -3797,6 +3812,8 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int { struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (!!(curr->flags & PF_IDLE)) { if (hardirq_count() || in_serving_softirq()) walt_sched_account_irqend(cpu, curr, delta); @@ -3808,6 +3825,8 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int static void android_rvh_flush_task(void *unused, struct task_struct *p) { + if (static_branch_unlikely(&walt_disabled)) + return; walt_task_dead(p); } @@ -3816,6 +3835,8 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st u64 wallclock = sched_ktime_clock(); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; wts->last_enqueued_ts = wallclock; sched_update_nr_prod(rq->cpu, true); @@ -3838,6 +3859,8 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st */ struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (p == wrq->ed_task) is_ed_task_present(rq, sched_ktime_clock()); @@ -3858,6 +3881,8 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p bool old_misfit, misfit; int change; + if (static_branch_unlikely(&walt_disabled)) + return; *need_update = false; if (!p) { @@ -3894,6 +3919,8 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) unsigned int old_load; struct walt_related_thread_group *grp = NULL; + if (static_branch_unlikely(&walt_disabled)) + return; rq_lock_irqsave(rq, &rf); old_load = task_load(p); wallclock = sched_ktime_clock(); @@ -3914,6 +3941,8 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct unsigned long flags; int cpu = p->cpu; + if (static_branch_unlikely(&walt_disabled)) + return; if (!sched_predl) return; @@ -3929,6 +3958,8 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) u32 old_load; struct walt_related_thread_group *grp; + if (static_branch_unlikely(&walt_disabled)) + return; set_window_start(rq); wallclock = sched_ktime_clock(); @@ -3958,6 +3989,8 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, u64 wallclock = sched_ktime_clock(); struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (likely(prev != next)) { if (!prev->on_rq) wts->last_sleep_ts = wallclock; @@ -3974,6 +4007,8 @@ static void android_rvh_resume_cpus(void *unused, struct cpumask *resuming_cpus, struct rq *rq; unsigned long flags; + if (static_branch_unlikely(&walt_disabled)) + return; /* * send a reschedule event on all resumed CPUs * which trigger newly idle load balance. @@ -3994,6 +4029,8 @@ static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (cpumask_subset(&wts->cpus_requested, cpus_requested)) *ret = set_cpus_allowed_ptr(p, &wts->cpus_requested); } @@ -4002,6 +4039,8 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; wts->last_sleep_ts = 0; wts->wake_up_idle = false; wts->boost = 0; @@ -4012,16 +4051,22 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) static void android_rvh_ttwu_cond(void *unused, bool *cond) { + if (static_branch_unlikely(&walt_disabled)) + return; *cond = sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT; } static void android_rvh_sched_exec(void *unused, bool *cond) { + if (static_branch_unlikely(&walt_disabled)) + return; *cond = true; } static void android_rvh_build_perf_domains(void *unused, bool *eas_check) { + if (static_branch_unlikely(&walt_disabled)) + return; *eas_check = true; } @@ -4054,6 +4099,7 @@ static void register_walt_hooks(void) } atomic64_t walt_irq_work_lastq_ws; +DEFINE_STATIC_KEY_TRUE(walt_disabled); static int walt_init_stop_handler(void *data) { @@ -4089,10 +4135,6 @@ static int walt_init_stop_handler(void *data) atomic64_set(&walt_irq_work_lastq_ws, window_start_ns); - register_walt_hooks(); - walt_lb_init(); - walt_rt_init(); - walt_cfs_init(); create_default_coloc_group(); walt_update_cluster_topology(); @@ -4101,7 +4143,6 @@ static int walt_init_stop_handler(void *data) raw_spin_unlock(&cpu_rq(cpu)->lock); } read_unlock(&tasklist_lock); - return 0; } @@ -4116,7 +4157,14 @@ static int walt_module_init(void) BUG_ON(alloc_related_thread_groups()); walt_init_cycle_counter(); init_clusters(); + + register_walt_hooks(); + walt_lb_init(); + walt_rt_init(); + walt_cfs_init(); + stop_machine(walt_init_stop_handler, NULL, NULL); + static_branch_disable(&walt_disabled); hdr = register_sysctl_table(walt_base_table); kmemleak_not_leak(hdr); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 0315be82852a..c3e485844ba4 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -9,6 +9,8 @@ #include "../../../kernel/sched/sched.h" #include "../../../fs/proc/internal.h" #include +#include + #include #ifdef CONFIG_HZ_300 @@ -38,6 +40,8 @@ /* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */ #define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1) +extern struct static_key_true walt_disabled; + enum task_event { PUT_PREV_TASK = 0, PICK_NEXT_TASK = 1, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index c99337e54ea2..5de798da3d14 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -686,6 +686,8 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); int sibling_count_hint = p->wake_q_head ? p->wake_q_head->count : 1; + if (static_branch_unlikely(&walt_disabled)) + return; *target_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, sync, sibling_count_hint); if (unlikely(*target_cpu < 0)) *target_cpu = prev_cpu; @@ -710,6 +712,8 @@ static unsigned long task_h_load(struct task_struct *p) static void walt_update_misfit_status(void *unused, struct task_struct *p, struct rq *rq, bool *need_update) { + if (static_branch_unlikely(&walt_disabled)) + return; *need_update = false; if (!p) { @@ -736,6 +740,8 @@ static inline struct task_struct *task_of(struct sched_entity *se) static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vruntime) { + if (static_branch_unlikely(&walt_disabled)) + return; if (entity_is_task(se)) { unsigned long thresh = sysctl_sched_latency; @@ -760,6 +766,8 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task) { struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (task && current->signal && (current->signal->oom_score_adj == 0) && (current->prio < DEFAULT_PRIO)) @@ -770,6 +778,8 @@ static void walt_binder_low_latency_clear(void *unused, struct binder_transactio { struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (wts->low_latency) wts->low_latency = false; } diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 648adfdf066d..24a37c47be0a 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -489,6 +489,8 @@ static void walt_lb_tick(void *unused, struct rq *rq) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (!rq->misfit_task_load) return; @@ -563,15 +565,18 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, { int this_cpu = this_rq->cpu; struct walt_rq *wrq = (struct walt_rq *) this_rq->android_vendor_data1; - int order_index = wrq->cluster->id; + int order_index; int cluster = 0; int busy_cpu; bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); bool help_min_cap; - if (unlikely(!cpu_array)) + if (static_branch_unlikely(&walt_disabled)) return; + /*Cluster isn't initialized until after WALT is enabled*/ + order_index = wrq->cluster->id; + /* * newly idle load balance is completely handled here, so * set done to skip the load balance by the caller. @@ -650,6 +655,8 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, int busiest_cpu = -1; struct cpumask src_mask; + if (static_branch_unlikely(&walt_disabled)) + return; *done = 1; *busiest = NULL; @@ -686,6 +693,8 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, struct task_struct *p, int new_cpu, int *detached) { + if (static_branch_unlikely(&walt_disabled)) + return; /* * WALT expects both source and destination rqs to be * held when set_task_cpu() is called on a queued task. @@ -713,6 +722,8 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, static void walt_nohz_balancer_kick(void *unused, struct rq *rq, unsigned int *flags, int *done) { + if (static_branch_unlikely(&walt_disabled)) + return; *done = 1; /* @@ -730,6 +741,8 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, { bool to_lower; + if (static_branch_unlikely(&walt_disabled)) + return; to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(task_cpu(p)); if (_walt_can_migrate_task(p, dst_cpu, to_lower)) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 87a783c9b974..862df51479f9 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -22,6 +22,8 @@ static void rt_energy_aware_wake_cpu(void *unused, struct task_struct *task, int cluster; int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0; + if (static_branch_unlikely(&walt_disabled)) + return; if (!ret) return; /* No targets found */