From 427c7e27f6aa8955318e47564188607183f36481 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 17 Aug 2022 16:53:00 -0700 Subject: [PATCH 1/4] kernel/sched/walt: start domain rebuilding and wait There is a race condition in cpufreq.c in which the energy model is not updated until after rebuild domains has completed. If this is performed out of order, the rebuild domains will fail and the perf domains will be freed, causing walt_find_energy_efficient_cpu to fail to perform active migration. After topology update is complete and walt initialization moves forward, rebuild the domains waiting for the completion of the rebuild before performing the walt update of the toplogy, guranteeing that the perf domains are populated properly even if cpufreq builds the domains the first time, out of order. Change-Id: I28499c4318580da54735347be0ea00867e68d94b Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 10 ++++++++++ kernel/sched/walt/walt.h | 1 + 2 files changed, 11 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 23f9a01f9a26..5f4bd7949cc5 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4440,6 +4440,14 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } +static DECLARE_COMPLETION(rebuild_domains_completion); +static void rebuild_sd_workfn(struct work_struct *work) +{ + rebuild_sched_domains(); + complete(&rebuild_domains_completion); +} +static DECLARE_WORK(rebuild_sd_work, rebuild_sd_workfn); + static void walt_do_sched_yield(void *unused, struct rq *rq) { struct task_struct *curr = rq->curr; @@ -4573,6 +4581,8 @@ static void walt_init(struct work_struct *work) walt_cfs_init(); walt_halt_init(); wait_for_completion_interruptible(&tick_sched_clock_completion); + schedule_work(&rebuild_sd_work); + wait_for_completion_interruptible(&rebuild_domains_completion); stop_machine(walt_init_stop_handler, NULL, NULL); hdr = register_sysctl_table(walt_base_table); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index c72e68120ee5..6274f7a49071 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -189,6 +189,7 @@ extern void walt_init_foreground_tg(struct task_group *tg); extern int register_walt_callback(void); extern int input_boost_init(void); extern int core_ctl_init(void); +extern void rebuild_sched_domains(void); extern atomic64_t walt_irq_work_lastq_ws; extern unsigned int __read_mostly sched_ravg_window; From 5ca19b7ae16cbe499556896f2eddc6dbc469eaac Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Aug 2022 16:37:20 -0700 Subject: [PATCH 2/4] kernel/sched/walt: rd->pd not getting updated soon enough If the energy model is not updated by the time rebalance_domains is called for the first time, the root domain -> perf_domain values will get erased completely, and will be invalid. The responsibility for this race is outside of the walt module. walt_find_energy_efficient_cpu() relies upon rd->pd being valid and cannot function properly without it. This is critical to proper walt functionality and it cannot move forward without it. Decisvely catch the failure. Change-Id: Iee8030afd487dcee1d9cc18c2d655ecc26938279 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 5f4bd7949cc5..be0db1478575 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4559,6 +4559,7 @@ static void walt_init(struct work_struct *work) { struct ctl_table_header *hdr; static atomic_t already_inited = ATOMIC_INIT(0); + struct root_domain *rd = cpu_rq(cpumask_first(cpu_active_mask))->rd; int i; might_sleep(); @@ -4600,6 +4601,11 @@ static void walt_init(struct work_struct *work) } topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_ARCH, cpu_online_mask); + + if (!rcu_dereference(rd->pd)) + WALT_BUG(WALT_BUG_WALT, NULL, + "root domain's perf-domain values not initialized rd->pd=%d.", + rd->pd); } static DECLARE_WORK(walt_init_work, walt_init); From c15a0101ac3ce74addf41ae00ea4c6428d107a90 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 3 Oct 2022 14:18:52 -0700 Subject: [PATCH 3/4] sched/walt: Update cpu capacity during thermal pressure Ensure that all thermal pressure gets accounted in capacity_orig as soon as it is set, rather than during the window rollover period. Change-Id: Ia22cb87f0d1fcb23f1a932f9b70637861b56ae33 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index be0db1478575..e3ebd1bdedc4 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3808,11 +3808,6 @@ static void walt_irq_work(struct irq_work *irq_work) level++; } - if (!is_migration) { - for_each_cpu(cpu, &lock_cpus) - update_cpu_capacity_helper(cpu); - } - __walt_irq_work_locked(is_migration, &lock_cpus); for_each_cpu(cpu, &lock_cpus) @@ -4440,6 +4435,13 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } +static void android_rvh_update_thermal_stats(void *unused, int cpu) +{ + if (unlikely(walt_disabled)) + return; + update_cpu_capacity_helper(cpu); +} + static DECLARE_COMPLETION(rebuild_domains_completion); static void rebuild_sd_workfn(struct work_struct *work) { @@ -4492,6 +4494,7 @@ static void register_walt_hooks(void) register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); register_trace_android_rvh_do_sched_yield(walt_do_sched_yield, NULL); + register_trace_android_rvh_update_thermal_stats(android_rvh_update_thermal_stats, NULL); } atomic64_t walt_irq_work_lastq_ws; From 8327be044c2921944c55c8ed8decc1a79e623b70 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 14 Oct 2022 15:09:36 -0700 Subject: [PATCH 4/4] kernel/sched/walt: ensure em_pd creation before rebuild_sched_domains rebuild_sched_domains() is called during walt_init with the assumption that cpufreq_online has actually completed before the execution of walt_init. This is only partially true. cpufreq_online() will notify update_topology_flags_workfn once the policies have been created. However, the em creation is after this point in time. This creates a race condition between cpufreq_online's creation of the energy model through it's call to register_em(policy), and walt's execution of rebuild_sched_domains. If walt init completes quickly, and rebuild_sched_domains() is executed quickly, it is possible it will complete before the energy model is registered. Thus, the perf domains will still be unpopulated, and the error detection for perf domains will cause the system to crash. Address this by ensuring that the energy model has run for each cpu in the system by checking the same field that em_create_pd must populate for build_perf_domains to work properly, the em_pd field in the cpu device. Additionally ensure that this is only attempted if the domains are not yet setup properly and it is known that corrective action must take place. Change-Id: I8f5150fe4e2e37b3820e809232f26ff207723601 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 51 +++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index e3ebd1bdedc4..4507cb2d02a2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -4443,12 +4444,37 @@ static void android_rvh_update_thermal_stats(void *unused, int cpu) } static DECLARE_COMPLETION(rebuild_domains_completion); +static void rebuild_sd_workfn(struct work_struct *work); +static DECLARE_WORK(rebuild_sd_work, rebuild_sd_workfn); + +/** rebuild_sd_workfn + * + * rebuild the sched domains (and therefore the perf + * domains). It is absolutely necessary that the + * em_pds are created for each cpu device before + * proceeding, and this must complete for walt to + * function properly. + */ static void rebuild_sd_workfn(struct work_struct *work) { + int cpu; + struct device *cpu_dev; + + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (cpu_dev->em_pd) + continue; + + WARN_ONCE(true, "must wait for perf domains to be created"); + schedule_work(&rebuild_sd_work); + + /* do not rebuild domains yet, and do not complete this action */ + return; + } + rebuild_sched_domains(); complete(&rebuild_domains_completion); } -static DECLARE_WORK(rebuild_sd_work, rebuild_sd_workfn); static void walt_do_sched_yield(void *unused, struct rq *rq) { @@ -4585,8 +4611,22 @@ static void walt_init(struct work_struct *work) walt_cfs_init(); walt_halt_init(); wait_for_completion_interruptible(&tick_sched_clock_completion); - schedule_work(&rebuild_sd_work); - wait_for_completion_interruptible(&rebuild_domains_completion); + + if (!rcu_dereference(rd->pd)) { + /* + * perf domains not properly configured. this is a must as + * create_util_to_cost depends on rd->pd being properly + * initialized. + */ + schedule_work(&rebuild_sd_work); + wait_for_completion_interruptible(&rebuild_domains_completion); + } + + if (!rcu_dereference(rd->pd)) + WALT_BUG(WALT_BUG_WALT, NULL, + "root domain's perf-domain values not initialized rd->pd=%d.", + rd->pd); + stop_machine(walt_init_stop_handler, NULL, NULL); hdr = register_sysctl_table(walt_base_table); @@ -4604,11 +4644,6 @@ static void walt_init(struct work_struct *work) } topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_ARCH, cpu_online_mask); - - if (!rcu_dereference(rd->pd)) - WALT_BUG(WALT_BUG_WALT, NULL, - "root domain's perf-domain values not initialized rd->pd=%d.", - rd->pd); } static DECLARE_WORK(walt_init_work, walt_init);