diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index fa8b913bb2d6..54b22401b581 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -50,6 +50,7 @@ struct waltgov_policy { s64 down_rate_delay_ns; unsigned int next_freq; unsigned int cached_raw_freq; + unsigned int driving_cpu; /* The next fields are only needed if fast switch cannot be used: */ struct irq_work irq_work; @@ -70,6 +71,7 @@ struct waltgov_cpu { unsigned long util; unsigned long max; unsigned int flags; + unsigned int reasons; }; DEFINE_PER_CPU(struct waltgov_callback *, waltgov_cb_data); @@ -224,19 +226,24 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, { struct cpufreq_policy *policy = wg_policy->policy; unsigned int freq, raw_freq, final_freq; + struct waltgov_cpu *wg_driv_cpu = &per_cpu(waltgov_cpu, wg_policy->driving_cpu); raw_freq = walt_map_util_freq(util, wg_policy, max, wg_cpu->cpu); freq = raw_freq; if (wg_policy->tunables->adaptive_high_freq) { - if (raw_freq < wg_policy->tunables->adaptive_low_freq) + if (raw_freq < wg_policy->tunables->adaptive_low_freq) { freq = wg_policy->tunables->adaptive_low_freq; - else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) + wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_LOW; + } else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) { freq = wg_policy->tunables->adaptive_high_freq; + wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_HIGH; + } } trace_waltgov_next_freq(policy->cpu, util, max, raw_freq, freq, policy->min, policy->max, - wg_policy->cached_raw_freq, wg_policy->need_freq_update); + wg_policy->cached_raw_freq, wg_policy->need_freq_update, + wg_driv_cpu->cpu, wg_driv_cpu->reasons); if (wg_policy->cached_raw_freq && freq == wg_policy->cached_raw_freq && !wg_policy->need_freq_update) @@ -259,7 +266,8 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) unsigned long util; wg_cpu->max = max; - util = cpu_util_freq_walt(wg_cpu->cpu, &wg_cpu->walt_load); + wg_cpu->reasons = 0; + util = cpu_util_freq_walt(wg_cpu->cpu, &wg_cpu->walt_load, &wg_cpu->reasons); return uclamp_rq_util_with(rq, util, NULL); } @@ -270,6 +278,16 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) #define DEFAULT_PRIME_RTG_BOOST_FREQ 0 #define DEFAULT_TARGET_LOAD_THRESH 1024 #define DEFAULT_TARGET_LOAD_SHIFT 4 +static inline void max_and_reason(unsigned long *cur_util, unsigned long boost_util, + struct waltgov_cpu *wg_cpu, unsigned int reason) +{ + if (boost_util >= *cur_util) { + *cur_util = boost_util; + wg_cpu->reasons = reason; + wg_cpu->wg_policy->driving_cpu = wg_cpu->cpu; + } +} + static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_util, unsigned long nl, unsigned long *util, unsigned long *max) @@ -281,22 +299,22 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut unsigned long pl = wg_cpu->walt_load.pl; if (is_rtg_boost) - *util = max(*util, wg_policy->rtg_boost_util); + max_and_reason(util, wg_policy->rtg_boost_util, wg_cpu, CPUFREQ_REASON_RTG_BOOST); is_hiload = (cpu_util >= mult_frac(wg_policy->avg_cap, wg_policy->tunables->hispeed_load, 100)); if (is_hiload && !is_migration) - *util = max(*util, wg_policy->hispeed_util); + max_and_reason(util, wg_policy->hispeed_util, wg_cpu, CPUFREQ_REASON_HISPEED); if (is_hiload && nl >= mult_frac(cpu_util, NL_RATIO, 100)) - *util = *max; + max_and_reason(util, *max, wg_cpu, CPUFREQ_REASON_NWD); if (wg_policy->tunables->pl) { if (sysctl_sched_conservative_pl) pl = mult_frac(pl, TARGET_LOAD, 100); - *util = max(*util, pl); + max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL); } } @@ -346,6 +364,7 @@ static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 tim if (j_util * max >= j_max * util) { util = j_util; max = j_max; + wg_policy->driving_cpu = j; } waltgov_walt_adjust(j_wg_cpu, j_util, j_nl, &util, &max); diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index cf32709f86c7..2407d60c3ec1 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #undef TRACE_SYSTEM @@ -421,9 +421,10 @@ TRACE_EVENT(sched_load_to_gov, int freq_aggr, u64 load, int policy, int big_task_rotation, unsigned int user_hint, - struct walt_rq *wrq), + struct walt_rq *wrq, + unsigned int reasons), TP_ARGS(rq, aggr_grp_load, tt_load, freq_aggr, load, policy, - big_task_rotation, user_hint, wrq), + big_task_rotation, user_hint, wrq, reasons), TP_STRUCT__entry( __field(int, cpu) @@ -440,6 +441,7 @@ TRACE_EVENT(sched_load_to_gov, __field(u64, load) __field(int, big_task_rotation) __field(unsigned int, user_hint) + __field(unsigned int, reasons) ), TP_fast_assign( @@ -458,14 +460,15 @@ TRACE_EVENT(sched_load_to_gov, __entry->load = load; __entry->big_task_rotation = big_task_rotation; __entry->user_hint = user_hint; + __entry->reasons = reasons; ), - TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr=%d tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d user_hint=%u", + TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr=%d tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d user_hint=%u reasons=0x%x", __entry->cpu, __entry->policy, __entry->ed_task_pid, __entry->aggr_grp_load, __entry->freq_aggr, __entry->tt_load, __entry->rq_ps, __entry->grp_rq_ps, __entry->nt_ps, __entry->grp_nt_ps, __entry->pl, __entry->load, - __entry->big_task_rotation, __entry->user_hint) + __entry->big_task_rotation, __entry->user_hint, __entry->reasons) ); TRACE_EVENT(core_ctl_eval_need, @@ -721,9 +724,10 @@ TRACE_EVENT(waltgov_util_update, TRACE_EVENT(waltgov_next_freq, TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, unsigned int raw_freq, unsigned int freq, unsigned int policy_min_freq, unsigned int policy_max_freq, - unsigned int cached_raw_freq, bool need_freq_update), + unsigned int cached_raw_freq, bool need_freq_update, unsigned int driving_cpu, + unsigned int reason), TP_ARGS(cpu, util, max, raw_freq, freq, policy_min_freq, policy_max_freq, - cached_raw_freq, need_freq_update), + cached_raw_freq, need_freq_update, driving_cpu, reason), TP_STRUCT__entry( __field(unsigned int, cpu) __field(unsigned long, util) @@ -735,6 +739,8 @@ TRACE_EVENT(waltgov_next_freq, __field(unsigned int, cached_raw_freq) __field(bool, need_freq_update) __field(unsigned int, rt_util) + __field(unsigned int, driving_cpu) + __field(unsigned int, reason) ), TP_fast_assign( __entry->cpu = cpu; @@ -746,9 +752,11 @@ TRACE_EVENT(waltgov_next_freq, __entry->policy_max_freq = policy_max_freq; __entry->cached_raw_freq = cached_raw_freq; __entry->need_freq_update = need_freq_update; - __entry->rt_util = cpu_util_rt(cpu_rq(cpu)); + __entry->rt_util = cpu_util_rt(cpu_rq(cpu)); + __entry->driving_cpu = driving_cpu; + __entry->reason = reason; ), - TP_printk("cpu=%u util=%lu max=%lu raw_freq=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u", + TP_printk("cpu=%u util=%lu max=%lu raw_freq=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u driv_cpu=%u reason=0x%x", __entry->cpu, __entry->util, __entry->max, @@ -758,7 +766,9 @@ TRACE_EVENT(waltgov_next_freq, __entry->policy_max_freq, __entry->cached_raw_freq, __entry->need_freq_update, - __entry->rt_util) + __entry->rt_util, + __entry->driving_cpu, + __entry->reason) ); TRACE_EVENT(walt_active_load_balance, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c2612cde457c..30052ac15cd7 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -590,30 +590,41 @@ should_apply_suh_freq_boost(struct walt_sched_cluster *cluster) return is_cluster_hosting_top_app(cluster); } -static inline u64 freq_policy_load(struct rq *rq) +static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_sched_cluster *cluster = wrq->cluster; u64 aggr_grp_load = cluster->aggr_grp_load; - u64 load, tt_load = 0; + u64 load, tt_load = 0, kload = 0; struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu_of(rq)); if (wrq->ed_task != NULL) { load = sched_ravg_window; + *reason = CPUFREQ_REASON_EARLY_DET; goto done; } - if (sched_freq_aggr_en) + if (sched_freq_aggr_en) { load = wrq->prev_runnable_sum + aggr_grp_load; + *reason = CPUFREQ_REASON_FREQ_AGR; + } else load = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; - if (cpu_ksoftirqd && READ_ONCE(cpu_ksoftirqd->__state) == TASK_RUNNING) - load = max_t(u64, load, task_load(cpu_ksoftirqd)); + if (cpu_ksoftirqd && READ_ONCE(cpu_ksoftirqd->__state) == TASK_RUNNING) { + kload = task_load(cpu_ksoftirqd); + if (kload > load) { + load = kload; + *reason = CPUFREQ_REASON_KSOFTIRQD; + } + } tt_load = top_task_load(rq); - load = max_t(u64, load, tt_load); + if (tt_load > load) { + load = tt_load; + *reason = CPUFREQ_REASON_TT_LOAD; + } if (should_apply_suh_freq_boost(cluster)) { if (is_suh_max()) @@ -621,26 +632,27 @@ static inline u64 freq_policy_load(struct rq *rq) else load = div64_u64(load * sysctl_sched_user_hint, (u64)100); + *reason = CPUFREQ_REASON_SUH; } done: trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en, load, 0, walt_rotation_enabled, - sysctl_sched_user_hint, wrq); + sysctl_sched_user_hint, wrq, *reason); return load; } static bool rtgb_active; static inline unsigned long -__cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) +__cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reason) { u64 util; struct rq *rq = cpu_rq(cpu); unsigned long capacity = capacity_orig_of(cpu); struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - util = div64_u64(freq_policy_load(rq), + util = div64_u64(freq_policy_load(rq, reason), sched_ravg_window >> SCHED_CAPACITY_SHIFT); if (walt_load) { @@ -665,7 +677,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) (max(orig, mult_frac(other, x, 100))) unsigned long -cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) +cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reason) { struct walt_cpu_load wl_other = {0}; unsigned long util = 0, util_other = 0; @@ -673,13 +685,13 @@ cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) int i, mpct = sysctl_sched_asym_cap_sibling_freq_match_pct; if (!cpumask_test_cpu(cpu, &asym_cap_sibling_cpus)) - return __cpu_util_freq_walt(cpu, walt_load); + return __cpu_util_freq_walt(cpu, walt_load, reason); for_each_cpu(i, &asym_cap_sibling_cpus) { if (i == cpu) - util = __cpu_util_freq_walt(cpu, walt_load); + util = __cpu_util_freq_walt(cpu, walt_load, reason); else - util_other = __cpu_util_freq_walt(i, &wl_other); + util_other = __cpu_util_freq_walt(i, &wl_other, reason); } if (cpu == cpumask_last(&asym_cap_sibling_cpus)) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index d1f8f3b0781a..9e573452d642 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _WALT_H @@ -297,6 +297,18 @@ extern unsigned int sched_lib_mask_force; #define WALT_CPUFREQ_EARLY_DET (1U << 4) #define WALT_CPUFREQ_BOOST_UPDATE (1U << 5) +#define CPUFREQ_REASON_PL (1U << 1) +#define CPUFREQ_REASON_EARLY_DET (1U << 2) +#define CPUFREQ_REASON_RTG_BOOST (1U << 3) +#define CPUFREQ_REASON_HISPEED (1U << 4) +#define CPUFREQ_REASON_NWD (1U << 5) +#define CPUFREQ_REASON_FREQ_AGR (1U << 6) +#define CPUFREQ_REASON_KSOFTIRQD (1U << 7) +#define CPUFREQ_REASON_TT_LOAD (1U << 8) +#define CPUFREQ_REASON_SUH (1U << 9) +#define CPUFREQ_REASON_ADAPTIVE_LOW (1U << 10) +#define CPUFREQ_REASON_ADAPTIVE_HIGH (1U << 11) + #define NO_BOOST 0 #define FULL_THROTTLE_BOOST 1 #define CONSERVATIVE_BOOST 2 @@ -366,7 +378,8 @@ static inline void waltgov_run_callback(struct rq *rq, unsigned int flags) cb->func(cb, walt_sched_clock(), flags); } -extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load); +extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, + unsigned int *reason); int waltgov_register(void); extern void walt_lb_init(void);