github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-31 03:27:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched/walt: Add parameter for CPUFREQ reasons

Browse Source 
Currently, cpufreq traces contain a "flags" parameter, which helps
understand why cpufreq updates were triggered.

In this patch, a "reasons" parameter is introduced to provide an accurate
guidance towards *why* the cpufreq chosen was selected, and the specific
reason behind any applied value adds that have potentially modified the
true load.

Change-Id: I88a62569e0a1ec1555a89161aa532db6c72df334
Signed-off-by: Stephen Dickey <quic_dickey@quicinc.com>
Signed-off-by: Shaleen Agrawal <quic_shalagra@quicinc.com>
This commit is contained in:
Stephen Dickey 2022-01-18 14:21:56 -08:00 committed by Rishabh Bhatnagar
parent a2ed1ebb5d
commit 498e57c028
4 changed files with 87 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
kernel/sched/walt/cpufreq_walt.c
View File

@ -50,6 +50,7 @@ struct waltgov_policy {
s64 down_rate_delay_ns;
unsigned int next_freq;
unsigned int cached_raw_freq;
unsigned int driving_cpu;
/* The next fields are only needed if fast switch cannot be used: */
struct irq_work irq_work;
@ -70,6 +71,7 @@ struct waltgov_cpu {
unsigned long util;
unsigned long max;
unsigned int flags;
unsigned int reasons;
};
DEFINE_PER_CPU(struct waltgov_callback *, waltgov_cb_data);
@ -224,19 +226,24 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy,
{
struct cpufreq_policy *policy = wg_policy->policy;
unsigned int freq, raw_freq, final_freq;
struct waltgov_cpu *wg_driv_cpu = &per_cpu(waltgov_cpu, wg_policy->driving_cpu);
raw_freq = walt_map_util_freq(util, wg_policy, max, wg_cpu->cpu);
freq = raw_freq;
if (wg_policy->tunables->adaptive_high_freq) {
if (raw_freq < wg_policy->tunables->adaptive_low_freq)
if (raw_freq < wg_policy->tunables->adaptive_low_freq) {
freq = wg_policy->tunables->adaptive_low_freq;
else if (raw_freq <= wg_policy->tunables->adaptive_high_freq)
wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_LOW;
} else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) {
freq = wg_policy->tunables->adaptive_high_freq;
wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_HIGH;
}
}
trace_waltgov_next_freq(policy->cpu, util, max, raw_freq, freq, policy->min, policy->max,
wg_policy->cached_raw_freq, wg_policy->need_freq_update);
wg_policy->cached_raw_freq, wg_policy->need_freq_update,
wg_driv_cpu->cpu, wg_driv_cpu->reasons);
if (wg_policy->cached_raw_freq && freq == wg_policy->cached_raw_freq &&
!wg_policy->need_freq_update)
@ -259,7 +266,8 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu)
unsigned long util;
wg_cpu->max = max;
util = cpu_util_freq_walt(wg_cpu->cpu, &wg_cpu->walt_load);
wg_cpu->reasons = 0;
util = cpu_util_freq_walt(wg_cpu->cpu, &wg_cpu->walt_load, &wg_cpu->reasons);
return uclamp_rq_util_with(rq, util, NULL);
}
@ -270,6 +278,16 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu)
#define DEFAULT_PRIME_RTG_BOOST_FREQ 0
#define DEFAULT_TARGET_LOAD_THRESH 1024
#define DEFAULT_TARGET_LOAD_SHIFT 4
static inline void max_and_reason(unsigned long *cur_util, unsigned long boost_util,
struct waltgov_cpu *wg_cpu, unsigned int reason)
{
if (boost_util >= *cur_util) {
*cur_util = boost_util;
wg_cpu->reasons = reason;
wg_cpu->wg_policy->driving_cpu = wg_cpu->cpu;
}
}
static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_util,
unsigned long nl, unsigned long *util,
unsigned long *max)
@ -281,22 +299,22 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut
unsigned long pl = wg_cpu->walt_load.pl;
if (is_rtg_boost)
*util = max(*util, wg_policy->rtg_boost_util);
max_and_reason(util, wg_policy->rtg_boost_util, wg_cpu, CPUFREQ_REASON_RTG_BOOST);
is_hiload = (cpu_util >= mult_frac(wg_policy->avg_cap,
wg_policy->tunables->hispeed_load,
100));
if (is_hiload && !is_migration)
*util = max(*util, wg_policy->hispeed_util);
max_and_reason(util, wg_policy->hispeed_util, wg_cpu, CPUFREQ_REASON_HISPEED);
if (is_hiload && nl >= mult_frac(cpu_util, NL_RATIO, 100))
*util = *max;
max_and_reason(util, *max, wg_cpu, CPUFREQ_REASON_NWD);
if (wg_policy->tunables->pl) {
if (sysctl_sched_conservative_pl)
pl = mult_frac(pl, TARGET_LOAD, 100);
*util = max(*util, pl);
max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL);
}
}
@ -346,6 +364,7 @@ static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 tim
if (j_util * max >= j_max * util) {
util = j_util;
max = j_max;
wg_policy->driving_cpu = j;
}
waltgov_walt_adjust(j_wg_cpu, j_util, j_nl, &util, &max);

30
kernel/sched/walt/trace.h
View File

@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
* Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
*/
#undef TRACE_SYSTEM
@ -421,9 +421,10 @@ TRACE_EVENT(sched_load_to_gov,
int freq_aggr, u64 load, int policy,
int big_task_rotation,
unsigned int user_hint,
struct walt_rq *wrq),
struct walt_rq *wrq,
unsigned int reasons),
TP_ARGS(rq, aggr_grp_load, tt_load, freq_aggr, load, policy,
big_task_rotation, user_hint, wrq),
big_task_rotation, user_hint, wrq, reasons),
TP_STRUCT__entry(
__field(int, cpu)
@ -440,6 +441,7 @@ TRACE_EVENT(sched_load_to_gov,
__field(u64, load)
__field(int, big_task_rotation)
__field(unsigned int, user_hint)
__field(unsigned int, reasons)
),
TP_fast_assign(
@ -458,14 +460,15 @@ TRACE_EVENT(sched_load_to_gov,
__entry->load = load;
__entry->big_task_rotation = big_task_rotation;
__entry->user_hint = user_hint;
__entry->reasons = reasons;
),
TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr=%d tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d user_hint=%u",
TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr=%d tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d user_hint=%u reasons=0x%x",
__entry->cpu, __entry->policy, __entry->ed_task_pid,
__entry->aggr_grp_load, __entry->freq_aggr,
__entry->tt_load, __entry->rq_ps, __entry->grp_rq_ps,
__entry->nt_ps, __entry->grp_nt_ps, __entry->pl, __entry->load,
__entry->big_task_rotation, __entry->user_hint)
__entry->big_task_rotation, __entry->user_hint, __entry->reasons)
);
TRACE_EVENT(core_ctl_eval_need,
@ -721,9 +724,10 @@ TRACE_EVENT(waltgov_util_update,
TRACE_EVENT(waltgov_next_freq,
TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, unsigned int raw_freq,
unsigned int freq, unsigned int policy_min_freq, unsigned int policy_max_freq,
unsigned int cached_raw_freq, bool need_freq_update),
unsigned int cached_raw_freq, bool need_freq_update, unsigned int driving_cpu,
unsigned int reason),
TP_ARGS(cpu, util, max, raw_freq, freq, policy_min_freq, policy_max_freq,
cached_raw_freq, need_freq_update),
cached_raw_freq, need_freq_update, driving_cpu, reason),
TP_STRUCT__entry(
__field(unsigned int, cpu)
__field(unsigned long, util)
@ -735,6 +739,8 @@ TRACE_EVENT(waltgov_next_freq,
__field(unsigned int, cached_raw_freq)
__field(bool, need_freq_update)
__field(unsigned int, rt_util)
__field(unsigned int, driving_cpu)
__field(unsigned int, reason)
),
TP_fast_assign(
__entry->cpu = cpu;
@ -746,9 +752,11 @@ TRACE_EVENT(waltgov_next_freq,
__entry->policy_max_freq = policy_max_freq;
__entry->cached_raw_freq = cached_raw_freq;
__entry->need_freq_update = need_freq_update;
__entry->rt_util = cpu_util_rt(cpu_rq(cpu));
__entry->rt_util = cpu_util_rt(cpu_rq(cpu));
__entry->driving_cpu = driving_cpu;
__entry->reason = reason;
),
TP_printk("cpu=%u util=%lu max=%lu raw_freq=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u",
TP_printk("cpu=%u util=%lu max=%lu raw_freq=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u driv_cpu=%u reason=0x%x",
__entry->cpu,
__entry->util,
__entry->max,
@ -758,7 +766,9 @@ TRACE_EVENT(waltgov_next_freq,
__entry->policy_max_freq,
__entry->cached_raw_freq,
__entry->need_freq_update,
__entry->rt_util)
__entry->rt_util,
__entry->driving_cpu,
__entry->reason)
);
TRACE_EVENT(walt_active_load_balance,

38
kernel/sched/walt/walt.c
View File

@ -590,30 +590,41 @@ should_apply_suh_freq_boost(struct walt_sched_cluster *cluster)
return is_cluster_hosting_top_app(cluster);
}
static inline u64 freq_policy_load(struct rq *rq)
static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_sched_cluster *cluster = wrq->cluster;
u64 aggr_grp_load = cluster->aggr_grp_load;
u64 load, tt_load = 0;
u64 load, tt_load = 0, kload = 0;
struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu_of(rq));
if (wrq->ed_task != NULL) {
load = sched_ravg_window;
*reason = CPUFREQ_REASON_EARLY_DET;
goto done;
}
if (sched_freq_aggr_en)
if (sched_freq_aggr_en) {
load = wrq->prev_runnable_sum + aggr_grp_load;
*reason = CPUFREQ_REASON_FREQ_AGR;
}
else
load = wrq->prev_runnable_sum +
wrq->grp_time.prev_runnable_sum;
if (cpu_ksoftirqd && READ_ONCE(cpu_ksoftirqd->__state) == TASK_RUNNING)
load = max_t(u64, load, task_load(cpu_ksoftirqd));
if (cpu_ksoftirqd && READ_ONCE(cpu_ksoftirqd->__state) == TASK_RUNNING) {
kload = task_load(cpu_ksoftirqd);
if (kload > load) {
load = kload;
*reason = CPUFREQ_REASON_KSOFTIRQD;
}
}
tt_load = top_task_load(rq);
load = max_t(u64, load, tt_load);
if (tt_load > load) {
load = tt_load;
*reason = CPUFREQ_REASON_TT_LOAD;
}
if (should_apply_suh_freq_boost(cluster)) {
if (is_suh_max())
@ -621,26 +632,27 @@ static inline u64 freq_policy_load(struct rq *rq)
else
load = div64_u64(load * sysctl_sched_user_hint,
(u64)100);
*reason = CPUFREQ_REASON_SUH;
}
done:
trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en,
load, 0, walt_rotation_enabled,
sysctl_sched_user_hint, wrq);
sysctl_sched_user_hint, wrq, *reason);
return load;
}
static bool rtgb_active;
static inline unsigned long
__cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load)
__cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reason)
{
u64 util;
struct rq *rq = cpu_rq(cpu);
unsigned long capacity = capacity_orig_of(cpu);
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
util = div64_u64(freq_policy_load(rq),
util = div64_u64(freq_policy_load(rq, reason),
sched_ravg_window >> SCHED_CAPACITY_SHIFT);
if (walt_load) {
@ -665,7 +677,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load)
(max(orig, mult_frac(other, x, 100)))
unsigned long
cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load)
cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reason)
{
struct walt_cpu_load wl_other = {0};
unsigned long util = 0, util_other = 0;
@ -673,13 +685,13 @@ cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load)
int i, mpct = sysctl_sched_asym_cap_sibling_freq_match_pct;
if (!cpumask_test_cpu(cpu, &asym_cap_sibling_cpus))
return __cpu_util_freq_walt(cpu, walt_load);
return __cpu_util_freq_walt(cpu, walt_load, reason);
for_each_cpu(i, &asym_cap_sibling_cpus) {
if (i == cpu)
util = __cpu_util_freq_walt(cpu, walt_load);
util = __cpu_util_freq_walt(cpu, walt_load, reason);
else
util_other = __cpu_util_freq_walt(i, &wl_other);
util_other = __cpu_util_freq_walt(i, &wl_other, reason);
}
if (cpu == cpumask_last(&asym_cap_sibling_cpus))

17
kernel/sched/walt/walt.h
View File

@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
* Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
*/
#ifndef _WALT_H
@ -297,6 +297,18 @@ extern unsigned int sched_lib_mask_force;
#define WALT_CPUFREQ_EARLY_DET (1U << 4)
#define WALT_CPUFREQ_BOOST_UPDATE (1U << 5)
#define CPUFREQ_REASON_PL (1U << 1)
#define CPUFREQ_REASON_EARLY_DET (1U << 2)
#define CPUFREQ_REASON_RTG_BOOST (1U << 3)
#define CPUFREQ_REASON_HISPEED (1U << 4)
#define CPUFREQ_REASON_NWD (1U << 5)
#define CPUFREQ_REASON_FREQ_AGR (1U << 6)
#define CPUFREQ_REASON_KSOFTIRQD (1U << 7)
#define CPUFREQ_REASON_TT_LOAD (1U << 8)
#define CPUFREQ_REASON_SUH (1U << 9)
#define CPUFREQ_REASON_ADAPTIVE_LOW (1U << 10)
#define CPUFREQ_REASON_ADAPTIVE_HIGH (1U << 11)
#define NO_BOOST 0
#define FULL_THROTTLE_BOOST 1
#define CONSERVATIVE_BOOST 2
@ -366,7 +378,8 @@ static inline void waltgov_run_callback(struct rq *rq, unsigned int flags)
cb->func(cb, walt_sched_clock(), flags);
}
extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load);
extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load,
unsigned int *reason);
int waltgov_register(void);
extern void walt_lb_init(void);