github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-31 11:37:06 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge "kernel/sched/walt: check migration disabled in allowed_ptr_locked"

Browse Source
This commit is contained in:
qctecmdr 2022-09-27 16:06:07 -07:00 committed by Gerrit - the friendly Code Review server
commit 4d36671e16
9 changed files with 346 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
kernel/sched/walt/core_ctl.c
View File

@ -392,7 +392,7 @@ static ssize_t show_not_preferred(const struct cluster_data *state, char *buf)
for (i = 0; i < state->num_cpus; i++) {
c = &per_cpu(cpu_state, i + state->first_cpu);
count += scnprintf(buf + count, PAGE_SIZE - count,
"CPU#%d: %u\n", c->cpu, c->not_preferred);
"%u ", c->not_preferred);
}
spin_unlock_irqrestore(&state_lock, flags);

121
kernel/sched/walt/cpufreq_walt.c
View File

@ -25,6 +25,8 @@ struct waltgov_tunables {
unsigned int rtg_boost_freq;
unsigned int adaptive_low_freq;
unsigned int adaptive_high_freq;
unsigned int adaptive_low_freq_kernel;
unsigned int adaptive_high_freq_kernel;
unsigned int target_load_thresh;
unsigned int target_load_shift;
bool pl;
@ -220,6 +222,18 @@ static inline unsigned long walt_map_util_freq(unsigned long util,
return (fmax + (fmax >> 2)) * util / cap;
}
static inline unsigned int get_adaptive_low_freq(struct waltgov_policy *wg_policy)
{
return(max(wg_policy->tunables->adaptive_low_freq,
wg_policy->tunables->adaptive_low_freq_kernel));
}
static inline unsigned int get_adaptive_high_freq(struct waltgov_policy *wg_policy)
{
return(max(wg_policy->tunables->adaptive_high_freq,
wg_policy->tunables->adaptive_high_freq_kernel));
}
static unsigned int get_next_freq(struct waltgov_policy *wg_policy,
unsigned long util, unsigned long max,
struct waltgov_cpu *wg_cpu, u64 time)
@ -232,11 +246,11 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy,
freq = raw_freq;
if (wg_policy->tunables->adaptive_high_freq) {
if (raw_freq < wg_policy->tunables->adaptive_low_freq) {
freq = wg_policy->tunables->adaptive_low_freq;
if (raw_freq < get_adaptive_low_freq(wg_policy)) {
freq = get_adaptive_low_freq(wg_policy);
wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_LOW;
} else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) {
freq = wg_policy->tunables->adaptive_high_freq;
} else if (raw_freq <= get_adaptive_high_freq(wg_policy)) {
freq = get_adaptive_high_freq(wg_policy);
wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_HIGH;
}
}
@ -296,10 +310,11 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut
bool is_migration = wg_cpu->flags & WALT_CPUFREQ_IC_MIGRATION;
bool is_rtg_boost = wg_cpu->walt_load.rtgb_active;
bool is_hiload;
bool is_ed_boost = wg_cpu->walt_load.ed_active;
bool big_task_rotation = wg_cpu->walt_load.big_task_rotation;
bool employ_ed_boost = wg_cpu->walt_load.ed_active && sysctl_ed_boost_pct;
unsigned long pl = wg_cpu->walt_load.pl;
if (is_ed_boost) {
if (employ_ed_boost) {
cpu_util = mult_frac(cpu_util, 100 + sysctl_ed_boost_pct, 100);
max_and_reason(util, cpu_util, wg_cpu, CPUFREQ_REASON_EARLY_DET);
}
@ -323,8 +338,11 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut
max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL);
}
if (is_ed_boost)
if (employ_ed_boost)
wg_cpu->reasons |= CPUFREQ_REASON_EARLY_DET;
if (big_task_rotation)
max_and_reason(util, *max, wg_cpu, CPUFREQ_REASON_BTR);
}
static inline unsigned long target_util(struct waltgov_policy *wg_policy,
@ -665,6 +683,91 @@ static ssize_t boost_store(struct gov_attr_set *attr_set, const char *buf,
return count;
}
/**
* cpufreq_walt_set_adaptive_freq() - set the waltgov adaptive freq for cpu
* @cpu: the cpu for which the values should be set
* @adaptive_low_freq: low freq
* @adaptive_high_freq:high_freq
*
* Configure the adaptive_low/high_freq for the cpu specified. This will impact all
* cpus governed by the policy (e.g. all cpus in a cluster). The actual value used
* for adaptive frequencies will be governed by the user space setting for the
* policy, and this value.
*
* Return: 0 if successful, error otherwise
*/
int cpufreq_walt_set_adaptive_freq(unsigned int cpu, unsigned int adaptive_low_freq,
unsigned int adaptive_high_freq)
{
struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu);
struct waltgov_policy *wg_policy = wg_cpu->wg_policy;
struct cpufreq_policy *policy = wg_policy->policy;
if (!cpu_possible(cpu))
return -EFAULT;
if (policy->min <= adaptive_low_freq && policy->max >= adaptive_high_freq) {
wg_policy->tunables->adaptive_low_freq_kernel = adaptive_low_freq;
wg_policy->tunables->adaptive_high_freq_kernel = adaptive_high_freq;
return 0;
}
return -EINVAL;
}
EXPORT_SYMBOL(cpufreq_walt_set_adaptive_freq);
/**
* cpufreq_walt_get_adaptive_freq() - get the waltgov adaptive freq for cpu
* @cpu: the cpu for which the values should be returned
* @adaptive_low_freq: pointer to write the current kernel adaptive_low_freq value
* @adaptive_high_freq:pointer to write the current kernel adaptive_high_freq value
*
* Get the currently active adaptive_low/high_freq for the cpu specified.
*
* Return: 0 if successful, error otherwise
*/
int cpufreq_walt_get_adaptive_freq(unsigned int cpu, unsigned int *adaptive_low_freq,
unsigned int *adaptive_high_freq)
{
struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu);
struct waltgov_policy *wg_policy = wg_cpu->wg_policy;
if (!cpu_possible(cpu))
return -EFAULT;
if (adaptive_low_freq && adaptive_high_freq) {
*adaptive_low_freq = get_adaptive_low_freq(wg_policy);
*adaptive_high_freq = get_adaptive_high_freq(wg_policy);
return 0;
}
return -EINVAL;
}
EXPORT_SYMBOL(cpufreq_walt_get_adaptive_freq);
/**
* cpufreq_walt_reset_adaptive_freq() - reset the waltgov adaptive freq for cpu
* @cpu: the cpu for which the values should be set
*
* Reset the kernel adaptive_low/high_freq to zero.
*
* Return: 0 if successful, error otherwise
*/
int cpufreq_walt_reset_adaptive_freq(unsigned int cpu)
{
struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu);
struct waltgov_policy *wg_policy = wg_cpu->wg_policy;
if (!cpu_possible(cpu))
return -EFAULT;
wg_policy->tunables->adaptive_low_freq_kernel = 0;
wg_policy->tunables->adaptive_high_freq_kernel = 0;
return 0;
}
EXPORT_SYMBOL(cpufreq_walt_reset_adaptive_freq);
#define WALTGOV_ATTR_RW(_name) \
static struct governor_attr _name = \
__ATTR(_name, 0644, show_##_name, store_##_name) \
@ -822,6 +925,8 @@ static void waltgov_tunables_save(struct cpufreq_policy *policy,
cached->boost = tunables->boost;
cached->adaptive_low_freq = tunables->adaptive_low_freq;
cached->adaptive_high_freq = tunables->adaptive_high_freq;
cached->adaptive_low_freq_kernel = tunables->adaptive_low_freq_kernel;
cached->adaptive_high_freq_kernel = tunables->adaptive_high_freq_kernel;
cached->target_load_thresh = tunables->target_load_thresh;
cached->target_load_shift = tunables->target_load_shift;
}
@ -844,6 +949,8 @@ static void waltgov_tunables_restore(struct cpufreq_policy *policy)
tunables->boost = cached->boost;
tunables->adaptive_low_freq = cached->adaptive_low_freq;
tunables->adaptive_high_freq = cached->adaptive_high_freq;
tunables->adaptive_low_freq_kernel = cached->adaptive_low_freq_kernel;
tunables->adaptive_high_freq_kernel = cached->adaptive_high_freq_kernel;
tunables->target_load_thresh = cached->target_load_thresh;
tunables->target_load_shift = cached->target_load_shift;
}

102
kernel/sched/walt/sysctl.c
View File

@ -19,6 +19,7 @@ static int __maybe_unused two = 2;
static int __maybe_unused four = 4;
static int one_hundred = 100;
static int one_thousand = 1000;
static int one_thousand_twenty_four = 1024;
static int two_thousand = 2000;
/*
@ -47,6 +48,8 @@ unsigned int sysctl_sched_wake_up_idle[2];
unsigned int sysctl_input_boost_ms;
unsigned int sysctl_input_boost_freq[8];
unsigned int sysctl_sched_boost_on_input;
unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS];
unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS];
/* sysctl nodes accesed by other files */
unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns;
@ -75,6 +78,8 @@ unsigned int sysctl_sched_long_running_rt_task_ms;
unsigned int sysctl_sched_idle_enough;
unsigned int sysctl_sched_cluster_util_thres_pct;
unsigned int sysctl_ed_boost_pct;
unsigned int sysctl_em_inflate_pct = 100;
unsigned int sysctl_em_inflate_thres = 1024;
/* range is [1 .. INT_MAX] */
static int sysctl_task_read_pid = 1;
@ -457,6 +462,69 @@ int sched_updown_migrate_handler(struct ctl_table *table, int write,
/* update individual cpu thresholds */
sched_update_updown_migrate_values(data == &sysctl_sched_capacity_margin_up_pct[0]);
unlock_mutex:
mutex_unlock(&mutex);
return ret;
}
int sched_updown_early_migrate_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos)
{
int ret, i;
unsigned int *data = (unsigned int *)table->data;
static DEFINE_MUTEX(mutex);
int cap_margin_levels = num_sched_clusters ? num_sched_clusters - 1 : 0;
int val[MAX_MARGIN_LEVELS];
struct ctl_table tmp = {
.data = &val,
.maxlen = sizeof(int) * cap_margin_levels,
.mode = table->mode,
};
if (cap_margin_levels <= 0)
return -EINVAL;
mutex_lock(&mutex);
if (!write) {
ret = proc_dointvec(table, write, buffer, lenp, ppos);
goto unlock_mutex;
}
ret = proc_dointvec(&tmp, write, buffer, lenp, ppos);
if (ret)
goto unlock_mutex;
for (i = 0; i < cap_margin_levels; i++) {
if (val[i] < 1024) {
ret = -EINVAL;
goto unlock_mutex;
}
}
/* check up pct is greater than dn pct */
if (data == &sysctl_sched_early_up[0]) {
for (i = 0; i < cap_margin_levels; i++) {
if (val[i] >= sysctl_sched_early_down[i]) {
ret = -EINVAL;
goto unlock_mutex;
}
}
} else {
for (i = 0; i < cap_margin_levels; i++) {
if (sysctl_sched_early_up[i] >= val[i]) {
ret = -EINVAL;
goto unlock_mutex;
}
}
}
/* all things checkout update the value */
for (i = 0; i < cap_margin_levels; i++)
data[i] = val[i];
unlock_mutex:
mutex_unlock(&mutex);
@ -721,6 +789,20 @@ struct ctl_table walt_table[] = {
.mode = 0644,
.proc_handler = sched_updown_migrate_handler,
},
{
.procname = "sched_early_upmigrate",
.data = &sysctl_sched_early_up,
.maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS,
.mode = 0644,
.proc_handler = sched_updown_early_migrate_handler,
},
{
.procname = "sched_early_downmigrate",
.data = &sysctl_sched_early_down,
.maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS,
.mode = 0644,
.proc_handler = sched_updown_early_migrate_handler,
},
{
.procname = "walt_rtg_cfs_boost_prio",
.data = &sysctl_walt_rtg_cfs_boost_prio,
@ -932,6 +1014,24 @@ struct ctl_table walt_table[] = {
.extra1 = SYSCTL_ZERO,
.extra2 = &one_hundred,
},
{
.procname = "sched_em_inflate_pct",
.data = &sysctl_em_inflate_pct,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax,
.extra1 = &one_hundred,
.extra2 = &one_thousand,
},
{
.procname = "sched_em_inflate_thres",
.data = &sysctl_em_inflate_thres,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = &one_thousand_twenty_four,
},
{ }
};
@ -951,6 +1051,8 @@ void walt_tunables(void)
for (i = 0; i < MAX_MARGIN_LEVELS; i++) {
sysctl_sched_capacity_margin_up_pct[i] = 95; /* ~5% margin */
sysctl_sched_capacity_margin_dn_pct[i] = 85; /* ~15% margin */
sysctl_sched_early_up[i] = 1077;
sysctl_sched_early_down[i] = 1204;
}
sysctl_sched_group_upmigrate_pct = 100;

22
kernel/sched/walt/trace.h
View File

@ -1055,6 +1055,28 @@ TRACE_EVENT(sched_compute_energy,
__entry->cluster_first_cpu2, __entry->s2, __entry->m2, __entry->c2)
)
TRACE_EVENT(sched_select_task_rt,
TP_PROTO(struct task_struct *p, int fastpath),
TP_ARGS(p, fastpath),
TP_STRUCT__entry(
__field(int, pid)
__array(char, comm, TASK_COMM_LEN)
__field(int, fastpath)
),
TP_fast_assign(
__entry->pid = p->pid;
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->fastpath = fastpath;
),
TP_printk("pid=%d comm=%s fastpath=%u",
__entry->pid, __entry->comm, __entry->fastpath)
);
TRACE_EVENT(sched_task_util,
TP_PROTO(struct task_struct *p, unsigned long candidates,

3
kernel/sched/walt/walt.c
View File

@ -464,7 +464,7 @@ static void update_task_cpu_cycles(struct task_struct *p, int cpu,
static inline bool is_ed_enabled(void)
{
return (walt_rotation_enabled || (boost_policy != SCHED_BOOST_NONE));
return (boost_policy != SCHED_BOOST_NONE);
}
static inline bool is_ed_task(struct task_struct *p, u64 wallclock)
@ -652,6 +652,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea
walt_load->pl = pl;
walt_load->ws = walt_load_reported_window;
walt_load->rtgb_active = rtgb_active;
walt_load->big_task_rotation = walt_rotation_enabled;
if (wrq->ed_task)
walt_load->ed_active = true;
else

71
kernel/sched/walt/walt.h
View File

@ -60,6 +60,7 @@ struct walt_cpu_load {
bool rtgb_active;
u64 ws;
bool ed_active;
bool big_task_rotation;
};
#define DECLARE_BITMAP_ARRAY(name, nr, bits) \
@ -219,6 +220,14 @@ extern unsigned int sysctl_sched_conservative_pl;
extern unsigned int sysctl_sched_hyst_min_coloc_ns;
extern unsigned int sysctl_sched_long_running_rt_task_ms;
extern unsigned int sysctl_ed_boost_pct;
extern unsigned int sysctl_em_inflate_pct;
extern unsigned int sysctl_em_inflate_thres;
extern int cpufreq_walt_set_adaptive_freq(unsigned int cpu, unsigned int adaptive_low_freq,
unsigned int adaptive_high_freq);
extern int cpufreq_walt_get_adaptive_freq(unsigned int cpu, unsigned int *adaptive_low_freq,
unsigned int *adaptive_high_freq);
extern int cpufreq_walt_reset_adaptive_freq(unsigned int cpu);
#define WALT_MANY_WAKEUP_DEFAULT 1000
extern unsigned int sysctl_sched_many_wakeup_threshold;
@ -307,6 +316,7 @@ extern unsigned int sched_lib_mask_force;
#define WALT_CPUFREQ_BOOST_UPDATE 0x20
#define CPUFREQ_REASON_LOAD 0
#define CPUFREQ_REASON_BTR 0x1
#define CPUFREQ_REASON_PL 0x2
#define CPUFREQ_REASON_EARLY_DET 0x4
#define CPUFREQ_REASON_RTG_BOOST 0x8
@ -740,6 +750,15 @@ static inline unsigned int walt_nr_rtg_high_prio(int cpu)
return wrq->walt_stats.nr_rtg_high_prio_tasks;
}
static inline bool task_in_related_thread_group(struct task_struct *p)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
return (rcu_access_pointer(wts->grp) != NULL);
}
extern unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS];
extern unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS];
static inline bool task_fits_capacity(struct task_struct *p,
long capacity,
int cpu)
@ -751,21 +770,27 @@ static inline bool task_fits_capacity(struct task_struct *p,
/*
* Derive upmigration/downmigrate margin wrt the src/dest CPU.
*/
if (src_wrq->cluster->id > dst_wrq->cluster->id)
if (src_wrq->cluster->id > dst_wrq->cluster->id) {
margin = sched_capacity_margin_down[cpu];
else
if (task_in_related_thread_group(p)) {
if (is_min_cluster_cpu(cpu))
margin = sysctl_sched_early_down[0];
else if (!is_max_cluster_cpu(cpu))
margin = sysctl_sched_early_down[1];
}
} else {
margin = sched_capacity_margin_up[task_cpu(p)];
if (task_in_related_thread_group(p)) {
if (is_min_cluster_cpu(task_cpu(p)))
margin = sysctl_sched_early_up[0];
else if (!is_max_cluster_cpu(task_cpu(p)))
margin = sysctl_sched_early_up[1];
}
}
return capacity * 1024 > uclamp_task_util(p) * margin;
}
static inline bool task_in_related_thread_group(struct task_struct *p)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
return (rcu_access_pointer(wts->grp) != NULL);
}
static inline bool task_fits_max(struct task_struct *p, int cpu)
{
unsigned long capacity = capacity_orig_of(cpu);
@ -985,7 +1010,6 @@ static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct
struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1;
struct walt_sched_cluster *cluster = wrq->cluster;
cpumask_t unhalted_cpus;
cpumask_t cluster_32bit_cpus;
int packing_cpu;
/* if idle_enough feature is not enabled */
@ -994,22 +1018,19 @@ static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct
if (!sysctl_sched_cluster_util_thres_pct)
return -1;
/* find all 32 bit capable cpus in this cluster */
cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask());
/* pack 32 bit and 64 bit tasks on the same cpu, if possible */
if (cpumask_weight(&cluster_32bit_cpus) > 0) {
packing_cpu = cpumask_first(&cluster_32bit_cpus);
} else {
/* find all unhalted active cpus */
cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask);
/* find all unhalted active cpus */
cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask);
/* find all unhalted active cpus in this cluster */
cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus);
/* find all unhalted active cpus in this cluster */
cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus);
/* return the first found unhalted, active cpu, in this cluster */
packing_cpu = cpumask_first(&unhalted_cpus);
}
if (is_compat_thread(task_thread_info(p)))
/* try to find a packing cpu within 32 bit subset */
cpumask_and(&unhalted_cpus, &unhalted_cpus, system_32bit_el0_cpumask());
/* return the first found unhalted, active cpu, in this cluster */
packing_cpu = cpumask_first(&unhalted_cpus);
/* packing cpu must be a valid cpu for runqueue lookup */
if (packing_cpu >= nr_cpu_ids)
@ -1031,6 +1052,10 @@ static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct
if (task_util(p) >= sysctl_sched_idle_enough)
return -1;
/* don't pack if running at a freq higher than 43.9pct of its fmax */
if (arch_scale_freq_capacity(packing_cpu) > 450)
return -1;
/* the packing cpu can be used, so pack! */
return packing_cpu;
}

51
kernel/sched/walt/walt_cfs.c
View File

@ -343,7 +343,8 @@ static void walt_find_best_target(struct sched_domain *sd,
if (fbt_env->skip_cpu == i)
continue;
if (wrq->num_mvp_tasks > 0)
if (wrq->num_mvp_tasks > 0 &&
per_task_boost(p) != TASK_BOOST_STRICT_MAX)
continue;
/*
@ -568,6 +569,16 @@ cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_ds
return util;
}
static inline unsigned long get_util_to_cost(int cpu, unsigned long util)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
if (cpu == 0 && util > sysctl_em_inflate_thres)
return mult_frac(wrq->cluster->util_to_cost[util], sysctl_em_inflate_pct, 100);
else
return wrq->cluster->util_to_cost[util];
}
/**
* walt_em_cpu_energy() - Estimates the energy consumed by the CPUs of a
performance domain
@ -586,9 +597,8 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd,
unsigned long max_util, unsigned long sum_util,
struct compute_energy_output *output, unsigned int x)
{
unsigned long scale_cpu;
unsigned long scale_cpu, cost;
int cpu;
struct walt_rq *wrq;
if (!sum_util)
return 0;
@ -651,14 +661,14 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd,
if (max_util >= 1024)
max_util = 1023;
wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
cost = get_util_to_cost(cpu, max_util);
if (output) {
output->cost[x] = wrq->cluster->util_to_cost[max_util];
output->cost[x] = cost;
output->max_util[x] = max_util;
output->sum_util[x] = sum_util;
}
return wrq->cluster->util_to_cost[max_util] * sum_util / scale_cpu;
return cost * sum_util / scale_cpu;
}
/*
@ -987,21 +997,23 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task,
if (unlikely(walt_disabled))
return;
if (task && current->signal &&
(current->signal->oom_score_adj == 0) &&
((current->prio < DEFAULT_PRIO) ||
(task->group_leader->prio < MAX_RT_PRIO)))
if (task && ((task_in_related_thread_group(current) &&
task->group_leader->prio < MAX_RT_PRIO) ||
(current->group_leader->prio < MAX_RT_PRIO &&
task_in_related_thread_group(task))))
wts->low_latency |= WALT_LOW_LATENCY_BINDER;
}
static void walt_binder_low_latency_clear(void *unused, struct binder_transaction *t)
{
struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1;
if (unlikely(walt_disabled))
return;
if (wts->low_latency & WALT_LOW_LATENCY_BINDER)
else
/*
* Clear low_latency flag if criterion above is not met, this
* will handle usecase where for a binder thread WALT_LOW_LATENCY_BINDER
* is set by one task and before WALT clears this flag after timer expiry
* some other task tries to use same binder thread.
*
* The only gets cleared when binder transaction is initiated
* and the above condition to set flasg is nto satisfied.
*/
wts->low_latency &= ~WALT_LOW_LATENCY_BINDER;
}
static void binder_set_priority_hook(void *data,
@ -1364,7 +1376,6 @@ void walt_cfs_init(void)
register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL);
register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL);
register_trace_binder_transaction_received(walt_binder_low_latency_clear, NULL);
register_trace_android_vh_binder_set_priority(binder_set_priority_hook, NULL);
register_trace_android_vh_binder_restore_priority(binder_restore_priority_hook, NULL);

15
kernel/sched/walt/walt_halt.c
View File

@ -500,17 +500,18 @@ static void android_rvh_get_nohz_timer_target(void *unused, int *cpu, bool *done
rcu_read_unlock();
}
static void android_rvh_set_cpus_allowed_ptr_locked(void *unused,
const struct cpumask *cpu_valid_mask,
const struct cpumask *new_mask,
unsigned int *dest_cpu)
static void android_rvh_set_cpus_allowed_by_task(void *unused,
const struct cpumask *cpu_valid_mask,
const struct cpumask *new_mask,
struct task_struct *p,
unsigned int *dest_cpu)
{
cpumask_t allowed_cpus;
if (unlikely(walt_disabled))
return;
if (cpu_halted(*dest_cpu)) {
if (cpu_halted(*dest_cpu) && !p->migration_disabled) {
/* remove halted cpus from the valid mask, and store locally */
cpumask_andnot(&allowed_cpus, cpu_valid_mask, cpu_halt_mask);
*dest_cpu = cpumask_any_and_distribute(&allowed_cpus, new_mask);
@ -573,8 +574,8 @@ void walt_halt_init(void)
sched_setscheduler_nocheck(walt_drain_thread, SCHED_FIFO, &param);
register_trace_android_rvh_get_nohz_timer_target(android_rvh_get_nohz_timer_target, NULL);
register_trace_android_rvh_set_cpus_allowed_ptr_locked(
android_rvh_set_cpus_allowed_ptr_locked, NULL);
register_trace_android_rvh_set_cpus_allowed_by_task(
android_rvh_set_cpus_allowed_by_task, NULL);
register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL);
register_trace_android_rvh_is_cpu_allowed(android_rvh_is_cpu_allowed, NULL);

21
kernel/sched/walt/walt_rt.c
View File

@ -225,6 +225,14 @@ static inline bool walt_should_honor_rt_sync(struct rq *rq, struct task_struct *
rq->rt.rt_nr_running <= 2;
}
enum rt_fastpaths {
NONE = 0,
NON_WAKEUP,
SYNC_WAKEUP,
CLUSTER_PACKING_FASTPATH,
};
static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int cpu,
int sd_flag, int wake_flags, int *new_cpu)
{
@ -235,13 +243,16 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c
int ret, target = -1, this_cpu;
struct cpumask *lowest_mask;
int packing_cpu;
int fastpath = NONE;
if (unlikely(walt_disabled))
return;
/* For anything but wake ups, just return the task_cpu */
if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK)
return;
if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) {
fastpath = NON_WAKEUP;
goto out;
}
this_cpu = raw_smp_processor_id();
this_cpu_rq = cpu_rq(this_cpu);
@ -252,8 +263,9 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c
if (sysctl_sched_sync_hint_enable && cpu_active(this_cpu) && !cpu_halted(this_cpu) &&
cpumask_test_cpu(this_cpu, task->cpus_ptr) &&
walt_should_honor_rt_sync(this_cpu_rq, task, sync)) {
fastpath = SYNC_WAKEUP;
*new_cpu = this_cpu;
return;
goto out;
}
*new_cpu = cpu; /* previous CPU as back up */
@ -292,6 +304,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c
/* create a fastpath for finding a packing cpu */
packing_cpu = walt_find_and_choose_cluster_packing_cpu(task_cpu(task), task);
if (packing_cpu >= 0) {
fastpath = CLUSTER_PACKING_FASTPATH;
*new_cpu = packing_cpu;
goto unlock;
}
@ -320,6 +333,8 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c
}
unlock:
rcu_read_unlock();
out:
trace_sched_select_task_rt(task, fastpath);
}