mirror of
https://github.com/torvalds/linux.git
synced 2026-07-31 03:27:03 +02:00
sched: walt: Improve the scheduler
This change is for general scheduler improvement. Change-Id: Ib1ec2f655193a71d401d7e51f7890537d4f75aef Signed-off-by: Shaleen Agrawal <shalagra@codeaurora.org>
This commit is contained in:
Shaleen Agrawal
Rishabh Bhatnagar
parent
741be7b858
commit
e483442bb7
|
|
@ -3625,7 +3625,7 @@ static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p)
|
|||
|
||||
static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
add_new_task_to_grp(new);
|
||||
}
|
||||
|
|
@ -3640,7 +3640,7 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long
|
|||
unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu);
|
||||
unsigned long thermal_cap;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
/*
|
||||
|
|
@ -3671,7 +3671,7 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu)
|
|||
unsigned long flags;
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
raw_spin_lock_irqsave(&rq->lock, flags);
|
||||
set_window_start(rq);
|
||||
|
|
@ -3682,14 +3682,14 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu)
|
|||
|
||||
static void android_rvh_sched_cpu_dying(void *unused, int cpu)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
clear_walt_request(cpu);
|
||||
}
|
||||
|
||||
static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsigned int new_cpu)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (new_cpu < 0)
|
||||
return;
|
||||
|
|
@ -3698,14 +3698,14 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign
|
|||
|
||||
static void android_rvh_sched_fork(void *unused, struct task_struct *p)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
init_new_task_load(p);
|
||||
}
|
||||
|
||||
static void android_rvh_new_task_stats(void *unused, struct task_struct *p)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
mark_task_starting(p);
|
||||
}
|
||||
|
|
@ -3714,7 +3714,7 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int
|
|||
{
|
||||
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (!!(curr->flags & PF_IDLE)) {
|
||||
if (hardirq_count() || in_serving_softirq())
|
||||
|
|
@ -3727,7 +3727,7 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int
|
|||
|
||||
static void android_rvh_flush_task(void *unused, struct task_struct *p)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
walt_task_dead(p);
|
||||
}
|
||||
|
|
@ -3737,7 +3737,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st
|
|||
u64 wallclock = sched_ktime_clock();
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
wts->last_enqueued_ts = wallclock;
|
||||
sched_update_nr_prod(rq->cpu, true);
|
||||
|
|
@ -3755,7 +3755,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st
|
|||
{
|
||||
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (p == wrq->ed_task)
|
||||
is_ed_task_present(rq, sched_ktime_clock());
|
||||
|
|
@ -3777,7 +3777,7 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p
|
|||
bool old_misfit, misfit;
|
||||
int change;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
*need_update = false;
|
||||
|
||||
|
|
@ -3815,7 +3815,7 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p)
|
|||
unsigned int old_load;
|
||||
struct walt_related_thread_group *grp = NULL;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
rq_lock_irqsave(rq, &rf);
|
||||
old_load = task_load(p);
|
||||
|
|
@ -3837,7 +3837,7 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct
|
|||
unsigned long flags;
|
||||
int cpu = p->cpu;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags);
|
||||
|
|
@ -3852,7 +3852,7 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq)
|
|||
u32 old_load;
|
||||
struct walt_related_thread_group *grp;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
set_window_start(rq);
|
||||
wallclock = sched_ktime_clock();
|
||||
|
|
@ -3876,7 +3876,7 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev,
|
|||
u64 wallclock = sched_ktime_clock();
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (likely(prev != next)) {
|
||||
if (!prev->on_rq)
|
||||
|
|
@ -3894,7 +3894,7 @@ static void android_rvh_resume_cpus(void *unused, struct cpumask *resuming_cpus,
|
|||
struct rq *rq;
|
||||
unsigned long flags;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
/*
|
||||
* send a reschedule event on all resumed CPUs
|
||||
|
|
@ -3916,7 +3916,7 @@ static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p,
|
|||
{
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (cpumask_subset(&wts->cpus_requested, cpus_requested))
|
||||
*ret = set_cpus_allowed_ptr(p, &wts->cpus_requested);
|
||||
|
|
@ -3926,7 +3926,7 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p)
|
|||
{
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
wts->last_sleep_ts = 0;
|
||||
wts->wake_up_idle = false;
|
||||
|
|
@ -3939,21 +3939,21 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p)
|
|||
|
||||
static void android_rvh_ttwu_cond(void *unused, bool *cond)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
*cond = sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT;
|
||||
}
|
||||
|
||||
static void android_rvh_sched_exec(void *unused, bool *cond)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
*cond = true;
|
||||
}
|
||||
|
||||
static void android_rvh_build_perf_domains(void *unused, bool *eas_check)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
*eas_check = true;
|
||||
}
|
||||
|
|
@ -3986,7 +3986,7 @@ static void register_walt_hooks(void)
|
|||
}
|
||||
|
||||
atomic64_t walt_irq_work_lastq_ws;
|
||||
DEFINE_STATIC_KEY_TRUE(walt_disabled);
|
||||
bool walt_disabled = true;
|
||||
|
||||
static int walt_init_stop_handler(void *data)
|
||||
{
|
||||
|
|
@ -4026,6 +4026,8 @@ static int walt_init_stop_handler(void *data)
|
|||
|
||||
walt_update_cluster_topology();
|
||||
|
||||
walt_disabled = false;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
raw_spin_unlock(&cpu_rq(cpu)->lock);
|
||||
}
|
||||
|
|
@ -4056,7 +4058,6 @@ static void walt_init(void)
|
|||
walt_cfs_init();
|
||||
|
||||
stop_machine(walt_init_stop_handler, NULL, NULL);
|
||||
static_branch_disable(&walt_disabled);
|
||||
|
||||
hdr = register_sysctl_table(walt_base_table);
|
||||
kmemleak_not_leak(hdr);
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@
|
|||
/* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */
|
||||
#define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1)
|
||||
|
||||
extern struct static_key_true walt_disabled;
|
||||
extern bool walt_disabled;
|
||||
|
||||
enum task_event {
|
||||
PUT_PREV_TASK = 0,
|
||||
|
|
|
|||
|
|
@ -665,7 +665,7 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu,
|
|||
int sync;
|
||||
int sibling_count_hint;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING);
|
||||
|
|
@ -684,7 +684,7 @@ static inline struct task_struct *task_of(struct sched_entity *se)
|
|||
|
||||
static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vruntime)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (entity_is_task(se)) {
|
||||
unsigned long thresh = sysctl_sched_latency;
|
||||
|
|
@ -710,7 +710,7 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task)
|
|||
{
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (task && current->signal &&
|
||||
(current->signal->oom_score_adj == 0) &&
|
||||
|
|
@ -723,7 +723,7 @@ static void walt_binder_low_latency_clear(void *unused, struct binder_transactio
|
|||
{
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (wts->low_latency & WALT_LOW_LATENCY_BINDER)
|
||||
wts->low_latency &= ~WALT_LOW_LATENCY_BINDER;
|
||||
|
|
|
|||
|
|
@ -491,7 +491,7 @@ static void walt_lb_tick(void *unused, struct rq *rq)
|
|||
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
if (!rq->misfit_task_load || !walt_fair_task(p))
|
||||
return;
|
||||
|
|
@ -633,7 +633,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
|
|||
bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD);
|
||||
bool help_min_cap = false;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
/*Cluster isn't initialized until after WALT is enabled*/
|
||||
|
|
@ -729,7 +729,7 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu,
|
|||
int busiest_cpu = -1;
|
||||
struct cpumask src_mask;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
*done = 1;
|
||||
*busiest = NULL;
|
||||
|
|
@ -767,7 +767,7 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq,
|
|||
struct task_struct *p,
|
||||
int new_cpu, int *detached)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
/*
|
||||
* WALT expects both source and destination rqs to be
|
||||
|
|
@ -796,7 +796,7 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq,
|
|||
static void walt_nohz_balancer_kick(void *unused, struct rq *rq,
|
||||
unsigned int *flags, int *done)
|
||||
{
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
*done = 1;
|
||||
|
||||
|
|
@ -815,7 +815,7 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p,
|
|||
{
|
||||
bool to_lower;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(task_cpu(p));
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task
|
|||
int cluster;
|
||||
int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
if (!ret)
|
||||
|
|
@ -119,7 +119,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c
|
|||
int ret, target = -1;
|
||||
struct cpumask *lowest_mask;
|
||||
|
||||
if (static_branch_unlikely(&walt_disabled))
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
/* For anything but wake ups, just return the task_cpu */
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user