github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-31 11:37:06 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge "sched/walt: Reorganize walt_rq structure"

Browse Source
This commit is contained in:
qctecmdr 2022-10-19 17:09:13 -07:00 committed by Gerrit - the friendly Code Review server
commit b8df6bcc05
8 changed files with 123 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
kernel/sched/walt/sched_avg.c
View File

@ -130,7 +130,7 @@ struct sched_avg_stats *sched_get_nr_running_avg(void)
/* load is already scaled, see freq_policy_load/prev_runnable_sum */
for_each_cpu(cpu, &cluster->cpus) {
struct rq *rq = cpu_rq(cpu);
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
/* compute the % this cpu's utilization of the cpu capacity,
* and sum it across all cpus
@ -303,7 +303,7 @@ unsigned int sched_get_cpu_util_pct(int cpu)
u64 util;
unsigned long capacity, flags;
unsigned int busy;
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
raw_spin_lock_irqsave(&rq->__lock, flags);

4
kernel/sched/walt/trace.c
View File

@ -29,7 +29,7 @@ const char *__window_print(struct trace_seq *p, const u32 *buf, int buf_len)
static inline s64 __rq_update_sum(struct rq *rq, bool curr, bool new)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
if (curr)
if (new)
@ -45,7 +45,7 @@ static inline s64 __rq_update_sum(struct rq *rq, bool curr, bool new)
static inline s64 __grp_update_sum(struct rq *rq, bool curr, bool new)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
if (curr)
if (new)

2
kernel/sched/walt/trace.h
View File

@ -952,7 +952,7 @@ TRACE_EVENT(sched_cpu_util,
),
TP_fast_assign(
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
__entry->cpu = cpu;
__entry->nr_running = cpu_rq(cpu)->nr_running;
__entry->cpu_util = cpu_util(cpu);

123
kernel/sched/walt/walt.c
View File

@ -47,6 +47,8 @@ const char *migrate_type_names[] = {
#define NEW_TASK_ACTIVE_TIME 100000000
DEFINE_PER_CPU(struct walt_rq, walt_rq);
unsigned int sysctl_sched_user_hint;
static u64 sched_clock_last;
static bool walt_clock_suspended;
@ -151,7 +153,7 @@ static inline void release_rq_locks_irqrestore(const cpumask_t *cpus,
static inline u64 walt_rq_clock(struct rq *rq)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
if (unlikely(walt_clock_suspended))
return sched_clock_last;
@ -241,7 +243,7 @@ void walt_rq_dump(int cpu)
struct rq *rq = cpu_rq(cpu);
struct task_struct *tsk = cpu_curr(cpu);
int i;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
/*
* Increment the task reference so that it can't be
@ -344,7 +346,7 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p,
wts->demand_scaled;
s64 pred_demand_delta = (s64)updated_pred_demand_scaled -
wts->pred_demand_scaled;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, task_load_delta,
pred_demand_delta);
@ -403,7 +405,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event)
{
s64 delta;
int nr_windows;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
u64 old_window_start = wrq->window_start;
bool full_window;
@ -442,7 +444,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event)
*/
static inline u64 read_cycle_counter(int cpu, u64 wallclock)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
if (wrq->last_cc_update != wallclock) {
wrq->cycles = qcom_cpufreq_get_cpu_cycle_counter(cpu);
@ -477,7 +479,7 @@ static bool is_ed_task_present(struct rq *rq, u64 wallclock, struct task_struct
{
struct task_struct *p;
int loop_max = 10;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
wrq->ed_task = NULL;
@ -509,7 +511,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even
*/
unsigned int walt_big_tasks(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->walt_stats.nr_big_tasks;
}
@ -518,7 +520,7 @@ static void clear_walt_request(int cpu)
{
struct rq *rq = cpu_rq(cpu);
unsigned long flags;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
clear_reserved(cpu);
if (wrq->push_task) {
@ -544,7 +546,7 @@ static void clear_walt_request(int cpu)
*/
static u32 top_task_load(struct rq *rq)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
int index = wrq->prev_top;
u8 prev = 1 - wrq->curr_table;
@ -577,7 +579,7 @@ should_apply_suh_freq_boost(struct walt_sched_cluster *cluster)
static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_sched_cluster *cluster = wrq->cluster;
u64 aggr_grp_load = cluster->aggr_grp_load;
u64 load, tt_load = 0, kload = 0;
@ -628,7 +630,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea
u64 util;
struct rq *rq = cpu_rq(cpu);
unsigned long capacity = capacity_orig_of(cpu);
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
util = scale_time_to_util(freq_policy_load(rq, reason));
@ -704,7 +706,7 @@ cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reaso
*/
static inline void account_load_subtractions(struct rq *rq)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
u64 ws = wrq->window_start;
u64 prev_ws = ws - wrq->prev_window_size;
struct load_subtractions *ls = wrq->load_subs;
@ -747,7 +749,7 @@ static inline void account_load_subtractions(struct rq *rq)
static inline void create_subtraction_entry(struct rq *rq, u64 ws, int index)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
wrq->load_subs[index].window_start = ws;
wrq->load_subs[index].subs = 0;
@ -769,7 +771,7 @@ static bool get_subtraction_index(struct rq *rq, u64 ws)
int i;
u64 oldest = ULLONG_MAX;
int oldest_index = 0;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
for (i = 0; i < NUM_TRACKED_WINDOWS; i++) {
u64 entry_ws = wrq->load_subs[i].window_start;
@ -790,7 +792,7 @@ static bool get_subtraction_index(struct rq *rq, u64 ws)
static void update_rq_load_subtractions(int index, struct rq *rq,
u32 sub_load, bool new_task)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
wrq->load_subs[index].subs += sub_load;
if (new_task)
@ -802,7 +804,7 @@ static void update_cluster_load_subtractions(struct task_struct *p,
{
struct walt_sched_cluster *cluster = cpu_cluster(cpu);
struct cpumask cluster_cpus = cluster->cpus;
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
u64 prev_ws = ws - wrq->prev_window_size;
int i;
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
@ -836,7 +838,7 @@ static inline void migrate_inter_cluster_subtraction(struct task_struct *p, int
bool new_task)
{
struct rq *src_rq = cpu_rq(task_cpu);
struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1;
struct walt_rq *src_wrq = &per_cpu(walt_rq, task_cpu);
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
if (src_wrq->curr_runnable_sum < wts->curr_window_cpu[task_cpu]) {
@ -893,8 +895,7 @@ static inline void migrate_inter_cluster_subtraction(struct task_struct *p, int
static inline void migrate_inter_cluster_addition(struct task_struct *p, int new_cpu,
bool new_task)
{
struct rq *dest_rq = cpu_rq(new_cpu);
struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1;
struct walt_rq *dest_wrq = &per_cpu(walt_rq, new_cpu);
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
@ -924,7 +925,7 @@ static void migrate_top_tasks_subtraction(struct task_struct *p, struct rq *src_
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
u32 curr_window = wts->curr_window;
u32 prev_window = wts->prev_window;
struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1;
struct walt_rq *src_wrq = &per_cpu(walt_rq, cpu_of(src_rq));
u8 src = src_wrq->curr_table;
u8 *src_table;
@ -966,7 +967,7 @@ static void migrate_top_tasks_addition(struct task_struct *p, struct rq *rq)
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
u32 curr_window = wts->curr_window;
u32 prev_window = wts->prev_window;
struct walt_rq *dst_wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *dst_wrq = &per_cpu(walt_rq, cpu_of(rq));
u8 dst = dst_wrq->curr_table;
u8 *dst_table;
@ -1015,7 +1016,7 @@ static void migrate_busy_time_subtraction(struct task_struct *p, int new_cpu)
bool new_task;
struct walt_related_thread_group *grp;
long pstate;
struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1;
struct walt_rq *src_wrq = &per_cpu(walt_rq, cpu_of(src_rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
if (!p->on_rq && READ_ONCE(p->__state) != TASK_WAKING)
@ -1103,11 +1104,10 @@ static void migrate_busy_time_addition(struct task_struct *p, int new_cpu, u64 w
u64 *dst_nt_curr_runnable_sum, *dst_nt_prev_runnable_sum;
bool new_task;
struct walt_related_thread_group *grp;
struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1;
struct walt_rq *dest_wrq = &per_cpu(walt_rq, new_cpu);
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
int src_cpu = wts->prev_cpu;
struct rq *src_rq = cpu_rq(src_cpu);
struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1;
struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu);
walt_update_task_ravg(p, dest_rq, TASK_UPDATE, wallclock, 0);
@ -1358,7 +1358,7 @@ static inline void clear_top_tasks_table(u8 *table)
static void update_top_tasks(struct task_struct *p, struct rq *rq,
u32 old_curr_window, int new_window, bool full_window)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
u8 curr = wrq->curr_table;
u8 prev = 1 - curr;
@ -1456,7 +1456,7 @@ static void update_top_tasks(struct task_struct *p, struct rq *rq,
static void rollover_top_tasks(struct rq *rq, bool full_window)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
u8 curr_table = wrq->curr_table;
u8 prev_table = 1 - curr_table;
int curr_top = wrq->curr_top;
@ -1482,7 +1482,7 @@ static void rollover_task_window(struct task_struct *p, bool full_window)
u32 *curr_cpu_windows = empty_windows;
u32 curr_window;
int i;
struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(task_rq(p)));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
/* Rollover the sum */
@ -1551,7 +1551,7 @@ static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p,
static inline u64 scale_exec_time(u64 delta, struct rq *rq, struct walt_task_struct *wts)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
delta = (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT;
@ -1572,7 +1572,7 @@ static inline unsigned int load_to_freq(struct rq *rq, unsigned int load)
static bool do_pl_notif(struct rq *rq)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
u64 prev = wrq->old_busy_time;
u64 pl = wrq->walt_stats.pred_demands_sum_scaled;
int cpu = cpu_of(rq);
@ -1589,7 +1589,7 @@ static bool do_pl_notif(struct rq *rq)
static void rollover_cpu_window(struct rq *rq, bool full_window)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
u64 curr_sum = wrq->curr_runnable_sum;
u64 nt_curr_sum = wrq->nt_curr_runnable_sum;
u64 grp_curr_sum = wrq->grp_time.curr_runnable_sum;
@ -1668,7 +1668,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
int p_is_curr_task = (p == rq->curr);
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
u64 mark_start = wts->mark_start;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
u64 window_start = wrq->window_start;
u32 window_size = wrq->prev_window_size;
u64 delta;
@ -1976,7 +1976,7 @@ static void update_history(struct rq *rq, struct task_struct *p,
u64 sum = 0;
u16 demand_scaled, pred_demand_scaled, runtime_scaled;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
/* Ignore windows where task had no activity */
if (!runtime || is_idle_task(p) || !samples)
@ -2113,7 +2113,7 @@ static u64 update_task_demand(struct task_struct *p, struct rq *rq,
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
u64 mark_start = wts->mark_start;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
u64 delta, window_start = wrq->window_start;
int new_window, nr_full_windows;
u32 window_size = sched_ravg_window;
@ -2177,7 +2177,7 @@ static u64 update_task_demand(struct task_struct *p, struct rq *rq,
static inline unsigned int cpu_cur_freq(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->cluster->cur_freq;
}
@ -2190,7 +2190,7 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event,
u64 cycles_delta;
u64 time_delta;
int cpu = cpu_of(rq);
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
lockdep_assert_held(&rq->__lock);
@ -2255,7 +2255,7 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event,
static inline void run_walt_irq_work_rollover(u64 old_window_start, struct rq *rq)
{
u64 result;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
if (old_window_start == wrq->window_start)
return;
@ -2273,7 +2273,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even
u64 wallclock, u64 irqtime)
{
u64 old_window_start;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
if (!wrq->window_start || wts->mark_start == wallclock)
@ -2498,7 +2498,7 @@ static void add_cluster(const struct cpumask *cpus, struct list_head *head)
BUG_ON(num_sched_clusters >= MAX_CLUSTERS);
for_each_cpu(i, cpus) {
wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1;
wrq = &per_cpu(walt_rq, i);
wrq->cluster = cluster;
}
@ -2514,7 +2514,7 @@ static void cleanup_clusters(struct list_head *head)
list_for_each_entry_safe(cluster, tmp, head, list) {
for_each_cpu(i, &cluster->cpus) {
wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1;
wrq = &per_cpu(walt_rq, i);
wrq->cluster = &init_cluster;
}
list_del(&cluster->list);
@ -2727,7 +2727,7 @@ static void walt_update_cluster_topology(void)
cluster->max_possible_freq = policy->cpuinfo.max_freq;
cluster->max_freq = policy->max;
for_each_cpu(i, &cluster->cpus) {
wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1;
wrq = &per_cpu(walt_rq, i);
cpumask_copy(&wrq->freq_domain_cpumask,
policy->related_cpus);
}
@ -2768,13 +2768,13 @@ static int cpufreq_notifier_trans(struct notifier_block *nb,
unsigned int cpu = freq->policy->cpu, new_freq = freq->new;
unsigned long flags;
struct walt_sched_cluster *cluster;
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
struct cpumask policy_cpus = wrq->freq_domain_cpumask;
int i, j;
if (use_cycle_counter)
return NOTIFY_DONE;
wrq = (struct walt_rq *) cpu_rq(cpumask_first(&policy_cpus))->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpumask_first(&policy_cpus));
if (wrq->cluster == &init_cluster)
return NOTIFY_DONE;
@ -2785,7 +2785,7 @@ static int cpufreq_notifier_trans(struct notifier_block *nb,
return NOTIFY_OK;
for_each_cpu(i, &policy_cpus) {
wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1;
wrq = &per_cpu(walt_rq, i);
cluster = wrq->cluster;
for_each_cpu(j, &cluster->cpus) {
@ -3315,7 +3315,7 @@ static void transfer_busy_time(struct rq *rq,
int cpu = cpu_of(rq);
bool new_task;
int i;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
wallclock = walt_sched_clock();
@ -3513,7 +3513,7 @@ static void walt_update_irqload(struct rq *rq)
u64 irq_delta = 0;
unsigned int nr_windows = 0;
u64 cur_irq_time;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
u64 last_irq_window = READ_ONCE(wrq->last_irq_window);
if (wrq->window_start > last_irq_window)
@ -3574,7 +3574,7 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc
raw_spin_lock(&cluster->load_lock);
for_each_cpu(cpu, &cluster->cpus) {
rq = cpu_rq(cpu);
wrq = (struct walt_rq *) rq->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpu_of(rq));
if (rq->curr) {
/* only update ravg for locked cpus */
if (cpumask_intersects(lock_cpus, &cluster->cpus)) {
@ -3634,7 +3634,7 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc
int wflag = 0;
rq = cpu_rq(cpu);
wrq = (struct walt_rq *) rq->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpu_of(rq));
if (is_migration) {
if (wrq->notif_pending) {
@ -3676,7 +3676,7 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc
*/
if (!is_migration) {
spin_lock_irqsave(&sched_ravg_window_lock, flags);
wrq = (struct walt_rq *) this_rq()->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpu_of(this_rq()));
if ((sched_ravg_window != new_sched_ravg_window) &&
(wc < wrq->window_start + new_sched_ravg_window)) {
sched_ravg_window_change_time = walt_sched_clock();
@ -3710,7 +3710,7 @@ static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus)
bool keep_locked = false;
for_each_cpu(cpu, &cluster->cpus) {
rq = cpu_rq(cpu);
wrq = (struct walt_rq *)rq->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpu_of(rq));
/* remove this cluster if it's not being notified */
if (wrq->notif_pending) {
@ -3819,7 +3819,7 @@ static void walt_irq_work(struct irq_work *irq_work)
raw_spin_unlock(&cpu_rq(cpu)->__lock);
if (!is_migration) {
wrq = (struct walt_rq *) this_rq()->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpu_of(this_rq()));
core_ctl_check(wrq->window_start);
}
}
@ -3921,7 +3921,7 @@ static void walt_init_once(void)
static void walt_sched_init_rq(struct rq *rq)
{
int j;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
if (cpu_of(rq) == 0)
walt_init_once();
@ -3987,7 +3987,7 @@ void sched_window_nr_ticks_change(void)
static void
walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, wts->demand_scaled,
@ -3997,7 +3997,7 @@ walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
static void
walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats,
@ -4007,7 +4007,7 @@ walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
static void inc_rq_walt_stats(struct rq *rq, struct task_struct *p)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
if (wts->misfit)
@ -4020,7 +4020,7 @@ static void inc_rq_walt_stats(struct rq *rq, struct task_struct *p)
static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
if (wts->misfit)
@ -4103,7 +4103,7 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int
return;
rq = cpu_rq(cpu);
wrq = (struct walt_rq *) rq->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpu_of(rq));
if (start) {
if (!wrq->window_start)
@ -4134,7 +4134,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq,
{
u64 wallclock;
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
bool double_enqueue = false;
if (unlikely(walt_disabled))
@ -4188,7 +4188,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq,
static void android_rvh_dequeue_task(void *unused, struct rq *rq,
struct task_struct *p, int flags)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
bool double_dequeue = false;
@ -4252,7 +4252,7 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p
return;
}
wrq = (struct walt_rq *) rq->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpu_of(rq));
wts = (struct walt_task_struct *) p->android_vendor_data1;
old_misfit = wts->misfit;
@ -4517,7 +4517,7 @@ static int walt_init_stop_handler(void *data)
walt_sched_init_rq(rq);
wrq = (struct walt_rq *) rq->android_vendor_data1;
wrq = &per_cpu(walt_rq, cpu_of(rq));
wrq->window_start = tick_sched_clock;
}
@ -4606,7 +4606,6 @@ static int walt_module_init(void)
{
/* compile time checks for vendor data size */
WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_struct, struct task_struct);
WALT_VENDOR_DATA_SIZE_TEST(struct walt_rq, struct rq);
WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_group, struct task_group);
register_trace_android_vh_update_topology_flags_workfn(

42
kernel/sched/walt/walt.h
View File

@ -134,6 +134,8 @@ struct walt_rq {
u32 enqueue_counter;
};
DECLARE_PER_CPU(struct walt_rq, walt_rq);
struct walt_sched_cluster {
raw_spinlock_t load_lock;
struct list_head list;
@ -250,7 +252,7 @@ extern struct list_head cluster_head;
static inline struct walt_sched_cluster *cpu_cluster(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->cluster;
}
@ -263,7 +265,7 @@ static inline u32 cpu_cycles_to_freq(u64 cycles, u64 period)
static inline unsigned int sched_cpu_legacy_freq(int cpu)
{
unsigned long curr_cap = arch_scale_freq_capacity(cpu);
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return (curr_cap * (u64) wrq->cluster->max_possible_freq) >>
SCHED_CAPACITY_SHIFT;
@ -431,7 +433,7 @@ static inline unsigned long task_util(struct task_struct *p)
static inline unsigned long cpu_util(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
u64 walt_cpu_util = wrq->walt_stats.cumulative_runnable_avg_scaled;
return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu));
@ -609,7 +611,7 @@ static inline bool asym_cap_sibling_group_has_capacity(int dst_cpu, int margin)
/* Is frequency of two cpus synchronized with each other? */
static inline int same_freq_domain(int src_cpu, int dst_cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, src_cpu);
if (src_cpu == dst_cpu)
return 1;
@ -667,14 +669,14 @@ static inline bool hmp_capable(void)
static inline bool is_max_cluster_cpu(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->cluster->id == max_possible_cluster_id;
}
static inline bool is_min_cluster_cpu(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->cluster->id == min_possible_cluster_id;
}
@ -692,7 +694,7 @@ static inline bool is_min_capacity_cluster(struct walt_sched_cluster *cluster)
static inline u64 sched_irqload(int cpu)
{
s64 delta;
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
delta = wrq->window_start - wrq->last_irq_window;
if (delta < SCHED_HIGH_IRQ_TIMEOUT)
@ -703,7 +705,7 @@ static inline u64 sched_irqload(int cpu)
static inline int sched_cpu_high_irqload(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->high_irqload;
}
@ -721,8 +723,8 @@ static inline unsigned int max_task_load(void)
static inline int same_cluster(int src_cpu, int dst_cpu)
{
struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1;
struct walt_rq *dest_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1;
struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu);
struct walt_rq *dest_wrq = &per_cpu(walt_rq, dst_cpu);
return src_wrq->cluster == dest_wrq->cluster;
}
@ -750,7 +752,7 @@ extern u64 get_rtgb_active_time(void);
static inline unsigned int walt_nr_rtg_high_prio(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->walt_stats.nr_rtg_high_prio_tasks;
}
@ -769,8 +771,8 @@ static inline bool task_fits_capacity(struct task_struct *p,
int cpu)
{
unsigned int margin;
struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1;
struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *src_wrq = &per_cpu(walt_rq, task_cpu(p));
struct walt_rq *dst_wrq = &per_cpu(walt_rq, cpu);
/*
* Derive upmigration/downmigrate margin wrt the src/dest CPU.
@ -836,7 +838,7 @@ extern bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p);
static inline unsigned int cpu_max_possible_freq(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->cluster->max_possible_freq;
}
@ -887,24 +889,21 @@ static inline bool walt_get_rtg_status(struct task_struct *p)
#define CPU_RESERVED 1
static inline int is_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return test_bit(CPU_RESERVED, &wrq->walt_flags);
}
static inline int mark_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return test_and_set_bit(CPU_RESERVED, &wrq->walt_flags);
}
static inline void clear_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
clear_bit(CPU_RESERVED, &wrq->walt_flags);
}
@ -1011,8 +1010,7 @@ extern struct cpumask __cpu_halt_mask;
*/
static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct task_struct *p)
{
struct rq *rq = cpu_rq(start_cpu);
struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, start_cpu);
struct walt_sched_cluster *cluster = wrq->cluster;
cpumask_t unhalted_cpus;
int packing_cpu;

36
kernel/sched/walt/walt_cfs.c
View File

@ -18,7 +18,7 @@ static void create_util_to_cost_pd(struct em_perf_domain *pd)
int util, cpu = cpumask_first(to_cpumask(pd->cpus));
unsigned long fmax;
unsigned long scale_cpu;
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
struct walt_sched_cluster *cluster = wrq->cluster;
fmax = (u64)pd->table[pd->nr_perf_states - 1].frequency;
@ -67,8 +67,8 @@ bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu)
bool base_test = cpumask_test_cpu(cpu, p->cpus_ptr) &&
cpu_active(cpu);
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
struct walt_rq *start_wrq = &per_cpu(walt_rq, start_cpu);
bool start_cap_test = (wrq->cluster->id >= start_wrq->cluster->id);
return base_test && start_cap_test;
@ -259,12 +259,12 @@ static void walt_find_best_target(struct sched_domain *sd,
cpumask_t visit_cpus;
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
int packing_cpu;
struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1;
struct walt_rq *prev_wrq = &per_cpu(walt_rq, prev_cpu);
struct walt_rq *start_wrq;
/* Find start CPU based on boost value */
start_cpu = fbt_env->start_cpu;
start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1;
start_wrq = &per_cpu(walt_rq, start_cpu);
/*
* For higher capacity worth I/O tasks, stop the search
@ -318,7 +318,7 @@ static void walt_find_best_target(struct sched_domain *sd,
unsigned long wake_cpu_util, new_cpu_util, new_util_cuml;
long spare_cap;
unsigned int idle_exit_latency = UINT_MAX;
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, i);
trace_sched_cpu_util(i, NULL);
/* record the prss as we visit cpus in a cluster */
@ -506,7 +506,7 @@ static void walt_find_best_target(struct sched_domain *sd,
static inline unsigned long
cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
unsigned long util = wrq->walt_stats.cumulative_runnable_avg_scaled;
bool queued = task_on_rq_queued(p);
@ -571,7 +571,7 @@ cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_ds
static inline unsigned long get_util_to_cost(int cpu, unsigned long util)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
if (cpu == 0 && util > sysctl_em_inflate_thres)
return mult_frac(wrq->cluster->util_to_cost[util], sysctl_em_inflate_pct, 100);
@ -748,8 +748,8 @@ static inline bool select_cpu_same_energy(int cpu, int best_cpu, int prev_cpu)
{
bool new_cpu_is_idle = available_idle_cpu(cpu);
bool best_cpu_is_idle = available_idle_cpu(best_cpu);
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *best_wrq = (struct walt_rq *) cpu_rq(best_cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
struct walt_rq *best_wrq = &per_cpu(walt_rq, best_cpu);
if (best_wrq->cluster->id < wrq->cluster->id)
return false;
@ -806,7 +806,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu,
int first_cpu;
bool energy_eval_needed = true;
struct compute_energy_output output;
struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1;
struct walt_rq *prev_wrq = &per_cpu(walt_rq, prev_cpu);
struct walt_rq *start_wrq;
if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu &&
@ -819,7 +819,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu,
walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost,
&energy_eval_needed);
start_cpu = cpumask_first(&cpu_array[order_index][0]);
start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1;
start_wrq = &per_cpu(walt_rq, start_cpu);
is_rtg = task_in_related_thread_group(p);
curr_is_rtg = task_in_related_thread_group(cpu_rq(cpu)->curr);
@ -1105,7 +1105,7 @@ static void walt_cfs_insert_mvp_task(struct walt_rq *wrq, struct walt_task_struc
void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
list_del_init(&wts->mvp_list);
@ -1122,7 +1122,7 @@ void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p)
*/
static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1;
u64 slice;
unsigned int limit;
@ -1178,7 +1178,7 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr
void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
int mvp_prio = walt_get_mvp_task_prio(p);
@ -1226,7 +1226,7 @@ void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p)
void walt_cfs_tick(struct rq *rq)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts = (struct walt_task_struct *) rq->curr->android_vendor_data1;
if (unlikely(walt_disabled))
@ -1258,7 +1258,7 @@ static void walt_cfs_check_preempt_wakeup(void *unused, struct rq *rq, struct ta
struct sched_entity *se, struct sched_entity *pse,
int next_buddy_marked, unsigned int granularity)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts_p = (struct walt_task_struct *) p->android_vendor_data1;
struct task_struct *c = rq->curr;
struct walt_task_struct *wts_c = (struct walt_task_struct *) rq->curr->android_vendor_data1;
@ -1321,7 +1321,7 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct
struct sched_entity **se, bool *repick, bool simple,
struct task_struct *prev)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_task_struct *wts;
struct task_struct *mvp;
struct cfs_rq *cfs_rq;

2
kernel/sched/walt/walt_halt.c
View File

@ -184,7 +184,7 @@ static int drain_rq_cpu_stop(void *data)
{
struct rq *rq = this_rq();
struct rq_flags rf;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq));
rq_lock_irqsave(rq, &rf);
migrate_tasks(rq, &rf);

36
kernel/sched/walt/walt_lb.c
View File

@ -11,7 +11,7 @@
static inline unsigned long walt_lb_cpu_util(int cpu)
{
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
return wrq->walt_stats.cumulative_runnable_avg_scaled;
}
@ -36,7 +36,7 @@ static int stop_walt_lb_active_migration(void *data)
int busiest_cpu = cpu_of(busiest_rq);
int target_cpu = busiest_rq->push_cpu;
struct rq *target_rq = cpu_rq(target_cpu);
struct walt_rq *wrq = (struct walt_rq *) busiest_rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, busiest_cpu);
struct task_struct *push_task;
int push_task_detached = 0;
@ -235,7 +235,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq)
static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu,
bool to_lower, bool to_higher, bool force)
{
struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, task_cpu(p));
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
/* Don't detach task if it is under active migration */
@ -271,8 +271,8 @@ static inline bool need_active_lb(struct task_struct *p, int dst_cpu,
int src_cpu)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1;
struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1;
struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu);
struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu);
if (cpu_rq(src_cpu)->active_balance)
return false;
@ -293,8 +293,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu)
unsigned long flags;
struct task_struct *pulled_task = NULL, *p;
bool active_balance = false, to_lower, to_higher;
struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1;
struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1;
struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu);
struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu);
struct walt_task_struct *wts;
struct task_struct *pull_me;
int task_visited;
@ -469,7 +469,7 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr
struct walt_rq *wrq;
for_each_cpu(i, src_mask) {
wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1;
wrq = &per_cpu(walt_rq, i);
trace_walt_lb_cpu_util(i, wrq);
if (cpu_rq(i)->nr_running < 2 || !cpu_rq(i)->cfs.h_nr_running)
@ -501,7 +501,7 @@ static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t
if (!cpu_active(i))
continue;
wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1;
wrq = &per_cpu(walt_rq, i);
trace_walt_lb_cpu_util(i, wrq);
util = walt_lb_cpu_util(i);
@ -566,7 +566,7 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t
* refactor this after final testing is done.
*/
for_each_cpu(i, src_mask) {
wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1;
wrq = &per_cpu(walt_rq, i);
if (!cpu_active(i))
continue;
@ -620,8 +620,8 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int
{
int fsrc_cpu = cpumask_first(src_mask);
int busiest_cpu;
struct walt_rq *fsrc_wrq = (struct walt_rq *) cpu_rq(fsrc_cpu)->android_vendor_data1;
struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1;
struct walt_rq *fsrc_wrq = &per_cpu(walt_rq, fsrc_cpu);
struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu);
if (dst_wrq->cluster->id == fsrc_wrq->cluster->id)
busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu,
@ -642,7 +642,7 @@ void walt_lb_tick(struct rq *rq)
int prev_cpu = rq->cpu, new_cpu, ret;
struct task_struct *p = rq->curr;
unsigned long flags;
struct walt_rq *prev_wrq = (struct walt_rq *) rq->android_vendor_data1;
struct walt_rq *prev_wrq = &per_cpu(walt_rq, cpu_of(rq));
struct walt_rq *new_wrq;
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
@ -673,7 +673,7 @@ void walt_lb_tick(struct rq *rq)
new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1);
rcu_read_unlock();
new_wrq = (struct walt_rq *) cpu_rq(new_cpu)->android_vendor_data1;
new_wrq = &per_cpu(walt_rq, new_cpu);
/* prevent active task migration to busy or same/lower capacity CPU */
if (!available_idle_cpu(new_cpu) || new_wrq->cluster->id <= prev_wrq->cluster->id)
@ -787,7 +787,7 @@ static bool should_help_min_cap(int this_cpu)
return false;
for_each_cpu(cpu, &cpu_array[0][0]) {
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
if (wrq->walt_stats.nr_big_tasks)
return true;
@ -803,7 +803,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq,
int *done)
{
int this_cpu = this_rq->cpu;
struct walt_rq *wrq = (struct walt_rq *) this_rq->android_vendor_data1;
struct walt_rq *wrq = &per_cpu(walt_rq, this_cpu);
int order_index;
int busy_cpu = -1;
bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD);
@ -1088,8 +1088,8 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p,
int dst_cpu, int *can_migrate)
{
bool to_lower, to_higher;
struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1;
struct walt_rq *task_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1;
struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu);
struct walt_rq *task_wrq = &per_cpu(walt_rq, task_cpu(p));
if (unlikely(walt_disabled))
return;