From ce976544d1efaf470a946f3982d64cdb4c386ae3 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Wed, 8 Jun 2022 14:17:11 -0700 Subject: [PATCH] sched/walt: Reorganize walt_rq structure Instead of having a field in struct rq for vendor data, maintain a per CPU variable of all the rq related members within the WALT module. Change-Id: I307142369fb27a4bfd73e2094e381091444e6b74 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/sched_avg.c | 4 +- kernel/sched/walt/trace.c | 4 +- kernel/sched/walt/trace.h | 2 +- kernel/sched/walt/walt.c | 123 +++++++++++++++++----------------- kernel/sched/walt/walt.h | 42 ++++++------ kernel/sched/walt/walt_cfs.c | 36 +++++----- kernel/sched/walt/walt_halt.c | 2 +- kernel/sched/walt/walt_lb.c | 36 +++++----- 8 files changed, 123 insertions(+), 126 deletions(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index c18e5c2234a9..b26a9c0057b1 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -130,7 +130,7 @@ struct sched_avg_stats *sched_get_nr_running_avg(void) /* load is already scaled, see freq_policy_load/prev_runnable_sum */ for_each_cpu(cpu, &cluster->cpus) { struct rq *rq = cpu_rq(cpu); - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); /* compute the % this cpu's utilization of the cpu capacity, * and sum it across all cpus @@ -303,7 +303,7 @@ unsigned int sched_get_cpu_util_pct(int cpu) u64 util; unsigned long capacity, flags; unsigned int busy; - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); raw_spin_lock_irqsave(&rq->__lock, flags); diff --git a/kernel/sched/walt/trace.c b/kernel/sched/walt/trace.c index cf9c34077f2c..c508fcaef0e2 100644 --- a/kernel/sched/walt/trace.c +++ b/kernel/sched/walt/trace.c @@ -29,7 +29,7 @@ const char *__window_print(struct trace_seq *p, const u32 *buf, int buf_len) static inline s64 __rq_update_sum(struct rq *rq, bool curr, bool new) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); if (curr) if (new) @@ -45,7 +45,7 @@ static inline s64 __rq_update_sum(struct rq *rq, bool curr, bool new) static inline s64 __grp_update_sum(struct rq *rq, bool curr, bool new) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); if (curr) if (new) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index e656b7fa1f80..4ba1d185b022 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -952,7 +952,7 @@ TRACE_EVENT(sched_cpu_util, ), TP_fast_assign( - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); __entry->cpu = cpu; __entry->nr_running = cpu_rq(cpu)->nr_running; __entry->cpu_util = cpu_util(cpu); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 88373f0d8bf2..c8a046d99045 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -47,6 +47,8 @@ const char *migrate_type_names[] = { #define NEW_TASK_ACTIVE_TIME 100000000 +DEFINE_PER_CPU(struct walt_rq, walt_rq); + unsigned int sysctl_sched_user_hint; static u64 sched_clock_last; static bool walt_clock_suspended; @@ -151,7 +153,7 @@ static inline void release_rq_locks_irqrestore(const cpumask_t *cpus, static inline u64 walt_rq_clock(struct rq *rq) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); if (unlikely(walt_clock_suspended)) return sched_clock_last; @@ -241,7 +243,7 @@ void walt_rq_dump(int cpu) struct rq *rq = cpu_rq(cpu); struct task_struct *tsk = cpu_curr(cpu); int i; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); /* * Increment the task reference so that it can't be @@ -344,7 +346,7 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, wts->demand_scaled; s64 pred_demand_delta = (s64)updated_pred_demand_scaled - wts->pred_demand_scaled; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, task_load_delta, pred_demand_delta); @@ -403,7 +405,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) { s64 delta; int nr_windows; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); u64 old_window_start = wrq->window_start; bool full_window; @@ -442,7 +444,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) */ static inline u64 read_cycle_counter(int cpu, u64 wallclock) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); if (wrq->last_cc_update != wallclock) { wrq->cycles = qcom_cpufreq_get_cpu_cycle_counter(cpu); @@ -477,7 +479,7 @@ static bool is_ed_task_present(struct rq *rq, u64 wallclock, struct task_struct { struct task_struct *p; int loop_max = 10; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); wrq->ed_task = NULL; @@ -509,7 +511,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even */ unsigned int walt_big_tasks(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->walt_stats.nr_big_tasks; } @@ -518,7 +520,7 @@ static void clear_walt_request(int cpu) { struct rq *rq = cpu_rq(cpu); unsigned long flags; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); clear_reserved(cpu); if (wrq->push_task) { @@ -544,7 +546,7 @@ static void clear_walt_request(int cpu) */ static u32 top_task_load(struct rq *rq) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); int index = wrq->prev_top; u8 prev = 1 - wrq->curr_table; @@ -577,7 +579,7 @@ should_apply_suh_freq_boost(struct walt_sched_cluster *cluster) static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_sched_cluster *cluster = wrq->cluster; u64 aggr_grp_load = cluster->aggr_grp_load; u64 load, tt_load = 0, kload = 0; @@ -628,7 +630,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea u64 util; struct rq *rq = cpu_rq(cpu); unsigned long capacity = capacity_orig_of(cpu); - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); util = scale_time_to_util(freq_policy_load(rq, reason)); @@ -704,7 +706,7 @@ cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reaso */ static inline void account_load_subtractions(struct rq *rq) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); u64 ws = wrq->window_start; u64 prev_ws = ws - wrq->prev_window_size; struct load_subtractions *ls = wrq->load_subs; @@ -747,7 +749,7 @@ static inline void account_load_subtractions(struct rq *rq) static inline void create_subtraction_entry(struct rq *rq, u64 ws, int index) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); wrq->load_subs[index].window_start = ws; wrq->load_subs[index].subs = 0; @@ -769,7 +771,7 @@ static bool get_subtraction_index(struct rq *rq, u64 ws) int i; u64 oldest = ULLONG_MAX; int oldest_index = 0; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); for (i = 0; i < NUM_TRACKED_WINDOWS; i++) { u64 entry_ws = wrq->load_subs[i].window_start; @@ -790,7 +792,7 @@ static bool get_subtraction_index(struct rq *rq, u64 ws) static void update_rq_load_subtractions(int index, struct rq *rq, u32 sub_load, bool new_task) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); wrq->load_subs[index].subs += sub_load; if (new_task) @@ -802,7 +804,7 @@ static void update_cluster_load_subtractions(struct task_struct *p, { struct walt_sched_cluster *cluster = cpu_cluster(cpu); struct cpumask cluster_cpus = cluster->cpus; - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); u64 prev_ws = ws - wrq->prev_window_size; int i; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -836,7 +838,7 @@ static inline void migrate_inter_cluster_subtraction(struct task_struct *p, int bool new_task) { struct rq *src_rq = cpu_rq(task_cpu); - struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_rq *src_wrq = &per_cpu(walt_rq, task_cpu); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (src_wrq->curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { @@ -893,8 +895,7 @@ static inline void migrate_inter_cluster_subtraction(struct task_struct *p, int static inline void migrate_inter_cluster_addition(struct task_struct *p, int new_cpu, bool new_task) { - struct rq *dest_rq = cpu_rq(new_cpu); - struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; + struct walt_rq *dest_wrq = &per_cpu(walt_rq, new_cpu); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -924,7 +925,7 @@ static void migrate_top_tasks_subtraction(struct task_struct *p, struct rq *src_ struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u32 curr_window = wts->curr_window; u32 prev_window = wts->prev_window; - struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_rq *src_wrq = &per_cpu(walt_rq, cpu_of(src_rq)); u8 src = src_wrq->curr_table; u8 *src_table; @@ -966,7 +967,7 @@ static void migrate_top_tasks_addition(struct task_struct *p, struct rq *rq) struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u32 curr_window = wts->curr_window; u32 prev_window = wts->prev_window; - struct walt_rq *dst_wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *dst_wrq = &per_cpu(walt_rq, cpu_of(rq)); u8 dst = dst_wrq->curr_table; u8 *dst_table; @@ -1015,7 +1016,7 @@ static void migrate_busy_time_subtraction(struct task_struct *p, int new_cpu) bool new_task; struct walt_related_thread_group *grp; long pstate; - struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_rq *src_wrq = &per_cpu(walt_rq, cpu_of(src_rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (!p->on_rq && READ_ONCE(p->__state) != TASK_WAKING) @@ -1103,11 +1104,10 @@ static void migrate_busy_time_addition(struct task_struct *p, int new_cpu, u64 w u64 *dst_nt_curr_runnable_sum, *dst_nt_prev_runnable_sum; bool new_task; struct walt_related_thread_group *grp; - struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; + struct walt_rq *dest_wrq = &per_cpu(walt_rq, new_cpu); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; int src_cpu = wts->prev_cpu; - struct rq *src_rq = cpu_rq(src_cpu); - struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu); walt_update_task_ravg(p, dest_rq, TASK_UPDATE, wallclock, 0); @@ -1358,7 +1358,7 @@ static inline void clear_top_tasks_table(u8 *table) static void update_top_tasks(struct task_struct *p, struct rq *rq, u32 old_curr_window, int new_window, bool full_window) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u8 curr = wrq->curr_table; u8 prev = 1 - curr; @@ -1456,7 +1456,7 @@ static void update_top_tasks(struct task_struct *p, struct rq *rq, static void rollover_top_tasks(struct rq *rq, bool full_window) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); u8 curr_table = wrq->curr_table; u8 prev_table = 1 - curr_table; int curr_top = wrq->curr_top; @@ -1482,7 +1482,7 @@ static void rollover_task_window(struct task_struct *p, bool full_window) u32 *curr_cpu_windows = empty_windows; u32 curr_window; int i; - struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(task_rq(p))); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; /* Rollover the sum */ @@ -1551,7 +1551,7 @@ static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, static inline u64 scale_exec_time(u64 delta, struct rq *rq, struct walt_task_struct *wts) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); delta = (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT; @@ -1572,7 +1572,7 @@ static inline unsigned int load_to_freq(struct rq *rq, unsigned int load) static bool do_pl_notif(struct rq *rq) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); u64 prev = wrq->old_busy_time; u64 pl = wrq->walt_stats.pred_demands_sum_scaled; int cpu = cpu_of(rq); @@ -1589,7 +1589,7 @@ static bool do_pl_notif(struct rq *rq) static void rollover_cpu_window(struct rq *rq, bool full_window) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); u64 curr_sum = wrq->curr_runnable_sum; u64 nt_curr_sum = wrq->nt_curr_runnable_sum; u64 grp_curr_sum = wrq->grp_time.curr_runnable_sum; @@ -1668,7 +1668,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, int p_is_curr_task = (p == rq->curr); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u64 mark_start = wts->mark_start; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); u64 window_start = wrq->window_start; u32 window_size = wrq->prev_window_size; u64 delta; @@ -1976,7 +1976,7 @@ static void update_history(struct rq *rq, struct task_struct *p, u64 sum = 0; u16 demand_scaled, pred_demand_scaled, runtime_scaled; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); /* Ignore windows where task had no activity */ if (!runtime || is_idle_task(p) || !samples) @@ -2113,7 +2113,7 @@ static u64 update_task_demand(struct task_struct *p, struct rq *rq, { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u64 mark_start = wts->mark_start; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); u64 delta, window_start = wrq->window_start; int new_window, nr_full_windows; u32 window_size = sched_ravg_window; @@ -2177,7 +2177,7 @@ static u64 update_task_demand(struct task_struct *p, struct rq *rq, static inline unsigned int cpu_cur_freq(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->cluster->cur_freq; } @@ -2190,7 +2190,7 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, u64 cycles_delta; u64 time_delta; int cpu = cpu_of(rq); - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; lockdep_assert_held(&rq->__lock); @@ -2255,7 +2255,7 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, static inline void run_walt_irq_work_rollover(u64 old_window_start, struct rq *rq) { u64 result; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); if (old_window_start == wrq->window_start) return; @@ -2273,7 +2273,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even u64 wallclock, u64 irqtime) { u64 old_window_start; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (!wrq->window_start || wts->mark_start == wallclock) @@ -2498,7 +2498,7 @@ static void add_cluster(const struct cpumask *cpus, struct list_head *head) BUG_ON(num_sched_clusters >= MAX_CLUSTERS); for_each_cpu(i, cpus) { - wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq = &per_cpu(walt_rq, i); wrq->cluster = cluster; } @@ -2514,7 +2514,7 @@ static void cleanup_clusters(struct list_head *head) list_for_each_entry_safe(cluster, tmp, head, list) { for_each_cpu(i, &cluster->cpus) { - wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq = &per_cpu(walt_rq, i); wrq->cluster = &init_cluster; } list_del(&cluster->list); @@ -2727,7 +2727,7 @@ static void walt_update_cluster_topology(void) cluster->max_possible_freq = policy->cpuinfo.max_freq; cluster->max_freq = policy->max; for_each_cpu(i, &cluster->cpus) { - wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq = &per_cpu(walt_rq, i); cpumask_copy(&wrq->freq_domain_cpumask, policy->related_cpus); } @@ -2768,13 +2768,13 @@ static int cpufreq_notifier_trans(struct notifier_block *nb, unsigned int cpu = freq->policy->cpu, new_freq = freq->new; unsigned long flags; struct walt_sched_cluster *cluster; - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); struct cpumask policy_cpus = wrq->freq_domain_cpumask; int i, j; if (use_cycle_counter) return NOTIFY_DONE; - wrq = (struct walt_rq *) cpu_rq(cpumask_first(&policy_cpus))->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpumask_first(&policy_cpus)); if (wrq->cluster == &init_cluster) return NOTIFY_DONE; @@ -2785,7 +2785,7 @@ static int cpufreq_notifier_trans(struct notifier_block *nb, return NOTIFY_OK; for_each_cpu(i, &policy_cpus) { - wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq = &per_cpu(walt_rq, i); cluster = wrq->cluster; for_each_cpu(j, &cluster->cpus) { @@ -3315,7 +3315,7 @@ static void transfer_busy_time(struct rq *rq, int cpu = cpu_of(rq); bool new_task; int i; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; wallclock = walt_sched_clock(); @@ -3513,7 +3513,7 @@ static void walt_update_irqload(struct rq *rq) u64 irq_delta = 0; unsigned int nr_windows = 0; u64 cur_irq_time; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); u64 last_irq_window = READ_ONCE(wrq->last_irq_window); if (wrq->window_start > last_irq_window) @@ -3574,7 +3574,7 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc raw_spin_lock(&cluster->load_lock); for_each_cpu(cpu, &cluster->cpus) { rq = cpu_rq(cpu); - wrq = (struct walt_rq *) rq->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpu_of(rq)); if (rq->curr) { /* only update ravg for locked cpus */ if (cpumask_intersects(lock_cpus, &cluster->cpus)) { @@ -3634,7 +3634,7 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc int wflag = 0; rq = cpu_rq(cpu); - wrq = (struct walt_rq *) rq->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpu_of(rq)); if (is_migration) { if (wrq->notif_pending) { @@ -3676,7 +3676,7 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc */ if (!is_migration) { spin_lock_irqsave(&sched_ravg_window_lock, flags); - wrq = (struct walt_rq *) this_rq()->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpu_of(this_rq())); if ((sched_ravg_window != new_sched_ravg_window) && (wc < wrq->window_start + new_sched_ravg_window)) { sched_ravg_window_change_time = walt_sched_clock(); @@ -3710,7 +3710,7 @@ static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus) bool keep_locked = false; for_each_cpu(cpu, &cluster->cpus) { rq = cpu_rq(cpu); - wrq = (struct walt_rq *)rq->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpu_of(rq)); /* remove this cluster if it's not being notified */ if (wrq->notif_pending) { @@ -3819,7 +3819,7 @@ static void walt_irq_work(struct irq_work *irq_work) raw_spin_unlock(&cpu_rq(cpu)->__lock); if (!is_migration) { - wrq = (struct walt_rq *) this_rq()->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpu_of(this_rq())); core_ctl_check(wrq->window_start); } } @@ -3921,7 +3921,7 @@ static void walt_init_once(void) static void walt_sched_init_rq(struct rq *rq) { int j; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); if (cpu_of(rq) == 0) walt_init_once(); @@ -3987,7 +3987,7 @@ void sched_window_nr_ticks_change(void) static void walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, wts->demand_scaled, @@ -3997,7 +3997,7 @@ walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) static void walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, @@ -4007,7 +4007,7 @@ walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) static void inc_rq_walt_stats(struct rq *rq, struct task_struct *p) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (wts->misfit) @@ -4020,7 +4020,7 @@ static void inc_rq_walt_stats(struct rq *rq, struct task_struct *p) static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (wts->misfit) @@ -4103,7 +4103,7 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int return; rq = cpu_rq(cpu); - wrq = (struct walt_rq *) rq->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpu_of(rq)); if (start) { if (!wrq->window_start) @@ -4134,7 +4134,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, { u64 wallclock; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); bool double_enqueue = false; if (unlikely(walt_disabled)) @@ -4188,7 +4188,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; bool double_dequeue = false; @@ -4252,7 +4252,7 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p return; } - wrq = (struct walt_rq *) rq->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpu_of(rq)); wts = (struct walt_task_struct *) p->android_vendor_data1; old_misfit = wts->misfit; @@ -4517,7 +4517,7 @@ static int walt_init_stop_handler(void *data) walt_sched_init_rq(rq); - wrq = (struct walt_rq *) rq->android_vendor_data1; + wrq = &per_cpu(walt_rq, cpu_of(rq)); wrq->window_start = tick_sched_clock; } @@ -4606,7 +4606,6 @@ static int walt_module_init(void) { /* compile time checks for vendor data size */ WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_struct, struct task_struct); - WALT_VENDOR_DATA_SIZE_TEST(struct walt_rq, struct rq); WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_group, struct task_group); register_trace_android_vh_update_topology_flags_workfn( diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index c32d670af629..1ded08bab0a2 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -134,6 +134,8 @@ struct walt_rq { u32 enqueue_counter; }; +DECLARE_PER_CPU(struct walt_rq, walt_rq); + struct walt_sched_cluster { raw_spinlock_t load_lock; struct list_head list; @@ -250,7 +252,7 @@ extern struct list_head cluster_head; static inline struct walt_sched_cluster *cpu_cluster(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->cluster; } @@ -263,7 +265,7 @@ static inline u32 cpu_cycles_to_freq(u64 cycles, u64 period) static inline unsigned int sched_cpu_legacy_freq(int cpu) { unsigned long curr_cap = arch_scale_freq_capacity(cpu); - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return (curr_cap * (u64) wrq->cluster->max_possible_freq) >> SCHED_CAPACITY_SHIFT; @@ -431,7 +433,7 @@ static inline unsigned long task_util(struct task_struct *p) static inline unsigned long cpu_util(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); u64 walt_cpu_util = wrq->walt_stats.cumulative_runnable_avg_scaled; return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu)); @@ -609,7 +611,7 @@ static inline bool asym_cap_sibling_group_has_capacity(int dst_cpu, int margin) /* Is frequency of two cpus synchronized with each other? */ static inline int same_freq_domain(int src_cpu, int dst_cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, src_cpu); if (src_cpu == dst_cpu) return 1; @@ -667,14 +669,14 @@ static inline bool hmp_capable(void) static inline bool is_max_cluster_cpu(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->cluster->id == max_possible_cluster_id; } static inline bool is_min_cluster_cpu(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->cluster->id == min_possible_cluster_id; } @@ -692,7 +694,7 @@ static inline bool is_min_capacity_cluster(struct walt_sched_cluster *cluster) static inline u64 sched_irqload(int cpu) { s64 delta; - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); delta = wrq->window_start - wrq->last_irq_window; if (delta < SCHED_HIGH_IRQ_TIMEOUT) @@ -703,7 +705,7 @@ static inline u64 sched_irqload(int cpu) static inline int sched_cpu_high_irqload(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->high_irqload; } @@ -721,8 +723,8 @@ static inline unsigned int max_task_load(void) static inline int same_cluster(int src_cpu, int dst_cpu) { - struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; - struct walt_rq *dest_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu); + struct walt_rq *dest_wrq = &per_cpu(walt_rq, dst_cpu); return src_wrq->cluster == dest_wrq->cluster; } @@ -750,7 +752,7 @@ extern u64 get_rtgb_active_time(void); static inline unsigned int walt_nr_rtg_high_prio(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->walt_stats.nr_rtg_high_prio_tasks; } @@ -769,8 +771,8 @@ static inline bool task_fits_capacity(struct task_struct *p, int cpu) { unsigned int margin; - struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1; - struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *src_wrq = &per_cpu(walt_rq, task_cpu(p)); + struct walt_rq *dst_wrq = &per_cpu(walt_rq, cpu); /* * Derive upmigration/downmigrate margin wrt the src/dest CPU. @@ -836,7 +838,7 @@ extern bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p); static inline unsigned int cpu_max_possible_freq(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->cluster->max_possible_freq; } @@ -887,24 +889,21 @@ static inline bool walt_get_rtg_status(struct task_struct *p) #define CPU_RESERVED 1 static inline int is_reserved(int cpu) { - struct rq *rq = cpu_rq(cpu); - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return test_bit(CPU_RESERVED, &wrq->walt_flags); } static inline int mark_reserved(int cpu) { - struct rq *rq = cpu_rq(cpu); - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return test_and_set_bit(CPU_RESERVED, &wrq->walt_flags); } static inline void clear_reserved(int cpu) { - struct rq *rq = cpu_rq(cpu); - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); clear_bit(CPU_RESERVED, &wrq->walt_flags); } @@ -1011,8 +1010,7 @@ extern struct cpumask __cpu_halt_mask; */ static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct task_struct *p) { - struct rq *rq = cpu_rq(start_cpu); - struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, start_cpu); struct walt_sched_cluster *cluster = wrq->cluster; cpumask_t unhalted_cpus; int packing_cpu; diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 61fd4f8debe7..b008c455b09a 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -18,7 +18,7 @@ static void create_util_to_cost_pd(struct em_perf_domain *pd) int util, cpu = cpumask_first(to_cpumask(pd->cpus)); unsigned long fmax; unsigned long scale_cpu; - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); struct walt_sched_cluster *cluster = wrq->cluster; fmax = (u64)pd->table[pd->nr_perf_states - 1].frequency; @@ -67,8 +67,8 @@ bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) bool base_test = cpumask_test_cpu(cpu, p->cpus_ptr) && cpu_active(cpu); - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; - struct walt_rq *start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); + struct walt_rq *start_wrq = &per_cpu(walt_rq, start_cpu); bool start_cap_test = (wrq->cluster->id >= start_wrq->cluster->id); return base_test && start_cap_test; @@ -259,12 +259,12 @@ static void walt_find_best_target(struct sched_domain *sd, cpumask_t visit_cpus; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; int packing_cpu; - struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1; + struct walt_rq *prev_wrq = &per_cpu(walt_rq, prev_cpu); struct walt_rq *start_wrq; /* Find start CPU based on boost value */ start_cpu = fbt_env->start_cpu; - start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; + start_wrq = &per_cpu(walt_rq, start_cpu); /* * For higher capacity worth I/O tasks, stop the search @@ -318,7 +318,7 @@ static void walt_find_best_target(struct sched_domain *sd, unsigned long wake_cpu_util, new_cpu_util, new_util_cuml; long spare_cap; unsigned int idle_exit_latency = UINT_MAX; - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, i); trace_sched_cpu_util(i, NULL); /* record the prss as we visit cpus in a cluster */ @@ -506,7 +506,7 @@ static void walt_find_best_target(struct sched_domain *sd, static inline unsigned long cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); unsigned long util = wrq->walt_stats.cumulative_runnable_avg_scaled; bool queued = task_on_rq_queued(p); @@ -571,7 +571,7 @@ cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_ds static inline unsigned long get_util_to_cost(int cpu, unsigned long util) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); if (cpu == 0 && util > sysctl_em_inflate_thres) return mult_frac(wrq->cluster->util_to_cost[util], sysctl_em_inflate_pct, 100); @@ -748,8 +748,8 @@ static inline bool select_cpu_same_energy(int cpu, int best_cpu, int prev_cpu) { bool new_cpu_is_idle = available_idle_cpu(cpu); bool best_cpu_is_idle = available_idle_cpu(best_cpu); - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; - struct walt_rq *best_wrq = (struct walt_rq *) cpu_rq(best_cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); + struct walt_rq *best_wrq = &per_cpu(walt_rq, best_cpu); if (best_wrq->cluster->id < wrq->cluster->id) return false; @@ -806,7 +806,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int first_cpu; bool energy_eval_needed = true; struct compute_energy_output output; - struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1; + struct walt_rq *prev_wrq = &per_cpu(walt_rq, prev_cpu); struct walt_rq *start_wrq; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && @@ -819,7 +819,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost, &energy_eval_needed); start_cpu = cpumask_first(&cpu_array[order_index][0]); - start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; + start_wrq = &per_cpu(walt_rq, start_cpu); is_rtg = task_in_related_thread_group(p); curr_is_rtg = task_in_related_thread_group(cpu_rq(cpu)->curr); @@ -1105,7 +1105,7 @@ static void walt_cfs_insert_mvp_task(struct walt_rq *wrq, struct walt_task_struc void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; list_del_init(&wts->mvp_list); @@ -1122,7 +1122,7 @@ void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p) */ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1; u64 slice; unsigned int limit; @@ -1178,7 +1178,7 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; int mvp_prio = walt_get_mvp_task_prio(p); @@ -1226,7 +1226,7 @@ void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) void walt_cfs_tick(struct rq *rq) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; if (unlikely(walt_disabled)) @@ -1258,7 +1258,7 @@ static void walt_cfs_check_preempt_wakeup(void *unused, struct rq *rq, struct ta struct sched_entity *se, struct sched_entity *pse, int next_buddy_marked, unsigned int granularity) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts_p = (struct walt_task_struct *) p->android_vendor_data1; struct task_struct *c = rq->curr; struct walt_task_struct *wts_c = (struct walt_task_struct *) rq->curr->android_vendor_data1; @@ -1321,7 +1321,7 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct struct sched_entity **se, bool *repick, bool simple, struct task_struct *prev) { - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_task_struct *wts; struct task_struct *mvp; struct cfs_rq *cfs_rq; diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index e6b61e00182f..833e879eb277 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -184,7 +184,7 @@ static int drain_rq_cpu_stop(void *data) { struct rq *rq = this_rq(); struct rq_flags rf; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu_of(rq)); rq_lock_irqsave(rq, &rf); migrate_tasks(rq, &rf); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index a4dc29d3f12a..02c14e6bbda7 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -11,7 +11,7 @@ static inline unsigned long walt_lb_cpu_util(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); return wrq->walt_stats.cumulative_runnable_avg_scaled; } @@ -36,7 +36,7 @@ static int stop_walt_lb_active_migration(void *data) int busiest_cpu = cpu_of(busiest_rq); int target_cpu = busiest_rq->push_cpu; struct rq *target_rq = cpu_rq(target_cpu); - struct walt_rq *wrq = (struct walt_rq *) busiest_rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, busiest_cpu); struct task_struct *push_task; int push_task_detached = 0; @@ -235,7 +235,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, bool to_lower, bool to_higher, bool force) { - struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, task_cpu(p)); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; /* Don't detach task if it is under active migration */ @@ -271,8 +271,8 @@ static inline bool need_active_lb(struct task_struct *p, int dst_cpu, int src_cpu) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; - struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu); + struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu); if (cpu_rq(src_cpu)->active_balance) return false; @@ -293,8 +293,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) unsigned long flags; struct task_struct *pulled_task = NULL, *p; bool active_balance = false, to_lower, to_higher; - struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; - struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu); + struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu); struct walt_task_struct *wts; struct task_struct *pull_me; int task_visited; @@ -469,7 +469,7 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr struct walt_rq *wrq; for_each_cpu(i, src_mask) { - wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq = &per_cpu(walt_rq, i); trace_walt_lb_cpu_util(i, wrq); if (cpu_rq(i)->nr_running < 2 || !cpu_rq(i)->cfs.h_nr_running) @@ -501,7 +501,7 @@ static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t if (!cpu_active(i)) continue; - wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq = &per_cpu(walt_rq, i); trace_walt_lb_cpu_util(i, wrq); util = walt_lb_cpu_util(i); @@ -566,7 +566,7 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t * refactor this after final testing is done. */ for_each_cpu(i, src_mask) { - wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq = &per_cpu(walt_rq, i); if (!cpu_active(i)) continue; @@ -620,8 +620,8 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int { int fsrc_cpu = cpumask_first(src_mask); int busiest_cpu; - struct walt_rq *fsrc_wrq = (struct walt_rq *) cpu_rq(fsrc_cpu)->android_vendor_data1; - struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + struct walt_rq *fsrc_wrq = &per_cpu(walt_rq, fsrc_cpu); + struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu); if (dst_wrq->cluster->id == fsrc_wrq->cluster->id) busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, @@ -642,7 +642,7 @@ void walt_lb_tick(struct rq *rq) int prev_cpu = rq->cpu, new_cpu, ret; struct task_struct *p = rq->curr; unsigned long flags; - struct walt_rq *prev_wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *prev_wrq = &per_cpu(walt_rq, cpu_of(rq)); struct walt_rq *new_wrq; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -673,7 +673,7 @@ void walt_lb_tick(struct rq *rq) new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1); rcu_read_unlock(); - new_wrq = (struct walt_rq *) cpu_rq(new_cpu)->android_vendor_data1; + new_wrq = &per_cpu(walt_rq, new_cpu); /* prevent active task migration to busy or same/lower capacity CPU */ if (!available_idle_cpu(new_cpu) || new_wrq->cluster->id <= prev_wrq->cluster->id) @@ -787,7 +787,7 @@ static bool should_help_min_cap(int this_cpu) return false; for_each_cpu(cpu, &cpu_array[0][0]) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, cpu); if (wrq->walt_stats.nr_big_tasks) return true; @@ -803,7 +803,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, int *done) { int this_cpu = this_rq->cpu; - struct walt_rq *wrq = (struct walt_rq *) this_rq->android_vendor_data1; + struct walt_rq *wrq = &per_cpu(walt_rq, this_cpu); int order_index; int busy_cpu = -1; bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); @@ -1088,8 +1088,8 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, int dst_cpu, int *can_migrate) { bool to_lower, to_higher; - struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; - struct walt_rq *task_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1; + struct walt_rq *dst_wrq = &per_cpu(walt_rq, dst_cpu); + struct walt_rq *task_wrq = &per_cpu(walt_rq, task_cpu(p)); if (unlikely(walt_disabled)) return;