github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-08-01 03:59:40 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched: Use bitshift over division

Browse Source 
Now that our window history size is a power of 2, we can take
advantage of this to replace division operations in a critical code
path by using bit-shifts instead.

Change-Id: Idc0ed37a87814783fcab2f4d5204b79fb9ce9996
Signed-off-by: Shaleen Agrawal <quic_shalagra@quicinc.com>
This commit is contained in:
Shaleen Agrawal 2022-01-24 17:02:45 -08:00 committed by Rishabh Bhatnagar
parent bb7363e104
commit 69bd1582b2
3 changed files with 17 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/linux/sched/walt.h
View File

@ -32,7 +32,13 @@ enum task_boost_type {
};
#define WALT_NR_CPUS 8
#define RAVG_HIST_SIZE_MAX 8
/*
* RAVG_HIST_SHIFT trick can only be used if RAVG_HIST_SIZE is a power of 2.
*/
#define RAVG_HIST_SIZE 8
#define RAVG_HIST_SHIFT 3
#define RAVG_HIST_MASK (RAVG_HIST_SIZE - 1)
/* wts->bucket_bitmask needs to be updated if NUM_BUSY_BUCKETS > 16 */
#define NUM_BUSY_BUCKETS 16
#define NUM_BUSY_BUCKETS_SHIFT 4
@ -90,7 +96,7 @@ struct walt_task_struct {
u64 mark_start;
u32 sum, demand;
u32 coloc_demand;
u32 sum_history[RAVG_HIST_SIZE_MAX];
u32 sum_history[RAVG_HIST_SIZE];
u32 curr_window_cpu[WALT_NR_CPUS];
u32 prev_window_cpu[WALT_NR_CPUS];
u32 curr_window, prev_window;

4
kernel/sched/walt/trace.h
View File

@ -83,7 +83,7 @@ TRACE_EVENT(sched_update_history,
__field(unsigned int, demand)
__field(unsigned int, coloc_demand)
__field(unsigned int, pred_demand_scaled)
__array(u32, hist, RAVG_HIST_SIZE_MAX)
__array(u32, hist, RAVG_HIST_SIZE)
__field(unsigned int, nr_big_tasks)
__field(int, cpu)
),
@ -98,7 +98,7 @@ TRACE_EVENT(sched_update_history,
__entry->coloc_demand = wts->coloc_demand;
__entry->pred_demand_scaled = wts->pred_demand_scaled;
memcpy(__entry->hist, wts->sum_history,
RAVG_HIST_SIZE_MAX * sizeof(u32));
RAVG_HIST_SIZE * sizeof(u32));
__entry->nr_big_tasks = wrq->walt_stats.nr_big_tasks;
__entry->cpu = rq->cpu;
),

16
kernel/sched/walt/walt.c
View File

@ -167,8 +167,6 @@ static inline u64 walt_rq_clock(struct rq *rq)
static unsigned int walt_cpu_high_irqload;
static __read_mostly unsigned int sched_ravg_hist_size = RAVG_HIST_SIZE_MAX;
static __read_mostly unsigned int sched_io_is_busy = 1;
/* Window size (in ns) */
@ -1935,10 +1933,10 @@ static void update_history(struct rq *rq, struct task_struct *p,
/* Push new 'runtime' value onto stack */
for (; samples > 0; samples--) {
hist[wts->cidx] = runtime;
wts->cidx = ++(wts->cidx) % sched_ravg_hist_size;
wts->cidx = ++(wts->cidx) & RAVG_HIST_MASK;
}
for (i = 0; i < sched_ravg_hist_size; i++) {
for (i = 0; i < RAVG_HIST_SIZE; i++) {
sum += hist[i];
if (hist[i] > max)
max = hist[i];
@ -1951,7 +1949,7 @@ static void update_history(struct rq *rq, struct task_struct *p,
} else if (sysctl_sched_window_stats_policy == WINDOW_STATS_MAX) {
demand = max;
} else {
avg = div64_u64(sum, sched_ravg_hist_size);
avg = sum >> RAVG_HIST_SHIFT;
if (sysctl_sched_window_stats_policy == WINDOW_STATS_AVG)
demand = avg;
else
@ -1980,7 +1978,7 @@ static void update_history(struct rq *rq, struct task_struct *p,
wts->demand = demand;
wts->demand_scaled = demand_scaled;
wts->coloc_demand = div64_u64(sum, sched_ravg_hist_size);
wts->coloc_demand = sum >> RAVG_HIST_SHIFT;
wts->pred_demand_scaled = pred_demand_scaled;
if (demand_scaled > sysctl_sched_min_task_util_for_colocation)
@ -2307,7 +2305,7 @@ static void init_new_task_load(struct task_struct *p)
wts->demand_scaled = init_load_windows_scaled;
wts->coloc_demand = init_load_windows;
wts->pred_demand_scaled = 0;
for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i)
for (i = 0; i < RAVG_HIST_SIZE; ++i)
wts->sum_history[i] = init_load_windows;
wts->misfit = false;
wts->rtg_high_prio = false;
@ -2850,7 +2848,7 @@ static void _set_preferred_cluster(struct walt_related_thread_group *grp)
}
if (wts->mark_start < wallclock -
(sched_ravg_window * sched_ravg_hist_size))
(sched_ravg_window * RAVG_HIST_SIZE))
continue;
combined_demand += wts->coloc_demand;
@ -3726,7 +3724,7 @@ void walt_fill_ta_data(struct core_ctl_notif_data *data)
list_for_each_entry(wts, &grp->tasks, grp_list) {
if (wts->mark_start < wallclock -
(sched_ravg_window * sched_ravg_hist_size))
(sched_ravg_window * RAVG_HIST_SIZE))
continue;
total_demand += wts->coloc_demand;