github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-31 03:27:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched: walt: Use pred_demand_scaled for all predictive ops

Browse Source 
Firstly, we change the bucket size to 16. This has two effects: firstly,
it helps improve power by predicting less aggressive midpoint values.
Secondly, it allows us optimize multiplication/division operations by
using bitshifts.

To enable these further overhead optimizations, translate pred_demand
to a 1024 unit value.

Change-Id: Ieb00f5bbce8aedefc5a023ce1e354dbfb0055543
Signed-off-by: Shaleen Agrawal <quic_shalagra@quicinc.com>
This commit is contained in:
Shaleen Agrawal 2022-02-09 13:04:59 -08:00 committed by Rishabh Bhatnagar
parent a8e14cc963
commit b53494ca46
4 changed files with 65 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/linux/sched/walt.h
View File

@ -32,7 +32,9 @@ enum task_boost_type {
#define WALT_NR_CPUS 8
#define RAVG_HIST_SIZE_MAX 8
#define NUM_BUSY_BUCKETS 10
/* wts->bucket_bitmask needs to be updated if NUM_BUSY_BUCKETS > 16 */
#define NUM_BUSY_BUCKETS 16
#define NUM_BUSY_BUCKETS_SHIFT 4
struct walt_related_thread_group {
int id;
@ -73,7 +75,8 @@ struct walt_task_struct {
*
* 'prev_window' represents the sum of all entries in prev_window_cpu
*
* 'pred_demand' represents task's current predicted cpu busy time
* 'pred_demand_scaled' represents task's current predicted cpu busy time
* in terms of 1024 units
*
* 'busy_buckets' groups historical busy time into different buckets
* used for prediction
@ -90,7 +93,6 @@ struct walt_task_struct {
u32 curr_window_cpu[WALT_NR_CPUS];
u32 prev_window_cpu[WALT_NR_CPUS];
u32 curr_window, prev_window;
u32 pred_demand;
u8 busy_buckets[NUM_BUSY_BUCKETS];
u16 bucket_bitmask;
u16 demand_scaled;

55
kernel/sched/walt/trace.h
View File

@ -24,39 +24,47 @@ extern const char *task_event_names[];
TRACE_EVENT(sched_update_pred_demand,
TP_PROTO(struct task_struct *p, u32 runtime, int pct,
unsigned int pred_demand, struct walt_task_struct *wts),
TP_PROTO(struct task_struct *p, u32 runtime,
unsigned int pred_demand_scaled, int start,
int first, int final, struct walt_task_struct *wts),
TP_ARGS(p, runtime, pct, pred_demand, wts),
TP_ARGS(p, runtime, pred_demand_scaled, start, first, final, wts),
TP_STRUCT__entry(
__array(char, comm, TASK_COMM_LEN)
__field(pid_t, pid)
__field(unsigned int, runtime)
__field(int, pct)
__field(unsigned int, pred_demand)
__field(unsigned int, pred_demand_scaled)
__array(u8, bucket, NUM_BUSY_BUCKETS)
__field(int, cpu)
__field(int, start)
__field(int, first)
__field(int, final)
),
TP_fast_assign(
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->runtime = runtime;
__entry->pct = pct;
__entry->pred_demand = pred_demand;
__entry->pred_demand_scaled = pred_demand_scaled;
memcpy(__entry->bucket, wts->busy_buckets,
NUM_BUSY_BUCKETS * sizeof(u8));
__entry->cpu = task_cpu(p);
__entry->start = start;
__entry->first = first;
__entry->final = final;
),
TP_printk("%d (%s): runtime %u pct %d cpu %d pred_demand %u (buckets: %u %u %u %u %u %u %u %u %u %u)",
TP_printk("%d (%s): runtime %u cpu %d pred_demand_scaled %u start %d first %d final %d (buckets: %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u)",
__entry->pid, __entry->comm,
__entry->runtime, __entry->pct, __entry->cpu,
__entry->pred_demand, __entry->bucket[0], __entry->bucket[1],
__entry->runtime, __entry->cpu,
__entry->pred_demand_scaled, __entry->start, __entry->first, __entry->final,
__entry->bucket[0], __entry->bucket[1],
__entry->bucket[2], __entry->bucket[3], __entry->bucket[4],
__entry->bucket[5], __entry->bucket[6], __entry->bucket[7],
__entry->bucket[8], __entry->bucket[9])
__entry->bucket[8], __entry->bucket[9], __entry->bucket[10],
__entry->bucket[11], __entry->bucket[12], __entry->bucket[13],
__entry->bucket[14], __entry->bucket[15])
);
TRACE_EVENT(sched_update_history,
@ -74,7 +82,7 @@ TRACE_EVENT(sched_update_history,
__field(enum task_event, evt)
__field(unsigned int, demand)
__field(unsigned int, coloc_demand)
__field(unsigned int, pred_demand)
__field(unsigned int, pred_demand_scaled)
__array(u32, hist, RAVG_HIST_SIZE_MAX)
__field(unsigned int, nr_big_tasks)
__field(int, cpu)
@ -88,18 +96,18 @@ TRACE_EVENT(sched_update_history,
__entry->evt = evt;
__entry->demand = wts->demand;
__entry->coloc_demand = wts->coloc_demand;
__entry->pred_demand = wts->pred_demand;
__entry->pred_demand_scaled = wts->pred_demand_scaled;
memcpy(__entry->hist, wts->sum_history,
RAVG_HIST_SIZE_MAX * sizeof(u32));
__entry->nr_big_tasks = wrq->walt_stats.nr_big_tasks;
__entry->cpu = rq->cpu;
),
TP_printk("%d (%s): runtime %u samples %d event %s demand %u coloc_demand %u pred_demand %u (hist: %u %u %u %u %u) cpu %d nr_big %u",
TP_printk("%d (%s): runtime %u samples %d event %s demand %u coloc_demand %u pred_demand_scaled %u (hist: %u %u %u %u %u) cpu %d nr_big %u",
__entry->pid, __entry->comm,
__entry->runtime, __entry->samples,
task_event_names[__entry->evt],
__entry->demand, __entry->coloc_demand, __entry->pred_demand,
__entry->demand, __entry->coloc_demand, __entry->pred_demand_scaled,
__entry->hist[0], __entry->hist[1],
__entry->hist[2], __entry->hist[3],
__entry->hist[4], __entry->cpu, __entry->nr_big_tasks)
@ -167,7 +175,7 @@ TRACE_EVENT(sched_update_task_ravg,
__field(unsigned int, coloc_demand)
__field(unsigned int, sum)
__field(int, cpu)
__field(unsigned int, pred_demand)
__field(unsigned int, pred_demand_scaled)
__field(u64, rq_cs)
__field(u64, rq_ps)
__field(u64, grp_cs)
@ -201,7 +209,7 @@ TRACE_EVENT(sched_update_task_ravg,
__entry->coloc_demand = wts->coloc_demand;
__entry->sum = wts->sum;
__entry->irqtime = irqtime;
__entry->pred_demand = wts->pred_demand;
__entry->pred_demand_scaled = wts->pred_demand_scaled;
__entry->rq_cs = wrq->curr_runnable_sum;
__entry->rq_ps = wrq->prev_runnable_sum;
__entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0;
@ -223,13 +231,13 @@ TRACE_EVENT(sched_update_task_ravg,
__entry->prev_top = wrq->prev_top;
),
TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u",
TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand_scaled %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u",
__entry->wallclock, __entry->win_start, __entry->delta,
task_event_names[__entry->evt], __entry->cpu,
__entry->cur_freq, __entry->cur_pid,
__entry->pid, __entry->comm, __entry->mark_start,
__entry->delta_m, __entry->demand, __entry->coloc_demand,
__entry->sum, __entry->irqtime, __entry->pred_demand,
__entry->sum, __entry->irqtime, __entry->pred_demand_scaled,
__entry->rq_cs, __entry->rq_ps, __entry->curr_window,
__window_print(p, __get_dynamic_array(curr_sum), nr_cpu_ids),
__entry->prev_window,
@ -1168,7 +1176,7 @@ TRACE_EVENT(sched_enq_deq_task,
__field(unsigned int, rt_nr_running)
__field(unsigned int, cpus_allowed)
__field(unsigned int, demand)
__field(unsigned int, pred_demand)
__field(unsigned int, pred_demand_scaled)
__field(bool, compat_thread)
__field(bool, mvp)
),
@ -1183,19 +1191,20 @@ TRACE_EVENT(sched_enq_deq_task,
__entry->rt_nr_running = task_rq(p)->rt.rt_nr_running;
__entry->cpus_allowed = cpus_allowed;
__entry->demand = task_load(p);
__entry->pred_demand = task_pl(p);
__entry->pred_demand_scaled =
((struct walt_task_struct *) p->android_vendor_data1)->pred_demand_scaled;
__entry->compat_thread = is_compat_thread(task_thread_info(p));
__entry->mvp = mvp;
),
TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d mvp=%d",
TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand_scaled=%u is_compat_t=%d mvp=%d",
__entry->cpu,
__entry->enqueue ? "enqueue" : "dequeue",
__entry->comm, __entry->pid,
__entry->prio, __entry->nr_running,
__entry->rt_nr_running,
__entry->cpus_allowed, __entry->demand,
__entry->pred_demand,
__entry->pred_demand_scaled,
__entry->compat_thread, __entry->mvp)
);

73
kernel/sched/walt/walt.c
View File

@ -1159,11 +1159,11 @@ static inline void bucket_increase(u8 *buckets, u16 *bucket_bitmask, int idx)
}
}
static inline int busy_to_bucket(u32 normalized_rt)
static inline int busy_to_bucket(u16 normalized_rt)
{
int bidx;
bidx = mult_frac(normalized_rt, NUM_BUSY_BUCKETS, max_task_load());
bidx = normalized_rt >> (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT);
bidx = min(bidx, NUM_BUSY_BUCKETS - 1);
/*
@ -1194,13 +1194,12 @@ static inline int busy_to_bucket(u32 normalized_rt)
* to use for prediction. Once found, it returns the midpoint of that bucket.
*/
static u32 get_pred_busy(struct task_struct *p,
int start, u32 runtime, u16 bucket_bitmask)
int start, u16 runtime_scaled, u16 bucket_bitmask)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
u32 dmin, dmax;
u64 cur_freq_runtime = 0;
u16 dmin, dmax;
int first = NUM_BUSY_BUCKETS, final = NUM_BUSY_BUCKETS;
u32 ret = runtime;
u16 ret = runtime_scaled;
u16 next_mask = bucket_bitmask >> start;
/* skip prediction for new tasks due to lack of history */
@ -1224,43 +1223,31 @@ static u32 get_pred_busy(struct task_struct *p,
dmin = 0;
final = 1;
} else {
dmin = mult_frac(final, max_task_load(), NUM_BUSY_BUCKETS);
dmin = final << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT);
}
dmax = mult_frac(final + 1, max_task_load(), NUM_BUSY_BUCKETS);
dmax = (final + 1) << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT);
/*
* when updating in middle of a window, runtime could be higher
* than all recorded history. Always predict at least runtime.
*/
ret = max(runtime, (dmin + dmax) / 2);
ret = max(runtime_scaled, (u16) (((u32)dmin + dmax) / 2));
out:
trace_sched_update_pred_demand(p, runtime,
mult_frac((unsigned int)cur_freq_runtime, 100,
sched_ravg_window), ret, wts);
trace_sched_update_pred_demand(p, runtime_scaled,
ret, start, first, final, wts);
return ret;
}
static inline u32 calc_pred_demand(struct task_struct *p)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
if (wts->pred_demand >= wts->curr_window)
return wts->pred_demand;
return get_pred_busy(p, busy_to_bucket(wts->curr_window),
wts->curr_window, wts->bucket_bitmask);
}
/*
* predictive demand of a task is calculated at the window roll-over.
* predictive demand of a task was calculated at the last window roll-over.
* if the task current window busy time exceeds the predicted
* demand, update it here to reflect the task needs.
*/
static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int event)
{
u32 new, old;
u16 new_scaled;
u16 new_pred_demand_scaled;
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
u16 curr_window_scaled;
if (is_idle_task(p))
return;
@ -1280,21 +1267,20 @@ static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int ev
return;
}
new = calc_pred_demand(p);
old = wts->pred_demand;
if (old >= new)
curr_window_scaled = scale_demand(wts->curr_window);
if (wts->pred_demand_scaled >= curr_window_scaled)
return;
new_scaled = scale_demand(new);
new_pred_demand_scaled = get_pred_busy(p, busy_to_bucket(curr_window_scaled),
curr_window_scaled, wts->bucket_bitmask);
if (task_on_rq_queued(p) && (!task_has_dl_policy(p) ||
!p->dl.dl_throttled))
fixup_walt_sched_stats_common(rq, p,
wts->demand_scaled,
new_scaled);
new_pred_demand_scaled);
wts->pred_demand = new;
wts->pred_demand_scaled = new_scaled;
wts->pred_demand_scaled = new_pred_demand_scaled;
}
static void clear_top_tasks_bitmap(unsigned long *bitmap)
@ -1801,17 +1787,17 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
new_window, full_window);
}
static inline u32 predict_and_update_buckets(
struct task_struct *p, u32 runtime) {
static inline u16 predict_and_update_buckets(
struct task_struct *p, u16 runtime_scaled) {
int bidx;
u32 pred_demand;
u32 pred_demand_scaled;
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
bidx = busy_to_bucket(runtime);
pred_demand = get_pred_busy(p, bidx, runtime, wts->bucket_bitmask);
bidx = busy_to_bucket(runtime_scaled);
pred_demand_scaled = get_pred_busy(p, bidx, runtime_scaled, wts->bucket_bitmask);
bucket_increase(wts->busy_buckets, &wts->bucket_bitmask, bidx);
return pred_demand;
return pred_demand_scaled;
}
static int
@ -1866,7 +1852,7 @@ static void update_history(struct rq *rq, struct task_struct *p,
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
u32 *hist = &wts->sum_history[0];
int i;
u32 max = 0, avg, demand, pred_demand;
u32 max = 0, avg, demand;
u64 sum = 0;
u16 demand_scaled, pred_demand_scaled;
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
@ -1900,9 +1886,8 @@ static void update_history(struct rq *rq, struct task_struct *p,
else
demand = max(avg, runtime);
}
pred_demand = predict_and_update_buckets(p, runtime);
pred_demand_scaled = predict_and_update_buckets(p, scale_demand(runtime));
demand_scaled = scale_demand(demand);
pred_demand_scaled = scale_demand(pred_demand);
/*
* A throttled deadline sched class task gets dequeued without
@ -1925,7 +1910,6 @@ static void update_history(struct rq *rq, struct task_struct *p,
wts->demand = demand;
wts->demand_scaled = demand_scaled;
wts->coloc_demand = div64_u64(sum, sched_ravg_hist_size);
wts->pred_demand = pred_demand;
wts->pred_demand_scaled = pred_demand_scaled;
if (demand_scaled > sysctl_sched_min_task_util_for_colocation)
@ -2251,7 +2235,6 @@ static void init_new_task_load(struct task_struct *p)
wts->demand = init_load_windows;
wts->demand_scaled = init_load_windows_scaled;
wts->coloc_demand = init_load_windows;
wts->pred_demand = 0;
wts->pred_demand_scaled = 0;
for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i)
wts->sum_history[i] = init_load_windows;

7
kernel/sched/walt/walt.h
View File

@ -788,13 +788,6 @@ static inline unsigned int task_load(struct task_struct *p)
return wts->demand;
}
static inline unsigned int task_pl(struct task_struct *p)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
return wts->pred_demand;
}
static inline bool task_in_related_thread_group(struct task_struct *p)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;