diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 15860b721795..08777104a28a 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -32,7 +32,9 @@ enum task_boost_type { #define WALT_NR_CPUS 8 #define RAVG_HIST_SIZE_MAX 8 -#define NUM_BUSY_BUCKETS 10 +/* wts->bucket_bitmask needs to be updated if NUM_BUSY_BUCKETS > 16 */ +#define NUM_BUSY_BUCKETS 16 +#define NUM_BUSY_BUCKETS_SHIFT 4 struct walt_related_thread_group { int id; @@ -73,7 +75,8 @@ struct walt_task_struct { * * 'prev_window' represents the sum of all entries in prev_window_cpu * - * 'pred_demand' represents task's current predicted cpu busy time + * 'pred_demand_scaled' represents task's current predicted cpu busy time + * in terms of 1024 units * * 'busy_buckets' groups historical busy time into different buckets * used for prediction @@ -90,7 +93,6 @@ struct walt_task_struct { u32 curr_window_cpu[WALT_NR_CPUS]; u32 prev_window_cpu[WALT_NR_CPUS]; u32 curr_window, prev_window; - u32 pred_demand; u8 busy_buckets[NUM_BUSY_BUCKETS]; u16 bucket_bitmask; u16 demand_scaled; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 925529525709..26c38c06214c 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -24,39 +24,47 @@ extern const char *task_event_names[]; TRACE_EVENT(sched_update_pred_demand, - TP_PROTO(struct task_struct *p, u32 runtime, int pct, - unsigned int pred_demand, struct walt_task_struct *wts), + TP_PROTO(struct task_struct *p, u32 runtime, + unsigned int pred_demand_scaled, int start, + int first, int final, struct walt_task_struct *wts), - TP_ARGS(p, runtime, pct, pred_demand, wts), + TP_ARGS(p, runtime, pred_demand_scaled, start, first, final, wts), TP_STRUCT__entry( __array(char, comm, TASK_COMM_LEN) __field(pid_t, pid) __field(unsigned int, runtime) - __field(int, pct) - __field(unsigned int, pred_demand) + __field(unsigned int, pred_demand_scaled) __array(u8, bucket, NUM_BUSY_BUCKETS) __field(int, cpu) + __field(int, start) + __field(int, first) + __field(int, final) ), TP_fast_assign( memcpy(__entry->comm, p->comm, TASK_COMM_LEN); __entry->pid = p->pid; __entry->runtime = runtime; - __entry->pct = pct; - __entry->pred_demand = pred_demand; + __entry->pred_demand_scaled = pred_demand_scaled; memcpy(__entry->bucket, wts->busy_buckets, NUM_BUSY_BUCKETS * sizeof(u8)); __entry->cpu = task_cpu(p); + __entry->start = start; + __entry->first = first; + __entry->final = final; ), - TP_printk("%d (%s): runtime %u pct %d cpu %d pred_demand %u (buckets: %u %u %u %u %u %u %u %u %u %u)", + TP_printk("%d (%s): runtime %u cpu %d pred_demand_scaled %u start %d first %d final %d (buckets: %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u)", __entry->pid, __entry->comm, - __entry->runtime, __entry->pct, __entry->cpu, - __entry->pred_demand, __entry->bucket[0], __entry->bucket[1], + __entry->runtime, __entry->cpu, + __entry->pred_demand_scaled, __entry->start, __entry->first, __entry->final, + __entry->bucket[0], __entry->bucket[1], __entry->bucket[2], __entry->bucket[3], __entry->bucket[4], __entry->bucket[5], __entry->bucket[6], __entry->bucket[7], - __entry->bucket[8], __entry->bucket[9]) + __entry->bucket[8], __entry->bucket[9], __entry->bucket[10], + __entry->bucket[11], __entry->bucket[12], __entry->bucket[13], + __entry->bucket[14], __entry->bucket[15]) ); TRACE_EVENT(sched_update_history, @@ -74,7 +82,7 @@ TRACE_EVENT(sched_update_history, __field(enum task_event, evt) __field(unsigned int, demand) __field(unsigned int, coloc_demand) - __field(unsigned int, pred_demand) + __field(unsigned int, pred_demand_scaled) __array(u32, hist, RAVG_HIST_SIZE_MAX) __field(unsigned int, nr_big_tasks) __field(int, cpu) @@ -88,18 +96,18 @@ TRACE_EVENT(sched_update_history, __entry->evt = evt; __entry->demand = wts->demand; __entry->coloc_demand = wts->coloc_demand; - __entry->pred_demand = wts->pred_demand; + __entry->pred_demand_scaled = wts->pred_demand_scaled; memcpy(__entry->hist, wts->sum_history, RAVG_HIST_SIZE_MAX * sizeof(u32)); __entry->nr_big_tasks = wrq->walt_stats.nr_big_tasks; __entry->cpu = rq->cpu; ), - TP_printk("%d (%s): runtime %u samples %d event %s demand %u coloc_demand %u pred_demand %u (hist: %u %u %u %u %u) cpu %d nr_big %u", + TP_printk("%d (%s): runtime %u samples %d event %s demand %u coloc_demand %u pred_demand_scaled %u (hist: %u %u %u %u %u) cpu %d nr_big %u", __entry->pid, __entry->comm, __entry->runtime, __entry->samples, task_event_names[__entry->evt], - __entry->demand, __entry->coloc_demand, __entry->pred_demand, + __entry->demand, __entry->coloc_demand, __entry->pred_demand_scaled, __entry->hist[0], __entry->hist[1], __entry->hist[2], __entry->hist[3], __entry->hist[4], __entry->cpu, __entry->nr_big_tasks) @@ -167,7 +175,7 @@ TRACE_EVENT(sched_update_task_ravg, __field(unsigned int, coloc_demand) __field(unsigned int, sum) __field(int, cpu) - __field(unsigned int, pred_demand) + __field(unsigned int, pred_demand_scaled) __field(u64, rq_cs) __field(u64, rq_ps) __field(u64, grp_cs) @@ -201,7 +209,7 @@ TRACE_EVENT(sched_update_task_ravg, __entry->coloc_demand = wts->coloc_demand; __entry->sum = wts->sum; __entry->irqtime = irqtime; - __entry->pred_demand = wts->pred_demand; + __entry->pred_demand_scaled = wts->pred_demand_scaled; __entry->rq_cs = wrq->curr_runnable_sum; __entry->rq_ps = wrq->prev_runnable_sum; __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; @@ -223,13 +231,13 @@ TRACE_EVENT(sched_update_task_ravg, __entry->prev_top = wrq->prev_top; ), - TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u", + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand_scaled %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u", __entry->wallclock, __entry->win_start, __entry->delta, task_event_names[__entry->evt], __entry->cpu, __entry->cur_freq, __entry->cur_pid, __entry->pid, __entry->comm, __entry->mark_start, __entry->delta_m, __entry->demand, __entry->coloc_demand, - __entry->sum, __entry->irqtime, __entry->pred_demand, + __entry->sum, __entry->irqtime, __entry->pred_demand_scaled, __entry->rq_cs, __entry->rq_ps, __entry->curr_window, __window_print(p, __get_dynamic_array(curr_sum), nr_cpu_ids), __entry->prev_window, @@ -1168,7 +1176,7 @@ TRACE_EVENT(sched_enq_deq_task, __field(unsigned int, rt_nr_running) __field(unsigned int, cpus_allowed) __field(unsigned int, demand) - __field(unsigned int, pred_demand) + __field(unsigned int, pred_demand_scaled) __field(bool, compat_thread) __field(bool, mvp) ), @@ -1183,19 +1191,20 @@ TRACE_EVENT(sched_enq_deq_task, __entry->rt_nr_running = task_rq(p)->rt.rt_nr_running; __entry->cpus_allowed = cpus_allowed; __entry->demand = task_load(p); - __entry->pred_demand = task_pl(p); + __entry->pred_demand_scaled = + ((struct walt_task_struct *) p->android_vendor_data1)->pred_demand_scaled; __entry->compat_thread = is_compat_thread(task_thread_info(p)); __entry->mvp = mvp; ), - TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d mvp=%d", + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand_scaled=%u is_compat_t=%d mvp=%d", __entry->cpu, __entry->enqueue ? "enqueue" : "dequeue", __entry->comm, __entry->pid, __entry->prio, __entry->nr_running, __entry->rt_nr_running, __entry->cpus_allowed, __entry->demand, - __entry->pred_demand, + __entry->pred_demand_scaled, __entry->compat_thread, __entry->mvp) ); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 58f8efc96cf7..ba4509306230 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1159,11 +1159,11 @@ static inline void bucket_increase(u8 *buckets, u16 *bucket_bitmask, int idx) } } -static inline int busy_to_bucket(u32 normalized_rt) +static inline int busy_to_bucket(u16 normalized_rt) { int bidx; - bidx = mult_frac(normalized_rt, NUM_BUSY_BUCKETS, max_task_load()); + bidx = normalized_rt >> (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); bidx = min(bidx, NUM_BUSY_BUCKETS - 1); /* @@ -1194,13 +1194,12 @@ static inline int busy_to_bucket(u32 normalized_rt) * to use for prediction. Once found, it returns the midpoint of that bucket. */ static u32 get_pred_busy(struct task_struct *p, - int start, u32 runtime, u16 bucket_bitmask) + int start, u16 runtime_scaled, u16 bucket_bitmask) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - u32 dmin, dmax; - u64 cur_freq_runtime = 0; + u16 dmin, dmax; int first = NUM_BUSY_BUCKETS, final = NUM_BUSY_BUCKETS; - u32 ret = runtime; + u16 ret = runtime_scaled; u16 next_mask = bucket_bitmask >> start; /* skip prediction for new tasks due to lack of history */ @@ -1224,43 +1223,31 @@ static u32 get_pred_busy(struct task_struct *p, dmin = 0; final = 1; } else { - dmin = mult_frac(final, max_task_load(), NUM_BUSY_BUCKETS); + dmin = final << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); } - dmax = mult_frac(final + 1, max_task_load(), NUM_BUSY_BUCKETS); + dmax = (final + 1) << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); /* * when updating in middle of a window, runtime could be higher * than all recorded history. Always predict at least runtime. */ - ret = max(runtime, (dmin + dmax) / 2); + ret = max(runtime_scaled, (u16) (((u32)dmin + dmax) / 2)); out: - trace_sched_update_pred_demand(p, runtime, - mult_frac((unsigned int)cur_freq_runtime, 100, - sched_ravg_window), ret, wts); + trace_sched_update_pred_demand(p, runtime_scaled, + ret, start, first, final, wts); return ret; } -static inline u32 calc_pred_demand(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - if (wts->pred_demand >= wts->curr_window) - return wts->pred_demand; - - return get_pred_busy(p, busy_to_bucket(wts->curr_window), - wts->curr_window, wts->bucket_bitmask); -} - /* - * predictive demand of a task is calculated at the window roll-over. + * predictive demand of a task was calculated at the last window roll-over. * if the task current window busy time exceeds the predicted * demand, update it here to reflect the task needs. */ static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int event) { - u32 new, old; - u16 new_scaled; + u16 new_pred_demand_scaled; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u16 curr_window_scaled; if (is_idle_task(p)) return; @@ -1280,21 +1267,20 @@ static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int ev return; } - new = calc_pred_demand(p); - old = wts->pred_demand; - - if (old >= new) + curr_window_scaled = scale_demand(wts->curr_window); + if (wts->pred_demand_scaled >= curr_window_scaled) return; - new_scaled = scale_demand(new); + new_pred_demand_scaled = get_pred_busy(p, busy_to_bucket(curr_window_scaled), + curr_window_scaled, wts->bucket_bitmask); + if (task_on_rq_queued(p) && (!task_has_dl_policy(p) || !p->dl.dl_throttled)) fixup_walt_sched_stats_common(rq, p, wts->demand_scaled, - new_scaled); + new_pred_demand_scaled); - wts->pred_demand = new; - wts->pred_demand_scaled = new_scaled; + wts->pred_demand_scaled = new_pred_demand_scaled; } static void clear_top_tasks_bitmap(unsigned long *bitmap) @@ -1801,17 +1787,17 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, new_window, full_window); } -static inline u32 predict_and_update_buckets( - struct task_struct *p, u32 runtime) { +static inline u16 predict_and_update_buckets( + struct task_struct *p, u16 runtime_scaled) { int bidx; - u32 pred_demand; + u32 pred_demand_scaled; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - bidx = busy_to_bucket(runtime); - pred_demand = get_pred_busy(p, bidx, runtime, wts->bucket_bitmask); + bidx = busy_to_bucket(runtime_scaled); + pred_demand_scaled = get_pred_busy(p, bidx, runtime_scaled, wts->bucket_bitmask); bucket_increase(wts->busy_buckets, &wts->bucket_bitmask, bidx); - return pred_demand; + return pred_demand_scaled; } static int @@ -1866,7 +1852,7 @@ static void update_history(struct rq *rq, struct task_struct *p, struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u32 *hist = &wts->sum_history[0]; int i; - u32 max = 0, avg, demand, pred_demand; + u32 max = 0, avg, demand; u64 sum = 0; u16 demand_scaled, pred_demand_scaled; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; @@ -1900,9 +1886,8 @@ static void update_history(struct rq *rq, struct task_struct *p, else demand = max(avg, runtime); } - pred_demand = predict_and_update_buckets(p, runtime); + pred_demand_scaled = predict_and_update_buckets(p, scale_demand(runtime)); demand_scaled = scale_demand(demand); - pred_demand_scaled = scale_demand(pred_demand); /* * A throttled deadline sched class task gets dequeued without @@ -1925,7 +1910,6 @@ static void update_history(struct rq *rq, struct task_struct *p, wts->demand = demand; wts->demand_scaled = demand_scaled; wts->coloc_demand = div64_u64(sum, sched_ravg_hist_size); - wts->pred_demand = pred_demand; wts->pred_demand_scaled = pred_demand_scaled; if (demand_scaled > sysctl_sched_min_task_util_for_colocation) @@ -2251,7 +2235,6 @@ static void init_new_task_load(struct task_struct *p) wts->demand = init_load_windows; wts->demand_scaled = init_load_windows_scaled; wts->coloc_demand = init_load_windows; - wts->pred_demand = 0; wts->pred_demand_scaled = 0; for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) wts->sum_history[i] = init_load_windows; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 8f059fe52197..5a973f5b12c7 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -788,13 +788,6 @@ static inline unsigned int task_load(struct task_struct *p) return wts->demand; } -static inline unsigned int task_pl(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return wts->pred_demand; -} - static inline bool task_in_related_thread_group(struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;