diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 853c7c169795..0bf24642e528 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -131,6 +131,8 @@ struct walt_task_struct { u64 total_exec; int mvp_prio; int cidx; + int load_boost; + int64_t boosted_task_load; }; #define wts_to_ts(wts) ({ \ diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 7a59a743cdc2..1be0123ebcbe 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -4,9 +4,11 @@ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ -#include "walt.h" #include +#include "walt.h" +#include "trace.h" + static int neg_three = -3; static int three = 3; static int two_hundred_fifty_five = 255; @@ -191,6 +193,7 @@ enum { PER_TASK_BOOST_PERIOD_MS, LOW_LATENCY, PIPELINE, + LOAD_BOOST, }; static int sched_task_handler(struct ctl_table *table, int write, @@ -247,6 +250,9 @@ static int sched_task_handler(struct ctl_table *table, int write, pid_and_val[1] = wts->low_latency & WALT_LOW_LATENCY_PIPELINE; break; + case LOAD_BOOST: + pid_and_val[1] = wts->load_boost; + break; default: ret = -EINVAL; goto put_task; @@ -259,7 +265,7 @@ static int sched_task_handler(struct ctl_table *table, int write, if (ret) goto unlock_mutex; - if (pid_and_val[0] <= 0 || pid_and_val[1] < 0) { + if (pid_and_val[0] <= 0) { ret = -ENOENT; goto unlock_mutex; } @@ -273,6 +279,10 @@ static int sched_task_handler(struct ctl_table *table, int write, wts = (struct walt_task_struct *) task->android_vendor_data1; param = (unsigned long)table->data; val = pid_and_val[1]; + if (param != LOAD_BOOST && val < 0) { + ret = -EINVAL; + goto put_task; + } switch (param) { case WAKE_UP_IDLE: wts->wake_up_idle = val; @@ -317,10 +327,23 @@ static int sched_task_handler(struct ctl_table *table, int write, else wts->low_latency &= ~WALT_LOW_LATENCY_PIPELINE; break; + case LOAD_BOOST: + if (pid_and_val[1] < -90 || pid_and_val[1] > 90) { + ret = -EINVAL; + goto put_task; + } + wts->load_boost = val; + if (val) + wts->boosted_task_load = mult_frac((int64_t)1024, (int64_t)val, 100); + else + wts->boosted_task_load = 0; + break; default: ret = -EINVAL; } + trace_sched_task_handler(task, param, val, CALLER_ADDR0, CALLER_ADDR1, + CALLER_ADDR2, CALLER_ADDR3, CALLER_ADDR4, CALLER_ADDR5); put_task: put_task_struct(task); unlock_mutex: @@ -836,6 +859,13 @@ struct ctl_table walt_table[] = { .mode = 0644, .proc_handler = sched_task_handler, }, + { + .procname = "task_load_boost", + .data = (int *) LOAD_BOOST, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, { .procname = "sched_task_read_pid", .data = &sysctl_task_read_pid, diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 4b48c88a5cd2..f780ea7cc85b 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1074,6 +1074,7 @@ TRACE_EVENT(sched_task_util, __field(int, task_boost) __field(bool, low_latency) __field(bool, iowaited) + __field(int, load_boost) ), TP_fast_assign( @@ -1099,16 +1100,18 @@ TRACE_EVENT(sched_task_util, __entry->low_latency = walt_low_latency_task(p); __entry->iowaited = ((struct walt_task_struct *) p->android_vendor_data1)->iowaited; + __entry->load_boost = + ((struct walt_task_struct *) p->android_vendor_data1)->load_boost; ), - TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d iowaited=%d", + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d iowaited=%d load_boost=%d", __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, __entry->candidates, __entry->best_energy_cpu, __entry->sync, __entry->need_idle, __entry->fastpath, __entry->placement_boost, __entry->latency, __entry->uclamp_boosted, __entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu, __entry->unfilter, __entry->cpus_allowed, __entry->task_boost, - __entry->low_latency, __entry->iowaited) + __entry->low_latency, __entry->iowaited, __entry->load_boost) ); /* @@ -1386,6 +1389,44 @@ TRACE_EVENT(halt_cpus, __entry->time, __entry->halt, __entry->success) ); +TRACE_EVENT(sched_task_handler, + TP_PROTO(struct task_struct *p, int param, int val, unsigned long c0, + unsigned long c1, unsigned long c2, unsigned long c3, + unsigned long c4, unsigned long c5), + + TP_ARGS(p, param, val, c0, c1, c2, c3, c4, c5), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, param) + __field(int, val) + __field(unsigned long, c0) + __field(unsigned long, c1) + __field(unsigned long, c2) + __field(unsigned long, c3) + __field(unsigned long, c4) + __field(unsigned long, c5) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->param = param; + __entry->val = val; + __entry->c0 = c0; + __entry->c1 = c1; + __entry->c2 = c2; + __entry->c3 = c3; + __entry->c4 = c4; + __entry->c5 = c5; + ), + + TP_printk("comm=%s pid=%d param=%d val=%d callers=%ps <- %ps <- %ps <- %ps <- %ps <- %ps", + __entry->comm, __entry->pid, __entry->param, __entry->val, __entry->c0, + __entry->c1, __entry->c2, __entry->c3, __entry->c4, __entry->c5) +); + TRACE_EVENT(update_cpu_capacity, TP_PROTO(int cpu, unsigned long rt_pressure, unsigned long capacity), diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c0a58e84e3f5..ea287de0c3fc 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1510,11 +1510,16 @@ static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, #define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -static inline u64 scale_exec_time(u64 delta, struct rq *rq) +static inline u64 scale_exec_time(u64 delta, struct rq *rq, struct walt_task_struct *wts) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - return (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT; + delta = (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT; + + if (wts->load_boost && wts->grp && wts->grp->skip_min) + delta = (delta * (1024 + wts->boosted_task_load) >> 10); + + return delta; } /* Convert busy time to frequency equivalent @@ -1679,7 +1684,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, delta = wallclock - mark_start; else delta = irqtime; - delta = scale_exec_time(delta, rq); + delta = scale_exec_time(delta, rq, wts); *curr_runnable_sum += delta; if (new_task) *nt_curr_runnable_sum += delta; @@ -1721,7 +1726,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * A full window hasn't elapsed, account partial * contribution to previous completed window. */ - delta = scale_exec_time(window_start - mark_start, rq); + delta = scale_exec_time(window_start - mark_start, rq, wts); wts->prev_window += delta; wts->prev_window_cpu[cpu] += delta; } else { @@ -1730,7 +1735,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * the contribution to the previous window is the * full window (window_size). */ - delta = scale_exec_time(window_size, rq); + delta = scale_exec_time(window_size, rq, wts); wts->prev_window = delta; wts->prev_window_cpu[cpu] = delta; } @@ -1740,7 +1745,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, *nt_prev_runnable_sum += delta; /* Account piece of busy time in the current window. */ - delta = scale_exec_time(wallclock - window_start, rq); + delta = scale_exec_time(wallclock - window_start, rq, wts); *curr_runnable_sum += delta; if (new_task) *nt_curr_runnable_sum += delta; @@ -1770,7 +1775,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * A full window hasn't elapsed, account partial * contribution to previous completed window. */ - delta = scale_exec_time(window_start - mark_start, rq); + delta = scale_exec_time(window_start - mark_start, rq, wts); if (!is_idle_task(p)) { wts->prev_window += delta; wts->prev_window_cpu[cpu] += delta; @@ -1781,7 +1786,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * the contribution to the previous window is the * full window (window_size). */ - delta = scale_exec_time(window_size, rq); + delta = scale_exec_time(window_size, rq, wts); if (!is_idle_task(p)) { wts->prev_window = delta; wts->prev_window_cpu[cpu] = delta; @@ -1793,7 +1798,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, *nt_prev_runnable_sum += delta; /* Account piece of busy time in the current window. */ - delta = scale_exec_time(wallclock - window_start, rq); + delta = scale_exec_time(wallclock - window_start, rq, wts); *curr_runnable_sum += delta; if (new_task) *nt_curr_runnable_sum += delta; @@ -1828,7 +1833,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * window then that is all that need be accounted. */ if (mark_start > window_start) { - *curr_runnable_sum += scale_exec_time(irqtime, rq); + *curr_runnable_sum += scale_exec_time(irqtime, rq, wts); return; } @@ -1839,12 +1844,12 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, delta = window_start - mark_start; if (delta > window_size) delta = window_size; - delta = scale_exec_time(delta, rq); + delta = scale_exec_time(delta, rq, wts); *prev_runnable_sum += delta; /* Process the remaining IRQ busy time in the current window. */ delta = wallclock - window_start; - wrq->curr_runnable_sum += scale_exec_time(delta, rq); + wrq->curr_runnable_sum += scale_exec_time(delta, rq, wts); return; } @@ -1995,7 +2000,7 @@ static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - delta = scale_exec_time(delta, rq); + delta = scale_exec_time(delta, rq, wts); wts->sum += delta; if (unlikely(wts->sum > sched_ravg_window)) wts->sum = sched_ravg_window; @@ -2101,7 +2106,7 @@ static u64 update_task_demand(struct task_struct *p, struct rq *rq, /* Push new sample(s) into task's demand history */ update_history(rq, p, wts->sum, 1, event); if (nr_full_windows) { - u64 scaled_window = scale_exec_time(window_size, rq); + u64 scaled_window = scale_exec_time(window_size, rq, wts); update_history(rq, p, scaled_window, nr_full_windows, event); runtime += nr_full_windows * scaled_window; @@ -2262,6 +2267,8 @@ static inline void __sched_fork_init(struct task_struct *p) wts->boost_period = false; wts->low_latency = false; wts->iowaited = false; + wts->load_boost = 0; + wts->boosted_task_load = 0; } static void init_new_task_load(struct task_struct *p)