github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-31 03:27:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

sched/walt: Improve the scheduler

Browse Source 
This change is for general scheduler improvements.

Change-Id: I611eb9aa071e4fac1fad962fb98139f9aff79dcd
Signed-off-by: Satya Durga Srinivasu Prabhala <quic_satyap@quicinc.com>
Signed-off-by: Sai Harshini Nimmala <quic_snimmala@quicinc.com>
This commit is contained in:
Satya Durga Srinivasu Prabhala 2022-04-07 23:14:28 -07:00 committed by Rishabh Bhatnagar
parent 52016a65f0
commit ec2d7224f4
4 changed files with 98 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/sched/walt.h
View File

@ -131,6 +131,8 @@ struct walt_task_struct {
u64 total_exec;
int mvp_prio;
int cidx;
int load_boost;
int64_t boosted_task_load;
};
#define wts_to_ts(wts) ({ \

34
kernel/sched/walt/sysctl.c
View File

@ -4,9 +4,11 @@
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#include "walt.h"
#include <trace/hooks/sched.h>
#include "walt.h"
#include "trace.h"
static int neg_three = -3;
static int three = 3;
static int two_hundred_fifty_five = 255;
@ -191,6 +193,7 @@ enum {
PER_TASK_BOOST_PERIOD_MS,
LOW_LATENCY,
PIPELINE,
LOAD_BOOST,
};
static int sched_task_handler(struct ctl_table *table, int write,
@ -247,6 +250,9 @@ static int sched_task_handler(struct ctl_table *table, int write,
pid_and_val[1] = wts->low_latency &
WALT_LOW_LATENCY_PIPELINE;
break;
case LOAD_BOOST:
pid_and_val[1] = wts->load_boost;
break;
default:
ret = -EINVAL;
goto put_task;
@ -259,7 +265,7 @@ static int sched_task_handler(struct ctl_table *table, int write,
if (ret)
goto unlock_mutex;
if (pid_and_val[0] <= 0 || pid_and_val[1] < 0) {
if (pid_and_val[0] <= 0) {
ret = -ENOENT;
goto unlock_mutex;
}
@ -273,6 +279,10 @@ static int sched_task_handler(struct ctl_table *table, int write,
wts = (struct walt_task_struct *) task->android_vendor_data1;
param = (unsigned long)table->data;
val = pid_and_val[1];
if (param != LOAD_BOOST && val < 0) {
ret = -EINVAL;
goto put_task;
}
switch (param) {
case WAKE_UP_IDLE:
wts->wake_up_idle = val;
@ -317,10 +327,23 @@ static int sched_task_handler(struct ctl_table *table, int write,
else
wts->low_latency &= ~WALT_LOW_LATENCY_PIPELINE;
break;
case LOAD_BOOST:
if (pid_and_val[1] < -90 || pid_and_val[1] > 90) {
ret = -EINVAL;
goto put_task;
}
wts->load_boost = val;
if (val)
wts->boosted_task_load = mult_frac((int64_t)1024, (int64_t)val, 100);
else
wts->boosted_task_load = 0;
break;
default:
ret = -EINVAL;
}
trace_sched_task_handler(task, param, val, CALLER_ADDR0, CALLER_ADDR1,
CALLER_ADDR2, CALLER_ADDR3, CALLER_ADDR4, CALLER_ADDR5);
put_task:
put_task_struct(task);
unlock_mutex:
@ -836,6 +859,13 @@ struct ctl_table walt_table[] = {
.mode = 0644,
.proc_handler = sched_task_handler,
},
{
.procname = "task_load_boost",
.data = (int *) LOAD_BOOST,
.maxlen = sizeof(unsigned int) * 2,
.mode = 0644,
.proc_handler = sched_task_handler,
},
{
.procname = "sched_task_read_pid",
.data = &sysctl_task_read_pid,

45
kernel/sched/walt/trace.h
View File

@ -1074,6 +1074,7 @@ TRACE_EVENT(sched_task_util,
__field(int, task_boost)
__field(bool, low_latency)
__field(bool, iowaited)
__field(int, load_boost)
),
TP_fast_assign(
@ -1099,16 +1100,18 @@ TRACE_EVENT(sched_task_util,
__entry->low_latency = walt_low_latency_task(p);
__entry->iowaited =
((struct walt_task_struct *) p->android_vendor_data1)->iowaited;
__entry->load_boost =
((struct walt_task_struct *) p->android_vendor_data1)->load_boost;
),
TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d iowaited=%d",
TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d iowaited=%d load_boost=%d",
__entry->pid, __entry->comm, __entry->util, __entry->prev_cpu,
__entry->candidates, __entry->best_energy_cpu, __entry->sync,
__entry->need_idle, __entry->fastpath, __entry->placement_boost,
__entry->latency, __entry->uclamp_boosted,
__entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu,
__entry->unfilter, __entry->cpus_allowed, __entry->task_boost,
__entry->low_latency, __entry->iowaited)
__entry->low_latency, __entry->iowaited, __entry->load_boost)
);
/*
@ -1386,6 +1389,44 @@ TRACE_EVENT(halt_cpus,
__entry->time, __entry->halt, __entry->success)
);
TRACE_EVENT(sched_task_handler,
TP_PROTO(struct task_struct *p, int param, int val, unsigned long c0,
unsigned long c1, unsigned long c2, unsigned long c3,
unsigned long c4, unsigned long c5),
TP_ARGS(p, param, val, c0, c1, c2, c3, c4, c5),
TP_STRUCT__entry(
__array(char, comm, TASK_COMM_LEN)
__field(pid_t, pid)
__field(int, param)
__field(int, val)
__field(unsigned long, c0)
__field(unsigned long, c1)
__field(unsigned long, c2)
__field(unsigned long, c3)
__field(unsigned long, c4)
__field(unsigned long, c5)
),
TP_fast_assign(
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->param = param;
__entry->val = val;
__entry->c0 = c0;
__entry->c1 = c1;
__entry->c2 = c2;
__entry->c3 = c3;
__entry->c4 = c4;
__entry->c5 = c5;
),
TP_printk("comm=%s pid=%d param=%d val=%d callers=%ps <- %ps <- %ps <- %ps <- %ps <- %ps",
__entry->comm, __entry->pid, __entry->param, __entry->val, __entry->c0,
__entry->c1, __entry->c2, __entry->c3, __entry->c4, __entry->c5)
);
TRACE_EVENT(update_cpu_capacity,
TP_PROTO(int cpu, unsigned long rt_pressure, unsigned long capacity),

35
kernel/sched/walt/walt.c
View File

@ -1510,11 +1510,16 @@ static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p,
#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y)
static inline u64 scale_exec_time(u64 delta, struct rq *rq)
static inline u64 scale_exec_time(u64 delta, struct rq *rq, struct walt_task_struct *wts)
{
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
return (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT;
delta = (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT;
if (wts->load_boost && wts->grp && wts->grp->skip_min)
delta = (delta * (1024 + wts->boosted_task_load) >> 10);
return delta;
}
/* Convert busy time to frequency equivalent
@ -1679,7 +1684,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
delta = wallclock - mark_start;
else
delta = irqtime;
delta = scale_exec_time(delta, rq);
delta = scale_exec_time(delta, rq, wts);
*curr_runnable_sum += delta;
if (new_task)
*nt_curr_runnable_sum += delta;
@ -1721,7 +1726,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
* A full window hasn't elapsed, account partial
* contribution to previous completed window.
*/
delta = scale_exec_time(window_start - mark_start, rq);
delta = scale_exec_time(window_start - mark_start, rq, wts);
wts->prev_window += delta;
wts->prev_window_cpu[cpu] += delta;
} else {
@ -1730,7 +1735,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
* the contribution to the previous window is the
* full window (window_size).
*/
delta = scale_exec_time(window_size, rq);
delta = scale_exec_time(window_size, rq, wts);
wts->prev_window = delta;
wts->prev_window_cpu[cpu] = delta;
}
@ -1740,7 +1745,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
*nt_prev_runnable_sum += delta;
/* Account piece of busy time in the current window. */
delta = scale_exec_time(wallclock - window_start, rq);
delta = scale_exec_time(wallclock - window_start, rq, wts);
*curr_runnable_sum += delta;
if (new_task)
*nt_curr_runnable_sum += delta;
@ -1770,7 +1775,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
* A full window hasn't elapsed, account partial
* contribution to previous completed window.
*/
delta = scale_exec_time(window_start - mark_start, rq);
delta = scale_exec_time(window_start - mark_start, rq, wts);
if (!is_idle_task(p)) {
wts->prev_window += delta;
wts->prev_window_cpu[cpu] += delta;
@ -1781,7 +1786,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
* the contribution to the previous window is the
* full window (window_size).
*/
delta = scale_exec_time(window_size, rq);
delta = scale_exec_time(window_size, rq, wts);
if (!is_idle_task(p)) {
wts->prev_window = delta;
wts->prev_window_cpu[cpu] = delta;
@ -1793,7 +1798,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
*nt_prev_runnable_sum += delta;
/* Account piece of busy time in the current window. */
delta = scale_exec_time(wallclock - window_start, rq);
delta = scale_exec_time(wallclock - window_start, rq, wts);
*curr_runnable_sum += delta;
if (new_task)
*nt_curr_runnable_sum += delta;
@ -1828,7 +1833,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
* window then that is all that need be accounted.
*/
if (mark_start > window_start) {
*curr_runnable_sum += scale_exec_time(irqtime, rq);
*curr_runnable_sum += scale_exec_time(irqtime, rq, wts);
return;
}
@ -1839,12 +1844,12 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
delta = window_start - mark_start;
if (delta > window_size)
delta = window_size;
delta = scale_exec_time(delta, rq);
delta = scale_exec_time(delta, rq, wts);
*prev_runnable_sum += delta;
/* Process the remaining IRQ busy time in the current window. */
delta = wallclock - window_start;
wrq->curr_runnable_sum += scale_exec_time(delta, rq);
wrq->curr_runnable_sum += scale_exec_time(delta, rq, wts);
return;
}
@ -1995,7 +2000,7 @@ static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta)
{
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
delta = scale_exec_time(delta, rq);
delta = scale_exec_time(delta, rq, wts);
wts->sum += delta;
if (unlikely(wts->sum > sched_ravg_window))
wts->sum = sched_ravg_window;
@ -2101,7 +2106,7 @@ static u64 update_task_demand(struct task_struct *p, struct rq *rq,
/* Push new sample(s) into task's demand history */
update_history(rq, p, wts->sum, 1, event);
if (nr_full_windows) {
u64 scaled_window = scale_exec_time(window_size, rq);
u64 scaled_window = scale_exec_time(window_size, rq, wts);
update_history(rq, p, scaled_window, nr_full_windows, event);
runtime += nr_full_windows * scaled_window;
@ -2262,6 +2267,8 @@ static inline void __sched_fork_init(struct task_struct *p)
wts->boost_period = false;
wts->low_latency = false;
wts->iowaited = false;
wts->load_boost = 0;
wts->boosted_task_load = 0;
}
static void init_new_task_load(struct task_struct *p)