From f8c92acbe701d01557dfe8e808decaf91e3991d4 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 27 Oct 2022 15:07:29 -0700 Subject: [PATCH] ANDROID: Sched: Add vendor hooks required for WALT Adding vendor hooks in several areas of the scheduler code. These are needed for implementing Qualcomm's load tracking scheme WALT via vendor modules. WALT was originally submitted upstream, but due to complexities it was decided that a better path forward would to be improve on existing PELT. Link to upstream submission of WALT can be found below. This reverts commit 7e2c96035407 ("Revert "ANDROID: Sched: Add restricted vendor hooks for scheduler""). This change effectively brings back the original change. Bug: 200103201 Bug: 243793188 Link: https://lore.kernel.org/lkml/1477638642-17428-2-git-send-email-markivx@codeaurora.org/ Signed-off-by: Ashay Jaiswal Signed-off-by: Sai Harshini Nimmala Change-Id: I03327dc00d35e6fabcf8b5e254c4da548943d1a7 --- include/trace/hooks/sched.h | 65 +++++++++++++++++++++++++++++++++++++ kernel/sched/core.c | 19 ++++++++++- kernel/sched/cputime.c | 11 +++++-- kernel/sched/fair.c | 7 +++- kernel/sched/topology.c | 1 + kernel/sched/vendor_hooks.c | 16 +++++++++ 6 files changed, 115 insertions(+), 4 deletions(-) diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index bc072cb3dc28..e88a567709ca 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -128,6 +128,71 @@ DECLARE_RESTRICTED_HOOK(android_rvh_sched_getaffinity, TP_PROTO(struct task_struct *p, struct cpumask *in_mask), TP_ARGS(p, in_mask), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_set_task_cpu, + TP_PROTO(struct task_struct *p, unsigned int new_cpu), + TP_ARGS(p, new_cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_try_to_wake_up, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_try_to_wake_up_success, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_fork, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_wake_up_new_task, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_new_task_stats, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_flush_task, + TP_PROTO(struct task_struct *prev), + TP_ARGS(prev), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_tick_entry, + TP_PROTO(struct rq *rq), + TP_ARGS(rq), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_schedule, + TP_PROTO(struct task_struct *prev, struct task_struct *next, struct rq *rq), + TP_ARGS(prev, next, rq), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_cpu_starting, + TP_PROTO(int cpu), + TP_ARGS(cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_cpu_dying, + TP_PROTO(int cpu), + TP_ARGS(cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_account_irq, + TP_PROTO(struct task_struct *curr, int cpu, s64 delta, bool start), + TP_ARGS(curr, cpu, delta, start), 1); + +struct sched_entity; +DECLARE_RESTRICTED_HOOK(android_rvh_place_entity, + TP_PROTO(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial, u64 *vruntime), + TP_ARGS(cfs_rq, se, initial, vruntime), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_build_perf_domains, + TP_PROTO(bool *eas_check), + TP_ARGS(eas_check), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_update_cpu_capacity, + TP_PROTO(int cpu, unsigned long *capacity), + TP_ARGS(cpu, capacity), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_update_misfit_status, + TP_PROTO(struct task_struct *p, struct rq *rq, bool *need_update), + TP_ARGS(p, rq, need_update), 1); + DECLARE_RESTRICTED_HOOK(android_rvh_sched_fork_init, TP_PROTO(struct task_struct *p), TP_ARGS(p), 1); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index e62200c7edaa..50f4a490bd68 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3184,6 +3184,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) p->se.nr_migrations++; rseq_migrate(p); perf_event_task_migrate(p); + trace_android_rvh_set_task_cpu(p, new_cpu); } __set_task_cpu(p, new_cpu); @@ -4252,6 +4253,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_cond_load_acquire(&p->on_cpu, !VAL); + trace_android_rvh_try_to_wake_up(p); + cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); if (task_cpu(p) != cpu) { if (p->in_iowait) { @@ -4271,8 +4274,10 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) unlock: raw_spin_unlock_irqrestore(&p->pi_lock, flags); out: - if (success) + if (success) { + trace_android_rvh_try_to_wake_up_success(p); ttwu_stat(p, task_cpu(p), wake_flags); + } preempt_enable(); return success; @@ -4623,6 +4628,8 @@ late_initcall(sched_core_sysctl_init); */ int sched_fork(unsigned long clone_flags, struct task_struct *p) { + trace_android_rvh_sched_fork(p); + __sched_fork(clone_flags, p); /* * We mark the process as NEW here. This guarantees that @@ -4748,6 +4755,8 @@ void wake_up_new_task(struct task_struct *p) struct rq_flags rf; struct rq *rq; + trace_android_rvh_wake_up_new_task(p); + raw_spin_lock_irqsave(&p->pi_lock, rf.flags); WRITE_ONCE(p->__state, TASK_RUNNING); #ifdef CONFIG_SMP @@ -4766,6 +4775,7 @@ void wake_up_new_task(struct task_struct *p) rq = __task_rq_lock(p, &rf); update_rq_clock(rq); post_init_entity_util_avg(p); + trace_android_rvh_new_task_stats(p); activate_task(rq, p, ENQUEUE_NOCLOCK); trace_sched_wakeup_new(p); @@ -5182,6 +5192,8 @@ static struct rq *finish_task_switch(struct task_struct *prev) if (prev->sched_class->task_dead) prev->sched_class->task_dead(prev); + trace_android_rvh_flush_task(prev); + /* Task is done with its stack. */ put_task_stack(prev); @@ -5546,6 +5558,7 @@ void scheduler_tick(void) rq_lock(rq, &rf); update_rq_clock(rq); + trace_android_rvh_tick_entry(rq); thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); @@ -6565,6 +6578,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) rq->last_seen_need_resched_ns = 0; #endif + trace_android_rvh_schedule(prev, next, rq); if (likely(prev != next)) { rq->nr_switches++; /* @@ -9580,6 +9594,7 @@ int sched_cpu_starting(unsigned int cpu) sched_core_cpu_starting(cpu); sched_rq_cpu_starting(cpu); sched_tick_start(cpu); + trace_android_rvh_sched_cpu_starting(cpu); return 0; } @@ -9653,6 +9668,8 @@ int sched_cpu_dying(unsigned int cpu) } rq_unlock_irqrestore(rq, &rf); + trace_android_rvh_sched_cpu_dying(cpu); + calc_load_migrate(rq); update_max_interval(); hrtick_clear(rq); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 3e0f79fee72e..a485f3b0201c 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -3,6 +3,7 @@ * Simple CPU accounting cgroup controller */ #include +#include #ifdef CONFIG_IRQ_TIME_ACCOUNTING @@ -54,6 +55,7 @@ void irqtime_account_irq(struct task_struct *curr, unsigned int offset) unsigned int pc; s64 delta; int cpu; + bool irq_start = true; if (!sched_clock_irqtime) return; @@ -69,10 +71,15 @@ void irqtime_account_irq(struct task_struct *curr, unsigned int offset) * in that case, so as not to confuse scheduler with a special task * that do not consume any time, but still wants to run. */ - if (pc & HARDIRQ_MASK) + if (pc & HARDIRQ_MASK) { irqtime_account_delta(irqtime, delta, CPUTIME_IRQ); - else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd()) + irq_start = false; + } else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd()) { irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ); + irq_start = false; + } + + trace_android_rvh_account_irq(curr, cpu, delta, irq_start); } static u64 irqtime_tick_accounted(u64 maxtime) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2396e2a33d19..e60a414c31cf 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4444,7 +4444,10 @@ static inline int task_fits_capacity(struct task_struct *p, static inline void update_misfit_status(struct task_struct *p, struct rq *rq) { - if (!sched_asym_cpucap_active()) + bool need_update = true; + + trace_android_rvh_update_misfit_status(p, rq, &need_update); + if (!sched_asym_cpucap_active() || !need_update) return; if (!p || p->nr_cpus_allowed == 1) { @@ -4554,6 +4557,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) /* ensure we never gain time by being placed backwards. */ se->vruntime = max_vruntime(se->vruntime, vruntime); + trace_android_rvh_place_entity(cfs_rq, se, initial, &vruntime); } static void check_enqueue_throttle(struct cfs_rq *cfs_rq); @@ -8756,6 +8760,7 @@ static void update_cpu_capacity(struct sched_domain *sd, int cpu) if (!capacity) capacity = 1; + trace_android_rvh_update_cpu_capacity(cpu, &capacity); cpu_rq(cpu)->cpu_capacity = capacity; trace_sched_cpu_capacity_tp(cpu_rq(cpu)); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 5275666b5eaa..c480058808bc 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -384,6 +384,7 @@ static bool build_perf_domains(const struct cpumask *cpu_map) * EAS is enabled for asymmetric CPU capacity topologies. * Allow vendor to override if desired. */ + trace_android_rvh_build_perf_domains(&eas_check); if (!per_cpu(sd_asym_cpucapacity, cpu) && !eas_check) { if (sched_debug()) { pr_info("rd %*pbl: CPUs do not have asymmetric capacities\n", diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c index 5a15a9b13c01..8de3c5830041 100644 --- a/kernel/sched/vendor_hooks.c +++ b/kernel/sched/vendor_hooks.c @@ -54,6 +54,22 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_enqueue_task_fair); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_task_fair); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_util_est_update); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_setscheduler_uclamp); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_cpu); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up_success); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_fork); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_wake_up_new_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_new_task_stats); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_flush_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tick_entry); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_schedule); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_cpu_starting); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_cpu_dying); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_account_irq); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_place_entity); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_build_perf_domains); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_cpu_capacity); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_misfit_status); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_rto_next_cpu); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_is_cpu_allowed); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_get_nohz_timer_target);