diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index bc072cb3dc28..e88a567709ca 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -128,6 +128,71 @@ DECLARE_RESTRICTED_HOOK(android_rvh_sched_getaffinity, TP_PROTO(struct task_struct *p, struct cpumask *in_mask), TP_ARGS(p, in_mask), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_set_task_cpu, + TP_PROTO(struct task_struct *p, unsigned int new_cpu), + TP_ARGS(p, new_cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_try_to_wake_up, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_try_to_wake_up_success, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_fork, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_wake_up_new_task, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_new_task_stats, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_flush_task, + TP_PROTO(struct task_struct *prev), + TP_ARGS(prev), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_tick_entry, + TP_PROTO(struct rq *rq), + TP_ARGS(rq), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_schedule, + TP_PROTO(struct task_struct *prev, struct task_struct *next, struct rq *rq), + TP_ARGS(prev, next, rq), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_cpu_starting, + TP_PROTO(int cpu), + TP_ARGS(cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_cpu_dying, + TP_PROTO(int cpu), + TP_ARGS(cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_account_irq, + TP_PROTO(struct task_struct *curr, int cpu, s64 delta, bool start), + TP_ARGS(curr, cpu, delta, start), 1); + +struct sched_entity; +DECLARE_RESTRICTED_HOOK(android_rvh_place_entity, + TP_PROTO(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial, u64 *vruntime), + TP_ARGS(cfs_rq, se, initial, vruntime), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_build_perf_domains, + TP_PROTO(bool *eas_check), + TP_ARGS(eas_check), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_update_cpu_capacity, + TP_PROTO(int cpu, unsigned long *capacity), + TP_ARGS(cpu, capacity), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_update_misfit_status, + TP_PROTO(struct task_struct *p, struct rq *rq, bool *need_update), + TP_ARGS(p, rq, need_update), 1); + DECLARE_RESTRICTED_HOOK(android_rvh_sched_fork_init, TP_PROTO(struct task_struct *p), TP_ARGS(p), 1); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index e62200c7edaa..50f4a490bd68 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3184,6 +3184,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) p->se.nr_migrations++; rseq_migrate(p); perf_event_task_migrate(p); + trace_android_rvh_set_task_cpu(p, new_cpu); } __set_task_cpu(p, new_cpu); @@ -4252,6 +4253,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_cond_load_acquire(&p->on_cpu, !VAL); + trace_android_rvh_try_to_wake_up(p); + cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); if (task_cpu(p) != cpu) { if (p->in_iowait) { @@ -4271,8 +4274,10 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) unlock: raw_spin_unlock_irqrestore(&p->pi_lock, flags); out: - if (success) + if (success) { + trace_android_rvh_try_to_wake_up_success(p); ttwu_stat(p, task_cpu(p), wake_flags); + } preempt_enable(); return success; @@ -4623,6 +4628,8 @@ late_initcall(sched_core_sysctl_init); */ int sched_fork(unsigned long clone_flags, struct task_struct *p) { + trace_android_rvh_sched_fork(p); + __sched_fork(clone_flags, p); /* * We mark the process as NEW here. This guarantees that @@ -4748,6 +4755,8 @@ void wake_up_new_task(struct task_struct *p) struct rq_flags rf; struct rq *rq; + trace_android_rvh_wake_up_new_task(p); + raw_spin_lock_irqsave(&p->pi_lock, rf.flags); WRITE_ONCE(p->__state, TASK_RUNNING); #ifdef CONFIG_SMP @@ -4766,6 +4775,7 @@ void wake_up_new_task(struct task_struct *p) rq = __task_rq_lock(p, &rf); update_rq_clock(rq); post_init_entity_util_avg(p); + trace_android_rvh_new_task_stats(p); activate_task(rq, p, ENQUEUE_NOCLOCK); trace_sched_wakeup_new(p); @@ -5182,6 +5192,8 @@ static struct rq *finish_task_switch(struct task_struct *prev) if (prev->sched_class->task_dead) prev->sched_class->task_dead(prev); + trace_android_rvh_flush_task(prev); + /* Task is done with its stack. */ put_task_stack(prev); @@ -5546,6 +5558,7 @@ void scheduler_tick(void) rq_lock(rq, &rf); update_rq_clock(rq); + trace_android_rvh_tick_entry(rq); thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); @@ -6565,6 +6578,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) rq->last_seen_need_resched_ns = 0; #endif + trace_android_rvh_schedule(prev, next, rq); if (likely(prev != next)) { rq->nr_switches++; /* @@ -9580,6 +9594,7 @@ int sched_cpu_starting(unsigned int cpu) sched_core_cpu_starting(cpu); sched_rq_cpu_starting(cpu); sched_tick_start(cpu); + trace_android_rvh_sched_cpu_starting(cpu); return 0; } @@ -9653,6 +9668,8 @@ int sched_cpu_dying(unsigned int cpu) } rq_unlock_irqrestore(rq, &rf); + trace_android_rvh_sched_cpu_dying(cpu); + calc_load_migrate(rq); update_max_interval(); hrtick_clear(rq); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 3e0f79fee72e..a485f3b0201c 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -3,6 +3,7 @@ * Simple CPU accounting cgroup controller */ #include +#include #ifdef CONFIG_IRQ_TIME_ACCOUNTING @@ -54,6 +55,7 @@ void irqtime_account_irq(struct task_struct *curr, unsigned int offset) unsigned int pc; s64 delta; int cpu; + bool irq_start = true; if (!sched_clock_irqtime) return; @@ -69,10 +71,15 @@ void irqtime_account_irq(struct task_struct *curr, unsigned int offset) * in that case, so as not to confuse scheduler with a special task * that do not consume any time, but still wants to run. */ - if (pc & HARDIRQ_MASK) + if (pc & HARDIRQ_MASK) { irqtime_account_delta(irqtime, delta, CPUTIME_IRQ); - else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd()) + irq_start = false; + } else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd()) { irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ); + irq_start = false; + } + + trace_android_rvh_account_irq(curr, cpu, delta, irq_start); } static u64 irqtime_tick_accounted(u64 maxtime) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2396e2a33d19..e60a414c31cf 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4444,7 +4444,10 @@ static inline int task_fits_capacity(struct task_struct *p, static inline void update_misfit_status(struct task_struct *p, struct rq *rq) { - if (!sched_asym_cpucap_active()) + bool need_update = true; + + trace_android_rvh_update_misfit_status(p, rq, &need_update); + if (!sched_asym_cpucap_active() || !need_update) return; if (!p || p->nr_cpus_allowed == 1) { @@ -4554,6 +4557,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) /* ensure we never gain time by being placed backwards. */ se->vruntime = max_vruntime(se->vruntime, vruntime); + trace_android_rvh_place_entity(cfs_rq, se, initial, &vruntime); } static void check_enqueue_throttle(struct cfs_rq *cfs_rq); @@ -8756,6 +8760,7 @@ static void update_cpu_capacity(struct sched_domain *sd, int cpu) if (!capacity) capacity = 1; + trace_android_rvh_update_cpu_capacity(cpu, &capacity); cpu_rq(cpu)->cpu_capacity = capacity; trace_sched_cpu_capacity_tp(cpu_rq(cpu)); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 5275666b5eaa..c480058808bc 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -384,6 +384,7 @@ static bool build_perf_domains(const struct cpumask *cpu_map) * EAS is enabled for asymmetric CPU capacity topologies. * Allow vendor to override if desired. */ + trace_android_rvh_build_perf_domains(&eas_check); if (!per_cpu(sd_asym_cpucapacity, cpu) && !eas_check) { if (sched_debug()) { pr_info("rd %*pbl: CPUs do not have asymmetric capacities\n", diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c index 5a15a9b13c01..8de3c5830041 100644 --- a/kernel/sched/vendor_hooks.c +++ b/kernel/sched/vendor_hooks.c @@ -54,6 +54,22 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_enqueue_task_fair); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_task_fair); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_util_est_update); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_setscheduler_uclamp); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_cpu); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up_success); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_fork); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_wake_up_new_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_new_task_stats); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_flush_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tick_entry); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_schedule); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_cpu_starting); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_cpu_dying); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_account_irq); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_place_entity); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_build_perf_domains); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_cpu_capacity); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_misfit_status); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_rto_next_cpu); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_is_cpu_allowed); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_get_nohz_timer_target);