diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fa7981d0d917..056c9d944d89 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -45,7 +45,7 @@ obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o -obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o perf_trace_user.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_CPU_PM) += sleep.o suspend.o obj-$(CONFIG_CPU_IDLE) += cpuidle.o diff --git a/kernel/sched/walt/perf_trace_counters.h b/kernel/sched/walt/perf_trace_counters.h new file mode 100644 index 000000000000..73d589a5628e --- /dev/null +++ b/kernel/sched/walt/perf_trace_counters.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2013-2014, 2017, 2021, The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM perf_trace_counters + +#if !defined(_PERF_TRACE_COUNTERS_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _PERF_TRACE_COUNTERS_H_ + +/* Ctr index for PMCNTENSET/CLR */ +#define CC 0x80000000 +#define C0 0x1 +#define C1 0x2 +#define C2 0x4 +#define C3 0x8 +#define C4 0x10 +#define C5 0x20 +#define C_ALL (CC | C0 | C1 | C2 | C3 | C4 | C5) +#define NUM_L1_CTRS 6 + +#include +#include +#include + +DECLARE_PER_CPU(u32, cntenset_val); +DECLARE_PER_CPU(u32, previous_ccnt); +DECLARE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts); +TRACE_EVENT(sched_switch_with_ctrs, + + TP_PROTO(pid_t prev, pid_t next), + + TP_ARGS(prev, next), + + TP_STRUCT__entry( + __field(pid_t, old_pid) + __field(pid_t, new_pid) + __field(u32, cctr) + __field(u32, ctr0) + __field(u32, ctr1) + __field(u32, ctr2) + __field(u32, ctr3) + __field(u32, ctr4) + __field(u32, ctr5) + ), + + TP_fast_assign( + u32 cpu = smp_processor_id(); + u32 i; + u32 cnten_val; + u32 total_ccnt = 0; + u32 total_cnt = 0; + u32 delta_l1_cnts[NUM_L1_CTRS]; + + __entry->old_pid = prev; + __entry->new_pid = next; + + cnten_val = per_cpu(cntenset_val, cpu); + + if (cnten_val & CC) { + /* Read value */ + total_ccnt = read_sysreg(pmccntr_el0); + __entry->cctr = total_ccnt - + per_cpu(previous_ccnt, cpu); + per_cpu(previous_ccnt, cpu) = total_ccnt; + } + for (i = 0; i < NUM_L1_CTRS; i++) { + if (cnten_val & (1 << i)) { + /* Select */ + write_sysreg(i, pmselr_el0); + isb(); + /* Read value */ + total_cnt = read_sysreg(pmxevcntr_el0); + delta_l1_cnts[i] = total_cnt - + per_cpu(previous_l1_cnts[i], cpu); + per_cpu(previous_l1_cnts[i], cpu) = + total_cnt; + } else + delta_l1_cnts[i] = 0; + } + + __entry->ctr0 = delta_l1_cnts[0]; + __entry->ctr1 = delta_l1_cnts[1]; + __entry->ctr2 = delta_l1_cnts[2]; + __entry->ctr3 = delta_l1_cnts[3]; + __entry->ctr4 = delta_l1_cnts[4]; + __entry->ctr5 = delta_l1_cnts[5]; + ), + + TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %u, CTR0: %u, CTR1: %u, CTR2: %u, CTR3: %u, CTR4: %u, CTR5: %u", + __entry->old_pid, __entry->new_pid, + __entry->cctr, + __entry->ctr0, __entry->ctr1, + __entry->ctr2, __entry->ctr3, + __entry->ctr4, __entry->ctr5) +); + +#endif +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../kernel/sched/walt + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE perf_trace_counters +#include diff --git a/kernel/sched/walt/walt_tp.c b/kernel/sched/walt/walt_tp.c index f809128fddd0..17708d91407e 100644 --- a/kernel/sched/walt/walt_tp.c +++ b/kernel/sched/walt/walt_tp.c @@ -3,12 +3,100 @@ * Copyright (c) 2021, The Linux Foundation. All rights reserved. */ +#include #include #include #include "trace.h" +#define CREATE_TRACE_POINTS +#include "perf_trace_counters.h" unsigned int sysctl_sched_dynamic_tp_enable; +#define USE_CPUHP_STATE CPUHP_AP_ONLINE_DYN + +DEFINE_PER_CPU(u32, cntenset_val); +DEFINE_PER_CPU(u32, previous_ccnt); +DEFINE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts); +DEFINE_PER_CPU(u32, old_pid); +DEFINE_PER_CPU(u32, hotplug_flag); + +static int tracectr_cpu_hotplug_coming_up(unsigned int cpu) +{ + per_cpu(hotplug_flag, cpu) = 1; + + return 0; +} + +static void setup_prev_cnts(u32 cpu, u32 cnten_val) +{ + int i; + + if (cnten_val & CC) + per_cpu(previous_ccnt, cpu) = + read_sysreg(pmccntr_el0); + + for (i = 0; i < NUM_L1_CTRS; i++) { + if (cnten_val & (1 << i)) { + /* Select */ + write_sysreg(i, pmselr_el0); + isb(); + /* Read value */ + per_cpu(previous_l1_cnts[i], cpu) = + read_sysreg(pmxevcntr_el0); + } + } +} + +void tracectr_notifier(void *ignore, bool preempt, + struct task_struct *prev, struct task_struct *next) +{ + u32 cnten_val; + int current_pid; + u32 cpu = task_cpu(next); + + if (!trace_sched_switch_with_ctrs_enabled()) + return; + + current_pid = next->pid; + if (per_cpu(old_pid, cpu) != -1) { + cnten_val = read_sysreg(pmcntenset_el0); + per_cpu(cntenset_val, cpu) = cnten_val; + /* Disable all the counters that were enabled */ + write_sysreg(cnten_val, pmcntenclr_el0); + + if (per_cpu(hotplug_flag, cpu) == 1) { + per_cpu(hotplug_flag, cpu) = 0; + setup_prev_cnts(cpu, cnten_val); + } else { + trace_sched_switch_with_ctrs(per_cpu(old_pid, cpu), + current_pid); + } + + /* Enable all the counters that were disabled */ + write_sysreg(cnten_val, pmcntenset_el0); + } + per_cpu(old_pid, cpu) = current_pid; +} + +static void register_sched_switch_ctrs(void) +{ + int cpu, rc; + + for_each_possible_cpu(cpu) + per_cpu(old_pid, cpu) = -1; + + rc = cpuhp_setup_state_nocalls(USE_CPUHP_STATE, "tracectr_cpu_hotplug", + tracectr_cpu_hotplug_coming_up, NULL); + if (rc >= 0) + register_trace_sched_switch(tracectr_notifier, NULL); +} + +static void unregister_sched_switch_ctrs(void) +{ + unregister_trace_sched_switch(tracectr_notifier, NULL); + cpuhp_remove_state_nocalls(USE_CPUHP_STATE); +} + static void sched_overutilized(void *data, struct root_domain *rd, bool overutilized) { @@ -23,11 +111,13 @@ static void sched_overutilized(void *data, struct root_domain *rd, static void walt_register_dynamic_tp_events(void) { register_trace_sched_overutilized_tp(sched_overutilized, NULL); + register_sched_switch_ctrs(); } static void walt_unregister_dynamic_tp_events(void) { unregister_trace_sched_overutilized_tp(sched_overutilized, NULL); + unregister_sched_switch_ctrs(); } int sched_dynamic_tp_handler(struct ctl_table *table, int write,