diff --git a/MAINTAINERS b/MAINTAINERS index 5114e6db7307..c82a7348baa3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23296,6 +23296,14 @@ F: drivers/perf/riscv_pmu.c F: drivers/perf/riscv_pmu_legacy.c F: drivers/perf/riscv_pmu_sbi.c +RISC-V QOS RESCTRL SUPPORT +M: Drew Fustini +R: yunhui cui +L: linux-riscv@lists.infradead.org +S: Supported +F: arch/riscv/include/asm/qos.h +F: arch/riscv/kernel/qos.c + RISC-V RPMI AND MPXY DRIVERS M: Rahul Pathak M: Anup Patel diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c2c2f5f3ed5e..f8a7c48b5a03 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -592,6 +592,24 @@ config RISCV_ISA_SVNAPOT If you don't know what to do here, say Y. +config RISCV_ISA_SSQOSID + bool "Ssqosid extension support for supervisor mode Quality of Service ID" + depends on 64BIT + default n + help + Adds support for the Ssqosid ISA extension (Supervisor-mode + Quality of Service ID). + + Ssqosid defines the srmcfg CSR which allows the system to tag the + running process with an RCID (Resource Control ID) and MCID + (Monitoring Counter ID). The RCID is used to determine resource + allocation. The MCID is used to track resource usage in event + counters. + + For example, a cache controller may use the RCID to apply a + cache partitioning scheme and use the MCID to track how much + cache a process, or a group of processes, is using. + config RISCV_ISA_SVPBMT bool "Svpbmt extension support for supervisor mode page-based memory types" depends on 64BIT && MMU diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index c8b94f3337c1..7f909629d21d 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -84,6 +84,10 @@ #define SATP_ASID_MASK _AC(0xFFFF, UL) #endif +/* SRMCFG fields */ +#define SRMCFG_RCID_MASK GENMASK(11, 0) +#define SRMCFG_MCID_MASK GENMASK(27, 16) + /* Exception cause high bit - is an interrupt if set */ #define CAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1)) @@ -346,6 +350,7 @@ #define CSR_STVAL 0x143 #define CSR_SIP 0x144 #define CSR_SATP 0x180 +#define CSR_SRMCFG 0x181 #define CSR_STIMECMP 0x14D #define CSR_STIMECMPH 0x15D diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 812517b2cec1..49a386d74cd3 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -123,6 +123,9 @@ struct thread_struct { /* A forced icache flush is not needed if migrating to the previous cpu. */ unsigned int prev_cpu; #endif +#ifdef CONFIG_RISCV_ISA_SSQOSID + u32 srmcfg; +#endif }; /* Whitelist the fstate from the task_struct for hardened usercopy */ diff --git a/arch/riscv/include/asm/qos.h b/arch/riscv/include/asm/qos.h new file mode 100644 index 000000000000..cf19e8438bb9 --- /dev/null +++ b/arch/riscv/include/asm/qos.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_QOS_H +#define _ASM_RISCV_QOS_H + +#include + +#ifdef CONFIG_RISCV_ISA_SSQOSID + +#include +#include +#include + +#include +#include + +/* cached value of srmcfg csr for each cpu */ +DECLARE_PER_CPU(u32, cpu_srmcfg); + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DECLARE_PER_CPU(u32, cpu_srmcfg_default); + +static inline void __switch_to_srmcfg(struct task_struct *next) +{ + u32 thread_srmcfg, default_srmcfg; + + thread_srmcfg = READ_ONCE(next->thread.srmcfg); + default_srmcfg = __this_cpu_read(cpu_srmcfg_default); + + /* + * RCID and MCID inherit from cpu_srmcfg_default independently. + * RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID are both 0, so a + * zero field means "unassigned" and takes the CPU default. + */ + if (thread_srmcfg == 0) { + thread_srmcfg = default_srmcfg; + } else { + u32 rcid = FIELD_GET(SRMCFG_RCID_MASK, thread_srmcfg); + u32 mcid = FIELD_GET(SRMCFG_MCID_MASK, thread_srmcfg); + + if (rcid == 0 || mcid == 0) { + if (rcid == 0) + rcid = FIELD_GET(SRMCFG_RCID_MASK, default_srmcfg); + if (mcid == 0) + mcid = FIELD_GET(SRMCFG_MCID_MASK, default_srmcfg); + thread_srmcfg = FIELD_PREP(SRMCFG_RCID_MASK, rcid) | + FIELD_PREP(SRMCFG_MCID_MASK, mcid); + } + } + + if (thread_srmcfg != __this_cpu_read(cpu_srmcfg)) { + /* + * No fence around the csrw. Ssqosid is silent on srmcfg + * ordering versus memory accesses, so a few accesses at the + * switch boundary may carry the previous RCID/MCID. The + * tagging inaccuracy is bounded and acceptable for QoS. + */ + __this_cpu_write(cpu_srmcfg, thread_srmcfg); + csr_write(CSR_SRMCFG, thread_srmcfg); + } +} + +static __always_inline bool has_srmcfg(void) +{ + return riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID); +} + +#else /* ! CONFIG_RISCV_ISA_SSQOSID */ + +struct task_struct; +static __always_inline bool has_srmcfg(void) { return false; } +static inline void __switch_to_srmcfg(struct task_struct *next) { } + +#endif /* CONFIG_RISCV_ISA_SSQOSID */ +#endif /* _ASM_RISCV_QOS_H */ diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index 0e71eb82f920..1c7ea53ec012 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef CONFIG_FPU extern void __fstate_save(struct task_struct *save_to); @@ -119,6 +120,8 @@ do { \ __switch_to_fpu(__prev, __next); \ if (has_vector() || has_xtheadvector()) \ __switch_to_vector(__prev, __next); \ + if (has_srmcfg()) \ + __switch_to_srmcfg(__next); \ if (switch_to_should_flush_icache(__next)) \ local_flush_icache_all(); \ __switch_to_envcfg(__next); \ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index cabb99cadfb6..ebe1c3588177 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -128,3 +128,5 @@ obj-$(CONFIG_ACPI_NUMA) += acpi_numa.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += bugs.o obj-$(CONFIG_RISCV_USER_CFI) += usercfi.o + +obj-$(CONFIG_RISCV_ISA_SSQOSID) += qos.o diff --git a/arch/riscv/kernel/qos.c b/arch/riscv/kernel/qos.c new file mode 100644 index 000000000000..99925c0e93dd --- /dev/null +++ b/arch/riscv/kernel/qos.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * Cached value of srmcfg csr for each cpu. Seeded to U32_MAX so the next + * __switch_to_srmcfg() unconditionally writes the CSR. The encoding + * MCID << 16 | RCID with both fields well under 16 bits can never + * produce this sentinel. This covers early-boot context switches that + * happen before riscv_srmcfg_init() runs as an arch_initcall. + */ +DEFINE_PER_CPU(u32, cpu_srmcfg) = U32_MAX; + +/* default srmcfg value for each cpu, set via resctrl cpu assignment */ +DEFINE_PER_CPU(u32, cpu_srmcfg_default); + +/* + * Invalidate the per-CPU srmcfg cache. Used as both the cpuhp startup + * and teardown callback. U32_MAX is not a valid srmcfg value + * (MCID << 16 | RCID, both fields under 16 bits), so the next + * __switch_to_srmcfg() always writes the CSR. + * + * Ssqosid leaves the CSR implementation-defined across hart stop/start, + * so the cached value cannot be trusted after online. The startup + * callback runs at CPUHP_AP_ONLINE_DYN, before CPUHP_AP_ACTIVE makes + * the CPU schedulable, so the cache is invalidated before any normal + * task runs and the CSR is written on that task's first switch. + * The teardown callback is not relied on. Idle and per-CPU kthreads keep + * switching as the CPU goes down and overwrite the sentinel with the CPU + * default, so it does not survive the offline period. + */ +static int riscv_srmcfg_reset_cache(unsigned int cpu) +{ + per_cpu(cpu_srmcfg, cpu) = U32_MAX; + return 0; +} + +/* + * CPU PM notifier: invalidate the cached srmcfg on resume from a deep + * idle / suspend. Ssqosid leaves CSR_SRMCFG state across low-power + * transitions implementation-defined, and the boot CPU never goes + * through the cpuhp online callback during system suspend, so without + * this hook __switch_to_srmcfg() would skip the CSR write when the + * outgoing task happens to share its srmcfg with the pre-suspend cache. + */ +static int riscv_srmcfg_pm_notify(struct notifier_block *nb, + unsigned long action, void *unused) +{ + switch (action) { + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + /* + * The CSR is implementation-defined across the low-power + * transition. Invalidate the cache and eagerly rewrite the + * CSR for the current task so it does not run mis-tagged + * until the next context switch. + */ + __this_cpu_write(cpu_srmcfg, U32_MAX); + __switch_to_srmcfg(current); + break; + } + return NOTIFY_OK; +} + +static struct notifier_block riscv_srmcfg_pm_nb = { + .notifier_call = riscv_srmcfg_pm_notify, +}; + +static int __init riscv_srmcfg_init(void) +{ + int err; + + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SSQOSID)) + return 0; + + /* + * cpuhp_setup_state() invokes the startup callback locally on every + * already-online CPU, so no separate seed loop is needed here. + */ + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/srmcfg:online", + riscv_srmcfg_reset_cache, riscv_srmcfg_reset_cache); + if (err < 0) + pr_warn("srmcfg: cpuhp setup failed (%d), cache not invalidated on CPU online\n", + err); + + /* + * Register the PM notifier even if the cpuhp setup failed. It is + * independent of the cpuhp state and guards suspend/resume. + */ + cpu_pm_register_notifier(&riscv_srmcfg_pm_nb); + return 0; +} +arch_initcall(riscv_srmcfg_init);