From 9106892e27cad34c87ab1c5631d9c101237bbee1 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:09 +0800 Subject: [PATCH 01/34] perf/x86: Move hybrid PMU initialization before x86_pmu_starting_cpu() The current approach initializes hybrid PMU structures immediately before registering them. This is risky as it can lead to key fields, such as 'capabilities', being inadvertently overwritten. Although no issues have arisen so far, this method is not ideal. It makes the PMU structure fields susceptible to being overwritten, especially with future changes that might initialize fields like 'capabilities' within init_hybrid_pmu() called by x86_pmu_starting_cpu(). To mitigate this potential problem, move the default hybrid structure initialization before calling x86_pmu_starting_cpu(). Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-2-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 8b3ea0adb965..75358a297222 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2218,8 +2218,20 @@ static int __init init_hw_perf_events(void) pmu.attr_update = x86_pmu.attr_update; - if (!is_hybrid()) + if (!is_hybrid()) { x86_pmu_show_pmu_cap(NULL); + } else { + int i; + + /* + * Init default ops. + * Must be called before registering x86_pmu_starting_cpu(), + * otherwise some key PMU fields, e.g., capabilities + * initialized in x86_pmu_starting_cpu(), would be overwritten. + */ + for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) + x86_pmu.hybrid_pmu[i].pmu = pmu; + } if (!x86_pmu.read) x86_pmu.read = _x86_pmu_read; @@ -2266,7 +2278,6 @@ static int __init init_hw_perf_events(void) for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { hybrid_pmu = &x86_pmu.hybrid_pmu[i]; - hybrid_pmu->pmu = pmu; hybrid_pmu->pmu.type = -1; hybrid_pmu->pmu.attr_update = x86_pmu.attr_update; hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE; From 62e074f55f4162e1898e7c1807f1501226a9c393 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:10 +0800 Subject: [PATCH 02/34] perf/x86/intel: Enable large PEBS sampling for XMMs Modern PEBS hardware supports directly sampling XMM registers, then large PEBS can be enabled for XMM registers just like other GPRs. Reported-by: Xudong Hao Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-3-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index cc13164d948f..e412e281266d 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4699,7 +4699,8 @@ static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event) flags &= ~PERF_SAMPLE_REGS_USER; if (event->attr.sample_regs_user & ~PEBS_GP_REGS) flags &= ~PERF_SAMPLE_REGS_USER; - if (event->attr.sample_regs_intr & ~PEBS_GP_REGS) + if (event->attr.sample_regs_intr & + ~(PEBS_GP_REGS | PERF_REG_EXTENDED_MASK)) flags &= ~PERF_SAMPLE_REGS_INTR; return flags; } From bbdf84fcedfa4f1e570e12e7657f7095f6e7d340 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:11 +0800 Subject: [PATCH 03/34] perf/x86/intel: Convert x86_perf_regs to per-cpu variables Currently, the drain_pebs() helpers, e.g., intel_pmu_drain_arch_pebs() define an on-stack x86_perf_regs. Upcoming patches will add new fields like *ymm_regs and *zmm_regs to the x86_perf_regs structure to support sampling for these SIMD registers. This would increase the stack size consumed by these helpers, potentially triggering the warning: "the frame size of 1048 bytes is larger than 1024 bytes [-Wframe-larger-than=]". To eliminate this warning, convert x86_perf_regs to per-cpu variables. Please note drain_pebs() can't be interrupted by other NMIs since either it's already in NMI context or PMU is already disabled. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-4-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 8940f0292229..d592a72796af 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -2917,6 +2917,8 @@ __intel_pmu_pebs_last_event(struct perf_event *event, } } +static DEFINE_PER_CPU(struct x86_perf_regs, x86_pebs_regs); + static __always_inline void __intel_pmu_pebs_events(struct perf_event *event, struct pt_regs *iregs, @@ -2926,8 +2928,8 @@ __intel_pmu_pebs_events(struct perf_event *event, setup_fn setup_sample) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - struct x86_perf_regs perf_regs; - struct pt_regs *regs = &perf_regs.regs; + struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs); + struct pt_regs *regs = &perf_regs->regs; void *at = get_next_pebs_record_by_bit(base, top, bit); int cnt = count; @@ -3175,8 +3177,8 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS]; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct debug_store *ds = cpuc->ds; - struct x86_perf_regs perf_regs; - struct pt_regs *regs = &perf_regs.regs; + struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs); + struct pt_regs *regs = &perf_regs->regs; struct pebs_basic *basic; void *base, *at, *top; u64 mask; @@ -3226,8 +3228,8 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS]; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); union arch_pebs_index index; - struct x86_perf_regs perf_regs; - struct pt_regs *regs = &perf_regs.regs; + struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs); + struct pt_regs *regs = &perf_regs->regs; void *base, *at, *top; u64 mask; From cb388bd530cc518b0e245fb4eabc40f3de538e4f Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:12 +0800 Subject: [PATCH 04/34] perf: Eliminate duplicate arch-specific function definitions Define default common __weak functions for perf_reg_value(), perf_reg_validate(), perf_reg_abi() and perf_get_regs_user(). This helps to eliminate the duplicated arch-specific definitions. No function changes intended. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-5-dapeng1.mi@linux.intel.com --- arch/arm/kernel/perf_regs.c | 6 ------ arch/arm64/kernel/perf_regs.c | 6 ------ arch/csky/kernel/perf_regs.c | 6 ------ arch/loongarch/kernel/perf_regs.c | 6 ------ arch/mips/kernel/perf_regs.c | 6 ------ arch/parisc/kernel/perf_regs.c | 6 ------ arch/riscv/kernel/perf_regs.c | 6 ------ arch/x86/kernel/perf_regs.c | 6 ------ include/linux/perf_regs.h | 32 ++++++------------------------- kernel/events/core.c | 22 +++++++++++++++++++++ 10 files changed, 28 insertions(+), 74 deletions(-) diff --git a/arch/arm/kernel/perf_regs.c b/arch/arm/kernel/perf_regs.c index 0529f90395c9..d575a4c3ca56 100644 --- a/arch/arm/kernel/perf_regs.c +++ b/arch/arm/kernel/perf_regs.c @@ -31,9 +31,3 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_32; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c index b4eece3eb17d..70e2f13f587f 100644 --- a/arch/arm64/kernel/perf_regs.c +++ b/arch/arm64/kernel/perf_regs.c @@ -98,9 +98,3 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_64; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/csky/kernel/perf_regs.c b/arch/csky/kernel/perf_regs.c index 09b7f88a2d6a..94601f37b596 100644 --- a/arch/csky/kernel/perf_regs.c +++ b/arch/csky/kernel/perf_regs.c @@ -31,9 +31,3 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_32; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/loongarch/kernel/perf_regs.c b/arch/loongarch/kernel/perf_regs.c index 263ac4ab5af6..8dd604f01745 100644 --- a/arch/loongarch/kernel/perf_regs.c +++ b/arch/loongarch/kernel/perf_regs.c @@ -45,9 +45,3 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return regs->regs[idx]; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/mips/kernel/perf_regs.c b/arch/mips/kernel/perf_regs.c index e686780d1647..7736d3c5ebd2 100644 --- a/arch/mips/kernel/perf_regs.c +++ b/arch/mips/kernel/perf_regs.c @@ -60,9 +60,3 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return (s64)v; /* Sign extend if 32-bit. */ } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/parisc/kernel/perf_regs.c b/arch/parisc/kernel/perf_regs.c index 10a1a5f06a18..b9fe1f2fcb9b 100644 --- a/arch/parisc/kernel/perf_regs.c +++ b/arch/parisc/kernel/perf_regs.c @@ -53,9 +53,3 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_64; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/riscv/kernel/perf_regs.c b/arch/riscv/kernel/perf_regs.c index fd304a248de6..3bba8deababb 100644 --- a/arch/riscv/kernel/perf_regs.c +++ b/arch/riscv/kernel/perf_regs.c @@ -35,9 +35,3 @@ u64 perf_reg_abi(struct task_struct *task) #endif } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index 624703af80a1..81204cb7f723 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -100,12 +100,6 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_32; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} #else /* CONFIG_X86_64 */ #define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \ (1ULL << PERF_REG_X86_ES) | \ diff --git a/include/linux/perf_regs.h b/include/linux/perf_regs.h index f632c5725f16..144bcc3ff19f 100644 --- a/include/linux/perf_regs.h +++ b/include/linux/perf_regs.h @@ -9,6 +9,12 @@ struct perf_regs { struct pt_regs *regs; }; +u64 perf_reg_value(struct pt_regs *regs, int idx); +int perf_reg_validate(u64 mask); +u64 perf_reg_abi(struct task_struct *task); +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs); + #ifdef CONFIG_HAVE_PERF_REGS #include @@ -16,35 +22,9 @@ struct perf_regs { #define PERF_REG_EXTENDED_MASK 0 #endif -u64 perf_reg_value(struct pt_regs *regs, int idx); -int perf_reg_validate(u64 mask); -u64 perf_reg_abi(struct task_struct *task); -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs); #else #define PERF_REG_EXTENDED_MASK 0 -static inline u64 perf_reg_value(struct pt_regs *regs, int idx) -{ - return 0; -} - -static inline int perf_reg_validate(u64 mask) -{ - return mask ? -ENOSYS : 0; -} - -static inline u64 perf_reg_abi(struct task_struct *task) -{ - return PERF_SAMPLE_REGS_ABI_NONE; -} - -static inline void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} #endif /* CONFIG_HAVE_PERF_REGS */ #endif /* _LINUX_PERF_REGS_H */ diff --git a/kernel/events/core.c b/kernel/events/core.c index a6c8e38a3110..c84340ee2982 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7818,6 +7818,28 @@ unsigned long perf_instruction_pointer(struct perf_event *event, 0 : perf_arch_instruction_pointer(regs); } +u64 __weak perf_reg_value(struct pt_regs *regs, int idx) +{ + return 0; +} + +int __weak perf_reg_validate(u64 mask) +{ + return mask ? -ENOSYS : 0; +} + +u64 __weak perf_reg_abi(struct task_struct *task) +{ + return PERF_SAMPLE_REGS_ABI_NONE; +} + +void __weak perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} + static void perf_output_sample_regs(struct perf_output_handle *handle, struct pt_regs *regs, u64 mask) From 9c05620b49718544ba163c2ccc3c5a6b1f0bc8b0 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:13 +0800 Subject: [PATCH 05/34] perf/x86: Use x86_perf_regs in NMI handlers Support for sampling additional register state in NMI context (e.g. vector registers and SSP) requires an x86-specific register container. The generic pt_regs structure cannot represent all of the required x86 register data, so switch x86 NMI handlers to x86_perf_regs. pt_regs is still passed to x86_pmu_handle_irq(), so there is no functional change to existing handling. AMD IBS NMI handling does not use x86_pmu_handle_irq(), so this conversion does not apply to IBS. IBS support for extended register sampling can be added separately in follow-up patches. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-6-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 5 ++++- arch/x86/xen/pmu.c | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 75358a297222..3368270c0ad1 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1800,9 +1800,11 @@ void perf_put_guest_lvtpc(void) EXPORT_SYMBOL_FOR_KVM(perf_put_guest_lvtpc); #endif /* CONFIG_PERF_GUEST_MEDIATED_PMU */ +static DEFINE_PER_CPU(struct x86_perf_regs, x86_intr_regs); static int perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) { + struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_intr_regs); u64 start_clock; u64 finish_clock; int ret; @@ -1826,7 +1828,8 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) return NMI_DONE; start_clock = sched_clock(); - ret = static_call(x86_pmu_handle_irq)(regs); + x86_regs->regs = *regs; + ret = static_call(x86_pmu_handle_irq)(&x86_regs->regs); finish_clock = sched_clock(); perf_sample_event_took(finish_clock - start_clock); diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 5f50a3ee08f5..3f4dd3f50f56 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -456,12 +456,14 @@ static void xen_convert_regs(const struct xen_pmu_regs *xen_regs, } } +static DEFINE_PER_CPU(struct x86_perf_regs, x86_xen_intr_regs); irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id) { int err, ret = IRQ_NONE; struct pt_regs regs = {0}; const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); uint8_t xenpmu_flags = get_xenpmu_flags(); + struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_xen_intr_regs); if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); @@ -472,7 +474,8 @@ irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id) xenpmu_flags | XENPMU_IRQ_PROCESSING; xen_convert_regs(&xenpmu_data->pmu.r.regs, ®s, xenpmu_data->pmu.pmu_flags); - if (x86_pmu.handle_irq(®s)) + x86_regs->regs = regs; + if (x86_pmu.handle_irq(&x86_regs->regs)) ret = IRQ_HANDLED; /* Write out cached context to HW */ From 03b89c9f202ee75c9997a8421f0e09a5590c3878 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Mon, 24 Aug 2026 16:27:14 +0800 Subject: [PATCH 06/34] x86/fpu/xstate: Add xsaves_nmi() helper Add xsaves_nmi() to save supported xsave states in NMI handler. This function is similar to xsaves(), but should only be called within the NMI handler. This function returns the actual register contents at the moment the NMI occurs. Currently the perf subsystem is the sole user of this helper. It uses this function to snapshot SIMD (XMM/YMM/ZMM) and APX eGPRs registers which would be added in subsequent patches. Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-7-dapeng1.mi@linux.intel.com --- arch/x86/include/asm/fpu/xstate.h | 1 + arch/x86/kernel/fpu/xstate.c | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h index 7a7dc9d56027..38fa8ff26559 100644 --- a/arch/x86/include/asm/fpu/xstate.h +++ b/arch/x86/include/asm/fpu/xstate.h @@ -110,6 +110,7 @@ int xfeature_size(int xfeature_nr); void xsaves(struct xregs_state *xsave, u64 mask); void xrstors(struct xregs_state *xsave, u64 mask); +void xsaves_nmi(struct xregs_state *xsave, u64 mask); int xfd_enable_feature(u64 xfd_err); diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index a7b6524a9dea..d39c01546dee 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -1474,6 +1474,29 @@ void xrstors(struct xregs_state *xstate, u64 mask) WARN_ON_ONCE(err); } +/** + * xsaves_nmi - Save selected components to a kernel xstate buffer in NMI + * @xstate: Pointer to the buffer + * @mask: Feature mask to select the components to save + * + * This function is similar to xsaves(), but should only be called within + * the NMI handler. This function returns the actual register contents at + * the moment the NMI occurs. + * + * Currently, the perf subsystem is the sole user of this helper. It uses + * the function to snapshot SIMD (XMM/YMM/ZMM) and APX eGPRs registers. + */ +void xsaves_nmi(struct xregs_state *xstate, u64 mask) +{ + int err; + + if (!in_nmi()) + return; + + XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err); + WARN_ON_ONCE(err); +} + #if IS_ENABLED(CONFIG_KVM) void fpstate_clear_xstate_component(struct fpstate *fpstate, unsigned int xfeature) { From 450d73dc5f9188ade87fe2d2d967cf18e63b280f Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:15 +0800 Subject: [PATCH 07/34] x86/fpu: Add update_fpu_state_and_flag() helper Add update_fpu_state_and_flag() as suggested by Peter and Dave. The helper saves user FPU state and then sets TIF_NEED_FPU_LOAD, ensuring the task FPU state is saved whenever the flag is set. Subsequent patches will use this guarantee in NMI context by checking TIF_NEED_FPU_LOAD before retrieving user FPU state from the saved task FPU state. Also add barrier() in the host/guest FPU state switch path and move clearing of fpu->__task_fpstate after switching back to host state. So fpu->__task_fpstate is always observed as host FPU state when non-NULL. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/20251204154721.GB2619703@noisy.programming.kicks-ass.net/ Link: https://patch.msgid.link/20260824082731.1013973-8-dapeng1.mi@linux.intel.com --- arch/x86/include/asm/fpu/sched.h | 5 +++-- arch/x86/kernel/fpu/core.c | 33 ++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h index 89004f4ca208..dcb2fa5f06d6 100644 --- a/arch/x86/include/asm/fpu/sched.h +++ b/arch/x86/include/asm/fpu/sched.h @@ -10,6 +10,8 @@ #include extern void save_fpregs_to_fpstate(struct fpu *fpu); +extern void update_fpu_state_and_flag(struct fpu *fpu, + struct task_struct *task); extern void fpu__drop(struct task_struct *tsk); extern int fpu_clone(struct task_struct *dst, u64 clone_flags, bool minimal, unsigned long shstk_addr); @@ -36,8 +38,7 @@ static inline void switch_fpu(struct task_struct *old, int cpu) !(old->flags & (PF_KTHREAD | PF_USER_WORKER))) { struct fpu *old_fpu = x86_task_fpu(old); - set_tsk_thread_flag(old, TIF_NEED_FPU_LOAD); - save_fpregs_to_fpstate(old_fpu); + update_fpu_state_and_flag(old_fpu, old); /* * The save operation preserved register state, so the * fpu_fpregs_owner_ctx is still @old_fpu. Store the diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index d1aeecd57f5e..047244e04d2f 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -213,6 +213,19 @@ void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask) } } +/* + * Save the FPU register state in fpu->fpstate->regs and set + * TIF_NEED_FPU_LOAD subsequently. + * + * Must be called with fpregs_lock() held, ensuring flag + * TIF_NEED_FPU_LOAD is set last. + */ +void update_fpu_state_and_flag(struct fpu *fpu, struct task_struct *task) +{ + save_fpregs_to_fpstate(fpu); + set_tsk_thread_flag(task, TIF_NEED_FPU_LOAD); +} + void fpu_reset_from_exception_fixup(void) { restore_fpregs_from_fpstate(&init_fpstate, XFEATURE_MASK_FPSTATE); @@ -383,13 +396,13 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) /* Swap fpstate */ if (enter_guest) { - fpu->__task_fpstate = cur_fps; + WRITE_ONCE(fpu->__task_fpstate, cur_fps); + barrier(); fpu->fpstate = guest_fps; guest_fps->in_use = true; } else { guest_fps->in_use = false; fpu->fpstate = fpu->__task_fpstate; - fpu->__task_fpstate = NULL; } cur_fps = fpu->fpstate; @@ -406,6 +419,16 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) xfd_update_state(cur_fps); } + /* + * Clear fpu->__task_fpstate after switching back to host state. + * A non-NULL __task_fpstate means guest state is still resident in + * hardware; reset it only once host state has been restored. + */ + if (!enter_guest) { + barrier(); + WRITE_ONCE(fpu->__task_fpstate, NULL); + } + fpregs_mark_activate(); fpregs_unlock(); return 0; @@ -481,10 +504,8 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask) this_cpu_write(kernel_fpu_allowed, false); if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) && - !test_thread_flag(TIF_NEED_FPU_LOAD)) { - set_thread_flag(TIF_NEED_FPU_LOAD); - save_fpregs_to_fpstate(x86_task_fpu(current)); - } + !test_thread_flag(TIF_NEED_FPU_LOAD)) + update_fpu_state_and_flag(x86_task_fpu(current), current); __cpu_invalidate_fpregs_state(); /* Put sane initial values into the control registers. */ From 05fe8825796e6b5557656c6c984486f0e66154fe Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Mon, 24 Aug 2026 16:27:16 +0800 Subject: [PATCH 08/34] perf: Move and enhance has_extended_regs() for arch-specific use Move has_extended_regs() to include/linux/perf_event.h so it can be used by arch-specific code. While moving it, enhance the check logic and rename it to event_has_extended_regs() to match existing perf event helper naming. Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-9-dapeng1.mi@linux.intel.com --- include/linux/perf_event.h | 10 ++++++++++ kernel/events/core.c | 8 +------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 5842552294c1..264785adaefd 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1534,6 +1534,16 @@ perf_event__output_id_sample(struct perf_event *event, extern void perf_log_lost_samples(struct perf_event *event, u64 lost); +static inline bool event_has_extended_regs(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + + return ((attr->sample_type & PERF_SAMPLE_REGS_USER) && + (attr->sample_regs_user & PERF_REG_EXTENDED_MASK)) || + ((attr->sample_type & PERF_SAMPLE_REGS_INTR) && + (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK)); +} + static inline bool event_has_any_exclude_flag(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; diff --git a/kernel/events/core.c b/kernel/events/core.c index c84340ee2982..5a0ae02edaf7 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -13127,12 +13127,6 @@ int perf_pmu_unregister(struct pmu *pmu) } EXPORT_SYMBOL_GPL(perf_pmu_unregister); -static inline bool has_extended_regs(struct perf_event *event) -{ - return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) || - (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK); -} - static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) { struct perf_event_context *ctx = NULL; @@ -13167,7 +13161,7 @@ static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) goto err_pmu; if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) && - has_extended_regs(event)) { + event_has_extended_regs(event)) { ret = -EOPNOTSUPP; goto err_destroy; } From c09466807467860f4d92b05bd4ca7db96f654ed0 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:17 +0800 Subject: [PATCH 09/34] perf/x86/intel: Centralize PERF_PMU_CAP_EXTENDED_REGS updates PERF_PMU_CAP_EXTENDED_REGS is currently updated in multiple paths: intel_ds_pebs_init() and __intel_update_pmu_caps() for adaptive and architectural PEBS. This split makes the capability flow harder to reason about and is awkward for upcoming XSAVES-based SIMD register sampling, which also needs consistent capability setup and validation. Refactor the update logic into a single helper, __intel_update_pmu_xregs_caps(), so adaptive and architectural PEBS share one implementation. Call this helper from intel_update_pmu_caps() to keep PMU capability handling centralized in one place and avoid duplicated or conflicting updates. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-10-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 16 ++++++++-------- arch/x86/events/intel/ds.c | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index e412e281266d..44332ecd5481 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -6162,11 +6162,14 @@ static inline bool intel_pmu_broken_perf_cap(void) return false; } -static inline void __intel_update_pmu_caps(struct pmu *pmu) +static inline void __intel_update_pmu_xregs_caps(struct pmu *pmu) { struct pmu *dest_pmu = pmu ? pmu : x86_get_pmu(smp_processor_id()); + u64 caps = hybrid(pmu, arch_pebs_cap).caps; - if (hybrid(pmu, arch_pebs_cap).caps & ARCH_PEBS_VECR_XMM) + if ((x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_XMM)) || + (!x86_pmu.arch_pebs && x86_pmu.intel_cap.pebs_format >= 4 && + x86_pmu.intel_cap.pebs_baseline)) dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; } @@ -6238,12 +6241,10 @@ static void update_pmu_cap_from_perfmonext(struct pmu *pmu) hybrid(pmu, arch_pebs_cap).counters = pebs_mask; hybrid(pmu, arch_pebs_cap).pdists = pdists_mask; - if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask)) { + if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask)) x86_pmu.arch_pebs = 0; - } else { - __intel_update_pmu_caps(pmu); + else __intel_update_large_pebs_flags(pmu); - } } else { WARN_ON(x86_pmu.arch_pebs == 1); x86_pmu.arch_pebs = 0; @@ -6267,6 +6268,7 @@ static void intel_update_pmu_caps(struct pmu *pmu) hybrid_pmu(pmu)->pmu_type == hybrid_big) hybrid(pmu, intel_cap).perf_metrics = 1; } + __intel_update_pmu_xregs_caps(pmu); } static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu) @@ -6420,8 +6422,6 @@ static void intel_pmu_cpu_starting(int cpu) } } - __intel_update_pmu_caps(cpuc->pmu); - if (!cpuc->shared_regs) return; diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index d592a72796af..e18906a3b31d 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -3396,7 +3396,6 @@ static void __init intel_ds_pebs_init(void) x86_pmu.flags |= PMU_FL_PEBS_ALL; x86_pmu.pebs_capable = ~0ULL; pebs_qual = "-baseline"; - x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; } else { /* Only basic record supported */ x86_pmu.large_pebs_flags &= From b84c962836848f4687a55e6ad219d8ee861a94ed Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:18 +0800 Subject: [PATCH 10/34] perf/x86: Enable XMM register sampling for non-PEBS events Previously, XMM register sampling was only available for PEBS events starting from Icelake. Extend the support to non-PEBS events using the xsaves instruction, thereby completing the feature set. To implement this, a 64-byte aligned buffer is required. A per-CPU ext_regs_buf is introduced to store SIMD and other registers, with an approximate size of 2K. The buffer is allocated using kzalloc_node(), ensuring natural and 64-byte alignment for all kmalloc() allocations with powers of 2. XMM sampling for non-PEBS events is supported in the REGS_INTR case. Support for REGS_USER will be added in a subsequent patch. For PEBS events, XMM register sampling data is directly retrieved from PEBS records. Future support for additional vector registers (YMM/ZMM/OPMASK) is planned. An ext_regs_mask is added to track the supported vector register groups. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-11-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 185 ++++++++++++++++++++++++++++-- arch/x86/events/intel/core.c | 24 +++- arch/x86/events/intel/ds.c | 8 +- arch/x86/events/perf_event.h | 13 +++ arch/x86/include/asm/fpu/xstate.h | 2 + arch/x86/include/asm/perf_event.h | 5 +- arch/x86/kernel/fpu/xstate.c | 2 +- 7 files changed, 219 insertions(+), 20 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 3368270c0ad1..1c3e2bfa2673 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -408,6 +408,53 @@ set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) return x86_pmu_extra_regs(val, event); } +static DEFINE_PER_CPU(struct xregs_state *, ext_regs_buf); + +static void release_ext_regs_buffers(void) +{ + int cpu; + + if (!x86_pmu.ext_regs_mask) + return; + + for_each_possible_cpu(cpu) { + kfree(per_cpu(ext_regs_buf, cpu)); + per_cpu(ext_regs_buf, cpu) = NULL; + } +} + +static void reserve_ext_regs_buffers(void) +{ + bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); + unsigned int size; + int cpu; + + if (!x86_pmu.ext_regs_mask) + return; + + /* Add 64 bytes to satisfy the XSAVE area's 64-byte alignment. */ + size = xstate_calculate_size(x86_pmu.ext_regs_mask, compacted) + 64; + + for_each_possible_cpu(cpu) { + per_cpu(ext_regs_buf, cpu) = kzalloc_node(size, GFP_KERNEL, + cpu_to_node(cpu)); + if (WARN_ON_ONCE(!per_cpu(ext_regs_buf, cpu))) + goto err; + } + + return; + +err: + release_ext_regs_buffers(); +} + +static inline struct xregs_state *get_ext_regs_buf(int cpu) +{ + void *buf = per_cpu(ext_regs_buf, cpu); + + return buf ? PTR_ALIGN(buf, 64) : NULL; +} + int x86_reserve_hardware(void) { int err = 0; @@ -420,6 +467,7 @@ int x86_reserve_hardware(void) } else { reserve_ds_buffers(); reserve_lbr_buffers(); + reserve_ext_regs_buffers(); } } if (!err) @@ -436,6 +484,7 @@ void x86_release_hardware(void) release_pmc_hardware(); release_ds_buffers(); release_lbr_buffers(); + release_ext_regs_buffers(); mutex_unlock(&pmc_reserve_mutex); } } @@ -653,18 +702,31 @@ int x86_pmu_hw_config(struct perf_event *event) return -EINVAL; } - /* sample_regs_user never support XMM registers */ - if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) - return -EINVAL; - /* - * Besides the general purpose registers, XMM registers may - * be collected in PEBS on some platforms, e.g. Icelake - */ - if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) { - if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) - return -EINVAL; + if (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) { + /* + * Besides the general purpose registers, XMM registers may + * be collected as well. + */ + if (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK) { + if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) + return -EINVAL; - if (!event->attr.precise_ip) + if (event->attr.precise_ip) { + u64 caps = hybrid(event->pmu, arch_pebs_cap).caps; + + if (x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) + return -EINVAL; + if (!x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline) + return -EINVAL; + } + if (!get_ext_regs_buf(raw_smp_processor_id())) + return -ENOMEM; + } + } + + if (event->attr.sample_type & PERF_SAMPLE_REGS_USER) { + /* XMM registers sampling for REGS_USER is not supported yet. */ + if (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) return -EINVAL; } @@ -1717,6 +1779,107 @@ static void x86_pmu_del(struct perf_event *event, int flags) static_call_cond(x86_pmu_del)(event); } +void x86_pmu_clear_perf_regs(struct pt_regs *regs) +{ + struct x86_perf_regs *perf_regs = container_of(regs, struct x86_perf_regs, regs); + + perf_regs->xmm_regs = NULL; +} + +static void update_perf_regs(struct x86_perf_regs *perf_regs, + struct xregs_state *xsave, u64 bitmap) +{ + u64 mask; + + if (!xsave) + return; + + /* Restrict to features actually saved by XSAVES */ + mask = bitmap & xsave->header.xfeatures; + + if (mask & XFEATURE_MASK_SSE) + perf_regs->xmm_space = xsave->i387.xmm_space; +} + +/* + * The x86 specific variant of perf_sample_regs_intr(). + * It would be extended to add more SIMD registers sampling support + * in later patches. + */ +static void x86_pmu_update_regs_intr(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs, + bool exclude_kernel) +{ + if (exclude_kernel && !user_mode(regs)) { + data->regs_intr.regs = NULL; + data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE; + } else { + data->regs_intr.regs = regs; + data->regs_intr.abi = perf_reg_abi(current); + } + + data->dyn_size += sizeof(u64); + if (data->regs_intr.regs) { + data->dyn_size += hweight64(event->attr.sample_regs_intr) * + sizeof(u64); + } + + /* + * Set PERF_SAMPLE_REGS_INTR to bypass perf_sample_regs_intr() call + * in perf_prepare_sample() function. + */ + data->sample_flags |= PERF_SAMPLE_REGS_INTR; +} + +static void x86_pmu_sample_xregs(struct perf_event *event, + struct perf_sample_data *data, + bool from_pebs) +{ + struct xregs_state *xsave = get_ext_regs_buf(smp_processor_id()); + u64 sample_type = event->attr.sample_type; + struct x86_perf_regs *perf_regs; + u64 intr_mask = 0; + + if (WARN_ON_ONCE(!xsave) || !in_nmi()) + return; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && data->regs_intr.regs) { + if (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK) + intr_mask |= XFEATURE_MASK_SSE; + + intr_mask &= x86_pmu.ext_regs_mask; + intr_mask = from_pebs ? 0 : intr_mask; + } + + if (intr_mask) { + perf_regs = container_of(data->regs_intr.regs, + struct x86_perf_regs, regs); + xsave->header.xfeatures = 0; + xsaves_nmi(xsave, intr_mask); + update_perf_regs(perf_regs, xsave, intr_mask); + } +} + +void x86_pmu_update_perf_regs(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs, + bool from_pebs) +{ + u64 sample_type = event->attr.sample_type; + + if (!((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK))) + return; + + if (sample_type & PERF_SAMPLE_REGS_INTR) { + x86_pmu_update_regs_intr(event, data, regs, + event->attr.exclude_kernel); + } + + x86_pmu_sample_xregs(event, data, from_pebs); +} + int x86_pmu_handle_irq(struct pt_regs *regs) { struct perf_sample_data data; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 44332ecd5481..ddf72614e739 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3925,6 +3925,9 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) if (has_branch_stack(event)) intel_pmu_lbr_save_brstack(&data, cpuc, event); + x86_pmu_clear_perf_regs(regs); + x86_pmu_update_perf_regs(event, &data, regs, false); + perf_event_overflow(event, &data, regs); } @@ -6165,12 +6168,23 @@ static inline bool intel_pmu_broken_perf_cap(void) static inline void __intel_update_pmu_xregs_caps(struct pmu *pmu) { struct pmu *dest_pmu = pmu ? pmu : x86_get_pmu(smp_processor_id()); - u64 caps = hybrid(pmu, arch_pebs_cap).caps; - if ((x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_XMM)) || - (!x86_pmu.arch_pebs && x86_pmu.intel_cap.pebs_format >= 4 && - x86_pmu.intel_cap.pebs_baseline)) - dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; + /* Only support the extension when XSAVES is available. */ + if (!boot_cpu_has(X86_FEATURE_XSAVES)) + return; + + if (!boot_cpu_has(X86_FEATURE_XMM) || + !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) + return; + + /* + * On current hybrid platforms, P-cores and E-cores expose the same + * XSAVE feature set. Therefore, using the global x86_pmu.ext_regs_mask + * is sufficient to represent the hardware-supported XSAVE features. + */ + x86_pmu.ext_regs_mask |= XFEATURE_MASK_SSE; + + dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; } static inline void __intel_update_large_pebs_flags(struct pmu *pmu) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index e18906a3b31d..e14f8c22ffb5 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -2515,7 +2515,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, return; perf_regs = container_of(regs, struct x86_perf_regs, regs); - perf_regs->xmm_regs = NULL; + x86_pmu_clear_perf_regs(regs); format_group = basic->format_group; @@ -2600,6 +2600,8 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, next_record += nr * sizeof(u64); } + x86_pmu_update_perf_regs(event, data, regs, true); + WARN_ONCE(next_record != __pebs + basic->format_size, "PEBS record size %u, expected %llu, config %llx\n", basic->format_size, @@ -2632,7 +2634,7 @@ static void setup_arch_pebs_sample_data(struct perf_event *event, return; perf_regs = container_of(regs, struct x86_perf_regs, regs); - perf_regs->xmm_regs = NULL; + x86_pmu_clear_perf_regs(regs); __setup_perf_sample_data(event, iregs, data); @@ -2734,6 +2736,8 @@ static void setup_arch_pebs_sample_data(struct perf_event *event, at = at + header->size; goto again; } + + x86_pmu_update_perf_regs(event, data, regs, true); } static inline void * diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 71ed5b2acea2..f1ab72094bac 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1028,6 +1028,12 @@ struct x86_pmu { struct extra_reg *extra_regs; unsigned int flags; + /* + * Extended regs, e.g., vector registers + * Utilize the same format as the XFEATURE_MASK_* + */ + u64 ext_regs_mask; + /* * Intel host/guest support (KVM) */ @@ -1315,6 +1321,13 @@ void x86_pmu_enable_event(struct perf_event *event); int x86_pmu_handle_irq(struct pt_regs *regs); +void x86_pmu_clear_perf_regs(struct pt_regs *regs); + +void x86_pmu_update_perf_regs(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs, + bool from_pebs); + void x86_pmu_show_pmu_cap(struct pmu *pmu); static inline int x86_pmu_num_counters(struct pmu *pmu) diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h index 38fa8ff26559..19dec5f0b1c7 100644 --- a/arch/x86/include/asm/fpu/xstate.h +++ b/arch/x86/include/asm/fpu/xstate.h @@ -112,6 +112,8 @@ void xsaves(struct xregs_state *xsave, u64 mask); void xrstors(struct xregs_state *xsave, u64 mask); void xsaves_nmi(struct xregs_state *xsave, u64 mask); +unsigned int xstate_calculate_size(u64 xfeatures, bool compacted); + int xfd_enable_feature(u64 xfd_err); #ifdef CONFIG_X86_64 diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 1eb13673e889..619e0ae915e1 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -728,7 +728,10 @@ extern void perf_events_lapic_init(void); struct pt_regs; struct x86_perf_regs { struct pt_regs regs; - u64 *xmm_regs; + union { + u64 *xmm_regs; + u32 *xmm_space; /* for xsaves */ + }; }; extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs); diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index d39c01546dee..3e7f5fb5bfaf 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -587,7 +587,7 @@ static bool __init check_xstate_against_struct(int nr) return true; } -static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted) +unsigned int xstate_calculate_size(u64 xfeatures, bool compacted) { unsigned int topmost = fls64(xfeatures) - 1; unsigned int offset, i; From edd9aec51213debb788d50ad8d7aeaac59fde6d6 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:19 +0800 Subject: [PATCH 11/34] perf/x86: Enable XMM register sampling for REGS_USER case Support XMM register sampling for the REGS_USER case. To handle simultaneous sampling of XMM registers for both REGS_INTR and REGS_USER cases, a per-CPU x86_user_regs is introduced to store REGS_USER-specific XMM registers. This prevents REGS_USER-specific XMM register data from being overwritten by REGS_INTR-specific data if they share the same x86_perf_regs structure. To sample user-space XMM registers, the x86_pmu_update_user_xregs() helper function is added. It checks if the user-space FPU state has been cached into task FPU state. If so, the user-space XMM register data can be directly retrieved from the cached task FPU state, as the corresponding hardware registers have been cleared or switched to kernel-space state or guest state. Otherwise, the data could be read from the hardware registers using the xsaves instruction. For PEBS events, x86_pmu_update_user_xregs() checks if the PEBS-sampled XMM register data belongs to user-space. If so, no further action is needed. Otherwise, the user-space XMM register data needs to be re-sampled using the same method as for non-PEBS events. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-12-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 154 ++++++++++++++++++++++++++++++++--- arch/x86/events/intel/core.c | 6 +- arch/x86/events/intel/ds.c | 5 +- 3 files changed, 147 insertions(+), 18 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 1c3e2bfa2673..20140643e21c 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -702,12 +702,12 @@ int x86_pmu_hw_config(struct perf_event *event) return -EINVAL; } - if (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) { + if (event->attr.sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) { /* * Besides the general purpose registers, XMM registers may * be collected as well. */ - if (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK) { + if (event_has_extended_regs(event)) { if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) return -EINVAL; @@ -724,12 +724,6 @@ int x86_pmu_hw_config(struct perf_event *event) } } - if (event->attr.sample_type & PERF_SAMPLE_REGS_USER) { - /* XMM registers sampling for REGS_USER is not supported yet. */ - if (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) - return -EINVAL; - } - return x86_setup_perfctr(event); } @@ -1832,14 +1826,129 @@ static void x86_pmu_update_regs_intr(struct perf_event *event, data->sample_flags |= PERF_SAMPLE_REGS_INTR; } +static DEFINE_PER_CPU(struct x86_perf_regs, x86_user_regs); + +static void x86_pmu_get_regs_user(struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs); + struct perf_regs regs_user; + + x86_pmu_clear_perf_regs(&x86_regs_user->regs); + + perf_get_regs_user(®s_user, regs); + data->regs_user.abi = regs_user.abi; + if (regs_user.regs) { + x86_regs_user->regs = *regs_user.regs; + data->regs_user.regs = &x86_regs_user->regs; + } else { + data->regs_user.regs = NULL; + } +} + +/* + * The x86 specific variant of perf_sample_regs_user(). + * Update data->regs_user fields for extended registers (e.g., SIMD). + */ +static void x86_pmu_update_regs_user(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs); + struct perf_event_attr *attr = &event->attr; + struct x86_perf_regs *perf_regs; + + /* + * PERF_SAMPLE_REGS_INTR and PERF_SAMPLE_REGS_USER can both be + * requested for one event. Keep user regs in a separate x86_perf_regs + * instance, so intr-reg collection does not overwrite user-reg data. + */ + if (user_mode(regs)) { + x86_pmu_clear_perf_regs(&x86_regs_user->regs); + perf_regs = container_of(regs, struct x86_perf_regs, regs); + /* Copy all sampled regs data to x86_regs_user. */ + *x86_regs_user = *perf_regs; + data->regs_user.regs = &x86_regs_user->regs; + data->regs_user.abi = perf_reg_abi(current); + } else if (is_user_task(current)) { + x86_pmu_get_regs_user(data, regs); + } else { + data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE; + data->regs_user.regs = NULL; + } + + data->dyn_size += sizeof(u64); + if (data->regs_user.regs) + data->dyn_size += hweight64(attr->sample_regs_user) * sizeof(u64); + + /* + * Set PERF_SAMPLE_REGS_USER to bypass perf_sample_regs_user() call + * in perf_prepare_sample() function. + */ + data->sample_flags |= PERF_SAMPLE_REGS_USER; +} + +/* + * This function retrieves cached user-space fpu registers (XMM/YMM/ZMM). + * If TIF_NEED_FPU_LOAD is set or PMI hits into guest, it indicates that + * the user-space FPU state is cached. Otherwise, the data should be read + * directly from the hardware registers. + */ +static inline u64 x86_pmu_update_user_xregs(struct perf_sample_data *data, + struct pt_regs *regs, + u64 mask, bool from_pebs) +{ + struct x86_perf_regs *perf_regs; + struct xregs_state *xsave; + struct fpu *fpu; + struct fpstate *fps; + u64 user_mask = mask; + + if (!is_user_task(current)) + return 0; + + if (data->regs_user.abi == PERF_SAMPLE_REGS_ABI_NONE) + return 0; + + /* + * If PEBS hits kernel space, need to re-sample extended + * registers for user space. + */ + if (user_mode(regs)) + user_mask = from_pebs ? 0 : mask; + + fpu = x86_task_fpu(current); + fps = READ_ONCE(fpu->__task_fpstate); + /* + * If fpu->__task_fpstate is set, it points to the cached user + * FPU state (e.g. with KVM guest-state swapping). Otherwise, + * when TIF_NEED_FPU_LOAD is set, fpu->fpstate holds the cached + * user state. If neither is true, the user state is live in hardware. + */ + if (user_mask && (test_thread_flag(TIF_NEED_FPU_LOAD) || fps)) { + perf_regs = container_of(data->regs_user.regs, + struct x86_perf_regs, regs); + if (!fps) + fps = fpu->fpstate; + xsave = &fps->regs.xsave; + + update_perf_regs(perf_regs, xsave, user_mask); + return 0; + } + + return user_mask; +} + static void x86_pmu_sample_xregs(struct perf_event *event, struct perf_sample_data *data, + struct pt_regs *regs, bool from_pebs) { struct xregs_state *xsave = get_ext_regs_buf(smp_processor_id()); u64 sample_type = event->attr.sample_type; struct x86_perf_regs *perf_regs; u64 intr_mask = 0; + u64 user_mask = 0; if (WARN_ON_ONCE(!xsave) || !in_nmi()) return; @@ -1852,13 +1961,31 @@ static void x86_pmu_sample_xregs(struct perf_event *event, intr_mask = from_pebs ? 0 : intr_mask; } + if ((sample_type & PERF_SAMPLE_REGS_USER) && data->regs_user.regs) { + if (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) + user_mask |= XFEATURE_MASK_SSE; + + user_mask &= x86_pmu.ext_regs_mask; + user_mask = x86_pmu_update_user_xregs(data, regs, + user_mask, from_pebs); + } + + if (user_mask | intr_mask) { + xsave->header.xfeatures = 0; + xsaves_nmi(xsave, user_mask | intr_mask); + } + if (intr_mask) { perf_regs = container_of(data->regs_intr.regs, struct x86_perf_regs, regs); - xsave->header.xfeatures = 0; - xsaves_nmi(xsave, intr_mask); update_perf_regs(perf_regs, xsave, intr_mask); } + + if (user_mask) { + perf_regs = container_of(data->regs_user.regs, + struct x86_perf_regs, regs); + update_perf_regs(perf_regs, xsave, user_mask); + } } void x86_pmu_update_perf_regs(struct perf_event *event, @@ -1868,16 +1995,17 @@ void x86_pmu_update_perf_regs(struct perf_event *event, { u64 sample_type = event->attr.sample_type; - if (!((sample_type & PERF_SAMPLE_REGS_INTR) && - (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK))) + if (!event_has_extended_regs(event)) return; if (sample_type & PERF_SAMPLE_REGS_INTR) { x86_pmu_update_regs_intr(event, data, regs, event->attr.exclude_kernel); } + if (sample_type & PERF_SAMPLE_REGS_USER) + x86_pmu_update_regs_user(event, data, regs); - x86_pmu_sample_xregs(event, data, from_pebs); + x86_pmu_sample_xregs(event, data, regs, from_pebs); } int x86_pmu_handle_irq(struct pt_regs *regs) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index ddf72614e739..5cb80833c7ed 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4695,15 +4695,15 @@ static void intel_pebs_aliases_skl(struct perf_event *event) static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event) { unsigned long flags = x86_pmu.large_pebs_flags; + u64 gprs_mask = PEBS_GP_REGS | PERF_REG_EXTENDED_MASK; if (event->attr.use_clockid) flags &= ~PERF_SAMPLE_TIME; if (!event->attr.exclude_kernel) flags &= ~PERF_SAMPLE_REGS_USER; - if (event->attr.sample_regs_user & ~PEBS_GP_REGS) + if (event->attr.sample_regs_user & ~gprs_mask) flags &= ~PERF_SAMPLE_REGS_USER; - if (event->attr.sample_regs_intr & - ~(PEBS_GP_REGS | PERF_REG_EXTENDED_MASK)) + if (event->attr.sample_regs_intr & ~gprs_mask) flags &= ~PERF_SAMPLE_REGS_INTR; return flags; } diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index e14f8c22ffb5..d216234803b8 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1733,8 +1733,7 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) if (gprs || (attr->precise_ip < 2) || tsx_weight) pebs_data_cfg |= PEBS_DATACFG_GP; - if ((sample_type & PERF_SAMPLE_REGS_INTR) && - (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK)) + if (event_has_extended_regs(event)) pebs_data_cfg |= PEBS_DATACFG_XMMS; if (sample_type & PERF_SAMPLE_BRANCH_STACK) { @@ -2937,6 +2936,8 @@ __intel_pmu_pebs_events(struct perf_event *event, void *at = get_next_pebs_record_by_bit(base, top, bit); int cnt = count; + x86_pmu_clear_perf_regs(regs); + if (!iregs) iregs = &dummy_iregs; From 918b7d6d1729952d51da4fa81db7b366a207de8f Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Mon, 24 Aug 2026 16:27:20 +0800 Subject: [PATCH 12/34] perf: Add sampling support for SIMD registers Users may be interested in sampling SIMD registers during profiling. The current sample_regs_* structure does not have sufficient space for all SIMD registers. To address this, new attribute fields sample_simd_{pred,vec}_reg_* are added to struct perf_event_attr to represent the SIMD registers that are expected to be sampled. Currently, the perf/x86 code supports XMM registers in sample_regs_*. To unify the configuration of SIMD registers and ensure a consistent method for configuring XMM and other SIMD registers, a new event attribute field, sample_simd_regs_enabled, is introduced. When sample_simd_regs_enabled is set, it indicates that all SIMD registers, including XMM, will be represented by the newly introduced sample_simd_{pred|vec}_reg_* fields. The original XMM space in sample_regs_* is reserved for future uses. Since SIMD registers are wider than 64 bits, a new output format is introduced. The number and width of SIMD registers are dumped first, followed by the register values. The number and width are based on the user's configuration. A new ABI, PERF_SAMPLE_REGS_ABI_SIMD, is added to indicate the new format. The enum perf_sample_regs_abi is now a bitmap. This change should not impact existing tools, as the version and bitmap remain the same for values 1 and 2. Additionally, two new __weak functions are introduced: - perf_simd_reg_value(): Retrieves the value of the requested SIMD register. - perf_simd_reg_validate(): Validates the configuration of the SIMD registers. A new flag, PERF_PMU_CAP_SIMD_REGS, is added to indicate that the PMU supports SIMD register dumping. An error is generated if sample_simd_{pred|vec}_reg_* is mistakenly set for a PMU that does not support this capability. Suggested-by: Peter Zijlstra (Intel) Signed-off-by: Kan Liang Co-developed-by: Dapeng Mi Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-13-dapeng1.mi@linux.intel.com --- include/linux/perf_event.h | 12 +++ include/linux/perf_regs.h | 4 + include/uapi/linux/perf_event.h | 49 ++++++++++- kernel/events/core.c | 141 +++++++++++++++++++++++++++++--- 4 files changed, 192 insertions(+), 14 deletions(-) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 264785adaefd..a2e7eae268cd 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -306,6 +306,7 @@ struct perf_event_pmu_context; #define PERF_PMU_CAP_AUX_PAUSE 0x0200 #define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400 #define PERF_PMU_CAP_MEDIATED_VPMU 0x0800 +#define PERF_PMU_CAP_SIMD_REGS 0x1000 /** * pmu::scope @@ -1534,6 +1535,17 @@ perf_event__output_id_sample(struct perf_event *event, extern void perf_log_lost_samples(struct perf_event *event, u64 lost); +static inline bool event_has_simd_regs(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + + if (!(event->attr.sample_type & + (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))) + return false; + + return attr->sample_simd_regs_enabled != 0; +} + static inline bool event_has_extended_regs(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; diff --git a/include/linux/perf_regs.h b/include/linux/perf_regs.h index 144bcc3ff19f..518f28c6a7d4 100644 --- a/include/linux/perf_regs.h +++ b/include/linux/perf_regs.h @@ -14,6 +14,10 @@ int perf_reg_validate(u64 mask); u64 perf_reg_abi(struct task_struct *task); void perf_get_regs_user(struct perf_regs *regs_user, struct pt_regs *regs); +int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, + u16 pred_qwords, u32 pred_mask); +u64 perf_simd_reg_value(struct pt_regs *regs, int idx, + u16 qwords_idx, bool pred); #ifdef CONFIG_HAVE_PERF_REGS #include diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index fd10aa8d697f..c49fc76292f7 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -314,8 +314,9 @@ enum { */ enum perf_sample_regs_abi { PERF_SAMPLE_REGS_ABI_NONE = 0, - PERF_SAMPLE_REGS_ABI_32 = 1, - PERF_SAMPLE_REGS_ABI_64 = 2, + PERF_SAMPLE_REGS_ABI_32 = (1 << 0), + PERF_SAMPLE_REGS_ABI_64 = (1 << 1), + PERF_SAMPLE_REGS_ABI_SIMD = (1 << 2), }; /* @@ -383,6 +384,7 @@ enum perf_event_read_format { #define PERF_ATTR_SIZE_VER7 128 /* Add: sig_data */ #define PERF_ATTR_SIZE_VER8 136 /* Add: config3 */ #define PERF_ATTR_SIZE_VER9 144 /* add: config4 */ +#define PERF_ATTR_SIZE_VER10 176 /* Add: sample_simd_{vec|pred}_reg_* */ /* * 'struct perf_event_attr' contains various attributes that define @@ -547,6 +549,29 @@ struct perf_event_attr { __u64 config3; /* extension of config2 */ __u64 config4; /* extension of config3 */ + + /* + * Defines the sampling SIMD/PRED(predicate) registers bitmap and + * qwords (8 bytes) length. + * + * sample_simd_regs_enabled != 0 indicates there are SIMD/PRED + * registers to be sampled, the SIMD/PRED registers bitmap and + * qwords length are represented in + * sample_simd_{vec|pred}_reg_{intr|user} and + * sample_simd_{vec|pred}_reg_qwords fields separately. + * + * sample_simd_regs_enabled == 0 indicates no SIMD/PRED registers + * are sampled. + */ + __u16 sample_simd_regs_enabled; + __u16 sample_simd_pred_reg_qwords; + __u16 sample_simd_vec_reg_qwords; + __u16 __reserved_4; + + __u32 sample_simd_pred_reg_intr; + __u32 sample_simd_pred_reg_user; + __u64 sample_simd_vec_reg_intr; + __u64 sample_simd_vec_reg_user; }; /* @@ -1020,7 +1045,15 @@ enum perf_event_type { * } && PERF_SAMPLE_BRANCH_STACK * * { u64 abi; # enum perf_sample_regs_abi - * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER + * u64 regs[weight(mask)]; + * struct { + * u64 nr_vectors; # 0 ... weight(sample_simd_vec_reg_user) + * u64 vector_qwords; # 0 ... sample_simd_vec_reg_qwords + * u64 nr_pred; # 0 ... weight(sample_simd_pred_reg_user) + * u64 pred_qwords; # 0 ... sample_simd_pred_reg_qwords + * u64 data[nr_vectors * vector_qwords + nr_pred * pred_qwords]; + * } && (abi & PERF_SAMPLE_REGS_ABI_SIMD) + * } && PERF_SAMPLE_REGS_USER * * { u64 size; * char data[size]; @@ -1047,7 +1080,15 @@ enum perf_event_type { * { u64 data_src; } && PERF_SAMPLE_DATA_SRC * { u64 transaction; } && PERF_SAMPLE_TRANSACTION * { u64 abi; # enum perf_sample_regs_abi - * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR + * u64 regs[weight(mask)]; + * struct { + * u64 nr_vectors; # 0 ... weight(sample_simd_vec_reg_intr) + * u64 vector_qwords; # 0 ... sample_simd_vec_reg_qwords + * u64 nr_pred; # 0 ... weight(sample_simd_pred_reg_intr) + * u64 pred_qwords; # 0 ... sample_simd_pred_reg_qwords + * u64 data[nr_vectors * vector_qwords + nr_pred * pred_qwords]; + * } && (abi & PERF_SAMPLE_REGS_ABI_SIMD) + * } && PERF_SAMPLE_REGS_INTR * { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR * { u64 cgroup;} && PERF_SAMPLE_CGROUP * { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE diff --git a/kernel/events/core.c b/kernel/events/core.c index 5a0ae02edaf7..3a87c6adcded 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7840,22 +7840,60 @@ void __weak perf_get_regs_user(struct perf_regs *regs_user, regs_user->abi = perf_reg_abi(current); } +#define word_for_each_set_bit(bit, val) \ + for (unsigned long long __v = (val); \ + __v && ((bit = __builtin_ctzll(__v)), 1); \ + __v &= __v - 1) + static void perf_output_sample_regs(struct perf_output_handle *handle, struct pt_regs *regs, u64 mask) { int bit; - DECLARE_BITMAP(_mask, 64); - bitmap_from_u64(_mask, mask); - for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) { - u64 val; - - val = perf_reg_value(regs, bit); + word_for_each_set_bit(bit, mask) { + u64 val = perf_reg_value(regs, bit); perf_output_put(handle, val); } } +static void +perf_output_sample_simd_regs(struct perf_output_handle *handle, + struct perf_event *event, + struct pt_regs *regs, + u64 mask, u32 pred_mask) +{ + u64 pred_qwords = event->attr.sample_simd_pred_reg_qwords; + u64 vec_qwords = event->attr.sample_simd_vec_reg_qwords; + u64 nr_vectors = hweight64(mask); + u64 nr_pred = hweight32(pred_mask); + int bit; + + perf_output_put(handle, nr_vectors); + perf_output_put(handle, vec_qwords); + perf_output_put(handle, nr_pred); + perf_output_put(handle, pred_qwords); + + if (nr_vectors) { + word_for_each_set_bit(bit, mask) { + for (int i = 0; i < vec_qwords; i++) { + u64 val = perf_simd_reg_value(regs, bit, + i, false); + perf_output_put(handle, val); + } + } + } + if (nr_pred) { + word_for_each_set_bit(bit, pred_mask) { + for (int i = 0; i < pred_qwords; i++) { + u64 val = perf_simd_reg_value(regs, bit, + i, true); + perf_output_put(handle, val); + } + } + } +} + static void perf_sample_regs_user(struct perf_regs *regs_user, struct pt_regs *regs) { @@ -7889,6 +7927,17 @@ static void perf_sample_regs_intr(struct perf_regs *regs_intr, } } +int __weak perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, + u16 pred_qwords, u32 pred_mask) +{ + return -EINVAL; +} + +u64 __weak perf_simd_reg_value(struct pt_regs *regs, int idx, + u16 qwords_idx, bool pred) +{ + return 0; +} /* * Get remaining task size from user stack pointer. @@ -8419,10 +8468,17 @@ void perf_output_sample(struct perf_output_handle *handle, perf_output_put(handle, abi); if (abi) { - u64 mask = event->attr.sample_regs_user; + struct perf_event_attr *attr = &event->attr; + u64 mask = attr->sample_regs_user; perf_output_sample_regs(handle, data->regs_user.regs, mask); + if (abi & PERF_SAMPLE_REGS_ABI_SIMD) { + perf_output_sample_simd_regs(handle, event, + data->regs_user.regs, + attr->sample_simd_vec_reg_user, + attr->sample_simd_pred_reg_user); + } } } @@ -8450,11 +8506,18 @@ void perf_output_sample(struct perf_output_handle *handle, perf_output_put(handle, abi); if (abi) { - u64 mask = event->attr.sample_regs_intr; + struct perf_event_attr *attr = &event->attr; + u64 mask = attr->sample_regs_intr; perf_output_sample_regs(handle, data->regs_intr.regs, mask); + if (abi & PERF_SAMPLE_REGS_ABI_SIMD) { + perf_output_sample_simd_regs(handle, event, + data->regs_intr.regs, + attr->sample_simd_vec_reg_intr, + attr->sample_simd_pred_reg_intr); + } } } @@ -8657,6 +8720,29 @@ static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d) return d * !!(flags & s); } +static u64 perf_update_xregs_size(struct perf_event *event, bool intr) +{ + u16 pred_qwords = event->attr.sample_simd_pred_reg_qwords; + u16 vec_qwords = event->attr.sample_simd_vec_reg_qwords; + u64 pred_mask; + u64 mask; + int size; + + if (intr) { + mask = event->attr.sample_simd_vec_reg_intr; + pred_mask = event->attr.sample_simd_pred_reg_intr; + } else { + mask = event->attr.sample_simd_vec_reg_user; + pred_mask = event->attr.sample_simd_pred_reg_user; + } + + size = sizeof(u64) * 4; + size += (hweight64(mask) * vec_qwords + + hweight64(pred_mask) * pred_qwords) * sizeof(u64); + + return size; +} + void perf_prepare_sample(struct perf_sample_data *data, struct perf_event *event, struct pt_regs *regs) @@ -8719,7 +8805,12 @@ void perf_prepare_sample(struct perf_sample_data *data, if (data->regs_user.regs) { u64 mask = event->attr.sample_regs_user; + size += hweight64(mask) * sizeof(u64); + if (event_has_simd_regs(event)) { + size += perf_update_xregs_size(event, false); + data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } } data->dyn_size += size; @@ -8784,6 +8875,10 @@ void perf_prepare_sample(struct perf_sample_data *data, u64 mask = event->attr.sample_regs_intr; size += hweight64(mask) * sizeof(u64); + if (event_has_simd_regs(event)) { + size += perf_update_xregs_size(event, true); + data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } } data->dyn_size += size; @@ -13160,6 +13255,12 @@ static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) if (ret) goto err_pmu; + if (!(pmu->capabilities & PERF_PMU_CAP_SIMD_REGS) && + event_has_simd_regs(event)) { + ret = -EOPNOTSUPP; + goto err_destroy; + } + if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) && event_has_extended_regs(event)) { ret = -EOPNOTSUPP; @@ -13656,7 +13757,8 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, attr->size = size; - if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) + if (attr->__reserved_1 || attr->__reserved_2 || + attr->__reserved_3 || attr->__reserved_4) return -EINVAL; if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) @@ -13705,6 +13807,14 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, ret = perf_reg_validate(attr->sample_regs_user); if (ret) return ret; + if (attr->sample_simd_regs_enabled) { + ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords, + attr->sample_simd_vec_reg_user, + attr->sample_simd_pred_reg_qwords, + attr->sample_simd_pred_reg_user); + if (ret) + return ret; + } } if (attr->sample_type & PERF_SAMPLE_STACK_USER) { @@ -13725,8 +13835,19 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, if (!attr->sample_max_stack) attr->sample_max_stack = sysctl_perf_event_max_stack; - if (attr->sample_type & PERF_SAMPLE_REGS_INTR) + if (attr->sample_type & PERF_SAMPLE_REGS_INTR) { ret = perf_reg_validate(attr->sample_regs_intr); + if (ret) + return ret; + if (attr->sample_simd_regs_enabled) { + ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords, + attr->sample_simd_vec_reg_intr, + attr->sample_simd_pred_reg_qwords, + attr->sample_simd_pred_reg_intr); + if (ret) + return ret; + } + } #ifndef CONFIG_CGROUP_PERF if (attr->sample_type & PERF_SAMPLE_CGROUP) From e9d76ada769c45e5363306d22cb10b323f6dc4b5 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:21 +0800 Subject: [PATCH 13/34] perf/x86: Support XMM sampling using sample_simd_vec_reg_* fields Support sampling of XMM registers using the sample_simd_vec_reg_* fields. When sample_simd_regs_enabled is set, the original XMM space in the sample_regs_* field is treated as reserved. An INVAL error will be reported to user space if any bit is set in the original XMM space while sample_simd_regs_enabled is set. The perf_reg_value function requires ABI information to understand the layout of sample_regs. To accommodate this, a new abi field is introduced in the struct x86_perf_regs to represent ABI information. Additionally, the x86 specific perf_simd_reg_value() function is implemented to retrieve the XMM register values. XMM sampling will be enabled in a subsequent patch that sets PERF_PMU_CAP_SIMD_REGS. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-14-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 84 +++++++++++++++++++++++---- arch/x86/events/intel/ds.c | 6 +- arch/x86/events/perf_event.h | 36 ++++++++++++ arch/x86/include/asm/perf_event.h | 1 + arch/x86/include/uapi/asm/perf_regs.h | 15 +++++ arch/x86/kernel/perf_regs.c | 75 +++++++++++++++++++++++- include/linux/perf_event.h | 1 + kernel/events/core.c | 2 +- 8 files changed, 204 insertions(+), 16 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 20140643e21c..f85163b95cfc 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -632,6 +632,34 @@ int x86_pmu_max_precise(struct pmu *pmu) return precise; } +static int pebs_simd_regs_validate(struct perf_event *event) +{ + u64 caps = hybrid(event->pmu, arch_pebs_cap).caps; + + if (event_needs_xmm(event) && + !x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline) + return -EINVAL; + if (event_needs_xmm(event) && + x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) + return -EINVAL; + + return 0; +} + +static int event_simd_regs_validate(struct perf_event *event) +{ + if (!get_ext_regs_buf(raw_smp_processor_id())) + return -ENOMEM; + /* sample_simd_regs_enabled repurposes legacy XMM reg-mask slots. */ + if (event_has_extended_regs(event)) + return -EINVAL; + if (event_needs_xmm(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) + return -EINVAL; + + return 0; +} + int x86_pmu_hw_config(struct perf_event *event) { if (event->attr.precise_ip) { @@ -703,11 +731,21 @@ int x86_pmu_hw_config(struct perf_event *event) } if (event->attr.sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) { - /* - * Besides the general purpose registers, XMM registers may - * be collected as well. - */ - if (event_has_extended_regs(event)) { + int ret; + + if (event->attr.sample_simd_regs_enabled) { + if (!(event->pmu->capabilities & PERF_PMU_CAP_SIMD_REGS)) + return -EINVAL; + + if (event->attr.precise_ip) { + ret = pebs_simd_regs_validate(event); + if (ret) + return ret; + } + ret = event_simd_regs_validate(event); + if (ret) + return ret; + } else if (event_has_extended_regs(event)) { if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) return -EINVAL; @@ -721,6 +759,8 @@ int x86_pmu_hw_config(struct perf_event *event) } if (!get_ext_regs_buf(raw_smp_processor_id())) return -ENOMEM; + if (!(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) + return -EINVAL; } } @@ -1777,6 +1817,7 @@ void x86_pmu_clear_perf_regs(struct pt_regs *regs) { struct x86_perf_regs *perf_regs = container_of(regs, struct x86_perf_regs, regs); + perf_regs->abi = PERF_SAMPLE_REGS_ABI_NONE; perf_regs->xmm_regs = NULL; } @@ -1797,14 +1838,15 @@ static void update_perf_regs(struct x86_perf_regs *perf_regs, /* * The x86 specific variant of perf_sample_regs_intr(). - * It would be extended to add more SIMD registers sampling support - * in later patches. + * Update data->regs_intr fields for extended registers (e.g., SIMD). */ static void x86_pmu_update_regs_intr(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs, bool exclude_kernel) { + struct x86_perf_regs *perf_regs; + if (exclude_kernel && !user_mode(regs)) { data->regs_intr.regs = NULL; data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE; @@ -1817,6 +1859,14 @@ static void x86_pmu_update_regs_intr(struct perf_event *event, if (data->regs_intr.regs) { data->dyn_size += hweight64(event->attr.sample_regs_intr) * sizeof(u64); + if (event_has_simd_regs(event)) { + data->dyn_size += perf_update_xregs_size(event, true); + data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } + + perf_regs = container_of(data->regs_intr.regs, + struct x86_perf_regs, regs); + perf_regs->abi = data->regs_intr.abi; } /* @@ -1878,8 +1928,15 @@ static void x86_pmu_update_regs_user(struct perf_event *event, } data->dyn_size += sizeof(u64); - if (data->regs_user.regs) + if (data->regs_user.regs) { data->dyn_size += hweight64(attr->sample_regs_user) * sizeof(u64); + if (event_has_simd_regs(event)) { + data->dyn_size += perf_update_xregs_size(event, false); + data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } + + x86_regs_user->abi = data->regs_user.abi; + } /* * Set PERF_SAMPLE_REGS_USER to bypass perf_sample_regs_user() call @@ -1954,7 +2011,7 @@ static void x86_pmu_sample_xregs(struct perf_event *event, return; if ((sample_type & PERF_SAMPLE_REGS_INTR) && data->regs_intr.regs) { - if (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK) + if (__event_needs_xmm(event, PERF_SAMPLE_REGS_INTR)) intr_mask |= XFEATURE_MASK_SSE; intr_mask &= x86_pmu.ext_regs_mask; @@ -1962,7 +2019,7 @@ static void x86_pmu_sample_xregs(struct perf_event *event, } if ((sample_type & PERF_SAMPLE_REGS_USER) && data->regs_user.regs) { - if (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) + if (__event_needs_xmm(event, PERF_SAMPLE_REGS_USER)) user_mask |= XFEATURE_MASK_SSE; user_mask &= x86_pmu.ext_regs_mask; @@ -1995,7 +2052,12 @@ void x86_pmu_update_perf_regs(struct perf_event *event, { u64 sample_type = event->attr.sample_type; - if (!event_has_extended_regs(event)) + if (!(sample_type & + (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))) + return; + + if (!event_needs_xmm(event) && + !event_has_simd_regs(event)) return; if (sample_type & PERF_SAMPLE_REGS_INTR) { diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index d216234803b8..84caae35b86e 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1733,8 +1733,10 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) if (gprs || (attr->precise_ip < 2) || tsx_weight) pebs_data_cfg |= PEBS_DATACFG_GP; - if (event_has_extended_regs(event)) - pebs_data_cfg |= PEBS_DATACFG_XMMS; + if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) { + if (event_needs_xmm(event)) + pebs_data_cfg |= PEBS_DATACFG_XMMS; + } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { /* diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index f1ab72094bac..57a8263bd7bf 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -147,6 +147,42 @@ static inline bool is_acr_self_reload_event(struct perf_event *event) return test_bit(hwc->idx, (unsigned long *)&hwc->config1); } +static inline bool __event_needs_xmm(struct perf_event *event, u64 sample_type) +{ + if (event->attr.sample_simd_regs_enabled) { + if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_XMM_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_simd_vec_reg_user > 0)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_simd_vec_reg_intr > 0)) + return true; + } else { + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) + return true; + } + + return false; +} + +static inline bool event_needs_xmm(struct perf_event *event) +{ + return __event_needs_xmm(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + struct amd_nb { int nb_id; /* NorthBridge id */ int refcnt; /* reference count */ diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 619e0ae915e1..a2b2123d008e 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -728,6 +728,7 @@ extern void perf_events_lapic_init(void); struct pt_regs; struct x86_perf_regs { struct pt_regs regs; + u64 abi; union { u64 *xmm_regs; u32 *xmm_space; /* for xsaves */ diff --git a/arch/x86/include/uapi/asm/perf_regs.h b/arch/x86/include/uapi/asm/perf_regs.h index 7c9d2bb3833b..edb35408e4cc 100644 --- a/arch/x86/include/uapi/asm/perf_regs.h +++ b/arch/x86/include/uapi/asm/perf_regs.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_PERF_REGS_H #define _ASM_X86_PERF_REGS_H +#include + enum perf_event_x86_regs { PERF_REG_X86_AX, PERF_REG_X86_BX, @@ -55,4 +57,17 @@ enum perf_event_x86_regs { #define PERF_REG_EXTENDED_MASK (~((1ULL << PERF_REG_X86_XMM0) - 1)) +enum { + PERF_X86_SIMD_XMM_REGS = 16, + PERF_X86_SIMD_VEC_REGS_MAX = PERF_X86_SIMD_XMM_REGS, +}; + +#define PERF_X86_SIMD_VEC_MASK __GENMASK_ULL(PERF_X86_SIMD_VEC_REGS_MAX - 1, 0) + +enum { + /* 1 qword = 8 bytes */ + PERF_X86_XMM_QWORDS = 2, + PERF_X86_SIMD_QWORDS_MAX = PERF_X86_XMM_QWORDS, +}; + #endif /* _ASM_X86_PERF_REGS_H */ diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index 81204cb7f723..bccf0fc27511 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -63,6 +63,9 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) { perf_regs = container_of(regs, struct x86_perf_regs, regs); + /* SIMD registers are moved to dedicated sample_simd_vec_reg */ + if (perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) + return 0; if (!perf_regs->xmm_regs) return 0; return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0]; @@ -74,6 +77,72 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return regs_get_register(regs, pt_regs_offset[idx]); } +u64 perf_simd_reg_value(struct pt_regs *regs, int idx, + u16 qwords_idx, bool pred) +{ + struct x86_perf_regs *perf_regs = + container_of(regs, struct x86_perf_regs, regs); + + if (!(perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD)) + return 0; + + if (pred) + return 0; + + if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_VEC_REGS_MAX || + qwords_idx >= PERF_X86_SIMD_QWORDS_MAX)) + return 0; + + if (qwords_idx < PERF_X86_XMM_QWORDS) { + if (!perf_regs->xmm_regs) + return 0; + return perf_regs->xmm_regs[idx * PERF_X86_XMM_QWORDS + + qwords_idx]; + } + + return 0; +} + +int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, + u16 pred_qwords, u32 pred_mask) +{ + unsigned long mask; + u64 size; + + if (!vec_qwords && !pred_qwords) { + if (vec_mask || pred_mask) + return -EINVAL; + } + + if (vec_qwords) { + if (vec_qwords != PERF_X86_XMM_QWORDS) + return -EINVAL; + if (vec_mask & ~PERF_X86_SIMD_VEC_MASK) + return -EINVAL; + /* Only full-register sampling is allowed. */ + mask = vec_mask; + if (vec_qwords == PERF_X86_XMM_QWORDS && mask && + !bitmap_full(&mask, PERF_X86_SIMD_XMM_REGS)) + return -EINVAL; + } + + /* PRED registers are not supported yet. */ + if (pred_qwords) + return -EINVAL; + + size = sizeof(u64) * 4; + size += (hweight64(vec_mask) * vec_qwords + + hweight32(pred_mask) * pred_qwords) * sizeof(u64); + /* + * INTR_REGS and USR_REGS could be sampled simultaneously, + * so roughly restrict the size to half of U16_MAX. + */ + if (size >= U16_MAX / 2) + return -EINVAL; + + return 0; +} + #define PERF_REG_X86_RESERVED (((1ULL << PERF_REG_X86_XMM0) - 1) & \ ~((1ULL << PERF_REG_X86_MAX) - 1)) @@ -89,7 +158,8 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) int perf_reg_validate(u64 mask) { - if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED))) + /* The mask could be 0 if only the SIMD registers are interested */ + if (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)) return -EINVAL; return 0; @@ -108,7 +178,8 @@ u64 perf_reg_abi(struct task_struct *task) int perf_reg_validate(u64 mask) { - if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED))) + /* The mask could be 0 if only the SIMD registers are interested */ + if (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)) return -EINVAL; return 0; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index a2e7eae268cd..7cc2345052e0 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1485,6 +1485,7 @@ static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *b br->reserved = 0; } +extern u64 perf_update_xregs_size(struct perf_event *event, bool intr); extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data, diff --git a/kernel/events/core.c b/kernel/events/core.c index 3a87c6adcded..2f84f5018369 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8720,7 +8720,7 @@ static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d) return d * !!(flags & s); } -static u64 perf_update_xregs_size(struct perf_event *event, bool intr) +u64 perf_update_xregs_size(struct perf_event *event, bool intr) { u16 pred_qwords = event->attr.sample_simd_pred_reg_qwords; u16 vec_qwords = event->attr.sample_simd_vec_reg_qwords; From b76210d3214730fd5d79387684ae0c56dd580229 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:22 +0800 Subject: [PATCH 14/34] perf/x86: Support YMM sampling using sample_simd_vec_reg_* fields Support sampling of YMM registers via the sample_simd_vec_reg_* fields. Each YMM register consists of 4 u64 words, assembled from two halves: XMM (the lower 2 u64 words) and YMMH (the upper 2 u64 words). Although both XMM and YMMH data can be retrieved with a single xsaves instruction, they are stored in separate locations. The perf_simd_reg_value() function is responsible for assembling these halves into a complete YMM register for output to userspace. Additionally, sample_simd_vec_reg_qwords should be set to 4 to indicate YMM sampling. YMM sampling will be enabled in a subsequent patch that sets PERF_PMU_CAP_SIMD_REGS. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-15-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 29 +++++++++++++++++++++------ arch/x86/events/perf_event.h | 26 ++++++++++++++++++++++++ arch/x86/include/asm/perf_event.h | 4 ++++ arch/x86/include/uapi/asm/perf_regs.h | 6 ++++-- arch/x86/kernel/perf_regs.c | 13 +++++++++++- 5 files changed, 69 insertions(+), 9 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index f85163b95cfc..3039a311e733 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -642,6 +642,9 @@ static int pebs_simd_regs_validate(struct perf_event *event) if (event_needs_xmm(event) && x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) return -EINVAL; + /* PEBS does not support YMM registers sampling yet. */ + if (event_needs_ymm(event)) + return -EINVAL; return 0; } @@ -656,6 +659,9 @@ static int event_simd_regs_validate(struct perf_event *event) if (event_needs_xmm(event) && !(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) return -EINVAL; + if (event_needs_ymm(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_YMM)) + return -EINVAL; return 0; } @@ -1819,6 +1825,7 @@ void x86_pmu_clear_perf_regs(struct pt_regs *regs) perf_regs->abi = PERF_SAMPLE_REGS_ABI_NONE; perf_regs->xmm_regs = NULL; + perf_regs->ymmh_regs = NULL; } static void update_perf_regs(struct x86_perf_regs *perf_regs, @@ -1834,6 +1841,8 @@ static void update_perf_regs(struct x86_perf_regs *perf_regs, if (mask & XFEATURE_MASK_SSE) perf_regs->xmm_space = xsave->i387.xmm_space; + if (mask & XFEATURE_MASK_YMM) + perf_regs->ymmh = get_xsave_addr(xsave, XFEATURE_YMM); } /* @@ -1996,6 +2005,18 @@ static inline u64 x86_pmu_update_user_xregs(struct perf_sample_data *data, return user_mask; } +static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type) +{ + u64 mask = 0; + + if (__event_needs_xmm(event, sample_type)) + mask |= XFEATURE_MASK_SSE; + if (__event_needs_ymm(event, sample_type)) + mask |= XFEATURE_MASK_YMM; + + return mask; +} + static void x86_pmu_sample_xregs(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs, @@ -2011,17 +2032,13 @@ static void x86_pmu_sample_xregs(struct perf_event *event, return; if ((sample_type & PERF_SAMPLE_REGS_INTR) && data->regs_intr.regs) { - if (__event_needs_xmm(event, PERF_SAMPLE_REGS_INTR)) - intr_mask |= XFEATURE_MASK_SSE; - + intr_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_INTR); intr_mask &= x86_pmu.ext_regs_mask; intr_mask = from_pebs ? 0 : intr_mask; } if ((sample_type & PERF_SAMPLE_REGS_USER) && data->regs_user.regs) { - if (__event_needs_xmm(event, PERF_SAMPLE_REGS_USER)) - user_mask |= XFEATURE_MASK_SSE; - + user_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_USER); user_mask &= x86_pmu.ext_regs_mask; user_mask = x86_pmu_update_user_xregs(data, regs, user_mask, from_pebs); diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 57a8263bd7bf..423835e4a864 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -183,6 +183,32 @@ static inline bool event_needs_xmm(struct perf_event *event) PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); } +static inline bool __event_needs_ymm(struct perf_event *event, u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_YMM_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_simd_vec_reg_user > 0)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_simd_vec_reg_intr > 0)) + return true; + + return false; +} + +static inline bool event_needs_ymm(struct perf_event *event) +{ + return __event_needs_ymm(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + struct amd_nb { int nb_id; /* NorthBridge id */ int refcnt; /* reference count */ diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index a2b2123d008e..da77845e1f02 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -733,6 +733,10 @@ struct x86_perf_regs { u64 *xmm_regs; u32 *xmm_space; /* for xsaves */ }; + union { + u64 *ymmh_regs; + struct ymmh_struct *ymmh; + }; }; extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs); diff --git a/arch/x86/include/uapi/asm/perf_regs.h b/arch/x86/include/uapi/asm/perf_regs.h index edb35408e4cc..d544f6d79871 100644 --- a/arch/x86/include/uapi/asm/perf_regs.h +++ b/arch/x86/include/uapi/asm/perf_regs.h @@ -59,7 +59,8 @@ enum perf_event_x86_regs { enum { PERF_X86_SIMD_XMM_REGS = 16, - PERF_X86_SIMD_VEC_REGS_MAX = PERF_X86_SIMD_XMM_REGS, + PERF_X86_SIMD_YMM_REGS = 16, + PERF_X86_SIMD_VEC_REGS_MAX = PERF_X86_SIMD_YMM_REGS, }; #define PERF_X86_SIMD_VEC_MASK __GENMASK_ULL(PERF_X86_SIMD_VEC_REGS_MAX - 1, 0) @@ -67,7 +68,8 @@ enum { enum { /* 1 qword = 8 bytes */ PERF_X86_XMM_QWORDS = 2, - PERF_X86_SIMD_QWORDS_MAX = PERF_X86_XMM_QWORDS, + PERF_X86_YMM_QWORDS = 4, + PERF_X86_SIMD_QWORDS_MAX = PERF_X86_YMM_QWORDS, }; #endif /* _ASM_X86_PERF_REGS_H */ diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index bccf0fc27511..0076974498ee 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -77,6 +77,8 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return regs_get_register(regs, pt_regs_offset[idx]); } +#define PERF_X86_YMMH_QWORDS (PERF_X86_YMM_QWORDS / 2) + u64 perf_simd_reg_value(struct pt_regs *regs, int idx, u16 qwords_idx, bool pred) { @@ -98,6 +100,11 @@ u64 perf_simd_reg_value(struct pt_regs *regs, int idx, return 0; return perf_regs->xmm_regs[idx * PERF_X86_XMM_QWORDS + qwords_idx]; + } else if (qwords_idx < PERF_X86_YMM_QWORDS) { + if (!perf_regs->ymmh_regs) + return 0; + return perf_regs->ymmh_regs[idx * PERF_X86_YMMH_QWORDS + + qwords_idx - PERF_X86_XMM_QWORDS]; } return 0; @@ -115,7 +122,8 @@ int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, } if (vec_qwords) { - if (vec_qwords != PERF_X86_XMM_QWORDS) + if (vec_qwords != PERF_X86_XMM_QWORDS && + vec_qwords != PERF_X86_YMM_QWORDS) return -EINVAL; if (vec_mask & ~PERF_X86_SIMD_VEC_MASK) return -EINVAL; @@ -124,6 +132,9 @@ int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, if (vec_qwords == PERF_X86_XMM_QWORDS && mask && !bitmap_full(&mask, PERF_X86_SIMD_XMM_REGS)) return -EINVAL; + if (vec_qwords == PERF_X86_YMM_QWORDS && mask && + !bitmap_full(&mask, PERF_X86_SIMD_YMM_REGS)) + return -EINVAL; } /* PRED registers are not supported yet. */ From 3807f6996a0bd9515ff69f3040d6e47f7b20e7a5 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:23 +0800 Subject: [PATCH 15/34] perf/x86: Support ZMM sampling using sample_simd_vec_reg_* fields Support sampling of ZMM registers via the sample_simd_vec_reg_* fields. Each ZMM register consists of 8 u64 words. Current x86 hardware supports up to 32 ZMM registers. For ZMM registers from ZMM0 to ZMM15, they are assembled from three parts: XMM (the lower 2 u64 words), YMMH (the middle 2 u64 words), and ZMMH (the upper 4 u64 words). The perf_simd_reg_value() function is responsible for assembling these three parts into a complete ZMM register for output to userspace. For ZMM registers ZMM16 to ZMM31, each register can be read as a whole and directly outputted to userspace. Additionally, sample_simd_vec_reg_qwords should be set to 8 to indicate ZMM sampling. ZMM sampling will be enabled in a subsequent patch that sets PERF_PMU_CAP_SIMD_REGS. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-16-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 22 ++++++++++- arch/x86/events/perf_event.h | 54 +++++++++++++++++++++++++++ arch/x86/include/asm/perf_event.h | 8 ++++ arch/x86/include/uapi/asm/perf_regs.h | 8 +++- arch/x86/kernel/perf_regs.c | 19 +++++++++- 5 files changed, 106 insertions(+), 5 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 3039a311e733..b0e3f1b9fa19 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -642,8 +642,10 @@ static int pebs_simd_regs_validate(struct perf_event *event) if (event_needs_xmm(event) && x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) return -EINVAL; - /* PEBS does not support YMM registers sampling yet. */ - if (event_needs_ymm(event)) + /* PEBS does not support YMM/ZMM registers sampling yet. */ + if (event_needs_ymm(event) || + event_needs_low16_zmm(event) || + event_needs_high16_zmm(event)) return -EINVAL; return 0; @@ -662,6 +664,12 @@ static int event_simd_regs_validate(struct perf_event *event) if (event_needs_ymm(event) && !(x86_pmu.ext_regs_mask & XFEATURE_MASK_YMM)) return -EINVAL; + if (event_needs_low16_zmm(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_ZMM_Hi256)) + return -EINVAL; + if (event_needs_high16_zmm(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_Hi16_ZMM)) + return -EINVAL; return 0; } @@ -1826,6 +1834,8 @@ void x86_pmu_clear_perf_regs(struct pt_regs *regs) perf_regs->abi = PERF_SAMPLE_REGS_ABI_NONE; perf_regs->xmm_regs = NULL; perf_regs->ymmh_regs = NULL; + perf_regs->zmmh_regs = NULL; + perf_regs->h16zmm_regs = NULL; } static void update_perf_regs(struct x86_perf_regs *perf_regs, @@ -1843,6 +1853,10 @@ static void update_perf_regs(struct x86_perf_regs *perf_regs, perf_regs->xmm_space = xsave->i387.xmm_space; if (mask & XFEATURE_MASK_YMM) perf_regs->ymmh = get_xsave_addr(xsave, XFEATURE_YMM); + if (mask & XFEATURE_MASK_ZMM_Hi256) + perf_regs->zmmh = get_xsave_addr(xsave, XFEATURE_ZMM_Hi256); + if (mask & XFEATURE_MASK_Hi16_ZMM) + perf_regs->h16zmm = get_xsave_addr(xsave, XFEATURE_Hi16_ZMM); } /* @@ -2013,6 +2027,10 @@ static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type) mask |= XFEATURE_MASK_SSE; if (__event_needs_ymm(event, sample_type)) mask |= XFEATURE_MASK_YMM; + if (__event_needs_low16_zmm(event, sample_type)) + mask |= XFEATURE_MASK_ZMM_Hi256; + if (__event_needs_high16_zmm(event, sample_type)) + mask |= XFEATURE_MASK_Hi16_ZMM; return mask; } diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 423835e4a864..8ee69ec6af9e 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -209,6 +209,60 @@ static inline bool event_needs_ymm(struct perf_event *event) PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); } +static inline bool __event_needs_low16_zmm(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_ZMM_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_simd_vec_reg_user > 0)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_simd_vec_reg_intr > 0)) + return true; + + return false; +} + +static inline bool event_needs_low16_zmm(struct perf_event *event) +{ + return __event_needs_low16_zmm(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + +static inline bool __event_needs_high16_zmm(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_ZMM_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (fls64(event->attr.sample_simd_vec_reg_user) > PERF_X86_H16ZMM_BASE)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (fls64(event->attr.sample_simd_vec_reg_intr) > PERF_X86_H16ZMM_BASE)) + return true; + + return false; +} + +static inline bool event_needs_high16_zmm(struct perf_event *event) +{ + return __event_needs_high16_zmm(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + struct amd_nb { int nb_id; /* NorthBridge id */ int refcnt; /* reference count */ diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index da77845e1f02..75394c4e8bc3 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -737,6 +737,14 @@ struct x86_perf_regs { u64 *ymmh_regs; struct ymmh_struct *ymmh; }; + union { + u64 *zmmh_regs; + struct avx_512_zmm_uppers_state *zmmh; + }; + union { + u64 *h16zmm_regs; + struct avx_512_hi16_state *h16zmm; + }; }; extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs); diff --git a/arch/x86/include/uapi/asm/perf_regs.h b/arch/x86/include/uapi/asm/perf_regs.h index d544f6d79871..b88d0b6822fd 100644 --- a/arch/x86/include/uapi/asm/perf_regs.h +++ b/arch/x86/include/uapi/asm/perf_regs.h @@ -60,16 +60,20 @@ enum perf_event_x86_regs { enum { PERF_X86_SIMD_XMM_REGS = 16, PERF_X86_SIMD_YMM_REGS = 16, - PERF_X86_SIMD_VEC_REGS_MAX = PERF_X86_SIMD_YMM_REGS, + PERF_X86_SIMD_ZMM_REGS = 32, + PERF_X86_SIMD_VEC_REGS_MAX = PERF_X86_SIMD_ZMM_REGS, }; #define PERF_X86_SIMD_VEC_MASK __GENMASK_ULL(PERF_X86_SIMD_VEC_REGS_MAX - 1, 0) +#define PERF_X86_H16ZMM_BASE 16 + enum { /* 1 qword = 8 bytes */ PERF_X86_XMM_QWORDS = 2, PERF_X86_YMM_QWORDS = 4, - PERF_X86_SIMD_QWORDS_MAX = PERF_X86_YMM_QWORDS, + PERF_X86_ZMM_QWORDS = 8, + PERF_X86_SIMD_QWORDS_MAX = PERF_X86_ZMM_QWORDS, }; #endif /* _ASM_X86_PERF_REGS_H */ diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index 0076974498ee..93370d465786 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -78,6 +78,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) } #define PERF_X86_YMMH_QWORDS (PERF_X86_YMM_QWORDS / 2) +#define PERF_X86_ZMMH_QWORDS (PERF_X86_ZMM_QWORDS / 2) u64 perf_simd_reg_value(struct pt_regs *regs, int idx, u16 qwords_idx, bool pred) @@ -95,6 +96,13 @@ u64 perf_simd_reg_value(struct pt_regs *regs, int idx, qwords_idx >= PERF_X86_SIMD_QWORDS_MAX)) return 0; + if (idx >= PERF_X86_H16ZMM_BASE) { + if (!perf_regs->h16zmm_regs) + return 0; + return perf_regs->h16zmm_regs[(idx - PERF_X86_H16ZMM_BASE) * + PERF_X86_ZMM_QWORDS + qwords_idx]; + } + if (qwords_idx < PERF_X86_XMM_QWORDS) { if (!perf_regs->xmm_regs) return 0; @@ -105,6 +113,11 @@ u64 perf_simd_reg_value(struct pt_regs *regs, int idx, return 0; return perf_regs->ymmh_regs[idx * PERF_X86_YMMH_QWORDS + qwords_idx - PERF_X86_XMM_QWORDS]; + } else if (qwords_idx < PERF_X86_ZMM_QWORDS) { + if (!perf_regs->zmmh_regs) + return 0; + return perf_regs->zmmh_regs[idx * PERF_X86_ZMMH_QWORDS + + qwords_idx - PERF_X86_YMM_QWORDS]; } return 0; @@ -123,7 +136,8 @@ int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, if (vec_qwords) { if (vec_qwords != PERF_X86_XMM_QWORDS && - vec_qwords != PERF_X86_YMM_QWORDS) + vec_qwords != PERF_X86_YMM_QWORDS && + vec_qwords != PERF_X86_ZMM_QWORDS) return -EINVAL; if (vec_mask & ~PERF_X86_SIMD_VEC_MASK) return -EINVAL; @@ -135,6 +149,9 @@ int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, if (vec_qwords == PERF_X86_YMM_QWORDS && mask && !bitmap_full(&mask, PERF_X86_SIMD_YMM_REGS)) return -EINVAL; + if (vec_qwords == PERF_X86_ZMM_QWORDS && mask && + !bitmap_full(&mask, PERF_X86_SIMD_ZMM_REGS)) + return -EINVAL; } /* PRED registers are not supported yet. */ From 74d55a827e31bc8a75fbc530951c6266d043ecc0 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:24 +0800 Subject: [PATCH 16/34] perf/x86: Support OPMASK sampling using sample_simd_pred_reg_* fields Support sampling of OPMASK registers via the sample_simd_pred_reg_* fields. Each OPMASK register consists of 1 u64 word. Current x86 hardware supports 8 OPMASK registers. The perf_simd_reg_value() function is responsible for outputting OPMASK value to userspace. Additionally, sample_simd_pred_reg_qwords should be set to 1 to indicate OPMASK sampling. OPMASK sampling will be enabled in a subsequent patch that sets PERF_PMU_CAP_SIMD_REGS. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-17-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 13 +++++++++++-- arch/x86/events/perf_event.h | 27 +++++++++++++++++++++++++++ arch/x86/include/asm/perf_event.h | 4 ++++ arch/x86/include/uapi/asm/perf_regs.h | 5 +++++ arch/x86/kernel/perf_regs.c | 24 +++++++++++++++++++----- 5 files changed, 66 insertions(+), 7 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index b0e3f1b9fa19..9bfb1810a9fe 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -642,10 +642,11 @@ static int pebs_simd_regs_validate(struct perf_event *event) if (event_needs_xmm(event) && x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) return -EINVAL; - /* PEBS does not support YMM/ZMM registers sampling yet. */ + /* PEBS does not support YMM/ZMM/OPMASK registers sampling yet. */ if (event_needs_ymm(event) || event_needs_low16_zmm(event) || - event_needs_high16_zmm(event)) + event_needs_high16_zmm(event) || + event_needs_opmask(event)) return -EINVAL; return 0; @@ -670,6 +671,9 @@ static int event_simd_regs_validate(struct perf_event *event) if (event_needs_high16_zmm(event) && !(x86_pmu.ext_regs_mask & XFEATURE_MASK_Hi16_ZMM)) return -EINVAL; + if (event_needs_opmask(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_OPMASK)) + return -EINVAL; return 0; } @@ -1836,6 +1840,7 @@ void x86_pmu_clear_perf_regs(struct pt_regs *regs) perf_regs->ymmh_regs = NULL; perf_regs->zmmh_regs = NULL; perf_regs->h16zmm_regs = NULL; + perf_regs->opmask_regs = NULL; } static void update_perf_regs(struct x86_perf_regs *perf_regs, @@ -1857,6 +1862,8 @@ static void update_perf_regs(struct x86_perf_regs *perf_regs, perf_regs->zmmh = get_xsave_addr(xsave, XFEATURE_ZMM_Hi256); if (mask & XFEATURE_MASK_Hi16_ZMM) perf_regs->h16zmm = get_xsave_addr(xsave, XFEATURE_Hi16_ZMM); + if (mask & XFEATURE_MASK_OPMASK) + perf_regs->opmask = get_xsave_addr(xsave, XFEATURE_OPMASK); } /* @@ -2031,6 +2038,8 @@ static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type) mask |= XFEATURE_MASK_ZMM_Hi256; if (__event_needs_high16_zmm(event, sample_type)) mask |= XFEATURE_MASK_Hi16_ZMM; + if (__event_needs_opmask(event, sample_type)) + mask |= XFEATURE_MASK_OPMASK; return mask; } diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 8ee69ec6af9e..8a74b7d2c130 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -263,6 +263,33 @@ static inline bool event_needs_high16_zmm(struct perf_event *event) PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); } +static inline bool __event_needs_opmask(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + if (event->attr.sample_simd_pred_reg_qwords != PERF_X86_OPMASK_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_simd_pred_reg_user > 0)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_simd_pred_reg_intr > 0)) + return true; + + return false; +} + +static inline bool event_needs_opmask(struct perf_event *event) +{ + return __event_needs_opmask(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + struct amd_nb { int nb_id; /* NorthBridge id */ int refcnt; /* reference count */ diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 75394c4e8bc3..49112e097e99 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -745,6 +745,10 @@ struct x86_perf_regs { u64 *h16zmm_regs; struct avx_512_hi16_state *h16zmm; }; + union { + u64 *opmask_regs; + struct avx_512_opmask_state *opmask; + }; }; extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs); diff --git a/arch/x86/include/uapi/asm/perf_regs.h b/arch/x86/include/uapi/asm/perf_regs.h index b88d0b6822fd..61aec60623f1 100644 --- a/arch/x86/include/uapi/asm/perf_regs.h +++ b/arch/x86/include/uapi/asm/perf_regs.h @@ -62,14 +62,19 @@ enum { PERF_X86_SIMD_YMM_REGS = 16, PERF_X86_SIMD_ZMM_REGS = 32, PERF_X86_SIMD_VEC_REGS_MAX = PERF_X86_SIMD_ZMM_REGS, + + PERF_X86_SIMD_OPMASK_REGS = 8, + PERF_X86_SIMD_PRED_REGS_MAX = PERF_X86_SIMD_OPMASK_REGS, }; +#define PERF_X86_SIMD_PRED_MASK __GENMASK(PERF_X86_SIMD_PRED_REGS_MAX - 1, 0) #define PERF_X86_SIMD_VEC_MASK __GENMASK_ULL(PERF_X86_SIMD_VEC_REGS_MAX - 1, 0) #define PERF_X86_H16ZMM_BASE 16 enum { /* 1 qword = 8 bytes */ + PERF_X86_OPMASK_QWORDS = 1, PERF_X86_XMM_QWORDS = 2, PERF_X86_YMM_QWORDS = 4, PERF_X86_ZMM_QWORDS = 8, diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index 93370d465786..87297da5e44b 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -89,8 +89,14 @@ u64 perf_simd_reg_value(struct pt_regs *regs, int idx, if (!(perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD)) return 0; - if (pred) - return 0; + if (pred) { + if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_PRED_REGS_MAX || + qwords_idx >= PERF_X86_OPMASK_QWORDS)) + return 0; + if (!perf_regs->opmask_regs) + return 0; + return perf_regs->opmask_regs[idx]; + } if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_VEC_REGS_MAX || qwords_idx >= PERF_X86_SIMD_QWORDS_MAX)) @@ -154,9 +160,17 @@ int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, return -EINVAL; } - /* PRED registers are not supported yet. */ - if (pred_qwords) - return -EINVAL; + if (pred_qwords) { + if (pred_qwords != PERF_X86_OPMASK_QWORDS) + return -EINVAL; + if (pred_mask & ~PERF_X86_SIMD_PRED_MASK) + return -EINVAL; + /* Only full-register sampling is allowed. */ + mask = pred_mask; + if (pred_qwords == PERF_X86_OPMASK_QWORDS && mask && + !bitmap_full(&mask, PERF_X86_SIMD_OPMASK_REGS)) + return -EINVAL; + } size = sizeof(u64) * 4; size += (hweight64(vec_mask) * vec_qwords + From a2c64c74c02957ca11975b5355e70210f7b934ad Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:25 +0800 Subject: [PATCH 17/34] perf: Enhance perf_reg_validate() with simd_enabled argument The upcoming patch will support x86 APX eGPRs sampling by using the reclaimed XMM register space to represent eGPRs in sample_regs_* fields. To differentiate between XMM and eGPRs in sample_regs_* fields, an additional argument, simd_enabled, is introduced to the perf_reg_validate() helper. If simd_enabled is set to 1, it indicates that eGPRs are represented in sample_regs_* fields for the x86 platform; otherwise, XMM registers are represented. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-18-dapeng1.mi@linux.intel.com --- arch/arm/kernel/perf_regs.c | 2 +- arch/arm64/kernel/perf_regs.c | 2 +- arch/csky/kernel/perf_regs.c | 2 +- arch/loongarch/kernel/perf_regs.c | 2 +- arch/mips/kernel/perf_regs.c | 2 +- arch/parisc/kernel/perf_regs.c | 2 +- arch/powerpc/perf/perf_regs.c | 2 +- arch/riscv/kernel/perf_regs.c | 2 +- arch/s390/kernel/perf_regs.c | 2 +- arch/x86/kernel/perf_regs.c | 4 ++-- include/linux/perf_regs.h | 2 +- kernel/events/core.c | 8 +++++--- 12 files changed, 17 insertions(+), 15 deletions(-) diff --git a/arch/arm/kernel/perf_regs.c b/arch/arm/kernel/perf_regs.c index d575a4c3ca56..838d701adf4d 100644 --- a/arch/arm/kernel/perf_regs.c +++ b/arch/arm/kernel/perf_regs.c @@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_ARM_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c index 70e2f13f587f..71a3e0238de4 100644 --- a/arch/arm64/kernel/perf_regs.c +++ b/arch/arm64/kernel/perf_regs.c @@ -77,7 +77,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_ARM64_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { u64 reserved_mask = REG_RESERVED; diff --git a/arch/csky/kernel/perf_regs.c b/arch/csky/kernel/perf_regs.c index 94601f37b596..c932a96afc56 100644 --- a/arch/csky/kernel/perf_regs.c +++ b/arch/csky/kernel/perf_regs.c @@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_CSKY_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; diff --git a/arch/loongarch/kernel/perf_regs.c b/arch/loongarch/kernel/perf_regs.c index 8dd604f01745..164514f40ae0 100644 --- a/arch/loongarch/kernel/perf_regs.c +++ b/arch/loongarch/kernel/perf_regs.c @@ -25,7 +25,7 @@ u64 perf_reg_abi(struct task_struct *tsk) } #endif /* CONFIG_32BIT */ -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask) return -EINVAL; diff --git a/arch/mips/kernel/perf_regs.c b/arch/mips/kernel/perf_regs.c index 7736d3c5ebd2..00a5201dbd5d 100644 --- a/arch/mips/kernel/perf_regs.c +++ b/arch/mips/kernel/perf_regs.c @@ -28,7 +28,7 @@ u64 perf_reg_abi(struct task_struct *tsk) } #endif /* CONFIG_32BIT */ -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask) return -EINVAL; diff --git a/arch/parisc/kernel/perf_regs.c b/arch/parisc/kernel/perf_regs.c index b9fe1f2fcb9b..4f21aab5405c 100644 --- a/arch/parisc/kernel/perf_regs.c +++ b/arch/parisc/kernel/perf_regs.c @@ -34,7 +34,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_PARISC_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; diff --git a/arch/powerpc/perf/perf_regs.c b/arch/powerpc/perf/perf_regs.c index 350dccb0143c..a01d8a903640 100644 --- a/arch/powerpc/perf/perf_regs.c +++ b/arch/powerpc/perf/perf_regs.c @@ -125,7 +125,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return regs_get_register(regs, pt_regs_offset[idx]); } -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; diff --git a/arch/riscv/kernel/perf_regs.c b/arch/riscv/kernel/perf_regs.c index 3bba8deababb..1ecc8760b88b 100644 --- a/arch/riscv/kernel/perf_regs.c +++ b/arch/riscv/kernel/perf_regs.c @@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_RISCV_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; diff --git a/arch/s390/kernel/perf_regs.c b/arch/s390/kernel/perf_regs.c index 7b305f1456f8..6496fd23c540 100644 --- a/arch/s390/kernel/perf_regs.c +++ b/arch/s390/kernel/perf_regs.c @@ -34,7 +34,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1UL << PERF_REG_S390_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index 87297da5e44b..494e06501210 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -198,7 +198,7 @@ int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, (1ULL << PERF_REG_X86_R14) | \ (1ULL << PERF_REG_X86_R15)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { /* The mask could be 0 if only the SIMD registers are interested */ if (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)) @@ -218,7 +218,7 @@ u64 perf_reg_abi(struct task_struct *task) (1ULL << PERF_REG_X86_FS) | \ (1ULL << PERF_REG_X86_GS)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { /* The mask could be 0 if only the SIMD registers are interested */ if (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)) diff --git a/include/linux/perf_regs.h b/include/linux/perf_regs.h index 518f28c6a7d4..09dbc2fc3859 100644 --- a/include/linux/perf_regs.h +++ b/include/linux/perf_regs.h @@ -10,7 +10,7 @@ struct perf_regs { }; u64 perf_reg_value(struct pt_regs *regs, int idx); -int perf_reg_validate(u64 mask); +int perf_reg_validate(u64 mask, bool simd_enabled); u64 perf_reg_abi(struct task_struct *task); void perf_get_regs_user(struct perf_regs *regs_user, struct pt_regs *regs); diff --git a/kernel/events/core.c b/kernel/events/core.c index 2f84f5018369..e053d8c8a794 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7823,7 +7823,7 @@ u64 __weak perf_reg_value(struct pt_regs *regs, int idx) return 0; } -int __weak perf_reg_validate(u64 mask) +int __weak perf_reg_validate(u64 mask, bool simd_enabled) { return mask ? -ENOSYS : 0; } @@ -13804,7 +13804,8 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, } if (attr->sample_type & PERF_SAMPLE_REGS_USER) { - ret = perf_reg_validate(attr->sample_regs_user); + ret = perf_reg_validate(attr->sample_regs_user, + attr->sample_simd_regs_enabled); if (ret) return ret; if (attr->sample_simd_regs_enabled) { @@ -13836,7 +13837,8 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, attr->sample_max_stack = sysctl_perf_event_max_stack; if (attr->sample_type & PERF_SAMPLE_REGS_INTR) { - ret = perf_reg_validate(attr->sample_regs_intr); + ret = perf_reg_validate(attr->sample_regs_intr, + attr->sample_simd_regs_enabled); if (ret) return ret; if (attr->sample_simd_regs_enabled) { From 578460d9af8d5816eff59642a8bb11945026b8d1 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:26 +0800 Subject: [PATCH 18/34] perf/x86: Support eGPRs sampling using sample_regs_* fields Support sampling of APX eGPRs (R16 ~ R31) via the sample_regs_* fields. To sample eGPRs, the sample_simd_regs_enabled field must be set. This allows the spare space (reclaimed from the original XMM space) in the sample_regs_* fields to be used for representing eGPRs. The perf_reg_value() function needs to check if the PERF_SAMPLE_REGS_ABI_SIMD flag is set first, and then determine whether to output eGPRs or legacy XMM registers to userspace. The perf_reg_validate() function first checks the simd_enabled argument to determine if the eGPRs bitmap is represented in sample_regs_* fields. It then validates the eGPRs bitmap accordingly. Currently, eGPRs sampling is only supported on the x86_64 architecture, as APX is only available on x86_64 platforms. APX eGPRs sampling will be enabled in a subsequent patch that sets PERF_PMU_CAP_SIMD_REGS. Suggested-by: Peter Zijlstra (Intel) Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-19-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 25 +++++++++++--- arch/x86/events/intel/core.c | 10 ++++-- arch/x86/events/perf_event.h | 25 ++++++++++++++ arch/x86/include/asm/perf_event.h | 4 +++ arch/x86/include/uapi/asm/perf_regs.h | 26 ++++++++++++++ arch/x86/kernel/perf_regs.c | 50 +++++++++++++++++---------- 6 files changed, 115 insertions(+), 25 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 9bfb1810a9fe..9cd79ac5570e 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -642,11 +642,12 @@ static int pebs_simd_regs_validate(struct perf_event *event) if (event_needs_xmm(event) && x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) return -EINVAL; - /* PEBS does not support YMM/ZMM/OPMASK registers sampling yet. */ + /* PEBS does not support YMM/ZMM/OPMASK/eGPR registers sampling yet. */ if (event_needs_ymm(event) || event_needs_low16_zmm(event) || event_needs_high16_zmm(event) || - event_needs_opmask(event)) + event_needs_opmask(event) || + event_needs_egprs(event)) return -EINVAL; return 0; @@ -654,10 +655,21 @@ static int pebs_simd_regs_validate(struct perf_event *event) static int event_simd_regs_validate(struct perf_event *event) { + u64 reserved = ~GENMASK_ULL(PERF_REG_MISC_MAX - 1, 0); + if (!get_ext_regs_buf(raw_smp_processor_id())) return -ENOMEM; - /* sample_simd_regs_enabled repurposes legacy XMM reg-mask slots. */ - if (event_has_extended_regs(event)) + /* + * The XMM space in the perf_event_x86_regs is reclaimed + * for eGPRs and other general registers. + */ + if (((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & reserved)) || + ((event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & reserved))) + return -EINVAL; + if (event_needs_egprs(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_APX)) return -EINVAL; if (event_needs_xmm(event) && !(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) @@ -1841,6 +1853,7 @@ void x86_pmu_clear_perf_regs(struct pt_regs *regs) perf_regs->zmmh_regs = NULL; perf_regs->h16zmm_regs = NULL; perf_regs->opmask_regs = NULL; + perf_regs->egpr_regs = NULL; } static void update_perf_regs(struct x86_perf_regs *perf_regs, @@ -1864,6 +1877,8 @@ static void update_perf_regs(struct x86_perf_regs *perf_regs, perf_regs->h16zmm = get_xsave_addr(xsave, XFEATURE_Hi16_ZMM); if (mask & XFEATURE_MASK_OPMASK) perf_regs->opmask = get_xsave_addr(xsave, XFEATURE_OPMASK); + if (mask & XFEATURE_MASK_APX) + perf_regs->egpr = get_xsave_addr(xsave, XFEATURE_APX); } /* @@ -2040,6 +2055,8 @@ static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type) mask |= XFEATURE_MASK_Hi16_ZMM; if (__event_needs_opmask(event, sample_type)) mask |= XFEATURE_MASK_OPMASK; + if (__event_needs_egprs(event, sample_type)) + mask |= XFEATURE_MASK_APX; return mask; } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 5cb80833c7ed..b2aa0bc7c97e 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4695,15 +4695,19 @@ static void intel_pebs_aliases_skl(struct perf_event *event) static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event) { unsigned long flags = x86_pmu.large_pebs_flags; - u64 gprs_mask = PEBS_GP_REGS | PERF_REG_EXTENDED_MASK; + u64 gprs_mask = event->attr.sample_simd_regs_enabled ? + PEBS_GP_REGS | PERF_X86_EGPRS_MASK : + PEBS_GP_REGS | PERF_REG_EXTENDED_MASK; if (event->attr.use_clockid) flags &= ~PERF_SAMPLE_TIME; if (!event->attr.exclude_kernel) flags &= ~PERF_SAMPLE_REGS_USER; - if (event->attr.sample_regs_user & ~gprs_mask) + if ((event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & ~gprs_mask)) flags &= ~PERF_SAMPLE_REGS_USER; - if (event->attr.sample_regs_intr & ~gprs_mask) + if ((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & ~gprs_mask)) flags &= ~PERF_SAMPLE_REGS_INTR; return flags; } diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 8a74b7d2c130..c052a38b8733 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -290,6 +290,31 @@ static inline bool event_needs_opmask(struct perf_event *event) PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); } +static inline bool __event_needs_egprs(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & PERF_X86_EGPRS_MASK)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & PERF_X86_EGPRS_MASK)) + return true; + + return false; +} + +static inline bool event_needs_egprs(struct perf_event *event) +{ + return __event_needs_egprs(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + struct amd_nb { int nb_id; /* NorthBridge id */ int refcnt; /* reference count */ diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 49112e097e99..bc05f8c17464 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -749,6 +749,10 @@ struct x86_perf_regs { u64 *opmask_regs; struct avx_512_opmask_state *opmask; }; + union { + u64 *egpr_regs; + struct apx_state *egpr; + }; }; extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs); diff --git a/arch/x86/include/uapi/asm/perf_regs.h b/arch/x86/include/uapi/asm/perf_regs.h index 61aec60623f1..977831bd7a9d 100644 --- a/arch/x86/include/uapi/asm/perf_regs.h +++ b/arch/x86/include/uapi/asm/perf_regs.h @@ -29,9 +29,34 @@ enum perf_event_x86_regs { PERF_REG_X86_R13, PERF_REG_X86_R14, PERF_REG_X86_R15, + /* + * The eGPRs and XMM have overlaps. Only one can be used + * at a time. The ABI PERF_SAMPLE_REGS_ABI_SIMD is used to + * distinguish which one is used. If PERF_SAMPLE_REGS_ABI_SIMD + * is set, then eGPRs is used, otherwise, XMM is used. + * + * Extended GPRs (eGPRs) + */ + PERF_REG_X86_R16, + PERF_REG_X86_R17, + PERF_REG_X86_R18, + PERF_REG_X86_R19, + PERF_REG_X86_R20, + PERF_REG_X86_R21, + PERF_REG_X86_R22, + PERF_REG_X86_R23, + PERF_REG_X86_R24, + PERF_REG_X86_R25, + PERF_REG_X86_R26, + PERF_REG_X86_R27, + PERF_REG_X86_R28, + PERF_REG_X86_R29, + PERF_REG_X86_R30, + PERF_REG_X86_R31, /* These are the limits for the GPRs. */ PERF_REG_X86_32_MAX = PERF_REG_X86_GS + 1, PERF_REG_X86_64_MAX = PERF_REG_X86_R15 + 1, + PERF_REG_MISC_MAX = PERF_REG_X86_R31 + 1, /* These all need two bits set because they are 128bit */ PERF_REG_X86_XMM0 = 32, @@ -56,6 +81,7 @@ enum perf_event_x86_regs { }; #define PERF_REG_EXTENDED_MASK (~((1ULL << PERF_REG_X86_XMM0) - 1)) +#define PERF_X86_EGPRS_MASK __GENMASK_ULL(PERF_REG_X86_R31, PERF_REG_X86_R16) enum { PERF_X86_SIMD_XMM_REGS = 16, diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index 494e06501210..7d4233ed859c 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -61,14 +61,24 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) { struct x86_perf_regs *perf_regs; - if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) { + if (idx > PERF_REG_X86_R15) { perf_regs = container_of(regs, struct x86_perf_regs, regs); - /* SIMD registers are moved to dedicated sample_simd_vec_reg */ - if (perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) + if (perf_regs->abi == PERF_SAMPLE_REGS_ABI_NONE) return 0; - if (!perf_regs->xmm_regs) - return 0; - return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0]; + + if (perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) { + if (idx <= PERF_REG_X86_R31) { + if (!perf_regs->egpr_regs) + return 0; + return perf_regs->egpr_regs[idx - PERF_REG_X86_R16]; + } + } else { + if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) { + if (!perf_regs->xmm_regs) + return 0; + return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0]; + } + } } if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset))) @@ -185,23 +195,22 @@ int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, return 0; } -#define PERF_REG_X86_RESERVED (((1ULL << PERF_REG_X86_XMM0) - 1) & \ - ~((1ULL << PERF_REG_X86_MAX) - 1)) +#define PERF_REG_X86_RESERVED (GENMASK_ULL(PERF_REG_X86_XMM0 - 1, PERF_REG_X86_AX) & \ + ~GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_AX)) +#define PERF_REG_X86_EXT_RESERVED (~GENMASK_ULL(PERF_REG_MISC_MAX - 1, PERF_REG_X86_AX)) #ifdef CONFIG_X86_32 -#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_R8) | \ - (1ULL << PERF_REG_X86_R9) | \ - (1ULL << PERF_REG_X86_R10) | \ - (1ULL << PERF_REG_X86_R11) | \ - (1ULL << PERF_REG_X86_R12) | \ - (1ULL << PERF_REG_X86_R13) | \ - (1ULL << PERF_REG_X86_R14) | \ - (1ULL << PERF_REG_X86_R15)) +#define REG_NOSUPPORT GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_R8) int perf_reg_validate(u64 mask, bool simd_enabled) { + if (!simd_enabled && + (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))) + return -EINVAL; + /* The mask could be 0 if only the SIMD registers are interested */ - if (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)) + if (simd_enabled && + (mask & ~GENMASK_ULL(PERF_REG_X86_GS, PERF_REG_X86_AX))) return -EINVAL; return 0; @@ -220,8 +229,13 @@ u64 perf_reg_abi(struct task_struct *task) int perf_reg_validate(u64 mask, bool simd_enabled) { + if (!simd_enabled && + (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))) + return -EINVAL; + /* The mask could be 0 if only the SIMD registers are interested */ - if (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)) + if (simd_enabled && + (mask & (REG_NOSUPPORT | PERF_REG_X86_EXT_RESERVED))) return -EINVAL; return 0; From 098cdd582a9baa7e4bcb26e80a22ebfddc74293a Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:27 +0800 Subject: [PATCH 19/34] perf/x86: Support SSP sampling using sample_regs_* fields Support sampling of CET SSP register via the sample_regs_* fields. To sample SSP, the sample_simd_regs_enabled field must be set. This allows the spare space (reclaimed from the original XMM space) in the sample_regs_* fields to be used for representing SSP. Similar to eGPRs sampling, the perf_reg_value() function needs to check if the PERF_SAMPLE_REGS_ABI_SIMD flag is set first, and then determine whether to output SSP or legacy XMM registers to userspace. Additionally, arch-PEBS supports sampling SSP, which is placed into the GPRs group. Also enables arch-PEBS-based SSP sampling. Currently, SSP sampling is only supported on the x86_64 architecture, as CET is only available on x86_64 platforms. SSP sampling will be enabled in a subsequent patch that sets PERF_PMU_CAP_SIMD_REGS. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-20-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 15 +++++++++++++++ arch/x86/events/intel/core.c | 3 ++- arch/x86/events/intel/ds.c | 13 +++++++++++-- arch/x86/events/perf_event.h | 25 +++++++++++++++++++++++++ arch/x86/include/asm/perf_event.h | 1 + arch/x86/include/uapi/asm/perf_regs.h | 7 ++++--- arch/x86/kernel/perf_regs.c | 5 +++++ 7 files changed, 63 insertions(+), 6 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 9cd79ac5570e..caa5a0fb3ed8 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -642,6 +642,10 @@ static int pebs_simd_regs_validate(struct perf_event *event) if (event_needs_xmm(event) && x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) return -EINVAL; + + if (event_needs_ssp(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_GPR))) + return -EINVAL; /* PEBS does not support YMM/ZMM/OPMASK/eGPR registers sampling yet. */ if (event_needs_ymm(event) || event_needs_low16_zmm(event) || @@ -671,6 +675,9 @@ static int event_simd_regs_validate(struct perf_event *event) if (event_needs_egprs(event) && !(x86_pmu.ext_regs_mask & XFEATURE_MASK_APX)) return -EINVAL; + if (event_needs_ssp(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_CET_USER)) + return -EINVAL; if (event_needs_xmm(event) && !(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) return -EINVAL; @@ -1854,11 +1861,13 @@ void x86_pmu_clear_perf_regs(struct pt_regs *regs) perf_regs->h16zmm_regs = NULL; perf_regs->opmask_regs = NULL; perf_regs->egpr_regs = NULL; + perf_regs->ssp = NULL; } static void update_perf_regs(struct x86_perf_regs *perf_regs, struct xregs_state *xsave, u64 bitmap) { + struct cet_user_state *cet; u64 mask; if (!xsave) @@ -1879,6 +1888,10 @@ static void update_perf_regs(struct x86_perf_regs *perf_regs, perf_regs->opmask = get_xsave_addr(xsave, XFEATURE_OPMASK); if (mask & XFEATURE_MASK_APX) perf_regs->egpr = get_xsave_addr(xsave, XFEATURE_APX); + if (mask & XFEATURE_MASK_CET_USER) { + cet = get_xsave_addr(xsave, XFEATURE_CET_USER); + perf_regs->ssp = cet ? &cet->user_ssp : NULL; + } } /* @@ -2057,6 +2070,8 @@ static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type) mask |= XFEATURE_MASK_OPMASK; if (__event_needs_egprs(event, sample_type)) mask |= XFEATURE_MASK_APX; + if (__event_needs_ssp(event, sample_type)) + mask |= XFEATURE_MASK_CET_USER; return mask; } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index b2aa0bc7c97e..6538ffe46f74 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4696,7 +4696,8 @@ static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event) { unsigned long flags = x86_pmu.large_pebs_flags; u64 gprs_mask = event->attr.sample_simd_regs_enabled ? - PEBS_GP_REGS | PERF_X86_EGPRS_MASK : + PEBS_GP_REGS | PERF_X86_EGPRS_MASK | + BIT_ULL(PERF_REG_X86_SSP) : PEBS_GP_REGS | PERF_REG_EXTENDED_MASK; if (event->attr.use_clockid) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 84caae35b86e..59f61e51c545 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1707,6 +1707,7 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) u64 sample_type = attr->sample_type; u64 pebs_data_cfg = 0; bool gprs, tsx_weight; + u64 xgprs_mask; if (!(sample_type & ~(PERF_SAMPLE_IP|PERF_SAMPLE_TIME)) && attr->precise_ip > 1) @@ -1721,10 +1722,13 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) * + precise_ip < 2 for the non event IP * + For RTM TSX weight we need GPRs for the abort code. */ + xgprs_mask = event->attr.sample_simd_regs_enabled ? + PEBS_GP_REGS | BIT_ULL(PERF_REG_X86_SSP) : + PEBS_GP_REGS; gprs = ((sample_type & PERF_SAMPLE_REGS_INTR) && - (attr->sample_regs_intr & PEBS_GP_REGS)) || + (attr->sample_regs_intr & xgprs_mask)) || ((sample_type & PERF_SAMPLE_REGS_USER) && - (attr->sample_regs_user & PEBS_GP_REGS)); + (attr->sample_regs_user & xgprs_mask)); tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) && ((attr->config & INTEL_ARCH_EVENT_MASK) == @@ -2673,6 +2677,11 @@ static void setup_arch_pebs_sample_data(struct perf_event *event, __setup_pebs_gpr_group(event, regs, (struct pebs_gprs *)gprs, sample_type); + + /* Currently only user space mode enables SSP. */ + if (user_mode(regs) && (sample_type & + (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))) + perf_regs->ssp = &gprs->ssp; } if (header->aux) { diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index c052a38b8733..1503ab68eca6 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -315,6 +315,31 @@ static inline bool event_needs_egprs(struct perf_event *event) PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); } +static inline bool __event_needs_ssp(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & BIT_ULL(PERF_REG_X86_SSP))) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & BIT_ULL(PERF_REG_X86_SSP))) + return true; + + return false; +} + +static inline bool event_needs_ssp(struct perf_event *event) +{ + return __event_needs_ssp(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + struct amd_nb { int nb_id; /* NorthBridge id */ int refcnt; /* reference count */ diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index bc05f8c17464..4302ef39c42e 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -753,6 +753,7 @@ struct x86_perf_regs { u64 *egpr_regs; struct apx_state *egpr; }; + u64 *ssp; }; extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs); diff --git a/arch/x86/include/uapi/asm/perf_regs.h b/arch/x86/include/uapi/asm/perf_regs.h index 977831bd7a9d..faaa82df688d 100644 --- a/arch/x86/include/uapi/asm/perf_regs.h +++ b/arch/x86/include/uapi/asm/perf_regs.h @@ -30,10 +30,10 @@ enum perf_event_x86_regs { PERF_REG_X86_R14, PERF_REG_X86_R15, /* - * The eGPRs and XMM have overlaps. Only one can be used + * The eGPRs/SSP and XMM have overlaps. Only one can be used * at a time. The ABI PERF_SAMPLE_REGS_ABI_SIMD is used to * distinguish which one is used. If PERF_SAMPLE_REGS_ABI_SIMD - * is set, then eGPRs is used, otherwise, XMM is used. + * is set, then eGPRs/SSP is used, otherwise, XMM is used. * * Extended GPRs (eGPRs) */ @@ -53,10 +53,11 @@ enum perf_event_x86_regs { PERF_REG_X86_R29, PERF_REG_X86_R30, PERF_REG_X86_R31, + PERF_REG_X86_SSP, /* These are the limits for the GPRs. */ PERF_REG_X86_32_MAX = PERF_REG_X86_GS + 1, PERF_REG_X86_64_MAX = PERF_REG_X86_R15 + 1, - PERF_REG_MISC_MAX = PERF_REG_X86_R31 + 1, + PERF_REG_MISC_MAX = PERF_REG_X86_SSP + 1, /* These all need two bits set because they are 128bit */ PERF_REG_X86_XMM0 = 32, diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index 7d4233ed859c..f8952f7c36cc 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -72,6 +72,11 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return 0; return perf_regs->egpr_regs[idx - PERF_REG_X86_R16]; } + if (idx == PERF_REG_X86_SSP) { + if (!perf_regs->ssp) + return 0; + return *perf_regs->ssp; + } } else { if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) { if (!perf_regs->xmm_regs) From c4fabb67a47f8103b80fadfcab3ba17e75784ab5 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:28 +0800 Subject: [PATCH 20/34] perf/x86/intel: Support arch-PEBS based SIMD/eGPRs sampling Support arch-PEBS based SIMD/eGPRs/SSP registers sampling. Arch-PEBS supports sampling all of these registers. SIMD and eGPR registers are placed into the XSAVE-Enabled Registers (XER) group with the layout described below. Field Name Registers Used Size XSTATE_BV XINUSE for groups 8 B Reserved Reserved 8 B SSER XMM0-XMM15 16 regs * 16 B = 256 B YMMHIR Upper 128 bits of YMM0-YMM15 16 regs * 16 B = 256 B EGPR R16-R31 16 regs * 8 B = 128 B OPMASKR K0-K7 8 regs * 8 B = 64 B ZMMHIR Upper 256 bits of ZMM0-ZMM15 16 regs * 32 B = 512 B Hi16ZMMR ZMM16-ZMM31 16 regs * 64 B = 1024 B Memory space in the output buffer is allocated for these sub-groups as long as the corresponding Format.XER[55:49] bits in the PEBS record header are set. However, the arch-PEBS hardware engine does not write the sub-group if it is not used (in INIT state). In such cases, the corresponding bit in the XSTATE_BV bitmap is set to 0. Therefore, the XSTATE_BV field is checked to determine if the register data is actually written for each PEBS record. If not, the register data is not outputted to userspace. Additionally, the MSRs IA32_PMC_{GPn|FXm}_CFG_C.[55:49] bits are used to manage which types of these registers need to be sampled. Arch-PEBS based SIMD/eGPRs sampling will be enabled in a subsequent patch that sets PERF_PMU_CAP_SIMD_REGS. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-21-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 20 ++++++--- arch/x86/events/intel/core.c | 15 +++++++ arch/x86/events/intel/ds.c | 67 +++++++++++++++++++++++++++++-- arch/x86/include/asm/msr-index.h | 7 ++++ arch/x86/include/asm/perf_event.h | 8 +++- 5 files changed, 106 insertions(+), 11 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index caa5a0fb3ed8..11799acd8c54 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -646,12 +646,20 @@ static int pebs_simd_regs_validate(struct perf_event *event) if (event_needs_ssp(event) && !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_GPR))) return -EINVAL; - /* PEBS does not support YMM/ZMM/OPMASK/eGPR registers sampling yet. */ - if (event_needs_ymm(event) || - event_needs_low16_zmm(event) || - event_needs_high16_zmm(event) || - event_needs_opmask(event) || - event_needs_egprs(event)) + if (event_needs_ymm(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_YMMH))) + return -EINVAL; + if (event_needs_egprs(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_EGPRS))) + return -EINVAL; + if (event_needs_opmask(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_OPMASK))) + return -EINVAL; + if (event_needs_low16_zmm(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_ZMMH))) + return -EINVAL; + if (event_needs_high16_zmm(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_H16ZMM))) return -EINVAL; return 0; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 6538ffe46f74..f87346200446 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3474,6 +3474,21 @@ static void intel_pmu_enable_event_ext(struct perf_event *event) if (pebs_data_cfg & PEBS_DATACFG_XMMS) ext |= ARCH_PEBS_VECR_XMM & cap.caps; + if (pebs_data_cfg & PEBS_DATACFG_YMMHS) + ext |= ARCH_PEBS_VECR_YMMH & cap.caps; + + if (pebs_data_cfg & PEBS_DATACFG_EGPRS) + ext |= ARCH_PEBS_VECR_EGPRS & cap.caps; + + if (pebs_data_cfg & PEBS_DATACFG_OPMASKS) + ext |= ARCH_PEBS_VECR_OPMASK & cap.caps; + + if (pebs_data_cfg & PEBS_DATACFG_ZMMHS) + ext |= ARCH_PEBS_VECR_ZMMH & cap.caps; + + if (pebs_data_cfg & PEBS_DATACFG_H16ZMMS) + ext |= ARCH_PEBS_VECR_H16ZMM & cap.caps; + if (pebs_data_cfg & PEBS_DATACFG_LBRS) ext |= ARCH_PEBS_LBR & cap.caps; diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 59f61e51c545..4335ae657dc4 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1740,6 +1740,16 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) { if (event_needs_xmm(event)) pebs_data_cfg |= PEBS_DATACFG_XMMS; + if (x86_pmu.arch_pebs && event_needs_ymm(event)) + pebs_data_cfg |= PEBS_DATACFG_YMMHS; + if (x86_pmu.arch_pebs && event_needs_low16_zmm(event)) + pebs_data_cfg |= PEBS_DATACFG_ZMMHS; + if (x86_pmu.arch_pebs && event_needs_high16_zmm(event)) + pebs_data_cfg |= PEBS_DATACFG_H16ZMMS; + if (x86_pmu.arch_pebs && event_needs_opmask(event)) + pebs_data_cfg |= PEBS_DATACFG_OPMASKS; + if (x86_pmu.arch_pebs && event_needs_egprs(event)) + pebs_data_cfg |= PEBS_DATACFG_EGPRS; } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { @@ -2694,14 +2704,63 @@ static void setup_arch_pebs_sample_data(struct perf_event *event, meminfo->tsx_tuning, ax); } - if (header->xmm) { + if (header->xmm || header->ymmh || header->egpr || + header->opmask || header->zmmh || header->h16zmm) { + struct arch_pebs_xer_header *xer_header = next_record; struct pebs_xmm *xmm; + struct ymmh_struct *ymmh; + struct avx_512_zmm_uppers_state *zmmh; + struct avx_512_hi16_state *h16zmm; + struct avx_512_opmask_state *opmask; + struct apx_state *egpr; next_record += sizeof(struct arch_pebs_xer_header); - xmm = next_record; - perf_regs->xmm_regs = xmm->xmm; - next_record = xmm + 1; + if (header->xmm) { + xmm = next_record; + /* + * Only output XMM regs to user space when arch-PEBS + * really writes data into xstate area. + */ + if (xer_header->xstate & XFEATURE_MASK_SSE) + perf_regs->xmm_regs = xmm->xmm; + next_record = xmm + 1; + } + + if (header->ymmh) { + ymmh = next_record; + if (xer_header->xstate & XFEATURE_MASK_YMM) + perf_regs->ymmh = ymmh; + next_record = ymmh + 1; + } + + if (header->egpr) { + egpr = next_record; + if (xer_header->xstate & XFEATURE_MASK_APX) + perf_regs->egpr = egpr; + next_record = egpr + 1; + } + + if (header->opmask) { + opmask = next_record; + if (xer_header->xstate & XFEATURE_MASK_OPMASK) + perf_regs->opmask = opmask; + next_record = opmask + 1; + } + + if (header->zmmh) { + zmmh = next_record; + if (xer_header->xstate & XFEATURE_MASK_ZMM_Hi256) + perf_regs->zmmh = zmmh; + next_record = zmmh + 1; + } + + if (header->h16zmm) { + h16zmm = next_record; + if (xer_header->xstate & XFEATURE_MASK_Hi16_ZMM) + perf_regs->h16zmm = h16zmm; + next_record = h16zmm + 1; + } } if (header->lbr) { diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 3a8e51a0c9e8..428eb44106a9 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -350,6 +350,13 @@ #define ARCH_PEBS_LBR_SHIFT 40 #define ARCH_PEBS_LBR (0x3ull << ARCH_PEBS_LBR_SHIFT) #define ARCH_PEBS_VECR_XMM BIT_ULL(49) +#define ARCH_PEBS_VECR_YMMH BIT_ULL(50) +#define ARCH_PEBS_VECR_EGPRS BIT_ULL(51) +#define ARCH_PEBS_VECR_OPMASK BIT_ULL(53) +#define ARCH_PEBS_VECR_ZMMH BIT_ULL(54) +#define ARCH_PEBS_VECR_H16ZMM BIT_ULL(55) +#define ARCH_PEBS_VECR_EXT_SHIFT 49 +#define ARCH_PEBS_VECR_EXT (0x7full << ARCH_PEBS_VECR_EXT_SHIFT) #define ARCH_PEBS_GPR BIT_ULL(61) #define ARCH_PEBS_AUX BIT_ULL(62) #define ARCH_PEBS_EN BIT_ULL(63) diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 4302ef39c42e..12f7db8c57b4 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -150,6 +150,11 @@ #define PEBS_DATACFG_LBRS BIT_ULL(3) #define PEBS_DATACFG_CNTR BIT_ULL(4) #define PEBS_DATACFG_METRICS BIT_ULL(5) +#define PEBS_DATACFG_YMMHS BIT_ULL(6) +#define PEBS_DATACFG_OPMASKS BIT_ULL(7) +#define PEBS_DATACFG_ZMMHS BIT_ULL(8) +#define PEBS_DATACFG_H16ZMMS BIT_ULL(9) +#define PEBS_DATACFG_EGPRS BIT_ULL(10) #define PEBS_DATACFG_LBR_SHIFT 24 #define PEBS_DATACFG_CNTR_SHIFT 32 #define PEBS_DATACFG_CNTR_MASK GENMASK_ULL(15, 0) @@ -547,7 +552,8 @@ struct arch_pebs_header { rsvd3:7, xmm:1, ymmh:1, - rsvd4:2, + egpr:1, + rsvd4:1, opmask:1, zmmh:1, h16zmm:1, From 00cf8daabe2bfd45e7283e44ad7cb3977089259b Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:29 +0800 Subject: [PATCH 21/34] perf/x86/intel: Advertise PERF_PMU_CAP_SIMD_REGS capability Set PERF_PMU_CAP_SIMD_REGS once XSAVES-backed extended state beyond XMM is available for sampling, e.g. YMM, ZMM, OPMASK, eGPRs, or SSP. Keep backward compatibility unchanged. When only XMM sampling is available, continue to expose only PERF_PMU_CAP_EXTENDED_REGS. Expose PERF_PMU_CAP_SIMD_REGS only when additional SIMD/predicate register state is supported. Co-developed-by: Kan Liang Signed-off-by: Kan Liang Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-22-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/core.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index f87346200446..a919168f9970 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -6204,7 +6204,27 @@ static inline void __intel_update_pmu_xregs_caps(struct pmu *pmu) */ x86_pmu.ext_regs_mask |= XFEATURE_MASK_SSE; + if (boot_cpu_has(X86_FEATURE_AVX) && + cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_YMM; + if (boot_cpu_has(X86_FEATURE_APX) && + cpu_has_xfeatures(XFEATURE_MASK_APX, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_APX; + if (boot_cpu_has(X86_FEATURE_AVX512F)) { + if (cpu_has_xfeatures(XFEATURE_MASK_OPMASK, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_OPMASK; + if (cpu_has_xfeatures(XFEATURE_MASK_ZMM_Hi256, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_ZMM_Hi256; + if (cpu_has_xfeatures(XFEATURE_MASK_Hi16_ZMM, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_Hi16_ZMM; + } + if (cpu_feature_enabled(X86_FEATURE_USER_SHSTK) && + cpu_has_xfeatures(XFEATURE_MASK_CET_USER, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_CET_USER; + dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; + if (x86_pmu.ext_regs_mask > XFEATURE_MASK_SSE) + dest_pmu->capabilities |= PERF_PMU_CAP_SIMD_REGS; } static inline void __intel_update_large_pebs_flags(struct pmu *pmu) From c34db2f950940fe8732899a8cc9ba12942be3ba9 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:30 +0800 Subject: [PATCH 22/34] perf/x86: Activate back-to-back NMI detection for arch-PEBS induced NMIs When two or more identical PEBS events with the same sampling period are programmed on a mix of PDIST and non-PDIST counters, multiple back-to-back NMIs can be triggered. The Linux PMI handler processes the first NMI and clears the GLOBAL_STATUS MSR. If a second NMI is triggered immediately after the first, it is recognized as a "suspicious NMI" because no bits are set in the GLOBAL_STATUS MSR (cleared by the first NMI). This issue does not lead to PEBS data corruption or data loss, but it does result in an annoying warning message. The current NMI handler supports back-to-back NMI detection, but it requires the PMI handler to return the count of actually processed events, which the PEBS handler does not currently do. Thus, modify the PEBS handlers to return the count of actually processed events, thereby activating back-to-back NMI detection and avoiding the "suspicious NMI" warning. Suggested-by: Andi Kleen Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-23-dapeng1.mi@linux.intel.com --- arch/x86/events/core.c | 2 +- arch/x86/events/intel/core.c | 31 +++++++++++++++++--------- arch/x86/events/intel/ds.c | 43 ++++++++++++++++++++++++------------ arch/x86/events/perf_event.h | 2 +- 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 11799acd8c54..ef9b8df2e1f7 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -93,7 +93,7 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling); DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task); -DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); +DEFINE_STATIC_CALL_RET0(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases); DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index a919168f9970..4bde7362d7b2 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3760,8 +3760,8 @@ static void intel_pmu_reset(void) * * The contents and other behavior of the guest event do not matter. */ -static void x86_pmu_handle_guest_pebs(struct pt_regs *regs, - struct perf_sample_data *data) +static int x86_pmu_handle_guest_pebs(struct pt_regs *regs, + struct perf_sample_data *data) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_host_mask; @@ -3769,11 +3769,11 @@ static void x86_pmu_handle_guest_pebs(struct pt_regs *regs, int bit; if (!unlikely(perf_guest_state())) - return; + return 0; if (!x86_pmu.pebs_ept || !x86_pmu.pebs_active || !guest_pebs_idxs) - return; + return 0; for_each_set_bit(bit, (unsigned long *)&guest_pebs_idxs, X86_PMC_IDX_MAX) { event = cpuc->events[bit]; @@ -3783,9 +3783,14 @@ static void x86_pmu_handle_guest_pebs(struct pt_regs *regs, perf_sample_data_init(data, 0, event->hw.last_period); perf_event_overflow(event, data, regs); - /* Inject one fake event is enough. */ - break; + /* + * Inject one fake event is enough. + * Returning 1 to inform PMI is handled. + */ + return 1; } + + return 0; } static int handle_pmi_common(struct pt_regs *regs, u64 status) @@ -3834,9 +3839,11 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) { u64 pebs_enabled = cpuc->pebs_enabled; - handled++; - x86_pmu_handle_guest_pebs(regs, &data); - static_call(x86_pmu_drain_pebs)(regs, &data); + handled += x86_pmu_handle_guest_pebs(regs, &data); + handled += static_call(x86_pmu_drain_pebs)(regs, &data); + /* Ensure no "suspicious NMI" warning for empty PEBS buffer. */ + if (!handled) + handled++; /* * PMI throttle may be triggered, which stops the PEBS event. @@ -3863,8 +3870,10 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) */ if (__test_and_clear_bit(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT, (unsigned long *)&status)) { - handled++; - static_call(x86_pmu_drain_pebs)(regs, &data); + handled += static_call(x86_pmu_drain_pebs)(regs, &data); + /* Ensure no "suspicious NMI" warning for empty PEBS buffer. */ + if (!handled) + handled++; if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] && is_pebs_counter_event_group(cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS])) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 4335ae657dc4..1b220cd4ca01 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -3021,7 +3021,7 @@ __intel_pmu_pebs_events(struct perf_event *event, __intel_pmu_pebs_last_event(event, iregs, regs, data, at, count, setup_sample); } -static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data) +static int intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct debug_store *ds = cpuc->ds; @@ -3030,7 +3030,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_ int n; if (!x86_pmu.pebs_active) - return; + return 0; at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base; top = (struct pebs_record_core *)(unsigned long)ds->pebs_index; @@ -3041,22 +3041,25 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_ ds->pebs_index = ds->pebs_buffer_base; if (!test_bit(0, cpuc->active_mask)) - return; + return 0; WARN_ON_ONCE(!event); if (!event->attr.precise_ip) - return; + return 0; n = top - at; if (n <= 0) { if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD) intel_pmu_save_and_restart_reload(event, 0); - return; + return 0; } __intel_pmu_pebs_events(event, iregs, data, at, top, 0, n, setup_pebs_fixed_sample_data); + + /* PMC0 only */ + return 1; } static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64 mask) @@ -3079,7 +3082,7 @@ static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64 } } -static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data) +static int intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct debug_store *ds = cpuc->ds; @@ -3088,11 +3091,12 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; short error[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; int max_pebs_events = intel_pmu_max_num_pebs(NULL); + u64 events_bitmap = 0; int bit, i, size; u64 mask; if (!x86_pmu.pebs_active) - return; + return 0; base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base; top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index; @@ -3108,7 +3112,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d if (unlikely(base >= top)) { intel_pmu_pebs_event_update_no_drain(cpuc, mask); - return; + return 0; } for (at = base; at < top; at += x86_pmu.pebs_record_size) { @@ -3172,6 +3176,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d if ((counts[bit] == 0) && (error[bit] == 0)) continue; + events_bitmap |= BIT_ULL(bit); event = cpuc->events[bit]; if (WARN_ON_ONCE(!event)) continue; @@ -3193,6 +3198,8 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d setup_pebs_fixed_sample_data); } } + + return hweight64(events_bitmap); } static __always_inline void @@ -3246,7 +3253,7 @@ __intel_pmu_handle_last_pebs_record(struct pt_regs *iregs, } -static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data) +static int intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data) { short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS]; @@ -3256,10 +3263,11 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d struct pt_regs *regs = &perf_regs->regs; struct pebs_basic *basic; void *base, *at, *top; + u64 events_bitmap = 0; u64 mask; if (!x86_pmu.pebs_active) - return; + return 0; base = (struct pebs_basic *)(unsigned long)ds->pebs_buffer_base; top = (struct pebs_basic *)(unsigned long)ds->pebs_index; @@ -3272,7 +3280,7 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d if (unlikely(base >= top)) { intel_pmu_pebs_event_update_no_drain(cpuc, mask); - return; + return 0; } if (!iregs) @@ -3287,6 +3295,7 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d continue; pebs_status = mask & basic->applicable_counters; + events_bitmap |= pebs_status; __intel_pmu_handle_pebs_record(iregs, regs, data, at, pebs_status, counts, last, setup_pebs_adaptive_sample_data); @@ -3294,10 +3303,12 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, counts, last, setup_pebs_adaptive_sample_data); + + return hweight64(events_bitmap); } -static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, - struct perf_sample_data *data) +static int intel_pmu_drain_arch_pebs(struct pt_regs *iregs, + struct perf_sample_data *data) { short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS]; @@ -3306,13 +3317,14 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs); struct pt_regs *regs = &perf_regs->regs; void *base, *at, *top; + u64 events_bitmap = 0; u64 mask; rdmsrq(MSR_IA32_PEBS_INDEX, index.whole); if (unlikely(!index.wr)) { intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX); - return; + return 0; } base = cpuc->pebs_vaddr; @@ -3351,6 +3363,7 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, basic = at + sizeof(struct arch_pebs_header); pebs_status = mask & basic->applicable_counters; + events_bitmap |= pebs_status; __intel_pmu_handle_pebs_record(iregs, regs, data, at, pebs_status, counts, last, setup_arch_pebs_sample_data); @@ -3370,6 +3383,8 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, counts, last, setup_arch_pebs_sample_data); + + return hweight64(events_bitmap); } static void __init intel_arch_pebs_init(void) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 1503ab68eca6..c9c52f3ee4fc 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1145,7 +1145,7 @@ struct x86_pmu { int pebs_record_size; int pebs_buffer_size; u64 pebs_events_mask; - void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data); + int (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data); struct event_constraint *pebs_constraints; void (*pebs_aliases)(struct perf_event *event); u64 (*pebs_latency_data)(struct perf_event *event, u64 status); From 68aca309e49c589f1922f4c5af2370e4df093175 Mon Sep 17 00:00:00 2001 From: Dapeng Mi Date: Mon, 24 Aug 2026 16:27:31 +0800 Subject: [PATCH 23/34] perf/x86/intel: Add sanity check for PEBS record/fragment size Prevent potential infinite loops by adding a sanity check for the corrupted PEBS record or fragment size which could happen in theory. If a corrupted PEBS fragment is detected, the entire PEBS record including the corrupted fragment and all subsequent records will be dropped and a NULL PEBS record is reported to user space. This ensures the integrity of PEBS data and prevents infinite loops in setup_arch_pebs_sample_data() again. Please note software has no way to figure out which events are impacted by the corrupted record, so the last record of each event would be discarded for all events if corrupted record is detected even though it may be a well-formed record for some events. Signed-off-by: Dapeng Mi Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260824082731.1013973-24-dapeng1.mi@linux.intel.com --- arch/x86/events/intel/ds.c | 77 ++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 1b220cd4ca01..2edff9754eb2 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -2657,6 +2657,9 @@ static void setup_arch_pebs_sample_data(struct perf_event *event, again: header = at; + if (!header->size) + return; + next_record = at + sizeof(struct arch_pebs_header); if (header->basic) { struct arch_pebs_basic *basic = next_record; @@ -2930,13 +2933,21 @@ __intel_pmu_pebs_last_event(struct perf_event *event, struct pt_regs *iregs, struct pt_regs *regs, struct perf_sample_data *data, - void *at, - int count, + void *at, int count, bool corrupted, setup_fn setup_sample) { struct hw_perf_event *hwc = &event->hw; - setup_sample(event, iregs, at, data, regs); + /* Skip parsing corrupted PEBS record. */ + if (corrupted) { + /* Clear stale register states in previous records. */ + memset(regs, 0, sizeof(*regs)); + x86_pmu_clear_perf_regs(regs); + perf_sample_data_init(data, 0, event->hw.last_period); + } else { + setup_sample(event, iregs, at, data, regs); + } + if (iregs == &dummy_iregs) { /* * The PEBS records may be drained in the non-overflow context, @@ -2954,12 +2965,16 @@ __intel_pmu_pebs_last_event(struct perf_event *event, } if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) { - if ((is_pebs_counter_event_group(event))) { - /* - * The value of each sample has been updated when setup - * the corresponding sample data. - */ - perf_event_update_userpage(event); + if (is_pebs_counter_event_group(event)) { + if (corrupted) { + intel_pmu_save_and_restart_reload(event, 1); + } else { + /* + * The value of each sample has been updated + * when setup the corresponding sample data. + */ + perf_event_update_userpage(event); + } } else { /* * Now, auto-reload is only enabled in fixed period mode. @@ -2983,7 +2998,7 @@ __intel_pmu_pebs_last_event(struct perf_event *event, * counters-snapshotting record, only needs to set the new * period for the counter. */ - if (is_pebs_counter_event_group(event)) + if (is_pebs_counter_event_group(event) && !corrupted) static_call(x86_pmu_set_period)(event); else intel_pmu_save_and_restart(event); @@ -3012,13 +3027,15 @@ __intel_pmu_pebs_events(struct perf_event *event, iregs = &dummy_iregs; while (cnt > 1) { - __intel_pmu_pebs_event(event, iregs, regs, data, at, setup_sample); + __intel_pmu_pebs_event(event, iregs, regs, data, + at, setup_sample); at += cpuc->pebs_record_size; at = get_next_pebs_record_by_bit(at, top, bit); cnt--; } - __intel_pmu_pebs_last_event(event, iregs, regs, data, at, count, setup_sample); + __intel_pmu_pebs_last_event(event, iregs, regs, data, at, + count, false, setup_sample); } static int intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data) @@ -3234,23 +3251,28 @@ static __always_inline void __intel_pmu_handle_last_pebs_record(struct pt_regs *iregs, struct pt_regs *regs, struct perf_sample_data *data, - u64 mask, short *counts, void **last, + u64 mask, short *counts, + void **last, bool corrupted, setup_fn setup_sample) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_event *event; + bool handled = false; int bit; for_each_set_bit(bit, (unsigned long *)&mask, X86_PMC_IDX_MAX) { if (!counts[bit]) continue; + handled = true; event = cpuc->events[bit]; - __intel_pmu_pebs_last_event(event, iregs, regs, data, last[bit], - counts[bit], setup_sample); + counts[bit], corrupted, setup_sample); } + /* All records are corrupted, reset sampling period. */ + if (!handled) + intel_pmu_pebs_event_update_no_drain(cpuc, mask); } static int intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data) @@ -3264,6 +3286,7 @@ static int intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_da struct pebs_basic *basic; void *base, *at, *top; u64 events_bitmap = 0; + bool corrupted = false; u64 mask; if (!x86_pmu.pebs_active) @@ -3291,6 +3314,10 @@ static int intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_da u64 pebs_status; basic = at; + if (WARN_ON_ONCE(!basic->format_size)) { + corrupted = true; + break; + } if (basic->format_size != cpuc->pebs_record_size) continue; @@ -3302,7 +3329,7 @@ static int intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_da } __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, counts, last, - setup_pebs_adaptive_sample_data); + corrupted, setup_pebs_adaptive_sample_data); return hweight64(events_bitmap); } @@ -3318,6 +3345,7 @@ static int intel_pmu_drain_arch_pebs(struct pt_regs *iregs, struct pt_regs *regs = &perf_regs->regs; void *base, *at, *top; u64 events_bitmap = 0; + bool corrupted = false; u64 mask; rdmsrq(MSR_IA32_PEBS_INDEX, index.whole); @@ -3352,8 +3380,10 @@ static int intel_pmu_drain_arch_pebs(struct pt_regs *iregs, header = at; - if (WARN_ON_ONCE(!header->size)) - break; + if (WARN_ON_ONCE(!header->size)) { + corrupted = true; + goto done; + } /* 1st fragment or single record must have basic group */ if (!header->basic) { @@ -3373,15 +3403,24 @@ static int intel_pmu_drain_arch_pebs(struct pt_regs *iregs, if (!header->size) break; at += header->size; + if (WARN_ON_ONCE(at >= top)) { + corrupted = true; + goto done; + } header = at; } /* Skip last fragment or the single record */ at += header->size; + if (WARN_ON_ONCE(at > top)) { + corrupted = true; + goto done; + } } +done: __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, - counts, last, + counts, last, corrupted, setup_arch_pebs_sample_data); return hweight64(events_bitmap); From df935e26ca9a8fd4ac7431b73348938250f80c6e Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Tue, 15 Sep 2026 17:45:16 -0700 Subject: [PATCH 24/34] perf/x86/amd/uncore: Turn amd_uncore_ctx events into a flexible array The events pointer was kzalloc_node()'d immediately after the parent struct allocation, with the count (pmu->num_counters) trivially available beforehand. Move events to the struct tail as a flexible array member and fold the two allocations into a single kzalloc_node() with struct_size(), dropping the separate kfree(ctx->events) in the free path. Assisted-by: Claude:Opus-4.7 Signed-off-by: Rosen Penev Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sandipan Das Link: https://patch.msgid.link/20260916004516.268192-1-rosenp@gmail.com --- arch/x86/events/amd/uncore.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 7181973b5b12..74b9c92d517a 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -39,11 +39,11 @@ static int pmu_version; struct amd_uncore_ctx { int refcnt; int cpu; - struct perf_event **events; unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)]; int nr_active; struct hrtimer hrtimer; u64 hrtimer_duration; + struct perf_event *events[]; }; struct amd_uncore_pmu { @@ -519,10 +519,8 @@ static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu) if (cpu == ctx->cpu) cpumask_clear_cpu(cpu, &pmu->active_mask); - if (!--ctx->refcnt) { - kfree(ctx->events); + if (!--ctx->refcnt) kfree(ctx); - } *per_cpu_ptr(pmu->ctx, cpu) = NULL; } @@ -567,18 +565,11 @@ static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu) /* Allocate context if sibling does not exist */ if (!curr) { node = cpu_to_node(cpu); - curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node); + curr = kzalloc_node(struct_size(curr, events, pmu->num_counters), GFP_KERNEL, node); if (!curr) goto fail; curr->cpu = cpu; - curr->events = kzalloc_node(sizeof(*curr->events) * - pmu->num_counters, - GFP_KERNEL, node); - if (!curr->events) { - kfree(curr); - goto fail; - } amd_uncore_init_hrtimer(curr); curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC; From 52a6457f59adf70d624ef1b1bed8d55cb7b73af6 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Fri, 4 Sep 2026 16:17:00 +0200 Subject: [PATCH 25/34] perf/x86/intel: Annotate x86_pmu::hybrid_pmu with __counted_by_ptr Add __counted_by_ptr() to x86_pmu::hybrid_pmu to improve access bounds checking via CONFIG_UBSAN_BOUNDS and CONFIG_FORTIFY_SOURCE. Signed-off-by: Thorsten Blum Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ian Rogers Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260904141659.224248-2-blum@kernel.org --- arch/x86/events/perf_event.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 6eb1bf4ca442..5deca1a95d30 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1245,7 +1245,7 @@ struct x86_pmu { * unique capabilities. */ int num_hybrid_pmus; - struct x86_hybrid_pmu *hybrid_pmu; + struct x86_hybrid_pmu *hybrid_pmu __counted_by_ptr(num_hybrid_pmus); enum intel_cpu_type (*get_hybrid_cpu_type) (void); }; From b7b84fff2bd630b593ee47a733381e3886aac1a7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:13 -0700 Subject: [PATCH 26/34] perf/x86/intel: Invert names of intel_ctrl_{guest,host}_mask Rename intel_ctrl_{guest,host}_mask to intel_ctrl_exclude_{host,guest}_mask to more accurately capture what they actually track. Specifically, an event that is excluded from the guest is NOT guaranteed to count in the host, and vice versa, as it legal (albeit bizarre) to configure an event to exclude both the host and the guest, i.e. to not count at all. Subjectively (though anyone who disagrees is wrong), aligning with perf_event_attr.exclude_{guest,host} also makes all related code much easier to follow. No functional change intended. Suggested-by: Jim Mattson Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-6-seanjc@google.com --- arch/x86/events/intel/core.c | 22 +++++++++++----------- arch/x86/events/intel/lbr.c | 2 +- arch/x86/events/perf_event.h | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index a73a3556959e..076518719d29 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2797,7 +2797,7 @@ static void __intel_pmu_enable_all(int added, bool pmi) } wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, - intel_ctrl & ~cpuc->intel_ctrl_guest_mask); + intel_ctrl & ~cpuc->intel_ctrl_exclude_host_mask); if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) { struct perf_event *event = @@ -2995,9 +2995,9 @@ static inline void intel_set_masks(struct perf_event *event, int idx) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); if (event->attr.exclude_host) - __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask); + __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask); if (event->attr.exclude_guest) - __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask); + __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_guest_mask); if (event_is_checkpointed(event)) __set_bit(idx, (unsigned long *)&cpuc->intel_cp_status); } @@ -3006,8 +3006,8 @@ static inline void intel_clear_masks(struct perf_event *event, int idx) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask); - __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask); + __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask); + __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_guest_mask); __clear_bit(idx, (unsigned long *)&cpuc->intel_cp_status); } @@ -3789,7 +3789,7 @@ static int x86_pmu_handle_guest_pebs(struct pt_regs *regs, struct perf_sample_data *data) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_host_mask; + u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_exclude_guest_mask; struct perf_event *event = NULL; int bit; @@ -5349,8 +5349,8 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) global_ctrl = (*nr)++; arr[global_ctrl] = (struct perf_guest_switch_msr){ .msr = MSR_CORE_PERF_GLOBAL_CTRL, - .host = intel_ctrl & ~cpuc->intel_ctrl_guest_mask, - .guest = intel_ctrl & ~cpuc->intel_ctrl_host_mask & ~pebs_mask, + .host = intel_ctrl & ~cpuc->intel_ctrl_exclude_host_mask, + .guest = intel_ctrl & ~cpuc->intel_ctrl_exclude_guest_mask & ~pebs_mask, }; if (!x86_pmu.ds_pebs) @@ -5387,8 +5387,8 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) * in the guest, and (d) _are_ excluded from counting in the host. */ guest_pebs_mask = pebs_mask & intel_ctrl & kvm_pmu->pebs_enable & - ~cpuc->intel_ctrl_host_mask & - cpuc->intel_ctrl_guest_mask; + ~cpuc->intel_ctrl_exclude_guest_mask & + cpuc->intel_ctrl_exclude_host_mask; /* * Disable counters where the guest PMC is different than the host PMC @@ -5404,7 +5404,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) * What exactly goes wrong if guest and host are using PEBS is * unknown. */ - if (pebs_mask & ~cpuc->intel_ctrl_guest_mask) + if (pebs_mask & ~cpuc->intel_ctrl_exclude_host_mask) guest_pebs_mask = 0; /* diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 22e2a06d5786..0b446d8912e4 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -714,7 +714,7 @@ static inline bool vlbr_exclude_host(void) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); return test_bit(INTEL_PMC_IDX_FIXED_VLBR, - (unsigned long *)&cpuc->intel_ctrl_guest_mask); + (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask); } void intel_pmu_lbr_enable_all(bool pmi) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 5deca1a95d30..7c3a84e08e04 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -540,8 +540,8 @@ struct cpu_hw_events { /* * Intel host/guest exclude bits */ - u64 intel_ctrl_guest_mask; - u64 intel_ctrl_host_mask; + u64 intel_ctrl_exclude_host_mask; + u64 intel_ctrl_exclude_guest_mask; struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX]; /* From a634e4536ec22a17e2391fd5ed38c06ad99cbf99 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:14 -0700 Subject: [PATCH 27/34] perf/x86: KVM: Have perf define a dedicated struct for getting guest PEBS data Have perf define a struct for getting guest PEBS data from KVM instead of poking into the kvm_pmu structure. Passing in an entire "struct kvm_pmu" _as an opaque pointer_ to get at four fields is silly, especially since one of the fields exists purely to convey information to perf, i.e. isn't used by KVM. Perf should also own its APIs, i.e. define what fields/data it needs, not rely on KVM to throw fields into data structures that effectively hold KVM-internal state. Opportunistically rephrase the comment about cross-mapped counters to explain *why* PEBS needs to be disabled. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260921191418.950933-7-seanjc@google.com --- arch/x86/events/core.c | 5 +++-- arch/x86/events/intel/core.c | 16 ++++++++-------- arch/x86/events/perf_event.h | 3 ++- arch/x86/include/asm/kvm_host.h | 9 --------- arch/x86/include/asm/perf_event.h | 12 ++++++++++-- arch/x86/kvm/vmx/pmu_intel.c | 17 ++++++++++++++--- arch/x86/kvm/vmx/vmx.c | 11 ++++++++--- arch/x86/kvm/vmx/vmx.h | 2 +- 8 files changed, 46 insertions(+), 29 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index ef9b8df2e1f7..9b1a9032e36d 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -862,9 +862,10 @@ void x86_pmu_disable_all(void) } } -struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data) +struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs) { - return static_call(x86_pmu_guest_get_msrs)(nr, data); + return static_call(x86_pmu_guest_get_msrs)(nr, guest_pebs); } EXPORT_SYMBOL_FOR_KVM(perf_guest_get_msrs); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 076518719d29..87274b3bf503 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include @@ -5328,11 +5327,11 @@ static int intel_pmu_hw_config(struct perf_event *event) * when it uses {RD,WR}MSR, which should be handled by the KVM context, * specifically in the intel_pmu_{get,set}_msr(). */ -static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) +static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; - struct kvm_pmu *kvm_pmu = (struct kvm_pmu *)data; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); u64 pebs_mask = cpuc->pebs_enabled & x86_pmu.pebs_capable; u64 guest_pebs_mask; @@ -5386,7 +5385,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) * the guest wants to use for PEBS, (c) are not excluded from counting * in the guest, and (d) _are_ excluded from counting in the host. */ - guest_pebs_mask = pebs_mask & intel_ctrl & kvm_pmu->pebs_enable & + guest_pebs_mask = pebs_mask & intel_ctrl & guest_pebs->enable & ~cpuc->intel_ctrl_exclude_guest_mask & cpuc->intel_ctrl_exclude_host_mask; @@ -5396,7 +5395,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) * PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the * wrong counter(s). */ - guest_pebs_mask &= ~kvm_pmu->host_cross_mapped_mask; + guest_pebs_mask &= ~guest_pebs->cross_mapped_mask; /* * FIXME: Allow guest and host usage of PEBS events to co-exist instead @@ -5415,14 +5414,14 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) arr[(*nr)++] = (struct perf_guest_switch_msr){ .msr = MSR_IA32_DS_AREA, .host = (unsigned long)cpuc->ds, - .guest = guest_pebs_mask ? kvm_pmu->ds_area : (unsigned long)cpuc->ds, + .guest = guest_pebs_mask ? guest_pebs->ds_area : (unsigned long)cpuc->ds, }; if (x86_pmu.intel_cap.pebs_baseline) { arr[(*nr)++] = (struct perf_guest_switch_msr){ .msr = MSR_PEBS_DATA_CFG, .host = cpuc->active_pebs_data_cfg, - .guest = guest_pebs_mask ? kvm_pmu->pebs_data_cfg : + .guest = guest_pebs_mask ? guest_pebs->data_cfg : cpuc->active_pebs_data_cfg, }; } @@ -5438,7 +5437,8 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) return arr; } -static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr, void *data) +static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 7c3a84e08e04..e274802ef062 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -1226,7 +1226,8 @@ struct x86_pmu { /* * Intel host/guest support (KVM) */ - struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data); + struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, + struct x86_guest_pebs *guest_pebs); /* * Check period value for PERF_EVENT_IOC_PERIOD ioctl. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 683bb8bf43a9..5e8aecfc795f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -608,15 +608,6 @@ struct kvm_pmu { u64 pebs_data_cfg; u64 pebs_data_cfg_rsvd; - /* - * If a guest counter is cross-mapped to host counter with different - * index, its PEBS capability will be temporarily disabled. - * - * The user should make sure that this mask is updated - * after disabling interrupts and before perf_guest_get_msrs(); - */ - u64 host_cross_mapped_mask; - /* * The gate to release perf_events not marked in * pmc_in_use only once in a vcpu time slice. diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 12f7db8c57b4..27f9c5bee7b0 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -819,11 +819,19 @@ extern void perf_load_guest_lvtpc(u32 guest_lvtpc); extern void perf_put_guest_lvtpc(void); #endif +struct x86_guest_pebs { + u64 enable; + u64 ds_area; + u64 data_cfg; + u64 cross_mapped_mask; +}; #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) -extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data); +extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs); extern void x86_perf_get_lbr(struct x86_pmu_lbr *lbr); #else -struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data); +struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs); static inline void x86_perf_get_lbr(struct x86_pmu_lbr *lbr) { memset(lbr, 0, sizeof(*lbr)); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index bfa8612fb450..a21e146ad4f2 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -747,11 +747,21 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) intel_pmu_release_guest_lbr_event(vcpu); } -void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu) +u64 intel_pmu_get_cross_mapped_mask(struct kvm_pmu *pmu) { - struct kvm_pmc *pmc = NULL; + u64 host_cross_mapped_mask; + struct kvm_pmc *pmc; int bit, hw_idx; + /* + * Provide a mask of counters that are cross-mapped between the guest + * and the host, i.e. where a guest PMC is mapped to a host PMC with a + * different index. PEBS records hold a PERF_GLOBAL_STATUS snapshot, + * and so PEBS-enabled counters need to hold the correct index so as + * not to confuse the guest. + */ + host_cross_mapped_mask = 0; + kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) { if (!pmc_is_locally_enabled(pmc) || !pmc_is_globally_enabled(pmc) || !pmc->perf_event) @@ -763,8 +773,9 @@ void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu) */ hw_idx = pmc->perf_event->hw.idx; if (hw_idx != pmc->idx && hw_idx > -1) - pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx); + host_cross_mapped_mask |= BIT_ULL(hw_idx); } + return host_cross_mapped_mask; } static bool intel_pmu_is_mediated_pmu_supported(struct x86_pmu_capability *host_pmu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..b097db296792 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7363,12 +7363,17 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) if (kvm_vcpu_has_mediated_pmu(&vmx->vcpu)) return; - pmu->host_cross_mapped_mask = 0; + struct x86_guest_pebs guest_pebs = { + .enable = pmu->pebs_enable, + .ds_area = pmu->ds_area, + .data_cfg = pmu->pebs_data_cfg, + }; + if (pmu->pebs_enable & pmu->global_ctrl) - intel_pmu_cross_mapped_check(pmu); + guest_pebs.cross_mapped_mask = intel_pmu_get_cross_mapped_mask(pmu); /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ - msrs = perf_guest_get_msrs(&nr_msrs, (void *)pmu); + msrs = perf_guest_get_msrs(&nr_msrs, &guest_pebs); if (!msrs) return; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..5c4e2cf6256c 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -664,7 +664,7 @@ static __always_inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) return container_of(vcpu, struct vcpu_vmx, vcpu); } -void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu); +u64 intel_pmu_get_cross_mapped_mask(struct kvm_pmu *pmu); int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu); void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu); From 31f7cf337ce717666ce29fbd793f06435eb48408 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:15 -0700 Subject: [PATCH 28/34] perf/x86/intel: KVM: Handle cross-mapped PEBS PMCs entirely within KVM Now that perf operates on a KVM-provided snapshot of PMU state, handled cross-mapped PEBS counters entirely in KVM by clearing unusable counters from the to-be-enabled mask instead of foisting the work on perf. No functional change intended. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-8-seanjc@google.com --- arch/x86/events/intel/core.c | 8 -------- arch/x86/include/asm/perf_event.h | 1 - arch/x86/kvm/vmx/vmx.c | 10 ++++++++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 87274b3bf503..1bc96b9fe0e4 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -5389,14 +5389,6 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, ~cpuc->intel_ctrl_exclude_guest_mask & cpuc->intel_ctrl_exclude_host_mask; - /* - * Disable counters where the guest PMC is different than the host PMC - * being used on behalf of the guest, as the PEBS record includes - * PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the - * wrong counter(s). - */ - guest_pebs_mask &= ~guest_pebs->cross_mapped_mask; - /* * FIXME: Allow guest and host usage of PEBS events to co-exist instead * of disabling guest PEBS entirely if the host is using PEBS. diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 27f9c5bee7b0..5c92d43bbef8 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -823,7 +823,6 @@ struct x86_guest_pebs { u64 enable; u64 ds_area; u64 data_cfg; - u64 cross_mapped_mask; }; #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b097db296792..8f7d0620f44c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7369,8 +7369,14 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) .data_cfg = pmu->pebs_data_cfg, }; - if (pmu->pebs_enable & pmu->global_ctrl) - guest_pebs.cross_mapped_mask = intel_pmu_get_cross_mapped_mask(pmu); + /* + * Disable counters where the guest PMC is different than the host PMC + * being used on behalf of the guest, as the PEBS record includes + * PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the + * wrong counter(s). + */ + if (guest_pebs.enable & pmu->global_ctrl) + guest_pebs.enable &= ~intel_pmu_get_cross_mapped_mask(pmu); /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ msrs = perf_guest_get_msrs(&nr_msrs, &guest_pebs); From 57c76478333012159b5cbec89527f925011cdefd Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:16 -0700 Subject: [PATCH 29/34] KVM: VMX: Drop a redundant pmu->global_ctrl check when processing pebs_enable Drop a redundant check that a PMC is globally enabled when looking for PEBS counters that are cross-mapped between the guest and the host. The for-loop explicitly iterates over pmu->global_ctrl, and since PEBS requires PMU v2+, kvm_pmu_has_perf_global_ctrl() must be true, and thus pmc_is_globally_enabled() is simply checking that the bit is set in pmu->global_ctrl. No functional change intended. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-9-seanjc@google.com --- arch/x86/kvm/vmx/pmu_intel.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index a21e146ad4f2..040b8e80cf86 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -763,8 +763,7 @@ u64 intel_pmu_get_cross_mapped_mask(struct kvm_pmu *pmu) host_cross_mapped_mask = 0; kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) { - if (!pmc_is_locally_enabled(pmc) || - !pmc_is_globally_enabled(pmc) || !pmc->perf_event) + if (!pmc_is_locally_enabled(pmc) || !pmc->perf_event) continue; /* From 193ef44e32619550478fce9d0b91edc0046ed8c3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:17 -0700 Subject: [PATCH 30/34] KVM: VMX: Only tell perf to enable PEBS counters for fully enabled PMCs When passing the guest's requested PEBS_ENABLE (or rather, KVM's version of PEBS_ENABLE on behalf of the guest), omit counters that are locally disable and/or don't have a perf event (due to contention), in addition to omitting counters that are cross-mapped in the host. In practice, this should be a nop as perf will already have disabled the associated counter, i.e. cpuc->pebs_enabled should have been cleared, but paranoia is cheap, and the existing code _looks_ wrong. Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-10-seanjc@google.com --- arch/x86/kvm/vmx/pmu_intel.c | 30 ++++++++++++++++-------------- arch/x86/kvm/vmx/vmx.c | 11 +---------- arch/x86/kvm/vmx/vmx.h | 15 ++++++++++++++- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 040b8e80cf86..472cf51298b4 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -747,34 +747,36 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) intel_pmu_release_guest_lbr_event(vcpu); } -u64 intel_pmu_get_cross_mapped_mask(struct kvm_pmu *pmu) +u64 __intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu) { - u64 host_cross_mapped_mask; + u64 guest_pebs_enable = pmu->pebs_enable & pmu->global_ctrl; + u64 pebs_enable = 0; struct kvm_pmc *pmc; int bit, hw_idx; /* - * Provide a mask of counters that are cross-mapped between the guest - * and the host, i.e. where a guest PMC is mapped to a host PMC with a - * different index. PEBS records hold a PERF_GLOBAL_STATUS snapshot, - * and so PEBS-enabled counters need to hold the correct index so as - * not to confuse the guest. + * Omit counters that are locally disabled, don't have a perf event, or + * ended up with a perf event that is using a different counter than + * the guest, i.e. where the guest PMC is different than the host PMC + * being used on behalf of the guest. PEBS records include + * PERF_GLOBAL_STATUS, and so using a counter with a different index + * means the guest will see overflow status for the wrong counter(s). */ - host_cross_mapped_mask = 0; - - kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) { + kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&guest_pebs_enable) { if (!pmc_is_locally_enabled(pmc) || !pmc->perf_event) continue; /* - * A negative index indicates the event isn't mapped to a + * Note, a negative index indicates the event isn't mapped to a * physical counter in the host, e.g. due to contention. */ hw_idx = pmc->perf_event->hw.idx; - if (hw_idx != pmc->idx && hw_idx > -1) - host_cross_mapped_mask |= BIT_ULL(hw_idx); + if (hw_idx != pmc->idx) + continue; + + pebs_enable |= BIT_ULL(pmc->idx); } - return host_cross_mapped_mask; + return pebs_enable; } static bool intel_pmu_is_mediated_pmu_supported(struct x86_pmu_capability *host_pmu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 8f7d0620f44c..2d0443562a16 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7364,20 +7364,11 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) return; struct x86_guest_pebs guest_pebs = { - .enable = pmu->pebs_enable, + .enable = intel_pmu_compute_pebs_enable(pmu), .ds_area = pmu->ds_area, .data_cfg = pmu->pebs_data_cfg, }; - /* - * Disable counters where the guest PMC is different than the host PMC - * being used on behalf of the guest, as the PEBS record includes - * PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the - * wrong counter(s). - */ - if (guest_pebs.enable & pmu->global_ctrl) - guest_pebs.enable &= ~intel_pmu_get_cross_mapped_mask(pmu); - /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ msrs = perf_guest_get_msrs(&nr_msrs, &guest_pebs); if (!msrs) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 5c4e2cf6256c..1db461060c7e 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -664,7 +664,20 @@ static __always_inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) return container_of(vcpu, struct vcpu_vmx, vcpu); } -u64 intel_pmu_get_cross_mapped_mask(struct kvm_pmu *pmu); +u64 __intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu); + +static inline u64 intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu) +{ + /* + * Avoid the function call overhead in the common case that the guest + * isn't using PEBS. + */ + if (!(pmu->pebs_enable & pmu->global_ctrl)) + return 0; + + return __intel_pmu_compute_pebs_enable(pmu); +} + int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu); void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu); From ba29babd881e09a7bb8b01fcfeda3d637d8115c5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 12:14:18 -0700 Subject: [PATCH 31/34] perf/x86/intel: Check only PMC bits in PEBS_ENABLED when detecting host PEBS usage When checking to see if guest PEBS usage should be disabled due to host PEBS usage, filter out non-counter bits in PEBS_ENABLE to avoid getting false positives in the unlikely scenario that "Enable Precise Store" is enabled without PEBS actually being active for any PMCs. Note, the other "hidden" usage of pebs_mask is already explicitly masked with intel_ctrl: .guest = intel_ctrl & ~cpuc->intel_ctrl_exclude_guest_mask & ~pebs_mask, I.e. removing the non-counter bits from pebs_mask doesn't change the value that is loaded into arr[global_ctrl].guest. Fixes: 854250329c02 ("KVM: x86/pmu: Disable guest PEBS temporarily in two rare situations") Closes: https://lore.kernel.org/all/20260508234018.F06AEC2BCB0@smtp.kernel.org Reported-by: Sashiko Bot Suggested-by: Dapeng Mi Signed-off-by: Sean Christopherson Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dapeng Mi Link: https://patch.msgid.link/20260921191418.950933-11-seanjc@google.com --- arch/x86/events/intel/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 1bc96b9fe0e4..0a34d674df59 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -5333,7 +5333,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); - u64 pebs_mask = cpuc->pebs_enabled & x86_pmu.pebs_capable; + u64 pebs_mask = intel_ctrl & cpuc->pebs_enabled & x86_pmu.pebs_capable; u64 guest_pebs_mask; int global_ctrl; @@ -5385,7 +5385,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, * the guest wants to use for PEBS, (c) are not excluded from counting * in the guest, and (d) _are_ excluded from counting in the host. */ - guest_pebs_mask = pebs_mask & intel_ctrl & guest_pebs->enable & + guest_pebs_mask = pebs_mask & guest_pebs->enable & ~cpuc->intel_ctrl_exclude_guest_mask & cpuc->intel_ctrl_exclude_host_mask; From 4a8557a4e5d2d5f96a989a428e38569451e74405 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 21 Sep 2026 08:30:14 -0700 Subject: [PATCH 32/34] perf/x86/amd/uncore: Remove redundant event slot scan amd_uncore_add() first checks the slot recorded in event->hw.idx. If that misses, it scans ctx->events[] for the event before looking for a free slot. Perf serializes ->add() and ->del() for an event. Initialization sets idx to -1. A successful ->add() claims a slot and records its index before returning, while ->del() clears the slot before resetting idx. CPU context migration follows the same delete/add sequence. Thus an installed event can only reside at the recorded index. If the direct check misses, the event is not present in ctx->events[]. Remove the redundant scan. This avoids walking all counters before each new slot search after multiplexing has scheduled an event out. On a host running a production workload in the Meta fleet, amd_uncore_add() was called 56,575 times per second from mux rotation. Its 16-counter DF PMU makes each failed search scan 16 pointers across two cache lines. Signed-off-by: Usama Arif Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sandipan Das Link: https://patch.msgid.link/20260921153108.2713179-2-usama.arif@linux.dev --- arch/x86/events/amd/uncore.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 74b9c92d517a..c1158f3691d4 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -206,17 +206,15 @@ static int amd_uncore_add(struct perf_event *event, int flags) struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); struct hw_perf_event *hwc = &event->hw; - /* are we already assigned? */ + /* + * Perf serializes ->add() and ->del() for an event. A successful + * ->add() records the claimed slot in hwc->idx before returning, and + * ->del() clears that slot before resetting hwc->idx. Therefore, an + * existing assignment must be at hwc->idx. + */ if (hwc->idx != -1 && ctx->events[hwc->idx] == event) goto out; - for (i = 0; i < pmu->num_counters; i++) { - if (ctx->events[i] == event) { - hwc->idx = i; - goto out; - } - } - /* if not, take the first available counter */ hwc->idx = -1; for (i = 0; i < pmu->num_counters; i++) { From 6350de8671b94afb7691d110f63bcda42f658a69 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 21 Sep 2026 08:30:15 -0700 Subject: [PATCH 33/34] perf/x86/amd/uncore: Free counter slot by index amd_uncore_del() scans ctx->events[] from slot zero and attempts a compare-exchange until it finds the event. amd_uncore_add() records the slot it claimed in event->hw.idx. Perf calls ->del() only after a successful ->add(), and the driver never moves an installed event between slots. The index therefore remains valid until deletion. Use the recorded index directly and retain the compare-exchange ownership check. Warn if it fails, as that means the driver lost track of the slot. Deleting all events from a full PMU with N counters now requires N compare-exchanges instead of N * (N + 1) / 2. On a host running a production workload in the Meta fleet, amd_uncore_del() was called 56,575 times per second from mux rotation. Signed-off-by: Usama Arif Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sandipan Das Link: https://patch.msgid.link/20260921153108.2713179-3-usama.arif@linux.dev --- arch/x86/events/amd/uncore.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index c1158f3691d4..7aa5a5b652a2 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -246,19 +246,15 @@ static int amd_uncore_add(struct perf_event *event, int flags) static void amd_uncore_del(struct perf_event *event, int flags) { - int i; struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); struct hw_perf_event *hwc = &event->hw; + struct perf_event *old = event; event->pmu->stop(event, PERF_EF_UPDATE); - for (i = 0; i < pmu->num_counters; i++) { - struct perf_event *tmp = event; - - if (try_cmpxchg(&ctx->events[i], &tmp, NULL)) - break; - } + /* ->del() follows a successful ->add(), so hwc->idx owns this slot. */ + WARN_ON_ONCE(!try_cmpxchg(&ctx->events[hwc->idx], &old, NULL)); hwc->idx = -1; } From f5a78ecccd98a04c47cea1cd63c2a2fdce8a4b48 Mon Sep 17 00:00:00 2001 From: vineash Date: Tue, 13 Oct 2026 16:13:20 +0800 Subject: [PATCH 34/34] perf/x86: Don't WARN when the ext_regs_buf allocation fails reserve_ext_regs_buffers() wraps the per-CPU ext_regs_buf allocation in WARN_ON_ONCE(). The allocator already prints a warning with a stack dump when a GFP_KERNEL allocation fails, so the WARN only repeats it, which Documentation/process/coding-style.rst asks not to do. The failure is also handled: the buffers are freed again, the event is still created, and events that ask for SIMD or extended register sampling are rejected with -ENOMEM in x86_pmu_hw_config() and event_simd_regs_validate(). syzkaller hit this with fault injection on next-20261002. The trigger is a plain hardware cycles event opened while no other x86 PMU event exists, with the allocation failed: !per_cpu(ext_regs_buf, cpu) WARNING: arch/x86/events/core.c:441 at reserve_ext_regs_buffers arch/x86/events/core.c:441 [inline] WARNING: arch/x86/events/core.c:441 at x86_reserve_hardware+0x6ff/0x880 arch/x86/events/core.c:458 Call Trace: __x86_pmu_event_init arch/x86/events/core.c:825 [inline] x86_pmu_event_init+0x109/0xc50 arch/x86/events/core.c:3003 perf_try_init_event+0x140/0xbc0 kernel/events/core.c:13261 perf_init_event kernel/events/core.c:13365 [inline] perf_event_alloc+0x2145/0x5680 kernel/events/core.c:13640 __do_sys_perf_event_open+0x669/0x2d40 kernel/events/core.c:14185 do_syscall_64+0x12c/0x7b0 arch/x86/entry/syscall_64.c:84 entry_SYSCALL_64_after_hwframe+0x77/0x7f Drop the WARN_ON_ONCE(). This matches reserve_ds_buffers() and reserve_lbr_buffers(), which also handle allocation failure without a warning. Fixes: b84c96283684 ("perf/x86: Enable XMM register sampling for non-PEBS events") Assisted-by: Claude:claude-opus-5-5 syzkaller Signed-off-by: vineash Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/179187920064.3441876.15687522011658301558@gmail.com --- arch/x86/events/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 9b1a9032e36d..4df792b139e0 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -438,7 +438,7 @@ static void reserve_ext_regs_buffers(void) for_each_possible_cpu(cpu) { per_cpu(ext_regs_buf, cpu) = kzalloc_node(size, GFP_KERNEL, cpu_to_node(cpu)); - if (WARN_ON_ONCE(!per_cpu(ext_regs_buf, cpu))) + if (!per_cpu(ext_regs_buf, cpu)) goto err; }