github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-10-22 23:01:49 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Perf events updates:

Browse Source 
- Support SIMD/eGPRs/SSP register sampling in perf
    (Dapeng Mi, Kan Liang)
 
  - Turn amd_uncore_ctx events into a flexible array (Rosen Penev)
 
  - Finish the work related to fixing the guest PEBS path to not
    set unwanted PERF_GLOBAL_CTRL bits, for which urgent fixes
    are upstream already (Sean Christopherson)
 
  - Drop redundant counter slot searches in the AMD uncore
    driver (Usama Arif)
 
  - Don't WARN when the ext_regs_buf allocation fails (vineash)
 
  - Annotate x86_pmu::hybrid_pmu with __counted_by_ptr (Thorsten Blum)
 
 Signed-off-by: Ingo Molnar <mingo@kernel.org>
 -----BEGIN PGP SIGNATURE-----
 
 iQJFBAABCgAvFiEEBpT5eoXrXCwVQwEKEnMQ0APhK1gFAmrUmW4RHG1pbmdvQGtl
 cm5lbC5vcmcACgkQEnMQ0APhK1hhcA/9GLLMd5OOBjw8i2ckWMOoi5UgmflkApBa
 BiAV9jEaIt3AaOS4hUe1Ttnrk02rWuq1NppwJm4n4foxysEI54ZwMfsWYeXLjdQ7
 Lw5Q+4/NwvuMPkE8pcasyM+OYE7BOnNRZijjqcDycYHcx0E464n4/sWFG9HQxhVa
 mNTLNvix9tIQTLjBTWWJdBDWYMvMlqvkjqCYNAf3CPDWP718pc29ilvZ3HW/qh2w
 iLELMP2+wPInfB7mDNy5aSdv0nLuSZHtmFshYSMK0d1MfIYd1a8p7XEYQ61iOCMz
 ZXKO3h21Ri0Ze7c3v6URlu3JsDv6lWiFyqW+ZyG9XFkmzam0Lk/HqFEg84QKUbPt
 9qiMxpLKQFhjumJRacZiswb/pKHGPbYQGkjt3wl5J5wSikiB57VnF+yPqwXFAlmx
 LIa7qLThGSy9Ju6dqbbpEkiiRkl7+Gb9zPAISbTjse9BdXiCUCKihUT689nAfmqT
 t7bTwocrYvSZiNK1ipEf+ARRb9lqLBF6LZ59WsFlVddMfiBb/f1x0b0M5sVEKucv
 JMOywESH6SFE0j533WA6WY6+iRG+FFa+k7mu78RO2nnKSXEXsLDQ5o3iyhQ2B4uD
 xaUq7J4DTRfu6XU1bK4HpBnKDgWzdkLrD+m8mTKPeNW5kMZC6QYakjQtqpC3Qks4
 liX/+yGieJs=
 =+I0p
 -----END PGP SIGNATURE-----

Merge tag 'perf-core-2026-10-18' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull perf events updates from Ingo Molnar:

 - Support SIMD/eGPRs/SSP register sampling in perf
   (Dapeng Mi, Kan Liang)

 - Turn amd_uncore_ctx events into a flexible array (Rosen Penev)

 - Finish the work related to fixing the guest PEBS path to not
   set unwanted PERF_GLOBAL_CTRL bits, for which urgent fixes
   are upstream already (Sean Christopherson)

 - Drop redundant counter slot searches in the AMD uncore
   driver (Usama Arif)

 - Don't WARN when the ext_regs_buf allocation fails (vineash)

 - Annotate x86_pmu::hybrid_pmu with __counted_by_ptr (Thorsten Blum)

* tag 'perf-core-2026-10-18' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (34 commits)
  perf/x86: Don't WARN when the ext_regs_buf allocation fails
  perf/x86/amd/uncore: Free counter slot by index
  perf/x86/amd/uncore: Remove redundant event slot scan
  perf/x86/intel: Check only PMC bits in PEBS_ENABLED when detecting host PEBS usage
  KVM: VMX: Only tell perf to enable PEBS counters for fully enabled PMCs
  KVM: VMX: Drop a redundant pmu->global_ctrl check when processing pebs_enable
  perf/x86/intel: KVM: Handle cross-mapped PEBS PMCs entirely within KVM
  perf/x86: KVM: Have perf define a dedicated struct for getting guest PEBS data
  perf/x86/intel: Invert names of intel_ctrl_{guest,host}_mask
  perf/x86/intel: Annotate x86_pmu::hybrid_pmu with __counted_by_ptr
  perf/x86/amd/uncore: Turn amd_uncore_ctx events into a flexible array
  perf/x86/intel: Add sanity check for PEBS record/fragment size
  perf/x86: Activate back-to-back NMI detection for arch-PEBS induced NMIs
  perf/x86/intel: Advertise PERF_PMU_CAP_SIMD_REGS capability
  perf/x86/intel: Support arch-PEBS based SIMD/eGPRs sampling
  perf/x86: Support SSP sampling using sample_regs_* fields
  perf/x86: Support eGPRs sampling using sample_regs_* fields
  perf: Enhance perf_reg_validate() with simd_enabled argument
  perf/x86: Support OPMASK sampling using sample_simd_pred_reg_* fields
  perf/x86: Support ZMM sampling using sample_simd_vec_reg_* fields
  ...
This commit is contained in:
Linus Torvalds 2026-10-20 13:00:41 -10:00
commit 4cbdd58dfa
32 changed files with 1575 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
arch/arm/kernel/perf_regs.c
View File

@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
#define REG_RESERVED (~((1ULL << PERF_REG_ARM_MAX) - 1))
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask || mask & REG_RESERVED)
return -EINVAL;
@ -31,9 +31,3 @@ u64 perf_reg_abi(struct task_struct *task)
return PERF_SAMPLE_REGS_ABI_32;
}
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}

8
arch/arm64/kernel/perf_regs.c
View File

@ -77,7 +77,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
#define REG_RESERVED (~((1ULL << PERF_REG_ARM64_MAX) - 1))
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
u64 reserved_mask = REG_RESERVED;
@ -98,9 +98,3 @@ u64 perf_reg_abi(struct task_struct *task)
return PERF_SAMPLE_REGS_ABI_64;
}
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}

8
arch/csky/kernel/perf_regs.c
View File

@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
#define REG_RESERVED (~((1ULL << PERF_REG_CSKY_MAX) - 1))
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask || mask & REG_RESERVED)
return -EINVAL;
@ -31,9 +31,3 @@ u64 perf_reg_abi(struct task_struct *task)
return PERF_SAMPLE_REGS_ABI_32;
}
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}

8
arch/loongarch/kernel/perf_regs.c
View File

@ -25,7 +25,7 @@ u64 perf_reg_abi(struct task_struct *tsk)
}
#endif /* CONFIG_32BIT */
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask)
return -EINVAL;
@ -45,9 +45,3 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
return regs->regs[idx];
}
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}

8
arch/mips/kernel/perf_regs.c
View File

@ -28,7 +28,7 @@ u64 perf_reg_abi(struct task_struct *tsk)
}
#endif /* CONFIG_32BIT */
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask)
return -EINVAL;
@ -60,9 +60,3 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
return (s64)v; /* Sign extend if 32-bit. */
}
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}

8
arch/parisc/kernel/perf_regs.c
View File

@ -34,7 +34,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
#define REG_RESERVED (~((1ULL << PERF_REG_PARISC_MAX) - 1))
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask || mask & REG_RESERVED)
return -EINVAL;
@ -53,9 +53,3 @@ u64 perf_reg_abi(struct task_struct *task)
return PERF_SAMPLE_REGS_ABI_64;
}
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}

2
arch/powerpc/perf/perf_regs.c
View File

@ -125,7 +125,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
return regs_get_register(regs, pt_regs_offset[idx]);
}
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask || mask & REG_RESERVED)
return -EINVAL;

8
arch/riscv/kernel/perf_regs.c
View File

@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
#define REG_RESERVED (~((1ULL << PERF_REG_RISCV_MAX) - 1))
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask || mask & REG_RESERVED)
return -EINVAL;
@ -35,9 +35,3 @@ u64 perf_reg_abi(struct task_struct *task)
#endif
}
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}

2
arch/s390/kernel/perf_regs.c
View File

@ -34,7 +34,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
#define REG_RESERVED (~((1UL << PERF_REG_S390_MAX) - 1))
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask || mask & REG_RESERVED)
return -EINVAL;

39
arch/x86/events/amd/uncore.c
View File

@ -39,11 +39,11 @@ static int pmu_version;
struct amd_uncore_ctx {
int refcnt;
int cpu;
struct perf_event **events;
unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)];
int nr_active;
struct hrtimer hrtimer;
u64 hrtimer_duration;
struct perf_event *events[];
};
struct amd_uncore_pmu {
@ -206,17 +206,15 @@ static int amd_uncore_add(struct perf_event *event, int flags)
struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
struct hw_perf_event *hwc = &event->hw;
/* are we already assigned? */
/*
* Perf serializes ->add() and ->del() for an event. A successful
* ->add() records the claimed slot in hwc->idx before returning, and
* ->del() clears that slot before resetting hwc->idx. Therefore, an
* existing assignment must be at hwc->idx.
*/
if (hwc->idx != -1 && ctx->events[hwc->idx] == event)
goto out;
for (i = 0; i < pmu->num_counters; i++) {
if (ctx->events[i] == event) {
hwc->idx = i;
goto out;
}
}
/* if not, take the first available counter */
hwc->idx = -1;
for (i = 0; i < pmu->num_counters; i++) {
@ -248,19 +246,15 @@ static int amd_uncore_add(struct perf_event *event, int flags)
static void amd_uncore_del(struct perf_event *event, int flags)
{
int i;
struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
struct hw_perf_event *hwc = &event->hw;
struct perf_event *old = event;
event->pmu->stop(event, PERF_EF_UPDATE);
for (i = 0; i < pmu->num_counters; i++) {
struct perf_event *tmp = event;
if (try_cmpxchg(&ctx->events[i], &tmp, NULL))
break;
}
/* ->del() follows a successful ->add(), so hwc->idx owns this slot. */
WARN_ON_ONCE(!try_cmpxchg(&ctx->events[hwc->idx], &old, NULL));
hwc->idx = -1;
}
@ -519,10 +513,8 @@ static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu)
if (cpu == ctx->cpu)
cpumask_clear_cpu(cpu, &pmu->active_mask);
if (!--ctx->refcnt) {
kfree(ctx->events);
if (!--ctx->refcnt)
kfree(ctx);
}
*per_cpu_ptr(pmu->ctx, cpu) = NULL;
}
@ -567,18 +559,11 @@ static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
/* Allocate context if sibling does not exist */
if (!curr) {
node = cpu_to_node(cpu);
curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node);
curr = kzalloc_node(struct_size(curr, events, pmu->num_counters), GFP_KERNEL, node);
if (!curr)
goto fail;
curr->cpu = cpu;
curr->events = kzalloc_node(sizeof(*curr->events) *
pmu->num_counters,
GFP_KERNEL, node);
if (!curr->events) {
kfree(curr);
goto fail;
}
amd_uncore_init_hrtimer(curr);
curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC;

488
arch/x86/events/core.c
View File

@ -93,7 +93,7 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling);
DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task);
DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs);
DEFINE_STATIC_CALL_RET0(x86_pmu_drain_pebs, *x86_pmu.drain_pebs);
DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);
DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter);
@ -408,6 +408,53 @@ set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
return x86_pmu_extra_regs(val, event);
}
static DEFINE_PER_CPU(struct xregs_state *, ext_regs_buf);
static void release_ext_regs_buffers(void)
{
int cpu;
if (!x86_pmu.ext_regs_mask)
return;
for_each_possible_cpu(cpu) {
kfree(per_cpu(ext_regs_buf, cpu));
per_cpu(ext_regs_buf, cpu) = NULL;
}
}
static void reserve_ext_regs_buffers(void)
{
bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);
unsigned int size;
int cpu;
if (!x86_pmu.ext_regs_mask)
return;
/* Add 64 bytes to satisfy the XSAVE area's 64-byte alignment. */
size = xstate_calculate_size(x86_pmu.ext_regs_mask, compacted) + 64;
for_each_possible_cpu(cpu) {
per_cpu(ext_regs_buf, cpu) = kzalloc_node(size, GFP_KERNEL,
cpu_to_node(cpu));
if (!per_cpu(ext_regs_buf, cpu))
goto err;
}
return;
err:
release_ext_regs_buffers();
}
static inline struct xregs_state *get_ext_regs_buf(int cpu)
{
void *buf = per_cpu(ext_regs_buf, cpu);
return buf ? PTR_ALIGN(buf, 64) : NULL;
}
int x86_reserve_hardware(void)
{
int err = 0;
@ -420,6 +467,7 @@ int x86_reserve_hardware(void)
} else {
reserve_ds_buffers();
reserve_lbr_buffers();
reserve_ext_regs_buffers();
}
}
if (!err)
@ -436,6 +484,7 @@ void x86_release_hardware(void)
release_pmc_hardware();
release_ds_buffers();
release_lbr_buffers();
release_ext_regs_buffers();
mutex_unlock(&pmc_reserve_mutex);
}
}
@ -583,6 +632,79 @@ int x86_pmu_max_precise(struct pmu *pmu)
return precise;
}
static int pebs_simd_regs_validate(struct perf_event *event)
{
u64 caps = hybrid(event->pmu, arch_pebs_cap).caps;
if (event_needs_xmm(event) &&
!x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline)
return -EINVAL;
if (event_needs_xmm(event) &&
x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM))
return -EINVAL;
if (event_needs_ssp(event) &&
!(x86_pmu.arch_pebs && (caps & ARCH_PEBS_GPR)))
return -EINVAL;
if (event_needs_ymm(event) &&
!(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_YMMH)))
return -EINVAL;
if (event_needs_egprs(event) &&
!(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_EGPRS)))
return -EINVAL;
if (event_needs_opmask(event) &&
!(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_OPMASK)))
return -EINVAL;
if (event_needs_low16_zmm(event) &&
!(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_ZMMH)))
return -EINVAL;
if (event_needs_high16_zmm(event) &&
!(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_H16ZMM)))
return -EINVAL;
return 0;
}
static int event_simd_regs_validate(struct perf_event *event)
{
u64 reserved = ~GENMASK_ULL(PERF_REG_MISC_MAX - 1, 0);
if (!get_ext_regs_buf(raw_smp_processor_id()))
return -ENOMEM;
/*
* The XMM space in the perf_event_x86_regs is reclaimed
* for eGPRs and other general registers.
*/
if (((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_regs_intr & reserved)) ||
((event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_regs_user & reserved)))
return -EINVAL;
if (event_needs_egprs(event) &&
!(x86_pmu.ext_regs_mask & XFEATURE_MASK_APX))
return -EINVAL;
if (event_needs_ssp(event) &&
!(x86_pmu.ext_regs_mask & XFEATURE_MASK_CET_USER))
return -EINVAL;
if (event_needs_xmm(event) &&
!(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE))
return -EINVAL;
if (event_needs_ymm(event) &&
!(x86_pmu.ext_regs_mask & XFEATURE_MASK_YMM))
return -EINVAL;
if (event_needs_low16_zmm(event) &&
!(x86_pmu.ext_regs_mask & XFEATURE_MASK_ZMM_Hi256))
return -EINVAL;
if (event_needs_high16_zmm(event) &&
!(x86_pmu.ext_regs_mask & XFEATURE_MASK_Hi16_ZMM))
return -EINVAL;
if (event_needs_opmask(event) &&
!(x86_pmu.ext_regs_mask & XFEATURE_MASK_OPMASK))
return -EINVAL;
return 0;
}
int x86_pmu_hw_config(struct perf_event *event)
{
if (event->attr.precise_ip) {
@ -653,19 +775,38 @@ int x86_pmu_hw_config(struct perf_event *event)
return -EINVAL;
}
/* sample_regs_user never support XMM registers */
if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
return -EINVAL;
/*
* Besides the general purpose registers, XMM registers may
* be collected in PEBS on some platforms, e.g. Icelake
*/
if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) {
if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
return -EINVAL;
if (event->attr.sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) {
int ret;
if (!event->attr.precise_ip)
return -EINVAL;
if (event->attr.sample_simd_regs_enabled) {
if (!(event->pmu->capabilities & PERF_PMU_CAP_SIMD_REGS))
return -EINVAL;
if (event->attr.precise_ip) {
ret = pebs_simd_regs_validate(event);
if (ret)
return ret;
}
ret = event_simd_regs_validate(event);
if (ret)
return ret;
} else if (event_has_extended_regs(event)) {
if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
return -EINVAL;
if (event->attr.precise_ip) {
u64 caps = hybrid(event->pmu, arch_pebs_cap).caps;
if (x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM))
return -EINVAL;
if (!x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline)
return -EINVAL;
}
if (!get_ext_regs_buf(raw_smp_processor_id()))
return -ENOMEM;
if (!(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE))
return -EINVAL;
}
}
return x86_setup_perfctr(event);
@ -721,9 +862,10 @@ void x86_pmu_disable_all(void)
}
}
struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data)
struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr,
struct x86_guest_pebs *guest_pebs)
{
return static_call(x86_pmu_guest_get_msrs)(nr, data);
return static_call(x86_pmu_guest_get_msrs)(nr, guest_pebs);
}
EXPORT_SYMBOL_FOR_KVM(perf_guest_get_msrs);
@ -1717,6 +1859,302 @@ static void x86_pmu_del(struct perf_event *event, int flags)
static_call_cond(x86_pmu_del)(event);
}
void x86_pmu_clear_perf_regs(struct pt_regs *regs)
{
struct x86_perf_regs *perf_regs = container_of(regs, struct x86_perf_regs, regs);
perf_regs->abi = PERF_SAMPLE_REGS_ABI_NONE;
perf_regs->xmm_regs = NULL;
perf_regs->ymmh_regs = NULL;
perf_regs->zmmh_regs = NULL;
perf_regs->h16zmm_regs = NULL;
perf_regs->opmask_regs = NULL;
perf_regs->egpr_regs = NULL;
perf_regs->ssp = NULL;
}
static void update_perf_regs(struct x86_perf_regs *perf_regs,
struct xregs_state *xsave, u64 bitmap)
{
struct cet_user_state *cet;
u64 mask;
if (!xsave)
return;
/* Restrict to features actually saved by XSAVES */
mask = bitmap & xsave->header.xfeatures;
if (mask & XFEATURE_MASK_SSE)
perf_regs->xmm_space = xsave->i387.xmm_space;
if (mask & XFEATURE_MASK_YMM)
perf_regs->ymmh = get_xsave_addr(xsave, XFEATURE_YMM);
if (mask & XFEATURE_MASK_ZMM_Hi256)
perf_regs->zmmh = get_xsave_addr(xsave, XFEATURE_ZMM_Hi256);
if (mask & XFEATURE_MASK_Hi16_ZMM)
perf_regs->h16zmm = get_xsave_addr(xsave, XFEATURE_Hi16_ZMM);
if (mask & XFEATURE_MASK_OPMASK)
perf_regs->opmask = get_xsave_addr(xsave, XFEATURE_OPMASK);
if (mask & XFEATURE_MASK_APX)
perf_regs->egpr = get_xsave_addr(xsave, XFEATURE_APX);
if (mask & XFEATURE_MASK_CET_USER) {
cet = get_xsave_addr(xsave, XFEATURE_CET_USER);
perf_regs->ssp = cet ? &cet->user_ssp : NULL;
}
}
/*
* The x86 specific variant of perf_sample_regs_intr().
* Update data->regs_intr fields for extended registers (e.g., SIMD).
*/
static void x86_pmu_update_regs_intr(struct perf_event *event,
struct perf_sample_data *data,
struct pt_regs *regs,
bool exclude_kernel)
{
struct x86_perf_regs *perf_regs;
if (exclude_kernel && !user_mode(regs)) {
data->regs_intr.regs = NULL;
data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE;
} else {
data->regs_intr.regs = regs;
data->regs_intr.abi = perf_reg_abi(current);
}
data->dyn_size += sizeof(u64);
if (data->regs_intr.regs) {
data->dyn_size += hweight64(event->attr.sample_regs_intr) *
sizeof(u64);
if (event_has_simd_regs(event)) {
data->dyn_size += perf_update_xregs_size(event, true);
data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD;
}
perf_regs = container_of(data->regs_intr.regs,
struct x86_perf_regs, regs);
perf_regs->abi = data->regs_intr.abi;
}
/*
* Set PERF_SAMPLE_REGS_INTR to bypass perf_sample_regs_intr() call
* in perf_prepare_sample() function.
*/
data->sample_flags |= PERF_SAMPLE_REGS_INTR;
}
static DEFINE_PER_CPU(struct x86_perf_regs, x86_user_regs);
static void x86_pmu_get_regs_user(struct perf_sample_data *data,
struct pt_regs *regs)
{
struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs);
struct perf_regs regs_user;
x86_pmu_clear_perf_regs(&x86_regs_user->regs);
perf_get_regs_user(&regs_user, regs);
data->regs_user.abi = regs_user.abi;
if (regs_user.regs) {
x86_regs_user->regs = *regs_user.regs;
data->regs_user.regs = &x86_regs_user->regs;
} else {
data->regs_user.regs = NULL;
}
}
/*
* The x86 specific variant of perf_sample_regs_user().
* Update data->regs_user fields for extended registers (e.g., SIMD).
*/
static void x86_pmu_update_regs_user(struct perf_event *event,
struct perf_sample_data *data,
struct pt_regs *regs)
{
struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs);
struct perf_event_attr *attr = &event->attr;
struct x86_perf_regs *perf_regs;
/*
* PERF_SAMPLE_REGS_INTR and PERF_SAMPLE_REGS_USER can both be
* requested for one event. Keep user regs in a separate x86_perf_regs
* instance, so intr-reg collection does not overwrite user-reg data.
*/
if (user_mode(regs)) {
x86_pmu_clear_perf_regs(&x86_regs_user->regs);
perf_regs = container_of(regs, struct x86_perf_regs, regs);
/* Copy all sampled regs data to x86_regs_user. */
*x86_regs_user = *perf_regs;
data->regs_user.regs = &x86_regs_user->regs;
data->regs_user.abi = perf_reg_abi(current);
} else if (is_user_task(current)) {
x86_pmu_get_regs_user(data, regs);
} else {
data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE;
data->regs_user.regs = NULL;
}
data->dyn_size += sizeof(u64);
if (data->regs_user.regs) {
data->dyn_size += hweight64(attr->sample_regs_user) * sizeof(u64);
if (event_has_simd_regs(event)) {
data->dyn_size += perf_update_xregs_size(event, false);
data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD;
}
x86_regs_user->abi = data->regs_user.abi;
}
/*
* Set PERF_SAMPLE_REGS_USER to bypass perf_sample_regs_user() call
* in perf_prepare_sample() function.
*/
data->sample_flags |= PERF_SAMPLE_REGS_USER;
}
/*
* This function retrieves cached user-space fpu registers (XMM/YMM/ZMM).
* If TIF_NEED_FPU_LOAD is set or PMI hits into guest, it indicates that
* the user-space FPU state is cached. Otherwise, the data should be read
* directly from the hardware registers.
*/
static inline u64 x86_pmu_update_user_xregs(struct perf_sample_data *data,
struct pt_regs *regs,
u64 mask, bool from_pebs)
{
struct x86_perf_regs *perf_regs;
struct xregs_state *xsave;
struct fpu *fpu;
struct fpstate *fps;
u64 user_mask = mask;
if (!is_user_task(current))
return 0;
if (data->regs_user.abi == PERF_SAMPLE_REGS_ABI_NONE)
return 0;
/*
* If PEBS hits kernel space, need to re-sample extended
* registers for user space.
*/
if (user_mode(regs))
user_mask = from_pebs ? 0 : mask;
fpu = x86_task_fpu(current);
fps = READ_ONCE(fpu->__task_fpstate);
/*
* If fpu->__task_fpstate is set, it points to the cached user
* FPU state (e.g. with KVM guest-state swapping). Otherwise,
* when TIF_NEED_FPU_LOAD is set, fpu->fpstate holds the cached
* user state. If neither is true, the user state is live in hardware.
*/
if (user_mask && (test_thread_flag(TIF_NEED_FPU_LOAD) || fps)) {
perf_regs = container_of(data->regs_user.regs,
struct x86_perf_regs, regs);
if (!fps)
fps = fpu->fpstate;
xsave = &fps->regs.xsave;
update_perf_regs(perf_regs, xsave, user_mask);
return 0;
}
return user_mask;
}
static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type)
{
u64 mask = 0;
if (__event_needs_xmm(event, sample_type))
mask |= XFEATURE_MASK_SSE;
if (__event_needs_ymm(event, sample_type))
mask |= XFEATURE_MASK_YMM;
if (__event_needs_low16_zmm(event, sample_type))
mask |= XFEATURE_MASK_ZMM_Hi256;
if (__event_needs_high16_zmm(event, sample_type))
mask |= XFEATURE_MASK_Hi16_ZMM;
if (__event_needs_opmask(event, sample_type))
mask |= XFEATURE_MASK_OPMASK;
if (__event_needs_egprs(event, sample_type))
mask |= XFEATURE_MASK_APX;
if (__event_needs_ssp(event, sample_type))
mask |= XFEATURE_MASK_CET_USER;
return mask;
}
static void x86_pmu_sample_xregs(struct perf_event *event,
struct perf_sample_data *data,
struct pt_regs *regs,
bool from_pebs)
{
struct xregs_state *xsave = get_ext_regs_buf(smp_processor_id());
u64 sample_type = event->attr.sample_type;
struct x86_perf_regs *perf_regs;
u64 intr_mask = 0;
u64 user_mask = 0;
if (WARN_ON_ONCE(!xsave) || !in_nmi())
return;
if ((sample_type & PERF_SAMPLE_REGS_INTR) && data->regs_intr.regs) {
intr_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_INTR);
intr_mask &= x86_pmu.ext_regs_mask;
intr_mask = from_pebs ? 0 : intr_mask;
}
if ((sample_type & PERF_SAMPLE_REGS_USER) && data->regs_user.regs) {
user_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_USER);
user_mask &= x86_pmu.ext_regs_mask;
user_mask = x86_pmu_update_user_xregs(data, regs,
user_mask, from_pebs);
}
if (user_mask | intr_mask) {
xsave->header.xfeatures = 0;
xsaves_nmi(xsave, user_mask | intr_mask);
}
if (intr_mask) {
perf_regs = container_of(data->regs_intr.regs,
struct x86_perf_regs, regs);
update_perf_regs(perf_regs, xsave, intr_mask);
}
if (user_mask) {
perf_regs = container_of(data->regs_user.regs,
struct x86_perf_regs, regs);
update_perf_regs(perf_regs, xsave, user_mask);
}
}
void x86_pmu_update_perf_regs(struct perf_event *event,
struct perf_sample_data *data,
struct pt_regs *regs,
bool from_pebs)
{
u64 sample_type = event->attr.sample_type;
if (!(sample_type &
(PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)))
return;
if (!event_needs_xmm(event) &&
!event_has_simd_regs(event))
return;
if (sample_type & PERF_SAMPLE_REGS_INTR) {
x86_pmu_update_regs_intr(event, data, regs,
event->attr.exclude_kernel);
}
if (sample_type & PERF_SAMPLE_REGS_USER)
x86_pmu_update_regs_user(event, data, regs);
x86_pmu_sample_xregs(event, data, regs, from_pebs);
}
int x86_pmu_handle_irq(struct pt_regs *regs)
{
struct perf_sample_data data;
@ -1800,9 +2238,11 @@ void perf_put_guest_lvtpc(void)
EXPORT_SYMBOL_FOR_KVM(perf_put_guest_lvtpc);
#endif /* CONFIG_PERF_GUEST_MEDIATED_PMU */
static DEFINE_PER_CPU(struct x86_perf_regs, x86_intr_regs);
static int
perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{
struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_intr_regs);
u64 start_clock;
u64 finish_clock;
int ret;
@ -1826,7 +2266,8 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
return NMI_DONE;
start_clock = sched_clock();
ret = static_call(x86_pmu_handle_irq)(regs);
x86_regs->regs = *regs;
ret = static_call(x86_pmu_handle_irq)(&x86_regs->regs);
finish_clock = sched_clock();
perf_sample_event_took(finish_clock - start_clock);
@ -2218,8 +2659,20 @@ static int __init init_hw_perf_events(void)
pmu.attr_update = x86_pmu.attr_update;
if (!is_hybrid())
if (!is_hybrid()) {
x86_pmu_show_pmu_cap(NULL);
} else {
int i;
/*
* Init default ops.
* Must be called before registering x86_pmu_starting_cpu(),
* otherwise some key PMU fields, e.g., capabilities
* initialized in x86_pmu_starting_cpu(), would be overwritten.
*/
for (i = 0; i < x86_pmu.num_hybrid_pmus; i++)
x86_pmu.hybrid_pmu[i].pmu = pmu;
}
if (!x86_pmu.read)
x86_pmu.read = _x86_pmu_read;
@ -2266,7 +2719,6 @@ static int __init init_hw_perf_events(void)
for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
hybrid_pmu = &x86_pmu.hybrid_pmu[i];
hybrid_pmu->pmu = pmu;
hybrid_pmu->pmu.type = -1;
hybrid_pmu->pmu.attr_update = x86_pmu.attr_update;
hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE;

154
arch/x86/events/intel/core.c
View File

@ -14,7 +14,6 @@
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/nmi.h>
#include <linux/kvm_host.h>
#include <asm/cpufeature.h>
#include <asm/cpuid/api.h>
@ -2797,7 +2796,7 @@ static void __intel_pmu_enable_all(int added, bool pmi)
}
wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL,
intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
intel_ctrl & ~cpuc->intel_ctrl_exclude_host_mask);
if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
struct perf_event *event =
@ -2995,9 +2994,9 @@ static inline void intel_set_masks(struct perf_event *event, int idx)
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
if (event->attr.exclude_host)
__set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask);
__set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask);
if (event->attr.exclude_guest)
__set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask);
__set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_guest_mask);
if (event_is_checkpointed(event))
__set_bit(idx, (unsigned long *)&cpuc->intel_cp_status);
}
@ -3006,8 +3005,8 @@ static inline void intel_clear_masks(struct perf_event *event, int idx)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
__clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask);
__clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask);
__clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask);
__clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_guest_mask);
__clear_bit(idx, (unsigned long *)&cpuc->intel_cp_status);
}
@ -3499,6 +3498,21 @@ static void intel_pmu_enable_event_ext(struct perf_event *event)
if (pebs_data_cfg & PEBS_DATACFG_XMMS)
ext |= ARCH_PEBS_VECR_XMM & cap.caps;
if (pebs_data_cfg & PEBS_DATACFG_YMMHS)
ext |= ARCH_PEBS_VECR_YMMH & cap.caps;
if (pebs_data_cfg & PEBS_DATACFG_EGPRS)
ext |= ARCH_PEBS_VECR_EGPRS & cap.caps;
if (pebs_data_cfg & PEBS_DATACFG_OPMASKS)
ext |= ARCH_PEBS_VECR_OPMASK & cap.caps;
if (pebs_data_cfg & PEBS_DATACFG_ZMMHS)
ext |= ARCH_PEBS_VECR_ZMMH & cap.caps;
if (pebs_data_cfg & PEBS_DATACFG_H16ZMMS)
ext |= ARCH_PEBS_VECR_H16ZMM & cap.caps;
if (pebs_data_cfg & PEBS_DATACFG_LBRS)
ext |= ARCH_PEBS_LBR & cap.caps;
@ -3770,20 +3784,20 @@ static void intel_pmu_reset(void)
*
* The contents and other behavior of the guest event do not matter.
*/
static void x86_pmu_handle_guest_pebs(struct pt_regs *regs,
struct perf_sample_data *data)
static int x86_pmu_handle_guest_pebs(struct pt_regs *regs,
struct perf_sample_data *data)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_host_mask;
u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_exclude_guest_mask;
struct perf_event *event = NULL;
int bit;
if (!unlikely(perf_guest_state()))
return;
return 0;
if (!x86_pmu.pebs_ept || !x86_pmu.pebs_active ||
!guest_pebs_idxs)
return;
return 0;
for_each_set_bit(bit, (unsigned long *)&guest_pebs_idxs, X86_PMC_IDX_MAX) {
event = cpuc->events[bit];
@ -3793,9 +3807,14 @@ static void x86_pmu_handle_guest_pebs(struct pt_regs *regs,
perf_sample_data_init(data, 0, event->hw.last_period);
perf_event_overflow(event, data, regs);
/* Inject one fake event is enough. */
break;
/*
* Inject one fake event is enough.
* Returning 1 to inform PMI is handled.
*/
return 1;
}
return 0;
}
static int handle_pmi_common(struct pt_regs *regs, u64 status)
@ -3844,9 +3863,11 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) {
u64 pebs_enabled = cpuc->pebs_enabled;
handled++;
x86_pmu_handle_guest_pebs(regs, &data);
static_call(x86_pmu_drain_pebs)(regs, &data);
handled += x86_pmu_handle_guest_pebs(regs, &data);
handled += static_call(x86_pmu_drain_pebs)(regs, &data);
/* Ensure no "suspicious NMI" warning for empty PEBS buffer. */
if (!handled)
handled++;
/*
* PMI throttle may be triggered, which stops the PEBS event.
@ -3873,8 +3894,10 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
*/
if (__test_and_clear_bit(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT,
(unsigned long *)&status)) {
handled++;
static_call(x86_pmu_drain_pebs)(regs, &data);
handled += static_call(x86_pmu_drain_pebs)(regs, &data);
/* Ensure no "suspicious NMI" warning for empty PEBS buffer. */
if (!handled)
handled++;
if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] &&
is_pebs_counter_event_group(cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS]))
@ -3950,6 +3973,9 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
if (has_branch_stack(event))
intel_pmu_lbr_save_brstack(&data, cpuc, event);
x86_pmu_clear_perf_regs(regs);
x86_pmu_update_perf_regs(event, &data, regs, false);
perf_event_overflow(event, &data, regs);
}
@ -4717,14 +4743,20 @@ static void intel_pebs_aliases_skl(struct perf_event *event)
static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event)
{
unsigned long flags = x86_pmu.large_pebs_flags;
u64 gprs_mask = event->attr.sample_simd_regs_enabled ?
PEBS_GP_REGS | PERF_X86_EGPRS_MASK |
BIT_ULL(PERF_REG_X86_SSP) :
PEBS_GP_REGS | PERF_REG_EXTENDED_MASK;
if (event->attr.use_clockid)
flags &= ~PERF_SAMPLE_TIME;
if (!event->attr.exclude_kernel)
flags &= ~PERF_SAMPLE_REGS_USER;
if (event->attr.sample_regs_user & ~PEBS_GP_REGS)
if ((event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_regs_user & ~gprs_mask))
flags &= ~PERF_SAMPLE_REGS_USER;
if (event->attr.sample_regs_intr & ~PEBS_GP_REGS)
if ((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_regs_intr & ~gprs_mask))
flags &= ~PERF_SAMPLE_REGS_INTR;
return flags;
}
@ -5295,13 +5327,13 @@ static int intel_pmu_hw_config(struct perf_event *event)
* when it uses {RD,WR}MSR, which should be handled by the KVM context,
* specifically in the intel_pmu_{get,set}_msr().
*/
static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr,
struct x86_guest_pebs *guest_pebs)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
struct kvm_pmu *kvm_pmu = (struct kvm_pmu *)data;
u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl);
u64 pebs_mask = cpuc->pebs_enabled & x86_pmu.pebs_capable;
u64 pebs_mask = intel_ctrl & cpuc->pebs_enabled & x86_pmu.pebs_capable;
u64 guest_pebs_mask;
int global_ctrl;
@ -5316,8 +5348,8 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
global_ctrl = (*nr)++;
arr[global_ctrl] = (struct perf_guest_switch_msr){
.msr = MSR_CORE_PERF_GLOBAL_CTRL,
.host = intel_ctrl & ~cpuc->intel_ctrl_guest_mask,
.guest = intel_ctrl & ~cpuc->intel_ctrl_host_mask & ~pebs_mask,
.host = intel_ctrl & ~cpuc->intel_ctrl_exclude_host_mask,
.guest = intel_ctrl & ~cpuc->intel_ctrl_exclude_guest_mask & ~pebs_mask,
};
if (!x86_pmu.ds_pebs)
@ -5353,17 +5385,9 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
* the guest wants to use for PEBS, (c) are not excluded from counting
* in the guest, and (d) _are_ excluded from counting in the host.
*/
guest_pebs_mask = pebs_mask & intel_ctrl & kvm_pmu->pebs_enable &
~cpuc->intel_ctrl_host_mask &
cpuc->intel_ctrl_guest_mask;
/*
* Disable counters where the guest PMC is different than the host PMC
* being used on behalf of the guest, as the PEBS record includes
* PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the
* wrong counter(s).
*/
guest_pebs_mask &= ~kvm_pmu->host_cross_mapped_mask;
guest_pebs_mask = pebs_mask & guest_pebs->enable &
~cpuc->intel_ctrl_exclude_guest_mask &
cpuc->intel_ctrl_exclude_host_mask;
/*
* FIXME: Allow guest and host usage of PEBS events to co-exist instead
@ -5371,7 +5395,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
* What exactly goes wrong if guest and host are using PEBS is
* unknown.
*/
if (pebs_mask & ~cpuc->intel_ctrl_guest_mask)
if (pebs_mask & ~cpuc->intel_ctrl_exclude_host_mask)
guest_pebs_mask = 0;
/*
@ -5382,14 +5406,14 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
arr[(*nr)++] = (struct perf_guest_switch_msr){
.msr = MSR_IA32_DS_AREA,
.host = (unsigned long)cpuc->ds,
.guest = guest_pebs_mask ? kvm_pmu->ds_area : (unsigned long)cpuc->ds,
.guest = guest_pebs_mask ? guest_pebs->ds_area : (unsigned long)cpuc->ds,
};
if (x86_pmu.intel_cap.pebs_baseline) {
arr[(*nr)++] = (struct perf_guest_switch_msr){
.msr = MSR_PEBS_DATA_CFG,
.host = cpuc->active_pebs_data_cfg,
.guest = guest_pebs_mask ? kvm_pmu->pebs_data_cfg :
.guest = guest_pebs_mask ? guest_pebs->data_cfg :
cpuc->active_pebs_data_cfg,
};
}
@ -5405,7 +5429,8 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
return arr;
}
static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr, void *data)
static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr,
struct x86_guest_pebs *guest_pebs)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
@ -6216,12 +6241,46 @@ static inline bool intel_pmu_broken_perf_cap(void)
return false;
}
static inline void __intel_update_pmu_caps(struct pmu *pmu)
static inline void __intel_update_pmu_xregs_caps(struct pmu *pmu)
{
struct pmu *dest_pmu = pmu ? pmu : x86_get_pmu(smp_processor_id());
if (hybrid(pmu, arch_pebs_cap).caps & ARCH_PEBS_VECR_XMM)
dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
/* Only support the extension when XSAVES is available. */
if (!boot_cpu_has(X86_FEATURE_XSAVES))
return;
if (!boot_cpu_has(X86_FEATURE_XMM) ||
!cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
return;
/*
* On current hybrid platforms, P-cores and E-cores expose the same
* XSAVE feature set. Therefore, using the global x86_pmu.ext_regs_mask
* is sufficient to represent the hardware-supported XSAVE features.
*/
x86_pmu.ext_regs_mask |= XFEATURE_MASK_SSE;
if (boot_cpu_has(X86_FEATURE_AVX) &&
cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL))
x86_pmu.ext_regs_mask |= XFEATURE_MASK_YMM;
if (boot_cpu_has(X86_FEATURE_APX) &&
cpu_has_xfeatures(XFEATURE_MASK_APX, NULL))
x86_pmu.ext_regs_mask |= XFEATURE_MASK_APX;
if (boot_cpu_has(X86_FEATURE_AVX512F)) {
if (cpu_has_xfeatures(XFEATURE_MASK_OPMASK, NULL))
x86_pmu.ext_regs_mask |= XFEATURE_MASK_OPMASK;
if (cpu_has_xfeatures(XFEATURE_MASK_ZMM_Hi256, NULL))
x86_pmu.ext_regs_mask |= XFEATURE_MASK_ZMM_Hi256;
if (cpu_has_xfeatures(XFEATURE_MASK_Hi16_ZMM, NULL))
x86_pmu.ext_regs_mask |= XFEATURE_MASK_Hi16_ZMM;
}
if (cpu_feature_enabled(X86_FEATURE_USER_SHSTK) &&
cpu_has_xfeatures(XFEATURE_MASK_CET_USER, NULL))
x86_pmu.ext_regs_mask |= XFEATURE_MASK_CET_USER;
dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
if (x86_pmu.ext_regs_mask > XFEATURE_MASK_SSE)
dest_pmu->capabilities |= PERF_PMU_CAP_SIMD_REGS;
}
static inline void __intel_update_large_pebs_flags(struct pmu *pmu)
@ -6292,12 +6351,10 @@ static void update_pmu_cap_from_perfmonext(struct pmu *pmu)
hybrid(pmu, arch_pebs_cap).counters = pebs_mask;
hybrid(pmu, arch_pebs_cap).pdists = pdists_mask;
if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask)) {
if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask))
x86_pmu.arch_pebs = 0;
} else {
__intel_update_pmu_caps(pmu);
else
__intel_update_large_pebs_flags(pmu);
}
} else {
WARN_ON(x86_pmu.arch_pebs == 1);
x86_pmu.arch_pebs = 0;
@ -6321,6 +6378,7 @@ static void intel_update_pmu_caps(struct pmu *pmu)
hybrid_pmu(pmu)->pmu_type == hybrid_big)
hybrid(pmu, intel_cap).perf_metrics = 1;
}
__intel_update_pmu_xregs_caps(pmu);
}
static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu)
@ -6474,8 +6532,6 @@ static void intel_pmu_cpu_starting(int cpu)
}
}
__intel_update_pmu_caps(cpuc->pmu);
if (!cpuc->shared_regs)
return;

232
arch/x86/events/intel/ds.c
View File

@ -1704,6 +1704,7 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)
u64 sample_type = attr->sample_type;
u64 pebs_data_cfg = 0;
bool gprs, tsx_weight;
u64 xgprs_mask;
if (!(sample_type & ~(PERF_SAMPLE_IP|PERF_SAMPLE_TIME)) &&
attr->precise_ip > 1)
@ -1718,10 +1719,13 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)
* + precise_ip < 2 for the non event IP
* + For RTM TSX weight we need GPRs for the abort code.
*/
xgprs_mask = event->attr.sample_simd_regs_enabled ?
PEBS_GP_REGS | BIT_ULL(PERF_REG_X86_SSP) :
PEBS_GP_REGS;
gprs = ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(attr->sample_regs_intr & PEBS_GP_REGS)) ||
(attr->sample_regs_intr & xgprs_mask)) ||
((sample_type & PERF_SAMPLE_REGS_USER) &&
(attr->sample_regs_user & PEBS_GP_REGS));
(attr->sample_regs_user & xgprs_mask));
tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) &&
((attr->config & INTEL_ARCH_EVENT_MASK) ==
@ -1730,9 +1734,20 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)
if (gprs || (attr->precise_ip < 2) || tsx_weight)
pebs_data_cfg |= PEBS_DATACFG_GP;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(attr->sample_regs_intr & PERF_REG_EXTENDED_MASK))
pebs_data_cfg |= PEBS_DATACFG_XMMS;
if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) {
if (event_needs_xmm(event))
pebs_data_cfg |= PEBS_DATACFG_XMMS;
if (x86_pmu.arch_pebs && event_needs_ymm(event))
pebs_data_cfg |= PEBS_DATACFG_YMMHS;
if (x86_pmu.arch_pebs && event_needs_low16_zmm(event))
pebs_data_cfg |= PEBS_DATACFG_ZMMHS;
if (x86_pmu.arch_pebs && event_needs_high16_zmm(event))
pebs_data_cfg |= PEBS_DATACFG_H16ZMMS;
if (x86_pmu.arch_pebs && event_needs_opmask(event))
pebs_data_cfg |= PEBS_DATACFG_OPMASKS;
if (x86_pmu.arch_pebs && event_needs_egprs(event))
pebs_data_cfg |= PEBS_DATACFG_EGPRS;
}
if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
/*
@ -2523,7 +2538,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
return;
perf_regs = container_of(regs, struct x86_perf_regs, regs);
perf_regs->xmm_regs = NULL;
x86_pmu_clear_perf_regs(regs);
format_group = basic->format_group;
@ -2608,6 +2623,8 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
next_record += nr * sizeof(u64);
}
x86_pmu_update_perf_regs(event, data, regs, true);
WARN_ONCE(next_record != __pebs + basic->format_size,
"PEBS record size %u, expected %llu, config %llx\n",
basic->format_size,
@ -2640,7 +2657,7 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
return;
perf_regs = container_of(regs, struct x86_perf_regs, regs);
perf_regs->xmm_regs = NULL;
x86_pmu_clear_perf_regs(regs);
__setup_perf_sample_data(event, iregs, data);
@ -2648,6 +2665,9 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
again:
header = at;
if (!header->size)
return;
next_record = at + sizeof(struct arch_pebs_header);
if (header->basic) {
struct arch_pebs_basic *basic = next_record;
@ -2678,6 +2698,11 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
__setup_pebs_gpr_group(event, regs,
(struct pebs_gprs *)gprs,
sample_type);
/* Currently only user space mode enables SSP. */
if (user_mode(regs) && (sample_type &
(PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)))
perf_regs->ssp = &gprs->ssp;
}
if (header->aux) {
@ -2690,14 +2715,63 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
meminfo->tsx_tuning, ax);
}
if (header->xmm) {
if (header->xmm || header->ymmh || header->egpr ||
header->opmask || header->zmmh || header->h16zmm) {
struct arch_pebs_xer_header *xer_header = next_record;
struct pebs_xmm *xmm;
struct ymmh_struct *ymmh;
struct avx_512_zmm_uppers_state *zmmh;
struct avx_512_hi16_state *h16zmm;
struct avx_512_opmask_state *opmask;
struct apx_state *egpr;
next_record += sizeof(struct arch_pebs_xer_header);
xmm = next_record;
perf_regs->xmm_regs = xmm->xmm;
next_record = xmm + 1;
if (header->xmm) {
xmm = next_record;
/*
* Only output XMM regs to user space when arch-PEBS
* really writes data into xstate area.
*/
if (xer_header->xstate & XFEATURE_MASK_SSE)
perf_regs->xmm_regs = xmm->xmm;
next_record = xmm + 1;
}
if (header->ymmh) {
ymmh = next_record;
if (xer_header->xstate & XFEATURE_MASK_YMM)
perf_regs->ymmh = ymmh;
next_record = ymmh + 1;
}
if (header->egpr) {
egpr = next_record;
if (xer_header->xstate & XFEATURE_MASK_APX)
perf_regs->egpr = egpr;
next_record = egpr + 1;
}
if (header->opmask) {
opmask = next_record;
if (xer_header->xstate & XFEATURE_MASK_OPMASK)
perf_regs->opmask = opmask;
next_record = opmask + 1;
}
if (header->zmmh) {
zmmh = next_record;
if (xer_header->xstate & XFEATURE_MASK_ZMM_Hi256)
perf_regs->zmmh = zmmh;
next_record = zmmh + 1;
}
if (header->h16zmm) {
h16zmm = next_record;
if (xer_header->xstate & XFEATURE_MASK_Hi16_ZMM)
perf_regs->h16zmm = h16zmm;
next_record = h16zmm + 1;
}
}
if (header->lbr) {
@ -2742,6 +2816,8 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
at = at + header->size;
goto again;
}
x86_pmu_update_perf_regs(event, data, regs, true);
}
static inline void *
@ -2865,13 +2941,21 @@ __intel_pmu_pebs_last_event(struct perf_event *event,
struct pt_regs *iregs,
struct pt_regs *regs,
struct perf_sample_data *data,
void *at,
int count,
void *at, int count, bool corrupted,
setup_fn setup_sample)
{
struct hw_perf_event *hwc = &event->hw;
setup_sample(event, iregs, at, data, regs);
/* Skip parsing corrupted PEBS record. */
if (corrupted) {
/* Clear stale register states in previous records. */
memset(regs, 0, sizeof(*regs));
x86_pmu_clear_perf_regs(regs);
perf_sample_data_init(data, 0, event->hw.last_period);
} else {
setup_sample(event, iregs, at, data, regs);
}
if (iregs == &dummy_iregs) {
/*
* The PEBS records may be drained in the non-overflow context,
@ -2889,12 +2973,16 @@ __intel_pmu_pebs_last_event(struct perf_event *event,
}
if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
if ((is_pebs_counter_event_group(event))) {
/*
* The value of each sample has been updated when setup
* the corresponding sample data.
*/
perf_event_update_userpage(event);
if (is_pebs_counter_event_group(event)) {
if (corrupted) {
intel_pmu_save_and_restart_reload(event, 1);
} else {
/*
* The value of each sample has been updated
* when setup the corresponding sample data.
*/
perf_event_update_userpage(event);
}
} else {
/*
* Now, auto-reload is only enabled in fixed period mode.
@ -2918,13 +3006,15 @@ __intel_pmu_pebs_last_event(struct perf_event *event,
* counters-snapshotting record, only needs to set the new
* period for the counter.
*/
if (is_pebs_counter_event_group(event))
if (is_pebs_counter_event_group(event) && !corrupted)
static_call(x86_pmu_set_period)(event);
else
intel_pmu_save_and_restart(event);
}
}
static DEFINE_PER_CPU(struct x86_perf_regs, x86_pebs_regs);
static __always_inline void
__intel_pmu_pebs_events(struct perf_event *event,
struct pt_regs *iregs,
@ -2934,25 +3024,29 @@ __intel_pmu_pebs_events(struct perf_event *event,
setup_fn setup_sample)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct x86_perf_regs perf_regs;
struct pt_regs *regs = &perf_regs.regs;
struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs);
struct pt_regs *regs = &perf_regs->regs;
void *at = get_next_pebs_record_by_bit(base, top, bit);
int cnt = count;
x86_pmu_clear_perf_regs(regs);
if (!iregs)
iregs = &dummy_iregs;
while (cnt > 1) {
__intel_pmu_pebs_event(event, iregs, regs, data, at, setup_sample);
__intel_pmu_pebs_event(event, iregs, regs, data,
at, setup_sample);
at += cpuc->pebs_record_size;
at = get_next_pebs_record_by_bit(at, top, bit);
cnt--;
}
__intel_pmu_pebs_last_event(event, iregs, regs, data, at, count, setup_sample);
__intel_pmu_pebs_last_event(event, iregs, regs, data, at,
count, false, setup_sample);
}
static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
static int intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct debug_store *ds = cpuc->ds;
@ -2961,7 +3055,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_
int n;
if (!x86_pmu.pebs_active)
return;
return 0;
at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
@ -2972,22 +3066,25 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_
ds->pebs_index = ds->pebs_buffer_base;
if (!test_bit(0, cpuc->active_mask))
return;
return 0;
WARN_ON_ONCE(!event);
if (!event->attr.precise_ip)
return;
return 0;
n = top - at;
if (n <= 0) {
if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
intel_pmu_save_and_restart_reload(event, 0);
return;
return 0;
}
__intel_pmu_pebs_events(event, iregs, data, at, top, 0, n,
setup_pebs_fixed_sample_data);
/* PMC0 only */
return 1;
}
static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64 mask)
@ -3010,7 +3107,7 @@ static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64
}
}
static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data)
static int intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct debug_store *ds = cpuc->ds;
@ -3019,11 +3116,12 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
short error[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
int max_pebs_events = intel_pmu_max_num_pebs(NULL);
u64 events_bitmap = 0;
int bit, i, size;
u64 mask;
if (!x86_pmu.pebs_active)
return;
return 0;
base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
@ -3039,7 +3137,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
if (unlikely(base >= top)) {
intel_pmu_pebs_event_update_no_drain(cpuc, mask);
return;
return 0;
}
for (at = base; at < top; at += x86_pmu.pebs_record_size) {
@ -3103,6 +3201,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
if ((counts[bit] == 0) && (error[bit] == 0))
continue;
events_bitmap |= BIT_ULL(bit);
event = cpuc->events[bit];
if (WARN_ON_ONCE(!event))
continue;
@ -3124,6 +3223,8 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
setup_pebs_fixed_sample_data);
}
}
return hweight64(events_bitmap);
}
static __always_inline void
@ -3158,39 +3259,46 @@ static __always_inline void
__intel_pmu_handle_last_pebs_record(struct pt_regs *iregs,
struct pt_regs *regs,
struct perf_sample_data *data,
u64 mask, short *counts, void **last,
u64 mask, short *counts,
void **last, bool corrupted,
setup_fn setup_sample)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct perf_event *event;
bool handled = false;
int bit;
for_each_set_bit(bit, (unsigned long *)&mask, X86_PMC_IDX_MAX) {
if (!counts[bit])
continue;
handled = true;
event = cpuc->events[bit];
__intel_pmu_pebs_last_event(event, iregs, regs, data, last[bit],
counts[bit], setup_sample);
counts[bit], corrupted, setup_sample);
}
/* All records are corrupted, reset sampling period. */
if (!handled)
intel_pmu_pebs_event_update_no_drain(cpuc, mask);
}
static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data)
static int intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data)
{
short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct debug_store *ds = cpuc->ds;
struct x86_perf_regs perf_regs;
struct pt_regs *regs = &perf_regs.regs;
struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs);
struct pt_regs *regs = &perf_regs->regs;
struct pebs_basic *basic;
void *base, *at, *top;
u64 events_bitmap = 0;
bool corrupted = false;
u64 mask;
if (!x86_pmu.pebs_active)
return;
return 0;
base = (struct pebs_basic *)(unsigned long)ds->pebs_buffer_base;
top = (struct pebs_basic *)(unsigned long)ds->pebs_index;
@ -3203,7 +3311,7 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
if (unlikely(base >= top)) {
intel_pmu_pebs_event_update_no_drain(cpuc, mask);
return;
return 0;
}
if (!iregs)
@ -3214,36 +3322,45 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
u64 pebs_status;
basic = at;
if (WARN_ON_ONCE(!basic->format_size)) {
corrupted = true;
break;
}
if (basic->format_size != cpuc->pebs_record_size)
continue;
pebs_status = mask & basic->applicable_counters;
events_bitmap |= pebs_status;
__intel_pmu_handle_pebs_record(iregs, regs, data, at,
pebs_status, counts, last,
setup_pebs_adaptive_sample_data);
}
__intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, counts, last,
setup_pebs_adaptive_sample_data);
corrupted, setup_pebs_adaptive_sample_data);
return hweight64(events_bitmap);
}
static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
struct perf_sample_data *data)
static int intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
struct perf_sample_data *data)
{
short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
union arch_pebs_index index;
struct x86_perf_regs perf_regs;
struct pt_regs *regs = &perf_regs.regs;
struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs);
struct pt_regs *regs = &perf_regs->regs;
void *base, *at, *top;
u64 events_bitmap = 0;
bool corrupted = false;
u64 mask;
rdmsrq(MSR_IA32_PEBS_INDEX, index.whole);
if (unlikely(!index.wr)) {
intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX);
return;
return 0;
}
base = cpuc->pebs_vaddr;
@ -3271,8 +3388,10 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
header = at;
if (WARN_ON_ONCE(!header->size))
break;
if (WARN_ON_ONCE(!header->size)) {
corrupted = true;
goto done;
}
/* 1st fragment or single record must have basic group */
if (!header->basic) {
@ -3282,6 +3401,7 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
basic = at + sizeof(struct arch_pebs_header);
pebs_status = mask & basic->applicable_counters;
events_bitmap |= pebs_status;
__intel_pmu_handle_pebs_record(iregs, regs, data, at,
pebs_status, counts, last,
setup_arch_pebs_sample_data);
@ -3291,16 +3411,27 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
if (!header->size)
break;
at += header->size;
if (WARN_ON_ONCE(at >= top)) {
corrupted = true;
goto done;
}
header = at;
}
/* Skip last fragment or the single record */
at += header->size;
if (WARN_ON_ONCE(at > top)) {
corrupted = true;
goto done;
}
}
done:
__intel_pmu_handle_last_pebs_record(iregs, regs, data, mask,
counts, last,
counts, last, corrupted,
setup_arch_pebs_sample_data);
return hweight64(events_bitmap);
}
static void __init intel_arch_pebs_init(void)
@ -3402,7 +3533,6 @@ static void __init intel_ds_pebs_init(void)
x86_pmu.flags |= PMU_FL_PEBS_ALL;
x86_pmu.pebs_capable = ~0ULL;
pebs_qual = "-baseline";
x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
} else {
/* Only basic record supported */
x86_pmu.large_pebs_flags &=

2
arch/x86/events/intel/lbr.c
View File

@ -714,7 +714,7 @@ static inline bool vlbr_exclude_host(void)
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
return test_bit(INTEL_PMC_IDX_FIXED_VLBR,
(unsigned long *)&cpuc->intel_ctrl_guest_mask);
(unsigned long *)&cpuc->intel_ctrl_exclude_host_mask);
}
void intel_pmu_lbr_enable_all(bool pmi)

217
arch/x86/events/perf_event.h
View File

@ -147,6 +147,199 @@ static inline bool is_acr_self_reload_event(struct perf_event *event)
return test_bit(hwc->idx, (unsigned long *)&hwc->config1);
}
static inline bool __event_needs_xmm(struct perf_event *event, u64 sample_type)
{
if (event->attr.sample_simd_regs_enabled) {
if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_XMM_QWORDS)
return false;
if ((sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_simd_vec_reg_user > 0))
return true;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_simd_vec_reg_intr > 0))
return true;
} else {
if ((sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
return true;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK))
return true;
}
return false;
}
static inline bool event_needs_xmm(struct perf_event *event)
{
return __event_needs_xmm(event,
PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
}
static inline bool __event_needs_ymm(struct perf_event *event, u64 sample_type)
{
if (!event->attr.sample_simd_regs_enabled)
return false;
if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_YMM_QWORDS)
return false;
if ((sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_simd_vec_reg_user > 0))
return true;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_simd_vec_reg_intr > 0))
return true;
return false;
}
static inline bool event_needs_ymm(struct perf_event *event)
{
return __event_needs_ymm(event,
PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
}
static inline bool __event_needs_low16_zmm(struct perf_event *event,
u64 sample_type)
{
if (!event->attr.sample_simd_regs_enabled)
return false;
if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_ZMM_QWORDS)
return false;
if ((sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_simd_vec_reg_user > 0))
return true;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_simd_vec_reg_intr > 0))
return true;
return false;
}
static inline bool event_needs_low16_zmm(struct perf_event *event)
{
return __event_needs_low16_zmm(event,
PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
}
static inline bool __event_needs_high16_zmm(struct perf_event *event,
u64 sample_type)
{
if (!event->attr.sample_simd_regs_enabled)
return false;
if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_ZMM_QWORDS)
return false;
if ((sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(fls64(event->attr.sample_simd_vec_reg_user) > PERF_X86_H16ZMM_BASE))
return true;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(fls64(event->attr.sample_simd_vec_reg_intr) > PERF_X86_H16ZMM_BASE))
return true;
return false;
}
static inline bool event_needs_high16_zmm(struct perf_event *event)
{
return __event_needs_high16_zmm(event,
PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
}
static inline bool __event_needs_opmask(struct perf_event *event,
u64 sample_type)
{
if (!event->attr.sample_simd_regs_enabled)
return false;
if (event->attr.sample_simd_pred_reg_qwords != PERF_X86_OPMASK_QWORDS)
return false;
if ((sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_simd_pred_reg_user > 0))
return true;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_simd_pred_reg_intr > 0))
return true;
return false;
}
static inline bool event_needs_opmask(struct perf_event *event)
{
return __event_needs_opmask(event,
PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
}
static inline bool __event_needs_egprs(struct perf_event *event,
u64 sample_type)
{
if (!event->attr.sample_simd_regs_enabled)
return false;
if ((sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_regs_user & PERF_X86_EGPRS_MASK))
return true;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_regs_intr & PERF_X86_EGPRS_MASK))
return true;
return false;
}
static inline bool event_needs_egprs(struct perf_event *event)
{
return __event_needs_egprs(event,
PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
}
static inline bool __event_needs_ssp(struct perf_event *event,
u64 sample_type)
{
if (!event->attr.sample_simd_regs_enabled)
return false;
if ((sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
(event->attr.sample_regs_user & BIT_ULL(PERF_REG_X86_SSP)))
return true;
if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
(event->attr.sample_regs_intr & BIT_ULL(PERF_REG_X86_SSP)))
return true;
return false;
}
static inline bool event_needs_ssp(struct perf_event *event)
{
return __event_needs_ssp(event,
PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
}
struct amd_nb {
int nb_id; /* NorthBridge id */
int refcnt; /* reference count */
@ -347,8 +540,8 @@ struct cpu_hw_events {
/*
* Intel host/guest exclude bits
*/
u64 intel_ctrl_guest_mask;
u64 intel_ctrl_host_mask;
u64 intel_ctrl_exclude_host_mask;
u64 intel_ctrl_exclude_guest_mask;
struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX];
/*
@ -948,7 +1141,7 @@ struct x86_pmu {
int pebs_record_size;
int pebs_buffer_size;
u64 pebs_events_mask;
void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
int (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
struct event_constraint *pebs_constraints;
void (*pebs_aliases)(struct perf_event *event);
u64 (*pebs_latency_data)(struct perf_event *event, u64 status);
@ -1024,10 +1217,17 @@ struct x86_pmu {
struct extra_reg *extra_regs;
unsigned int flags;
/*
* Extended regs, e.g., vector registers
* Utilize the same format as the XFEATURE_MASK_*
*/
u64 ext_regs_mask;
/*
* Intel host/guest support (KVM)
*/
struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data);
struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr,
struct x86_guest_pebs *guest_pebs);
/*
* Check period value for PERF_EVENT_IOC_PERIOD ioctl.
@ -1046,7 +1246,7 @@ struct x86_pmu {
* unique capabilities.
*/
int num_hybrid_pmus;
struct x86_hybrid_pmu *hybrid_pmu;
struct x86_hybrid_pmu *hybrid_pmu __counted_by_ptr(num_hybrid_pmus);
enum intel_cpu_type (*get_hybrid_cpu_type) (void);
};
@ -1311,6 +1511,13 @@ void x86_pmu_enable_event(struct perf_event *event);
int x86_pmu_handle_irq(struct pt_regs *regs);
void x86_pmu_clear_perf_regs(struct pt_regs *regs);
void x86_pmu_update_perf_regs(struct perf_event *event,
struct perf_sample_data *data,
struct pt_regs *regs,
bool from_pebs);
void x86_pmu_show_pmu_cap(struct pmu *pmu);
static inline int x86_pmu_num_counters(struct pmu *pmu)

5
arch/x86/include/asm/fpu/sched.h
View File

@ -10,6 +10,8 @@
#include <asm/trace/fpu.h>
extern void save_fpregs_to_fpstate(struct fpu *fpu);
extern void update_fpu_state_and_flag(struct fpu *fpu,
struct task_struct *task);
extern void fpu__drop(struct task_struct *tsk);
extern int fpu_clone(struct task_struct *dst, u64 clone_flags, bool minimal,
unsigned long shstk_addr);
@ -35,8 +37,7 @@ static inline void switch_fpu(struct task_struct *old, int cpu)
!(old->flags & (PF_KTHREAD | PF_USER_WORKER))) {
struct fpu *old_fpu = x86_task_fpu(old);
set_tsk_thread_flag(old, TIF_NEED_FPU_LOAD);
save_fpregs_to_fpstate(old_fpu);
update_fpu_state_and_flag(old_fpu, old);
/*
* The save operation preserved register state, so the
* fpu_fpregs_owner_ctx is still @old_fpu. Store the

1
arch/x86/include/asm/fpu/xstate.h
View File

@ -110,6 +110,7 @@ int xfeature_size(int xfeature_nr);
void xsaves(struct xregs_state *xsave, u64 mask);
void xrstors(struct xregs_state *xsave, u64 mask);
void xsaves_nmi(struct xregs_state *xsave, u64 mask);
unsigned int xstate_calculate_size(u64 xfeatures, bool compacted);

9
arch/x86/include/asm/kvm_host.h
View File

@ -608,15 +608,6 @@ struct kvm_pmu {
u64 pebs_data_cfg;
u64 pebs_data_cfg_rsvd;
/*
* If a guest counter is cross-mapped to host counter with different
* index, its PEBS capability will be temporarily disabled.
*
* The user should make sure that this mask is updated
* after disabling interrupts and before perf_guest_get_msrs();
*/
u64 host_cross_mapped_mask;
/*
* The gate to release perf_events not marked in
* pmc_in_use only once in a vcpu time slice.

7
arch/x86/include/asm/msr-index.h
View File

@ -352,6 +352,13 @@
#define ARCH_PEBS_LBR_SHIFT 40
#define ARCH_PEBS_LBR (0x3ull << ARCH_PEBS_LBR_SHIFT)
#define ARCH_PEBS_VECR_XMM BIT_ULL(49)
#define ARCH_PEBS_VECR_YMMH BIT_ULL(50)
#define ARCH_PEBS_VECR_EGPRS BIT_ULL(51)
#define ARCH_PEBS_VECR_OPMASK BIT_ULL(53)
#define ARCH_PEBS_VECR_ZMMH BIT_ULL(54)
#define ARCH_PEBS_VECR_H16ZMM BIT_ULL(55)
#define ARCH_PEBS_VECR_EXT_SHIFT 49
#define ARCH_PEBS_VECR_EXT (0x7full << ARCH_PEBS_VECR_EXT_SHIFT)
#define ARCH_PEBS_GPR BIT_ULL(61)
#define ARCH_PEBS_AUX BIT_ULL(62)
#define ARCH_PEBS_EN BIT_ULL(63)

46
arch/x86/include/asm/perf_event.h
View File

@ -150,6 +150,11 @@
#define PEBS_DATACFG_LBRS BIT_ULL(3)
#define PEBS_DATACFG_CNTR BIT_ULL(4)
#define PEBS_DATACFG_METRICS BIT_ULL(5)
#define PEBS_DATACFG_YMMHS BIT_ULL(6)
#define PEBS_DATACFG_OPMASKS BIT_ULL(7)
#define PEBS_DATACFG_ZMMHS BIT_ULL(8)
#define PEBS_DATACFG_H16ZMMS BIT_ULL(9)
#define PEBS_DATACFG_EGPRS BIT_ULL(10)
#define PEBS_DATACFG_LBR_SHIFT 24
#define PEBS_DATACFG_CNTR_SHIFT 32
#define PEBS_DATACFG_CNTR_MASK GENMASK_ULL(15, 0)
@ -547,7 +552,8 @@ struct arch_pebs_header {
rsvd3:7,
xmm:1,
ymmh:1,
rsvd4:2,
egpr:1,
rsvd4:1,
opmask:1,
zmmh:1,
h16zmm:1,
@ -728,7 +734,32 @@ extern void perf_events_lapic_init(void);
struct pt_regs;
struct x86_perf_regs {
struct pt_regs regs;
u64 *xmm_regs;
u64 abi;
union {
u64 *xmm_regs;
u32 *xmm_space; /* for xsaves */
};
union {
u64 *ymmh_regs;
struct ymmh_struct *ymmh;
};
union {
u64 *zmmh_regs;
struct avx_512_zmm_uppers_state *zmmh;
};
union {
u64 *h16zmm_regs;
struct avx_512_hi16_state *h16zmm;
};
union {
u64 *opmask_regs;
struct avx_512_opmask_state *opmask;
};
union {
u64 *egpr_regs;
struct apx_state *egpr;
};
u64 *ssp;
};
extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs);
@ -788,11 +819,18 @@ extern void perf_load_guest_lvtpc(u32 guest_lvtpc);
extern void perf_put_guest_lvtpc(void);
#endif
struct x86_guest_pebs {
u64 enable;
u64 ds_area;
u64 data_cfg;
};
#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data);
extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr,
struct x86_guest_pebs *guest_pebs);
extern void x86_perf_get_lbr(struct x86_pmu_lbr *lbr);
#else
struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data);
struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr,
struct x86_guest_pebs *guest_pebs);
static inline void x86_perf_get_lbr(struct x86_pmu_lbr *lbr)
{
memset(lbr, 0, sizeof(*lbr));

53
arch/x86/include/uapi/asm/perf_regs.h
View File

@ -2,6 +2,8 @@
#ifndef _ASM_X86_PERF_REGS_H
#define _ASM_X86_PERF_REGS_H
#include <linux/bits.h>
enum perf_event_x86_regs {
PERF_REG_X86_AX,
PERF_REG_X86_BX,
@ -27,9 +29,35 @@ enum perf_event_x86_regs {
PERF_REG_X86_R13,
PERF_REG_X86_R14,
PERF_REG_X86_R15,
/*
* The eGPRs/SSP and XMM have overlaps. Only one can be used
* at a time. The ABI PERF_SAMPLE_REGS_ABI_SIMD is used to
* distinguish which one is used. If PERF_SAMPLE_REGS_ABI_SIMD
* is set, then eGPRs/SSP is used, otherwise, XMM is used.
*
* Extended GPRs (eGPRs)
*/
PERF_REG_X86_R16,
PERF_REG_X86_R17,
PERF_REG_X86_R18,
PERF_REG_X86_R19,
PERF_REG_X86_R20,
PERF_REG_X86_R21,
PERF_REG_X86_R22,
PERF_REG_X86_R23,
PERF_REG_X86_R24,
PERF_REG_X86_R25,
PERF_REG_X86_R26,
PERF_REG_X86_R27,
PERF_REG_X86_R28,
PERF_REG_X86_R29,
PERF_REG_X86_R30,
PERF_REG_X86_R31,
PERF_REG_X86_SSP,
/* These are the limits for the GPRs. */
PERF_REG_X86_32_MAX = PERF_REG_X86_GS + 1,
PERF_REG_X86_64_MAX = PERF_REG_X86_R15 + 1,
PERF_REG_MISC_MAX = PERF_REG_X86_SSP + 1,
/* These all need two bits set because they are 128bit */
PERF_REG_X86_XMM0 = 32,
@ -54,5 +82,30 @@ enum perf_event_x86_regs {
};
#define PERF_REG_EXTENDED_MASK (~((1ULL << PERF_REG_X86_XMM0) - 1))
#define PERF_X86_EGPRS_MASK __GENMASK_ULL(PERF_REG_X86_R31, PERF_REG_X86_R16)
enum {
PERF_X86_SIMD_XMM_REGS = 16,
PERF_X86_SIMD_YMM_REGS = 16,
PERF_X86_SIMD_ZMM_REGS = 32,
PERF_X86_SIMD_VEC_REGS_MAX = PERF_X86_SIMD_ZMM_REGS,
PERF_X86_SIMD_OPMASK_REGS = 8,
PERF_X86_SIMD_PRED_REGS_MAX = PERF_X86_SIMD_OPMASK_REGS,
};
#define PERF_X86_SIMD_PRED_MASK __GENMASK(PERF_X86_SIMD_PRED_REGS_MAX - 1, 0)
#define PERF_X86_SIMD_VEC_MASK __GENMASK_ULL(PERF_X86_SIMD_VEC_REGS_MAX - 1, 0)
#define PERF_X86_H16ZMM_BASE 16
enum {
/* 1 qword = 8 bytes */
PERF_X86_OPMASK_QWORDS = 1,
PERF_X86_XMM_QWORDS = 2,
PERF_X86_YMM_QWORDS = 4,
PERF_X86_ZMM_QWORDS = 8,
PERF_X86_SIMD_QWORDS_MAX = PERF_X86_ZMM_QWORDS,
};
#endif /* _ASM_X86_PERF_REGS_H */

33
arch/x86/kernel/fpu/core.c
View File

@ -213,6 +213,19 @@ void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask)
}
}
/*
* Save the FPU register state in fpu->fpstate->regs and set
* TIF_NEED_FPU_LOAD subsequently.
*
* Must be called with fpregs_lock() held, ensuring flag
* TIF_NEED_FPU_LOAD is set last.
*/
void update_fpu_state_and_flag(struct fpu *fpu, struct task_struct *task)
{
save_fpregs_to_fpstate(fpu);
set_tsk_thread_flag(task, TIF_NEED_FPU_LOAD);
}
void fpu_reset_from_exception_fixup(void)
{
restore_fpregs_from_fpstate(&init_fpstate, XFEATURE_MASK_FPSTATE);
@ -383,13 +396,13 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest)
/* Swap fpstate */
if (enter_guest) {
fpu->__task_fpstate = cur_fps;
WRITE_ONCE(fpu->__task_fpstate, cur_fps);
barrier();
fpu->fpstate = guest_fps;
guest_fps->in_use = true;
} else {
guest_fps->in_use = false;
fpu->fpstate = fpu->__task_fpstate;
fpu->__task_fpstate = NULL;
}
cur_fps = fpu->fpstate;
@ -406,6 +419,16 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest)
xfd_update_state(cur_fps);
}
/*
* Clear fpu->__task_fpstate after switching back to host state.
* A non-NULL __task_fpstate means guest state is still resident in
* hardware; reset it only once host state has been restored.
*/
if (!enter_guest) {
barrier();
WRITE_ONCE(fpu->__task_fpstate, NULL);
}
fpregs_mark_activate();
fpregs_unlock();
return 0;
@ -481,10 +504,8 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask)
this_cpu_write(kernel_fpu_allowed, false);
if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) &&
!test_thread_flag(TIF_NEED_FPU_LOAD)) {
set_thread_flag(TIF_NEED_FPU_LOAD);
save_fpregs_to_fpstate(x86_task_fpu(current));
}
!test_thread_flag(TIF_NEED_FPU_LOAD))
update_fpu_state_and_flag(x86_task_fpu(current), current);
__cpu_invalidate_fpregs_state();
/* Put sane initial values into the control registers. */

23
arch/x86/kernel/fpu/xstate.c
View File

@ -1491,6 +1491,29 @@ void xrstors(struct xregs_state *xstate, u64 mask)
WARN_ON_ONCE(err);
}
/**
* xsaves_nmi - Save selected components to a kernel xstate buffer in NMI
* @xstate: Pointer to the buffer
* @mask: Feature mask to select the components to save
*
* This function is similar to xsaves(), but should only be called within
* the NMI handler. This function returns the actual register contents at
* the moment the NMI occurs.
*
* Currently, the perf subsystem is the sole user of this helper. It uses
* the function to snapshot SIMD (XMM/YMM/ZMM) and APX eGPRs registers.
*/
void xsaves_nmi(struct xregs_state *xstate, u64 mask)
{
int err;
if (!in_nmi())
return;
XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err);
WARN_ON_ONCE(err);
}
#if IS_ENABLED(CONFIG_KVM)
void fpstate_clear_xstate_component(struct fpstate *fpstate, unsigned int xfeature)
{

172
arch/x86/kernel/perf_regs.c
View File

@ -61,11 +61,29 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
{
struct x86_perf_regs *perf_regs;
if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) {
if (idx > PERF_REG_X86_R15) {
perf_regs = container_of(regs, struct x86_perf_regs, regs);
if (!perf_regs->xmm_regs)
if (perf_regs->abi == PERF_SAMPLE_REGS_ABI_NONE)
return 0;
return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0];
if (perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) {
if (idx <= PERF_REG_X86_R31) {
if (!perf_regs->egpr_regs)
return 0;
return perf_regs->egpr_regs[idx - PERF_REG_X86_R16];
}
if (idx == PERF_REG_X86_SSP) {
if (!perf_regs->ssp)
return 0;
return *perf_regs->ssp;
}
} else {
if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) {
if (!perf_regs->xmm_regs)
return 0;
return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0];
}
}
}
if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset)))
@ -74,22 +92,130 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
return regs_get_register(regs, pt_regs_offset[idx]);
}
#define PERF_REG_X86_RESERVED (((1ULL << PERF_REG_X86_XMM0) - 1) & \
~((1ULL << PERF_REG_X86_MAX) - 1))
#define PERF_X86_YMMH_QWORDS (PERF_X86_YMM_QWORDS / 2)
#define PERF_X86_ZMMH_QWORDS (PERF_X86_ZMM_QWORDS / 2)
u64 perf_simd_reg_value(struct pt_regs *regs, int idx,
u16 qwords_idx, bool pred)
{
struct x86_perf_regs *perf_regs =
container_of(regs, struct x86_perf_regs, regs);
if (!(perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD))
return 0;
if (pred) {
if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_PRED_REGS_MAX ||
qwords_idx >= PERF_X86_OPMASK_QWORDS))
return 0;
if (!perf_regs->opmask_regs)
return 0;
return perf_regs->opmask_regs[idx];
}
if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_VEC_REGS_MAX ||
qwords_idx >= PERF_X86_SIMD_QWORDS_MAX))
return 0;
if (idx >= PERF_X86_H16ZMM_BASE) {
if (!perf_regs->h16zmm_regs)
return 0;
return perf_regs->h16zmm_regs[(idx - PERF_X86_H16ZMM_BASE) *
PERF_X86_ZMM_QWORDS + qwords_idx];
}
if (qwords_idx < PERF_X86_XMM_QWORDS) {
if (!perf_regs->xmm_regs)
return 0;
return perf_regs->xmm_regs[idx * PERF_X86_XMM_QWORDS +
qwords_idx];
} else if (qwords_idx < PERF_X86_YMM_QWORDS) {
if (!perf_regs->ymmh_regs)
return 0;
return perf_regs->ymmh_regs[idx * PERF_X86_YMMH_QWORDS +
qwords_idx - PERF_X86_XMM_QWORDS];
} else if (qwords_idx < PERF_X86_ZMM_QWORDS) {
if (!perf_regs->zmmh_regs)
return 0;
return perf_regs->zmmh_regs[idx * PERF_X86_ZMMH_QWORDS +
qwords_idx - PERF_X86_YMM_QWORDS];
}
return 0;
}
int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask,
u16 pred_qwords, u32 pred_mask)
{
unsigned long mask;
u64 size;
if (!vec_qwords && !pred_qwords) {
if (vec_mask || pred_mask)
return -EINVAL;
}
if (vec_qwords) {
if (vec_qwords != PERF_X86_XMM_QWORDS &&
vec_qwords != PERF_X86_YMM_QWORDS &&
vec_qwords != PERF_X86_ZMM_QWORDS)
return -EINVAL;
if (vec_mask & ~PERF_X86_SIMD_VEC_MASK)
return -EINVAL;
/* Only full-register sampling is allowed. */
mask = vec_mask;
if (vec_qwords == PERF_X86_XMM_QWORDS && mask &&
!bitmap_full(&mask, PERF_X86_SIMD_XMM_REGS))
return -EINVAL;
if (vec_qwords == PERF_X86_YMM_QWORDS && mask &&
!bitmap_full(&mask, PERF_X86_SIMD_YMM_REGS))
return -EINVAL;
if (vec_qwords == PERF_X86_ZMM_QWORDS && mask &&
!bitmap_full(&mask, PERF_X86_SIMD_ZMM_REGS))
return -EINVAL;
}
if (pred_qwords) {
if (pred_qwords != PERF_X86_OPMASK_QWORDS)
return -EINVAL;
if (pred_mask & ~PERF_X86_SIMD_PRED_MASK)
return -EINVAL;
/* Only full-register sampling is allowed. */
mask = pred_mask;
if (pred_qwords == PERF_X86_OPMASK_QWORDS && mask &&
!bitmap_full(&mask, PERF_X86_SIMD_OPMASK_REGS))
return -EINVAL;
}
size = sizeof(u64) * 4;
size += (hweight64(vec_mask) * vec_qwords +
hweight32(pred_mask) * pred_qwords) * sizeof(u64);
/*
* INTR_REGS and USR_REGS could be sampled simultaneously,
* so roughly restrict the size to half of U16_MAX.
*/
if (size >= U16_MAX / 2)
return -EINVAL;
return 0;
}
#define PERF_REG_X86_RESERVED (GENMASK_ULL(PERF_REG_X86_XMM0 - 1, PERF_REG_X86_AX) & \
~GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_AX))
#define PERF_REG_X86_EXT_RESERVED (~GENMASK_ULL(PERF_REG_MISC_MAX - 1, PERF_REG_X86_AX))
#ifdef CONFIG_X86_32
#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_R8) | \
(1ULL << PERF_REG_X86_R9) | \
(1ULL << PERF_REG_X86_R10) | \
(1ULL << PERF_REG_X86_R11) | \
(1ULL << PERF_REG_X86_R12) | \
(1ULL << PERF_REG_X86_R13) | \
(1ULL << PERF_REG_X86_R14) | \
(1ULL << PERF_REG_X86_R15))
#define REG_NOSUPPORT GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_R8)
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))
if (!simd_enabled &&
(!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED))))
return -EINVAL;
/* The mask could be 0 if only the SIMD registers are interested */
if (simd_enabled &&
(mask & ~GENMASK_ULL(PERF_REG_X86_GS, PERF_REG_X86_AX)))
return -EINVAL;
return 0;
@ -100,21 +226,21 @@ u64 perf_reg_abi(struct task_struct *task)
return PERF_SAMPLE_REGS_ABI_32;
}
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}
#else /* CONFIG_X86_64 */
#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \
(1ULL << PERF_REG_X86_ES) | \
(1ULL << PERF_REG_X86_FS) | \
(1ULL << PERF_REG_X86_GS))
int perf_reg_validate(u64 mask)
int perf_reg_validate(u64 mask, bool simd_enabled)
{
if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))
if (!simd_enabled &&
(!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED))))
return -EINVAL;
/* The mask could be 0 if only the SIMD registers are interested */
if (simd_enabled &&
(mask & (REG_NOSUPPORT | PERF_REG_X86_EXT_RESERVED)))
return -EINVAL;
return 0;

28
arch/x86/kvm/vmx/pmu_intel.c
View File

@ -750,24 +750,36 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu)
intel_pmu_release_guest_lbr_event(vcpu);
}
void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu)
u64 __intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu)
{
struct kvm_pmc *pmc = NULL;
u64 guest_pebs_enable = pmu->pebs_enable & pmu->global_ctrl;
u64 pebs_enable = 0;
struct kvm_pmc *pmc;
int bit, hw_idx;
kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) {
if (!pmc_is_locally_enabled(pmc) ||
!pmc_is_globally_enabled(pmc) || !pmc->perf_event)
/*
* Omit counters that are locally disabled, don't have a perf event, or
* ended up with a perf event that is using a different counter than
* the guest, i.e. where the guest PMC is different than the host PMC
* being used on behalf of the guest. PEBS records include
* PERF_GLOBAL_STATUS, and so using a counter with a different index
* means the guest will see overflow status for the wrong counter(s).
*/
kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&guest_pebs_enable) {
if (!pmc_is_locally_enabled(pmc) || !pmc->perf_event)
continue;
/*
* A negative index indicates the event isn't mapped to a
* Note, a negative index indicates the event isn't mapped to a
* physical counter in the host, e.g. due to contention.
*/
hw_idx = pmc->perf_event->hw.idx;
if (hw_idx != pmc->idx && hw_idx > -1)
pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx);
if (hw_idx != pmc->idx)
continue;
pebs_enable |= BIT_ULL(pmc->idx);
}
return pebs_enable;
}
static bool intel_pmu_is_mediated_pmu_supported(struct x86_pmu_capability *host_pmu)

10
arch/x86/kvm/vmx/vmx.c
View File

@ -7363,12 +7363,14 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
if (kvm_vcpu_has_mediated_pmu(&vmx->vcpu))
return;
pmu->host_cross_mapped_mask = 0;
if (pmu->pebs_enable & pmu->global_ctrl)
intel_pmu_cross_mapped_check(pmu);
struct x86_guest_pebs guest_pebs = {
.enable = intel_pmu_compute_pebs_enable(pmu),
.ds_area = pmu->ds_area,
.data_cfg = pmu->pebs_data_cfg,
};
/* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */
msrs = perf_guest_get_msrs(&nr_msrs, (void *)pmu);
msrs = perf_guest_get_msrs(&nr_msrs, &guest_pebs);
if (!msrs)
return;

15
arch/x86/kvm/vmx/vmx.h
View File

@ -664,7 +664,20 @@ static __always_inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
return container_of(vcpu, struct vcpu_vmx, vcpu);
}
void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu);
u64 __intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu);
static inline u64 intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu)
{
/*
* Avoid the function call overhead in the common case that the guest
* isn't using PEBS.
*/
if (!(pmu->pebs_enable & pmu->global_ctrl))
return 0;
return __intel_pmu_compute_pebs_enable(pmu);
}
int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu);
void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu);

5
arch/x86/xen/pmu.c
View File

@ -456,12 +456,14 @@ static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
}
}
static DEFINE_PER_CPU(struct x86_perf_regs, x86_xen_intr_regs);
irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
{
int err, ret = IRQ_NONE;
struct pt_regs regs = {0};
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
uint8_t xenpmu_flags = get_xenpmu_flags();
struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_xen_intr_regs);
if (!xenpmu_data) {
pr_warn_once("%s: pmudata not initialized\n", __func__);
@ -472,7 +474,8 @@ irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
xenpmu_flags | XENPMU_IRQ_PROCESSING;
xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
xenpmu_data->pmu.pmu_flags);
if (x86_pmu.handle_irq(&regs))
x86_regs->regs = regs;
if (x86_pmu.handle_irq(&x86_regs->regs))
ret = IRQ_HANDLED;
/* Write out cached context to HW */

23
include/linux/perf_event.h
View File

@ -306,6 +306,7 @@ struct perf_event_pmu_context;
#define PERF_PMU_CAP_AUX_PAUSE 0x0200
#define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400
#define PERF_PMU_CAP_MEDIATED_VPMU 0x0800
#define PERF_PMU_CAP_SIMD_REGS 0x1000
/**
* pmu::scope
@ -1467,6 +1468,7 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data,
return size;
}
extern u64 perf_update_xregs_size(struct perf_event *event, bool intr);
extern void perf_output_sample(struct perf_output_handle *handle,
struct perf_event_header *header,
struct perf_sample_data *data,
@ -1518,6 +1520,27 @@ perf_event__output_id_sample(struct perf_event *event,
extern void
perf_log_lost_samples(struct perf_event *event, u64 lost);
static inline bool event_has_simd_regs(struct perf_event *event)
{
struct perf_event_attr *attr = &event->attr;
if (!(event->attr.sample_type &
(PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)))
return false;
return attr->sample_simd_regs_enabled != 0;
}
static inline bool event_has_extended_regs(struct perf_event *event)
{
struct perf_event_attr *attr = &event->attr;
return ((attr->sample_type & PERF_SAMPLE_REGS_USER) &&
(attr->sample_regs_user & PERF_REG_EXTENDED_MASK)) ||
((attr->sample_type & PERF_SAMPLE_REGS_INTR) &&
(attr->sample_regs_intr & PERF_REG_EXTENDED_MASK));
}
static inline bool event_has_any_exclude_flag(struct perf_event *event)
{
struct perf_event_attr *attr = &event->attr;

36
include/linux/perf_regs.h
View File

@ -9,6 +9,16 @@ struct perf_regs {
struct pt_regs *regs;
};
u64 perf_reg_value(struct pt_regs *regs, int idx);
int perf_reg_validate(u64 mask, bool simd_enabled);
u64 perf_reg_abi(struct task_struct *task);
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs);
int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask,
u16 pred_qwords, u32 pred_mask);
u64 perf_simd_reg_value(struct pt_regs *regs, int idx,
u16 qwords_idx, bool pred);
#ifdef CONFIG_HAVE_PERF_REGS
#include <asm/perf_regs.h>
@ -16,35 +26,9 @@ struct perf_regs {
#define PERF_REG_EXTENDED_MASK 0
#endif
u64 perf_reg_value(struct pt_regs *regs, int idx);
int perf_reg_validate(u64 mask);
u64 perf_reg_abi(struct task_struct *task);
void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs);
#else
#define PERF_REG_EXTENDED_MASK 0
static inline u64 perf_reg_value(struct pt_regs *regs, int idx)
{
return 0;
}
static inline int perf_reg_validate(u64 mask)
{
return mask ? -ENOSYS : 0;
}
static inline u64 perf_reg_abi(struct task_struct *task)
{
return PERF_SAMPLE_REGS_ABI_NONE;
}
static inline void perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}
#endif /* CONFIG_HAVE_PERF_REGS */
#endif /* _LINUX_PERF_REGS_H */

49
include/uapi/linux/perf_event.h
View File

@ -314,8 +314,9 @@ enum {
*/
enum perf_sample_regs_abi {
PERF_SAMPLE_REGS_ABI_NONE = 0,
PERF_SAMPLE_REGS_ABI_32 = 1,
PERF_SAMPLE_REGS_ABI_64 = 2,
PERF_SAMPLE_REGS_ABI_32 = (1 << 0),
PERF_SAMPLE_REGS_ABI_64 = (1 << 1),
PERF_SAMPLE_REGS_ABI_SIMD = (1 << 2),
};
/*
@ -383,6 +384,7 @@ enum perf_event_read_format {
#define PERF_ATTR_SIZE_VER7 128 /* Add: sig_data */
#define PERF_ATTR_SIZE_VER8 136 /* Add: config3 */
#define PERF_ATTR_SIZE_VER9 144 /* add: config4 */
#define PERF_ATTR_SIZE_VER10 176 /* Add: sample_simd_{vec|pred}_reg_* */
/*
* 'struct perf_event_attr' contains various attributes that define
@ -547,6 +549,29 @@ struct perf_event_attr {
__u64 config3; /* extension of config2 */
__u64 config4; /* extension of config3 */
/*
* Defines the sampling SIMD/PRED(predicate) registers bitmap and
* qwords (8 bytes) length.
*
* sample_simd_regs_enabled != 0 indicates there are SIMD/PRED
* registers to be sampled, the SIMD/PRED registers bitmap and
* qwords length are represented in
* sample_simd_{vec|pred}_reg_{intr|user} and
* sample_simd_{vec|pred}_reg_qwords fields separately.
*
* sample_simd_regs_enabled == 0 indicates no SIMD/PRED registers
* are sampled.
*/
__u16 sample_simd_regs_enabled;
__u16 sample_simd_pred_reg_qwords;
__u16 sample_simd_vec_reg_qwords;
__u16 __reserved_4;
__u32 sample_simd_pred_reg_intr;
__u32 sample_simd_pred_reg_user;
__u64 sample_simd_vec_reg_intr;
__u64 sample_simd_vec_reg_user;
};
/*
@ -1020,7 +1045,15 @@ enum perf_event_type {
* } && PERF_SAMPLE_BRANCH_STACK
*
* { u64 abi; # enum perf_sample_regs_abi
* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
* u64 regs[weight(mask)];
* struct {
* u64 nr_vectors; # 0 ... weight(sample_simd_vec_reg_user)
* u64 vector_qwords; # 0 ... sample_simd_vec_reg_qwords
* u64 nr_pred; # 0 ... weight(sample_simd_pred_reg_user)
* u64 pred_qwords; # 0 ... sample_simd_pred_reg_qwords
* u64 data[nr_vectors * vector_qwords + nr_pred * pred_qwords];
* } && (abi & PERF_SAMPLE_REGS_ABI_SIMD)
* } && PERF_SAMPLE_REGS_USER
*
* { u64 size;
* char data[size];
@ -1047,7 +1080,15 @@ enum perf_event_type {
* { u64 data_src; } && PERF_SAMPLE_DATA_SRC
* { u64 transaction; } && PERF_SAMPLE_TRANSACTION
* { u64 abi; # enum perf_sample_regs_abi
* u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR
* u64 regs[weight(mask)];
* struct {
* u64 nr_vectors; # 0 ... weight(sample_simd_vec_reg_intr)
* u64 vector_qwords; # 0 ... sample_simd_vec_reg_qwords
* u64 nr_pred; # 0 ... weight(sample_simd_pred_reg_intr)
* u64 pred_qwords; # 0 ... sample_simd_pred_reg_qwords
* u64 data[nr_vectors * vector_qwords + nr_pred * pred_qwords];
* } && (abi & PERF_SAMPLE_REGS_ABI_SIMD)
* } && PERF_SAMPLE_REGS_INTR
* { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR
* { u64 cgroup;} && PERF_SAMPLE_CGROUP
* { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE

177
kernel/events/core.c
View File

@ -7831,22 +7831,82 @@ unsigned long perf_instruction_pointer(struct perf_event *event,
0 : perf_arch_instruction_pointer(regs);
}
u64 __weak perf_reg_value(struct pt_regs *regs, int idx)
{
return 0;
}
int __weak perf_reg_validate(u64 mask, bool simd_enabled)
{
return mask ? -ENOSYS : 0;
}
u64 __weak perf_reg_abi(struct task_struct *task)
{
return PERF_SAMPLE_REGS_ABI_NONE;
}
void __weak perf_get_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
regs_user->regs = task_pt_regs(current);
regs_user->abi = perf_reg_abi(current);
}
#define word_for_each_set_bit(bit, val) \
for (unsigned long long __v = (val); \
__v && ((bit = __builtin_ctzll(__v)), 1); \
__v &= __v - 1)
static void
perf_output_sample_regs(struct perf_output_handle *handle,
struct pt_regs *regs, u64 mask)
{
int bit;
DECLARE_BITMAP(_mask, 64);
bitmap_from_u64(_mask, mask);
for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) {
u64 val;
val = perf_reg_value(regs, bit);
word_for_each_set_bit(bit, mask) {
u64 val = perf_reg_value(regs, bit);
perf_output_put(handle, val);
}
}
static void
perf_output_sample_simd_regs(struct perf_output_handle *handle,
struct perf_event *event,
struct pt_regs *regs,
u64 mask, u32 pred_mask)
{
u64 pred_qwords = event->attr.sample_simd_pred_reg_qwords;
u64 vec_qwords = event->attr.sample_simd_vec_reg_qwords;
u64 nr_vectors = hweight64(mask);
u64 nr_pred = hweight32(pred_mask);
int bit;
perf_output_put(handle, nr_vectors);
perf_output_put(handle, vec_qwords);
perf_output_put(handle, nr_pred);
perf_output_put(handle, pred_qwords);
if (nr_vectors) {
word_for_each_set_bit(bit, mask) {
for (int i = 0; i < vec_qwords; i++) {
u64 val = perf_simd_reg_value(regs, bit,
i, false);
perf_output_put(handle, val);
}
}
}
if (nr_pred) {
word_for_each_set_bit(bit, pred_mask) {
for (int i = 0; i < pred_qwords; i++) {
u64 val = perf_simd_reg_value(regs, bit,
i, true);
perf_output_put(handle, val);
}
}
}
}
static void perf_sample_regs_user(struct perf_regs *regs_user,
struct pt_regs *regs)
{
@ -7880,6 +7940,17 @@ static void perf_sample_regs_intr(struct perf_regs *regs_intr,
}
}
int __weak perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask,
u16 pred_qwords, u32 pred_mask)
{
return -EINVAL;
}
u64 __weak perf_simd_reg_value(struct pt_regs *regs, int idx,
u16 qwords_idx, bool pred)
{
return 0;
}
/*
* Get remaining task size from user stack pointer.
@ -8415,10 +8486,17 @@ void perf_output_sample(struct perf_output_handle *handle,
perf_output_put(handle, abi);
if (abi) {
u64 mask = event->attr.sample_regs_user;
struct perf_event_attr *attr = &event->attr;
u64 mask = attr->sample_regs_user;
perf_output_sample_regs(handle,
data->regs_user.regs,
mask);
if (abi & PERF_SAMPLE_REGS_ABI_SIMD) {
perf_output_sample_simd_regs(handle, event,
data->regs_user.regs,
attr->sample_simd_vec_reg_user,
attr->sample_simd_pred_reg_user);
}
}
}
@ -8446,11 +8524,18 @@ void perf_output_sample(struct perf_output_handle *handle,
perf_output_put(handle, abi);
if (abi) {
u64 mask = event->attr.sample_regs_intr;
struct perf_event_attr *attr = &event->attr;
u64 mask = attr->sample_regs_intr;
perf_output_sample_regs(handle,
data->regs_intr.regs,
mask);
if (abi & PERF_SAMPLE_REGS_ABI_SIMD) {
perf_output_sample_simd_regs(handle, event,
data->regs_intr.regs,
attr->sample_simd_vec_reg_intr,
attr->sample_simd_pred_reg_intr);
}
}
}
@ -8653,6 +8738,29 @@ static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
return d * !!(flags & s);
}
u64 perf_update_xregs_size(struct perf_event *event, bool intr)
{
u16 pred_qwords = event->attr.sample_simd_pred_reg_qwords;
u16 vec_qwords = event->attr.sample_simd_vec_reg_qwords;
u64 pred_mask;
u64 mask;
int size;
if (intr) {
mask = event->attr.sample_simd_vec_reg_intr;
pred_mask = event->attr.sample_simd_pred_reg_intr;
} else {
mask = event->attr.sample_simd_vec_reg_user;
pred_mask = event->attr.sample_simd_pred_reg_user;
}
size = sizeof(u64) * 4;
size += (hweight64(mask) * vec_qwords +
hweight64(pred_mask) * pred_qwords) * sizeof(u64);
return size;
}
void perf_prepare_sample(struct perf_sample_data *data,
struct perf_event *event,
struct pt_regs *regs)
@ -8715,7 +8823,12 @@ void perf_prepare_sample(struct perf_sample_data *data,
if (data->regs_user.regs) {
u64 mask = event->attr.sample_regs_user;
size += hweight64(mask) * sizeof(u64);
if (event_has_simd_regs(event)) {
size += perf_update_xregs_size(event, false);
data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD;
}
}
data->dyn_size += size;
@ -8780,6 +8893,10 @@ void perf_prepare_sample(struct perf_sample_data *data,
u64 mask = event->attr.sample_regs_intr;
size += hweight64(mask) * sizeof(u64);
if (event_has_simd_regs(event)) {
size += perf_update_xregs_size(event, true);
data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD;
}
}
data->dyn_size += size;
@ -13071,12 +13188,6 @@ int perf_pmu_unregister(struct pmu *pmu)
}
EXPORT_SYMBOL_GPL(perf_pmu_unregister);
static inline bool has_extended_regs(struct perf_event *event)
{
return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||
(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);
}
static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
{
struct perf_event_context *ctx = NULL;
@ -13110,8 +13221,14 @@ static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
if (ret)
goto err_pmu;
if (!(pmu->capabilities & PERF_PMU_CAP_SIMD_REGS) &&
event_has_simd_regs(event)) {
ret = -EOPNOTSUPP;
goto err_destroy;
}
if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
has_extended_regs(event)) {
event_has_extended_regs(event)) {
ret = -EOPNOTSUPP;
goto err_destroy;
}
@ -13605,7 +13722,8 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
attr->size = size;
if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
if (attr->__reserved_1 || attr->__reserved_2 ||
attr->__reserved_3 || attr->__reserved_4)
return -EINVAL;
if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
@ -13651,9 +13769,18 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
}
if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
ret = perf_reg_validate(attr->sample_regs_user);
ret = perf_reg_validate(attr->sample_regs_user,
attr->sample_simd_regs_enabled);
if (ret)
return ret;
if (attr->sample_simd_regs_enabled) {
ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords,
attr->sample_simd_vec_reg_user,
attr->sample_simd_pred_reg_qwords,
attr->sample_simd_pred_reg_user);
if (ret)
return ret;
}
}
if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
@ -13674,8 +13801,20 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
if (!attr->sample_max_stack)
attr->sample_max_stack = sysctl_perf_event_max_stack;
if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
ret = perf_reg_validate(attr->sample_regs_intr);
if (attr->sample_type & PERF_SAMPLE_REGS_INTR) {
ret = perf_reg_validate(attr->sample_regs_intr,
attr->sample_simd_regs_enabled);
if (ret)
return ret;
if (attr->sample_simd_regs_enabled) {
ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords,
attr->sample_simd_vec_reg_intr,
attr->sample_simd_pred_reg_qwords,
attr->sample_simd_pred_reg_intr);
if (ret)
return ret;
}
}
#ifndef CONFIG_CGROUP_PERF
if (attr->sample_type & PERF_SAMPLE_CGROUP)