github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-31 11:37:06 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge cfeb2522c3 ("Merge tag 'perf-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip") into android-mainline

Browse Source 
Steps on the way to 5.19-rc1

Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
Change-Id: Icd6673702c20fb9535c00cf92945709c3b728778
This commit is contained in:
Greg Kroah-Hartman 2022-06-06 11:22:24 +02:00
commit 1cb27f2c55
99 changed files with 2821 additions and 1254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -1339,7 +1339,7 @@ install: sub_make_done :=
# ---------------------------------------------------------------------------
# Tools
ifdef CONFIG_STACK_VALIDATION
ifdef CONFIG_OBJTOOL
prepare: tools/objtool
endif

18
arch/Kconfig
View File

@ -54,6 +54,7 @@ config JUMP_LABEL
bool "Optimize very unlikely/likely branches"
depends on HAVE_ARCH_JUMP_LABEL
depends on CC_HAS_ASM_GOTO
select OBJTOOL if HAVE_JUMP_LABEL_HACK
help
This option enables a transparent branch optimization that
makes certain almost-always-true or almost-always-false branch
@ -1034,11 +1035,23 @@ config ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
depends on MMU
select ARCH_HAS_ELF_RANDOMIZE
config HAVE_OBJTOOL
bool
config HAVE_JUMP_LABEL_HACK
bool
config HAVE_NOINSTR_HACK
bool
config HAVE_NOINSTR_VALIDATION
bool
config HAVE_STACK_VALIDATION
bool
help
Architecture supports the 'objtool check' host tool command, which
performs compile-time stack metadata validation.
Architecture supports objtool compile-time frame pointer rule
validation.
config HAVE_RELIABLE_STACKTRACE
bool
@ -1308,6 +1321,7 @@ config HAVE_STATIC_CALL
config HAVE_STATIC_CALL_INLINE
bool
depends on HAVE_STATIC_CALL
select OBJTOOL
config HAVE_PREEMPT_DYNAMIC
bool

1
arch/arm/kernel/signal.c
View File

@ -708,6 +708,7 @@ static_assert(offsetof(siginfo_t, si_upper) == 0x18);
static_assert(offsetof(siginfo_t, si_pkey) == 0x14);
static_assert(offsetof(siginfo_t, si_perf_data) == 0x10);
static_assert(offsetof(siginfo_t, si_perf_type) == 0x14);
static_assert(offsetof(siginfo_t, si_perf_flags) == 0x18);
static_assert(offsetof(siginfo_t, si_band) == 0x0c);
static_assert(offsetof(siginfo_t, si_fd) == 0x10);
static_assert(offsetof(siginfo_t, si_call_addr) == 0x0c);

8
arch/arm64/kernel/entry-common.c
View File

@ -75,7 +75,7 @@ static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)
if (interrupts_enabled(regs)) {
if (regs->exit_rcu) {
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
rcu_irq_exit();
lockdep_hardirqs_on(CALLER_ADDR0);
return;
@ -121,7 +121,7 @@ static __always_inline void enter_from_user_mode(struct pt_regs *regs)
static __always_inline void __exit_to_user_mode(void)
{
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
user_enter_irqoff();
lockdep_hardirqs_on(CALLER_ADDR0);
}
@ -179,7 +179,7 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs)
ftrace_nmi_exit();
if (restore) {
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
}
rcu_nmi_exit();
@ -215,7 +215,7 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
if (restore) {
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
}
rcu_nmi_exit();

1
arch/arm64/kernel/signal.c
View File

@ -1179,6 +1179,7 @@ static_assert(offsetof(siginfo_t, si_upper) == 0x28);
static_assert(offsetof(siginfo_t, si_pkey) == 0x20);
static_assert(offsetof(siginfo_t, si_perf_data) == 0x18);
static_assert(offsetof(siginfo_t, si_perf_type) == 0x20);
static_assert(offsetof(siginfo_t, si_perf_flags) == 0x24);
static_assert(offsetof(siginfo_t, si_band) == 0x10);
static_assert(offsetof(siginfo_t, si_fd) == 0x18);
static_assert(offsetof(siginfo_t, si_call_addr) == 0x10);

1
arch/arm64/kernel/signal32.c
View File

@ -487,6 +487,7 @@ static_assert(offsetof(compat_siginfo_t, si_upper) == 0x18);
static_assert(offsetof(compat_siginfo_t, si_pkey) == 0x14);
static_assert(offsetof(compat_siginfo_t, si_perf_data) == 0x10);
static_assert(offsetof(compat_siginfo_t, si_perf_type) == 0x14);
static_assert(offsetof(compat_siginfo_t, si_perf_flags) == 0x18);
static_assert(offsetof(compat_siginfo_t, si_band) == 0x0c);
static_assert(offsetof(compat_siginfo_t, si_fd) == 0x10);
static_assert(offsetof(compat_siginfo_t, si_call_addr) == 0x0c);

1
arch/m68k/kernel/signal.c
View File

@ -625,6 +625,7 @@ static inline void siginfo_build_tests(void)
/* _sigfault._perf */
BUILD_BUG_ON(offsetof(siginfo_t, si_perf_data) != 0x10);
BUILD_BUG_ON(offsetof(siginfo_t, si_perf_type) != 0x14);
BUILD_BUG_ON(offsetof(siginfo_t, si_perf_flags) != 0x18);
/* _sigpoll */
BUILD_BUG_ON(offsetof(siginfo_t, si_band) != 0x0c);

1
arch/sparc/kernel/signal32.c
View File

@ -779,5 +779,6 @@ static_assert(offsetof(compat_siginfo_t, si_upper) == 0x18);
static_assert(offsetof(compat_siginfo_t, si_pkey) == 0x14);
static_assert(offsetof(compat_siginfo_t, si_perf_data) == 0x10);
static_assert(offsetof(compat_siginfo_t, si_perf_type) == 0x14);
static_assert(offsetof(compat_siginfo_t, si_perf_flags) == 0x18);
static_assert(offsetof(compat_siginfo_t, si_band) == 0x0c);
static_assert(offsetof(compat_siginfo_t, si_fd) == 0x10);

1
arch/sparc/kernel/signal_64.c
View File

@ -590,5 +590,6 @@ static_assert(offsetof(siginfo_t, si_upper) == 0x28);
static_assert(offsetof(siginfo_t, si_pkey) == 0x20);
static_assert(offsetof(siginfo_t, si_perf_data) == 0x18);
static_assert(offsetof(siginfo_t, si_perf_type) == 0x20);
static_assert(offsetof(siginfo_t, si_perf_flags) == 0x24);
static_assert(offsetof(siginfo_t, si_band) == 0x10);
static_assert(offsetof(siginfo_t, si_fd) == 0x14);

20
arch/x86/Kconfig
View File

@ -188,7 +188,7 @@ config X86
select HAVE_CONTEXT_TRACKING if X86_64
select HAVE_CONTEXT_TRACKING_OFFSTACK if HAVE_CONTEXT_TRACKING
select HAVE_C_RECORDMCOUNT
select HAVE_OBJTOOL_MCOUNT if STACK_VALIDATION
select HAVE_OBJTOOL_MCOUNT if HAVE_OBJTOOL
select HAVE_BUILDTIME_MCOUNT_SORT
select HAVE_DEBUG_KMEMLEAK
select HAVE_DMA_CONTIGUOUS
@ -212,6 +212,7 @@ config X86
select HAVE_IOREMAP_PROT
select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64
select HAVE_IRQ_TIME_ACCOUNTING
select HAVE_JUMP_LABEL_HACK if HAVE_OBJTOOL
select HAVE_KERNEL_BZIP2
select HAVE_KERNEL_GZIP
select HAVE_KERNEL_LZ4
@ -230,7 +231,10 @@ config X86
select HAVE_MOD_ARCH_SPECIFIC
select HAVE_MOVE_PMD
select HAVE_MOVE_PUD
select HAVE_NOINSTR_HACK if HAVE_OBJTOOL
select HAVE_NMI
select HAVE_NOINSTR_VALIDATION if HAVE_OBJTOOL
select HAVE_OBJTOOL if X86_64
select HAVE_OPTPROBES
select HAVE_PCSPKR_PLATFORM
select HAVE_PERF_EVENTS
@ -239,17 +243,17 @@ config X86
select HAVE_PCI
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
select MMU_GATHER_RCU_TABLE_FREE if PARAVIRT
select MMU_GATHER_RCU_TABLE_FREE if PARAVIRT
select HAVE_POSIX_CPU_TIMERS_TASK_WORK
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION
select HAVE_RELIABLE_STACKTRACE if UNWINDER_ORC || STACK_VALIDATION
select HAVE_FUNCTION_ARG_ACCESS_API
select HAVE_SETUP_PER_CPU_AREA
select HAVE_SOFTIRQ_ON_OWN_STACK
select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR
select HAVE_STACK_VALIDATION if X86_64
select HAVE_STACK_VALIDATION if HAVE_OBJTOOL
select HAVE_STATIC_CALL
select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION
select HAVE_STATIC_CALL_INLINE if HAVE_OBJTOOL
select HAVE_PREEMPT_DYNAMIC_CALL
select HAVE_RSEQ
select HAVE_SYSCALL_TRACEPOINTS
@ -268,7 +272,6 @@ config X86
select RTC_MC146818_LIB
select SPARSE_IRQ
select SRCU
select STACK_VALIDATION if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE)
select SYSCTL_EXCEPTION_TRACE
select THREAD_INFO_IN_TASK
select TRACE_IRQFLAGS_SUPPORT
@ -459,6 +462,7 @@ config GOLDFISH
config RETPOLINE
bool "Avoid speculative indirect branches in kernel"
select OBJTOOL if HAVE_OBJTOOL
default y
help
Compile kernel with the retpoline compiler options to guard against
@ -472,6 +476,7 @@ config CC_HAS_SLS
config SLS
bool "Mitigate Straight-Line-Speculation"
depends on CC_HAS_SLS && X86_64
select OBJTOOL if HAVE_OBJTOOL
default n
help
Compile the kernel with straight-line-speculation options to guard
@ -1859,9 +1864,10 @@ config CC_HAS_IBT
config X86_KERNEL_IBT
prompt "Indirect Branch Tracking"
bool
depends on X86_64 && CC_HAS_IBT && STACK_VALIDATION
depends on X86_64 && CC_HAS_IBT && HAVE_OBJTOOL
# https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f
depends on !LD_IS_LLD || LLD_VERSION >= 140000
select OBJTOOL
help
Build the kernel with support for Indirect Branch Tracking, a
hardware support course-grain forward-edge Control Flow Integrity

2
arch/x86/Kconfig.debug
View File

@ -237,7 +237,7 @@ choice
config UNWINDER_ORC
bool "ORC unwinder"
depends on X86_64
select STACK_VALIDATION
select OBJTOOL
help
This option enables the ORC (Oops Rewind Capability) unwinder for
unwinding kernel stack traces. It uses a custom data format which is

8
arch/x86/events/Kconfig
View File

@ -44,4 +44,12 @@ config PERF_EVENTS_AMD_UNCORE
To compile this driver as a module, choose M here: the
module will be called 'amd-uncore'.
config PERF_EVENTS_AMD_BRS
depends on PERF_EVENTS && CPU_SUP_AMD
bool "AMD Zen3 Branch Sampling support"
help
Enable AMD Zen3 branch sampling support (BRS) which samples up to
16 consecutive taken branches in registers.
endmenu

1
arch/x86/events/amd/Makefile
View File

@ -1,5 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CPU_SUP_AMD) += core.o
obj-$(CONFIG_PERF_EVENTS_AMD_BRS) += brs.o
obj-$(CONFIG_PERF_EVENTS_AMD_POWER) += power.o
obj-$(CONFIG_X86_LOCAL_APIC) += ibs.o
obj-$(CONFIG_PERF_EVENTS_AMD_UNCORE) += amd-uncore.o

367
arch/x86/events/amd/brs.c Normal file
View File

@ -0,0 +1,367 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Implement support for AMD Fam19h Branch Sampling feature
* Based on specifications published in AMD PPR Fam19 Model 01
*
* Copyright 2021 Google LLC
* Contributed by Stephane Eranian <eranian@google.com>
*/
#include <linux/kernel.h>
#include <linux/jump_label.h>
#include <asm/msr.h>
#include <asm/cpufeature.h>
#include "../perf_event.h"
#define BRS_POISON 0xFFFFFFFFFFFFFFFEULL /* mark limit of valid entries */
/* Debug Extension Configuration register layout */
union amd_debug_extn_cfg {
__u64 val;
struct {
__u64 rsvd0:2, /* reserved */
brsmen:1, /* branch sample enable */
rsvd4_3:2,/* reserved - must be 0x3 */
vb:1, /* valid branches recorded */
rsvd2:10, /* reserved */
msroff:4, /* index of next entry to write */
rsvd3:4, /* reserved */
pmc:3, /* #PMC holding the sampling event */
rsvd4:37; /* reserved */
};
};
static inline unsigned int brs_from(int idx)
{
return MSR_AMD_SAMP_BR_FROM + 2 * idx;
}
static inline unsigned int brs_to(int idx)
{
return MSR_AMD_SAMP_BR_FROM + 2 * idx + 1;
}
static inline void set_debug_extn_cfg(u64 val)
{
/* bits[4:3] must always be set to 11b */
wrmsrl(MSR_AMD_DBG_EXTN_CFG, val | 3ULL << 3);
}
static inline u64 get_debug_extn_cfg(void)
{
u64 val;
rdmsrl(MSR_AMD_DBG_EXTN_CFG, val);
return val;
}
static bool __init amd_brs_detect(void)
{
if (!cpu_feature_enabled(X86_FEATURE_BRS))
return false;
switch (boot_cpu_data.x86) {
case 0x19: /* AMD Fam19h (Zen3) */
x86_pmu.lbr_nr = 16;
/* No hardware filtering supported */
x86_pmu.lbr_sel_map = NULL;
x86_pmu.lbr_sel_mask = 0;
break;
default:
return false;
}
return true;
}
/*
* Current BRS implementation does not support branch type or privilege level
* filtering. Therefore, this function simply enforces these limitations. No need for
* a br_sel_map. Software filtering is not supported because it would not correlate well
* with a sampling period.
*/
int amd_brs_setup_filter(struct perf_event *event)
{
u64 type = event->attr.branch_sample_type;
/* No BRS support */
if (!x86_pmu.lbr_nr)
return -EOPNOTSUPP;
/* Can only capture all branches, i.e., no filtering */
if ((type & ~PERF_SAMPLE_BRANCH_PLM_ALL) != PERF_SAMPLE_BRANCH_ANY)
return -EINVAL;
return 0;
}
/* tos = top of stack, i.e., last valid entry written */
static inline int amd_brs_get_tos(union amd_debug_extn_cfg *cfg)
{
/*
* msroff: index of next entry to write so top-of-stack is one off
* if BRS is full then msroff is set back to 0.
*/
return (cfg->msroff ? cfg->msroff : x86_pmu.lbr_nr) - 1;
}
/*
* make sure we have a sane BRS offset to begin with
* especially with kexec
*/
void amd_brs_reset(void)
{
if (!cpu_feature_enabled(X86_FEATURE_BRS))
return;
/*
* Reset config
*/
set_debug_extn_cfg(0);
/*
* Mark first entry as poisoned
*/
wrmsrl(brs_to(0), BRS_POISON);
}
int __init amd_brs_init(void)
{
if (!amd_brs_detect())
return -EOPNOTSUPP;
pr_cont("%d-deep BRS, ", x86_pmu.lbr_nr);
return 0;
}
void amd_brs_enable(void)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
union amd_debug_extn_cfg cfg;
/* Activate only on first user */
if (++cpuc->brs_active > 1)
return;
cfg.val = 0; /* reset all fields */
cfg.brsmen = 1; /* enable branch sampling */
/* Set enable bit */
set_debug_extn_cfg(cfg.val);
}
void amd_brs_enable_all(void)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
if (cpuc->lbr_users)
amd_brs_enable();
}
void amd_brs_disable(void)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
union amd_debug_extn_cfg cfg;
/* Check if active (could be disabled via x86_pmu_disable_all()) */
if (!cpuc->brs_active)
return;
/* Only disable for last user */
if (--cpuc->brs_active)
return;
/*
* Clear the brsmen bit but preserve the others as they contain
* useful state such as vb and msroff
*/
cfg.val = get_debug_extn_cfg();
/*
* When coming in on interrupt and BRS is full, then hw will have
* already stopped BRS, no need to issue wrmsr again
*/
if (cfg.brsmen) {
cfg.brsmen = 0;
set_debug_extn_cfg(cfg.val);
}
}
void amd_brs_disable_all(void)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
if (cpuc->lbr_users)
amd_brs_disable();
}
static bool amd_brs_match_plm(struct perf_event *event, u64 to)
{
int type = event->attr.branch_sample_type;
int plm_k = PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_HV;
int plm_u = PERF_SAMPLE_BRANCH_USER;
if (!(type & plm_k) && kernel_ip(to))
return 0;
if (!(type & plm_u) && !kernel_ip(to))
return 0;
return 1;
}
/*
* Caller must ensure amd_brs_inuse() is true before calling
* return:
*/
void amd_brs_drain(void)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct perf_event *event = cpuc->events[0];
struct perf_branch_entry *br = cpuc->lbr_entries;
union amd_debug_extn_cfg cfg;
u32 i, nr = 0, num, tos, start;
u32 shift = 64 - boot_cpu_data.x86_virt_bits;
/*
* BRS event forced on PMC0,
* so check if there is an event.
* It is possible to have lbr_users > 0 but the event
* not yet scheduled due to long latency PMU irq
*/
if (!event)
goto empty;
cfg.val = get_debug_extn_cfg();
/* Sanity check [0-x86_pmu.lbr_nr] */
if (WARN_ON_ONCE(cfg.msroff >= x86_pmu.lbr_nr))
goto empty;
/* No valid branch */
if (cfg.vb == 0)
goto empty;
/*
* msr.off points to next entry to be written
* tos = most recent entry index = msr.off - 1
* BRS register buffer saturates, so we know we have
* start < tos and that we have to read from start to tos
*/
start = 0;
tos = amd_brs_get_tos(&cfg);
num = tos - start + 1;
/*
* BRS is only one pass (saturation) from MSROFF to depth-1
* MSROFF wraps to zero when buffer is full
*/
for (i = 0; i < num; i++) {
u32 brs_idx = tos - i;
u64 from, to;
rdmsrl(brs_to(brs_idx), to);
/* Entry does not belong to us (as marked by kernel) */
if (to == BRS_POISON)
break;
/*
* Sign-extend SAMP_BR_TO to 64 bits, bits 61-63 are reserved.
* Necessary to generate proper virtual addresses suitable for
* symbolization
*/
to = (u64)(((s64)to << shift) >> shift);
if (!amd_brs_match_plm(event, to))
continue;
rdmsrl(brs_from(brs_idx), from);
perf_clear_branch_entry_bitfields(br+nr);
br[nr].from = from;
br[nr].to = to;
nr++;
}
empty:
/* Record number of sampled branches */
cpuc->lbr_stack.nr = nr;
}
/*
* Poison most recent entry to prevent reuse by next task
* required because BRS entry are not tagged by PID
*/
static void amd_brs_poison_buffer(void)
{
union amd_debug_extn_cfg cfg;
unsigned int idx;
/* Get current state */
cfg.val = get_debug_extn_cfg();
/* idx is most recently written entry */
idx = amd_brs_get_tos(&cfg);
/* Poison target of entry */
wrmsrl(brs_to(idx), BRS_POISON);
}
/*
* On context switch in, we need to make sure no samples from previous user
* are left in the BRS.
*
* On ctxswin, sched_in = true, called after the PMU has started
* On ctxswout, sched_in = false, called before the PMU is stopped
*/
void amd_pmu_brs_sched_task(struct perf_event_context *ctx, bool sched_in)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
/* no active users */
if (!cpuc->lbr_users)
return;
/*
* On context switch in, we need to ensure we do not use entries
* from previous BRS user on that CPU, so we poison the buffer as
* a faster way compared to resetting all entries.
*/
if (sched_in)
amd_brs_poison_buffer();
}
/*
* called from ACPI processor_idle.c or acpi_pad.c
* with interrupts disabled
*/
void perf_amd_brs_lopwr_cb(bool lopwr_in)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
union amd_debug_extn_cfg cfg;
/*
* on mwait in, we may end up in non C0 state.
* we must disable branch sampling to avoid holding the NMI
* for too long. We disable it in hardware but we
* keep the state in cpuc, so we can re-enable.
*
* The hardware will deliver the NMI if needed when brsmen cleared
*/
if (cpuc->brs_active) {
cfg.val = get_debug_extn_cfg();
cfg.brsmen = !lopwr_in;
set_debug_extn_cfg(cfg.val);
}
}
DEFINE_STATIC_CALL_NULL(perf_lopwr_cb, perf_amd_brs_lopwr_cb);
EXPORT_STATIC_CALL_TRAMP_GPL(perf_lopwr_cb);
void __init amd_brs_lopwr_init(void)
{
static_call_update(perf_lopwr_cb, perf_amd_brs_lopwr_cb);
}

505
arch/x86/events/amd/core.c
View File

@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/perf_event.h>
#include <linux/jump_label.h>
#include <linux/export.h>
#include <linux/types.h>
#include <linux/init.h>
@ -7,6 +8,7 @@
#include <linux/delay.h>
#include <linux/jiffies.h>
#include <asm/apicdef.h>
#include <asm/apic.h>
#include <asm/nmi.h>
#include "../perf_event.h"
@ -18,6 +20,9 @@ static unsigned long perf_nmi_window;
#define AMD_MERGE_EVENT ((0xFULL << 32) | 0xFFULL)
#define AMD_MERGE_EVENT_ENABLE (AMD_MERGE_EVENT | ARCH_PERFMON_EVENTSEL_ENABLE)
/* PMC Enable and Overflow bits for PerfCntrGlobal* registers */
static u64 amd_pmu_global_cntr_mask __read_mostly;
static __initconst const u64 amd_hw_cache_event_ids
[PERF_COUNT_HW_CACHE_MAX]
[PERF_COUNT_HW_CACHE_OP_MAX]
@ -325,8 +330,16 @@ static inline bool amd_is_pair_event_code(struct hw_perf_event *hwc)
}
}
#define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */
static inline int amd_is_brs_event(struct perf_event *e)
{
return (e->hw.config & AMD64_RAW_EVENT_MASK) == AMD_FAM19H_BRS_EVENT;
}
static int amd_core_hw_config(struct perf_event *event)
{
int ret = 0;
if (event->attr.exclude_host && event->attr.exclude_guest)
/*
* When HO == GO == 1 the hardware treats that as GO == HO == 0
@ -343,7 +356,66 @@ static int amd_core_hw_config(struct perf_event *event)
if ((x86_pmu.flags & PMU_FL_PAIR) && amd_is_pair_event_code(&event->hw))
event->hw.flags |= PERF_X86_EVENT_PAIR;
return 0;
/*
* if branch stack is requested
*/
if (has_branch_stack(event)) {
/*
* Due to interrupt holding, BRS is not recommended in
* counting mode.
*/
if (!is_sampling_event(event))
return -EINVAL;
/*
* Due to the way BRS operates by holding the interrupt until
* lbr_nr entries have been captured, it does not make sense
* to allow sampling on BRS with an event that does not match
* what BRS is capturing, i.e., retired taken branches.
* Otherwise the correlation with the event's period is even
* more loose:
*
* With retired taken branch:
* Effective P = P + 16 + X
* With any other event:
* Effective P = P + Y + X
*
* Where X is the number of taken branches due to interrupt
* skid. Skid is large.
*
* Where Y is the occurences of the event while BRS is
* capturing the lbr_nr entries.
*
* By using retired taken branches, we limit the impact on the
* Y variable. We know it cannot be more than the depth of
* BRS.
*/
if (!amd_is_brs_event(event))
return -EINVAL;
/*
* BRS implementation does not work with frequency mode
* reprogramming of the period.
*/
if (event->attr.freq)
return -EINVAL;
/*
* The kernel subtracts BRS depth from period, so it must
* be big enough.
*/
if (event->attr.sample_period <= x86_pmu.lbr_nr)
return -EINVAL;
/*
* Check if we can allow PERF_SAMPLE_BRANCH_STACK
*/
ret = amd_brs_setup_filter(event);
/* only set in case of success */
if (!ret)
event->hw.flags |= PERF_X86_EVENT_AMD_BRS;
}
return ret;
}
static inline int amd_is_nb_event(struct hw_perf_event *hwc)
@ -366,7 +438,7 @@ static int amd_pmu_hw_config(struct perf_event *event)
if (event->attr.precise_ip && get_ibs_caps())
return -ENOENT;
if (has_branch_stack(event))
if (has_branch_stack(event) && !x86_pmu.lbr_nr)
return -EOPNOTSUPP;
ret = x86_pmu_hw_config(event);
@ -510,6 +582,18 @@ static struct amd_nb *amd_alloc_nb(int cpu)
return nb;
}
static void amd_pmu_cpu_reset(int cpu)
{
if (x86_pmu.version < 2)
return;
/* Clear enable bits i.e. PerfCntrGlobalCtl.PerfCntrEn */
wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 0);
/* Clear overflow bits i.e. PerfCntrGLobalStatus.PerfCntrOvfl */
wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, amd_pmu_global_cntr_mask);
}
static int amd_pmu_cpu_prepare(int cpu)
{
struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
@ -555,6 +639,9 @@ static void amd_pmu_cpu_starting(int cpu)
cpuc->amd_nb->nb_id = nb_id;
cpuc->amd_nb->refcnt++;
amd_brs_reset();
amd_pmu_cpu_reset(cpu);
}
static void amd_pmu_cpu_dead(int cpu)
@ -574,8 +661,54 @@ static void amd_pmu_cpu_dead(int cpu)
cpuhw->amd_nb = NULL;
}
amd_pmu_cpu_reset(cpu);
}
static inline void amd_pmu_set_global_ctl(u64 ctl)
{
wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
}
static inline u64 amd_pmu_get_global_status(void)
{
u64 status;
/* PerfCntrGlobalStatus is read-only */
rdmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status);
return status & amd_pmu_global_cntr_mask;
}
static inline void amd_pmu_ack_global_status(u64 status)
{
/*
* PerfCntrGlobalStatus is read-only but an overflow acknowledgment
* mechanism exists; writing 1 to a bit in PerfCntrGlobalStatusClr
* clears the same bit in PerfCntrGlobalStatus
*/
/* Only allow modifications to PerfCntrGlobalStatus.PerfCntrOvfl */
status &= amd_pmu_global_cntr_mask;
wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status);
}
static bool amd_pmu_test_overflow_topbit(int idx)
{
u64 counter;
rdmsrl(x86_pmu_event_addr(idx), counter);
return !(counter & BIT_ULL(x86_pmu.cntval_bits - 1));
}
static bool amd_pmu_test_overflow_status(int idx)
{
return amd_pmu_get_global_status() & BIT_ULL(idx);
}
DEFINE_STATIC_CALL(amd_pmu_test_overflow, amd_pmu_test_overflow_topbit);
/*
* When a PMC counter overflows, an NMI is used to process the event and
* reset the counter. NMI latency can result in the counter being updated
@ -588,7 +721,6 @@ static void amd_pmu_cpu_dead(int cpu)
static void amd_pmu_wait_on_overflow(int idx)
{
unsigned int i;
u64 counter;
/*
* Wait for the counter to be reset if it has overflowed. This loop
@ -596,8 +728,7 @@ static void amd_pmu_wait_on_overflow(int idx)
* forever...
*/
for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) {
rdmsrl(x86_pmu_event_addr(idx), counter);
if (counter & (1ULL << (x86_pmu.cntval_bits - 1)))
if (!static_call(amd_pmu_test_overflow)(idx))
break;
/* Might be in IRQ context, so can't sleep */
@ -605,13 +736,11 @@ static void amd_pmu_wait_on_overflow(int idx)
}
}
static void amd_pmu_disable_all(void)
static void amd_pmu_check_overflow(void)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
int idx;
x86_pmu_disable_all();
/*
* This shouldn't be called from NMI context, but add a safeguard here
* to return, since if we're in NMI context we can't wait for an NMI
@ -634,6 +763,47 @@ static void amd_pmu_disable_all(void)
}
}
static void amd_pmu_enable_event(struct perf_event *event)
{
x86_pmu_enable_event(event);
}
static void amd_pmu_enable_all(int added)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
int idx;
amd_brs_enable_all();
for (idx = 0; idx < x86_pmu.num_counters; idx++) {
/* only activate events which are marked as active */
if (!test_bit(idx, cpuc->active_mask))
continue;
amd_pmu_enable_event(cpuc->events[idx]);
}
}
static void amd_pmu_v2_enable_event(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
/*
* Testing cpu_hw_events.enabled should be skipped in this case unlike
* in x86_pmu_enable_event().
*
* Since cpu_hw_events.enabled is set only after returning from
* x86_pmu_start(), the PMCs must be programmed and kept ready.
* Counting starts only after x86_pmu_enable_all() is called.
*/
__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
}
static void amd_pmu_v2_enable_all(int added)
{
amd_pmu_set_global_ctl(amd_pmu_global_cntr_mask);
}
static void amd_pmu_disable_event(struct perf_event *event)
{
x86_pmu_disable_event(event);
@ -651,6 +821,32 @@ static void amd_pmu_disable_event(struct perf_event *event)
amd_pmu_wait_on_overflow(event->hw.idx);
}
static void amd_pmu_disable_all(void)
{
amd_brs_disable_all();
x86_pmu_disable_all();
amd_pmu_check_overflow();
}
static void amd_pmu_v2_disable_all(void)
{
/* Disable all PMCs */
amd_pmu_set_global_ctl(0);
amd_pmu_check_overflow();
}
static void amd_pmu_add_event(struct perf_event *event)
{
if (needs_branch_stack(event))
amd_pmu_brs_add(event);
}
static void amd_pmu_del_event(struct perf_event *event)
{
if (needs_branch_stack(event))
amd_pmu_brs_del(event);
}
/*
* Because of NMI latency, if multiple PMC counters are active or other sources
* of NMIs are received, the perf NMI handler can handle one or more overflowed
@ -669,13 +865,8 @@ static void amd_pmu_disable_event(struct perf_event *event)
* handled a counter. When an un-handled NMI is received, it will be claimed
* only if arriving within that window.
*/
static int amd_pmu_handle_irq(struct pt_regs *regs)
static inline int amd_pmu_adjust_nmi_window(int handled)
{
int handled;
/* Process any counter overflows */
handled = x86_pmu_handle_irq(regs);
/*
* If a counter was handled, record a timestamp such that un-handled
* NMIs will be claimed if arriving within that window.
@ -692,6 +883,113 @@ static int amd_pmu_handle_irq(struct pt_regs *regs)
return NMI_HANDLED;
}
static int amd_pmu_handle_irq(struct pt_regs *regs)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
int handled;
int pmu_enabled;
/*
* Save the PMU state.
* It needs to be restored when leaving the handler.
*/
pmu_enabled = cpuc->enabled;
cpuc->enabled = 0;
/* stop everything (includes BRS) */
amd_pmu_disable_all();
/* Drain BRS is in use (could be inactive) */
if (cpuc->lbr_users)
amd_brs_drain();
/* Process any counter overflows */
handled = x86_pmu_handle_irq(regs);
cpuc->enabled = pmu_enabled;
if (pmu_enabled)
amd_pmu_enable_all(0);
return amd_pmu_adjust_nmi_window(handled);
}
static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct perf_sample_data data;
struct hw_perf_event *hwc;
struct perf_event *event;
int handled = 0, idx;
u64 status, mask;
bool pmu_enabled;
/*
* Save the PMU state as it needs to be restored when leaving the
* handler
*/
pmu_enabled = cpuc->enabled;
cpuc->enabled = 0;
/* Stop counting */
amd_pmu_v2_disable_all();
status = amd_pmu_get_global_status();
/* Check if any overflows are pending */
if (!status)
goto done;
for (idx = 0; idx < x86_pmu.num_counters; idx++) {
if (!test_bit(idx, cpuc->active_mask))
continue;
event = cpuc->events[idx];
hwc = &event->hw;
x86_perf_event_update(event);
mask = BIT_ULL(idx);
if (!(status & mask))
continue;
/* Event overflow */
handled++;
perf_sample_data_init(&data, 0, hwc->last_period);
if (!x86_perf_event_set_period(event))
continue;
if (perf_event_overflow(event, &data, regs))
x86_pmu_stop(event, 0);
status &= ~mask;
}
/*
* It should never be the case that some overflows are not handled as
* the corresponding PMCs are expected to be inactive according to the
* active_mask
*/
WARN_ON(status > 0);
/* Clear overflow bits */
amd_pmu_ack_global_status(~status);
/*
* Unmasking the LVTPC is not required as the Mask (M) bit of the LVT
* PMI entry is not set by the local APIC when a PMC overflow occurs
*/
inc_irq_stat(apic_perf_irqs);
done:
cpuc->enabled = pmu_enabled;
/* Resume counting only if PMU is active */
if (pmu_enabled)
amd_pmu_v2_enable_all(0);
return amd_pmu_adjust_nmi_window(handled);
}
static struct event_constraint *
amd_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
struct perf_event *event)
@ -897,6 +1195,51 @@ static void amd_put_event_constraints_f17h(struct cpu_hw_events *cpuc,
--cpuc->n_pair;
}
/*
* Because of the way BRS operates with an inactive and active phases, and
* the link to one counter, it is not possible to have two events using BRS
* scheduled at the same time. There would be an issue with enforcing the
* period of each one and given that the BRS saturates, it would not be possible
* to guarantee correlated content for all events. Therefore, in situations
* where multiple events want to use BRS, the kernel enforces mutual exclusion.
* Exclusion is enforced by chosing only one counter for events using BRS.
* The event scheduling logic will then automatically multiplex the
* events and ensure that at most one event is actively using BRS.
*
* The BRS counter could be any counter, but there is no constraint on Fam19h,
* therefore all counters are equal and thus we pick the first one: PMC0
*/
static struct event_constraint amd_fam19h_brs_cntr0_constraint =
EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK);
static struct event_constraint amd_fam19h_brs_pair_cntr0_constraint =
__EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK, 1, 0, PERF_X86_EVENT_PAIR);
static struct event_constraint *
amd_get_event_constraints_f19h(struct cpu_hw_events *cpuc, int idx,
struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
bool has_brs = has_amd_brs(hwc);
/*
* In case BRS is used with an event requiring a counter pair,
* the kernel allows it but only on counter 0 & 1 to enforce
* multiplexing requiring to protect BRS in case of multiple
* BRS users
*/
if (amd_is_pair_event_code(hwc)) {
return has_brs ? &amd_fam19h_brs_pair_cntr0_constraint
: &pair_constraint;
}
if (has_brs)
return &amd_fam19h_brs_cntr0_constraint;
return &unconstrained;
}
static ssize_t amd_event_sysfs_show(char *page, u64 config)
{
u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT) |
@ -905,12 +1248,31 @@ static ssize_t amd_event_sysfs_show(char *page, u64 config)
return x86_event_sysfs_show(page, config, event);
}
static void amd_pmu_sched_task(struct perf_event_context *ctx,
bool sched_in)
{
if (sched_in && x86_pmu.lbr_nr)
amd_pmu_brs_sched_task(ctx, sched_in);
}
static u64 amd_pmu_limit_period(struct perf_event *event, u64 left)
{
/*
* Decrease period by the depth of the BRS feature to get the last N
* taken branches and approximate the desired period
*/
if (has_branch_stack(event) && left > x86_pmu.lbr_nr)
left -= x86_pmu.lbr_nr;
return left;
}
static __initconst const struct x86_pmu amd_pmu = {
.name = "AMD",
.handle_irq = amd_pmu_handle_irq,
.disable_all = amd_pmu_disable_all,
.enable_all = x86_pmu_enable_all,
.enable = x86_pmu_enable_event,
.enable_all = amd_pmu_enable_all,
.enable = amd_pmu_enable_event,
.disable = amd_pmu_disable_event,
.hw_config = amd_pmu_hw_config,
.schedule_events = x86_schedule_events,
@ -920,6 +1282,8 @@ static __initconst const struct x86_pmu amd_pmu = {
.event_map = amd_pmu_event_map,
.max_events = ARRAY_SIZE(amd_perfmon_event_map),
.num_counters = AMD64_NUM_COUNTERS,
.add = amd_pmu_add_event,
.del = amd_pmu_del_event,
.cntval_bits = 48,
.cntval_mask = (1ULL << 48) - 1,
.apic = 1,
@ -938,8 +1302,55 @@ static __initconst const struct x86_pmu amd_pmu = {
.amd_nb_constraints = 1,
};
static ssize_t branches_show(struct device *cdev,
struct device_attribute *attr,
char *buf)
{
return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu.lbr_nr);
}
static DEVICE_ATTR_RO(branches);
static struct attribute *amd_pmu_brs_attrs[] = {
&dev_attr_branches.attr,
NULL,
};
static umode_t
amd_brs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
{
return x86_pmu.lbr_nr ? attr->mode : 0;
}
static struct attribute_group group_caps_amd_brs = {
.name = "caps",
.attrs = amd_pmu_brs_attrs,
.is_visible = amd_brs_is_visible,
};
EVENT_ATTR_STR(branch-brs, amd_branch_brs,
"event=" __stringify(AMD_FAM19H_BRS_EVENT)"\n");
static struct attribute *amd_brs_events_attrs[] = {
EVENT_PTR(amd_branch_brs),
NULL,
};
static struct attribute_group group_events_amd_brs = {
.name = "events",
.attrs = amd_brs_events_attrs,
.is_visible = amd_brs_is_visible,
};
static const struct attribute_group *amd_attr_update[] = {
&group_caps_amd_brs,
&group_events_amd_brs,
NULL,
};
static int __init amd_core_pmu_init(void)
{
union cpuid_0x80000022_ebx ebx;
u64 even_ctr_mask = 0ULL;
int i;
@ -957,6 +1368,27 @@ static int __init amd_core_pmu_init(void)
x86_pmu.eventsel = MSR_F15H_PERF_CTL;
x86_pmu.perfctr = MSR_F15H_PERF_CTR;
x86_pmu.num_counters = AMD64_NUM_COUNTERS_CORE;
/* Check for Performance Monitoring v2 support */
if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) {
ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
/* Update PMU version for later usage */
x86_pmu.version = 2;
/* Find the number of available Core PMCs */
x86_pmu.num_counters = ebx.split.num_core_pmc;
amd_pmu_global_cntr_mask = (1ULL << x86_pmu.num_counters) - 1;
/* Update PMC handling functions */
x86_pmu.enable_all = amd_pmu_v2_enable_all;
x86_pmu.disable_all = amd_pmu_v2_disable_all;
x86_pmu.enable = amd_pmu_v2_enable_event;
x86_pmu.handle_irq = amd_pmu_v2_handle_irq;
static_call_update(amd_pmu_test_overflow, amd_pmu_test_overflow_status);
}
/*
* AMD Core perfctr has separate MSRs for the NB events, see
* the amd/uncore.c driver.
@ -989,6 +1421,23 @@ static int __init amd_core_pmu_init(void)
x86_pmu.flags |= PMU_FL_PAIR;
}
/*
* BRS requires special event constraints and flushing on ctxsw.
*/
if (boot_cpu_data.x86 >= 0x19 && !amd_brs_init()) {
x86_pmu.get_event_constraints = amd_get_event_constraints_f19h;
x86_pmu.sched_task = amd_pmu_sched_task;
x86_pmu.limit_period = amd_pmu_limit_period;
/*
* put_event_constraints callback same as Fam17h, set above
*/
/* branch sampling must be stopped when entering low power */
amd_brs_lopwr_init();
}
x86_pmu.attr_update = amd_attr_update;
pr_cont("core perfctr, ");
return 0;
}
@ -1023,6 +1472,24 @@ __init int amd_pmu_init(void)
return 0;
}
static inline void amd_pmu_reload_virt(void)
{
if (x86_pmu.version >= 2) {
/*
* Clear global enable bits, reprogram the PERF_CTL
* registers with updated perf_ctr_virt_mask and then
* set global enable bits once again
*/
amd_pmu_v2_disable_all();
amd_pmu_enable_all(0);
amd_pmu_v2_enable_all(0);
return;
}
amd_pmu_disable_all();
amd_pmu_enable_all(0);
}
void amd_pmu_enable_virt(void)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
@ -1030,8 +1497,7 @@ void amd_pmu_enable_virt(void)
cpuc->perf_ctr_virt_mask = 0;
/* Reload all events */
amd_pmu_disable_all();
x86_pmu_enable_all(0);
amd_pmu_reload_virt();
}
EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
@ -1048,7 +1514,6 @@ void amd_pmu_disable_virt(void)
cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
/* Reload all events */
amd_pmu_disable_all();
x86_pmu_enable_all(0);
amd_pmu_reload_virt();
}
EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);

209
arch/x86/events/amd/ibs.c
View File

@ -94,10 +94,6 @@ struct perf_ibs {
unsigned int fetch_ignore_if_zero_rip : 1;
struct cpu_perf_ibs __percpu *pcpu;
struct attribute **format_attrs;
struct attribute_group format_group;
const struct attribute_group *attr_groups[2];
u64 (*get_count)(u64 config);
};
@ -304,6 +300,16 @@ static int perf_ibs_init(struct perf_event *event)
hwc->config_base = perf_ibs->msr;
hwc->config = config;
/*
* rip recorded by IbsOpRip will not be consistent with rsp and rbp
* recorded as part of interrupt regs. Thus we need to use rip from
* interrupt regs while unwinding call stack. Setting _EARLY flag
* makes sure we unwind call-stack before perf sample rip is set to
* IbsOpRip.
*/
if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY;
return 0;
}
@ -518,16 +524,118 @@ static void perf_ibs_del(struct perf_event *event, int flags)
static void perf_ibs_read(struct perf_event *event) { }
/*
* We need to initialize with empty group if all attributes in the
* group are dynamic.
*/
static struct attribute *attrs_empty[] = {
NULL,
};
static struct attribute_group empty_format_group = {
.name = "format",
.attrs = attrs_empty,
};
static struct attribute_group empty_caps_group = {
.name = "caps",
.attrs = attrs_empty,
};
static const struct attribute_group *empty_attr_groups[] = {
&empty_format_group,
&empty_caps_group,
NULL,
};
PMU_FORMAT_ATTR(rand_en, "config:57");
PMU_FORMAT_ATTR(cnt_ctl, "config:19");
PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59");
PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16");
PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1");
static struct attribute *ibs_fetch_format_attrs[] = {
static umode_t
zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i)
{
return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0;
}
static struct attribute *rand_en_attrs[] = {
&format_attr_rand_en.attr,
NULL,
};
static struct attribute *ibs_op_format_attrs[] = {
NULL, /* &format_attr_cnt_ctl.attr if IBS_CAPS_OPCNT */
static struct attribute *fetch_l3missonly_attrs[] = {
&fetch_l3missonly.attr.attr,
NULL,
};
static struct attribute *zen4_ibs_extensions_attrs[] = {
&zen4_ibs_extensions.attr.attr,
NULL,
};
static struct attribute_group group_rand_en = {
.name = "format",
.attrs = rand_en_attrs,
};
static struct attribute_group group_fetch_l3missonly = {
.name = "format",
.attrs = fetch_l3missonly_attrs,
.is_visible = zen4_ibs_extensions_is_visible,
};
static struct attribute_group group_zen4_ibs_extensions = {
.name = "caps",
.attrs = zen4_ibs_extensions_attrs,
.is_visible = zen4_ibs_extensions_is_visible,
};
static const struct attribute_group *fetch_attr_groups[] = {
&group_rand_en,
&empty_caps_group,
NULL,
};
static const struct attribute_group *fetch_attr_update[] = {
&group_fetch_l3missonly,
&group_zen4_ibs_extensions,
NULL,
};
static umode_t
cnt_ctl_is_visible(struct kobject *kobj, struct attribute *attr, int i)
{
return ibs_caps & IBS_CAPS_OPCNT ? attr->mode : 0;
}
static struct attribute *cnt_ctl_attrs[] = {
&format_attr_cnt_ctl.attr,
NULL,
};
static struct attribute *op_l3missonly_attrs[] = {
&op_l3missonly.attr.attr,
NULL,
};
static struct attribute_group group_cnt_ctl = {
.name = "format",
.attrs = cnt_ctl_attrs,
.is_visible = cnt_ctl_is_visible,
};
static struct attribute_group group_op_l3missonly = {
.name = "format",
.attrs = op_l3missonly_attrs,
.is_visible = zen4_ibs_extensions_is_visible,
};
static const struct attribute_group *op_attr_update[] = {
&group_cnt_ctl,
&group_op_l3missonly,
&group_zen4_ibs_extensions,
NULL,
};
@ -551,7 +659,6 @@ static struct perf_ibs perf_ibs_fetch = {
.max_period = IBS_FETCH_MAX_CNT << 4,
.offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK },
.offset_max = MSR_AMD64_IBSFETCH_REG_COUNT,
.format_attrs = ibs_fetch_format_attrs,
.get_count = get_ibs_fetch_count,
};
@ -577,7 +684,6 @@ static struct perf_ibs perf_ibs_op = {
.max_period = IBS_OP_MAX_CNT << 4,
.offset_mask = { MSR_AMD64_IBSOP_REG_MASK },
.offset_max = MSR_AMD64_IBSOP_REG_COUNT,
.format_attrs = ibs_op_format_attrs,
.get_count = get_ibs_op_count,
};
@ -687,6 +793,14 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
data.raw = &raw;
}
/*
* rip recorded by IbsOpRip will not be consistent with rsp and rbp
* recorded as part of interrupt regs. Thus we need to use rip from
* interrupt regs while unwinding call stack.
*/
if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
data.callchain = perf_callchain(event, iregs);
throttle = perf_event_overflow(event, &data, &regs);
out:
if (throttle) {
@ -739,17 +853,6 @@ static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
perf_ibs->pcpu = pcpu;
/* register attributes */
if (perf_ibs->format_attrs[0]) {
memset(&perf_ibs->format_group, 0, sizeof(perf_ibs->format_group));
perf_ibs->format_group.name = "format";
perf_ibs->format_group.attrs = perf_ibs->format_attrs;
memset(&perf_ibs->attr_groups, 0, sizeof(perf_ibs->attr_groups));
perf_ibs->attr_groups[0] = &perf_ibs->format_group;
perf_ibs->pmu.attr_groups = perf_ibs->attr_groups;
}
ret = perf_pmu_register(&perf_ibs->pmu, name, -1);
if (ret) {
perf_ibs->pcpu = NULL;
@ -759,10 +862,8 @@ static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
return ret;
}
static __init void perf_event_ibs_init(void)
static __init int perf_ibs_fetch_init(void)
{
struct attribute **attr = ibs_op_format_attrs;
/*
* Some chips fail to reset the fetch count when it is written; instead
* they need a 0-1 transition of IbsFetchEn.
@ -773,12 +874,19 @@ static __init void perf_event_ibs_init(void)
if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model < 0x10)
perf_ibs_fetch.fetch_ignore_if_zero_rip = 1;
perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch");
if (ibs_caps & IBS_CAPS_ZEN4)
perf_ibs_fetch.config_mask |= IBS_FETCH_L3MISSONLY;
if (ibs_caps & IBS_CAPS_OPCNT) {
perf_ibs_fetch.pmu.attr_groups = fetch_attr_groups;
perf_ibs_fetch.pmu.attr_update = fetch_attr_update;
return perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch");
}
static __init int perf_ibs_op_init(void)
{
if (ibs_caps & IBS_CAPS_OPCNT)
perf_ibs_op.config_mask |= IBS_OP_CNT_CTL;
*attr++ = &format_attr_cnt_ctl.attr;
}
if (ibs_caps & IBS_CAPS_OPCNTEXT) {
perf_ibs_op.max_period |= IBS_OP_MAX_CNT_EXT_MASK;
@ -786,15 +894,52 @@ static __init void perf_event_ibs_init(void)
perf_ibs_op.cnt_mask |= IBS_OP_MAX_CNT_EXT_MASK;
}
perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
if (ibs_caps & IBS_CAPS_ZEN4)
perf_ibs_op.config_mask |= IBS_OP_L3MISSONLY;
perf_ibs_op.pmu.attr_groups = empty_attr_groups;
perf_ibs_op.pmu.attr_update = op_attr_update;
return perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
}
static __init int perf_event_ibs_init(void)
{
int ret;
ret = perf_ibs_fetch_init();
if (ret)
return ret;
ret = perf_ibs_op_init();
if (ret)
goto err_op;
ret = register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs");
if (ret)
goto err_nmi;
register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs");
pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps);
return 0;
err_nmi:
perf_pmu_unregister(&perf_ibs_op.pmu);
free_percpu(perf_ibs_op.pcpu);
perf_ibs_op.pcpu = NULL;
err_op:
perf_pmu_unregister(&perf_ibs_fetch.pmu);
free_percpu(perf_ibs_fetch.pcpu);
perf_ibs_fetch.pcpu = NULL;
return ret;
}
#else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */
static __init void perf_event_ibs_init(void) { }
static __init int perf_event_ibs_init(void)
{
return 0;
}
#endif
@ -1064,9 +1209,7 @@ static __init int amd_ibs_init(void)
x86_pmu_amd_ibs_starting_cpu,
x86_pmu_amd_ibs_dying_cpu);
perf_event_ibs_init();
return 0;
return perf_event_ibs_init();
}
/* Since we need the pci subsystem to init ibs we can't do this earlier: */

12
arch/x86/events/core.c
View File

@ -1338,6 +1338,10 @@ static void x86_pmu_enable(struct pmu *pmu)
if (hwc->state & PERF_HES_ARCH)
continue;
/*
* if cpuc->enabled = 0, then no wrmsr as
* per x86_pmu_enable_event()
*/
x86_pmu_start(event, PERF_EF_RELOAD);
}
cpuc->n_added = 0;
@ -1704,11 +1708,15 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
* event overflow
*/
handled++;
perf_sample_data_init(&data, 0, event->hw.last_period);
if (!x86_perf_event_set_period(event))
continue;
perf_sample_data_init(&data, 0, event->hw.last_period);
if (has_branch_stack(event))
data.br_stack = &cpuc->lbr_stack;
if (perf_event_overflow(event, &data, regs))
x86_pmu_stop(event, 0);
}
@ -1837,7 +1845,7 @@ ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, cha
/* string trumps id */
if (pmu_attr->event_str)
return sprintf(page, "%s", pmu_attr->event_str);
return sprintf(page, "%s\n", pmu_attr->event_str);
return x86_pmu.events_sysfs_show(page, config);
}

2
arch/x86/events/intel/core.c
View File

@ -6216,7 +6216,9 @@ __init int intel_pmu_init(void)
case INTEL_FAM6_ALDERLAKE:
case INTEL_FAM6_ALDERLAKE_L:
case INTEL_FAM6_ALDERLAKE_N:
case INTEL_FAM6_RAPTORLAKE:
case INTEL_FAM6_RAPTORLAKE_P:
/*
* Alder Lake has 2 types of CPU, core and atom.
*

2
arch/x86/events/intel/cstate.c
View File

@ -682,7 +682,9 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = {
X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &icl_cstates),
X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_cstates),
X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &adl_cstates),
X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &adl_cstates),
X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_cstates),
X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &adl_cstates),
{ },
};
MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);

36
arch/x86/events/intel/lbr.c
View File

@ -769,6 +769,7 @@ void intel_pmu_lbr_disable_all(void)
void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
{
unsigned long mask = x86_pmu.lbr_nr - 1;
struct perf_branch_entry *br = cpuc->lbr_entries;
u64 tos = intel_pmu_lbr_tos();
int i;
@ -784,15 +785,11 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
cpuc->lbr_entries[i].from = msr_lastbranch.from;
cpuc->lbr_entries[i].to = msr_lastbranch.to;
cpuc->lbr_entries[i].mispred = 0;
cpuc->lbr_entries[i].predicted = 0;
cpuc->lbr_entries[i].in_tx = 0;
cpuc->lbr_entries[i].abort = 0;
cpuc->lbr_entries[i].cycles = 0;
cpuc->lbr_entries[i].type = 0;
cpuc->lbr_entries[i].reserved = 0;
perf_clear_branch_entry_bitfields(br);
br->from = msr_lastbranch.from;
br->to = msr_lastbranch.to;
br++;
}
cpuc->lbr_stack.nr = i;
cpuc->lbr_stack.hw_idx = tos;
@ -807,6 +804,7 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
{
bool need_info = false, call_stack = false;
unsigned long mask = x86_pmu.lbr_nr - 1;
struct perf_branch_entry *br = cpuc->lbr_entries;
u64 tos = intel_pmu_lbr_tos();
int i;
int out = 0;
@ -878,15 +876,14 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
if (abort && x86_pmu.lbr_double_abort && out > 0)
out--;
cpuc->lbr_entries[out].from = from;
cpuc->lbr_entries[out].to = to;
cpuc->lbr_entries[out].mispred = mis;
cpuc->lbr_entries[out].predicted = pred;
cpuc->lbr_entries[out].in_tx = in_tx;
cpuc->lbr_entries[out].abort = abort;
cpuc->lbr_entries[out].cycles = cycles;
cpuc->lbr_entries[out].type = 0;
cpuc->lbr_entries[out].reserved = 0;
perf_clear_branch_entry_bitfields(br+out);
br[out].from = from;
br[out].to = to;
br[out].mispred = mis;
br[out].predicted = pred;
br[out].in_tx = in_tx;
br[out].abort = abort;
br[out].cycles = cycles;
out++;
}
cpuc->lbr_stack.nr = out;
@ -951,6 +948,8 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
to = rdlbr_to(i, lbr);
info = rdlbr_info(i, lbr);
perf_clear_branch_entry_bitfields(e);
e->from = from;
e->to = to;
e->mispred = get_lbr_mispred(info);
@ -959,7 +958,6 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
e->abort = !!(info & LBR_INFO_ABORT);
e->cycles = get_lbr_cycles(info);
e->type = get_lbr_br_type(info);
e->reserved = 0;
}
cpuc->lbr_stack.nr = i;

2
arch/x86/events/intel/uncore.c
View File

@ -1828,7 +1828,9 @@ static const struct x86_cpu_id intel_uncore_match[] __initconst = {
X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &rkl_uncore_init),
X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_uncore_init),
X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &adl_uncore_init),
X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &adl_uncore_init),
X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_uncore_init),
X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &adl_uncore_init),
X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &spr_uncore_init),
X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init),
{},

454
arch/x86/events/intel/uncore_snb.c
View File

@ -79,10 +79,43 @@
#define PCI_DEVICE_ID_INTEL_ADL_14_IMC 0x4650
#define PCI_DEVICE_ID_INTEL_ADL_15_IMC 0x4668
#define PCI_DEVICE_ID_INTEL_ADL_16_IMC 0x4670
#define PCI_DEVICE_ID_INTEL_ADL_17_IMC 0x4614
#define PCI_DEVICE_ID_INTEL_ADL_18_IMC 0x4617
#define PCI_DEVICE_ID_INTEL_ADL_19_IMC 0x4618
#define PCI_DEVICE_ID_INTEL_ADL_20_IMC 0x461B
#define PCI_DEVICE_ID_INTEL_ADL_21_IMC 0x461C
#define PCI_DEVICE_ID_INTEL_RPL_1_IMC 0xA700
#define PCI_DEVICE_ID_INTEL_RPL_2_IMC 0xA702
#define PCI_DEVICE_ID_INTEL_RPL_3_IMC 0xA706
#define PCI_DEVICE_ID_INTEL_RPL_4_IMC 0xA709
#define PCI_DEVICE_ID_INTEL_RPL_5_IMC 0xA701
#define PCI_DEVICE_ID_INTEL_RPL_6_IMC 0xA703
#define PCI_DEVICE_ID_INTEL_RPL_7_IMC 0xA704
#define PCI_DEVICE_ID_INTEL_RPL_8_IMC 0xA705
#define PCI_DEVICE_ID_INTEL_RPL_9_IMC 0xA706
#define PCI_DEVICE_ID_INTEL_RPL_10_IMC 0xA707
#define PCI_DEVICE_ID_INTEL_RPL_11_IMC 0xA708
#define PCI_DEVICE_ID_INTEL_RPL_12_IMC 0xA709
#define PCI_DEVICE_ID_INTEL_RPL_13_IMC 0xA70a
#define PCI_DEVICE_ID_INTEL_RPL_14_IMC 0xA70b
#define PCI_DEVICE_ID_INTEL_RPL_15_IMC 0xA715
#define PCI_DEVICE_ID_INTEL_RPL_16_IMC 0xA716
#define PCI_DEVICE_ID_INTEL_RPL_17_IMC 0xA717
#define PCI_DEVICE_ID_INTEL_RPL_18_IMC 0xA718
#define PCI_DEVICE_ID_INTEL_RPL_19_IMC 0xA719
#define PCI_DEVICE_ID_INTEL_RPL_20_IMC 0xA71A
#define PCI_DEVICE_ID_INTEL_RPL_21_IMC 0xA71B
#define PCI_DEVICE_ID_INTEL_RPL_22_IMC 0xA71C
#define PCI_DEVICE_ID_INTEL_RPL_23_IMC 0xA728
#define PCI_DEVICE_ID_INTEL_RPL_24_IMC 0xA729
#define PCI_DEVICE_ID_INTEL_RPL_25_IMC 0xA72A
#define IMC_UNCORE_DEV(a) \
{ \
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_##a##_IMC), \
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0), \
}
/* SNB event control */
#define SNB_UNC_CTL_EV_SEL_MASK 0x000000ff
@ -849,242 +882,80 @@ static struct intel_uncore_type *snb_pci_uncores[] = {
};
static const struct pci_device_id snb_uncore_pci_ids[] = {
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SNB_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
IMC_UNCORE_DEV(SNB),
{ /* end: all zeroes */ },
};
static const struct pci_device_id ivb_uncore_pci_ids[] = {
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IVB_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IVB_E3_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
IMC_UNCORE_DEV(IVB),
IMC_UNCORE_DEV(IVB_E3),
{ /* end: all zeroes */ },
};
static const struct pci_device_id hsw_uncore_pci_ids[] = {
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_HSW_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_HSW_U_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
IMC_UNCORE_DEV(HSW),
IMC_UNCORE_DEV(HSW_U),
{ /* end: all zeroes */ },
};
static const struct pci_device_id bdw_uncore_pci_ids[] = {
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BDW_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
IMC_UNCORE_DEV(BDW),
{ /* end: all zeroes */ },
};
static const struct pci_device_id skl_uncore_pci_ids[] = {
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_Y_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_U_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_HD_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_HQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_SD_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_SQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_E3_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_Y_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_U_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_UQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_SD_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_SQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_HQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_WQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_2U_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4U_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4H_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6H_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_2S_D_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_D_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_D_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_D_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_W_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_W_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_W_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_S_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_S_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_S_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_AML_YD_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_AML_YQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_UQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_4_UQ_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_UD_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_H1_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_H2_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_H3_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_U1_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_U2_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_U3_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S1_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S2_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S3_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S4_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S5_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
IMC_UNCORE_DEV(SKL_Y),
IMC_UNCORE_DEV(SKL_U),
IMC_UNCORE_DEV(SKL_HD),
IMC_UNCORE_DEV(SKL_HQ),
IMC_UNCORE_DEV(SKL_SD),
IMC_UNCORE_DEV(SKL_SQ),
IMC_UNCORE_DEV(SKL_E3),
IMC_UNCORE_DEV(KBL_Y),
IMC_UNCORE_DEV(KBL_U),
IMC_UNCORE_DEV(KBL_UQ),
IMC_UNCORE_DEV(KBL_SD),
IMC_UNCORE_DEV(KBL_SQ),
IMC_UNCORE_DEV(KBL_HQ),
IMC_UNCORE_DEV(KBL_WQ),
IMC_UNCORE_DEV(CFL_2U),
IMC_UNCORE_DEV(CFL_4U),
IMC_UNCORE_DEV(CFL_4H),
IMC_UNCORE_DEV(CFL_6H),
IMC_UNCORE_DEV(CFL_2S_D),
IMC_UNCORE_DEV(CFL_4S_D),
IMC_UNCORE_DEV(CFL_6S_D),
IMC_UNCORE_DEV(CFL_8S_D),
IMC_UNCORE_DEV(CFL_4S_W),
IMC_UNCORE_DEV(CFL_6S_W),
IMC_UNCORE_DEV(CFL_8S_W),
IMC_UNCORE_DEV(CFL_4S_S),
IMC_UNCORE_DEV(CFL_6S_S),
IMC_UNCORE_DEV(CFL_8S_S),
IMC_UNCORE_DEV(AML_YD),
IMC_UNCORE_DEV(AML_YQ),
IMC_UNCORE_DEV(WHL_UQ),
IMC_UNCORE_DEV(WHL_4_UQ),
IMC_UNCORE_DEV(WHL_UD),
IMC_UNCORE_DEV(CML_H1),
IMC_UNCORE_DEV(CML_H2),
IMC_UNCORE_DEV(CML_H3),
IMC_UNCORE_DEV(CML_U1),
IMC_UNCORE_DEV(CML_U2),
IMC_UNCORE_DEV(CML_U3),
IMC_UNCORE_DEV(CML_S1),
IMC_UNCORE_DEV(CML_S2),
IMC_UNCORE_DEV(CML_S3),
IMC_UNCORE_DEV(CML_S4),
IMC_UNCORE_DEV(CML_S5),
{ /* end: all zeroes */ },
};
static const struct pci_device_id icl_uncore_pci_ids[] = {
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICL_U_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICL_U2_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_RKL_1_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_RKL_2_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
IMC_UNCORE_DEV(ICL_U),
IMC_UNCORE_DEV(ICL_U2),
IMC_UNCORE_DEV(RKL_1),
IMC_UNCORE_DEV(RKL_2),
{ /* end: all zeroes */ },
};
@ -1326,106 +1197,57 @@ void nhm_uncore_cpu_init(void)
/* Tiger Lake MMIO uncore support */
static const struct pci_device_id tgl_uncore_pci_ids[] = {
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U1_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U2_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U3_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U4_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_H_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_1_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_2_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_3_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_4_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_5_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_6_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_7_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_8_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_9_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_10_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_11_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_12_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_13_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_14_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_15_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ADL_16_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_RPL_1_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_RPL_2_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_RPL_3_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
{ /* IMC */
PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_RPL_4_IMC),
.driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
},
IMC_UNCORE_DEV(TGL_U1),
IMC_UNCORE_DEV(TGL_U2),
IMC_UNCORE_DEV(TGL_U3),
IMC_UNCORE_DEV(TGL_U4),
IMC_UNCORE_DEV(TGL_H),
IMC_UNCORE_DEV(ADL_1),
IMC_UNCORE_DEV(ADL_2),
IMC_UNCORE_DEV(ADL_3),
IMC_UNCORE_DEV(ADL_4),
IMC_UNCORE_DEV(ADL_5),
IMC_UNCORE_DEV(ADL_6),
IMC_UNCORE_DEV(ADL_7),
IMC_UNCORE_DEV(ADL_8),
IMC_UNCORE_DEV(ADL_9),
IMC_UNCORE_DEV(ADL_10),
IMC_UNCORE_DEV(ADL_11),
IMC_UNCORE_DEV(ADL_12),
IMC_UNCORE_DEV(ADL_13),
IMC_UNCORE_DEV(ADL_14),
IMC_UNCORE_DEV(ADL_15),
IMC_UNCORE_DEV(ADL_16),
IMC_UNCORE_DEV(ADL_17),
IMC_UNCORE_DEV(ADL_18),
IMC_UNCORE_DEV(ADL_19),
IMC_UNCORE_DEV(ADL_20),
IMC_UNCORE_DEV(ADL_21),
IMC_UNCORE_DEV(RPL_1),
IMC_UNCORE_DEV(RPL_2),
IMC_UNCORE_DEV(RPL_3),
IMC_UNCORE_DEV(RPL_4),
IMC_UNCORE_DEV(RPL_5),
IMC_UNCORE_DEV(RPL_6),
IMC_UNCORE_DEV(RPL_7),
IMC_UNCORE_DEV(RPL_8),
IMC_UNCORE_DEV(RPL_9),
IMC_UNCORE_DEV(RPL_10),
IMC_UNCORE_DEV(RPL_11),
IMC_UNCORE_DEV(RPL_12),
IMC_UNCORE_DEV(RPL_13),
IMC_UNCORE_DEV(RPL_14),
IMC_UNCORE_DEV(RPL_15),
IMC_UNCORE_DEV(RPL_16),
IMC_UNCORE_DEV(RPL_17),
IMC_UNCORE_DEV(RPL_18),
IMC_UNCORE_DEV(RPL_19),
IMC_UNCORE_DEV(RPL_20),
IMC_UNCORE_DEV(RPL_21),
IMC_UNCORE_DEV(RPL_22),
IMC_UNCORE_DEV(RPL_23),
IMC_UNCORE_DEV(RPL_24),
IMC_UNCORE_DEV(RPL_25),
{ /* end: all zeroes */ }
};

2
arch/x86/events/msr.c
View File

@ -103,7 +103,9 @@ static bool test_intel(int idx, void *data)
case INTEL_FAM6_ROCKETLAKE:
case INTEL_FAM6_ALDERLAKE:
case INTEL_FAM6_ALDERLAKE_L:
case INTEL_FAM6_ALDERLAKE_N:
case INTEL_FAM6_RAPTORLAKE:
case INTEL_FAM6_RAPTORLAKE_P:
if (idx == PERF_MSR_SMI || idx == PERF_MSR_PPERF)
return true;
break;

123
arch/x86/events/perf_event.h
View File

@ -67,22 +67,23 @@ static inline bool constraint_match(struct event_constraint *c, u64 ecode)
/*
* struct hw_perf_event.flags flags
*/
#define PERF_X86_EVENT_PEBS_LDLAT 0x0001 /* ld+ldlat data address sampling */
#define PERF_X86_EVENT_PEBS_ST 0x0002 /* st data address sampling */
#define PERF_X86_EVENT_PEBS_ST_HSW 0x0004 /* haswell style datala, store */
#define PERF_X86_EVENT_PEBS_LD_HSW 0x0008 /* haswell style datala, load */
#define PERF_X86_EVENT_PEBS_NA_HSW 0x0010 /* haswell style datala, unknown */
#define PERF_X86_EVENT_EXCL 0x0020 /* HT exclusivity on counter */
#define PERF_X86_EVENT_DYNAMIC 0x0040 /* dynamic alloc'd constraint */
#define PERF_X86_EVENT_PEBS_LDLAT 0x00001 /* ld+ldlat data address sampling */
#define PERF_X86_EVENT_PEBS_ST 0x00002 /* st data address sampling */
#define PERF_X86_EVENT_PEBS_ST_HSW 0x00004 /* haswell style datala, store */
#define PERF_X86_EVENT_PEBS_LD_HSW 0x00008 /* haswell style datala, load */
#define PERF_X86_EVENT_PEBS_NA_HSW 0x00010 /* haswell style datala, unknown */
#define PERF_X86_EVENT_EXCL 0x00020 /* HT exclusivity on counter */
#define PERF_X86_EVENT_DYNAMIC 0x00040 /* dynamic alloc'd constraint */
#define PERF_X86_EVENT_EXCL_ACCT 0x0100 /* accounted EXCL event */
#define PERF_X86_EVENT_AUTO_RELOAD 0x0200 /* use PEBS auto-reload */
#define PERF_X86_EVENT_LARGE_PEBS 0x0400 /* use large PEBS */
#define PERF_X86_EVENT_PEBS_VIA_PT 0x0800 /* use PT buffer for PEBS */
#define PERF_X86_EVENT_PAIR 0x1000 /* Large Increment per Cycle */
#define PERF_X86_EVENT_LBR_SELECT 0x2000 /* Save/Restore MSR_LBR_SELECT */
#define PERF_X86_EVENT_TOPDOWN 0x4000 /* Count Topdown slots/metrics events */
#define PERF_X86_EVENT_PEBS_STLAT 0x8000 /* st+stlat data address sampling */
#define PERF_X86_EVENT_EXCL_ACCT 0x00100 /* accounted EXCL event */
#define PERF_X86_EVENT_AUTO_RELOAD 0x00200 /* use PEBS auto-reload */
#define PERF_X86_EVENT_LARGE_PEBS 0x00400 /* use large PEBS */
#define PERF_X86_EVENT_PEBS_VIA_PT 0x00800 /* use PT buffer for PEBS */
#define PERF_X86_EVENT_PAIR 0x01000 /* Large Increment per Cycle */
#define PERF_X86_EVENT_LBR_SELECT 0x02000 /* Save/Restore MSR_LBR_SELECT */
#define PERF_X86_EVENT_TOPDOWN 0x04000 /* Count Topdown slots/metrics events */
#define PERF_X86_EVENT_PEBS_STLAT 0x08000 /* st+stlat data address sampling */
#define PERF_X86_EVENT_AMD_BRS 0x10000 /* AMD Branch Sampling */
static inline bool is_topdown_count(struct perf_event *event)
{
@ -325,6 +326,8 @@ struct cpu_hw_events {
* AMD specific bits
*/
struct amd_nb *amd_nb;
int brs_active; /* BRS is enabled */
/* Inverted mask of bits to clear in the perf_ctr ctrl registers */
u64 perf_ctr_virt_mask;
int n_pair; /* Large increment events */
@ -1105,6 +1108,11 @@ int x86_pmu_hw_config(struct perf_event *event);
void x86_pmu_disable_all(void);
static inline bool has_amd_brs(struct hw_perf_event *hwc)
{
return hwc->flags & PERF_X86_EVENT_AMD_BRS;
}
static inline bool is_counter_pair(struct hw_perf_event *hwc)
{
return hwc->flags & PERF_X86_EVENT_PAIR;
@ -1211,6 +1219,75 @@ static inline bool fixed_counter_disabled(int i, struct pmu *pmu)
int amd_pmu_init(void);
#ifdef CONFIG_PERF_EVENTS_AMD_BRS
int amd_brs_init(void);
void amd_brs_disable(void);
void amd_brs_enable(void);
void amd_brs_enable_all(void);
void amd_brs_disable_all(void);
void amd_brs_drain(void);
void amd_brs_lopwr_init(void);
void amd_brs_disable_all(void);
int amd_brs_setup_filter(struct perf_event *event);
void amd_brs_reset(void);
static inline void amd_pmu_brs_add(struct perf_event *event)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
perf_sched_cb_inc(event->ctx->pmu);
cpuc->lbr_users++;
/*
* No need to reset BRS because it is reset
* on brs_enable() and it is saturating
*/
}
static inline void amd_pmu_brs_del(struct perf_event *event)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
cpuc->lbr_users--;
WARN_ON_ONCE(cpuc->lbr_users < 0);
perf_sched_cb_dec(event->ctx->pmu);
}
void amd_pmu_brs_sched_task(struct perf_event_context *ctx, bool sched_in);
#else
static inline int amd_brs_init(void)
{
return 0;
}
static inline void amd_brs_disable(void) {}
static inline void amd_brs_enable(void) {}
static inline void amd_brs_drain(void) {}
static inline void amd_brs_lopwr_init(void) {}
static inline void amd_brs_disable_all(void) {}
static inline int amd_brs_setup_filter(struct perf_event *event)
{
return 0;
}
static inline void amd_brs_reset(void) {}
static inline void amd_pmu_brs_add(struct perf_event *event)
{
}
static inline void amd_pmu_brs_del(struct perf_event *event)
{
}
static inline void amd_pmu_brs_sched_task(struct perf_event_context *ctx, bool sched_in)
{
}
static inline void amd_brs_enable_all(void)
{
}
#endif
#else /* CONFIG_CPU_SUP_AMD */
static inline int amd_pmu_init(void)
@ -1218,6 +1295,22 @@ static inline int amd_pmu_init(void)
return 0;
}
static inline int amd_brs_init(void)
{
return -EOPNOTSUPP;
}
static inline void amd_brs_drain(void)
{
}
static inline void amd_brs_enable_all(void)
{
}
static inline void amd_brs_disable_all(void)
{
}
#endif /* CONFIG_CPU_SUP_AMD */
static inline int is_pebs_pt(struct perf_event *event)

2
arch/x86/include/asm/amd-ibs.h
View File

@ -49,7 +49,7 @@ union ibs_op_ctl {
};
};
/* MSR 0xc0011035: IBS Op Data 2 */
/* MSR 0xc0011035: IBS Op Data 1 */
union ibs_op_data {
__u64 val;
struct {

21
arch/x86/include/asm/cmpxchg_32.h
View File

@ -42,6 +42,9 @@ static inline void set_64bit(volatile u64 *ptr, u64 value)
#define arch_cmpxchg64_local(ptr, o, n) \
((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
(unsigned long long)(n)))
#define arch_try_cmpxchg64(ptr, po, n) \
__try_cmpxchg64((ptr), (unsigned long long *)(po), \
(unsigned long long)(n))
#endif
static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
@ -70,6 +73,24 @@ static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
return prev;
}
static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new)
{
bool success;
u64 old = *pold;
asm volatile(LOCK_PREFIX "cmpxchg8b %[ptr]"
CC_SET(z)
: CC_OUT(z) (success),
[ptr] "+m" (*ptr),
"+A" (old)
: "b" ((u32)new),
"c" ((u32)(new >> 32))
: "memory");
if (unlikely(!success))
*pold = old;
return success;
}
#ifndef CONFIG_X86_CMPXCHG64
/*
* Building a kernel capable running on 80386 and 80486. It may be necessary

6
arch/x86/include/asm/cmpxchg_64.h
View File

@ -19,6 +19,12 @@ static inline void set_64bit(volatile u64 *ptr, u64 val)
arch_cmpxchg_local((ptr), (o), (n)); \
})
#define arch_try_cmpxchg64(ptr, po, n) \
({ \
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
arch_try_cmpxchg((ptr), (po), (n)); \
})
#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX16)
#endif /* _ASM_X86_CMPXCHG_64_H */

2
arch/x86/include/asm/cpu_entry_area.h
View File

@ -143,7 +143,7 @@ extern void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags);
extern struct cpu_entry_area *get_cpu_entry_area(int cpu);
static inline struct entry_stack *cpu_entry_stack(int cpu)
static __always_inline struct entry_stack *cpu_entry_stack(int cpu)
{
return &get_cpu_entry_area(cpu)->entry_stack_page.stack;
}

3
arch/x86/include/asm/cpufeatures.h
View File

@ -211,7 +211,7 @@
#define X86_FEATURE_SSBD ( 7*32+17) /* Speculative Store Bypass Disable */
#define X86_FEATURE_MBA ( 7*32+18) /* Memory Bandwidth Allocation */
#define X86_FEATURE_RSB_CTXSW ( 7*32+19) /* "" Fill RSB on context switches */
/* FREE! ( 7*32+20) */
#define X86_FEATURE_PERFMON_V2 ( 7*32+20) /* AMD Performance Monitoring Version 2 */
#define X86_FEATURE_USE_IBPB ( 7*32+21) /* "" Indirect Branch Prediction Barrier enabled */
#define X86_FEATURE_USE_IBRS_FW ( 7*32+22) /* "" Use IBRS during runtime firmware calls */
#define X86_FEATURE_SPEC_STORE_BYPASS_DISABLE ( 7*32+23) /* "" Disable Speculative Store Bypass. */
@ -316,6 +316,7 @@
#define X86_FEATURE_VIRT_SSBD (13*32+25) /* Virtualized Speculative Store Bypass Disable */
#define X86_FEATURE_AMD_SSB_NO (13*32+26) /* "" Speculative Store Bypass is fixed in hardware. */
#define X86_FEATURE_CPPC (13*32+27) /* Collaborative Processor Performance Control */
#define X86_FEATURE_BRS (13*32+31) /* Branch Sampling available */
/* Thermal and Power Management Leaf, CPUID level 0x00000006 (EAX), word 14 */
#define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */

6
arch/x86/include/asm/jump_label.h
View File

@ -20,7 +20,7 @@
_ASM_PTR "%c0 + %c1 - .\n\t" \
".popsection \n\t"
#ifdef CONFIG_STACK_VALIDATION
#ifdef CONFIG_HAVE_JUMP_LABEL_HACK
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{
@ -34,7 +34,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
return true;
}
#else
#else /* !CONFIG_HAVE_JUMP_LABEL_HACK */
static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
{
@ -48,7 +48,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key, co
return true;
}
#endif /* STACK_VALIDATION */
#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */
static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
{

9
arch/x86/include/asm/msr-index.h
View File

@ -533,6 +533,11 @@
#define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16)
#define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24)
/* AMD Performance Counter Global Status and Control MSRs */
#define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300
#define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301
#define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302
/* Fam 17h MSRs */
#define MSR_F17H_IRPERF 0xc00000e9
@ -697,6 +702,10 @@
#define MSR_IA32_PERF_CTL 0x00000199
#define INTEL_PERF_CTL_MASK 0xffff
/* AMD Branch Sampling configuration */
#define MSR_AMD_DBG_EXTN_CFG 0xc000010f
#define MSR_AMD_SAMP_BR_FROM 0xc0010300
#define MSR_IA32_MPERF 0x000000e7
#define MSR_IA32_APERF 0x000000e8

2
arch/x86/include/asm/page_64.h
View File

@ -16,7 +16,7 @@ extern unsigned long page_offset_base;
extern unsigned long vmalloc_base;
extern unsigned long vmemmap_base;
static inline unsigned long __phys_addr_nodebug(unsigned long x)
static __always_inline unsigned long __phys_addr_nodebug(unsigned long x)
{
unsigned long y = x - __START_KERNEL_map;

43
arch/x86/include/asm/perf_event.h
View File

@ -2,6 +2,8 @@
#ifndef _ASM_X86_PERF_EVENT_H
#define _ASM_X86_PERF_EVENT_H
#include <linux/static_call.h>
/*
* Performance event hw details:
*/
@ -184,6 +186,18 @@ union cpuid28_ecx {
unsigned int full;
};
/*
* AMD "Extended Performance Monitoring and Debug" CPUID
* detection/enumeration details:
*/
union cpuid_0x80000022_ebx {
struct {
/* Number of Core Performance Counters */
unsigned int num_core_pmc:4;
} split;
unsigned int full;
};
struct x86_pmu_capability {
int version;
int num_counters_gp;
@ -370,6 +384,11 @@ struct pebs_xmm {
u64 xmm[16*2]; /* two entries for each register */
};
/*
* AMD Extended Performance Monitoring and Debug cpuid feature detection
*/
#define EXT_PERFMON_DEBUG_FEATURES 0x80000022
/*
* IBS cpuid feature detection
*/
@ -391,6 +410,7 @@ struct pebs_xmm {
#define IBS_CAPS_OPBRNFUSE (1U<<8)
#define IBS_CAPS_FETCHCTLEXTD (1U<<9)
#define IBS_CAPS_OPDATA4 (1U<<10)
#define IBS_CAPS_ZEN4 (1U<<11)
#define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \
| IBS_CAPS_FETCHSAM \
@ -404,6 +424,7 @@ struct pebs_xmm {
#define IBSCTL_LVT_OFFSET_MASK 0x0F
/* IBS fetch bits/masks */
#define IBS_FETCH_L3MISSONLY (1ULL<<59)
#define IBS_FETCH_RAND_EN (1ULL<<57)
#define IBS_FETCH_VAL (1ULL<<49)
#define IBS_FETCH_ENABLE (1ULL<<48)
@ -420,6 +441,7 @@ struct pebs_xmm {
#define IBS_OP_CNT_CTL (1ULL<<19)
#define IBS_OP_VAL (1ULL<<18)
#define IBS_OP_ENABLE (1ULL<<17)
#define IBS_OP_L3MISSONLY (1ULL<<16)
#define IBS_OP_MAX_CNT 0x0000FFFFULL
#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */
#define IBS_OP_MAX_CNT_EXT_MASK (0x7FULL<<20) /* separate upper 7 bits */
@ -518,6 +540,27 @@ static inline void intel_pt_handle_vmx(int on)
#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD)
extern void amd_pmu_enable_virt(void);
extern void amd_pmu_disable_virt(void);
#if defined(CONFIG_PERF_EVENTS_AMD_BRS)
#define PERF_NEEDS_LOPWR_CB 1
/*
* architectural low power callback impacts
* drivers/acpi/processor_idle.c
* drivers/acpi/acpi_pad.c
*/
extern void perf_amd_brs_lopwr_cb(bool lopwr_in);
DECLARE_STATIC_CALL(perf_lopwr_cb, perf_amd_brs_lopwr_cb);
static inline void perf_lopwr_cb(bool lopwr_in)
{
static_call_mod(perf_lopwr_cb)(lopwr_in);
}
#endif /* PERF_NEEDS_LOPWR_CB */
#else
static inline void amd_pmu_enable_virt(void) { }
static inline void amd_pmu_disable_virt(void) { }

8
arch/x86/include/asm/svm.h
View File

@ -569,23 +569,23 @@ struct vmcb {
(offsetof(struct ghcb_save_area, field) / sizeof(u64))
#define DEFINE_GHCB_ACCESSORS(field) \
static inline bool ghcb_##field##_is_valid(const struct ghcb *ghcb) \
static __always_inline bool ghcb_##field##_is_valid(const struct ghcb *ghcb) \
{ \
return test_bit(GHCB_BITMAP_IDX(field), \
(unsigned long *)&ghcb->save.valid_bitmap); \
} \
\
static inline u64 ghcb_get_##field(struct ghcb *ghcb) \
static __always_inline u64 ghcb_get_##field(struct ghcb *ghcb) \
{ \
return ghcb->save.field; \
} \
\
static inline u64 ghcb_get_##field##_if_valid(struct ghcb *ghcb) \
static __always_inline u64 ghcb_get_##field##_if_valid(struct ghcb *ghcb) \
{ \
return ghcb_##field##_is_valid(ghcb) ? ghcb->save.field : 0; \
} \
\
static inline void ghcb_set_##field(struct ghcb *ghcb, u64 value) \
static __always_inline void ghcb_set_##field(struct ghcb *ghcb, u64 value) \
{ \
__set_bit(GHCB_BITMAP_IDX(field), \
(unsigned long *)&ghcb->save.valid_bitmap); \

6
arch/x86/kernel/alternative.c
View File

@ -338,7 +338,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
}
}
#if defined(CONFIG_RETPOLINE) && defined(CONFIG_STACK_VALIDATION)
#if defined(CONFIG_RETPOLINE) && defined(CONFIG_OBJTOOL)
/*
* CALL/JMP *%\reg
@ -507,11 +507,11 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)
}
}
#else /* !RETPOLINES || !CONFIG_STACK_VALIDATION */
#else /* !CONFIG_RETPOLINE || !CONFIG_OBJTOOL */
void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { }
#endif /* CONFIG_RETPOLINE && CONFIG_STACK_VALIDATION */
#endif /* CONFIG_RETPOLINE && CONFIG_OBJTOOL */
#ifdef CONFIG_X86_KERNEL_IBT

1
arch/x86/kernel/cpu/scattered.c
View File

@ -43,6 +43,7 @@ static const struct cpuid_bit cpuid_bits[] = {
{ X86_FEATURE_CPB, CPUID_EDX, 9, 0x80000007, 0 },
{ X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 },
{ X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 },
{ X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 },
{ 0, 0, 0, 0, 0 }
};

2
arch/x86/kernel/signal_compat.c
View File

@ -149,8 +149,10 @@ static inline void signal_compat_build_tests(void)
BUILD_BUG_ON(offsetof(siginfo_t, si_perf_data) != 0x18);
BUILD_BUG_ON(offsetof(siginfo_t, si_perf_type) != 0x20);
BUILD_BUG_ON(offsetof(siginfo_t, si_perf_flags) != 0x24);
BUILD_BUG_ON(offsetof(compat_siginfo_t, si_perf_data) != 0x10);
BUILD_BUG_ON(offsetof(compat_siginfo_t, si_perf_type) != 0x14);
BUILD_BUG_ON(offsetof(compat_siginfo_t, si_perf_flags) != 0x18);
CHECK_CSI_OFFSET(_sigpoll);
CHECK_CSI_SIZE (_sigpoll, 2*sizeof(int));

7
drivers/acpi/acpi_pad.c
View File

@ -17,6 +17,7 @@
#include <linux/tick.h>
#include <linux/slab.h>
#include <linux/acpi.h>
#include <linux/perf_event.h>
#include <asm/mwait.h>
#include <xen/xen.h>
@ -164,6 +165,9 @@ static int power_saving_thread(void *data)
tsc_marked_unstable = 1;
}
local_irq_disable();
perf_lopwr_cb(true);
tick_broadcast_enable();
tick_broadcast_enter();
stop_critical_timings();
@ -172,6 +176,9 @@ static int power_saving_thread(void *data)
start_critical_timings();
tick_broadcast_exit();
perf_lopwr_cb(false);
local_irq_enable();
if (time_before(expire_time, jiffies)) {

5
drivers/acpi/processor_idle.c
View File

@ -21,6 +21,7 @@
#include <linux/cpuidle.h>
#include <linux/cpu.h>
#include <linux/minmax.h>
#include <linux/perf_event.h>
#include <acpi/processor.h>
/*
@ -544,6 +545,8 @@ static void wait_for_freeze(void)
*/
static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx)
{
perf_lopwr_cb(true);
if (cx->entry_method == ACPI_CSTATE_FFH) {
/* Call into architectural FFH based C-state */
acpi_processor_ffh_cstate_enter(cx);
@ -554,6 +557,8 @@ static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx)
inb(cx->address);
wait_for_freeze();
}
perf_lopwr_cb(false);
}
/**

28
include/asm-generic/qrwlock.h
View File

@ -33,8 +33,8 @@ extern void queued_read_lock_slowpath(struct qrwlock *lock);
extern void queued_write_lock_slowpath(struct qrwlock *lock);
/**
* queued_read_trylock - try to acquire read lock of a queue rwlock
* @lock : Pointer to queue rwlock structure
* queued_read_trylock - try to acquire read lock of a queued rwlock
* @lock : Pointer to queued rwlock structure
* Return: 1 if lock acquired, 0 if failed
*/
static inline int queued_read_trylock(struct qrwlock *lock)
@ -52,8 +52,8 @@ static inline int queued_read_trylock(struct qrwlock *lock)
}
/**
* queued_write_trylock - try to acquire write lock of a queue rwlock
* @lock : Pointer to queue rwlock structure
* queued_write_trylock - try to acquire write lock of a queued rwlock
* @lock : Pointer to queued rwlock structure
* Return: 1 if lock acquired, 0 if failed
*/
static inline int queued_write_trylock(struct qrwlock *lock)
@ -68,8 +68,8 @@ static inline int queued_write_trylock(struct qrwlock *lock)
_QW_LOCKED));
}
/**
* queued_read_lock - acquire read lock of a queue rwlock
* @lock: Pointer to queue rwlock structure
* queued_read_lock - acquire read lock of a queued rwlock
* @lock: Pointer to queued rwlock structure
*/
static inline void queued_read_lock(struct qrwlock *lock)
{
@ -84,8 +84,8 @@ static inline void queued_read_lock(struct qrwlock *lock)
}
/**
* queued_write_lock - acquire write lock of a queue rwlock
* @lock : Pointer to queue rwlock structure
* queued_write_lock - acquire write lock of a queued rwlock
* @lock : Pointer to queued rwlock structure
*/
static inline void queued_write_lock(struct qrwlock *lock)
{
@ -98,8 +98,8 @@ static inline void queued_write_lock(struct qrwlock *lock)
}
/**
* queued_read_unlock - release read lock of a queue rwlock
* @lock : Pointer to queue rwlock structure
* queued_read_unlock - release read lock of a queued rwlock
* @lock : Pointer to queued rwlock structure
*/
static inline void queued_read_unlock(struct qrwlock *lock)
{
@ -110,8 +110,8 @@ static inline void queued_read_unlock(struct qrwlock *lock)
}
/**
* queued_write_unlock - release write lock of a queue rwlock
* @lock : Pointer to queue rwlock structure
* queued_write_unlock - release write lock of a queued rwlock
* @lock : Pointer to queued rwlock structure
*/
static inline void queued_write_unlock(struct qrwlock *lock)
{
@ -120,7 +120,7 @@ static inline void queued_write_unlock(struct qrwlock *lock)
/**
* queued_rwlock_is_contended - check if the lock is contended
* @lock : Pointer to queue rwlock structure
* @lock : Pointer to queued rwlock structure
* Return: 1 if lock contended, 0 otherwise
*/
static inline int queued_rwlock_is_contended(struct qrwlock *lock)
@ -130,7 +130,7 @@ static inline int queued_rwlock_is_contended(struct qrwlock *lock)
/*
* Remapping rwlock architecture specific functions to the corresponding
* queue rwlock functions.
* queued rwlock functions.
*/
#define arch_read_lock(l) queued_read_lock(l)
#define arch_write_lock(l) queued_write_lock(l)

2
include/asm-generic/qrwlock_types.h
View File

@ -7,7 +7,7 @@
#include <asm/spinlock_types.h>
/*
* The queue read/write lock data structure
* The queued read/write lock data structure
*/
typedef struct qrwlock {

72
include/linux/atomic/atomic-arch-fallback.h
View File

@ -147,6 +147,76 @@
#endif /* arch_try_cmpxchg_relaxed */
#ifndef arch_try_cmpxchg64_relaxed
#ifdef arch_try_cmpxchg64
#define arch_try_cmpxchg64_acquire arch_try_cmpxchg64
#define arch_try_cmpxchg64_release arch_try_cmpxchg64
#define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64
#endif /* arch_try_cmpxchg64 */
#ifndef arch_try_cmpxchg64
#define arch_try_cmpxchg64(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg64((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg64 */
#ifndef arch_try_cmpxchg64_acquire
#define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg64_acquire */
#ifndef arch_try_cmpxchg64_release
#define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg64_release */
#ifndef arch_try_cmpxchg64_relaxed
#define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg64_relaxed */
#else /* arch_try_cmpxchg64_relaxed */
#ifndef arch_try_cmpxchg64_acquire
#define arch_try_cmpxchg64_acquire(...) \
__atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
#endif
#ifndef arch_try_cmpxchg64_release
#define arch_try_cmpxchg64_release(...) \
__atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
#endif
#ifndef arch_try_cmpxchg64
#define arch_try_cmpxchg64(...) \
__atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
#endif
#endif /* arch_try_cmpxchg64_relaxed */
#ifndef arch_atomic_read_acquire
static __always_inline int
arch_atomic_read_acquire(const atomic_t *v)
@ -2386,4 +2456,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)
#endif
#endif /* _LINUX_ATOMIC_FALLBACK_H */
// 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae
// b5e87bdd5ede61470c29f7a7e4de781af3770f09

40
include/linux/atomic/atomic-instrumented.h
View File

@ -2006,6 +2006,44 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_mb(); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64_acquire(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64_release(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_release(); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define try_cmpxchg64_relaxed(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})
#define cmpxchg_local(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
@ -2045,4 +2083,4 @@ atomic_long_dec_if_positive(atomic_long_t *v)
})
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
// 87c974b93032afd42143613434d1a7788fa598f9
// 764f741eb77a7ad565dc8d99ce2837d5542e8aee

1
include/linux/compat.h
View File

@ -235,6 +235,7 @@ typedef struct compat_siginfo {
struct {
compat_ulong_t _data;
u32 _type;
u32 _flags;
} _perf;
};
} _sigfault;

6
include/linux/compiler.h
View File

@ -109,7 +109,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
#endif
/* Unreachable code */
#ifdef CONFIG_STACK_VALIDATION
#ifdef CONFIG_OBJTOOL
/*
* These macros help objtool understand GCC code flow for unreachable code.
* The __COUNTER__ based labels are a hack to make each instance of the macros
@ -128,10 +128,10 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
/* Annotate a C jump table to allow objtool to follow the code flow */
#define __annotate_jump_table __section(".rodata..c_jump_table")
#else
#else /* !CONFIG_OBJTOOL */
#define annotate_unreachable()
#define __annotate_jump_table
#endif
#endif /* CONFIG_OBJTOOL */
#ifndef unreachable
# define unreachable() do { \

6
include/linux/instrumentation.h
View File

@ -2,7 +2,7 @@
#ifndef __LINUX_INSTRUMENTATION_H
#define __LINUX_INSTRUMENTATION_H
#if defined(CONFIG_DEBUG_ENTRY) && defined(CONFIG_STACK_VALIDATION)
#ifdef CONFIG_NOINSTR_VALIDATION
#include <linux/stringify.h>
@ -53,9 +53,9 @@
".popsection\n\t" : : "i" (c)); \
})
#define instrumentation_end() __instrumentation_end(__COUNTER__)
#else
#else /* !CONFIG_NOINSTR_VALIDATION */
# define instrumentation_begin() do { } while(0)
# define instrumentation_end() do { } while(0)
#endif
#endif /* CONFIG_NOINSTR_VALIDATION */
#endif /* __LINUX_INSTRUMENTATION_H */

18
include/linux/interrupt.h
View File

@ -222,24 +222,6 @@ devm_request_any_context_irq(struct device *dev, unsigned int irq,
extern void devm_free_irq(struct device *dev, unsigned int irq, void *dev_id);
/*
* On lockdep we dont want to enable hardirqs in hardirq
* context. Use local_irq_enable_in_hardirq() to annotate
* kernel code that has to do this nevertheless (pretty much
* the only valid case is for old/broken hardware that is
* insanely slow).
*
* NOTE: in theory this might break fragile code that relies
* on hardirq delivery - in practice we dont seem to have such
* places left. So the only effect should be slightly increased
* irqs-off latencies.
*/
#ifdef CONFIG_LOCKDEP
# define local_irq_enable_in_hardirq() do { } while (0)
#else
# define local_irq_enable_in_hardirq() local_irq_enable()
#endif
bool irq_has_action(unsigned int irq);
extern void disable_irq_nosync(unsigned int irq);
extern bool disable_hardirq(unsigned int irq);

4
include/linux/irqflags.h
View File

@ -20,13 +20,13 @@
#ifdef CONFIG_PROVE_LOCKING
extern void lockdep_softirqs_on(unsigned long ip);
extern void lockdep_softirqs_off(unsigned long ip);
extern void lockdep_hardirqs_on_prepare(unsigned long ip);
extern void lockdep_hardirqs_on_prepare(void);
extern void lockdep_hardirqs_on(unsigned long ip);
extern void lockdep_hardirqs_off(unsigned long ip);
#else
static inline void lockdep_softirqs_on(unsigned long ip) { }
static inline void lockdep_softirqs_off(unsigned long ip) { }
static inline void lockdep_hardirqs_on_prepare(unsigned long ip) { }
static inline void lockdep_hardirqs_on_prepare(void) { }
static inline void lockdep_hardirqs_on(unsigned long ip) { }
static inline void lockdep_hardirqs_off(unsigned long ip) { }
#endif

2
include/linux/kvm_host.h
View File

@ -453,7 +453,7 @@ static __always_inline void guest_state_enter_irqoff(void)
{
instrumentation_begin();
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
instrumentation_end();
guest_context_enter_irqoff();

6
include/linux/objtool.h
View File

@ -38,7 +38,7 @@ struct unwind_hint {
#define UNWIND_HINT_TYPE_REGS_PARTIAL 2
#define UNWIND_HINT_TYPE_FUNC 3
#ifdef CONFIG_STACK_VALIDATION
#ifdef CONFIG_OBJTOOL
#include <asm/asm.h>
@ -159,7 +159,7 @@ struct unwind_hint {
#endif /* __ASSEMBLY__ */
#else /* !CONFIG_STACK_VALIDATION */
#else /* !CONFIG_OBJTOOL */
#ifndef __ASSEMBLY__
@ -181,6 +181,6 @@ struct unwind_hint {
.endm
#endif
#endif /* CONFIG_STACK_VALIDATION */
#endif /* CONFIG_OBJTOOL */
#endif /* _LINUX_OBJTOOL_H */

22
include/linux/perf_event.h
View File

@ -1063,6 +1063,22 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,
data->txn = 0;
}
/*
* Clear all bitfields in the perf_branch_entry.
* The to and from fields are not cleared because they are
* systematically modified by caller.
*/
static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br)
{
br->mispred = 0;
br->predicted = 0;
br->in_tx = 0;
br->abort = 0;
br->cycles = 0;
br->type = 0;
br->reserved = 0;
}
extern void perf_output_sample(struct perf_output_handle *handle,
struct perf_event_header *header,
struct perf_sample_data *data,
@ -1678,4 +1694,10 @@ typedef int (perf_snapshot_branch_stack_t)(struct perf_branch_entry *entries,
unsigned int cnt);
DECLARE_STATIC_CALL(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);
#ifndef PERF_NEEDS_LOPWR_CB
static inline void perf_lopwr_cb(bool mode)
{
}
#endif
#endif /* _LINUX_PERF_EVENT_H */

2
include/linux/sched/signal.h
View File

@ -320,7 +320,7 @@ int send_sig_mceerr(int code, void __user *, short, struct task_struct *);
int force_sig_bnderr(void __user *addr, void __user *lower, void __user *upper);
int force_sig_pkuerr(void __user *addr, u32 pkey);
int force_sig_perf(void __user *addr, u32 type, u64 sig_data);
int send_sig_perf(void __user *addr, u32 type, u64 sig_data);
int force_sig_ptrace_errno_trap(int errno, void __user *addr);
int force_sig_fault_trapno(int sig, int code, void __user *addr, int trapno);

2
include/linux/sched/task_stack.h
View File

@ -16,7 +16,7 @@
* try_get_task_stack() instead. task_stack_page will return a pointer
* that could get freed out from under you.
*/
static inline void *task_stack_page(const struct task_struct *task)
static __always_inline void *task_stack_page(const struct task_struct *task)
{
return task->stack;
}

63
include/trace/events/lock.h
View File

@ -5,11 +5,22 @@
#if !defined(_TRACE_LOCK_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_LOCK_H
#include <linux/lockdep.h>
#include <linux/sched.h>
#include <linux/tracepoint.h>
/* flags for lock:contention_begin */
#define LCB_F_SPIN (1U << 0)
#define LCB_F_READ (1U << 1)
#define LCB_F_WRITE (1U << 2)
#define LCB_F_RT (1U << 3)
#define LCB_F_PERCPU (1U << 4)
#define LCB_F_MUTEX (1U << 5)
#ifdef CONFIG_LOCKDEP
#include <linux/lockdep.h>
TRACE_EVENT(lock_acquire,
TP_PROTO(struct lockdep_map *lock, unsigned int subclass,
@ -78,8 +89,54 @@ DEFINE_EVENT(lock, lock_acquired,
TP_ARGS(lock, ip)
);
#endif
#endif
#endif /* CONFIG_LOCK_STAT */
#endif /* CONFIG_LOCKDEP */
TRACE_EVENT(contention_begin,
TP_PROTO(void *lock, unsigned int flags),
TP_ARGS(lock, flags),
TP_STRUCT__entry(
__field(void *, lock_addr)
__field(unsigned int, flags)
),
TP_fast_assign(
__entry->lock_addr = lock;
__entry->flags = flags;
),
TP_printk("%p (flags=%s)", __entry->lock_addr,
__print_flags(__entry->flags, "|",
{ LCB_F_SPIN, "SPIN" },
{ LCB_F_READ, "READ" },
{ LCB_F_WRITE, "WRITE" },
{ LCB_F_RT, "RT" },
{ LCB_F_PERCPU, "PERCPU" },
{ LCB_F_MUTEX, "MUTEX" }
))
);
TRACE_EVENT(contention_end,
TP_PROTO(void *lock, int ret),
TP_ARGS(lock, ret),
TP_STRUCT__entry(
__field(void *, lock_addr)
__field(int, ret)
),
TP_fast_assign(
__entry->lock_addr = lock;
__entry->ret = ret;
),
TP_printk("%p (ret=%d)", __entry->lock_addr, __entry->ret)
);
#endif /* _TRACE_LOCK_H */

7
include/uapi/asm-generic/siginfo.h
View File

@ -99,6 +99,7 @@ union __sifields {
struct {
unsigned long _data;
__u32 _type;
__u32 _flags;
} _perf;
};
} _sigfault;
@ -164,6 +165,7 @@ typedef struct siginfo {
#define si_pkey _sifields._sigfault._addr_pkey._pkey
#define si_perf_data _sifields._sigfault._perf._data
#define si_perf_type _sifields._sigfault._perf._type
#define si_perf_flags _sifields._sigfault._perf._flags
#define si_band _sifields._sigpoll._band
#define si_fd _sifields._sigpoll._fd
#define si_call_addr _sifields._sigsys._call_addr
@ -270,6 +272,11 @@ typedef struct siginfo {
* that are of the form: ((PTRACE_EVENT_XXX << 8) | SIGTRAP)
*/
/*
* Flags for si_perf_flags if SIGTRAP si_code is TRAP_PERF.
*/
#define TRAP_PERF_FLAG_ASYNC (1u << 0)
/*
* SIGCHLD si_codes
*/

6
kernel/entry/common.c
View File

@ -126,7 +126,7 @@ static __always_inline void __exit_to_user_mode(void)
{
instrumentation_begin();
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
instrumentation_end();
user_enter_irqoff();
@ -416,7 +416,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
instrumentation_begin();
/* Tell the tracer that IRET will enable interrupts */
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
instrumentation_end();
rcu_irq_exit();
lockdep_hardirqs_on(CALLER_ADDR0);
@ -465,7 +465,7 @@ void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
ftrace_nmi_exit();
if (irq_state.lockdep) {
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
}
instrumentation_end();

4
kernel/events/core.c
View File

@ -6429,8 +6429,8 @@ static void perf_sigtrap(struct perf_event *event)
if (current->flags & PF_EXITING)
return;
force_sig_perf((void __user *)event->pending_addr,
event->attr.type, event->attr.sig_data);
send_sig_perf((void __user *)event->pending_addr,
event->attr.type, event->attr.sig_data);
}
static void perf_pending_event_disable(struct perf_event *event)

2
kernel/futex/pi.c
View File

@ -1005,7 +1005,7 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl
rt_mutex_init_waiter(&rt_waiter);
/*
* On PREEMPT_RT_FULL, when hb->lock becomes an rt_mutex, we must not
* On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not
* hold it while doing rt_mutex_start_proxy(), because then it will
* include hb->lock in the blocking chain, even through we'll not in
* fact hold it while blocking. This will lead it to report -EDEADLK

23
kernel/locking/lockdep.c
View File

@ -60,7 +60,6 @@
#include "lockdep_internals.h"
#define CREATE_TRACE_POINTS
#include <trace/events/lock.h>
#ifdef CONFIG_PROVE_LOCKING
@ -1380,7 +1379,7 @@ static struct lock_list *alloc_list_entry(void)
*/
static int add_lock_to_list(struct lock_class *this,
struct lock_class *links_to, struct list_head *head,
unsigned long ip, u16 distance, u8 dep,
u16 distance, u8 dep,
const struct lock_trace *trace)
{
struct lock_list *entry;
@ -3133,19 +3132,15 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev,
* to the previous lock's dependency list:
*/
ret = add_lock_to_list(hlock_class(next), hlock_class(prev),
&hlock_class(prev)->locks_after,
next->acquire_ip, distance,
calc_dep(prev, next),
*trace);
&hlock_class(prev)->locks_after, distance,
calc_dep(prev, next), *trace);
if (!ret)
return 0;
ret = add_lock_to_list(hlock_class(prev), hlock_class(next),
&hlock_class(next)->locks_before,
next->acquire_ip, distance,
calc_depb(prev, next),
*trace);
&hlock_class(next)->locks_before, distance,
calc_depb(prev, next), *trace);
if (!ret)
return 0;
@ -4236,14 +4231,13 @@ static void __trace_hardirqs_on_caller(void)
/**
* lockdep_hardirqs_on_prepare - Prepare for enabling interrupts
* @ip: Caller address
*
* Invoked before a possible transition to RCU idle from exit to user or
* guest mode. This ensures that all RCU operations are done before RCU
* stops watching. After the RCU transition lockdep_hardirqs_on() has to be
* invoked to set the final state.
*/
void lockdep_hardirqs_on_prepare(unsigned long ip)
void lockdep_hardirqs_on_prepare(void)
{
if (unlikely(!debug_locks))
return;
@ -4840,8 +4834,7 @@ EXPORT_SYMBOL_GPL(__lockdep_no_validate__);
static void
print_lock_nested_lock_not_held(struct task_struct *curr,
struct held_lock *hlock,
unsigned long ip)
struct held_lock *hlock)
{
if (!debug_locks_off())
return;
@ -5017,7 +5010,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
chain_key = iterate_chain_key(chain_key, hlock_id(hlock));
if (nest_lock && !__lock_is_held(nest_lock, -1)) {
print_lock_nested_lock_not_held(curr, hlock, ip);
print_lock_nested_lock_not_held(curr, hlock);
return 0;
}

18
kernel/locking/mutex.c
View File

@ -30,6 +30,9 @@
#include <linux/debug_locks.h>
#include <linux/osq_lock.h>
#define CREATE_TRACE_POINTS
#include <trace/events/lock.h>
#ifndef CONFIG_PREEMPT_RT
#include "mutex.h"
@ -599,12 +602,14 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
preempt_disable();
mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
if (__mutex_trylock(lock) ||
mutex_optimistic_spin(lock, ww_ctx, NULL)) {
/* got the lock, yay! */
lock_acquired(&lock->dep_map, ip);
if (ww_ctx)
ww_mutex_set_context_fastpath(ww, ww_ctx);
trace_contention_end(lock, 0);
preempt_enable();
return 0;
}
@ -641,6 +646,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
}
set_current_state(state);
trace_contention_begin(lock, LCB_F_MUTEX);
for (;;) {
bool first;
@ -680,10 +686,16 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
* state back to RUNNING and fall through the next schedule(),
* or we must see its unlock and acquire.
*/
if (__mutex_trylock_or_handoff(lock, first) ||
(first && mutex_optimistic_spin(lock, ww_ctx, &waiter)))
if (__mutex_trylock_or_handoff(lock, first))
break;
if (first) {
trace_contention_begin(lock, LCB_F_MUTEX | LCB_F_SPIN);
if (mutex_optimistic_spin(lock, ww_ctx, &waiter))
break;
trace_contention_begin(lock, LCB_F_MUTEX);
}
raw_spin_lock(&lock->wait_lock);
}
raw_spin_lock(&lock->wait_lock);
@ -707,6 +719,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
skip_wait:
/* got the lock - cleanup and rejoice! */
lock_acquired(&lock->dep_map, ip);
trace_contention_end(lock, 0);
if (ww_ctx)
ww_mutex_lock_acquired(ww, ww_ctx);
@ -719,6 +732,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
__set_current_state(TASK_RUNNING);
__mutex_remove_waiter(lock, &waiter);
err_early_kill:
trace_contention_end(lock, ret);
raw_spin_unlock(&lock->wait_lock);
debug_mutex_free_waiter(&waiter);
mutex_release(&lock->dep_map, ip);

5
kernel/locking/percpu-rwsem.c
View File

@ -9,6 +9,7 @@
#include <linux/sched/task.h>
#include <linux/sched/debug.h>
#include <linux/errno.h>
#include <trace/events/lock.h>
int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
const char *name, struct lock_class_key *key)
@ -171,9 +172,11 @@ bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try)
if (try)
return false;
trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_READ);
preempt_enable();
percpu_rwsem_wait(sem, /* .reader = */ true);
preempt_disable();
trace_contention_end(sem, 0);
return true;
}
@ -216,6 +219,7 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
{
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_WRITE);
/* Notify readers to take the slow path. */
rcu_sync_enter(&sem->rss);
@ -237,6 +241,7 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
/* Wait for all active readers to complete. */
rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);
trace_contention_end(sem, 0);
}
EXPORT_SYMBOL_GPL(percpu_down_write);

17
kernel/locking/qrwlock.c
View File

@ -12,10 +12,11 @@
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/spinlock.h>
#include <trace/events/lock.h>
/**
* queued_read_lock_slowpath - acquire read lock of a queue rwlock
* @lock: Pointer to queue rwlock structure
* queued_read_lock_slowpath - acquire read lock of a queued rwlock
* @lock: Pointer to queued rwlock structure
*/
void queued_read_lock_slowpath(struct qrwlock *lock)
{
@ -34,6 +35,8 @@ void queued_read_lock_slowpath(struct qrwlock *lock)
}
atomic_sub(_QR_BIAS, &lock->cnts);
trace_contention_begin(lock, LCB_F_SPIN | LCB_F_READ);
/*
* Put the reader into the wait queue
*/
@ -51,17 +54,21 @@ void queued_read_lock_slowpath(struct qrwlock *lock)
* Signal the next one in queue to become queue head
*/
arch_spin_unlock(&lock->wait_lock);
trace_contention_end(lock, 0);
}
EXPORT_SYMBOL(queued_read_lock_slowpath);
/**
* queued_write_lock_slowpath - acquire write lock of a queue rwlock
* @lock : Pointer to queue rwlock structure
* queued_write_lock_slowpath - acquire write lock of a queued rwlock
* @lock : Pointer to queued rwlock structure
*/
void queued_write_lock_slowpath(struct qrwlock *lock)
{
int cnts;
trace_contention_begin(lock, LCB_F_SPIN | LCB_F_WRITE);
/* Put the writer into the wait queue */
arch_spin_lock(&lock->wait_lock);
@ -79,5 +86,7 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
} while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));
unlock:
arch_spin_unlock(&lock->wait_lock);
trace_contention_end(lock, 0);
}
EXPORT_SYMBOL(queued_write_lock_slowpath);

5
kernel/locking/qspinlock.c
View File

@ -22,6 +22,7 @@
#include <linux/prefetch.h>
#include <asm/byteorder.h>
#include <asm/qspinlock.h>
#include <trace/events/lock.h>
/*
* Include queued spinlock statistics code
@ -401,6 +402,8 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
trace_contention_begin(lock, LCB_F_SPIN);
/*
* 4 nodes are allocated based on the assumption that there will
* not be nested NMIs taking spinlocks. That may not be true in
@ -554,6 +557,8 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
pv_kick_node(lock, next);
release:
trace_contention_end(lock, 0);
/*
* release the node
*/

11
kernel/locking/rtmutex.c
View File

@ -24,6 +24,8 @@
#include <linux/sched/wake_q.h>
#include <linux/ww_mutex.h>
#include <trace/events/lock.h>
#include "rtmutex_common.h"
#ifndef WW_RT
@ -1579,6 +1581,8 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
set_current_state(state);
trace_contention_begin(lock, LCB_F_RT);
ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);
if (likely(!ret))
ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);
@ -1601,6 +1605,9 @@ static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
* unconditionally. We might have to fix that up.
*/
fixup_rt_mutex_waiters(lock);
trace_contention_end(lock, ret);
return ret;
}
@ -1683,6 +1690,8 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)
/* Save current state and set state to TASK_RTLOCK_WAIT */
current_save_and_set_rtlock_wait_state();
trace_contention_begin(lock, LCB_F_RT);
task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);
for (;;) {
@ -1712,6 +1721,8 @@ static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)
*/
fixup_rt_mutex_waiters(lock);
debug_rt_mutex_free_waiter(&waiter);
trace_contention_end(lock, 0);
}
static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)

7
kernel/locking/rwbase_rt.c
View File

@ -112,6 +112,8 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
* Reader2 to call up_read(), which might be unbound.
*/
trace_contention_begin(rwb, LCB_F_RT | LCB_F_READ);
/*
* For rwlocks this returns 0 unconditionally, so the below
* !ret conditionals are optimized out.
@ -130,6 +132,8 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,
raw_spin_unlock_irq(&rtm->wait_lock);
if (!ret)
rwbase_rtmutex_unlock(rtm);
trace_contention_end(rwb, ret);
return ret;
}
@ -247,11 +251,13 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
goto out_unlock;
rwbase_set_and_save_current_state(state);
trace_contention_begin(rwb, LCB_F_RT | LCB_F_WRITE);
for (;;) {
/* Optimized out for rwlocks */
if (rwbase_signal_pending_state(state, current)) {
rwbase_restore_current_state();
__rwbase_write_unlock(rwb, 0, flags);
trace_contention_end(rwb, -EINTR);
return -EINTR;
}
@ -265,6 +271,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
set_current_state(state);
}
rwbase_restore_current_state();
trace_contention_end(rwb, 0);
out_unlock:
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);

130
kernel/locking/rwsem.c
View File

@ -27,6 +27,7 @@
#include <linux/export.h>
#include <linux/rwsem.h>
#include <linux/atomic.h>
#include <trace/events/lock.h>
#ifndef CONFIG_PREEMPT_RT
#include "lock_events.h"
@ -375,16 +376,19 @@ rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
*
* Both rwsem_mark_wake() and rwsem_try_write_lock() contain a full 'copy' of
* this function. Modify with care.
*
* Return: true if wait_list isn't empty and false otherwise
*/
static inline void
static inline bool
rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
{
lockdep_assert_held(&sem->wait_lock);
list_del(&waiter->list);
if (likely(!list_empty(&sem->wait_list)))
return;
return true;
atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count);
return false;
}
/*
@ -558,6 +562,33 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
}
}
/*
* Remove a waiter and try to wake up other waiters in the wait queue
* This function is called from the out_nolock path of both the reader and
* writer slowpaths with wait_lock held. It releases the wait_lock and
* optionally wake up waiters before it returns.
*/
static inline void
rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter,
struct wake_q_head *wake_q)
__releases(&sem->wait_lock)
{
bool first = rwsem_first_waiter(sem) == waiter;
wake_q_init(wake_q);
/*
* If the wait_list isn't empty and the waiter to be deleted is
* the first waiter, we wake up the remaining waiters as they may
* be eligible to acquire or spin on the lock.
*/
if (rwsem_del_waiter(sem, waiter) && first)
rwsem_mark_wake(sem, RWSEM_WAKE_ANY, wake_q);
raw_spin_unlock_irq(&sem->wait_lock);
if (!wake_q_empty(wake_q))
wake_up_q(wake_q);
}
/*
* This function must be called with the sem->wait_lock held to prevent
* race conditions between checking the rwsem wait list and setting the
@ -901,7 +932,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
*/
static inline void clear_nonspinnable(struct rw_semaphore *sem)
{
if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE))
if (unlikely(rwsem_test_oflags(sem, RWSEM_NONSPINNABLE)))
atomic_long_andnot(RWSEM_NONSPINNABLE, &sem->owner);
}
@ -925,6 +956,31 @@ rwsem_spin_on_owner(struct rw_semaphore *sem)
}
#endif
/*
* Prepare to wake up waiter(s) in the wait queue by putting them into the
* given wake_q if the rwsem lock owner isn't a writer. If rwsem is likely
* reader-owned, wake up read lock waiters in queue front or wake up any
* front waiter otherwise.
* This is being called from both reader and writer slow paths.
*/
static inline void rwsem_cond_wake_waiter(struct rw_semaphore *sem, long count,
struct wake_q_head *wake_q)
{
enum rwsem_wake_type wake_type;
if (count & RWSEM_WRITER_MASK)
return;
if (count & RWSEM_READER_MASK) {
wake_type = RWSEM_WAKE_READERS;
} else {
wake_type = RWSEM_WAKE_ANY;
clear_nonspinnable(sem);
}
rwsem_mark_wake(sem, wake_type, wake_q);
}
/*
* Wait for the read lock to be granted
*/
@ -935,7 +991,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
long rcnt = (count >> RWSEM_READER_SHIFT);
struct rwsem_waiter waiter;
DEFINE_WAKE_Q(wake_q);
bool wake = false;
/*
* To prevent a constant stream of readers from starving a sleeping
@ -977,12 +1032,11 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
if (list_empty(&sem->wait_list)) {
/*
* In case the wait queue is empty and the lock isn't owned
* by a writer or has the handoff bit set, this reader can
* exit the slowpath and return immediately as its
* RWSEM_READER_BIAS has already been set in the count.
* by a writer, this reader can exit the slowpath and return
* immediately as its RWSEM_READER_BIAS has already been set
* in the count.
*/
if (!(atomic_long_read(&sem->count) &
(RWSEM_WRITER_MASK | RWSEM_FLAG_HANDOFF))) {
if (!(atomic_long_read(&sem->count) & RWSEM_WRITER_MASK)) {
/* Provide lock ACQUIRE */
smp_acquire__after_ctrl_dep();
raw_spin_unlock_irq(&sem->wait_lock);
@ -997,22 +1051,13 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
/* we're now waiting on the lock, but no longer actively locking */
count = atomic_long_add_return(adjustment, &sem->count);
/*
* If there are no active locks, wake the front queued process(es).
*
* If there are no writers and we are first in the queue,
* wake our own waiter to join the existing active readers !
*/
if (!(count & RWSEM_LOCK_MASK)) {
clear_nonspinnable(sem);
wake = true;
}
if (wake || (!(count & RWSEM_WRITER_MASK) &&
(adjustment & RWSEM_FLAG_WAITERS)))
rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
rwsem_cond_wake_waiter(sem, count, &wake_q);
raw_spin_unlock_irq(&sem->wait_lock);
wake_up_q(&wake_q);
if (!wake_q_empty(&wake_q))
wake_up_q(&wake_q);
trace_contention_begin(sem, LCB_F_READ);
/* wait to be given the lock */
for (;;) {
@ -1035,13 +1080,14 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
__set_current_state(TASK_RUNNING);
lockevent_inc(rwsem_rlock);
trace_contention_end(sem, 0);
return sem;
out_nolock:
rwsem_del_waiter(sem, &waiter);
raw_spin_unlock_irq(&sem->wait_lock);
rwsem_del_wake_waiter(sem, &waiter, &wake_q);
__set_current_state(TASK_RUNNING);
lockevent_inc(rwsem_rlock_fail);
trace_contention_end(sem, -EINTR);
return ERR_PTR(-EINTR);
}
@ -1051,7 +1097,6 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
static struct rw_semaphore __sched *
rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
{
long count;
struct rwsem_waiter waiter;
DEFINE_WAKE_Q(wake_q);
@ -1075,23 +1120,8 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
/* we're now waiting on the lock */
if (rwsem_first_waiter(sem) != &waiter) {
count = atomic_long_read(&sem->count);
/*
* If there were already threads queued before us and:
* 1) there are no active locks, wake the front
* queued process(es) as the handoff bit might be set.
* 2) there are no active writers and some readers, the lock
* must be read owned; so we try to wake any read lock
* waiters that were queued ahead of us.
*/
if (count & RWSEM_WRITER_MASK)
goto wait;
rwsem_mark_wake(sem, (count & RWSEM_READER_MASK)
? RWSEM_WAKE_READERS
: RWSEM_WAKE_ANY, &wake_q);
rwsem_cond_wake_waiter(sem, atomic_long_read(&sem->count),
&wake_q);
if (!wake_q_empty(&wake_q)) {
/*
* We want to minimize wait_lock hold time especially
@ -1099,16 +1129,16 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
*/
raw_spin_unlock_irq(&sem->wait_lock);
wake_up_q(&wake_q);
wake_q_init(&wake_q); /* Used again, reinit */
raw_spin_lock_irq(&sem->wait_lock);
}
} else {
atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count);
}
wait:
/* wait until we successfully acquire the lock */
set_current_state(state);
trace_contention_begin(sem, LCB_F_WRITE);
for (;;) {
if (rwsem_try_write_lock(sem, &waiter)) {
/* rwsem_try_write_lock() implies ACQUIRE on success */
@ -1148,17 +1178,15 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
__set_current_state(TASK_RUNNING);
raw_spin_unlock_irq(&sem->wait_lock);
lockevent_inc(rwsem_wlock);
trace_contention_end(sem, 0);
return sem;
out_nolock:
__set_current_state(TASK_RUNNING);
raw_spin_lock_irq(&sem->wait_lock);
rwsem_del_waiter(sem, &waiter);
if (!list_empty(&sem->wait_list))
rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
raw_spin_unlock_irq(&sem->wait_lock);
wake_up_q(&wake_q);
rwsem_del_wake_waiter(sem, &waiter, &wake_q);
lockevent_inc(rwsem_wlock_fail);
trace_contention_end(sem, -EINTR);
return ERR_PTR(-EINTR);
}

15
kernel/locking/semaphore.c
View File

@ -32,6 +32,7 @@
#include <linux/semaphore.h>
#include <linux/spinlock.h>
#include <linux/ftrace.h>
#include <trace/events/lock.h>
static noinline void __down(struct semaphore *sem);
static noinline int __down_interruptible(struct semaphore *sem);
@ -205,7 +206,7 @@ struct semaphore_waiter {
* constant, and thus optimised away by the compiler. Likewise the
* 'timeout' parameter for the cases without timeouts.
*/
static inline int __sched __down_common(struct semaphore *sem, long state,
static inline int __sched ___down_common(struct semaphore *sem, long state,
long timeout)
{
struct semaphore_waiter waiter;
@ -236,6 +237,18 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
return -EINTR;
}
static inline int __sched __down_common(struct semaphore *sem, long state,
long timeout)
{
int ret;
trace_contention_begin(sem, 0);
ret = ___down_common(sem, state, timeout);
trace_contention_end(sem, ret);
return ret;
}
static noinline void __sched __down(struct semaphore *sem)
{
__down_common(sem, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);

4
kernel/sched/clock.c
View File

@ -287,7 +287,7 @@ notrace static u64 sched_clock_local(struct sched_clock_data *scd)
clock = wrap_max(clock, min_clock);
clock = wrap_min(clock, max_clock);
if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock)
if (!try_cmpxchg64(&scd->clock, &old_clock, clock))
goto again;
return clock;
@ -349,7 +349,7 @@ notrace static u64 sched_clock_remote(struct sched_clock_data *scd)
val = remote_clock;
}
if (cmpxchg64(ptr, old_val, val) != old_val)
if (!try_cmpxchg64(ptr, &old_val, val))
goto again;
return val;

2
kernel/sched/idle.c
View File

@ -102,7 +102,7 @@ void __cpuidle default_idle_call(void)
* last -- this is very similar to the entry code.
*/
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(_THIS_IP_);
lockdep_hardirqs_on_prepare();
rcu_idle_enter();
lockdep_hardirqs_on(_THIS_IP_);

18
kernel/signal.c
View File

@ -1805,7 +1805,7 @@ int force_sig_pkuerr(void __user *addr, u32 pkey)
}
#endif
int force_sig_perf(void __user *addr, u32 type, u64 sig_data)
int send_sig_perf(void __user *addr, u32 type, u64 sig_data)
{
struct kernel_siginfo info;
@ -1817,7 +1817,18 @@ int force_sig_perf(void __user *addr, u32 type, u64 sig_data)
info.si_perf_data = sig_data;
info.si_perf_type = type;
return force_sig_info(&info);
/*
* Signals generated by perf events should not terminate the whole
* process if SIGTRAP is blocked, however, delivering the signal
* asynchronously is better than not delivering at all. But tell user
* space if the signal was asynchronous, so it can clearly be
* distinguished from normal synchronous ones.
*/
info.si_perf_flags = sigismember(&current->blocked, info.si_signo) ?
TRAP_PERF_FLAG_ASYNC :
0;
return send_sig_info(info.si_signo, &info, current);
}
/**
@ -3432,6 +3443,7 @@ void copy_siginfo_to_external32(struct compat_siginfo *to,
to->si_addr = ptr_to_compat(from->si_addr);
to->si_perf_data = from->si_perf_data;
to->si_perf_type = from->si_perf_type;
to->si_perf_flags = from->si_perf_flags;
break;
case SIL_CHLD:
to->si_pid = from->si_pid;
@ -3509,6 +3521,7 @@ static int post_copy_siginfo_from_user32(kernel_siginfo_t *to,
to->si_addr = compat_ptr(from->si_addr);
to->si_perf_data = from->si_perf_data;
to->si_perf_type = from->si_perf_type;
to->si_perf_flags = from->si_perf_flags;
break;
case SIL_CHLD:
to->si_pid = from->si_pid;
@ -4722,6 +4735,7 @@ static inline void siginfo_buildtime_checks(void)
CHECK_OFFSET(si_pkey);
CHECK_OFFSET(si_perf_data);
CHECK_OFFSET(si_perf_type);
CHECK_OFFSET(si_perf_flags);
/* sigpoll */
CHECK_OFFSET(si_band);

1
kernel/trace/Kconfig
View File

@ -729,6 +729,7 @@ config FTRACE_MCOUNT_USE_OBJTOOL
depends on !FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY
depends on !FTRACE_MCOUNT_USE_CC
depends on FTRACE_MCOUNT_RECORD
select OBJTOOL
config FTRACE_MCOUNT_USE_RECORDMCOUNT
def_bool y

4
kernel/trace/trace_preemptirq.c
View File

@ -54,7 +54,7 @@ void trace_hardirqs_on(void)
this_cpu_write(tracing_irq_cpu, 0);
}
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
lockdep_hardirqs_on(CALLER_ADDR0);
}
EXPORT_SYMBOL(trace_hardirqs_on);
@ -111,7 +111,7 @@ __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
this_cpu_write(tracing_irq_cpu, 0);
}
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
lockdep_hardirqs_on_prepare();
lockdep_hardirqs_on(CALLER_ADDR0);
}
EXPORT_SYMBOL(trace_hardirqs_on_caller);

22
lib/Kconfig.debug
View File

@ -485,24 +485,25 @@ config FRAME_POINTER
larger and slower, but it gives very useful debugging information
in case of kernel bugs. (precise oopses/stacktraces/warnings)
config OBJTOOL
bool
config STACK_VALIDATION
bool "Compile-time stack metadata validation"
depends on HAVE_STACK_VALIDATION
depends on HAVE_STACK_VALIDATION && UNWINDER_FRAME_POINTER
select OBJTOOL
default n
help
Add compile-time checks to validate stack metadata, including frame
pointers (if CONFIG_FRAME_POINTER is enabled). This helps ensure
that runtime stack traces are more reliable.
This is also a prerequisite for generation of ORC unwind data, which
is needed for CONFIG_UNWINDER_ORC.
Validate frame pointer rules at compile-time. This helps ensure that
runtime stack traces are more reliable.
For more information, see
tools/objtool/Documentation/stack-validation.txt.
config VMLINUX_VALIDATION
config NOINSTR_VALIDATION
bool
depends on STACK_VALIDATION && DEBUG_ENTRY
depends on HAVE_NOINSTR_VALIDATION && DEBUG_ENTRY
select OBJTOOL
default y
config VMLINUX_MAP
@ -2035,10 +2036,11 @@ config KCOV
bool "Code coverage for fuzzing"
depends on ARCH_HAS_KCOV
depends on CC_HAS_SANCOV_TRACE_PC || GCC_PLUGINS
depends on !ARCH_WANTS_NO_INSTR || STACK_VALIDATION || \
depends on !ARCH_WANTS_NO_INSTR || HAVE_NOINSTR_HACK || \
GCC_VERSION >= 120000 || CLANG_VERSION >= 130000
select DEBUG_FS
select GCC_PLUGIN_SANCOV if !CC_HAS_SANCOV_TRACE_PC
select OBJTOOL if HAVE_NOINSTR_HACK
help
KCOV exposes kernel code coverage information in a form suitable
for coverage-guided fuzzing (randomized testing).

4
lib/Kconfig.kcsan
View File

@ -187,7 +187,9 @@ config KCSAN_WEAK_MEMORY
# We can either let objtool nop __tsan_func_{entry,exit}() and builtin
# atomics instrumentation in .noinstr.text, or use a compiler that can
# implement __no_kcsan to really remove all instrumentation.
depends on STACK_VALIDATION || CC_IS_GCC || CLANG_VERSION >= 140000
depends on !ARCH_WANTS_NO_INSTR || HAVE_NOINSTR_HACK || \
CC_IS_GCC || CLANG_VERSION >= 140000
select OBJTOOL if HAVE_NOINSTR_HACK
help
Enable support for modeling a subset of weak memory, which allows
detecting a subset of data races due to missing memory barriers.

2
lib/Kconfig.ubsan
View File

@ -94,7 +94,7 @@ config UBSAN_UNREACHABLE
bool "Perform checking for unreachable code"
# objtool already handles unreachable checking and gets angry about
# seeing UBSan instrumentation located in unreachable places.
depends on !STACK_VALIDATION
depends on !(OBJTOOL && (STACK_VALIDATION || UNWINDER_ORC || X86_SMAP))
depends on $(cc-option,-fsanitize=unreachable)
help
This option enables -fsanitize=unreachable which checks for control

25
scripts/Makefile.build
View File

@ -222,25 +222,29 @@ cmd_record_mcount = $(if $(findstring $(strip $(CC_FLAGS_FTRACE)),$(_c_flags)),
$(sub_cmd_record_mcount))
endif # CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT
ifdef CONFIG_STACK_VALIDATION
ifdef CONFIG_OBJTOOL
objtool := $(objtree)/tools/objtool/objtool
objtool_args = \
$(if $(CONFIG_UNWINDER_ORC),orc generate,check) \
$(if $(part-of-module), --module) \
$(if $(CONFIG_X86_KERNEL_IBT), --lto --ibt) \
$(if $(CONFIG_FRAME_POINTER),, --no-fp) \
$(if $(CONFIG_GCOV_KERNEL), --no-unreachable) \
$(if $(CONFIG_RETPOLINE), --retpoline) \
--uaccess \
$(if $(CONFIG_HAVE_JUMP_LABEL_HACK), --hacks=jump_label) \
$(if $(CONFIG_HAVE_NOINSTR_HACK), --hacks=noinstr) \
$(if $(CONFIG_X86_KERNEL_IBT), --ibt) \
$(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \
$(if $(CONFIG_SLS), --sls)
$(if $(CONFIG_UNWINDER_ORC), --orc) \
$(if $(CONFIG_RETPOLINE), --retpoline) \
$(if $(CONFIG_SLS), --sls) \
$(if $(CONFIG_STACK_VALIDATION), --stackval) \
$(if $(CONFIG_HAVE_STATIC_CALL_INLINE), --static-call) \
--uaccess \
$(if $(linked-object), --link) \
$(if $(part-of-module), --module) \
$(if $(CONFIG_GCOV_KERNEL), --no-unreachable)
cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@)
cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)
endif # CONFIG_STACK_VALIDATION
endif # CONFIG_OBJTOOL
ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
@ -303,6 +307,7 @@ quiet_cmd_cc_prelink_modules = LD [M] $@
# modules into native code
$(obj)/%.prelink.o: objtool-enabled = y
$(obj)/%.prelink.o: part-of-module := y
$(obj)/%.prelink.o: linked-object := y
$(obj)/%.prelink.o: $(obj)/%.o FORCE
$(call if_changed,cc_prelink_modules)

31
scripts/atomic/gen-atomic-fallback.sh
View File

@ -164,41 +164,44 @@ gen_xchg_fallbacks()
gen_try_cmpxchg_fallback()
{
local cmpxchg="$1"; shift;
local order="$1"; shift;
cat <<EOF
#ifndef arch_try_cmpxchg${order}
#define arch_try_cmpxchg${order}(_ptr, _oldp, _new) \\
#ifndef arch_try_${cmpxchg}${order}
#define arch_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
({ \\
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\
___r = arch_cmpxchg${order}((_ptr), ___o, (_new)); \\
___r = arch_${cmpxchg}${order}((_ptr), ___o, (_new)); \\
if (unlikely(___r != ___o)) \\
*___op = ___r; \\
likely(___r == ___o); \\
})
#endif /* arch_try_cmpxchg${order} */
#endif /* arch_try_${cmpxchg}${order} */
EOF
}
gen_try_cmpxchg_fallbacks()
{
printf "#ifndef arch_try_cmpxchg_relaxed\n"
printf "#ifdef arch_try_cmpxchg\n"
local cmpxchg="$1"; shift;
gen_basic_fallbacks "arch_try_cmpxchg"
printf "#ifndef arch_try_${cmpxchg}_relaxed\n"
printf "#ifdef arch_try_${cmpxchg}\n"
printf "#endif /* arch_try_cmpxchg */\n\n"
gen_basic_fallbacks "arch_try_${cmpxchg}"
printf "#endif /* arch_try_${cmpxchg} */\n\n"
for order in "" "_acquire" "_release" "_relaxed"; do
gen_try_cmpxchg_fallback "${order}"
gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"
done
printf "#else /* arch_try_cmpxchg_relaxed */\n"
printf "#else /* arch_try_${cmpxchg}_relaxed */\n"
gen_order_fallbacks "arch_try_cmpxchg"
gen_order_fallbacks "arch_try_${cmpxchg}"
printf "#endif /* arch_try_cmpxchg_relaxed */\n\n"
printf "#endif /* arch_try_${cmpxchg}_relaxed */\n\n"
}
cat << EOF
@ -218,7 +221,9 @@ for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do
gen_xchg_fallbacks "${xchg}"
done
gen_try_cmpxchg_fallbacks
for cmpxchg in "cmpxchg" "cmpxchg64"; do
gen_try_cmpxchg_fallbacks "${cmpxchg}"
done
grep '^[a-z]' "$1" | while read name meta args; do
gen_proto "${meta}" "${name}" "atomic" "int" ${args}

2
scripts/atomic/gen-atomic-instrumented.sh
View File

@ -166,7 +166,7 @@ grep '^[a-z]' "$1" | while read name meta args; do
done
for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do
for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do
for order in "" "_acquire" "_release" "_relaxed"; do
gen_xchg "${xchg}" "${order}" ""
printf "\n"

150
scripts/faddr2line
View File

@ -44,17 +44,6 @@
set -o errexit
set -o nounset
READELF="${CROSS_COMPILE:-}readelf"
ADDR2LINE="${CROSS_COMPILE:-}addr2line"
SIZE="${CROSS_COMPILE:-}size"
NM="${CROSS_COMPILE:-}nm"
command -v awk >/dev/null 2>&1 || die "awk isn't installed"
command -v ${READELF} >/dev/null 2>&1 || die "readelf isn't installed"
command -v ${ADDR2LINE} >/dev/null 2>&1 || die "addr2line isn't installed"
command -v ${SIZE} >/dev/null 2>&1 || die "size isn't installed"
command -v ${NM} >/dev/null 2>&1 || die "nm isn't installed"
usage() {
echo "usage: faddr2line [--list] <object file> <func+offset> <func+offset>..." >&2
exit 1
@ -69,6 +58,14 @@ die() {
exit 1
}
READELF="${CROSS_COMPILE:-}readelf"
ADDR2LINE="${CROSS_COMPILE:-}addr2line"
AWK="awk"
command -v ${AWK} >/dev/null 2>&1 || die "${AWK} isn't installed"
command -v ${READELF} >/dev/null 2>&1 || die "${READELF} isn't installed"
command -v ${ADDR2LINE} >/dev/null 2>&1 || die "${ADDR2LINE} isn't installed"
# Try to figure out the source directory prefix so we can remove it from the
# addr2line output. HACK ALERT: This assumes that start_kernel() is in
# init/main.c! This only works for vmlinux. Otherwise it falls back to
@ -76,7 +73,7 @@ die() {
find_dir_prefix() {
local objfile=$1
local start_kernel_addr=$(${READELF} -sW $objfile | awk '$8 == "start_kernel" {printf "0x%s", $2}')
local start_kernel_addr=$(${READELF} --symbols --wide $objfile | ${AWK} '$8 == "start_kernel" {printf "0x%s", $2}')
[[ -z $start_kernel_addr ]] && return
local file_line=$(${ADDR2LINE} -e $objfile $start_kernel_addr)
@ -97,86 +94,133 @@ __faddr2line() {
local dir_prefix=$3
local print_warnings=$4
local func=${func_addr%+*}
local sym_name=${func_addr%+*}
local offset=${func_addr#*+}
offset=${offset%/*}
local size=
[[ $func_addr =~ "/" ]] && size=${func_addr#*/}
local user_size=
[[ $func_addr =~ "/" ]] && user_size=${func_addr#*/}
if [[ -z $func ]] || [[ -z $offset ]] || [[ $func = $func_addr ]]; then
if [[ -z $sym_name ]] || [[ -z $offset ]] || [[ $sym_name = $func_addr ]]; then
warn "bad func+offset $func_addr"
DONE=1
return
fi
# Go through each of the object's symbols which match the func name.
# In rare cases there might be duplicates.
file_end=$(${SIZE} -Ax $objfile | awk '$1 == ".text" {print $2}')
while read symbol; do
local fields=($symbol)
local sym_base=0x${fields[0]}
local sym_type=${fields[1]}
local sym_end=${fields[3]}
# In rare cases there might be duplicates, in which case we print all
# matches.
while read line; do
local fields=($line)
local sym_addr=0x${fields[1]}
local sym_elf_size=${fields[2]}
local sym_sec=${fields[6]}
# calculate the size
local sym_size=$(($sym_end - $sym_base))
if [[ -z $sym_size ]] || [[ $sym_size -le 0 ]]; then
warn "bad symbol size: base: $sym_base end: $sym_end"
# Get the section size:
local sec_size=$(${READELF} --section-headers --wide $objfile |
sed 's/\[ /\[/' |
${AWK} -v sec=$sym_sec '$1 == "[" sec "]" { print "0x" $6; exit }')
if [[ -z $sec_size ]]; then
warn "bad section size: section: $sym_sec"
DONE=1
return
fi
# Calculate the symbol size.
#
# Unfortunately we can't use the ELF size, because kallsyms
# also includes the padding bytes in its size calculation. For
# kallsyms, the size calculation is the distance between the
# symbol and the next symbol in a sorted list.
local sym_size
local cur_sym_addr
local found=0
while read line; do
local fields=($line)
cur_sym_addr=0x${fields[1]}
local cur_sym_elf_size=${fields[2]}
local cur_sym_name=${fields[7]:-}
if [[ $cur_sym_addr = $sym_addr ]] &&
[[ $cur_sym_elf_size = $sym_elf_size ]] &&
[[ $cur_sym_name = $sym_name ]]; then
found=1
continue
fi
if [[ $found = 1 ]]; then
sym_size=$(($cur_sym_addr - $sym_addr))
[[ $sym_size -lt $sym_elf_size ]] && continue;
found=2
break
fi
done < <(${READELF} --symbols --wide $objfile | ${AWK} -v sec=$sym_sec '$7 == sec' | sort --key=2)
if [[ $found = 0 ]]; then
warn "can't find symbol: sym_name: $sym_name sym_sec: $sym_sec sym_addr: $sym_addr sym_elf_size: $sym_elf_size"
DONE=1
return
fi
# If nothing was found after the symbol, assume it's the last
# symbol in the section.
[[ $found = 1 ]] && sym_size=$(($sec_size - $sym_addr))
if [[ -z $sym_size ]] || [[ $sym_size -le 0 ]]; then
warn "bad symbol size: sym_addr: $sym_addr cur_sym_addr: $cur_sym_addr"
DONE=1
return
fi
sym_size=0x$(printf %x $sym_size)
# calculate the address
local addr=$(($sym_base + $offset))
# Calculate the section address from user-supplied offset:
local addr=$(($sym_addr + $offset))
if [[ -z $addr ]] || [[ $addr = 0 ]]; then
warn "bad address: $sym_base + $offset"
warn "bad address: $sym_addr + $offset"
DONE=1
return
fi
addr=0x$(printf %x $addr)
# weed out non-function symbols
if [[ $sym_type != t ]] && [[ $sym_type != T ]]; then
# If the user provided a size, make sure it matches the symbol's size:
if [[ -n $user_size ]] && [[ $user_size -ne $sym_size ]]; then
[[ $print_warnings = 1 ]] &&
echo "skipping $func address at $addr due to non-function symbol of type '$sym_type'"
continue
fi
# if the user provided a size, make sure it matches the symbol's size
if [[ -n $size ]] && [[ $size -ne $sym_size ]]; then
[[ $print_warnings = 1 ]] &&
echo "skipping $func address at $addr due to size mismatch ($size != $sym_size)"
echo "skipping $sym_name address at $addr due to size mismatch ($user_size != $sym_size)"
continue;
fi
# make sure the provided offset is within the symbol's range
# Make sure the provided offset is within the symbol's range:
if [[ $offset -gt $sym_size ]]; then
[[ $print_warnings = 1 ]] &&
echo "skipping $func address at $addr due to size mismatch ($offset > $sym_size)"
echo "skipping $sym_name address at $addr due to size mismatch ($offset > $sym_size)"
continue
fi
# separate multiple entries with a blank line
# In case of duplicates or multiple addresses specified on the
# cmdline, separate multiple entries with a blank line:
[[ $FIRST = 0 ]] && echo
FIRST=0
# pass real address to addr2line
echo "$func+$offset/$sym_size:"
local file_lines=$(${ADDR2LINE} -fpie $objfile $addr | sed "s; $dir_prefix\(\./\)*; ;")
[[ -z $file_lines ]] && return
echo "$sym_name+$offset/$sym_size:"
# Pass section address to addr2line and strip absolute paths
# from the output:
local output=$(${ADDR2LINE} -fpie $objfile $addr | sed "s; $dir_prefix\(\./\)*; ;")
[[ -z $output ]] && continue
# Default output (non --list):
if [[ $LIST = 0 ]]; then
echo "$file_lines" | while read -r line
echo "$output" | while read -r line
do
echo $line
done
DONE=1;
return
continue
fi
# show each line with context
echo "$file_lines" | while read -r line
# For --list, show each line with its corresponding source code:
echo "$output" | while read -r line
do
echo
echo $line
@ -184,12 +228,12 @@ __faddr2line() {
n1=$[$n-5]
n2=$[$n+5]
f=$(echo $line | sed 's/.*at \(.\+\):.*/\1/g')
awk 'NR>=strtonum("'$n1'") && NR<=strtonum("'$n2'") { if (NR=='$n') printf(">%d<", NR); else printf(" %d ", NR); printf("\t%s\n", $0)}' $f
${AWK} 'NR>=strtonum("'$n1'") && NR<=strtonum("'$n2'") { if (NR=='$n') printf(">%d<", NR); else printf(" %d ", NR); printf("\t%s\n", $0)}' $f
done
DONE=1
done < <(${NM} -n $objfile | awk -v fn=$func -v end=$file_end '$3 == fn { found=1; line=$0; start=$1; next } found == 1 { found=0; print line, "0x"$1 } END {if (found == 1) print line, end; }')
done < <(${READELF} --symbols --wide $objfile | ${AWK} -v fn=$sym_name '$4 == "FUNC" && $8 == fn')
}
[[ $# -lt 2 ]] && usage

66
scripts/link-vmlinux.sh
View File

@ -108,16 +108,22 @@ objtool_link()
local objtoolcmd;
local objtoolopt;
if is_enabled CONFIG_STACK_VALIDATION && \
( is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT ); then
if ! is_enabled CONFIG_OBJTOOL; then
return;
fi
# Don't perform vmlinux validation unless explicitly requested,
# but run objtool on vmlinux.o now that we have an object file.
if is_enabled CONFIG_UNWINDER_ORC; then
objtoolcmd="orc generate"
if is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT; then
# For LTO and IBT, objtool doesn't run on individual
# translation units. Run everything on vmlinux instead.
if is_enabled CONFIG_HAVE_JUMP_LABEL_HACK; then
objtoolopt="${objtoolopt} --hacks=jump_label"
fi
objtoolopt="${objtoolopt} --lto"
if is_enabled CONFIG_HAVE_NOINSTR_HACK; then
objtoolopt="${objtoolopt} --hacks=noinstr"
fi
if is_enabled CONFIG_X86_KERNEL_IBT; then
objtoolopt="${objtoolopt} --ibt"
@ -126,34 +132,44 @@ objtool_link()
if is_enabled CONFIG_FTRACE_MCOUNT_USE_OBJTOOL; then
objtoolopt="${objtoolopt} --mcount"
fi
fi
if is_enabled CONFIG_VMLINUX_VALIDATION; then
objtoolopt="${objtoolopt} --noinstr"
fi
if is_enabled CONFIG_UNWINDER_ORC; then
objtoolopt="${objtoolopt} --orc"
fi
if [ -n "${objtoolopt}" ]; then
if [ -z "${objtoolcmd}" ]; then
objtoolcmd="check"
fi
objtoolopt="${objtoolopt} --vmlinux"
if ! is_enabled CONFIG_FRAME_POINTER; then
objtoolopt="${objtoolopt} --no-fp"
fi
if is_enabled CONFIG_GCOV_KERNEL; then
objtoolopt="${objtoolopt} --no-unreachable"
fi
if is_enabled CONFIG_RETPOLINE; then
objtoolopt="${objtoolopt} --retpoline"
fi
objtoolopt="${objtoolopt} --uaccess"
if is_enabled CONFIG_SLS; then
objtoolopt="${objtoolopt} --sls"
fi
if is_enabled CONFIG_STACK_VALIDATION; then
objtoolopt="${objtoolopt} --stackval"
fi
if is_enabled CONFIG_HAVE_STATIC_CALL_INLINE; then
objtoolopt="${objtoolopt} --static-call"
fi
objtoolopt="${objtoolopt} --uaccess"
fi
if is_enabled CONFIG_NOINSTR_VALIDATION; then
objtoolopt="${objtoolopt} --noinstr"
fi
if [ -n "${objtoolopt}" ]; then
if is_enabled CONFIG_GCOV_KERNEL; then
objtoolopt="${objtoolopt} --no-unreachable"
fi
objtoolopt="${objtoolopt} --link"
info OBJTOOL ${1}
tools/objtool/objtool ${objtoolcmd} ${objtoolopt} ${1}
tools/objtool/objtool ${objtoolopt} ${1}
fi
}

29
scripts/objdump-func Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
#
# Disassemble a single function.
#
# usage: objdump-func <file> <func>
set -o errexit
set -o nounset
OBJDUMP="${CROSS_COMPILE:-}objdump"
command -v gawk >/dev/null 2>&1 || die "gawk isn't installed"
usage() {
echo "usage: objdump-func <file> <func>" >&2
exit 1
}
[[ $# -lt 2 ]] && usage
OBJ=$1; shift
FUNC=$1; shift
# Secret feature to allow adding extra objdump args at the end
EXTRA_ARGS=$@
# Note this also matches compiler-added suffixes like ".cold", etc
${OBJDUMP} -wdr $EXTRA_ARGS $OBJ | gawk -M -v f=$FUNC '/^$/ { P=0; } $0 ~ "<" f "(\\..*)?>:" { P=1; O=strtonum("0x" $1); } { if (P) { o=strtonum("0x" $1); printf("%04x ", o-O); print $0; } }'

2
scripts/package/builddeb
View File

@ -67,7 +67,7 @@ deploy_kernel_headers () {
) > debian/hdrsrcfiles
{
if is_enabled CONFIG_STACK_VALIDATION; then
if is_enabled CONFIG_OBJTOOL; then
echo tools/objtool/objtool
fi

2
tools/arch/x86/include/asm/amd-ibs.h
View File

@ -49,7 +49,7 @@ union ibs_op_ctl {
};
};
/* MSR 0xc0011035: IBS Op Data 2 */
/* MSR 0xc0011035: IBS Op Data 1 */
union ibs_op_data {
__u64 val;
struct {

6
tools/include/linux/objtool.h
View File

@ -38,7 +38,7 @@ struct unwind_hint {
#define UNWIND_HINT_TYPE_REGS_PARTIAL 2
#define UNWIND_HINT_TYPE_FUNC 3
#ifdef CONFIG_STACK_VALIDATION
#ifdef CONFIG_OBJTOOL
#include <asm/asm.h>
@ -159,7 +159,7 @@ struct unwind_hint {
#endif /* __ASSEMBLY__ */
#else /* !CONFIG_STACK_VALIDATION */
#else /* !CONFIG_OBJTOOL */
#ifndef __ASSEMBLY__
@ -181,6 +181,6 @@ struct unwind_hint {
.endm
#endif
#endif /* CONFIG_STACK_VALIDATION */
#endif /* CONFIG_OBJTOOL */
#endif /* _LINUX_OBJTOOL_H */

17
tools/lib/subcmd/parse-options.c
View File

@ -806,9 +806,9 @@ static int option__cmp(const void *va, const void *vb)
static struct option *options__order(const struct option *opts)
{
int nr_opts = 0, len;
int nr_opts = 0, nr_group = 0, len;
const struct option *o = opts;
struct option *ordered;
struct option *opt, *ordered, *group;
for (o = opts; o->type != OPTION_END; o++)
++nr_opts;
@ -819,7 +819,18 @@ static struct option *options__order(const struct option *opts)
goto out;
memcpy(ordered, opts, len);
qsort(ordered, nr_opts, sizeof(*o), option__cmp);
/* sort each option group individually */
for (opt = group = ordered; opt->type != OPTION_END; opt++) {
if (opt->type == OPTION_GROUP) {
qsort(group, nr_group, sizeof(*opt), option__cmp);
group = opt + 1;
nr_group = 0;
continue;
}
nr_group++;
}
qsort(group, nr_group, sizeof(*opt), option__cmp);
out:
return ordered;
}

12
tools/objtool/Build
View File

@ -2,17 +2,15 @@ objtool-y += arch/$(SRCARCH)/
objtool-y += weak.o
objtool-$(SUBCMD_CHECK) += check.o
objtool-$(SUBCMD_CHECK) += special.o
objtool-$(SUBCMD_ORC) += check.o
objtool-$(SUBCMD_ORC) += orc_gen.o
objtool-$(SUBCMD_ORC) += orc_dump.o
objtool-y += check.o
objtool-y += special.o
objtool-y += builtin-check.o
objtool-y += builtin-orc.o
objtool-y += elf.o
objtool-y += objtool.o
objtool-$(BUILD_ORC) += orc_gen.o
objtool-$(BUILD_ORC) += orc_dump.o
objtool-y += libstring.o
objtool-y += libctype.o
objtool-y += str_error_r.o

122
tools/objtool/Documentation/stack-validation.txt → tools/objtool/Documentation/objtool.txt
View File

@ -1,15 +1,103 @@
Compile-time stack metadata validation
======================================
Objtool
=======
The kernel CONFIG_OBJTOOL option enables a host tool named 'objtool'
which runs at compile time. It can do various validations and
transformations on .o files.
Objtool has become an integral part of the x86-64 kernel toolchain. The
kernel depends on it for a variety of security and performance features
(and other types of features as well).
Overview
Features
--------
The kernel CONFIG_STACK_VALIDATION option enables a host tool named
objtool which runs at compile time. It has a "check" subcommand which
analyzes every .o file and ensures the validity of its stack metadata.
It enforces a set of rules on asm code and C inline assembly code so
that stack traces can be reliable.
Objtool has the following features:
- Stack unwinding metadata validation -- useful for helping to ensure
stack traces are reliable for live patching
- ORC unwinder metadata generation -- a faster and more precise
alternative to frame pointer based unwinding
- Retpoline validation -- ensures that all indirect calls go through
retpoline thunks, for Spectre v2 mitigations
- Retpoline call site annotation -- annotates all retpoline thunk call
sites, enabling the kernel to patch them inline, to prevent "thunk
funneling" for both security and performance reasons
- Non-instrumentation validation -- validates non-instrumentable
("noinstr") code rules, preventing instrumentation in low-level C
entry code
- Static call annotation -- annotates static call sites, enabling the
kernel to implement inline static calls, a faster alternative to some
indirect branches
- Uaccess validation -- validates uaccess rules for a proper
implementation of Supervisor Mode Access Protection (SMAP)
- Straight Line Speculation validation -- validates certain SLS
mitigations
- Indirect Branch Tracking validation -- validates Intel CET IBT rules
to ensure that all functions referenced by function pointers have
corresponding ENDBR instructions
- Indirect Branch Tracking annotation -- annotates unused ENDBR
instruction sites, enabling the kernel to "seal" them (replace them
with NOPs) to further harden IBT
- Function entry annotation -- annotates function entries, enabling
kernel function tracing
- Other toolchain hacks which will go unmentioned at this time...
Each feature can be enabled individually or in combination using the
objtool cmdline.
Objects
-------
Typically, objtool runs on every translation unit (TU, aka ".o file") in
the kernel. If a TU is part of a kernel module, the '--module' option
is added.
However:
- If noinstr validation is enabled, it also runs on vmlinux.o, with all
options removed and '--noinstr' added.
- If IBT or LTO is enabled, it doesn't run on TUs at all. Instead it
runs on vmlinux.o and linked modules, with all options.
In summary:
A) Legacy mode:
TU: objtool [--module] <options>
vmlinux: N/A
module: N/A
B) CONFIG_NOINSTR_VALIDATION=y && !(CONFIG_X86_KERNEL_IBT=y || CONFIG_LTO=y):
TU: objtool [--module] <options> // no --noinstr
vmlinux: objtool --noinstr // other options removed
module: N/A
C) CONFIG_X86_KERNEL_IBT=y || CONFIG_LTO=y:
TU: N/A
vmlinux: objtool --noinstr <options>
module: objtool --module --noinstr <options>
Stack validation
----------------
Objtool's stack validation feature analyzes every .o file and ensures
the validity of its stack metadata. It enforces a set of rules on asm
code and C inline assembly code so that stack traces can be reliable.
For each function, it recursively follows all possible code paths and
validates the correct frame pointer state at each instruction.
@ -20,14 +108,6 @@ alternative execution paths to a given instruction (or set of
instructions). Similarly, it knows how to follow switch statements, for
which gcc sometimes uses jump tables.
(Objtool also has an 'orc generate' subcommand which generates debuginfo
for the ORC unwinder. See Documentation/x86/orc-unwinder.rst in the
kernel tree for more details.)
Why do we need stack metadata validation?
-----------------------------------------
Here are some of the benefits of validating stack metadata:
a) More reliable stack traces for frame pointer enabled kernels
@ -113,9 +193,6 @@ c) Higher live patching compatibility rate
For more details, see the livepatch documentation in the Linux kernel
source tree at Documentation/livepatch/livepatch.rst.
Rules
-----
To achieve the validation, objtool enforces the following rules:
1. Each callable function must be annotated as such with the ELF
@ -177,7 +254,8 @@ Another possible cause for errors in C code is if the Makefile removes
-fno-omit-frame-pointer or adds -fomit-frame-pointer to the gcc options.
Here are some examples of common warnings reported by objtool, what
they mean, and suggestions for how to fix them.
they mean, and suggestions for how to fix them. When in doubt, ping
the objtool maintainers.
1. file.o: warning: objtool: func()+0x128: call without frame pointer save/setup
@ -358,3 +436,7 @@ ignore it:
OBJECT_FILES_NON_STANDARD := y
to the Makefile.
NOTE: OBJECT_FILES_NON_STANDARD doesn't work for link time validation of
vmlinux.o or a linked module. So it should only be used for files which
aren't linked into vmlinux or a module.

10
tools/objtool/Makefile
View File

@ -39,15 +39,13 @@ CFLAGS += $(if $(elfshdr),,-DLIBELF_USE_DEPRECATED)
AWK = awk
SUBCMD_CHECK := n
SUBCMD_ORC := n
BUILD_ORC := n
ifeq ($(SRCARCH),x86)
SUBCMD_CHECK := y
SUBCMD_ORC := y
BUILD_ORC := y
endif
export SUBCMD_CHECK SUBCMD_ORC
export BUILD_ORC
export srctree OUTPUT CFLAGS SRCARCH AWK
include $(srctree)/tools/build/Makefile.include
@ -65,7 +63,7 @@ $(LIBSUBCMD): fixdep FORCE
clean:
$(call QUIET_CLEAN, objtool) $(RM) $(OBJTOOL)
$(Q)find $(OUTPUT) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete
$(Q)$(RM) $(OUTPUT)arch/x86/inat-tables.c $(OUTPUT)fixdep
$(Q)$(RM) $(OUTPUT)arch/x86/lib/inat-tables.c $(OUTPUT)fixdep $(LIBSUBCMD)
FORCE:

2
tools/objtool/arch/x86/decode.c
View File

@ -581,7 +581,7 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec
break;
case 0xc7: /* mov imm, r/m */
if (!noinstr)
if (!opts.noinstr)
break;
if (insn.length == 3+4+4 && !strncmp(sec->name, ".init.text", 10)) {

2
tools/objtool/arch/x86/special.c
View File

@ -20,7 +20,7 @@ void arch_handle_alternative(unsigned short feature, struct special_alt *alt)
* find paths that see the STAC but take the NOP instead of
* CLAC and the other way around.
*/
if (uaccess)
if (opts.uaccess)
alt->skip_orig = true;
else
alt->skip_alt = true;

156
tools/objtool/builtin-check.c
View File

@ -3,28 +3,21 @@
* Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
*/
/*
* objtool check:
*
* This command analyzes every .o file and ensures the validity of its stack
* trace metadata. It enforces a set of rules on asm code and C inline
* assembly code so that stack traces can be reliable.
*
* For more information, see tools/objtool/Documentation/stack-validation.txt.
*/
#include <subcmd/parse-options.h>
#include <string.h>
#include <stdlib.h>
#include <objtool/builtin.h>
#include <objtool/objtool.h>
bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
lto, vmlinux, mcount, noinstr, backup, sls, dryrun,
ibt;
#define ERROR(format, ...) \
fprintf(stderr, \
"error: objtool: " format "\n", \
##__VA_ARGS__)
struct opts opts;
static const char * const check_usage[] = {
"objtool check [<options>] file.o",
"objtool <actions> [<options>] file.o",
NULL,
};
@ -33,22 +26,64 @@ static const char * const env_usage[] = {
NULL,
};
static int parse_dump(const struct option *opt, const char *str, int unset)
{
if (!str || !strcmp(str, "orc")) {
opts.dump_orc = true;
return 0;
}
return -1;
}
static int parse_hacks(const struct option *opt, const char *str, int unset)
{
bool found = false;
/*
* Use strstr() as a lazy method of checking for comma-separated
* options.
*
* No string provided == enable all options.
*/
if (!str || strstr(str, "jump_label")) {
opts.hack_jump_label = true;
found = true;
}
if (!str || strstr(str, "noinstr")) {
opts.hack_noinstr = true;
found = true;
}
return found ? 0 : -1;
}
const struct option check_options[] = {
OPT_BOOLEAN('f', "no-fp", &no_fp, "Skip frame pointer validation"),
OPT_BOOLEAN('u', "no-unreachable", &no_unreachable, "Skip 'unreachable instruction' warnings"),
OPT_BOOLEAN('r', "retpoline", &retpoline, "Validate retpoline assumptions"),
OPT_BOOLEAN('m', "module", &module, "Indicates the object will be part of a kernel module"),
OPT_BOOLEAN('b', "backtrace", &backtrace, "unwind on error"),
OPT_BOOLEAN('a', "uaccess", &uaccess, "enable uaccess checking"),
OPT_BOOLEAN('s', "stats", &stats, "print statistics"),
OPT_BOOLEAN(0, "lto", &lto, "whole-archive like runs"),
OPT_BOOLEAN('n', "noinstr", &noinstr, "noinstr validation for vmlinux.o"),
OPT_BOOLEAN('l', "vmlinux", &vmlinux, "vmlinux.o validation"),
OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"),
OPT_BOOLEAN('B', "backup", &backup, "create .orig files before modification"),
OPT_BOOLEAN('S', "sls", &sls, "validate straight-line-speculation"),
OPT_BOOLEAN(0, "dry-run", &dryrun, "don't write the modifications"),
OPT_BOOLEAN(0, "ibt", &ibt, "validate ENDBR placement"),
OPT_GROUP("Actions:"),
OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr", "patch toolchain bugs/limitations", parse_hacks),
OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"),
OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"),
OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"),
OPT_BOOLEAN('o', "orc", &opts.orc, "generate ORC metadata"),
OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"),
OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"),
OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"),
OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"),
OPT_BOOLEAN('u', "uaccess", &opts.uaccess, "validate uaccess rules for SMAP"),
OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump),
OPT_GROUP("Options:"),
OPT_BOOLEAN(0, "backtrace", &opts.backtrace, "unwind on error"),
OPT_BOOLEAN(0, "backup", &opts.backup, "create .orig files before modification"),
OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"),
OPT_BOOLEAN(0, "link", &opts.link, "object is a linked object"),
OPT_BOOLEAN(0, "module", &opts.module, "object is part of a kernel module"),
OPT_BOOLEAN(0, "no-unreachable", &opts.no_unreachable, "skip 'unreachable instruction' warnings"),
OPT_BOOLEAN(0, "sec-address", &opts.sec_address, "print section addresses in warnings"),
OPT_BOOLEAN(0, "stats", &opts.stats, "print statistics"),
OPT_END(),
};
@ -79,7 +114,59 @@ int cmd_parse_options(int argc, const char **argv, const char * const usage[])
return argc;
}
int cmd_check(int argc, const char **argv)
static bool opts_valid(void)
{
if (opts.hack_jump_label ||
opts.hack_noinstr ||
opts.ibt ||
opts.mcount ||
opts.noinstr ||
opts.orc ||
opts.retpoline ||
opts.sls ||
opts.stackval ||
opts.static_call ||
opts.uaccess) {
if (opts.dump_orc) {
ERROR("--dump can't be combined with other options");
return false;
}
return true;
}
if (opts.dump_orc)
return true;
ERROR("At least one command required");
return false;
}
static bool link_opts_valid(struct objtool_file *file)
{
if (opts.link)
return true;
if (has_multiple_files(file->elf)) {
ERROR("Linked object detected, forcing --link");
opts.link = true;
return true;
}
if (opts.noinstr) {
ERROR("--noinstr requires --link");
return false;
}
if (opts.ibt) {
ERROR("--ibt requires --link");
return false;
}
return true;
}
int objtool_run(int argc, const char **argv)
{
const char *objname;
struct objtool_file *file;
@ -88,10 +175,19 @@ int cmd_check(int argc, const char **argv)
argc = cmd_parse_options(argc, argv, check_usage);
objname = argv[0];
if (!opts_valid())
return 1;
if (opts.dump_orc)
return orc_dump(objname);
file = objtool_open_read(objname);
if (!file)
return 1;
if (!link_opts_valid(file))
return 1;
ret = check(file);
if (ret)
return ret;

73
tools/objtool/builtin-orc.c
View File

@ -1,73 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
*/
/*
* objtool orc:
*
* This command analyzes a .o file and adds .orc_unwind and .orc_unwind_ip
* sections to it, which is used by the in-kernel ORC unwinder.
*
* This command is a superset of "objtool check".
*/
#include <string.h>
#include <objtool/builtin.h>
#include <objtool/objtool.h>
static const char *orc_usage[] = {
"objtool orc generate [<options>] file.o",
"objtool orc dump file.o",
NULL,
};
int cmd_orc(int argc, const char **argv)
{
const char *objname;
argc--; argv++;
if (argc <= 0)
usage_with_options(orc_usage, check_options);
if (!strncmp(argv[0], "gen", 3)) {
struct objtool_file *file;
int ret;
argc = cmd_parse_options(argc, argv, orc_usage);
objname = argv[0];
file = objtool_open_read(objname);
if (!file)
return 1;
ret = check(file);
if (ret)
return ret;
if (list_empty(&file->insn_list))
return 0;
ret = orc_create(file);
if (ret)
return ret;
if (!file->elf->changed)
return 0;
return elf_write(file->elf);
}
if (!strcmp(argv[0], "dump")) {
if (argc != 2)
usage_with_options(orc_usage, check_options);
objname = argv[1];
return orc_dump(objname);
}
usage_with_options(orc_usage, check_options);
return 0;
}

471
tools/objtool/check.c
View File

@ -5,6 +5,7 @@
#include <string.h>
#include <stdlib.h>
#include <inttypes.h>
#include <sys/mman.h>
#include <arch/elf.h>
@ -263,7 +264,8 @@ static void init_cfi_state(struct cfi_state *cfi)
cfi->drap_offset = -1;
}
static void init_insn_state(struct insn_state *state, struct section *sec)
static void init_insn_state(struct objtool_file *file, struct insn_state *state,
struct section *sec)
{
memset(state, 0, sizeof(*state));
init_cfi_state(&state->cfi);
@ -273,7 +275,7 @@ static void init_insn_state(struct insn_state *state, struct section *sec)
* not correctly determine insn->call_dest->sec (external symbols do
* not have a section).
*/
if (vmlinux && noinstr && sec)
if (opts.link && opts.noinstr && sec)
state->noinstr = sec->noinstr;
}
@ -339,7 +341,7 @@ static void *cfi_hash_alloc(unsigned long size)
if (cfi_hash == (void *)-1L) {
WARN("mmap fail cfi_hash");
cfi_hash = NULL;
} else if (stats) {
} else if (opts.stats) {
printf("cfi_bits: %d\n", cfi_bits);
}
@ -434,7 +436,7 @@ static int decode_instructions(struct objtool_file *file)
}
}
if (stats)
if (opts.stats)
printf("nr_insns: %lu\n", nr_insns);
return 0;
@ -497,7 +499,7 @@ static int init_pv_ops(struct objtool_file *file)
struct symbol *sym;
int idx, nr;
if (!noinstr)
if (!opts.noinstr)
return 0;
file->pv_ops = NULL;
@ -560,12 +562,12 @@ static int add_dead_ends(struct objtool_file *file)
else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
insn = find_last_insn(file, reloc->sym->sec);
if (!insn) {
WARN("can't find unreachable insn at %s+0x%lx",
WARN("can't find unreachable insn at %s+0x%" PRIx64,
reloc->sym->sec->name, reloc->addend);
return -1;
}
} else {
WARN("can't find unreachable insn at %s+0x%lx",
WARN("can't find unreachable insn at %s+0x%" PRIx64,
reloc->sym->sec->name, reloc->addend);
return -1;
}
@ -595,12 +597,12 @@ static int add_dead_ends(struct objtool_file *file)
else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
insn = find_last_insn(file, reloc->sym->sec);
if (!insn) {
WARN("can't find reachable insn at %s+0x%lx",
WARN("can't find reachable insn at %s+0x%" PRIx64,
reloc->sym->sec->name, reloc->addend);
return -1;
}
} else {
WARN("can't find reachable insn at %s+0x%lx",
WARN("can't find reachable insn at %s+0x%" PRIx64,
reloc->sym->sec->name, reloc->addend);
return -1;
}
@ -668,7 +670,7 @@ static int create_static_call_sections(struct objtool_file *file)
key_sym = find_symbol_by_name(file->elf, tmp);
if (!key_sym) {
if (!module) {
if (!opts.module) {
WARN("static_call: can't find static_call_key symbol: %s", tmp);
return -1;
}
@ -761,7 +763,7 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file)
list_for_each_entry(insn, &file->endbr_list, call_node)
idx++;
if (stats) {
if (opts.stats) {
printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
printf("ibt: superfluous ENDBR: %d\n", idx);
@ -1028,7 +1030,7 @@ static void add_uaccess_safe(struct objtool_file *file)
struct symbol *func;
const char **name;
if (!uaccess)
if (!opts.uaccess)
return;
for (name = uaccess_safe_builtin; *name; name++) {
@ -1144,7 +1146,7 @@ static void annotate_call_site(struct objtool_file *file,
* attribute so they need a little help, NOP out any such calls from
* noinstr text.
*/
if (insn->sec->noinstr && sym->profiling_func) {
if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
if (reloc) {
reloc->type = R_NONE;
elf_write_reloc(file->elf, reloc);
@ -1170,7 +1172,7 @@ static void annotate_call_site(struct objtool_file *file,
return;
}
if (mcount && sym->fentry) {
if (opts.mcount && sym->fentry) {
if (sibling)
WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
@ -1256,7 +1258,7 @@ static bool is_first_func_insn(struct objtool_file *file, struct instruction *in
if (insn->offset == insn->func->offset)
return true;
if (ibt) {
if (opts.ibt) {
struct instruction *prev = prev_insn_same_sym(file, insn);
if (prev && prev->type == INSN_ENDBR &&
@ -1592,7 +1594,7 @@ static int handle_jump_alt(struct objtool_file *file,
return -1;
}
if (special_alt->key_addend & 2) {
if (opts.hack_jump_label && special_alt->key_addend & 2) {
struct reloc *reloc = insn_reloc(file, orig_insn);
if (reloc) {
@ -1699,7 +1701,7 @@ static int add_special_section_alts(struct objtool_file *file)
free(special_alt);
}
if (stats) {
if (opts.stats) {
printf("jl\\\tNOP\tJMP\n");
printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
@ -1945,7 +1947,7 @@ static int read_unwind_hints(struct objtool_file *file)
insn->hint = true;
if (ibt && hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
if (opts.ibt && hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
if (sym && sym->bind == STB_GLOBAL &&
@ -2806,7 +2808,7 @@ static int update_cfi_state(struct instruction *insn,
}
/* detect when asm code uses rbp as a scratch register */
if (!no_fp && insn->func && op->src.reg == CFI_BP &&
if (opts.stackval && insn->func && op->src.reg == CFI_BP &&
cfa->base != CFI_BP)
cfi->bp_scratch = true;
break;
@ -3182,114 +3184,6 @@ static struct instruction *next_insn_to_validate(struct objtool_file *file,
return next_insn_same_sec(file, insn);
}
static struct instruction *
validate_ibt_reloc(struct objtool_file *file, struct reloc *reloc)
{
struct instruction *dest;
struct section *sec;
unsigned long off;
sec = reloc->sym->sec;
off = reloc->sym->offset;
if ((reloc->sec->base->sh.sh_flags & SHF_EXECINSTR) &&
(reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32))
off += arch_dest_reloc_offset(reloc->addend);
else
off += reloc->addend;
dest = find_insn(file, sec, off);
if (!dest)
return NULL;
if (dest->type == INSN_ENDBR) {
if (!list_empty(&dest->call_node))
list_del_init(&dest->call_node);
return NULL;
}
if (reloc->sym->static_call_tramp)
return NULL;
return dest;
}
static void warn_noendbr(const char *msg, struct section *sec, unsigned long offset,
struct instruction *dest)
{
WARN_FUNC("%srelocation to !ENDBR: %s", sec, offset, msg,
offstr(dest->sec, dest->offset));
}
static void validate_ibt_dest(struct objtool_file *file, struct instruction *insn,
struct instruction *dest)
{
if (dest->func && dest->func == insn->func) {
/*
* Anything from->to self is either _THIS_IP_ or IRET-to-self.
*
* There is no sane way to annotate _THIS_IP_ since the compiler treats the
* relocation as a constant and is happy to fold in offsets, skewing any
* annotation we do, leading to vast amounts of false-positives.
*
* There's also compiler generated _THIS_IP_ through KCOV and
* such which we have no hope of annotating.
*
* As such, blanket accept self-references without issue.
*/
return;
}
if (dest->noendbr)
return;
warn_noendbr("", insn->sec, insn->offset, dest);
}
static void validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
{
struct instruction *dest;
struct reloc *reloc;
switch (insn->type) {
case INSN_CALL:
case INSN_CALL_DYNAMIC:
case INSN_JUMP_CONDITIONAL:
case INSN_JUMP_UNCONDITIONAL:
case INSN_JUMP_DYNAMIC:
case INSN_JUMP_DYNAMIC_CONDITIONAL:
case INSN_RETURN:
/*
* We're looking for code references setting up indirect code
* flow. As such, ignore direct code flow and the actual
* dynamic branches.
*/
return;
case INSN_NOP:
/*
* handle_group_alt() will create INSN_NOP instruction that
* don't belong to any section, ignore all NOP since they won't
* carry a (useful) relocation anyway.
*/
return;
default:
break;
}
for (reloc = insn_reloc(file, insn);
reloc;
reloc = find_reloc_by_dest_range(file->elf, insn->sec,
reloc->offset + 1,
(insn->offset + insn->len) - (reloc->offset + 1))) {
dest = validate_ibt_reloc(file, reloc);
if (dest)
validate_ibt_dest(file, insn, dest);
}
}
/*
* Follow the branch starting at the given instruction, and recursively follow
* any other branches (jumps). Meanwhile, track the frame pointer state at
@ -3363,7 +3257,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
ret = validate_branch(file, func, alt->insn, state);
if (ret) {
if (backtrace)
if (opts.backtrace)
BT_FUNC("(alt)", insn);
return ret;
}
@ -3379,11 +3273,6 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
switch (insn->type) {
case INSN_RETURN:
if (sls && !insn->retpoline_safe &&
next_insn && next_insn->type != INSN_TRAP) {
WARN_FUNC("missing int3 after ret",
insn->sec, insn->offset);
}
return validate_return(func, insn, &state);
case INSN_CALL:
@ -3392,7 +3281,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
if (ret)
return ret;
if (!no_fp && func && !is_fentry_call(insn) &&
if (opts.stackval && func && !is_fentry_call(insn) &&
!has_valid_stack_frame(&state)) {
WARN_FUNC("call without frame pointer save/setup",
sec, insn->offset);
@ -3415,7 +3304,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
ret = validate_branch(file, func,
insn->jump_dest, state);
if (ret) {
if (backtrace)
if (opts.backtrace)
BT_FUNC("(branch)", insn);
return ret;
}
@ -3427,13 +3316,6 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
break;
case INSN_JUMP_DYNAMIC:
if (sls && !insn->retpoline_safe &&
next_insn && next_insn->type != INSN_TRAP) {
WARN_FUNC("missing int3 after indirect jump",
insn->sec, insn->offset);
}
/* fallthrough */
case INSN_JUMP_DYNAMIC_CONDITIONAL:
if (is_sibling_call(insn)) {
ret = validate_sibling_call(file, insn, &state);
@ -3499,9 +3381,6 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
break;
}
if (ibt)
validate_ibt_insn(file, insn);
if (insn->dead_end)
return 0;
@ -3528,7 +3407,7 @@ static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
if (!file->hints)
return 0;
init_insn_state(&state, sec);
init_insn_state(file, &state, sec);
if (sec) {
insn = find_insn(file, sec, 0);
@ -3541,7 +3420,7 @@ static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
if (insn->hint && !insn->visited && !insn->ignore) {
ret = validate_branch(file, insn->func, insn, state);
if (ret && backtrace)
if (ret && opts.backtrace)
BT_FUNC("<=== (hint)", insn);
warnings += ret;
}
@ -3571,7 +3450,7 @@ static int validate_retpoline(struct objtool_file *file)
* loaded late, they very much do need retpoline in their
* .init.text
*/
if (!strcmp(insn->sec->name, ".init.text") && !module)
if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
continue;
WARN_FUNC("indirect %s found in RETPOLINE build",
@ -3614,14 +3493,14 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio
return true;
/*
* Whole archive runs might encounder dead code from weak symbols.
* Whole archive runs might encounter dead code from weak symbols.
* This is where the linker will have dropped the weak symbol in
* favour of a regular symbol, but leaves the code in place.
*
* In this case we'll find a piece of code (whole function) that is not
* covered by a !section symbol. Ignore them.
*/
if (!insn->func && lto) {
if (opts.link && !insn->func) {
int size = find_symbol_hole_containing(insn->sec, insn->offset);
unsigned long end = insn->offset + size;
@ -3728,7 +3607,7 @@ static int validate_symbol(struct objtool_file *file, struct section *sec,
state->uaccess = sym->uaccess_safe;
ret = validate_branch(file, insn->func, insn, *state);
if (ret && backtrace)
if (ret && opts.backtrace)
BT_FUNC("<=== (sym)", insn);
return ret;
}
@ -3743,7 +3622,7 @@ static int validate_section(struct objtool_file *file, struct section *sec)
if (func->type != STT_FUNC)
continue;
init_insn_state(&state, sec);
init_insn_state(file, &state, sec);
set_func_state(&state.cfi);
warnings += validate_symbol(file, sec, func, &state);
@ -3752,7 +3631,7 @@ static int validate_section(struct objtool_file *file, struct section *sec)
return warnings;
}
static int validate_vmlinux_functions(struct objtool_file *file)
static int validate_noinstr_sections(struct objtool_file *file)
{
struct section *sec;
int warnings = 0;
@ -3787,48 +3666,208 @@ static int validate_functions(struct objtool_file *file)
return warnings;
}
static void mark_endbr_used(struct instruction *insn)
{
if (!list_empty(&insn->call_node))
list_del_init(&insn->call_node);
}
static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
{
struct instruction *dest;
struct reloc *reloc;
unsigned long off;
int warnings = 0;
/*
* Looking for function pointer load relocations. Ignore
* direct/indirect branches:
*/
switch (insn->type) {
case INSN_CALL:
case INSN_CALL_DYNAMIC:
case INSN_JUMP_CONDITIONAL:
case INSN_JUMP_UNCONDITIONAL:
case INSN_JUMP_DYNAMIC:
case INSN_JUMP_DYNAMIC_CONDITIONAL:
case INSN_RETURN:
case INSN_NOP:
return 0;
default:
break;
}
for (reloc = insn_reloc(file, insn);
reloc;
reloc = find_reloc_by_dest_range(file->elf, insn->sec,
reloc->offset + 1,
(insn->offset + insn->len) - (reloc->offset + 1))) {
/*
* static_call_update() references the trampoline, which
* doesn't have (or need) ENDBR. Skip warning in that case.
*/
if (reloc->sym->static_call_tramp)
continue;
off = reloc->sym->offset;
if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
off += arch_dest_reloc_offset(reloc->addend);
else
off += reloc->addend;
dest = find_insn(file, reloc->sym->sec, off);
if (!dest)
continue;
if (dest->type == INSN_ENDBR) {
mark_endbr_used(dest);
continue;
}
if (dest->func && dest->func == insn->func) {
/*
* Anything from->to self is either _THIS_IP_ or
* IRET-to-self.
*
* There is no sane way to annotate _THIS_IP_ since the
* compiler treats the relocation as a constant and is
* happy to fold in offsets, skewing any annotation we
* do, leading to vast amounts of false-positives.
*
* There's also compiler generated _THIS_IP_ through
* KCOV and such which we have no hope of annotating.
*
* As such, blanket accept self-references without
* issue.
*/
continue;
}
if (dest->noendbr)
continue;
WARN_FUNC("relocation to !ENDBR: %s",
insn->sec, insn->offset,
offstr(dest->sec, dest->offset));
warnings++;
}
return warnings;
}
static int validate_ibt_data_reloc(struct objtool_file *file,
struct reloc *reloc)
{
struct instruction *dest;
dest = find_insn(file, reloc->sym->sec,
reloc->sym->offset + reloc->addend);
if (!dest)
return 0;
if (dest->type == INSN_ENDBR) {
mark_endbr_used(dest);
return 0;
}
if (dest->noendbr)
return 0;
WARN_FUNC("data relocation to !ENDBR: %s",
reloc->sec->base, reloc->offset,
offstr(dest->sec, dest->offset));
return 1;
}
/*
* Validate IBT rules and remove used ENDBR instructions from the seal list.
* Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
* NOPs) later, in create_ibt_endbr_seal_sections().
*/
static int validate_ibt(struct objtool_file *file)
{
struct section *sec;
struct reloc *reloc;
struct instruction *insn;
int warnings = 0;
for_each_insn(file, insn)
warnings += validate_ibt_insn(file, insn);
for_each_sec(file, sec) {
bool is_data;
/* already done in validate_branch() */
/* Already done by validate_ibt_insn() */
if (sec->sh.sh_flags & SHF_EXECINSTR)
continue;
if (!sec->reloc)
continue;
if (!strncmp(sec->name, ".orc", 4))
/*
* These sections can reference text addresses, but not with
* the intent to indirect branch to them.
*/
if (!strncmp(sec->name, ".discard", 8) ||
!strncmp(sec->name, ".debug", 6) ||
!strcmp(sec->name, ".altinstructions") ||
!strcmp(sec->name, ".ibt_endbr_seal") ||
!strcmp(sec->name, ".orc_unwind_ip") ||
!strcmp(sec->name, ".parainstructions") ||
!strcmp(sec->name, ".retpoline_sites") ||
!strcmp(sec->name, ".smp_locks") ||
!strcmp(sec->name, ".static_call_sites") ||
!strcmp(sec->name, "_error_injection_whitelist") ||
!strcmp(sec->name, "_kprobe_blacklist") ||
!strcmp(sec->name, "__bug_table") ||
!strcmp(sec->name, "__ex_table") ||
!strcmp(sec->name, "__jump_table") ||
!strcmp(sec->name, "__mcount_loc") ||
!strcmp(sec->name, "__tracepoints"))
continue;
if (!strncmp(sec->name, ".discard", 8))
list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
warnings += validate_ibt_data_reloc(file, reloc);
}
return warnings;
}
static int validate_sls(struct objtool_file *file)
{
struct instruction *insn, *next_insn;
int warnings = 0;
for_each_insn(file, insn) {
next_insn = next_insn_same_sec(file, insn);
if (insn->retpoline_safe)
continue;
if (!strncmp(sec->name, ".debug", 6))
continue;
switch (insn->type) {
case INSN_RETURN:
if (!next_insn || next_insn->type != INSN_TRAP) {
WARN_FUNC("missing int3 after ret",
insn->sec, insn->offset);
warnings++;
}
if (!strcmp(sec->name, "_error_injection_whitelist"))
continue;
if (!strcmp(sec->name, "_kprobe_blacklist"))
continue;
is_data = strstr(sec->name, ".data") || strstr(sec->name, ".rodata");
list_for_each_entry(reloc, &sec->reloc->reloc_list, list) {
struct instruction *dest;
dest = validate_ibt_reloc(file, reloc);
if (is_data && dest && !dest->noendbr)
warn_noendbr("data ", sec, reloc->offset, dest);
break;
case INSN_JUMP_DYNAMIC:
if (!next_insn || next_insn->type != INSN_TRAP) {
WARN_FUNC("missing int3 after indirect jump",
insn->sec, insn->offset);
warnings++;
}
break;
default:
break;
}
}
return 0;
return warnings;
}
static int validate_reachable_instructions(struct objtool_file *file)
@ -3853,16 +3892,6 @@ int check(struct objtool_file *file)
{
int ret, warnings = 0;
if (lto && !(vmlinux || module)) {
fprintf(stderr, "--lto requires: --vmlinux or --module\n");
return 1;
}
if (ibt && !lto) {
fprintf(stderr, "--ibt requires: --lto\n");
return 1;
}
arch_initial_func_cfi_state(&initial_func_cfi);
init_cfi_state(&init_cfi);
init_cfi_state(&func_cfi);
@ -3883,73 +3912,89 @@ int check(struct objtool_file *file)
if (list_empty(&file->insn_list))
goto out;
if (vmlinux && !lto) {
ret = validate_vmlinux_functions(file);
if (ret < 0)
goto out;
warnings += ret;
goto out;
}
if (retpoline) {
if (opts.retpoline) {
ret = validate_retpoline(file);
if (ret < 0)
return ret;
warnings += ret;
}
ret = validate_functions(file);
if (ret < 0)
goto out;
warnings += ret;
if (opts.stackval || opts.orc || opts.uaccess) {
ret = validate_functions(file);
if (ret < 0)
goto out;
warnings += ret;
ret = validate_unwind_hints(file, NULL);
if (ret < 0)
goto out;
warnings += ret;
ret = validate_unwind_hints(file, NULL);
if (ret < 0)
goto out;
warnings += ret;
if (ibt) {
if (!warnings) {
ret = validate_reachable_instructions(file);
if (ret < 0)
goto out;
warnings += ret;
}
} else if (opts.noinstr) {
ret = validate_noinstr_sections(file);
if (ret < 0)
goto out;
warnings += ret;
}
if (opts.ibt) {
ret = validate_ibt(file);
if (ret < 0)
goto out;
warnings += ret;
}
if (!warnings) {
ret = validate_reachable_instructions(file);
if (opts.sls) {
ret = validate_sls(file);
if (ret < 0)
goto out;
warnings += ret;
}
ret = create_static_call_sections(file);
if (ret < 0)
goto out;
warnings += ret;
if (opts.static_call) {
ret = create_static_call_sections(file);
if (ret < 0)
goto out;
warnings += ret;
}
if (retpoline) {
if (opts.retpoline) {
ret = create_retpoline_sites_sections(file);
if (ret < 0)
goto out;
warnings += ret;
}
if (mcount) {
if (opts.mcount) {
ret = create_mcount_loc_sections(file);
if (ret < 0)
goto out;
warnings += ret;
}
if (ibt) {
if (opts.ibt) {
ret = create_ibt_endbr_seal_sections(file);
if (ret < 0)
goto out;
warnings += ret;
}
if (stats) {
if (opts.orc && !list_empty(&file->insn_list)) {
ret = orc_create(file);
if (ret < 0)
goto out;
warnings += ret;
}
if (opts.stats) {
printf("nr_insns_visited: %ld\n", nr_insns_visited);
printf("nr_cfi: %ld\n", nr_cfi);
printf("nr_cfi_reused: %ld\n", nr_cfi_reused);

217
tools/objtool/elf.c
View File

@ -355,7 +355,7 @@ static int read_sections(struct elf *elf)
elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name));
}
if (stats) {
if (opts.stats) {
printf("nr_sections: %lu\n", (unsigned long)sections_nr);
printf("section_bits: %d\n", elf->section_bits);
}
@ -374,9 +374,15 @@ static void elf_add_symbol(struct elf *elf, struct symbol *sym)
struct list_head *entry;
struct rb_node *pnode;
INIT_LIST_HEAD(&sym->pv_target);
sym->alias = sym;
sym->type = GELF_ST_TYPE(sym->sym.st_info);
sym->bind = GELF_ST_BIND(sym->sym.st_info);
if (sym->type == STT_FILE)
elf->num_files++;
sym->offset = sym->sym.st_value;
sym->len = sym->sym.st_size;
@ -435,8 +441,6 @@ static int read_symbols(struct elf *elf)
return -1;
}
memset(sym, 0, sizeof(*sym));
INIT_LIST_HEAD(&sym->pv_target);
sym->alias = sym;
sym->idx = i;
@ -475,7 +479,7 @@ static int read_symbols(struct elf *elf)
elf_add_symbol(elf, sym);
}
if (stats) {
if (opts.stats) {
printf("nr_symbols: %lu\n", (unsigned long)symbols_nr);
printf("symbol_bits: %d\n", elf->symbol_bits);
}
@ -546,7 +550,7 @@ static struct section *elf_create_reloc_section(struct elf *elf,
int reltype);
int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
unsigned int type, struct symbol *sym, long addend)
unsigned int type, struct symbol *sym, s64 addend)
{
struct reloc *reloc;
@ -600,24 +604,21 @@ static void elf_dirty_reloc_sym(struct elf *elf, struct symbol *sym)
}
/*
* Move the first global symbol, as per sh_info, into a new, higher symbol
* index. This fees up the shndx for a new local symbol.
* The libelf API is terrible; gelf_update_sym*() takes a data block relative
* index value, *NOT* the symbol index. As such, iterate the data blocks and
* adjust index until it fits.
*
* If no data block is found, allow adding a new data block provided the index
* is only one past the end.
*/
static int elf_move_global_symbol(struct elf *elf, struct section *symtab,
struct section *symtab_shndx)
static int elf_update_symbol(struct elf *elf, struct section *symtab,
struct section *symtab_shndx, struct symbol *sym)
{
Elf_Data *data, *shndx_data = NULL;
Elf32_Word first_non_local;
struct symbol *sym;
Elf_Scn *s;
first_non_local = symtab->sh.sh_info;
sym = find_symbol_by_index(elf, first_non_local);
if (!sym) {
WARN("no non-local symbols !?");
return first_non_local;
}
Elf32_Word shndx = sym->sec ? sym->sec->idx : SHN_UNDEF;
Elf_Data *symtab_data = NULL, *shndx_data = NULL;
Elf64_Xword entsize = symtab->sh.sh_entsize;
int max_idx, idx = sym->idx;
Elf_Scn *s, *t = NULL;
s = elf_getscn(elf->elf, symtab->idx);
if (!s) {
@ -625,79 +626,124 @@ static int elf_move_global_symbol(struct elf *elf, struct section *symtab,
return -1;
}
data = elf_newdata(s);
if (!data) {
WARN_ELF("elf_newdata");
return -1;
}
data->d_buf = &sym->sym;
data->d_size = sizeof(sym->sym);
data->d_align = 1;
data->d_type = ELF_T_SYM;
sym->idx = symtab->sh.sh_size / sizeof(sym->sym);
elf_dirty_reloc_sym(elf, sym);
symtab->sh.sh_info += 1;
symtab->sh.sh_size += data->d_size;
symtab->changed = true;
if (symtab_shndx) {
s = elf_getscn(elf->elf, symtab_shndx->idx);
if (!s) {
t = elf_getscn(elf->elf, symtab_shndx->idx);
if (!t) {
WARN_ELF("elf_getscn");
return -1;
}
}
shndx_data = elf_newdata(s);
if (!shndx_data) {
WARN_ELF("elf_newshndx_data");
for (;;) {
/* get next data descriptor for the relevant sections */
symtab_data = elf_getdata(s, symtab_data);
if (t)
shndx_data = elf_getdata(t, shndx_data);
/* end-of-list */
if (!symtab_data) {
void *buf;
if (idx) {
/* we don't do holes in symbol tables */
WARN("index out of range");
return -1;
}
/* if @idx == 0, it's the next contiguous entry, create it */
symtab_data = elf_newdata(s);
if (t)
shndx_data = elf_newdata(t);
buf = calloc(1, entsize);
if (!buf) {
WARN("malloc");
return -1;
}
symtab_data->d_buf = buf;
symtab_data->d_size = entsize;
symtab_data->d_align = 1;
symtab_data->d_type = ELF_T_SYM;
symtab->sh.sh_size += entsize;
symtab->changed = true;
if (t) {
shndx_data->d_buf = &sym->sec->idx;
shndx_data->d_size = sizeof(Elf32_Word);
shndx_data->d_align = sizeof(Elf32_Word);
shndx_data->d_type = ELF_T_WORD;
symtab_shndx->sh.sh_size += sizeof(Elf32_Word);
symtab_shndx->changed = true;
}
break;
}
/* empty blocks should not happen */
if (!symtab_data->d_size) {
WARN("zero size data");
return -1;
}
shndx_data->d_buf = &sym->sec->idx;
shndx_data->d_size = sizeof(Elf32_Word);
shndx_data->d_align = 4;
shndx_data->d_type = ELF_T_WORD;
/* is this the right block? */
max_idx = symtab_data->d_size / entsize;
if (idx < max_idx)
break;
symtab_shndx->sh.sh_size += 4;
symtab_shndx->changed = true;
/* adjust index and try again */
idx -= max_idx;
}
return first_non_local;
/* something went side-ways */
if (idx < 0) {
WARN("negative index");
return -1;
}
/* setup extended section index magic and write the symbol */
if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) {
sym->sym.st_shndx = shndx;
if (!shndx_data)
shndx = 0;
} else {
sym->sym.st_shndx = SHN_XINDEX;
if (!shndx_data) {
WARN("no .symtab_shndx");
return -1;
}
}
if (!gelf_update_symshndx(symtab_data, shndx_data, idx, &sym->sym, shndx)) {
WARN_ELF("gelf_update_symshndx");
return -1;
}
return 0;
}
static struct symbol *
elf_create_section_symbol(struct elf *elf, struct section *sec)
{
struct section *symtab, *symtab_shndx;
Elf_Data *shndx_data = NULL;
struct symbol *sym;
Elf32_Word shndx;
Elf32_Word first_non_local, new_idx;
struct symbol *sym, *old;
symtab = find_section_by_name(elf, ".symtab");
if (symtab) {
symtab_shndx = find_section_by_name(elf, ".symtab_shndx");
if (symtab_shndx)
shndx_data = symtab_shndx->data;
} else {
WARN("no .symtab");
return NULL;
}
sym = malloc(sizeof(*sym));
sym = calloc(1, sizeof(*sym));
if (!sym) {
perror("malloc");
return NULL;
}
memset(sym, 0, sizeof(*sym));
sym->idx = elf_move_global_symbol(elf, symtab, symtab_shndx);
if (sym->idx < 0) {
WARN("elf_move_global_symbol");
return NULL;
}
sym->name = sec->name;
sym->sec = sec;
@ -707,24 +753,41 @@ elf_create_section_symbol(struct elf *elf, struct section *sec)
// st_other 0
// st_value 0
// st_size 0
shndx = sec->idx;
if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) {
sym->sym.st_shndx = shndx;
if (!shndx_data)
shndx = 0;
} else {
sym->sym.st_shndx = SHN_XINDEX;
if (!shndx_data) {
WARN("no .symtab_shndx");
/*
* Move the first global symbol, as per sh_info, into a new, higher
* symbol index. This fees up a spot for a new local symbol.
*/
first_non_local = symtab->sh.sh_info;
new_idx = symtab->sh.sh_size / symtab->sh.sh_entsize;
old = find_symbol_by_index(elf, first_non_local);
if (old) {
old->idx = new_idx;
hlist_del(&old->hash);
elf_hash_add(symbol, &old->hash, old->idx);
elf_dirty_reloc_sym(elf, old);
if (elf_update_symbol(elf, symtab, symtab_shndx, old)) {
WARN("elf_update_symbol move");
return NULL;
}
new_idx = first_non_local;
}
if (!gelf_update_symshndx(symtab->data, shndx_data, sym->idx, &sym->sym, shndx)) {
WARN_ELF("gelf_update_symshndx");
sym->idx = new_idx;
if (elf_update_symbol(elf, symtab, symtab_shndx, sym)) {
WARN("elf_update_symbol");
return NULL;
}
/*
* Either way, we added a LOCAL symbol.
*/
symtab->sh.sh_info += 1;
elf_add_symbol(elf, sym);
return sym;
@ -843,7 +906,7 @@ static int read_relocs(struct elf *elf)
tot_reloc += nr_reloc;
}
if (stats) {
if (opts.stats) {
printf("max_reloc: %lu\n", max_reloc);
printf("tot_reloc: %lu\n", tot_reloc);
printf("reloc_bits: %d\n", elf->reloc_bits);
@ -1222,7 +1285,7 @@ int elf_write(struct elf *elf)
struct section *sec;
Elf_Scn *s;
if (dryrun)
if (opts.dryrun)
return 0;
/* Update changed relocation sections and section headers: */

34
tools/objtool/include/objtool/builtin.h
View File

@ -8,13 +8,37 @@
#include <subcmd/parse-options.h>
extern const struct option check_options[];
extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
lto, vmlinux, mcount, noinstr, backup, sls, dryrun,
ibt;
struct opts {
/* actions: */
bool dump_orc;
bool hack_jump_label;
bool hack_noinstr;
bool ibt;
bool mcount;
bool noinstr;
bool orc;
bool retpoline;
bool sls;
bool stackval;
bool static_call;
bool uaccess;
/* options: */
bool backtrace;
bool backup;
bool dryrun;
bool link;
bool module;
bool no_unreachable;
bool sec_address;
bool stats;
};
extern struct opts opts;
extern int cmd_parse_options(int argc, const char **argv, const char * const usage[]);
extern int cmd_check(int argc, const char **argv);
extern int cmd_orc(int argc, const char **argv);
extern int objtool_run(int argc, const char **argv);
#endif /* _BUILTIN_H */

16
tools/objtool/include/objtool/elf.h
View File

@ -73,7 +73,7 @@ struct reloc {
struct symbol *sym;
unsigned long offset;
unsigned int type;
long addend;
s64 addend;
int idx;
bool jump_table_start;
};
@ -86,7 +86,7 @@ struct elf {
int fd;
bool changed;
char *name;
unsigned int text_size;
unsigned int text_size, num_files;
struct list_head sections;
int symbol_bits;
@ -131,11 +131,21 @@ static inline u32 reloc_hash(struct reloc *reloc)
return sec_offset_hash(reloc->sec, reloc->offset);
}
/*
* Try to see if it's a whole archive (vmlinux.o or module).
*
* Note this will miss the case where a module only has one source file.
*/
static inline bool has_multiple_files(struct elf *elf)
{
return elf->num_files > 1;
}
struct elf *elf_open_read(const char *name, int flags);
struct section *elf_create_section(struct elf *elf, const char *name, unsigned int sh_flags, size_t entsize, int nr);
int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
unsigned int type, struct symbol *sym, long addend);
unsigned int type, struct symbol *sym, s64 addend);
int elf_add_reloc_to_insn(struct elf *elf, struct section *sec,
unsigned long offset, unsigned int type,
struct section *insn_sec, unsigned long insn_off);

35
tools/objtool/include/objtool/warn.h
View File

@ -11,34 +11,33 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <objtool/builtin.h>
#include <objtool/elf.h>
extern const char *objname;
static inline char *offstr(struct section *sec, unsigned long offset)
{
struct symbol *func;
char *name, *str;
unsigned long name_off;
bool is_text = (sec->sh.sh_flags & SHF_EXECINSTR);
struct symbol *sym = NULL;
char *str;
int len;
func = find_func_containing(sec, offset);
if (!func)
func = find_symbol_containing(sec, offset);
if (func) {
name = func->name;
name_off = offset - func->offset;
if (is_text)
sym = find_func_containing(sec, offset);
if (!sym)
sym = find_symbol_containing(sec, offset);
if (sym) {
str = malloc(strlen(sym->name) + strlen(sec->name) + 40);
len = sprintf(str, "%s+0x%lx", sym->name, offset - sym->offset);
if (opts.sec_address)
sprintf(str+len, " (%s+0x%lx)", sec->name, offset);
} else {
name = sec->name;
name_off = offset;
str = malloc(strlen(sec->name) + 20);
sprintf(str, "%s+0x%lx", sec->name, offset);
}
str = malloc(strlen(name) + 20);
if (func)
sprintf(str, "%s()+0x%lx", name, name_off);
else
sprintf(str, "%s+0x%lx", name, name_off);
return str;
}

103
tools/objtool/objtool.c
View File

@ -3,16 +3,6 @@
* Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
*/
/*
* objtool:
*
* The 'check' subcmd analyzes every .o file and ensures the validity of its
* stack trace metadata. It enforces a set of rules on asm code and C inline
* assembly code so that stack traces can be reliable.
*
* For more information, see tools/objtool/Documentation/stack-validation.txt.
*/
#include <stdio.h>
#include <stdbool.h>
#include <string.h>
@ -26,20 +16,6 @@
#include <objtool/objtool.h>
#include <objtool/warn.h>
struct cmd_struct {
const char *name;
int (*fn)(int, const char **);
const char *help;
};
static const char objtool_usage_string[] =
"objtool COMMAND [ARGS]";
static struct cmd_struct objtool_cmds[] = {
{"check", cmd_check, "Perform stack metadata validation on an object file" },
{"orc", cmd_orc, "Generate in-place ORC unwind tables for an object file" },
};
bool help;
const char *objname;
@ -118,7 +94,7 @@ struct objtool_file *objtool_open_read(const char *_objname)
if (!file.elf)
return NULL;
if (backup && !objtool_create_backup(objname)) {
if (opts.backup && !objtool_create_backup(objname)) {
WARN("can't create backup file");
return NULL;
}
@ -129,7 +105,7 @@ struct objtool_file *objtool_open_read(const char *_objname)
INIT_LIST_HEAD(&file.static_call_list);
INIT_LIST_HEAD(&file.mcount_loc_list);
INIT_LIST_HEAD(&file.endbr_list);
file.ignore_unreachables = no_unreachable;
file.ignore_unreachables = opts.no_unreachable;
file.hints = false;
return &file;
@ -137,7 +113,7 @@ struct objtool_file *objtool_open_read(const char *_objname)
void objtool_pv_add(struct objtool_file *f, int idx, struct symbol *func)
{
if (!noinstr)
if (!opts.noinstr)
return;
if (!f->pv_ops) {
@ -161,70 +137,6 @@ void objtool_pv_add(struct objtool_file *f, int idx, struct symbol *func)
f->pv_ops[idx].clean = false;
}
static void cmd_usage(void)
{
unsigned int i, longest = 0;
printf("\n usage: %s\n\n", objtool_usage_string);
for (i = 0; i < ARRAY_SIZE(objtool_cmds); i++) {
if (longest < strlen(objtool_cmds[i].name))
longest = strlen(objtool_cmds[i].name);
}
puts(" Commands:");
for (i = 0; i < ARRAY_SIZE(objtool_cmds); i++) {
printf(" %-*s ", longest, objtool_cmds[i].name);
puts(objtool_cmds[i].help);
}
printf("\n");
if (!help)
exit(129);
exit(0);
}
static void handle_options(int *argc, const char ***argv)
{
while (*argc > 0) {
const char *cmd = (*argv)[0];
if (cmd[0] != '-')
break;
if (!strcmp(cmd, "--help") || !strcmp(cmd, "-h")) {
help = true;
break;
} else {
fprintf(stderr, "Unknown option: %s\n", cmd);
cmd_usage();
}
(*argv)++;
(*argc)--;
}
}
static void handle_internal_command(int argc, const char **argv)
{
const char *cmd = argv[0];
unsigned int i, ret;
for (i = 0; i < ARRAY_SIZE(objtool_cmds); i++) {
struct cmd_struct *p = objtool_cmds+i;
if (strcmp(p->name, cmd))
continue;
ret = p->fn(argc, argv);
exit(ret);
}
cmd_usage();
}
int main(int argc, const char **argv)
{
static const char *UNUSED = "OBJTOOL_NOT_IMPLEMENTED";
@ -233,14 +145,7 @@ int main(int argc, const char **argv)
exec_cmd_init("objtool", UNUSED, UNUSED, UNUSED);
pager_init(UNUSED);
argv++;
argc--;
handle_options(&argc, &argv);
if (!argc || help)
cmd_usage();
handle_internal_command(argc, argv);
objtool_run(argc, argv);
return 0;
}

9
tools/objtool/weak.c
View File

@ -15,17 +15,12 @@
return ENOSYS; \
})
int __weak check(struct objtool_file *file)
{
UNSUPPORTED("check subcommand");
}
int __weak orc_dump(const char *_objname)
{
UNSUPPORTED("orc");
UNSUPPORTED("ORC");
}
int __weak orc_create(struct objtool_file *file)
{
UNSUPPORTED("orc");
UNSUPPORTED("ORC");
}