diff --git a/arch/arm64/include/asm/sdei.h b/arch/arm64/include/asm/sdei.h index b2248bd3cb58..a78a5337ff85 100644 --- a/arch/arm64/include/asm/sdei.h +++ b/arch/arm64/include/asm/sdei.h @@ -44,7 +44,8 @@ asmlinkage unsigned long __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg); unsigned long do_sdei_event(struct pt_regs *regs, - struct sdei_registered_event *arg); + struct sdei_registered_event *arg, + unsigned long vbar); unsigned long sdei_arch_get_entry_point(int conduit); #define sdei_arch_get_entry_point(x) sdei_arch_get_entry_point(x) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 11d3f888fcd8..36b85b9e1575 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -31,6 +31,7 @@ #include #include #include +#include /* * Where the context being returned to had an active percpu GPR critical @@ -1040,6 +1041,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) asmlinkage noinstr unsigned long __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) { + unsigned long vbar; irqentry_state_t state; unsigned long ret; @@ -1065,10 +1067,27 @@ __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) else if (cpu_has_pan()) set_pstate_pan(0); + /* + * With KPTI, VBAR_EL1 holds the trampoline vectors from the return to + * EL0 until the next exception from EL0 maps the kernel. They only + * handle exceptions from EL0: an exception taken here would run their + * empty current-EL half and never come back. Switch to the kernel + * vectors. The IRQ-like resume keeps using the interrupted VBAR, so it + * still goes through the trampoline. + */ + vbar = read_sysreg(vbar_el1); + if (arm64_kernel_unmapped_at_el0() && + vbar == (unsigned long)raw_cpu_read(this_cpu_vector)) { + write_sysreg((unsigned long)vectors, vbar_el1); + isb(); + } + state = irqentry_nmi_enter(regs); - ret = do_sdei_event(regs, arg); + ret = do_sdei_event(regs, arg, vbar); irqentry_nmi_exit(regs, state); + write_sysreg(vbar, vbar_el1); + return ret; } #endif /* CONFIG_ARM_SDE_INTERFACE */ diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c index 778f2a1faac8..260c13106009 100644 --- a/arch/arm64/kernel/sdei.c +++ b/arch/arm64/kernel/sdei.c @@ -200,16 +200,18 @@ unsigned long sdei_arch_get_entry_point(int conduit) * SDEI_EV_HANDLED - success, return to the interrupted context. * SDEI_EV_FAILED - failure, return this error code to firmware. * virtual-address - success, return to this address. + * + * vbar is the interrupted context's VBAR_EL1, for the IRQ-like resume. */ unsigned long __kprobes do_sdei_event(struct pt_regs *regs, - struct sdei_registered_event *arg) + struct sdei_registered_event *arg, + unsigned long vbar) { u32 mode; int i, err = 0; int clobbered_registers = 4; u64 elr = read_sysreg(elr_el1); u32 kernel_mode = read_sysreg(CurrentEL) | 1; /* +SPSel */ - unsigned long vbar = read_sysreg(vbar_el1); if (arm64_kernel_unmapped_at_el0()) clobbered_registers++;