diff --git a/arch/arm64/include/asm/sdei.h b/arch/arm64/include/asm/sdei.h index b2248bd3cb58..a78a5337ff85 100644 --- a/arch/arm64/include/asm/sdei.h +++ b/arch/arm64/include/asm/sdei.h @@ -44,7 +44,8 @@ asmlinkage unsigned long __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg); unsigned long do_sdei_event(struct pt_regs *regs, - struct sdei_registered_event *arg); + struct sdei_registered_event *arg, + unsigned long vbar); unsigned long sdei_arch_get_entry_point(int conduit); #define sdei_arch_get_entry_point(x) sdei_arch_get_entry_point(x) diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h index e9ce68d50ba4..a5abf0615594 100644 --- a/arch/arm64/include/asm/suspend.h +++ b/arch/arm64/include/asm/suspend.h @@ -2,7 +2,7 @@ #ifndef __ASM_SUSPEND_H #define __ASM_SUSPEND_H -#define NR_CTX_REGS 14 +#define NR_CTX_REGS 15 #define NR_CALLEE_SAVED_REGS 12 /* diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 11d3f888fcd8..beb56891fc4f 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -31,6 +31,7 @@ #include #include #include +#include /* * Where the context being returned to had an active percpu GPR critical @@ -1040,6 +1041,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) asmlinkage noinstr unsigned long __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) { + unsigned long elr, spsr, esr, far, vbar; irqentry_state_t state; unsigned long ret; @@ -1065,10 +1067,46 @@ __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) else if (cpu_has_pan()) set_pstate_pan(0); + /* + * Nothing saved the interrupted context's exception registers on the + * way in, and it may be in the middle of exception entry or return + * (SDEI ignores DAIF). Save them before anything can take an exception + * and overwrite them. + */ + elr = read_sysreg(elr_el1); + spsr = read_sysreg(spsr_el1); + esr = read_sysreg(esr_el1); + far = read_sysreg(far_el1); + + /* + * With KPTI, VBAR_EL1 holds the trampoline vectors from the return to + * EL0 until the next exception from EL0 maps the kernel. They only + * handle exceptions from EL0: an exception taken here would run their + * empty current-EL half and never come back. Switch to the kernel + * vectors. The IRQ-like resume keeps using the interrupted VBAR, so it + * still goes through the trampoline. + */ + vbar = read_sysreg(vbar_el1); + if (arm64_kernel_unmapped_at_el0() && + vbar == (unsigned long)raw_cpu_read(this_cpu_vector)) { + write_sysreg((unsigned long)vectors, vbar_el1); + isb(); + } + state = irqentry_nmi_enter(regs); - ret = do_sdei_event(regs, arg); + ret = do_sdei_event(regs, arg, vbar); irqentry_nmi_exit(regs, state); + /* + * Put them back unconditionally. An unchanged ELR does not rule out a + * nested exception: one may have returned to the same address. + */ + write_sysreg(elr, elr_el1); + write_sysreg(spsr, spsr_el1); + write_sysreg(esr, esr_el1); + write_sysreg(far, far_el1); + write_sysreg(vbar, vbar_el1); + return ret; } #endif /* CONFIG_ARM_SDE_INTERFACE */ diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index 854d872dfd0f..14f453dd0946 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -112,6 +112,8 @@ int load_other_segments(struct kimage *image, vfree(headers); goto out_err; } + + vfree(image->elf_headers); image->elf_headers = headers; image->elf_load_addr = kbuf.mem; image->elf_headers_sz = headers_sz; diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c index 778f2a1faac8..1b902da8ba43 100644 --- a/arch/arm64/kernel/sdei.c +++ b/arch/arm64/kernel/sdei.c @@ -195,21 +195,49 @@ unsigned long sdei_arch_get_entry_point(int conduit) return 0; } +/* + * Exceptions an SDEI handler may take without harm once __sdei_handler() has + * restored the interrupted context's exception registers. Their handlers are + * safe in NMI context: + * - EL1 data aborts on translation, permission and access-flag faults: + * faulthandler_disabled() is true in NMI context, so do_page_fault() never + * takes the mmap lock. This covers the _nofault() accessors used by the + * generic dump_stack()/show_regs() code (e.g. print_worker_info()). + * - BRK: kprobes, WARN/BUG, kCFI and UBSAN traps, all NMI-capable. + * Anything else taken at EL1 is fatal on its own, or unexpected here. + */ +static bool sdei_nested_exception_ok(unsigned long esr) +{ + switch (ESR_ELx_EC(esr)) { + case ESR_ELx_EC_DABT_CUR: + return esr_fsc_is_translation_fault(esr) || + esr_fsc_is_permission_fault(esr) || + esr_fsc_is_access_flag_fault(esr); + case ESR_ELx_EC_BRK64: + return true; + default: + return false; + } +} + /* * do_sdei_event() returns one of: * SDEI_EV_HANDLED - success, return to the interrupted context. * SDEI_EV_FAILED - failure, return this error code to firmware. * virtual-address - success, return to this address. + * + * vbar is the interrupted context's VBAR_EL1, for the IRQ-like resume. */ unsigned long __kprobes do_sdei_event(struct pt_regs *regs, - struct sdei_registered_event *arg) + struct sdei_registered_event *arg, + unsigned long vbar) { u32 mode; int i, err = 0; int clobbered_registers = 4; u64 elr = read_sysreg(elr_el1); + u64 nested_elr, nested_esr, nested_far; u32 kernel_mode = read_sysreg(CurrentEL) | 1; /* +SPSel */ - unsigned long vbar = read_sysreg(vbar_el1); if (arm64_kernel_unmapped_at_el0()) clobbered_registers++; @@ -221,18 +249,27 @@ unsigned long __kprobes do_sdei_event(struct pt_regs *regs, } err = sdei_event_handler(regs, arg); + + /* + * A synchronous exception taken by the handler has overwritten the + * exception registers; __sdei_handler() puts them back. Only the last + * one's syndrome survives; report it if it is not one known to be + * harmless. + * + * If the event interrupted a KVM world switch, VBAR points at the KVM + * hyp vectors and any such exception is a hyp panic instead. + */ + nested_elr = read_sysreg(elr_el1); + nested_esr = read_sysreg(esr_el1); + nested_far = read_sysreg(far_el1); + if (nested_elr != elr && !sdei_nested_exception_ok(nested_esr)) { + pr_warn_once("unexpected exception during handler: ESR %#llx FAR %#llx near %pS\n", + nested_esr, nested_far, (void *)nested_elr); + } + if (err) return SDEI_EV_FAILED; - if (elr != read_sysreg(elr_el1)) { - /* - * We took a synchronous exception from the SDEI handler. - * This could deadlock, and if you interrupt KVM it will - * hyp-panic instead. - */ - pr_warn("unsafe: exception during handler\n"); - } - mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK); /* diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 32e0771d6477..e212270d4e09 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -61,7 +61,9 @@ int valid_phys_addr_range(phys_addr_t addr, size_t size) */ int valid_mmap_phys_addr_range(unsigned long pfn, size_t size) { - return !(((pfn << PAGE_SHIFT) + size) & ~PHYS_MASK); + const phys_addr_t limit = PHYS_MASK + 1; + + return size <= limit && pfn <= (limit - size) >> PAGE_SHIFT; } static int __init adjust_protection_map(void) diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 12aacc74f764..b1516630904d 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -113,6 +113,10 @@ SYM_FUNC_START(cpu_do_suspend) alternative_if ARM64_HAS_TCR2 mrs x2, REG_TCR2_EL1 str x2, [x0, #104] +alternative_else_nop_endif +alternative_if ARM64_SME + mrs_s x2, SYS_TPIDR2_EL0 + str x2, [x0, #112] alternative_else_nop_endif ret SYM_FUNC_END(cpu_do_suspend) @@ -152,6 +156,10 @@ alternative_if ARM64_HAS_TCR2 ldr x2, [x0, #104] msr REG_TCR2_EL1, x2 alternative_else_nop_endif +alternative_if ARM64_SME + ldr x2, [x0, #112] + msr_s SYS_TPIDR2_EL0, x2 +alternative_else_nop_endif msr sctlr_el1, x12 set_this_cpu_offset x13