arm64 fixes:

- Fix nested exceptions taken by SDEI event handlers. With KPTI, an
    event interrupting user space left the trampoline vectors installed
    and any exception in the handler looped forever. Also save and
    restore ELR/SPSR/ESR/FAR around the handler since an event can
    interrupt exception entry or return where these registers are live
 
  - Save and restore TPIDR2_EL0 across CPU suspend. Its value is UNKNOWN
    after a warm reset and the task initiating suspend could return to
    user space with a corrupted value
 
  - Fix an overflow in valid_mmap_phys_addr_range() allowing /dev/mem to
    map out-of-range PFNs
 
  - Fix an elf_headers memory leak in the kexec_file segment placement
    retry loop
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEE5RElWfyWxS+3PLO2a9axLQDIXvEFAmruHxAACgkQa9axLQDI
 XvF6+A/+JH00oYTBOvNR2InAyew8Sa2daiPfPetwSv7rtmY1xiPa01DHo3m+SFrL
 bQUxoDArXoD2pI8TgYn8Qtxe1+YkmI+dvo26fS9e6BEwF4NBe5GmF/50i7f9+QuR
 kBXlutxBDfXxHiZi9rxu+R3YcvUhShhQSAhxbZcT97E0hgFnRMGqmexQv9e9N95u
 ydpGle03cuVnDVbrHSKxAA9gTqQvf+dDFRKWXq3HuELbJs66Pj5e2aSREs2Dsd6H
 9/xZKtgm1pL6S7azXsov7m/uMACH9vpD4NYwC0XwmQrRRrZGoiGvZMY7gLGsMQYe
 QtvlbK1ZQAM/+epkKCbb58zAqq8LqD+VbZ+3T3ZPryr0xVYUNybnxTJyKqXPoSKz
 fNjPyikHRSO/7qs1akJYl2a54FygwckmHQnsRKFkemiYd8QNXWD1wxDh70/vfr7A
 1NKmudo/JXCGLzs/Jov1m3RBcrCB7N8AFcCfFFKt9myXPJVXTpQfomeiUYkNMjac
 /eHm3g8C3fVSnlbX5RYIonTuyihDk8DtI/lv+eE2lX64ue7ovMP3rtXo5jy2NxDo
 e/ls+sJS5hW36P6d2SKUFH0P6MsmuqRR6GHLHaG/uURUg4eCVLPc1oxXniAaAthz
 f3SmvFXXXrTkEQKyxY3Um+gy9i71x+tukQokADPvw29O8qhZovY=
 =5F6y
 -----END PGP SIGNATURE-----

Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 fixes from Catalin Marinas:
 "Fix SDEI (software delegated exception interface) handlers taking
  nested exceptions (looping on the KPTI trampoline vectors and
  clobbering the interrupted context's exception registers), TPIDR2_EL0
  preservation across CPU suspend, a kexec_file_load() memory leak and
  an overflow in the /dev/mem valid_mmap_phys_addr_range() check:

   - Fix nested exceptions taken by SDEI event handlers.

     With KPTI, an event interrupting user space left the trampoline
     vectors installed and any exception in the handler looped forever.
     Also save and restore ELR/SPSR/ESR/FAR around the handler since an
     event can interrupt exception entry or return where these registers
     are live

   - Save and restore TPIDR2_EL0 across CPU suspend.

     Its value is UNKNOWN after a warm reset and the task initiating
     suspend could return to user space with a corrupted value

   - Fix an overflow in valid_mmap_phys_addr_range() allowing /dev/mem
     to map out-of-range PFNs

   - Fix an elf_headers memory leak in the kexec_file segment placement
     retry loop"

* tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux:
  arm64: sdei: Preserve the interrupted context's exception registers
  arm64: sdei: Run the event handler on the kernel vectors
  arm64: kexec_file: Fix elf_headers memory leak in retry loop
  arm64: mm: validate /dev/mem mmap ranges without overflow
  arm64: suspend: Save and restore TPIDR2_EL0 across CPU suspend

arch/arm64/include/asm/sdei.h

@@ -44,7 +44,8 @@ asmlinkage unsigned long __sdei_handler(struct pt_regs *regs,
 					struct sdei_registered_event *arg);
 
 unsigned long do_sdei_event(struct pt_regs *regs,
-			    struct sdei_registered_event *arg);
+			    struct sdei_registered_event *arg,
+			    unsigned long vbar);
 
 unsigned long sdei_arch_get_entry_point(int conduit);
 #define sdei_arch_get_entry_point(x)	sdei_arch_get_entry_point(x)

arch/arm64/include/asm/suspend.h

@@ -2,7 +2,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 14
+#define NR_CTX_REGS 15
 #define NR_CALLEE_SAVED_REGS 12
 
 /*

arch/arm64/kernel/entry-common.c

@@ -31,6 +31,7 @@
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
 #include <asm/system_misc.h>
+#include <asm/vectors.h>
 
 /*
  * Where the context being returned to had an active percpu GPR critical
@@ -1040,6 +1041,7 @@ asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
 asmlinkage noinstr unsigned long
 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
 {
+	unsigned long elr, spsr, esr, far, vbar;
 	irqentry_state_t state;
 	unsigned long ret;
 
@@ -1065,10 +1067,46 @@ __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
 	else if (cpu_has_pan())
 		set_pstate_pan(0);
 
+	/*
+	 * Nothing saved the interrupted context's exception registers on the
+	 * way in, and it may be in the middle of exception entry or return
+	 * (SDEI ignores DAIF). Save them before anything can take an exception
+	 * and overwrite them.
+	 */
+	elr = read_sysreg(elr_el1);
+	spsr = read_sysreg(spsr_el1);
+	esr = read_sysreg(esr_el1);
+	far = read_sysreg(far_el1);
+
+	/*
+	 * With KPTI, VBAR_EL1 holds the trampoline vectors from the return to
+	 * EL0 until the next exception from EL0 maps the kernel. They only
+	 * handle exceptions from EL0: an exception taken here would run their
+	 * empty current-EL half and never come back. Switch to the kernel
+	 * vectors. The IRQ-like resume keeps using the interrupted VBAR, so it
+	 * still goes through the trampoline.
+	 */
+	vbar = read_sysreg(vbar_el1);
+	if (arm64_kernel_unmapped_at_el0() &&
+	    vbar == (unsigned long)raw_cpu_read(this_cpu_vector)) {
+		write_sysreg((unsigned long)vectors, vbar_el1);
+		isb();
+	}
+
 	state = irqentry_nmi_enter(regs);
-	ret = do_sdei_event(regs, arg);
+	ret = do_sdei_event(regs, arg, vbar);
 	irqentry_nmi_exit(regs, state);
 
+	/*
+	 * Put them back unconditionally. An unchanged ELR does not rule out a
+	 * nested exception: one may have returned to the same address.
+	 */
+	write_sysreg(elr, elr_el1);
+	write_sysreg(spsr, spsr_el1);
+	write_sysreg(esr, esr_el1);
+	write_sysreg(far, far_el1);
+	write_sysreg(vbar, vbar_el1);
+
 	return ret;
 }
 #endif /* CONFIG_ARM_SDE_INTERFACE */

arch/arm64/kernel/machine_kexec_file.c

@@ -112,6 +112,8 @@ int load_other_segments(struct kimage *image,
 			vfree(headers);
 			goto out_err;
 		}
+
+		vfree(image->elf_headers);
 		image->elf_headers = headers;
 		image->elf_load_addr = kbuf.mem;
 		image->elf_headers_sz = headers_sz;

arch/arm64/kernel/sdei.c

@@ -195,21 +195,49 @@ unsigned long sdei_arch_get_entry_point(int conduit)
 	return 0;
 }
 
+/*
+ * Exceptions an SDEI handler may take without harm once __sdei_handler() has
+ * restored the interrupted context's exception registers. Their handlers are
+ * safe in NMI context:
+ *  - EL1 data aborts on translation, permission and access-flag faults:
+ *    faulthandler_disabled() is true in NMI context, so do_page_fault() never
+ *    takes the mmap lock. This covers the _nofault() accessors used by the
+ *    generic dump_stack()/show_regs() code (e.g. print_worker_info()).
+ *  - BRK: kprobes, WARN/BUG, kCFI and UBSAN traps, all NMI-capable.
+ * Anything else taken at EL1 is fatal on its own, or unexpected here.
+ */
+static bool sdei_nested_exception_ok(unsigned long esr)
+{
+	switch (ESR_ELx_EC(esr)) {
+	case ESR_ELx_EC_DABT_CUR:
+		return esr_fsc_is_translation_fault(esr) ||
+		       esr_fsc_is_permission_fault(esr) ||
+		       esr_fsc_is_access_flag_fault(esr);
+	case ESR_ELx_EC_BRK64:
+		return true;
+	default:
+		return false;
+	}
+}
+
 /*
  * do_sdei_event() returns one of:
  *  SDEI_EV_HANDLED -  success, return to the interrupted context.
  *  SDEI_EV_FAILED  -  failure, return this error code to firmware.
  *  virtual-address -  success, return to this address.
+ *
+ * vbar is the interrupted context's VBAR_EL1, for the IRQ-like resume.
  */
 unsigned long __kprobes do_sdei_event(struct pt_regs *regs,
-				      struct sdei_registered_event *arg)
+				      struct sdei_registered_event *arg,
+				      unsigned long vbar)
 {
 	u32 mode;
 	int i, err = 0;
 	int clobbered_registers = 4;
 	u64 elr = read_sysreg(elr_el1);
+	u64 nested_elr, nested_esr, nested_far;
 	u32 kernel_mode = read_sysreg(CurrentEL) | 1;	/* +SPSel */
-	unsigned long vbar = read_sysreg(vbar_el1);
 
 	if (arm64_kernel_unmapped_at_el0())
 		clobbered_registers++;
@@ -221,18 +249,27 @@ unsigned long __kprobes do_sdei_event(struct pt_regs *regs,
 	}
 
 	err = sdei_event_handler(regs, arg);
+
+	/*
+	 * A synchronous exception taken by the handler has overwritten the
+	 * exception registers; __sdei_handler() puts them back. Only the last
+	 * one's syndrome survives; report it if it is not one known to be
+	 * harmless.
+	 *
+	 * If the event interrupted a KVM world switch, VBAR points at the KVM
+	 * hyp vectors and any such exception is a hyp panic instead.
+	 */
+	nested_elr = read_sysreg(elr_el1);
+	nested_esr = read_sysreg(esr_el1);
+	nested_far = read_sysreg(far_el1);
+	if (nested_elr != elr && !sdei_nested_exception_ok(nested_esr)) {
+		pr_warn_once("unexpected exception during handler: ESR %#llx FAR %#llx near %pS\n",
+			     nested_esr, nested_far, (void *)nested_elr);
+	}
+
 	if (err)
 		return SDEI_EV_FAILED;
 
-	if (elr != read_sysreg(elr_el1)) {
-		/*
-		 * We took a synchronous exception from the SDEI handler.
-		 * This could deadlock, and if you interrupt KVM it will
-		 * hyp-panic instead.
-		 */
-		pr_warn("unsafe: exception during handler\n");
-	}
-
 	mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);
 
 	/*

arch/arm64/mm/mmap.c

@@ -61,7 +61,9 @@ int valid_phys_addr_range(phys_addr_t addr, size_t size)
  */
 int valid_mmap_phys_addr_range(unsigned long pfn, size_t size)
 {
-	return !(((pfn << PAGE_SHIFT) + size) & ~PHYS_MASK);
+	const phys_addr_t limit = PHYS_MASK + 1;
+
+	return size <= limit && pfn <= (limit - size) >> PAGE_SHIFT;
 }
 
 static int __init adjust_protection_map(void)

arch/arm64/mm/proc.S

@@ -113,6 +113,10 @@ SYM_FUNC_START(cpu_do_suspend)
 alternative_if ARM64_HAS_TCR2
 	mrs	x2, REG_TCR2_EL1
 	str	x2, [x0, #104]
+alternative_else_nop_endif
+alternative_if ARM64_SME
+	mrs_s	x2, SYS_TPIDR2_EL0
+	str	x2, [x0, #112]
 alternative_else_nop_endif
 	ret
 SYM_FUNC_END(cpu_do_suspend)
@@ -152,6 +156,10 @@ alternative_if ARM64_HAS_TCR2
 	ldr	x2, [x0, #104]
 	msr	REG_TCR2_EL1, x2
 alternative_else_nop_endif
+alternative_if ARM64_SME
+	ldr	x2, [x0, #112]
+	msr_s	SYS_TPIDR2_EL0, x2
+alternative_else_nop_endif
 
 	msr	sctlr_el1, x12
 	set_this_cpu_offset x13