From b0dc553cfc9d3bc2c7b8672b0b2fcf0edf0c3b6e Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 20 Jan 2021 21:09:50 -0800 Subject: [PATCH 1/3] x86/fpu: Make the EFI FPU calling convention explicit EFI uses kernel_fpu_begin() to conform to the UEFI calling convention. This specifically requires initializing FCW (FPU Control Word), whereas no sane 64-bit kernel code should use legacy 387 operations that reference FCW. This should allow to safely change the default semantics of kernel_fpu_begin() to stop initializing FCW on 64-bit kernels. [ bp: Massage commit message a little. ] Signed-off-by: Andy Lutomirski Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/25d392fff64680e0f4bb8cf0b1003314dc29eafe.1611205691.git.luto@kernel.org --- arch/x86/include/asm/efi.h | 24 ++++++++++++++++++++---- arch/x86/platform/efi/efi_64.c | 4 ++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index c98f78330b09..c81e68f00071 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -68,17 +68,33 @@ extern unsigned long efi_fw_vendor, efi_config_table; #f " called with too many arguments (" #p ">" #n ")"); \ }) +static inline void efi_fpu_begin(void) +{ + /* + * The UEFI calling convention (UEFI spec 2.3.2 and 2.3.4) requires + * that FCW and MXCSR (64-bit) must be initialized prior to calling + * UEFI code. (Oddly the spec does not require that the FPU stack + * be empty.) + */ + kernel_fpu_begin_mask(KFPU_387 | KFPU_MXCSR); +} + +static inline void efi_fpu_end(void) +{ + kernel_fpu_end(); +} + #ifdef CONFIG_X86_32 #define arch_efi_call_virt_setup() \ ({ \ - kernel_fpu_begin(); \ + efi_fpu_begin(); \ firmware_restrict_branch_speculation_start(); \ }) #define arch_efi_call_virt_teardown() \ ({ \ firmware_restrict_branch_speculation_end(); \ - kernel_fpu_end(); \ + efi_fpu_end(); \ }) #define arch_efi_call_virt(p, f, args...) p->f(args) @@ -107,7 +123,7 @@ struct efi_scratch { #define arch_efi_call_virt_setup() \ ({ \ efi_sync_low_kernel_mappings(); \ - kernel_fpu_begin(); \ + efi_fpu_begin(); \ firmware_restrict_branch_speculation_start(); \ efi_switch_mm(&efi_mm); \ }) @@ -119,7 +135,7 @@ struct efi_scratch { ({ \ efi_switch_mm(efi_scratch.prev_mm); \ firmware_restrict_branch_speculation_end(); \ - kernel_fpu_end(); \ + efi_fpu_end(); \ }) #ifdef CONFIG_KASAN diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index e1e8d4e3a213..cf7b3bfe1a1e 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -850,7 +850,7 @@ efi_set_virtual_address_map(unsigned long memory_map_size, virtual_map); efi_switch_mm(&efi_mm); - kernel_fpu_begin(); + efi_fpu_begin(); /* Disable interrupts around EFI calls: */ local_irq_save(flags); @@ -859,7 +859,7 @@ efi_set_virtual_address_map(unsigned long memory_map_size, descriptor_version, virtual_map); local_irq_restore(flags); - kernel_fpu_end(); + efi_fpu_end(); /* grab the virtually remapped EFI runtime services table pointer */ efi.runtime = READ_ONCE(systab->runtime); From 49200d17d27d5cc1aede2d1bb2a78dbfc1563e65 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 20 Jan 2021 21:09:51 -0800 Subject: [PATCH 2/3] x86/fpu/64: Don't FNINIT in kernel_fpu_begin() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remaining callers of kernel_fpu_begin() in 64-bit kernels don't use 387 instructions, so there's no need to sanitize the FPU state. Skip it to get most of the performance we lost back. Reported-by: Krzysztof Olędzki Signed-off-by: Andy Lutomirski Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/57f8841ccbf9f3c25a23196c888f5f6ec5887577.1611205691.git.luto@kernel.org --- arch/x86/include/asm/fpu/api.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h index 67a4f1cb2aac..ed33a14188f6 100644 --- a/arch/x86/include/asm/fpu/api.h +++ b/arch/x86/include/asm/fpu/api.h @@ -32,7 +32,19 @@ extern void fpregs_mark_activate(void); /* Code that is unaware of kernel_fpu_begin_mask() can use this */ static inline void kernel_fpu_begin(void) { +#ifdef CONFIG_X86_64 + /* + * Any 64-bit code that uses 387 instructions must explicitly request + * KFPU_387. + */ + kernel_fpu_begin_mask(KFPU_MXCSR); +#else + /* + * 32-bit kernel code may use 387 operations as well as SSE2, etc, + * as long as it checks that the CPU has the required capability. + */ kernel_fpu_begin_mask(KFPU_387 | KFPU_MXCSR); +#endif } /* From 0a74d61c7d842b583f33f74d7a9e93201826f4c5 Mon Sep 17 00:00:00 2001 From: Yejune Deng Date: Fri, 22 Jan 2021 15:19:25 +0800 Subject: [PATCH 3/3] x86/fpu/xstate: Use sizeof() instead of a constant Use sizeof() instead of a constant in fpstate_sanitize_xstate(). Remove use of the address of the 0th array element of ->st_space and ->xmm_space which is equivalent to the array address itself: No code changed: # arch/x86/kernel/fpu/xstate.o: text data bss dec hex filename 9694 899 4 10597 2965 xstate.o.before 9694 899 4 10597 2965 xstate.o.after md5: 5a43fc70bad8e2a1784f67f01b71aabb xstate.o.before.asm 5a43fc70bad8e2a1784f67f01b71aabb xstate.o.after.asm [ bp: Massage commit message. ] Signed-off-by: Yejune Deng Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20210122071925.41285-1-yejune.deng@gmail.com --- arch/x86/kernel/fpu/xstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 5d8047441a0a..683749b80ae2 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -167,14 +167,14 @@ void fpstate_sanitize_xstate(struct fpu *fpu) fx->fop = 0; fx->rip = 0; fx->rdp = 0; - memset(&fx->st_space[0], 0, 128); + memset(fx->st_space, 0, sizeof(fx->st_space)); } /* * SSE is in init state */ if (!(xfeatures & XFEATURE_MASK_SSE)) - memset(&fx->xmm_space[0], 0, 256); + memset(fx->xmm_space, 0, sizeof(fx->xmm_space)); /* * First two features are FPU and SSE, which above we handled