From 52705e72e265406255f83dbd0c725fddc5bd2c83 Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" Date: Mon, 20 Apr 2026 18:43:52 +0200 Subject: [PATCH 1/4] x86/entry: Zap the #VC entry user and kernel macros Drop the separate kernel and user macros in favor of calling a single #VC C handler which multiplexes between the kernel and user #VC entry points by looking at CS's RPL. Zap unused DEFINE_IDTENTRY_VC while at it. There should be no functionality change resulting from this - just code simplification. Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Nikunj A. Dadhania Reviewed-by: Tom Lendacky Acked-by: Joerg Roedel Tested-by: Nikunj A. Dadhania Link: https://patch.msgid.link/20260420164352.32129-1-bp@kernel.org --- arch/x86/coco/sev/internal.h | 3 +++ arch/x86/coco/sev/vc-handle.c | 12 ++++++++++-- arch/x86/entry/entry_64.S | 4 ++-- arch/x86/entry/entry_fred.c | 10 ---------- arch/x86/include/asm/idtentry.h | 29 +++-------------------------- 5 files changed, 18 insertions(+), 40 deletions(-) diff --git a/arch/x86/coco/sev/internal.h b/arch/x86/coco/sev/internal.h index b1d0c66a651a..b9632c0fc391 100644 --- a/arch/x86/coco/sev/internal.h +++ b/arch/x86/coco/sev/internal.h @@ -70,6 +70,9 @@ void svsm_pval_pages(struct snp_psc_desc *desc); int svsm_perform_call_protocol(struct svsm_call *call); bool snp_svsm_vtpm_probe(void); +noinstr void kernel_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code); +noinstr void user_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code); + static inline u64 sev_es_rd_ghcb_msr(void) { return native_rdmsrq(MSR_AMD64_SEV_ES_GHCB); diff --git a/arch/x86/coco/sev/vc-handle.c b/arch/x86/coco/sev/vc-handle.c index d98b5c08ef00..96b62b49b2b5 100644 --- a/arch/x86/coco/sev/vc-handle.c +++ b/arch/x86/coco/sev/vc-handle.c @@ -954,7 +954,7 @@ static __always_inline bool vc_is_db(unsigned long error_code) * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode * and will panic when an error happens. */ -DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) +noinstr void kernel_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code) { irqentry_state_t irq_state; @@ -1006,7 +1006,7 @@ DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode * and will kill the current task with SIGBUS when an error happens. */ -DEFINE_IDTENTRY_VC_USER(exc_vmm_communication) +noinstr void user_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code) { /* * Handle #DB before calling into !noinstr code to avoid recursive #DB. @@ -1032,6 +1032,14 @@ DEFINE_IDTENTRY_VC_USER(exc_vmm_communication) irqentry_exit_to_user_mode(regs); } +DEFINE_IDTENTRY_RAW_ERRORCODE(exc_vmm_communication) +{ + if (user_mode(regs)) + return user_exc_vmm_communication(regs, error_code); + else + return kernel_exc_vmm_communication(regs, error_code); +} + bool __init handle_vc_boot_ghcb(struct pt_regs *regs) { unsigned long exit_code = regs->orig_ax; diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 42447b1e1dff..c6d996593f32 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -492,7 +492,7 @@ SYM_CODE_START(\asmsym) movq %rsp, %rdi /* pt_regs pointer */ - call kernel_\cfunc + call \cfunc /* * No need to switch back to the IST stack. The current stack is either @@ -503,7 +503,7 @@ SYM_CODE_START(\asmsym) /* Switch to the regular task stack */ .Lfrom_usermode_switch_stack_\@: - idtentry_body user_\cfunc, has_error_code=1 + idtentry_body \cfunc, has_error_code=1 _ASM_NOKPROBE(\asmsym) SYM_CODE_END(\asmsym) diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c index fbe2d10dd737..fb3594ddf731 100644 --- a/arch/x86/entry/entry_fred.c +++ b/arch/x86/entry/entry_fred.c @@ -177,16 +177,6 @@ static noinstr void fred_extint(struct pt_regs *regs) } } -#ifdef CONFIG_AMD_MEM_ENCRYPT -noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code) -{ - if (user_mode(regs)) - return user_exc_vmm_communication(regs, error_code); - else - return kernel_exc_vmm_communication(regs, error_code); -} -#endif - static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code) { /* Optimize for #PF. That's the only exception which matters performance wise */ diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 42bf6a58ec36..20f548702404 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -340,17 +340,14 @@ static __always_inline void __##func(struct pt_regs *regs) __visible void noist_##func(struct pt_regs *regs) /** - * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point + * DECLARE_IDTENTRY_VC - Declare a function for the VC entry point * @vector: Vector number (ignored for C) * @func: Function name of the entry point * - * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the - * safe_stack C handler. + * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE. */ #define DECLARE_IDTENTRY_VC(vector, func) \ - DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \ - __visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code); \ - __visible noinstr void user_##func(struct pt_regs *regs, unsigned long error_code) + DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); /** * DEFINE_IDTENTRY_IST - Emit code for IST entry points @@ -391,26 +388,6 @@ static __always_inline void __##func(struct pt_regs *regs) #define DEFINE_IDTENTRY_DF(func) \ DEFINE_IDTENTRY_RAW_ERRORCODE(func) -/** - * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler - * when raised from kernel mode - * @func: Function name of the entry point - * - * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE - */ -#define DEFINE_IDTENTRY_VC_KERNEL(func) \ - DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func) - -/** - * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler - * when raised from user mode - * @func: Function name of the entry point - * - * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE - */ -#define DEFINE_IDTENTRY_VC_USER(func) \ - DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func) - #else /* CONFIG_X86_64 */ /** From 39f1de2fffb3dc1751153e4c3d9138ccd958e8b1 Mon Sep 17 00:00:00 2001 From: "Tycho Andersen (AMD)" Date: Wed, 29 Apr 2026 09:56:35 -0600 Subject: [PATCH 2/4] x86/sev: Do not initialize SNP if missing CPUs The SEV firmware checks that the SNP enable bit is set on each CPU during SNP initialization, and will fail if not. If there are some CPUs offline, they will not run the setup functions, so SNP initialization will always fail. Skip the IPIs in this case and return an error so that the CCP driver can skip the SNP_INIT that will fail. Also print the CPU masks in order to leave breadcrumbs so people can figure out what happened. [ bp: Massage commit message. ] Suggested-by: Borislav Petkov (AMD) Signed-off-by: Tycho Andersen (AMD) Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Nikunj A Dadhania Reviewed-by: Tom Lendacky Link: https://20260429155636.540040-1-tycho@kernel.org --- arch/x86/include/asm/sev.h | 4 ++-- arch/x86/virt/svm/sev.c | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index 09e605c85de4..594cfa19cbd4 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -661,7 +661,7 @@ static inline void snp_leak_pages(u64 pfn, unsigned int pages) { __snp_leak_pages(pfn, pages, true); } -void snp_prepare(void); +int snp_prepare(void); void snp_shutdown(void); #else static inline bool snp_probe_rmptable_info(void) { return false; } @@ -679,7 +679,7 @@ static inline void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp) static inline void snp_leak_pages(u64 pfn, unsigned int npages) {} static inline void kdump_sev_callback(void) { } static inline void snp_fixup_e820_tables(void) {} -static inline void snp_prepare(void) {} +static inline int snp_prepare(void) { return -ENODEV; } static inline void snp_shutdown(void) {} #endif diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index 41f76f15caa1..8bcdce98f6dc 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -511,8 +511,9 @@ static void clear_hsave_pa(void *arg) wrmsrq(MSR_VM_HSAVE_PA, 0); } -void snp_prepare(void) +int snp_prepare(void) { + int ret; u64 val; /* @@ -521,12 +522,20 @@ void snp_prepare(void) */ rdmsrq(MSR_AMD64_SYSCFG, val); if (val & MSR_AMD64_SYSCFG_SNP_EN) - return; + return 0; clear_rmp(); cpus_read_lock(); + if (!cpumask_equal(cpu_online_mask, cpu_present_mask)) { + ret = -EOPNOTSUPP; + pr_warn("SNP init failed: not all CPUs online. (%*pbl online <-> %*pbl present masks).\n", + cpumask_pr_args(cpu_online_mask), + cpumask_pr_args(cpu_present_mask)); + goto unlock; + } + /* * MtrrFixDramModEn is not shared between threads on a core, * therefore it must be set on all CPUs prior to enabling SNP. @@ -537,7 +546,12 @@ void snp_prepare(void) /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ on_each_cpu(clear_hsave_pa, NULL, 1); + ret = 0; + +unlock: cpus_read_unlock(); + + return ret; } EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp"); From e9d29f4a9183b5bd355aaf996bd16ecb69e52eb1 Mon Sep 17 00:00:00 2001 From: "Tycho Andersen (AMD)" Date: Wed, 29 Apr 2026 09:56:36 -0600 Subject: [PATCH 3/4] crypto/ccp: Skip SNP_INIT if preparation fails If snp_prepare() fails, SNP_INIT will fail, so skip it and return early. Note that this is not a change in initialization behavior: if SNP_INIT fails even before this change, it will still return an error and __sev_snp_init_locked() will fail initialization of other SEV modes. Signed-off-by: Tycho Andersen (AMD) Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Nikunj A Dadhania Link: https://lore.kernel.org/20260429155636.540040-1-tycho@kernel.org --- drivers/crypto/ccp/sev-dev.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index d1e9e0ac63b6..78f98aee7a66 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1374,7 +1374,9 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) return -EOPNOTSUPP; } - snp_prepare(); + rc = snp_prepare(); + if (rc) + return rc; /* * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list From 9d8460a1c7a6b0f2dc6302e5d0f31d4e8c2a7913 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 18 May 2026 10:22:30 +0000 Subject: [PATCH 4/4] x86/sev: Remove redundant ghcbs_initialized checks around __sev_{get,put}_ghcb() After 3645eb7e3915 ("x86/fred: Fix early boot failures on SEV-ES/SNP guests"), __sev_{get,put}_ghcb() handle the early-boot GHCB fallback internally, making the ghcbs_initialized guards in __set_pages_state() and svsm_perform_call_protocol() redundant. Remove them. Also initialize state->ghcb to NULL in the early-boot path of __sev_get_ghcb() so that the ghcb_state is well-defined for all callers, even though __sev_put_ghcb() currently returns early before reading it. No functional change intended. Suggested-by: Tom Lendacky Signed-off-by: Nikunj A Dadhania Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Tom Lendacky Link: https://patch.msgid.link/20260518102230.3394603-1-nikunj@amd.com --- arch/x86/coco/sev/core.c | 8 ++------ arch/x86/coco/sev/noinstr.c | 4 +++- arch/x86/coco/sev/svsm.c | 10 ++-------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index 7ed3da998489..ecd77d3217f3 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -367,17 +367,13 @@ static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long local_irq_save(flags); - if (sev_cfg.ghcbs_initialized) - ghcb = __sev_get_ghcb(&state); - else - ghcb = boot_ghcb; + ghcb = __sev_get_ghcb(&state); /* Invoke the hypervisor to perform the page state changes */ if (!ghcb || vmgexit_psc(ghcb, data)) sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); - if (sev_cfg.ghcbs_initialized) - __sev_put_ghcb(&state); + __sev_put_ghcb(&state); local_irq_restore(flags); diff --git a/arch/x86/coco/sev/noinstr.c b/arch/x86/coco/sev/noinstr.c index 5afd663a1c21..e1e03f12fc7b 100644 --- a/arch/x86/coco/sev/noinstr.c +++ b/arch/x86/coco/sev/noinstr.c @@ -121,8 +121,10 @@ noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) WARN_ON(!irqs_disabled()); - if (!sev_cfg.ghcbs_initialized) + if (!sev_cfg.ghcbs_initialized) { + state->ghcb = NULL; return boot_ghcb; + } data = this_cpu_read(runtime_data); ghcb = &data->ghcb_page; diff --git a/arch/x86/coco/sev/svsm.c b/arch/x86/coco/sev/svsm.c index 2acf4a76afe7..916d62cd17dc 100644 --- a/arch/x86/coco/sev/svsm.c +++ b/arch/x86/coco/sev/svsm.c @@ -74,20 +74,14 @@ int svsm_perform_call_protocol(struct svsm_call *call) flags = native_local_irq_save(); - if (sev_cfg.ghcbs_initialized) - ghcb = __sev_get_ghcb(&state); - else if (boot_ghcb) - ghcb = boot_ghcb; - else - ghcb = NULL; + ghcb = __sev_get_ghcb(&state); do { ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call) : __pi_svsm_perform_msr_protocol(call); } while (ret == -EAGAIN); - if (sev_cfg.ghcbs_initialized) - __sev_put_ghcb(&state); + __sev_put_ghcb(&state); native_local_irq_restore(flags);