github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-30 10:04:04 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

x86/sev: Disentangle #VC handling code from startup code

Browse Source 
Most of the SEV support code used to reside in a single C source file
that was included in two places: the core kernel, and the decompressor.

The code that is actually shared with the decompressor was moved into a
separate, shared source file under startup/, on the basis that the
decompressor also executes from the early 1:1 mapping of memory.

However, while the elaborate #VC handling and instruction decoding that
it involves is also performed by the decompressor, it does not actually
occur in the core kernel at early boot, and therefore, does not need to
be part of the confined early startup code.

So split off the #VC handling code and move it back into arch/x86/coco
where it came from, into another C source file that is included from
both the decompressor and the core kernel.

Code movement only - no functional change intended.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: David Woodhouse <dwmw@amazon.co.uk>
Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kevin Loughlin <kevinloughlin@google.com>
Cc: Len Brown <len.brown@intel.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: linux-efi@vger.kernel.org
Link: https://lore.kernel.org/r/20250504095230.2932860-31-ardb+git@google.com
This commit is contained in:
Ard Biesheuvel 2025-05-04 11:52:36 +02:00 committed by Ingo Molnar
parent 5297886f0c
commit ed4d95d033
11 changed files with 1694 additions and 1643 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/x86/boot/compressed/misc.h
View File

@ -150,6 +150,7 @@ void sev_prep_identity_maps(unsigned long top_level_pgt);
enum es_result vc_decode_insn(struct es_em_ctxt *ctxt);
bool insn_has_rep_prefix(struct insn *insn);
void sev_insn_decode_init(void);
bool early_setup_ghcb(void);
#else
static inline void sev_enable(struct boot_params *bp)
{

83
arch/x86/boot/compressed/sev-handle-vc.c
View File

@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include "misc.h"
#include "sev.h"
#include <linux/kernel.h>
#include <linux/string.h>
@ -8,6 +9,9 @@
#include <asm/pgtable_types.h>
#include <asm/ptrace.h>
#include <asm/sev.h>
#include <asm/trapnr.h>
#include <asm/trap_pf.h>
#include <asm/fpu/xcr.h>
#define __BOOT_COMPRESSED
@ -49,3 +53,82 @@ enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
}
extern void sev_insn_decode_init(void) __alias(inat_init_tables);
/*
* Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
* doesn't use segments.
*/
static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
{
return 0UL;
}
static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
void *dst, char *buf, size_t size)
{
memcpy(dst, buf, size);
return ES_OK;
}
static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
void *src, char *buf, size_t size)
{
memcpy(buf, src, size);
return ES_OK;
}
static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
{
return ES_OK;
}
static bool fault_in_kernel_space(unsigned long address)
{
return false;
}
#define sev_printk(fmt, ...)
#include "../../coco/sev/vc-shared.c"
void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
{
struct es_em_ctxt ctxt;
enum es_result result;
if (!boot_ghcb && !early_setup_ghcb())
sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
vc_ghcb_invalidate(boot_ghcb);
result = vc_init_em_ctxt(&ctxt, regs, exit_code);
if (result != ES_OK)
goto finish;
result = vc_check_opcode_bytes(&ctxt, exit_code);
if (result != ES_OK)
goto finish;
switch (exit_code) {
case SVM_EXIT_RDTSC:
case SVM_EXIT_RDTSCP:
result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
break;
case SVM_EXIT_IOIO:
result = vc_handle_ioio(boot_ghcb, &ctxt);
break;
case SVM_EXIT_CPUID:
result = vc_handle_cpuid(boot_ghcb, &ctxt);
break;
default:
result = ES_UNSUPPORTED;
break;
}
finish:
if (result == ES_OK)
vc_finish_insn(&ctxt);
else if (result != ES_RETRY)
sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
}

96
arch/x86/boot/compressed/sev.c
View File

@ -24,63 +24,11 @@
#include <asm/cpuid.h>
#include "error.h"
#include "../msr.h"
#include "sev.h"
static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
struct ghcb *boot_ghcb;
/*
* Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
* doesn't use segments.
*/
static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
{
return 0UL;
}
static inline u64 sev_es_rd_ghcb_msr(void)
{
struct msr m;
boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
return m.q;
}
static inline void sev_es_wr_ghcb_msr(u64 val)
{
struct msr m;
m.q = val;
boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
}
static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
void *dst, char *buf, size_t size)
{
memcpy(dst, buf, size);
return ES_OK;
}
static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
void *src, char *buf, size_t size)
{
memcpy(buf, src, size);
return ES_OK;
}
static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
{
return ES_OK;
}
static bool fault_in_kernel_space(unsigned long address)
{
return false;
}
#undef __init
#define __init
@ -182,7 +130,7 @@ void snp_set_page_shared(unsigned long paddr)
__page_state_change(paddr, SNP_PAGE_STATE_SHARED);
}
static bool early_setup_ghcb(void)
bool early_setup_ghcb(void)
{
if (set_page_decrypted((unsigned long)&boot_ghcb_page))
return false;
@ -266,46 +214,6 @@ bool sev_es_check_ghcb_fault(unsigned long address)
return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
}
void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
{
struct es_em_ctxt ctxt;
enum es_result result;
if (!boot_ghcb && !early_setup_ghcb())
sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
vc_ghcb_invalidate(boot_ghcb);
result = vc_init_em_ctxt(&ctxt, regs, exit_code);
if (result != ES_OK)
goto finish;
result = vc_check_opcode_bytes(&ctxt, exit_code);
if (result != ES_OK)
goto finish;
switch (exit_code) {
case SVM_EXIT_RDTSC:
case SVM_EXIT_RDTSCP:
result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
break;
case SVM_EXIT_IOIO:
result = vc_handle_ioio(boot_ghcb, &ctxt);
break;
case SVM_EXIT_CPUID:
result = vc_handle_cpuid(boot_ghcb, &ctxt);
break;
default:
result = ES_UNSUPPORTED;
break;
}
finish:
if (result == ES_OK)
vc_finish_insn(&ctxt);
else if (result != ES_RETRY)
sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
}
/*
* SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
* guest side implementation for proper functioning of the guest. If any

19
arch/x86/boot/compressed/sev.h
View File

@ -10,10 +10,29 @@
#ifdef CONFIG_AMD_MEM_ENCRYPT
#include "../msr.h"
void snp_accept_memory(phys_addr_t start, phys_addr_t end);
u64 sev_get_status(void);
bool early_is_sevsnp_guest(void);
static inline u64 sev_es_rd_ghcb_msr(void)
{
struct msr m;
boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
return m.q;
}
static inline void sev_es_wr_ghcb_msr(u64 val)
{
struct msr m;
m.q = val;
boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
}
#else
static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }

519
arch/x86/boot/startup/sev-shared.c
View File

@ -14,13 +14,9 @@
#ifndef __BOOT_COMPRESSED
#define error(v) pr_err(v)
#define has_cpuflag(f) boot_cpu_has(f)
#define sev_printk(fmt, ...) printk(fmt, ##__VA_ARGS__)
#define sev_printk_rtl(fmt, ...) printk_ratelimited(fmt, ##__VA_ARGS__)
#else
#undef WARN
#define WARN(condition, format...) (!!(condition))
#define sev_printk(fmt, ...)
#define sev_printk_rtl(fmt, ...)
#undef vc_forward_exception
#define vc_forward_exception(c) panic("SNP: Hypervisor requested exception\n")
#endif
@ -36,16 +32,6 @@
struct svsm_ca *boot_svsm_caa __ro_after_init;
u64 boot_svsm_caa_pa __ro_after_init;
/* I/O parameters for CPUID-related helpers */
struct cpuid_leaf {
u32 fn;
u32 subfn;
u32 eax;
u32 ebx;
u32 ecx;
u32 edx;
};
/*
* Since feature negotiation related variables are set early in the boot
* process they must reside in the .data section so as not to be zeroed
@ -151,33 +137,6 @@ bool sev_es_negotiate_protocol(void)
return true;
}
static bool vc_decoding_needed(unsigned long exit_code)
{
/* Exceptions don't require to decode the instruction */
return !(exit_code >= SVM_EXIT_EXCP_BASE &&
exit_code <= SVM_EXIT_LAST_EXCP);
}
static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
struct pt_regs *regs,
unsigned long exit_code)
{
enum es_result ret = ES_OK;
memset(ctxt, 0, sizeof(*ctxt));
ctxt->regs = regs;
if (vc_decoding_needed(exit_code))
ret = vc_decode_insn(ctxt);
return ret;
}
static void vc_finish_insn(struct es_em_ctxt *ctxt)
{
ctxt->regs->ip += ctxt->insn.length;
}
static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
{
u32 ret;
@ -627,7 +586,7 @@ snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
* Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
* should be treated as fatal by caller.
*/
static int __head
int __head
snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
{
const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
@ -737,391 +696,6 @@ void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
}
static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
unsigned long address,
bool write)
{
if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
ctxt->fi.vector = X86_TRAP_PF;
ctxt->fi.error_code = X86_PF_USER;
ctxt->fi.cr2 = address;
if (write)
ctxt->fi.error_code |= X86_PF_WRITE;
return ES_EXCEPTION;
}
return ES_OK;
}
static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
void *src, char *buf,
unsigned int data_size,
unsigned int count,
bool backwards)
{
int i, b = backwards ? -1 : 1;
unsigned long address = (unsigned long)src;
enum es_result ret;
ret = vc_insn_string_check(ctxt, address, false);
if (ret != ES_OK)
return ret;
for (i = 0; i < count; i++) {
void *s = src + (i * data_size * b);
char *d = buf + (i * data_size);
ret = vc_read_mem(ctxt, s, d, data_size);
if (ret != ES_OK)
break;
}
return ret;
}
static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
void *dst, char *buf,
unsigned int data_size,
unsigned int count,
bool backwards)
{
int i, s = backwards ? -1 : 1;
unsigned long address = (unsigned long)dst;
enum es_result ret;
ret = vc_insn_string_check(ctxt, address, true);
if (ret != ES_OK)
return ret;
for (i = 0; i < count; i++) {
void *d = dst + (i * data_size * s);
char *b = buf + (i * data_size);
ret = vc_write_mem(ctxt, d, b, data_size);
if (ret != ES_OK)
break;
}
return ret;
}
#define IOIO_TYPE_STR BIT(2)
#define IOIO_TYPE_IN 1
#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
#define IOIO_TYPE_OUT 0
#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
#define IOIO_REP BIT(3)
#define IOIO_ADDR_64 BIT(9)
#define IOIO_ADDR_32 BIT(8)
#define IOIO_ADDR_16 BIT(7)
#define IOIO_DATA_32 BIT(6)
#define IOIO_DATA_16 BIT(5)
#define IOIO_DATA_8 BIT(4)
#define IOIO_SEG_ES (0 << 10)
#define IOIO_SEG_DS (3 << 10)
static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
{
struct insn *insn = &ctxt->insn;
size_t size;
u64 port;
*exitinfo = 0;
switch (insn->opcode.bytes[0]) {
/* INS opcodes */
case 0x6c:
case 0x6d:
*exitinfo |= IOIO_TYPE_INS;
*exitinfo |= IOIO_SEG_ES;
port = ctxt->regs->dx & 0xffff;
break;
/* OUTS opcodes */
case 0x6e:
case 0x6f:
*exitinfo |= IOIO_TYPE_OUTS;
*exitinfo |= IOIO_SEG_DS;
port = ctxt->regs->dx & 0xffff;
break;
/* IN immediate opcodes */
case 0xe4:
case 0xe5:
*exitinfo |= IOIO_TYPE_IN;
port = (u8)insn->immediate.value & 0xffff;
break;
/* OUT immediate opcodes */
case 0xe6:
case 0xe7:
*exitinfo |= IOIO_TYPE_OUT;
port = (u8)insn->immediate.value & 0xffff;
break;
/* IN register opcodes */
case 0xec:
case 0xed:
*exitinfo |= IOIO_TYPE_IN;
port = ctxt->regs->dx & 0xffff;
break;
/* OUT register opcodes */
case 0xee:
case 0xef:
*exitinfo |= IOIO_TYPE_OUT;
port = ctxt->regs->dx & 0xffff;
break;
default:
return ES_DECODE_FAILED;
}
*exitinfo |= port << 16;
switch (insn->opcode.bytes[0]) {
case 0x6c:
case 0x6e:
case 0xe4:
case 0xe6:
case 0xec:
case 0xee:
/* Single byte opcodes */
*exitinfo |= IOIO_DATA_8;
size = 1;
break;
default:
/* Length determined by instruction parsing */
*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
: IOIO_DATA_32;
size = (insn->opnd_bytes == 2) ? 2 : 4;
}
switch (insn->addr_bytes) {
case 2:
*exitinfo |= IOIO_ADDR_16;
break;
case 4:
*exitinfo |= IOIO_ADDR_32;
break;
case 8:
*exitinfo |= IOIO_ADDR_64;
break;
}
if (insn_has_rep_prefix(insn))
*exitinfo |= IOIO_REP;
return vc_ioio_check(ctxt, (u16)port, size);
}
static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
{
struct pt_regs *regs = ctxt->regs;
u64 exit_info_1, exit_info_2;
enum es_result ret;
ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
if (ret != ES_OK)
return ret;
if (exit_info_1 & IOIO_TYPE_STR) {
/* (REP) INS/OUTS */
bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
unsigned int io_bytes, exit_bytes;
unsigned int ghcb_count, op_count;
unsigned long es_base;
u64 sw_scratch;
/*
* For the string variants with rep prefix the amount of in/out
* operations per #VC exception is limited so that the kernel
* has a chance to take interrupts and re-schedule while the
* instruction is emulated.
*/
io_bytes = (exit_info_1 >> 4) & 0x7;
ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
exit_info_2 = min(op_count, ghcb_count);
exit_bytes = exit_info_2 * io_bytes;
es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
/* Read bytes of OUTS into the shared buffer */
if (!(exit_info_1 & IOIO_TYPE_IN)) {
ret = vc_insn_string_read(ctxt,
(void *)(es_base + regs->si),
ghcb->shared_buffer, io_bytes,
exit_info_2, df);
if (ret)
return ret;
}
/*
* Issue an VMGEXIT to the HV to consume the bytes from the
* shared buffer or to have it write them into the shared buffer
* depending on the instruction: OUTS or INS.
*/
sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
ghcb_set_sw_scratch(ghcb, sw_scratch);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
exit_info_1, exit_info_2);
if (ret != ES_OK)
return ret;
/* Read bytes from shared buffer into the guest's destination. */
if (exit_info_1 & IOIO_TYPE_IN) {
ret = vc_insn_string_write(ctxt,
(void *)(es_base + regs->di),
ghcb->shared_buffer, io_bytes,
exit_info_2, df);
if (ret)
return ret;
if (df)
regs->di -= exit_bytes;
else
regs->di += exit_bytes;
} else {
if (df)
regs->si -= exit_bytes;
else
regs->si += exit_bytes;
}
if (exit_info_1 & IOIO_REP)
regs->cx -= exit_info_2;
ret = regs->cx ? ES_RETRY : ES_OK;
} else {
/* IN/OUT into/from rAX */
int bits = (exit_info_1 & 0x70) >> 1;
u64 rax = 0;
if (!(exit_info_1 & IOIO_TYPE_IN))
rax = lower_bits(regs->ax, bits);
ghcb_set_rax(ghcb, rax);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
if (ret != ES_OK)
return ret;
if (exit_info_1 & IOIO_TYPE_IN) {
if (!ghcb_rax_is_valid(ghcb))
return ES_VMM_ERROR;
regs->ax = lower_bits(ghcb->save.rax, bits);
}
}
return ret;
}
static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
{
struct pt_regs *regs = ctxt->regs;
struct cpuid_leaf leaf;
int ret;
leaf.fn = regs->ax;
leaf.subfn = regs->cx;
ret = snp_cpuid(ghcb, ctxt, &leaf);
if (!ret) {
regs->ax = leaf.eax;
regs->bx = leaf.ebx;
regs->cx = leaf.ecx;
regs->dx = leaf.edx;
}
return ret;
}
static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
struct es_em_ctxt *ctxt)
{
struct pt_regs *regs = ctxt->regs;
u32 cr4 = native_read_cr4();
enum es_result ret;
int snp_cpuid_ret;
snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
if (!snp_cpuid_ret)
return ES_OK;
if (snp_cpuid_ret != -EOPNOTSUPP)
return ES_VMM_ERROR;
ghcb_set_rax(ghcb, regs->ax);
ghcb_set_rcx(ghcb, regs->cx);
if (cr4 & X86_CR4_OSXSAVE)
/* Safe to read xcr0 */
ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
else
/* xgetbv will cause #GP - use reset value for xcr0 */
ghcb_set_xcr0(ghcb, 1);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
if (ret != ES_OK)
return ret;
if (!(ghcb_rax_is_valid(ghcb) &&
ghcb_rbx_is_valid(ghcb) &&
ghcb_rcx_is_valid(ghcb) &&
ghcb_rdx_is_valid(ghcb)))
return ES_VMM_ERROR;
regs->ax = ghcb->save.rax;
regs->bx = ghcb->save.rbx;
regs->cx = ghcb->save.rcx;
regs->dx = ghcb->save.rdx;
return ES_OK;
}
static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
struct es_em_ctxt *ctxt,
unsigned long exit_code)
{
bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
enum es_result ret;
/*
* The hypervisor should not be intercepting RDTSC/RDTSCP when Secure
* TSC is enabled. A #VC exception will be generated if the RDTSC/RDTSCP
* instructions are being intercepted. If this should occur and Secure
* TSC is enabled, guest execution should be terminated as the guest
* cannot rely on the TSC value provided by the hypervisor.
*/
if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
return ES_VMM_ERROR;
ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
if (ret != ES_OK)
return ret;
if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
(!rdtscp || ghcb_rcx_is_valid(ghcb))))
return ES_VMM_ERROR;
ctxt->regs->ax = ghcb->save.rax;
ctxt->regs->dx = ghcb->save.rdx;
if (rdtscp)
ctxt->regs->cx = ghcb->save.rcx;
return ES_OK;
}
struct cc_setup_data {
struct setup_data header;
u32 cc_blob_address;
@ -1238,97 +812,6 @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr,
}
}
static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt,
unsigned long exit_code)
{
unsigned int opcode = (unsigned int)ctxt->insn.opcode.value;
u8 modrm = ctxt->insn.modrm.value;
switch (exit_code) {
case SVM_EXIT_IOIO:
case SVM_EXIT_NPF:
/* handled separately */
return ES_OK;
case SVM_EXIT_CPUID:
if (opcode == 0xa20f)
return ES_OK;
break;
case SVM_EXIT_INVD:
if (opcode == 0x080f)
return ES_OK;
break;
case SVM_EXIT_MONITOR:
/* MONITOR and MONITORX instructions generate the same error code */
if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa))
return ES_OK;
break;
case SVM_EXIT_MWAIT:
/* MWAIT and MWAITX instructions generate the same error code */
if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb))
return ES_OK;
break;
case SVM_EXIT_MSR:
/* RDMSR */
if (opcode == 0x320f ||
/* WRMSR */
opcode == 0x300f)
return ES_OK;
break;
case SVM_EXIT_RDPMC:
if (opcode == 0x330f)
return ES_OK;
break;
case SVM_EXIT_RDTSC:
if (opcode == 0x310f)
return ES_OK;
break;
case SVM_EXIT_RDTSCP:
if (opcode == 0x010f && modrm == 0xf9)
return ES_OK;
break;
case SVM_EXIT_READ_DR7:
if (opcode == 0x210f &&
X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
return ES_OK;
break;
case SVM_EXIT_VMMCALL:
if (opcode == 0x010f && modrm == 0xd9)
return ES_OK;
break;
case SVM_EXIT_WRITE_DR7:
if (opcode == 0x230f &&
X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
return ES_OK;
break;
case SVM_EXIT_WBINVD:
if (opcode == 0x90f)
return ES_OK;
break;
default:
break;
}
sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n",
opcode, exit_code, ctxt->regs->ip);
return ES_UNSUPPORTED;
}
/*
* Maintain the GPA of the SVSM Calling Area (CA) in order to utilize the SVSM
* services needed when not running in VMPL0.

1022
arch/x86/boot/startup/sev-startup.c
View File

File diff suppressed because it is too large Load Diff

2
arch/x86/coco/sev/Makefile
View File

@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
obj-y += core.o sev-nmi.o
obj-y += core.o sev-nmi.o vc-handle.o
# Clang 14 and older may fail to respect __no_sanitize_undefined when inlining
UBSAN_SANITIZE_sev-nmi.o := n

1061
arch/x86/coco/sev/vc-handle.c Normal file
View File

File diff suppressed because it is too large Load Diff

504
arch/x86/coco/sev/vc-shared.c Normal file
View File

@ -0,0 +1,504 @@
// SPDX-License-Identifier: GPL-2.0
static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt,
unsigned long exit_code)
{
unsigned int opcode = (unsigned int)ctxt->insn.opcode.value;
u8 modrm = ctxt->insn.modrm.value;
switch (exit_code) {
case SVM_EXIT_IOIO:
case SVM_EXIT_NPF:
/* handled separately */
return ES_OK;
case SVM_EXIT_CPUID:
if (opcode == 0xa20f)
return ES_OK;
break;
case SVM_EXIT_INVD:
if (opcode == 0x080f)
return ES_OK;
break;
case SVM_EXIT_MONITOR:
/* MONITOR and MONITORX instructions generate the same error code */
if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa))
return ES_OK;
break;
case SVM_EXIT_MWAIT:
/* MWAIT and MWAITX instructions generate the same error code */
if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb))
return ES_OK;
break;
case SVM_EXIT_MSR:
/* RDMSR */
if (opcode == 0x320f ||
/* WRMSR */
opcode == 0x300f)
return ES_OK;
break;
case SVM_EXIT_RDPMC:
if (opcode == 0x330f)
return ES_OK;
break;
case SVM_EXIT_RDTSC:
if (opcode == 0x310f)
return ES_OK;
break;
case SVM_EXIT_RDTSCP:
if (opcode == 0x010f && modrm == 0xf9)
return ES_OK;
break;
case SVM_EXIT_READ_DR7:
if (opcode == 0x210f &&
X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
return ES_OK;
break;
case SVM_EXIT_VMMCALL:
if (opcode == 0x010f && modrm == 0xd9)
return ES_OK;
break;
case SVM_EXIT_WRITE_DR7:
if (opcode == 0x230f &&
X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
return ES_OK;
break;
case SVM_EXIT_WBINVD:
if (opcode == 0x90f)
return ES_OK;
break;
default:
break;
}
sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n",
opcode, exit_code, ctxt->regs->ip);
return ES_UNSUPPORTED;
}
static bool vc_decoding_needed(unsigned long exit_code)
{
/* Exceptions don't require to decode the instruction */
return !(exit_code >= SVM_EXIT_EXCP_BASE &&
exit_code <= SVM_EXIT_LAST_EXCP);
}
static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
struct pt_regs *regs,
unsigned long exit_code)
{
enum es_result ret = ES_OK;
memset(ctxt, 0, sizeof(*ctxt));
ctxt->regs = regs;
if (vc_decoding_needed(exit_code))
ret = vc_decode_insn(ctxt);
return ret;
}
static void vc_finish_insn(struct es_em_ctxt *ctxt)
{
ctxt->regs->ip += ctxt->insn.length;
}
static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
unsigned long address,
bool write)
{
if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
ctxt->fi.vector = X86_TRAP_PF;
ctxt->fi.error_code = X86_PF_USER;
ctxt->fi.cr2 = address;
if (write)
ctxt->fi.error_code |= X86_PF_WRITE;
return ES_EXCEPTION;
}
return ES_OK;
}
static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
void *src, char *buf,
unsigned int data_size,
unsigned int count,
bool backwards)
{
int i, b = backwards ? -1 : 1;
unsigned long address = (unsigned long)src;
enum es_result ret;
ret = vc_insn_string_check(ctxt, address, false);
if (ret != ES_OK)
return ret;
for (i = 0; i < count; i++) {
void *s = src + (i * data_size * b);
char *d = buf + (i * data_size);
ret = vc_read_mem(ctxt, s, d, data_size);
if (ret != ES_OK)
break;
}
return ret;
}
static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
void *dst, char *buf,
unsigned int data_size,
unsigned int count,
bool backwards)
{
int i, s = backwards ? -1 : 1;
unsigned long address = (unsigned long)dst;
enum es_result ret;
ret = vc_insn_string_check(ctxt, address, true);
if (ret != ES_OK)
return ret;
for (i = 0; i < count; i++) {
void *d = dst + (i * data_size * s);
char *b = buf + (i * data_size);
ret = vc_write_mem(ctxt, d, b, data_size);
if (ret != ES_OK)
break;
}
return ret;
}
#define IOIO_TYPE_STR BIT(2)
#define IOIO_TYPE_IN 1
#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
#define IOIO_TYPE_OUT 0
#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
#define IOIO_REP BIT(3)
#define IOIO_ADDR_64 BIT(9)
#define IOIO_ADDR_32 BIT(8)
#define IOIO_ADDR_16 BIT(7)
#define IOIO_DATA_32 BIT(6)
#define IOIO_DATA_16 BIT(5)
#define IOIO_DATA_8 BIT(4)
#define IOIO_SEG_ES (0 << 10)
#define IOIO_SEG_DS (3 << 10)
static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
{
struct insn *insn = &ctxt->insn;
size_t size;
u64 port;
*exitinfo = 0;
switch (insn->opcode.bytes[0]) {
/* INS opcodes */
case 0x6c:
case 0x6d:
*exitinfo |= IOIO_TYPE_INS;
*exitinfo |= IOIO_SEG_ES;
port = ctxt->regs->dx & 0xffff;
break;
/* OUTS opcodes */
case 0x6e:
case 0x6f:
*exitinfo |= IOIO_TYPE_OUTS;
*exitinfo |= IOIO_SEG_DS;
port = ctxt->regs->dx & 0xffff;
break;
/* IN immediate opcodes */
case 0xe4:
case 0xe5:
*exitinfo |= IOIO_TYPE_IN;
port = (u8)insn->immediate.value & 0xffff;
break;
/* OUT immediate opcodes */
case 0xe6:
case 0xe7:
*exitinfo |= IOIO_TYPE_OUT;
port = (u8)insn->immediate.value & 0xffff;
break;
/* IN register opcodes */
case 0xec:
case 0xed:
*exitinfo |= IOIO_TYPE_IN;
port = ctxt->regs->dx & 0xffff;
break;
/* OUT register opcodes */
case 0xee:
case 0xef:
*exitinfo |= IOIO_TYPE_OUT;
port = ctxt->regs->dx & 0xffff;
break;
default:
return ES_DECODE_FAILED;
}
*exitinfo |= port << 16;
switch (insn->opcode.bytes[0]) {
case 0x6c:
case 0x6e:
case 0xe4:
case 0xe6:
case 0xec:
case 0xee:
/* Single byte opcodes */
*exitinfo |= IOIO_DATA_8;
size = 1;
break;
default:
/* Length determined by instruction parsing */
*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
: IOIO_DATA_32;
size = (insn->opnd_bytes == 2) ? 2 : 4;
}
switch (insn->addr_bytes) {
case 2:
*exitinfo |= IOIO_ADDR_16;
break;
case 4:
*exitinfo |= IOIO_ADDR_32;
break;
case 8:
*exitinfo |= IOIO_ADDR_64;
break;
}
if (insn_has_rep_prefix(insn))
*exitinfo |= IOIO_REP;
return vc_ioio_check(ctxt, (u16)port, size);
}
static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
{
struct pt_regs *regs = ctxt->regs;
u64 exit_info_1, exit_info_2;
enum es_result ret;
ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
if (ret != ES_OK)
return ret;
if (exit_info_1 & IOIO_TYPE_STR) {
/* (REP) INS/OUTS */
bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
unsigned int io_bytes, exit_bytes;
unsigned int ghcb_count, op_count;
unsigned long es_base;
u64 sw_scratch;
/*
* For the string variants with rep prefix the amount of in/out
* operations per #VC exception is limited so that the kernel
* has a chance to take interrupts and re-schedule while the
* instruction is emulated.
*/
io_bytes = (exit_info_1 >> 4) & 0x7;
ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
exit_info_2 = min(op_count, ghcb_count);
exit_bytes = exit_info_2 * io_bytes;
es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
/* Read bytes of OUTS into the shared buffer */
if (!(exit_info_1 & IOIO_TYPE_IN)) {
ret = vc_insn_string_read(ctxt,
(void *)(es_base + regs->si),
ghcb->shared_buffer, io_bytes,
exit_info_2, df);
if (ret)
return ret;
}
/*
* Issue an VMGEXIT to the HV to consume the bytes from the
* shared buffer or to have it write them into the shared buffer
* depending on the instruction: OUTS or INS.
*/
sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
ghcb_set_sw_scratch(ghcb, sw_scratch);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
exit_info_1, exit_info_2);
if (ret != ES_OK)
return ret;
/* Read bytes from shared buffer into the guest's destination. */
if (exit_info_1 & IOIO_TYPE_IN) {
ret = vc_insn_string_write(ctxt,
(void *)(es_base + regs->di),
ghcb->shared_buffer, io_bytes,
exit_info_2, df);
if (ret)
return ret;
if (df)
regs->di -= exit_bytes;
else
regs->di += exit_bytes;
} else {
if (df)
regs->si -= exit_bytes;
else
regs->si += exit_bytes;
}
if (exit_info_1 & IOIO_REP)
regs->cx -= exit_info_2;
ret = regs->cx ? ES_RETRY : ES_OK;
} else {
/* IN/OUT into/from rAX */
int bits = (exit_info_1 & 0x70) >> 1;
u64 rax = 0;
if (!(exit_info_1 & IOIO_TYPE_IN))
rax = lower_bits(regs->ax, bits);
ghcb_set_rax(ghcb, rax);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
if (ret != ES_OK)
return ret;
if (exit_info_1 & IOIO_TYPE_IN) {
if (!ghcb_rax_is_valid(ghcb))
return ES_VMM_ERROR;
regs->ax = lower_bits(ghcb->save.rax, bits);
}
}
return ret;
}
static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
{
struct pt_regs *regs = ctxt->regs;
struct cpuid_leaf leaf;
int ret;
leaf.fn = regs->ax;
leaf.subfn = regs->cx;
ret = snp_cpuid(ghcb, ctxt, &leaf);
if (!ret) {
regs->ax = leaf.eax;
regs->bx = leaf.ebx;
regs->cx = leaf.ecx;
regs->dx = leaf.edx;
}
return ret;
}
static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
struct es_em_ctxt *ctxt)
{
struct pt_regs *regs = ctxt->regs;
u32 cr4 = native_read_cr4();
enum es_result ret;
int snp_cpuid_ret;
snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
if (!snp_cpuid_ret)
return ES_OK;
if (snp_cpuid_ret != -EOPNOTSUPP)
return ES_VMM_ERROR;
ghcb_set_rax(ghcb, regs->ax);
ghcb_set_rcx(ghcb, regs->cx);
if (cr4 & X86_CR4_OSXSAVE)
/* Safe to read xcr0 */
ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
else
/* xgetbv will cause #GP - use reset value for xcr0 */
ghcb_set_xcr0(ghcb, 1);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
if (ret != ES_OK)
return ret;
if (!(ghcb_rax_is_valid(ghcb) &&
ghcb_rbx_is_valid(ghcb) &&
ghcb_rcx_is_valid(ghcb) &&
ghcb_rdx_is_valid(ghcb)))
return ES_VMM_ERROR;
regs->ax = ghcb->save.rax;
regs->bx = ghcb->save.rbx;
regs->cx = ghcb->save.rcx;
regs->dx = ghcb->save.rdx;
return ES_OK;
}
static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
struct es_em_ctxt *ctxt,
unsigned long exit_code)
{
bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
enum es_result ret;
/*
* The hypervisor should not be intercepting RDTSC/RDTSCP when Secure
* TSC is enabled. A #VC exception will be generated if the RDTSC/RDTSCP
* instructions are being intercepted. If this should occur and Secure
* TSC is enabled, guest execution should be terminated as the guest
* cannot rely on the TSC value provided by the hypervisor.
*/
if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
return ES_VMM_ERROR;
ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
if (ret != ES_OK)
return ret;
if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
(!rdtscp || ghcb_rcx_is_valid(ghcb))))
return ES_VMM_ERROR;
ctxt->regs->ax = ghcb->save.rax;
ctxt->regs->dx = ghcb->save.rdx;
if (rdtscp)
ctxt->regs->cx = ghcb->save.rcx;
return ES_OK;
}

8
arch/x86/include/asm/sev-internal.h
View File

@ -3,7 +3,6 @@
#define DR7_RESET_VALUE 0x400
extern struct ghcb boot_ghcb_page;
extern struct ghcb *boot_ghcb;
extern u64 sev_hv_features;
extern u64 sev_secrets_pa;
@ -59,8 +58,6 @@ DECLARE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
void early_set_pages_state(unsigned long vaddr, unsigned long paddr,
unsigned long npages, enum psc_op op);
void __noreturn sev_es_terminate(unsigned int set, unsigned int reason);
DECLARE_PER_CPU(struct svsm_ca *, svsm_caa);
DECLARE_PER_CPU(u64, svsm_caa_pa);
@ -100,11 +97,6 @@ static __always_inline void sev_es_wr_ghcb_msr(u64 val)
native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
}
enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
struct es_em_ctxt *ctxt,
u64 exit_code, u64 exit_info_1,
u64 exit_info_2);
void snp_register_ghcb_early(unsigned long paddr);
bool sev_es_negotiate_protocol(void);
bool sev_es_check_cpu_features(void);

22
arch/x86/include/asm/sev.h
View File

@ -521,6 +521,28 @@ static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
}
void vc_forward_exception(struct es_em_ctxt *ctxt);
/* I/O parameters for CPUID-related helpers */
struct cpuid_leaf {
u32 fn;
u32 subfn;
u32 eax;
u32 ebx;
u32 ecx;
u32 edx;
};
int snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf);
void __noreturn sev_es_terminate(unsigned int set, unsigned int reason);
enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
struct es_em_ctxt *ctxt,
u64 exit_code, u64 exit_info_1,
u64 exit_info_2);
extern struct ghcb *boot_ghcb;
#else /* !CONFIG_AMD_MEM_ENCRYPT */
#define snp_vmpl 0