diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 8c62c6d4d5c1..cb8ac4b9b0d7 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -488,8 +488,10 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, if (irqfd->irq_entry.type == KVM_IRQ_ROUTING_MSI) { ret = kvm_pi_update_irte(irqfd, &irqfd->irq_entry); - if (ret) + if (ret) { kvm->arch.nr_possible_bypass_irqs--; + irqfd->producer = NULL; + } } spin_unlock_irq(&kvm->irqfds.lock); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 6f30bbdddb5a..38bba9a1114c 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -3371,6 +3371,12 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, struct kvm_lapic *apic) { int vector; + + if (unlikely(!pv_eoi_enabled(vcpu))) { + __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention); + return; + } + /* * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host * and KVM_PV_EOI_ENABLED in guest memory as follows: @@ -3382,8 +3388,6 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu, * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset: * -> host enabled PV EOI, guest executed EOI. */ - BUG_ON(!pv_eoi_enabled(vcpu)); - if (pv_eoi_test_and_clr_pending(vcpu)) return; vector = apic_set_eoi(apic); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 427229347876..944aaea6501f 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2129,8 +2129,9 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (ret) return ret; + /* Do not allow SNP VM migration until additional state transfer is implemented */ if (kvm->arch.vm_type != source_kvm->arch.vm_type || - sev_guest(kvm) || !sev_guest(source_kvm)) { + sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm)) { ret = -EINVAL; goto out_unlock; } @@ -2851,8 +2852,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) * disallow out-of-band SEV/SEV-ES init if the target is already an * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being * created after SEV/SEV-ES initialization, e.g. to init intercepts. + * Also do not allow SNP VM mirroring until additional state transfer is implemented. */ - if (sev_guest(kvm) || !sev_guest(source_kvm) || + if (sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm) || is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) { ret = -EINVAL; goto e_unlock; diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 0db25bba17f6..93de876c318c 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -490,7 +490,7 @@ TRACE_EVENT(kvm_inj_exception, TP_printk("%s%s%s%s%s", __print_symbolic(__entry->exception, kvm_trace_sym_exc), !__entry->has_error ? "" : " (", - !__entry->has_error ? "" : __print_symbolic(__entry->error_code, { }), + !__entry->has_error ? "" : __print_symbolic(__entry->error_code), !__entry->has_error ? "" : ")", __entry->reinjected ? " [reinjected]" : "") ); diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6957bb6f5cf7..bb0eb40b4448 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -582,6 +582,9 @@ static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { + gpa_t vtpr_gpa = vmcs12->virtual_apic_page_addr + APIC_TASKPRI; + u32 vtpr; + if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) return 0; @@ -591,6 +594,32 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4)) return -EINVAL; + /* + * Do the illegal vTPR vs. TPR Threshold consistency check if and only + * if KVM is configured to WARN on missed consistency checks, otherwise + * it's a waste of time. KVM needs to rely on hardware to fully detect + * an illegal combination due to the vTPR being writable by L1 at all + * times (it's an in-memory value, not a VMCS field). I.e. even if the + * check passes now, it might fail at the actual VM-Enter. + * + * If reading guest memory fails, skip the check as KVM's de facto ABI + * for VMX instruction accesses to non-existent memory is to provide + * PCI Bus Error semantics (reads return 0xFFs), in which case the vTPR + * is guaranteed to greater than or equal to the threshold. + * + * Note! Deliberately use the VM-scoped API when reading guest memory, + * to ensure the read doesn't hit SMRAM when restoring L2 state on RSM, + * and only perform the check when in KVM_RUN, to avoid a false failure + * if userspace hasn't yet configured memslots during state restore. + */ + if (warn_on_missed_cc && vcpu->wants_to_run && + nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && + !nested_cpu_has_vid(vmcs12) && + !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && + !kvm_read_guest(vcpu->kvm, vtpr_gpa, &vtpr, sizeof(vtpr)) && + CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > ((vtpr >> 4) & GENMASK(3, 0)))) + return -EINVAL; + return 0; } @@ -3104,38 +3133,6 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, return 0; } -static int nested_vmx_check_controls_late(struct kvm_vcpu *vcpu, - struct vmcs12 *vmcs12) -{ - void *vapic = to_vmx(vcpu)->nested.virtual_apic_map.hva; - u32 vtpr = vapic ? (*(u32 *)(vapic + APIC_TASKPRI)) >> 4 : 0; - - /* - * Don't bother with the consistency checks if KVM isn't configured to - * WARN on missed consistency checks, as KVM needs to rely on hardware - * to fully detect an illegal vTPR vs. TRP Threshold combination due to - * the vTPR being writable by L1 at all times (it's an in-memory value, - * not a VMCS field). I.e. even if the check passes now, it might fail - * at the actual VM-Enter. - * - * Keying off the module param also allows treating an invalid vAPIC - * mapping as a consistency check failure without increasing the risk - * of breaking a "real" VM. - */ - if (!warn_on_missed_cc) - return 0; - - if ((exec_controls_get(to_vmx(vcpu)) & CPU_BASED_TPR_SHADOW) && - nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && - !nested_cpu_has_vid(vmcs12) && - !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && - (CC(!vapic) || - CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > (vtpr & GENMASK(3, 0))))) - return -EINVAL; - - return 0; -} - static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { @@ -3661,19 +3658,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, &vmx->nested.pre_vmenter_ssp_tbl); /* - * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled. In the - * event of a "late" VM-Fail, i.e. a VM-Fail detected by hardware but - * not KVM, KVM must unwind its software model to the pre-VM-Entry host - * state. When EPT is disabled, GUEST_CR3 holds KVM's shadow CR3, not - * L1's "real" CR3, which causes nested_vmx_restore_host_state() to - * corrupt vcpu->arch.cr3. Stuffing vmcs01.GUEST_CR3 results in the - * unwind naturally setting arch.cr3 to the correct value. Smashing - * vmcs01.GUEST_CR3 is safe because nested VM-Exits, and the unwind, - * reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is guaranteed to be - * overwritten with a shadow CR3 prior to re-entering L1. + * Stash L1's CR3, so that in the event of a "late" VM-Fail, i.e. a + * VM-Fail detected by hardware but not KVM, KVM can unwind its + * software model to the pre-VM-Entry host state. When EPT is + * disabled, GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, + * and so simply restoring from vmcs01.GUEST_CR3 would corrupt + * vcpu->arch.cr3. */ - if (!enable_ept) - vmcs_writel(GUEST_CR3, vcpu->arch.cr3); + vmx->nested.pre_vmenter_cr3 = kvm_read_cr3(vcpu); vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); @@ -3685,11 +3677,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, return NVMX_VMENTRY_KVM_INTERNAL_ERROR; } - if (nested_vmx_check_controls_late(vcpu, vmcs12)) { - vmx_switch_vmcs(vcpu, &vmx->vmcs01); - return NVMX_VMENTRY_VMFAIL; - } - if (nested_vmx_check_guest_state(vcpu, vmcs12, &entry_failure_code)) { exit_reason.basic = EXIT_REASON_INVALID_STATE; @@ -4990,7 +4977,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); nested_ept_uninit_mmu_context(vcpu); - vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); + vcpu->arch.cr3 = vmx->nested.pre_vmenter_cr3; kvm_register_mark_available(vcpu, VCPU_REG_CR3); /* diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 989ab29b8c6f..545b03d9d10b 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2797,7 +2797,11 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) goto out; } - if (init_vm->cpuid.padding) { + /* + * Reject the request if userspace changes cpuid.nent between the + * initial read and the subsequent copy. + */ + if (init_vm->cpuid.padding || init_vm->cpuid.nent != nr_user_entries) { ret = -EINVAL; goto out; } diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index de9de0d2016c..dc8517f15bc4 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -159,6 +159,13 @@ struct nested_vmx { bool has_preemption_timer_deadline; bool preemption_timer_expired; + /* + * Used to restore L1's CR3 if hardware detects a VM-Fail Consistency + * Check that KVM does not, in which case KVM needs to unwind CR3 back + * to its pre-VM-Enter state, NOT to vmcs01.HOST_CR3. + */ + unsigned long pre_vmenter_cr3; + /* * Used to snapshot MSRs that are conditionally loaded on VM-Enter in * order to propagate the guest's pre-VM-Enter value into vmcs02. For diff --git a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c index 42bc023d5193..d59abb198d86 100644 --- a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c +++ b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c @@ -313,6 +313,49 @@ static void test_sev_mirror_parameters(void) kvm_vm_free(vm_no_vcpu); } +static void test_sev_snp_migrate_reject(void) +{ + struct kvm_vm *src_vm, *dst_vm; + int ret; + + src_vm = vm_create_barebones_type(KVM_X86_SNP_VM); + snp_vm_init(src_vm); + __vm_vcpu_add(src_vm, 0); + vm_sev_launch(src_vm, snp_default_policy(), NULL); + + dst_vm = vm_create_barebones_type(KVM_X86_SNP_VM); + __vm_vcpu_add(dst_vm, 0); + + ret = __sev_migrate_from(dst_vm, src_vm); + TEST_ASSERT(ret == -1 && errno == EINVAL, + "SNP VM migration should be rejected. ret: %d, errno: %d", + ret, errno); + + kvm_vm_free(src_vm); + kvm_vm_free(dst_vm); +} + +static void test_sev_snp_mirror_reject(void) +{ + struct kvm_vm *src_vm, *dst_vm; + int ret; + + src_vm = vm_create_barebones_type(KVM_X86_SNP_VM); + snp_vm_init(src_vm); + __vm_vcpu_add(src_vm, 0); + vm_sev_launch(src_vm, snp_default_policy(), NULL); + + dst_vm = aux_vm_create(false); + + ret = __sev_mirror_create(dst_vm, src_vm); + TEST_ASSERT(ret == -1 && errno == EINVAL, + "SNP VM mirroring should be rejected. ret: %d, errno: %d", + ret, errno); + + kvm_vm_free(src_vm); + kvm_vm_free(dst_vm); +} + static void test_sev_move_copy(void) { struct kvm_vm *dst_vm, *dst2_vm, *dst3_vm, *sev_vm, *mirror_vm, @@ -384,12 +427,16 @@ int main(int argc, char *argv[]) test_sev_migrate_parameters(); if (kvm_has_cap(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM)) test_sev_move_copy(); + if (kvm_cpu_has(X86_FEATURE_SEV_SNP)) + test_sev_snp_migrate_reject(); } if (kvm_has_cap(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM)) { test_sev_mirror(/* es= */ false); if (have_sev_es) test_sev_mirror(/* es= */ true); test_sev_mirror_parameters(); + if (kvm_cpu_has(X86_FEATURE_SEV_SNP)) + test_sev_snp_mirror_reject(); } return 0; }