diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index dc2aef72ffdd..f89e9f8a6cc2 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2359,6 +2359,16 @@ static struct kvm_mmu_page *kvm_mmu_find_shadow_page(struct kvm *kvm, continue; } + /* + * Note! Unlike fully direct MMUs, direct SPs in indirect MMUs + * can have an arbitrary number of parents, e.g if the guest is + * mapping a huge PTE at many points in its page tables. + */ + if (pte_list_count(&sp->parent_ptes) >= RMAP_RECYCLE_THRESHOLD) { + kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); + break; + } + /* unsync and write-flooding only apply to indirect SPs. */ if (sp->role.direct) goto out; diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index dc86c2ae70c9..939a10fc1110 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -339,6 +339,13 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, gpa_t real_gpa; gfn_t gfn; + /* + * KVM might sleep/resched when retrying the walk if the updating A/D + * fails because the gPTE changed between reading the gPTE and trying + * the CMPXHG to set A/D bits. + */ + might_sleep(); + trace_kvm_mmu_pagetable_walk(addr, access); retry_walk: walker->level = w->cpu_role.base.level; @@ -481,8 +488,10 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, addr, write_fault); if (unlikely(ret < 0)) goto error; - else if (ret) + else if (ret) { + cond_resched(); goto retry_walk; + } } return 1; diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index af0ef678433d..07554089ec05 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -1889,6 +1889,21 @@ static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst, /* 'clean' and 'hv_enlightenments' are not changed by KVM */ } +bool nested_svm_check_vmcb01_save(struct kvm_vcpu *vcpu, + struct vmcb_save_area *save) +{ + struct vmcb_save_area_cached save_cached; + + if (!(save->cr0 & X86_CR0_PG) || + !(save->cr0 & X86_CR0_PE) || + (save->rflags & X86_EFLAGS_VM) || + ((save->efer & EFER_LMA) && !(save->efer & EFER_LME))) + return false; + + __nested_copy_vmcb_save_to_cache(&save_cached, save); + return nested_vmcb_check_save(vcpu, &save_cached, false); +} + static int svm_get_nested_state(struct kvm_vcpu *vcpu, struct kvm_nested_state __user *user_kvm_nested_state, u32 user_data_size) @@ -1968,7 +1983,6 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, &user_kvm_nested_state->data.svm[0]; struct vmcb_control_area *ctl; struct vmcb_save_area *save; - struct vmcb_save_area_cached save_cached; struct vmcb_ctrl_area_cached ctl_cached; bool use_separate_l2_pat; unsigned long cr0; @@ -2040,12 +2054,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, * KVM_GET_NESTED_STATE, and (b) it is not loaded at emulated * #VMEXIT. */ - __nested_copy_vmcb_save_to_cache(&save_cached, save); - if (!(save->cr0 & X86_CR0_PG) || - !(save->cr0 & X86_CR0_PE) || - (save->rflags & X86_EFLAGS_VM) || - ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) || - !nested_vmcb_check_save(vcpu, &save_cached, false)) + if (!nested_svm_check_vmcb01_save(vcpu, save)) goto out_free; /* diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 136069dfcd44..e30ee80624b7 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5223,6 +5223,8 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) */ svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, m_save.map.hva + 0x400); + if (!nested_svm_check_vmcb01_save(vcpu, &svm->vmcb01.ptr->save)) + return 1; /* * Enter the nested guest now diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 6d0242966dc8..cd3242811e43 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -911,6 +911,8 @@ void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, struct vmcb_save_area *save); void nested_sync_control_from_vmcb02(struct vcpu_svm *svm); +bool nested_svm_check_vmcb01_save(struct kvm_vcpu *vcpu, + struct vmcb_save_area *save); void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb); diff --git a/arch/x86/kvm/vmx/hyperv_evmcs.h b/arch/x86/kvm/vmx/hyperv_evmcs.h index bc08fe40590e..052b62b8ed8e 100644 --- a/arch/x86/kvm/vmx/hyperv_evmcs.h +++ b/arch/x86/kvm/vmx/hyperv_evmcs.h @@ -136,6 +136,8 @@ static __always_inline int evmcs_field_offset(unsigned long field, if (unlikely(index >= nr_evmcs_1_fields)) return -ENOENT; + index = array_index_nospec(index, nr_evmcs_1_fields); + evmcs_field = &vmcs_field_to_evmcs_1[index]; /* diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 516ba03bf608..879183c1bfa5 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5059,6 +5059,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) __func__, j, h.index, h.value); goto vmabort; } + break; } } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 32563a91fba9..26e7beaff8a0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6558,10 +6558,12 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, } else if (vcpu->arch.complete_userspace_io) { writeback = false; r = 0; - } else if (r == EMULATION_RESTART) + } else if (r == EMULATION_RESTART) { + cond_resched(); goto restart; - else + } else { r = 1; + } writeback: if (writeback) { @@ -9730,29 +9732,31 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) kvm_x86_call(update_exception_bitmap)(vcpu); /* - * On the standard CR0/CR4/EFER modification paths, there are several - * complex conditions determining whether the MMU has to be reset and/or - * which PCIDs have to be flushed. However, CR0.WP and the paging-related - * bits in CR4 and EFER are irrelevant if CR0.PG was '0'; and a reset+flush - * is needed anyway if CR0.PG was '1' (which can only happen for INIT, as - * CR0 will be '0' prior to RESET). So we only need to check CR0.PG here. + * Unconditionally reset the MMU, even though super strictly speaking, + * a MMU reset is only necessary if and only one of the page role bits + * is changing. All of the CR0/CR4/EFER bits effectively get covered + * by checking CR0.PG=1, as CR0.WP and the paging-related bits in CR4 + * and EFER are irrelevant if CR0.PG was '0'. Subtly, if the vCPU was + * in guest mode (L2) or SMM (although emulating INIT in SMM _should_ + * be impossible), an MMU reset is also necessary as "guest mode" and + * "smm" are both encoded in the shadow page role. As a result, the + * only scenarios where an MMU reset is *not* needed is if the vCPU is + * in Real Mode or Unpaged Protected Mode, and micro-optimizing those + * paths are not worth the risk of running with a stale MMU context. */ - if (old_cr0 & X86_CR0_PG) { - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - kvm_mmu_reset_context(vcpu); - } + kvm_mmu_reset_context(vcpu); /* - * Intel's SDM states that all TLB entries are flushed on INIT. AMD's - * APM states the TLBs are untouched by INIT, but it also states that - * the TLBs are flushed on "External initialization of the processor." - * Flush the guest TLB regardless of vendor, there is no meaningful - * benefit in relying on the guest to flush the TLB immediately after - * INIT. A spurious TLB flush is benign and likely negligible from a - * performance perspective. + * Intel's SDM states that all TLB entries are flushed on INIT "as with + * a hardware reset", and AMD's APM states the TLBs are flushed on + * "External initialization of the processor". Flush the guest TLB for + * both RESET and INIT, regardless of vendor, even though both the SDM + * and APM contradict the above statements in other tables. There is + * no meaningful benefit in relying on the guest to flush the TLB, and + * a spurious TLB flush is benign and likely negligible from a + * performance perspective (especially for RESET). */ - if (init_event) - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_reset);