x86:

- Check validity of vmcb01 save area copied from SMRAM on RSM
 
 - Conditionally resched when retrying gPTE walk on A/D update failure
 
 - Conditionally resched when restarting REP string emulation
 
 - Process each host MSR entry at most once when unwinding failed VM-Entry
 
 - Cap number of parents of page table pages, to avoid quadratic teardown
 
 - Unconditionally reset MMU context when emulating INIT
 
 - Flush guest TLB when emulating a RESET to strictly honor the spec
 
 - Add array_index_nospec() to evmcs_field_offset()
 -----BEGIN PGP SIGNATURE-----
 
 iQFIBAABCAAyFiEE8TM4V0tmI4mGbHaCv/vSX3jHroMFAmrpFz4UHHBib256aW5p
 QHJlZGhhdC5jb20ACgkQv/vSX3jHroM8PQf8CUtTIZTsJMDj9fjyudqdGzacmvxp
 1ThdTxDFselGIrE+Kqm8b+LMOZJ3FK5GFOhnJtfUhuHk8nJyGau1uZi7hLaQwzmF
 ZXzcQdcErYcPRXw6aIH034Ys4+K+QgsJvu1l9Kg/VGJ+mElQWlZjEuzW/B5iGW6f
 r0gj06XqbN6ipj+5YezyClkUkfsclPDBKfF+tEW3m3wyCCnF1asmyrWzm71NuGBq
 CeKjiuX7sXWXAapbXZfhXD0MrQV1wZpEEVhvcnoOzwMTOaRL6IEwcluKYlVi5uC2
 Vv+5VbYUTQZaYiLFcJQh4OGL8lz/HuQgzFhFJ4XIABpFDb8uWRRqyE+KdA==
 =TjJZ
 -----END PGP SIGNATURE-----

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull KVM fixes from Paolo Bonzini:
 "x86:
   - Check validity of vmcb01 save area copied from SMRAM on RSM
   - Conditionally resched when retrying gPTE walk on A/D update failure
   - Conditionally resched when restarting REP string emulation
   - Process each host MSR entry at most once when unwinding failed VM-Entry
   - Cap number of parents of page table pages, to avoid quadratic teardown
   - Unconditionally reset MMU context when emulating INIT
   - Flush guest TLB when emulating a RESET to strictly honor the spec
   - Add array_index_nospec() to evmcs_field_offset()"

* tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm:
  KVM: VMX: Add array_index_nospec() to evmcs_field_offset()
  KVM: x86: Flush guest TLB when emulating a RESET to strictly honor the spec
  KVM: x86: Unconditionally reset MMU context when emulating INIT
  KVM: x86/mmu: Cap number of parent PTEs per SP at 1000 (the arbitrary rmap limit)
  KVM: nVMX: Process each host MSR entry at most once when unwinding failed VM-Entry
  KVM: x86: Conditionally resched when restarting REP string emulation
  KVM: x86/mmu: Conditionally resched when retrying gPTE walk on A/D update failure
  KVM: nSVM: Check validity of vmcs01 save area copied from SMRAM on RSM

arch/x86/kvm/mmu/mmu.c

@@ -2359,6 +2359,16 @@ static struct kvm_mmu_page *kvm_mmu_find_shadow_page(struct kvm *kvm,
 			continue;
 		}
 
+		/*
+		 * Note!  Unlike fully direct MMUs, direct SPs in indirect MMUs
+		 * can have an arbitrary number of parents, e.g if the guest is
+		 * mapping a huge PTE at many points in its page tables.
+		 */
+		if (pte_list_count(&sp->parent_ptes) >= RMAP_RECYCLE_THRESHOLD) {
+			kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
+			break;
+		}
+
 		/* unsync and write-flooding only apply to indirect SPs. */
 		if (sp->role.direct)
 			goto out;

arch/x86/kvm/mmu/paging_tmpl.h

@@ -339,6 +339,13 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 	gpa_t real_gpa;
 	gfn_t gfn;
 
+	/*
+	 * KVM might sleep/resched when retrying the walk if the updating A/D
+	 * fails because the gPTE changed between reading the gPTE and trying
+	 * the CMPXHG to set A/D bits.
+	 */
+	might_sleep();
+
 	trace_kvm_mmu_pagetable_walk(addr, access);
 retry_walk:
 	walker->level = w->cpu_role.base.level;
@@ -481,8 +488,10 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 							addr, write_fault);
 		if (unlikely(ret < 0))
 			goto error;
-		else if (ret)
+		else if (ret) {
+			cond_resched();
 			goto retry_walk;
+		}
 	}
 
 	return 1;

arch/x86/kvm/svm/nested.c

@@ -1889,6 +1889,21 @@ static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst,
 	/* 'clean' and 'hv_enlightenments' are not changed by KVM */
 }
 
+bool nested_svm_check_vmcb01_save(struct kvm_vcpu *vcpu,
+				  struct vmcb_save_area *save)
+{
+	struct vmcb_save_area_cached save_cached;
+
+	if (!(save->cr0 & X86_CR0_PG) ||
+	    !(save->cr0 & X86_CR0_PE) ||
+	    (save->rflags & X86_EFLAGS_VM) ||
+	    ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)))
+		return false;
+
+	__nested_copy_vmcb_save_to_cache(&save_cached, save);
+	return nested_vmcb_check_save(vcpu, &save_cached, false);
+}
+
 static int svm_get_nested_state(struct kvm_vcpu *vcpu,
 				struct kvm_nested_state __user *user_kvm_nested_state,
 				u32 user_data_size)
@@ -1968,7 +1983,6 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 		&user_kvm_nested_state->data.svm[0];
 	struct vmcb_control_area *ctl;
 	struct vmcb_save_area *save;
-	struct vmcb_save_area_cached save_cached;
 	struct vmcb_ctrl_area_cached ctl_cached;
 	bool use_separate_l2_pat;
 	unsigned long cr0;
@@ -2040,12 +2054,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 	 * KVM_GET_NESTED_STATE, and (b) it is not loaded at emulated
 	 * #VMEXIT.
 	 */
-	__nested_copy_vmcb_save_to_cache(&save_cached, save);
-	if (!(save->cr0 & X86_CR0_PG) ||
-	    !(save->cr0 & X86_CR0_PE) ||
-	    (save->rflags & X86_EFLAGS_VM) ||
-	    ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) ||
-	    !nested_vmcb_check_save(vcpu, &save_cached, false))
+	if (!nested_svm_check_vmcb01_save(vcpu, save))
 		goto out_free;
 
 	/*

arch/x86/kvm/svm/svm.c

@@ -5223,6 +5223,8 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
 	 */
 
 	svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, m_save.map.hva + 0x400);
+	if (!nested_svm_check_vmcb01_save(vcpu, &svm->vmcb01.ptr->save))
+		return 1;
 
 	/*
 	 * Enter the nested guest now

arch/x86/kvm/svm/svm.h

@@ -911,6 +911,8 @@ void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm,
 void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm,
 				    struct vmcb_save_area *save);
 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm);
+bool nested_svm_check_vmcb01_save(struct kvm_vcpu *vcpu,
+				  struct vmcb_save_area *save);
 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb);
 
 

arch/x86/kvm/vmx/hyperv_evmcs.h

@@ -136,6 +136,8 @@ static __always_inline int evmcs_field_offset(unsigned long field,
 	if (unlikely(index >= nr_evmcs_1_fields))
 		return -ENOENT;
 
+	index = array_index_nospec(index, nr_evmcs_1_fields);
+
 	evmcs_field = &vmcs_field_to_evmcs_1[index];
 
 	/*

arch/x86/kvm/vmx/nested.c

@@ -5059,6 +5059,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
 					__func__, j, h.index, h.value);
 				goto vmabort;
 			}
+			break;
 		}
 	}
 

arch/x86/kvm/x86.c

@@ -6558,10 +6558,12 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 	} else if (vcpu->arch.complete_userspace_io) {
 		writeback = false;
 		r = 0;
-	} else if (r == EMULATION_RESTART)
+	} else if (r == EMULATION_RESTART) {
+		cond_resched();
 		goto restart;
-	else
+	} else {
 		r = 1;
+	}
 
 writeback:
 	if (writeback) {
@@ -9730,29 +9732,31 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	kvm_x86_call(update_exception_bitmap)(vcpu);
 
 	/*
-	 * On the standard CR0/CR4/EFER modification paths, there are several
-	 * complex conditions determining whether the MMU has to be reset and/or
-	 * which PCIDs have to be flushed.  However, CR0.WP and the paging-related
-	 * bits in CR4 and EFER are irrelevant if CR0.PG was '0'; and a reset+flush
-	 * is needed anyway if CR0.PG was '1' (which can only happen for INIT, as
-	 * CR0 will be '0' prior to RESET).  So we only need to check CR0.PG here.
+	 * Unconditionally reset the MMU, even though super strictly speaking,
+	 * a MMU reset is only necessary if and only one of the page role bits
+	 * is changing.  All of the CR0/CR4/EFER bits effectively get covered
+	 * by checking CR0.PG=1, as CR0.WP and the paging-related bits in CR4
+	 * and EFER are irrelevant if CR0.PG was '0'.  Subtly, if the vCPU was
+	 * in guest mode (L2) or SMM (although emulating INIT in SMM _should_
+	 * be impossible), an MMU reset is also necessary as "guest mode" and
+	 * "smm" are both encoded in the shadow page role.  As a result, the
+	 * only scenarios where an MMU reset is *not* needed is if the vCPU is
+	 * in Real Mode or Unpaged Protected Mode, and micro-optimizing those
+	 * paths are not worth the risk of running with a stale MMU context.
 	 */
-	if (old_cr0 & X86_CR0_PG) {
-		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
-		kvm_mmu_reset_context(vcpu);
-	}
+	kvm_mmu_reset_context(vcpu);
 
 	/*
-	 * Intel's SDM states that all TLB entries are flushed on INIT.  AMD's
-	 * APM states the TLBs are untouched by INIT, but it also states that
-	 * the TLBs are flushed on "External initialization of the processor."
-	 * Flush the guest TLB regardless of vendor, there is no meaningful
-	 * benefit in relying on the guest to flush the TLB immediately after
-	 * INIT.  A spurious TLB flush is benign and likely negligible from a
-	 * performance perspective.
+	 * Intel's SDM states that all TLB entries are flushed on INIT "as with
+	 * a hardware reset", and AMD's APM states the TLBs are flushed on
+	 * "External initialization of the processor".  Flush the guest TLB for
+	 * both RESET and INIT, regardless of vendor, even though both the SDM
+	 * and APM contradict the above statements in other tables.  There is
+	 * no meaningful benefit in relying on the guest to flush the TLB, and
+	 * a spurious TLB flush is benign and likely negligible from a
+	 * performance perspective (especially for RESET).
 	 */
-	if (init_event)
-		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
+	kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_reset);