From 6978adcc3e73d91fbf371f08cdd924cd05771ac6 Mon Sep 17 00:00:00 2001 From: Hao Zhang Date: Mon, 29 Jun 2026 14:10:41 +0800 Subject: [PATCH 1/4] KVM: VMX: Use cached vcpu_vmx pointer in MSR and segment helpers vmx_get_msr() and vmx_set_msr() already cache to_vmx(vcpu) in a local 'vmx' pointer, but a few cases still open-code to_vmx(vcpu). Use the cached pointer for consistency. Likewise, cache to_vmx(vcpu) in vmx_get_segment_base() instead of open-coding it in both the real-mode check and the VMCS read path. No functional change intended. Signed-off-by: Hao Zhang Link: https://patch.msgid.link/tencent_A78DC401911634111A3391650CB00FCD0409@qq.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3681d565f177..f360849b7f77 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2159,7 +2159,7 @@ int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) !guest_has_spec_ctrl_msr(vcpu)) return 1; - msr_info->data = to_vmx(vcpu)->spec_ctrl; + msr_info->data = vmx->spec_ctrl; break; case MSR_IA32_SYSENTER_CS: msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); @@ -2191,7 +2191,7 @@ int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!msr_info->host_initiated && !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC)) return 1; - msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash + msr_info->data = vmx->msr_ia32_sgxlepubkeyhash [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; break; case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: @@ -2404,7 +2404,7 @@ int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) vmx_guest_debugctl_write(vcpu, data); - if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && + if (intel_pmu_lbr_is_enabled(vcpu) && !vmx->lbr_desc.event && (data & DEBUGCTLMSR_LBR)) intel_pmu_create_guest_lbr_event(vcpu); return 0; @@ -2483,7 +2483,7 @@ int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_IA32_MCG_EXT_CTL: if ((!msr_info->host_initiated && - !(to_vmx(vcpu)->msr_ia32_feature_control & + !(vmx->msr_ia32_feature_control & FEAT_CTL_LMCE_ENABLED)) || (data & ~MCG_EXT_CTL_LMCE_EN)) return 1; @@ -3678,13 +3678,14 @@ void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) { + struct vcpu_vmx *vmx = to_vmx(vcpu); struct kvm_segment s; - if (to_vmx(vcpu)->rmode.vm86_active) { + if (vmx->rmode.vm86_active) { vmx_get_segment(vcpu, &s, seg); return s.base; } - return vmx_read_guest_seg_base(to_vmx(vcpu), seg); + return vmx_read_guest_seg_base(vmx, seg); } static int __vmx_get_cpl(struct kvm_vcpu *vcpu, bool no_cache) From 05a0b701d1089fb57beeb8982f23c3bbafe0fa8b Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Wed, 22 Jul 2026 23:01:28 +0000 Subject: [PATCH 2/4] KVM: nVMX: Service local TLB flushes on failed nested VM-Enter KVM services local TLB flushes on "full" nested VM-Exits (through __nested_vmx_vmexit()), but not if a nested VM-Enter fails (e.g. due to failed VMCS checks in nested_vmx_enter_non_root_mode()). However, it is possible that KVM had queued TLB flushes that need to be performed, even if the nested VM-Enter was not successful. For example, if VPID is disabled for L2 (via nested_vmx_transition_tlb_flush(), or if via the MSR load lists, as the SDM says: If any MSR is being loaded in such a way that would architecturally require a TLB flush, the TLBs are updated so that, after VM entry, the logical processor will not use any translations that were cached before the transition. The SDM is unclear about when the TLB flush should occur, and whether or not a failed VM entry would flush the TLB, so it is safer to always do the TLB flush in this case. More concretely, KVM also updates the last VPID L1 used for L2 in nested_vmx_transition_tlb_flush() (i.e. last_vpid), even if the VM entry ultimately fails. With the current code, KVM could miss a TLB flush if L1 changes L2's VPID, then does a failed VM entry followed by a successful one, as the failed VM entry would update last_vpid but not actually flush the TLB. Servicing local TLB flushes on failed VM entries makes sure that the TLB is always flushed when last_vpid is updated. Fixes: 5c614b3583e7 ("KVM: nVMX: nested VPID emulation") Cc: stable@vger.kernel.org Reported-by: Sashiko # Internal review Suggested-by: Sean Christopherson Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260722230128.1587363-1-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 0635e92471c8..5d5b5022438d 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3765,6 +3765,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, vmentry_fail_vmexit_guest_mode: if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) vcpu->arch.tsc_offset -= vmcs12->tsc_offset; + + /* + * Handle any TLB flush requests that were queued for L2 if KVM made it + * far enough along to switch to L2 context. Note, loading host state + * will generate any flushes for L1 required by VM-Exit. + */ + kvm_service_local_tlb_flush_requests(vcpu); + leave_guest_mode(vcpu); vmentry_fail_vmexit: From d0dea3ed847c6b6feb00a096ca13a8f95da95066 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 31 Jul 2026 10:19:25 -0700 Subject: [PATCH 3/4] KVM: VMX: Bury all of the VMX preemption timer code under CONFIG_X86_64=y Double down on using the VMX preemption timer only for 64-bit kernels, and bury the setup and runtime adjustment code, and all global variables, under CONFIG_X86_64=y. This will allow addressing a widespread Intel erratum without running afoul of unused-but-set-variable and __udivdi3() errors on 32-bit kernels. No functional change intended. Reviewed-by: Binbin Wu Reviewed-by: Chao Gao Link: https://patch.msgid.link/20260731171926.2629627-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 122 +++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 53 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index f360849b7f77..97b2860ace66 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -150,10 +150,12 @@ module_param(dump_invalid_vmcs, bool, 0644); #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ +#ifdef CONFIG_X86_64 static int __read_mostly cpu_preemption_timer_multi; static bool __read_mostly enable_preemption_timer = 1; -#ifdef CONFIG_X86_64 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); +#else +#define enable_preemption_timer false #endif extern bool __read_mostly allow_smaller_maxphyaddr; @@ -7408,32 +7410,6 @@ static void vmx_refresh_guest_perf_global_control(struct kvm_vcpu *vcpu) pmu->global_ctrl = vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL); } -static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u64 tscl; - u32 delta_tsc; - - if (force_immediate_exit) { - vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); - vmx->loaded_vmcs->hv_timer_soft_disabled = false; - } else if (vmx->hv_deadline_tsc != -1) { - tscl = rdtsc(); - if (vmx->hv_deadline_tsc > tscl) - /* set_hv_timer ensures the delta fits in 32-bits */ - delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> - cpu_preemption_timer_multi); - else - delta_tsc = 0; - - vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); - vmx->loaded_vmcs->hv_timer_soft_disabled = false; - } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { - vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); - vmx->loaded_vmcs->hv_timer_soft_disabled = true; - } -} - void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) { if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { @@ -7519,6 +7495,8 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, guest_state_exit_irqoff(); } +static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit); + fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) { bool force_immediate_exit = run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT; @@ -8328,6 +8306,36 @@ static inline int u64_shl_div_u64(u64 a, unsigned int shift, return 0; } +static __init void vmx_setup_preemption_timer(void) +{ + if (!cpu_has_vmx_preemption_timer()) + enable_preemption_timer = false; + + if (enable_preemption_timer) { + u64 use_timer_freq = 5000ULL * 1000 * 1000; + + cpu_preemption_timer_multi = + vmx_misc_preemption_timer_rate(vmcs_config.misc); + + if (tsc_khz) + use_timer_freq = (u64)tsc_khz * 1000; + use_timer_freq >>= cpu_preemption_timer_multi; + + /* + * KVM "disables" the preemption timer by setting it to its max + * value. Don't use the timer if it might cause spurious exits + * at a rate faster than 0.1 Hz (of uninterrupted guest time). + */ + if (use_timer_freq > 0xffffffffu / 10) + enable_preemption_timer = false; + } + + if (!enable_preemption_timer) { + vt_x86_ops.set_hv_timer = NULL; + vt_x86_ops.cancel_hv_timer = NULL; + } +} + int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired) { @@ -8372,6 +8380,39 @@ void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) { to_vmx(vcpu)->hv_deadline_tsc = -1; } + +static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u64 tscl; + u32 delta_tsc; + + if (force_immediate_exit) { + vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); + vmx->loaded_vmcs->hv_timer_soft_disabled = false; + } else if (vmx->hv_deadline_tsc != -1) { + tscl = rdtsc(); + if (vmx->hv_deadline_tsc > tscl) + /* set_hv_timer ensures the delta fits in 32-bits */ + delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> + cpu_preemption_timer_multi); + else + delta_tsc = 0; + + vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); + vmx->loaded_vmcs->hv_timer_soft_disabled = false; + } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { + vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); + vmx->loaded_vmcs->hv_timer_soft_disabled = true; + } +} +#else +static __init void vmx_setup_preemption_timer(void) { } + +static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) +{ + BUILD_BUG_ON(1); +} #endif void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) @@ -8734,32 +8775,7 @@ __init int vmx_hardware_setup(void) if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) enable_pml = 0; - if (!cpu_has_vmx_preemption_timer()) - enable_preemption_timer = false; - - if (enable_preemption_timer) { - u64 use_timer_freq = 5000ULL * 1000 * 1000; - - cpu_preemption_timer_multi = - vmx_misc_preemption_timer_rate(vmcs_config.misc); - - if (tsc_khz) - use_timer_freq = (u64)tsc_khz * 1000; - use_timer_freq >>= cpu_preemption_timer_multi; - - /* - * KVM "disables" the preemption timer by setting it to its max - * value. Don't use the timer if it might cause spurious exits - * at a rate faster than 0.1 Hz (of uninterrupted guest time). - */ - if (use_timer_freq > 0xffffffffu / 10) - enable_preemption_timer = false; - } - - if (!enable_preemption_timer) { - vt_x86_ops.set_hv_timer = NULL; - vt_x86_ops.cancel_hv_timer = NULL; - } + vmx_setup_preemption_timer(); kvm_caps.supported_mce_cap |= MCG_LMCE_P; kvm_caps.supported_mce_cap |= MCG_CMCI_P; From 0aaedada606aef1732cf0e123403a1e8b8b38c0d Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Fri, 31 Jul 2026 10:19:26 -0700 Subject: [PATCH 4/4] KVM: VMX: Cap VMX preemption timer to work around Intel erratum Due to a widespread Intel erratum (e.g. EMR158), programming the VMX-preemption timer with certain large values may cause the timer to expire earlier than expected. The recommended workaround is to cap the VMX-preemption timer value to strictly less than: 2^25 * CPUID.15H:EBX[31:0] / CPUID.15H:EAX[31:0]. Calculate the maximum "safe" preemption timer value during hardware setup based on CPUID 15H when available, and use the adjusted max value in all locations where KVM currently hardcodes the max architectural value, including in the subtle case where KVM soft-disables the timer. Don't apply the workaround when running as a VM, because absent explicit enumeration to state the bug is present (or not), it's L0's responsibility to faithfully emulate/virtualize the VMX preemption timer. WARN if the above logic would result in a max value of zero and fall back to the maximum architectural value, as the expectation is that real hardware will never provide problematic EAX/EBX values (which is another reason to ignore the erratum when running as a VM; there's less chance of a false positive on the WARN due to L0 providing an unanticipated ratio). Reported-by: Sean Christopherson Closes: https://lore.kernel.org/all/Zn9X0yFxZi_Mrlnt@google.com/ Suggested-by: Chao Gao Assisted-by: Gemini:Gemini-Next Reviewed-by: Chao Gao Signed-off-by: Jim Mattson Reviewed-by: Binbin Wu [sean: track inclusive max instead of exclusive limit, massage changelog] Link: https://patch.msgid.link/20260731171926.2629627-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 97b2860ace66..de3d92bd36b7 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -153,6 +153,7 @@ module_param(dump_invalid_vmcs, bool, 0644); #ifdef CONFIG_X86_64 static int __read_mostly cpu_preemption_timer_multi; static bool __read_mostly enable_preemption_timer = 1; +static u64 __ro_after_init preemption_timer_max_value; module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); #else #define enable_preemption_timer false @@ -8306,6 +8307,33 @@ static inline int u64_shl_div_u64(u64 a, unsigned int shift, return 0; } +/* + * Workaround for a widespread Intel erratum (e.g. EMR158) where the + * VMX-preemption timer may expire earlier than expected when programmed + * with large values. The workaround is to cap the timer value to strictly + * less than 2^25 * CPUID.15H:EBX / CPUID.15H:EAX. + */ +static __init u64 calc_preemption_timer_max_value(void) +{ + const u64 ARCHITECTURAL_MAX_VALUE = UINT_MAX; + u32 eax, ebx, ecx, edx; + + if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) + return ARCHITECTURAL_MAX_VALUE; + + if (cpuid_eax(0) < 0x15) + return ARCHITECTURAL_MAX_VALUE; + + cpuid(0x15, &eax, &ebx, &ecx, &edx); + if (!eax || !ebx) + return ARCHITECTURAL_MAX_VALUE; + + if (WARN_ON_ONCE(!(((u64)ebx << 25) / eax))) + return ARCHITECTURAL_MAX_VALUE; + + return min((((u64)ebx << 25) / eax) - 1, ARCHITECTURAL_MAX_VALUE); +} + static __init void vmx_setup_preemption_timer(void) { if (!cpu_has_vmx_preemption_timer()) @@ -8317,6 +8345,8 @@ static __init void vmx_setup_preemption_timer(void) cpu_preemption_timer_multi = vmx_misc_preemption_timer_rate(vmcs_config.misc); + preemption_timer_max_value = calc_preemption_timer_max_value(); + if (tsc_khz) use_timer_freq = (u64)tsc_khz * 1000; use_timer_freq >>= cpu_preemption_timer_multi; @@ -8326,7 +8356,7 @@ static __init void vmx_setup_preemption_timer(void) * value. Don't use the timer if it might cause spurious exits * at a rate faster than 0.1 Hz (of uninterrupted guest time). */ - if (use_timer_freq > 0xffffffffu / 10) + if (use_timer_freq > preemption_timer_max_value / 10) enable_preemption_timer = false; } @@ -8363,12 +8393,12 @@ int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, return -ERANGE; /* - * If the delta tsc can't fit in the 32 bit after the multi shift, - * we can't use the preemption timer. + * If the delta tsc exceeds the preemption timer limit after the + * multi shift, we can't use the preemption timer. * It's possible that it fits on later vmentries, but checking * on every vmentry is costly so we just use an hrtimer. */ - if (delta_tsc >> (cpu_preemption_timer_multi + 32)) + if ((delta_tsc >> cpu_preemption_timer_multi) > preemption_timer_max_value) return -ERANGE; vmx->hv_deadline_tsc = tscl + delta_tsc; @@ -8402,7 +8432,7 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); vmx->loaded_vmcs->hv_timer_soft_disabled = false; } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { - vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); + vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, preemption_timer_max_value); vmx->loaded_vmcs->hv_timer_soft_disabled = true; } }