github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-23 13:14:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM: x86: Add static calls for nested virtualization ops

Browse Source 
Use static calls to invoke nested virtualization ops, as many of the calls
are in relatively hot paths when L2 is active, e.g. checking for events,
and because there's no reason not use static calls these days.

Opportunistically use a RET0 static call for get_evmcs_version() instead
of manually checking for a non-NULL vendor hook.

Link: https://patch.msgid.link/20260630202828.440724-3-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
This commit is contained in:
Sean Christopherson 2026-06-30 13:28:27 -07:00
parent b65699be2c
commit 4b9819a506
7 changed files with 86 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
arch/x86/include/asm/kvm-x86-nested-ops.h Normal file
View File

@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0 */
#if !defined(KVM_X86_NESTED_OP) || \
!defined(KVM_X86_NESTED_OP_OPTIONAL) || \
!defined(KVM_X86_NESTED_OP_OPTIONAL_RET0)
#error Missing one or more KVM_X86_NESTED_OP #defines
#else
/*
* KVM_X86_NESTED_OP() and KVM_X86_NESTED_OP_OPTIONAL() are used to help
* generate both DECLARE/DEFINE_STATIC_CALL() invocations and
* "static_call_update()" calls.
*
* KVM_X86_NESTED_OP_OPTIONAL() can be used for those functions that can have
* a NULL definition. KVM_X86_NESTED_OP_OPTIONAL_RET0() can be used likewise
* to make a definition optional, but in this case the default will
* be __static_call_return0.
*/
KVM_X86_NESTED_OP(leave_nested)
KVM_X86_NESTED_OP(is_exception_vmexit)
KVM_X86_NESTED_OP(check_events)
KVM_X86_NESTED_OP_OPTIONAL_RET0(has_events)
KVM_X86_NESTED_OP(triple_fault)
KVM_X86_NESTED_OP(get_state)
KVM_X86_NESTED_OP(set_state)
KVM_X86_NESTED_OP(get_nested_state_pages)
KVM_X86_NESTED_OP_OPTIONAL_RET0(write_log_dirty)
KVM_X86_NESTED_OP(translate_nested_gpa)
#ifdef CONFIG_KVM_HYPERV
KVM_X86_NESTED_OP_OPTIONAL(enable_evmcs)
KVM_X86_NESTED_OP_OPTIONAL_RET0(get_evmcs_version)
KVM_X86_NESTED_OP(hv_inject_synthetic_vmexit_post_tlb_flush)
#endif
#endif
#undef KVM_X86_NESTED_OP
#undef KVM_X86_NESTED_OP_OPTIONAL
#undef KVM_X86_NESTED_OP_OPTIONAL_RET0

8
arch/x86/include/asm/kvm_host.h
View File

@ -1786,6 +1786,14 @@ extern struct kvm_x86_ops kvm_x86_ops;
#define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP
#include <asm/kvm-x86-ops.h>
#define kvm_nested_call(func) static_call(kvm_x86_nested_##func)
#define KVM_X86_NESTED_OP(func) \
DECLARE_STATIC_CALL(kvm_x86_nested_##func, *(((struct kvm_x86_nested_ops *)0)->func));
#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP
#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP
#include <asm/kvm-x86-nested-ops.h>
#define __KVM_HAVE_ARCH_VM_ALLOC
static inline struct kvm *kvm_arch_alloc_vm(void)
{

7
arch/x86/kvm/hyperv.c
View File

@ -2410,7 +2410,7 @@ static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result)
ret = kvm_skip_emulated_instruction(vcpu);
if (tlb_lock_count)
kvm_x86_ops.nested_ops->hv_inject_synthetic_vmexit_post_tlb_flush(vcpu);
kvm_nested_call(hv_inject_synthetic_vmexit_post_tlb_flush)(vcpu);
return ret;
}
@ -2791,9 +2791,8 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid,
};
int i, nent = ARRAY_SIZE(cpuid_entries);
if (kvm_x86_ops.nested_ops->enabled &&
kvm_x86_ops.nested_ops->get_evmcs_version)
evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu);
if (kvm_x86_ops.nested_ops->enabled)
evmcs_ver = kvm_nested_call(get_evmcs_version)(vcpu);
if (cpuid->nent < nent)
return -E2BIG;

5
arch/x86/kvm/mmu.h
View File

@ -385,9 +385,8 @@ static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
{
if (!mmu_is_nested(vcpu) || w == &vcpu->arch.ngpa_walk)
return gpa;
return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access,
exception,
pte_access);
return kvm_nested_call(translate_nested_gpa)(vcpu, gpa, access,
exception, pte_access);
}
static inline bool kvm_has_mirrored_tdp(const struct kvm *kvm)

2
arch/x86/kvm/mmu/paging_tmpl.h
View File

@ -235,7 +235,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
!(pte & PT_GUEST_DIRTY_MASK)) {
trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
#if PTTYPE == PTTYPE_EPT
if (kvm_x86_ops.nested_ops->write_log_dirty(vcpu, addr))
if (kvm_nested_call(write_log_dirty)(vcpu, addr))
return -EINVAL;
#endif
pte |= PT_GUEST_DIRTY_MASK;

53
arch/x86/kvm/x86.c
View File

@ -174,6 +174,13 @@ EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl);
#define KVM_X86_NESTED_OP(func) \
DEFINE_STATIC_CALL_NULL(kvm_x86_nested_##func, \
*(((struct kvm_x86_nested_ops *)0)->func));
#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP
#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP
#include <asm/kvm-x86-nested-ops.h>
unsigned int min_timer_period_us = 200;
module_param(min_timer_period_us, uint, 0644);
@ -463,7 +470,7 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr,
* wants to intercept the exception.
*/
if (is_guest_mode(vcpu) &&
kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {
kvm_nested_call(is_exception_vmexit)(vcpu, nr, error_code)) {
kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,
has_payload, payload);
return;
@ -2356,7 +2363,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
break;
case KVM_CAP_NESTED_STATE:
r = kvm_x86_ops.nested_ops->enabled ?
kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
kvm_nested_call(get_state)(NULL, NULL, 0) : 0;
break;
#ifdef CONFIG_KVM_HYPERV
case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
@ -3379,7 +3386,7 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
if (!kvm_x86_ops.nested_ops->enabled ||
!kvm_x86_ops.nested_ops->enable_evmcs)
return -ENOTTY;
r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
r = kvm_nested_call(enable_evmcs)(vcpu, &vmcs_version);
if (!r) {
user_ptr = (void __user *)(uintptr_t)cap->args[0];
if (copy_to_user(user_ptr, &vmcs_version,
@ -3751,8 +3758,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
if (get_user(user_data_size, &user_kvm_nested_state->size))
break;
r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
user_data_size);
r = kvm_nested_call(get_state)(vcpu, user_kvm_nested_state, user_data_size);
if (r < 0)
break;
@ -3796,7 +3802,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
break;
idx = srcu_read_lock(&vcpu->kvm->srcu);
r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
r = kvm_nested_call(set_state)(vcpu, user_kvm_nested_state, &kvm_state);
srcu_read_unlock(&vcpu->kvm->srcu, idx);
break;
}
@ -6914,6 +6920,20 @@ static void kvm_setup_efer_caps(void)
kvm_enable_efer_bits(EFER_AUTOIBRS);
}
static void kvm_nested_ops_update(const struct kvm_x86_nested_ops *nested_ops)
{
#define __KVM_X86_NESTED_OP(func) \
static_call_update(kvm_x86_nested_##func, nested_ops->func);
#define KVM_X86_NESTED_OP(func) \
WARN_ON(!nested_ops->func); __KVM_X86_NESTED_OP(func)
#define KVM_X86_NESTED_OP_OPTIONAL __KVM_X86_NESTED_OP
#define KVM_X86_NESTED_OP_OPTIONAL_RET0(func) \
static_call_update(kvm_x86_nested_##func, (void *)nested_ops->func ? : \
(void *)__static_call_return0);
#include <asm/kvm-x86-nested-ops.h>
#undef __KVM_X86_NESTED_OP
}
static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
{
memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
@ -6929,6 +6949,8 @@ static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
#include <asm/kvm-x86-ops.h>
#undef __KVM_X86_OP
kvm_nested_ops_update(kvm_x86_ops.nested_ops);
kvm_pmu_ops_update(ops->pmu_ops);
}
@ -7465,11 +7487,11 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu)
int kvm_check_nested_events(struct kvm_vcpu *vcpu)
{
if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
kvm_x86_ops.nested_ops->triple_fault(vcpu);
kvm_nested_call(triple_fault)(vcpu);
return 1;
}
return kvm_x86_ops.nested_ops->check_events(vcpu);
return kvm_nested_call(check_events)(vcpu);
}
static void kvm_inject_exception(struct kvm_vcpu *vcpu)
@ -7707,9 +7729,7 @@ static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,
kvm_x86_call(enable_irq_window)(vcpu);
}
if (is_guest_mode(vcpu) &&
kvm_x86_ops.nested_ops->has_events &&
kvm_x86_ops.nested_ops->has_events(vcpu, true))
if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, true))
*req_immediate_exit = true;
/*
@ -8032,7 +8052,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
}
if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) {
r = 0;
goto out;
}
@ -8084,7 +8104,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
}
if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
if (is_guest_mode(vcpu))
kvm_x86_ops.nested_ops->triple_fault(vcpu);
kvm_nested_call(triple_fault)(vcpu);
if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
@ -8502,9 +8522,7 @@ bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
if (kvm_hv_has_stimer_pending(vcpu))
return true;
if (is_guest_mode(vcpu) &&
kvm_x86_ops.nested_ops->has_events &&
kvm_x86_ops.nested_ops->has_events(vcpu, false))
if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, false))
return true;
if (kvm_xen_has_pending_events(vcpu))
@ -8907,8 +8925,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
* a pending VM-Exit if L1 wants to intercept the exception.
*/
if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&
kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,
ex->error_code)) {
kvm_nested_call(is_exception_vmexit)(vcpu, ex->vector, ex->error_code)) {
kvm_queue_exception_vmexit(vcpu, ex->vector,
ex->has_error_code, ex->error_code,
ex->has_payload, ex->payload);

2
arch/x86/kvm/x86.h
View File

@ -93,7 +93,7 @@ int kvm_check_nested_events(struct kvm_vcpu *vcpu);
/* Forcibly leave the nested mode in cases like a vCPU reset */
static inline void kvm_leave_nested(struct kvm_vcpu *vcpu)
{
kvm_x86_ops.nested_ops->leave_nested(vcpu);
kvm_nested_call(leave_nested)(vcpu);
}
/*