From f16044caeab6248a2b1a2e4de42f5de01c27cd5b Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 31 Aug 2026 22:26:32 +0100 Subject: [PATCH 001/506] KVM: x86/xen: Rename 'longmode' to 'is_64bit' in hypercall handling Rename the local 'longmode' variable and function parameter to 'is_64bit' throughout the Xen hypercall handling code. This distinguishes it from the VM-wide kvm->arch.xen.long_mode which represents the Xen shared_info layout mode. The 'is_64bit' parameter indicates whether the vCPU was in 64-bit mode when it made the hypercall, which determines how to parse the hypercall arguments. The UAPI field name (vcpu->run->xen.u.hcall.longmode) is unchanged. Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse Link: https://patch.msgid.link/20260831213632.81023-2-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index d113bd0f1c3d..f14f12eaa63e 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1485,7 +1485,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, return ret; } -static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, +static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, u64 param, u64 *r) { struct sched_poll sched_poll; @@ -1497,7 +1497,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, !(vcpu->kvm->arch.xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND)) return false; - if (IS_ENABLED(CONFIG_64BIT) && !longmode) { + if (IS_ENABLED(CONFIG_64BIT) && !is_64bit) { struct compat_sched_poll sp32; /* Sanity check that the compat struct definition is correct */ @@ -1594,12 +1594,12 @@ static void cancel_evtchn_poll(struct timer_list *t) kvm_vcpu_kick(vcpu); } -static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool longmode, +static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd, u64 param, u64 *r) { switch (cmd) { case SCHEDOP_poll: - if (kvm_xen_schedop_poll(vcpu, longmode, param, r)) + if (kvm_xen_schedop_poll(vcpu, is_64bit, param, r)) return true; fallthrough; case SCHEDOP_yield: @@ -1618,7 +1618,7 @@ struct compat_vcpu_set_singleshot_timer { uint32_t flags; } __attribute__((packed)); -static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, +static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd, int vcpu_id, u64 param, u64 *r) { struct vcpu_set_singleshot_timer oneshot; @@ -1662,7 +1662,7 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, flags) != sizeof_field(struct vcpu_set_singleshot_timer, flags)); - if (kvm_read_guest_virt(vcpu, param, &oneshot, longmode ? sizeof(oneshot) : + if (kvm_read_guest_virt(vcpu, param, &oneshot, is_64bit ? sizeof(oneshot) : sizeof(struct compat_vcpu_set_singleshot_timer), &e)) { *r = -EFAULT; return true; @@ -1694,7 +1694,7 @@ static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout, int kvm_xen_hypercall(struct kvm_vcpu *vcpu) { - bool longmode; + bool is_64bit; u64 input, params[6], r = -ENOSYS; bool handled = false; u8 cpl; @@ -1704,8 +1704,8 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) kvm_hv_hypercall_enabled(vcpu)) return kvm_hv_hypercall(vcpu); - longmode = is_64_bit_hypercall(vcpu); - if (!longmode) { + is_64bit = is_64_bit_hypercall(vcpu); + if (!is_64bit) { input = kvm_eax_read(vcpu); params[0] = kvm_ebx_read(vcpu); params[1] = kvm_ecx_read(vcpu); @@ -1751,17 +1751,17 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r); break; case __HYPERVISOR_sched_op: - handled = kvm_xen_hcall_sched_op(vcpu, longmode, params[0], + handled = kvm_xen_hcall_sched_op(vcpu, is_64bit, params[0], params[1], &r); break; case __HYPERVISOR_vcpu_op: - handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1], + handled = kvm_xen_hcall_vcpu_op(vcpu, is_64bit, params[0], params[1], params[2], &r); break; case __HYPERVISOR_set_timer_op: { u64 timeout = params[0]; /* In 32-bit mode, the 64-bit timeout is in two 32-bit params. */ - if (!longmode) + if (!is_64bit) timeout |= params[1] << 32; handled = kvm_xen_hcall_set_timer_op(vcpu, timeout, &r); break; @@ -1776,7 +1776,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handle_in_userspace: vcpu->run->exit_reason = KVM_EXIT_XEN; vcpu->run->xen.type = KVM_EXIT_XEN_HCALL; - vcpu->run->xen.u.hcall.longmode = longmode; + vcpu->run->xen.u.hcall.longmode = is_64bit; vcpu->run->xen.u.hcall.cpl = cpl; vcpu->run->xen.u.hcall.input = input; vcpu->run->xen.u.hcall.params[0] = params[0]; From c152f0a08366107a4333b3260a60803c36120bfa Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 31 Aug 2026 22:26:33 +0100 Subject: [PATCH 002/506] KVM: x86/xen: Introduce kvm_xen_has_64bit_shinfo() macro Add a kvm_xen_has_64bit_shinfo() helper macro to replace the repeated pattern of 'IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode' throughout the Xen emulation code. The macro uses READ_ONCE() to ensure a consistent snapshot of the flag, which can be changed by another vCPU at any time. This is the KVM equivalent of Xen's !has_32bit_shinfo(). Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-3-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 16 ++++++++-------- arch/x86/kvm/xen.h | 5 +++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index f14f12eaa63e..da543852504e 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -73,7 +73,7 @@ static int kvm_xen_shared_info_init(struct kvm *kvm) BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00); BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c); - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct shared_info *shinfo = gpc->khva; wc_sec_hi = &shinfo->wc_sec_hi; @@ -389,7 +389,7 @@ static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic) BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) != sizeof(vx->runstate_times)); - if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(v->kvm)) { user_len = sizeof(struct vcpu_runstate_info); times_ofs = offsetof(struct vcpu_runstate_info, state_entry_time); @@ -676,7 +676,7 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) } /* Now gpc->khva is a valid kernel address for the vcpu_info */ - if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(v->kvm)) { struct vcpu_info *vi = gpc->khva; asm volatile(LOCK_PREFIX "orq %0, %1\n" @@ -993,7 +993,7 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) * address, that's actually OK. kvm_xen_update_runstate_guest() * will cope. */ - if (IS_ENABLED(CONFIG_64BIT) && vcpu->kvm->arch.xen.long_mode) + if (kvm_xen_has_64bit_shinfo(vcpu->kvm)) sz = sizeof(struct vcpu_runstate_info); else sz = sizeof(struct compat_vcpu_runstate_info); @@ -1441,7 +1441,7 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) static inline int max_evtchn_port(struct kvm *kvm) { - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) + if (kvm_xen_has_64bit_shinfo(kvm)) return EVTCHN_2L_NR_CHANNELS; else return COMPAT_EVTCHN_2L_NR_CHANNELS; @@ -1463,7 +1463,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, goto out_rcu; ret = false; - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; } else { @@ -1844,7 +1844,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) if (!kvm_gpc_check(gpc, PAGE_SIZE)) goto out_rcu; - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; @@ -1885,7 +1885,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) goto out_rcu; } - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct vcpu_info *vcpu_info = gpc->khva; if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index f372855857a8..9d04e350bdb1 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -235,6 +235,11 @@ struct compat_shared_info { #define COMPAT_EVTCHN_2L_NR_CHANNELS (8 * \ sizeof_field(struct compat_shared_info, \ evtchn_pending)) + +/* Latched VM-wide mode; the KVM equivalent of Xen's !has_32bit_shinfo(). */ +#define kvm_xen_has_64bit_shinfo(kvm) \ + (IS_ENABLED(CONFIG_64BIT) && READ_ONCE((kvm)->arch.xen.long_mode)) + struct compat_vcpu_runstate_info { int state; uint64_t state_entry_time; From a9c11e5f31461ad8870028670474288c12a4bf07 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 31 Aug 2026 22:26:34 +0100 Subject: [PATCH 003/506] KVM: x86/xen: Rename max_evtchn_port() to kvm_max_evtchn_port() Rename in preparation for adding a variant that takes a latched bool argument for use in paths that need a consistent snapshot of the shinfo mode. No functional change. Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-4-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index da543852504e..dafcdbc86fb6 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1439,7 +1439,7 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result); } -static inline int max_evtchn_port(struct kvm *kvm) +static inline int kvm_max_evtchn_port(struct kvm *kvm) { if (kvm_xen_has_64bit_shinfo(kvm)) return EVTCHN_2L_NR_CHANNELS; @@ -1546,7 +1546,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, } for (i = 0; i < sched_poll.nr_ports; i++) { - if (ports[i] >= max_evtchn_port(vcpu->kvm)) { + if (ports[i] >= kvm_max_evtchn_port(vcpu->kvm)) { *r = -EINVAL; goto out; } @@ -1833,7 +1833,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx); } - if (xe->port >= max_evtchn_port(kvm)) + if (xe->port >= kvm_max_evtchn_port(kvm)) return -EINVAL; rc = -EWOULDBLOCK; @@ -1995,7 +1995,7 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, struct kvm_vcpu *vcpu; /* - * Don't check for the port being within range of max_evtchn_port(). + * Don't check for the port being within range of kvm_max_evtchn_port(). * Userspace can configure what ever targets it likes; events just won't * be delivered if/while the target is invalid, just like userspace can * configure MSIs which target non-existent APICs. @@ -2004,8 +2004,8 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, * can be restored *independently* of other things like creating vCPUs, * without imposing an ordering dependency on userspace. In this * particular case, the problematic ordering would be with setting the - * Xen 'long mode' flag, which changes max_evtchn_port() to allow 4096 - * instead of 1024 event channels. + * Xen 'long mode' flag, which changes kvm_max_evtchn_port() to allow + * 4096 instead of 1024 event channels. */ /* We only support 2 level event channels for now */ @@ -2042,7 +2042,7 @@ int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn * struct kvm_xen_evtchn e; int ret; - if (!uxe->port || uxe->port >= max_evtchn_port(kvm)) + if (!uxe->port || uxe->port >= kvm_max_evtchn_port(kvm)) return -EINVAL; /* We only support 2 level event channels for now */ @@ -2152,7 +2152,7 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, case EVTCHNSTAT_interdomain: if (data->u.evtchn.deliver.port.port) { - if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm)) + if (data->u.evtchn.deliver.port.port >= kvm_max_evtchn_port(kvm)) goto out_noeventfd; /* -EINVAL */ } else { eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd); @@ -2270,7 +2270,7 @@ static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data if (data->u.evtchn.flags == KVM_XEN_EVTCHN_RESET) return kvm_xen_eventfd_reset(kvm); - if (!port || port >= max_evtchn_port(kvm)) + if (!port || port >= kvm_max_evtchn_port(kvm)) return -EINVAL; if (data->u.evtchn.flags == KVM_XEN_EVTCHN_DEASSIGN) From 5831fb3e6b39c7ce875be0881131800405409572 Mon Sep 17 00:00:00 2001 From: Hyunwoo Kim Date: Mon, 31 Aug 2026 22:26:35 +0100 Subject: [PATCH 004/506] KVM: x86/xen: Latch shinfo mode in kvm_xen_set_evtchn_fast() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kvm_xen_set_evtchn_fast() assumes the port range check in max_evtchn_port() and the bitmap layout selection observe the same shinfo mode, but each calls kvm_xen_has_64bit_shinfo() separately. If the guest changes the mode in between, a port accepted by the 64-bit range check is handled with the 32-bit layout, and port_word_bit falls outside evtchn_pending_sel. Latch kvm_xen_has_64bit_shinfo() once on entry so the range check and both layout computations use the same value. In practice this is harmless: the evtchn_pending bitmap is at the same offset in both native and compat shared_info layouts, so a stale mode just results in setting a bit in what the guest (in its new compat mode) considers the evtchn_mask, wallclock, or the arch_shared_info fields which follow it — all of which are in the guest's own page. Even with this fix, the same corruption can occur if 64-bit mode is latched and the guest switches to 32-bit mode immediately afterward. Like Xen, KVM makes no attempt to *convert* when shinfo mode is changed. Only the wallclock field is updated in the new location. This fix is for internal consistency rather than correcting any observable bug. Fixes: 14243b387137 ("KVM: x86/xen: Add KVM_IRQ_ROUTING_XEN_EVTCHN and event channel delivery") Reported-by: Hyunwoo Kim Closes: https://lore.kernel.org/all/aiHPPUk5DY7rH-zL@v4bel/ Signed-off-by: Hyunwoo Kim [dwmw2: Rework on top of long_mode/has_64bit_shinfo cleanups] Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-5-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index dafcdbc86fb6..f4c89388cae6 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1439,14 +1439,19 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result); } -static inline int kvm_max_evtchn_port(struct kvm *kvm) +static inline int max_evtchn_port(bool has_64bit_shinfo) { - if (kvm_xen_has_64bit_shinfo(kvm)) + if (has_64bit_shinfo) return EVTCHN_2L_NR_CHANNELS; else return COMPAT_EVTCHN_2L_NR_CHANNELS; } +static inline int kvm_max_evtchn_port(struct kvm *kvm) +{ + return max_evtchn_port(kvm_xen_has_64bit_shinfo(kvm)); +} + static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, evtchn_port_t *ports) { @@ -1816,8 +1821,9 @@ static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) { struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; - struct kvm_vcpu *vcpu; + bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(kvm); unsigned long *pending_bits, *mask_bits; + struct kvm_vcpu *vcpu; unsigned long flags; int port_word_bit; bool kick_vcpu = false; @@ -1833,7 +1839,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx); } - if (xe->port >= kvm_max_evtchn_port(kvm)) + if (xe->port >= max_evtchn_port(has_64bit_shinfo)) return -EINVAL; rc = -EWOULDBLOCK; @@ -1844,7 +1850,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) if (!kvm_gpc_check(gpc, PAGE_SIZE)) goto out_rcu; - if (kvm_xen_has_64bit_shinfo(kvm)) { + if (has_64bit_shinfo) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; @@ -1885,7 +1891,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) goto out_rcu; } - if (kvm_xen_has_64bit_shinfo(kvm)) { + if (has_64bit_shinfo) { struct vcpu_info *vcpu_info = gpc->khva; if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); From c72ed7c36d2a5b60e24148076db9c4bc932cb6c5 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 31 Aug 2026 22:26:36 +0100 Subject: [PATCH 005/506] KVM: x86/xen: Latch shinfo mode in kvm_xen_schedop_poll() kvm_xen_schedop_poll() validates port numbers against kvm_max_evtchn_port() and then calls wait_pending_event() which reads the shinfo mode again to select the bitmap layout. Latch kvm_xen_has_64bit_shinfo() once and pass it to both max_evtchn_port() and wait_pending_event(). As with the previous fix to kvm_xen_set_evtchn_fast(), this is harmless in practice for the same reasons: the inconsistency can only corrupt fields in the guest's own shared_info page, and the same corruption can occur anyway if the mode changes immediately after the latch. Fixes: d518b9d0fc80 ("KVM: x86/xen: handle PV spinlocks slowpath") Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-6-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index f4c89388cae6..d0bdc6e8eb28 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1452,8 +1452,8 @@ static inline int kvm_max_evtchn_port(struct kvm *kvm) return max_evtchn_port(kvm_xen_has_64bit_shinfo(kvm)); } -static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, - evtchn_port_t *ports) +static bool wait_pending_event(struct kvm_vcpu *vcpu, bool has_64bit_shinfo, + int nr_ports, evtchn_port_t *ports) { struct kvm *kvm = vcpu->kvm; struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; @@ -1468,7 +1468,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, goto out_rcu; ret = false; - if (kvm_xen_has_64bit_shinfo(kvm)) { + if (has_64bit_shinfo) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; } else { @@ -1493,6 +1493,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, u64 param, u64 *r) { + bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(vcpu->kvm); struct sched_poll sched_poll; evtchn_port_t port, *ports; struct x86_exception e; @@ -1551,7 +1552,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, } for (i = 0; i < sched_poll.nr_ports; i++) { - if (ports[i] >= kvm_max_evtchn_port(vcpu->kvm)) { + if (ports[i] >= max_evtchn_port(has_64bit_shinfo)) { *r = -EINVAL; goto out; } @@ -1564,7 +1565,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask); - if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) { + if (!wait_pending_event(vcpu, has_64bit_shinfo, sched_poll.nr_ports, ports)) { kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); if (sched_poll.timeout) From eacdb76c5f04956035c5d7f7e1973028f30684a2 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 31 Aug 2026 22:26:37 +0100 Subject: [PATCH 006/506] KVM: x86/xen: Enforce 4-byte alignment of vcpu_info registration Xen's map_guest_area() enforces that vcpu_info is aligned to sizeof(xen_ulong_t). KVM has no such check, allowing a guest to register vcpu_info at an arbitrary byte alignment. Enforce unconditional 4-byte alignment regardless of the current shinfo mode. This is sufficient because subsequent commits ensure that all locked atomic operations on vcpu_info fields use at most 32-bit accesses. Return -ENXIO on failure, matching Xen's map_guest_area() behaviour for unaligned requests. Originally observed in review of an unrelated patch: https://lore.kernel.org/all/20260604193554.1BA311F00893@smtp.kernel.org/ Cc: stable@vger.kernel.org Fixes: 73e69a86347a ("KVM: x86/xen: register vcpu info") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260604193554.1BA311F00893@smtp.kernel.org Assisted-by: Kiro:claude-opus-4.6-1m Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-7-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index d0bdc6e8eb28..15c508b7b28b 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -941,6 +941,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + r = -ENXIO; + if (!IS_ALIGNED(data->u.gpa, sizeof(u32))) + break; + r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_info_cache, data->u.gpa, sizeof(struct vcpu_info)); } else { @@ -950,6 +954,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + r = -ENXIO; + if (!IS_ALIGNED(data->u.hva, sizeof(u32))) + break; + r = kvm_gpc_activate_hva(&vcpu->arch.xen.vcpu_info_cache, data->u.hva, sizeof(struct vcpu_info)); } From 1835551d4304abb1edcbf5d1ba71945279051236 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 31 Aug 2026 22:26:38 +0100 Subject: [PATCH 007/506] KVM: x86/xen: Use 32-bit locked bts for vcpu_info evtchn_pending_sel Replace test_and_set_bit() on vcpu_info->evtchn_pending_sel with an explicit 'lock btsl' in kvm_xen_set_evtchn_fast(). The generic test_and_set_bit() uses a 64-bit locked operation ('lock btsq') on x86-64, and the address of the per-vCPU info is guest-controlled and only required to be 32-bit aligned, so an 8-byte access can generate a split-lock #AC exception. Since evtchn_pending_sel is at most 64 bits wide and port_word_bit ranges 0-63, a 32-bit 'lock btsl' suffices for both native and compat vcpu_info layouts, and only requires the 4-byte alignment that is already guaranteed by the registration path. This also eliminates the bogus cast of compat_vcpu_info's 32-bit evtchn_pending_sel to 'unsigned long *' which was the original source of the split-lock hazard. Note, KVM reuses the local gpc. The atomic accesses to pending_bits is on the page-aligned per-VM shared_info structure, i.e. doesn't need the same treatment as the access is guaranteed to be 64-bit aligned. Fixes: 14243b387137 ("KVM: x86/xen: Add KVM_IRQ_ROUTING_XEN_EVTCHN and event channel delivery") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260604193554.1BA311F00893@smtp.kernel.org Suggested-by: Sean Christopherson Signed-off-by: David Woodhouse Assisted-by: Claude:claude-mythos-5 Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-8-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 47 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 15c508b7b28b..0f983e068498 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1831,7 +1831,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) { struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(kvm); - unsigned long *pending_bits, *mask_bits; + unsigned long *pending_bits, *mask_bits, vi_pending_sel_ofs; struct kvm_vcpu *vcpu; unsigned long flags; int port_word_bit; @@ -1864,11 +1864,18 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; port_word_bit = xe->port / 64; + + vi_pending_sel_ofs = offsetof(struct vcpu_info, evtchn_pending_sel); } else { struct compat_shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; port_word_bit = xe->port / 32; + + vi_pending_sel_ofs = offsetof(struct compat_vcpu_info, evtchn_pending_sel); + + /* test_and_set_bit() needs 64-bit alignment, but that's OK */ + BUILD_BUG_ON(offsetof(struct compat_shared_info, evtchn_pending) & 7); } /* @@ -1884,6 +1891,8 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) rc = -ENOTCONN; /* Masked */ kvm_xen_check_poller(vcpu, xe->port); } else { + bool old; + rc = 1; /* Delivered to the bitmap in shared_info. */ /* Now switch to the vCPU's vcpu_info to set the index and pending_sel */ read_unlock_irqrestore(&gpc->lock, flags); @@ -1900,19 +1909,29 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) goto out_rcu; } - if (has_64bit_shinfo) { - struct vcpu_info *vcpu_info = gpc->khva; - if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { - WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); - kick_vcpu = true; - } - } else { - struct compat_vcpu_info *vcpu_info = gpc->khva; - if (!test_and_set_bit(port_word_bit, - (unsigned long *)&vcpu_info->evtchn_pending_sel)) { - WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); - kick_vcpu = true; - } + /* + * Explicitly use a 32-bit btsl instead of test_and_set_bit(), + * which would use btsq on x86-64. The vcpu_info is guest- + * controlled and only required to be 32-bit aligned, so a + * 64-bit access could generate a split-lock #AC. + * + * Note, this does not apply to the test_and_set_bit() on + * pending_bits above: that is in the per-VM shared_info, which + * is page aligned, so the access is guaranteed to be 64-bit + * aligned. + */ + old = GEN_BINARY_RMWcc(LOCK_PREFIX "btsl", + *(u32 *)(gpc->khva + vi_pending_sel_ofs), + c, "Ir", port_word_bit); + if (!old) { + struct vcpu_info *vi = gpc->khva; + + /* No need for compat handling */ + BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) != + offsetof(struct compat_vcpu_info, evtchn_upcall_pending)); + + WRITE_ONCE(vi->evtchn_upcall_pending, 1); + kick_vcpu = true; } /* For the per-vCPU lapic vector, deliver it as MSI. */ From 301a5ced32e08e96212b413e896ef4c32d1ce961 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 31 Aug 2026 22:26:39 +0100 Subject: [PATCH 008/506] KVM: x86/xen: Use 32-bit atomics if vCPU's evtchn_pending_sel isn't aligned When propagating pending Xen events from KVM's "cache" to the guest-visible structure, use two 32-bit atomic operations to do the bitwise-OR into the guest-controlled structure if the structure isn't 64-bit aligned, i.e. if the guest registered its vcpu_info in 32-bit mode and then switched to 64-bit mode, in which case using a 64-bit atomic OR will generate a split-lock #AC (if enabled). Opportunistically isolate the clearing of the bits from KVM's cache, as that structure is KVM-controlled, i.e. is guaranteed to be 64-bit aligned. This will allow dropping the open-coded inline asm blobs in the future. [dwmw2: Cast to u64 before shifting; evtchn_pending_sel is unsigned long, so >> 32 is undefined on 32-bit even though the branch is unreachable there] Fixes: 14243b387137 ("KVM: x86/xen: Add KVM_IRQ_ROUTING_XEN_EVTCHN and event channel delivery") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260604193554.1BA311F00893@smtp.kernel.org Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202608071502.rYOi3Pg8-lkp@intel.com/ Signed-off-by: Sean Christopherson Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-9-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 0f983e068498..780eec2fb528 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -679,13 +679,28 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) if (kvm_xen_has_64bit_shinfo(v->kvm)) { struct vcpu_info *vi = gpc->khva; - asm volatile(LOCK_PREFIX "orq %0, %1\n" - "notq %0\n" - LOCK_PREFIX "andq %0, %2\n" - : "=r" (evtchn_pending_sel), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel)); + if (IS_ALIGNED((unsigned long)&vi->evtchn_pending_sel, sizeof(u64))) + asm volatile(LOCK_PREFIX "orq %[src], %[dst]\n" + : [dst] "+m" (vi->evtchn_pending_sel) + : [src] "r" (evtchn_pending_sel)); + else + /* + * The cast keeps the shift well-defined on 32-bit, + * where evtchn_pending_sel is 32 bits wide and this + * branch is unreachable anyway (this is inside + * kvm_xen_has_64bit_shinfo(), which is gated on + * IS_ENABLED(CONFIG_64BIT)). + */ + asm volatile(LOCK_PREFIX "orl %[src_lo], %[dst_lo]\n" + LOCK_PREFIX "orl %[src_hi], %[dst_hi]\n" + : [dst_lo] "+m" (vi->evtchn_pending_sel), + [dst_hi] "+m" (*(((u32 *)&vi->evtchn_pending_sel) + 1)) + : [src_lo] "r" ((u32)evtchn_pending_sel), + [src_hi] "r" ((u32)((u64)evtchn_pending_sel >> 32))); + + asm volatile(LOCK_PREFIX "andq %1, %0\n" + : "+m" (v->arch.xen.evtchn_pending_sel) + : "r" (~evtchn_pending_sel)); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } else { u32 evtchn_pending_sel32 = evtchn_pending_sel; From 83efc5ae38c99bc261f1edecee8d29b23fa25c0e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 31 Aug 2026 22:26:40 +0100 Subject: [PATCH 009/506] KVM: x86/xen: Use atomic*() APIs instead of open coded equivalents Replace the open coded atomic asm blobs in the Xen event injection code with equivalent atomic{,64}_xxx() operations. Casting the event channel to atomic types is ugly, but not as ugly as asm blobs. No functional change intended. Signed-off-by: Sean Christopherson Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-10-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 780eec2fb528..a4befaaa39af 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -678,12 +678,12 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) /* Now gpc->khva is a valid kernel address for the vcpu_info */ if (kvm_xen_has_64bit_shinfo(v->kvm)) { struct vcpu_info *vi = gpc->khva; + void *vi_pending_sel = &vi->evtchn_pending_sel; - if (IS_ALIGNED((unsigned long)&vi->evtchn_pending_sel, sizeof(u64))) - asm volatile(LOCK_PREFIX "orq %[src], %[dst]\n" - : [dst] "+m" (vi->evtchn_pending_sel) - : [src] "r" (evtchn_pending_sel)); - else + if (IS_ALIGNED((unsigned long)vi_pending_sel, sizeof(u64))) { + atomic64_or(evtchn_pending_sel, vi_pending_sel); + } else { + atomic_or(evtchn_pending_sel, vi_pending_sel); /* * The cast keeps the shift well-defined on 32-bit, * where evtchn_pending_sel is 32 bits wide and this @@ -691,28 +691,17 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) * kvm_xen_has_64bit_shinfo(), which is gated on * IS_ENABLED(CONFIG_64BIT)). */ - asm volatile(LOCK_PREFIX "orl %[src_lo], %[dst_lo]\n" - LOCK_PREFIX "orl %[src_hi], %[dst_hi]\n" - : [dst_lo] "+m" (vi->evtchn_pending_sel), - [dst_hi] "+m" (*(((u32 *)&vi->evtchn_pending_sel) + 1)) - : [src_lo] "r" ((u32)evtchn_pending_sel), - [src_hi] "r" ((u32)((u64)evtchn_pending_sel >> 32))); + atomic_or((u64)evtchn_pending_sel >> 32, + vi_pending_sel + 4); + } - asm volatile(LOCK_PREFIX "andq %1, %0\n" - : "+m" (v->arch.xen.evtchn_pending_sel) - : "r" (~evtchn_pending_sel)); + atomic64_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } else { - u32 evtchn_pending_sel32 = evtchn_pending_sel; struct compat_vcpu_info *vi = gpc->khva; - asm volatile(LOCK_PREFIX "orl %0, %1\n" - "notl %0\n" - LOCK_PREFIX "andl %0, %2\n" - : "=r" (evtchn_pending_sel32), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel32)); + atomic_or(evtchn_pending_sel, (void *)&vi->evtchn_pending_sel); + atomic_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } From aa8ad943f242ef14d136c5440b1178de77466c9e Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 31 Aug 2026 22:26:41 +0100 Subject: [PATCH 010/506] KVM: x86/xen: Take kvm->srcu in __kvm_xen_has_interrupt() kvm_gpc_check() checks the cached memslot generation against the current one, which dereferences kvm->memslots and therefore requires kvm->srcu to be held. __kvm_xen_has_interrupt() does not take it. Most callers do happen to hold kvm->srcu, but not all of them: - kvm_emulate_halt() on the VM-Exit path, via kvm_vcpu_has_events() and kvm_cpu_has_extint(). vcpu_enter_guest() drops the vCPU's SRCU lock before entering the guest, so it is not held on the way back out. - kvm_vcpu_block() -> kvm_vcpu_check_block() -> kvm_arch_vcpu_runnable(), which is the case the existing comment in this function describes. On a PROVE_RCU kernel the former produces: WARNING: suspicious RCU usage include/linux/kvm_host.h:1092 suspicious rcu_dereference_check() usage! ... kvm_gpc_check+0x344/0x3e0 [kvm] __kvm_xen_has_interrupt+0x83/0x310 [kvm] kvm_cpu_has_extint+0x1ff/0x370 [kvm] kvm_cpu_has_interrupt+0x16/0x100 [kvm] kvm_vcpu_has_events+0x4ce/0x690 [kvm] kvm_emulate_halt+0x52/0x1f0 [kvm] vmx_vcpu_run+0x988/0x2630 [kvm_intel] Use guard(srcu) so that the three existing early returns don't each need an explicit unlock. SRCU read sections nest, so this is harmless on the paths which already hold it, and srcu_read_lock() does not sleep, so it is also safe in the atomic case which this function already handles. Fixes: 7caf9571563e ("KVM: x86/xen: Use gfn_to_pfn_cache for vcpu_info") Cc: stable@vger.kernel.org Signed-off-by: David Woodhouse Assisted-by: Claude:claude-mythos-5 Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-11-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index a4befaaa39af..75225a3b1246 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -732,6 +732,16 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) BUILD_BUG_ON(sizeof(rc) != sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending)); + /* + * kvm_gpc_check() checks the memslot generation, so kvm->srcu must be + * held. Most callers hold it already, but this is also reached from + * kvm_emulate_halt() on the VM-Exit path and from kvm_vcpu_block(), + * where vcpu_enter_guest() has already dropped the vCPU's SRCU lock. + * Taking SRCU does not sleep, so it is safe even in the atomic case + * which is handled below. + */ + guard(srcu)(&v->kvm->srcu); + read_lock_irqsave(&gpc->lock, flags); while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) { read_unlock_irqrestore(&gpc->lock, flags); From f7b8f2ca58c89fb363d62a119e3201ab807543f1 Mon Sep 17 00:00:00 2001 From: Chengfeng Ye Date: Mon, 31 Aug 2026 22:26:42 +0100 Subject: [PATCH 011/506] KVM: x86/xen: Mark poll_evtchn accesses with READ_ONCE()/WRITE_ONCE() Use READ_ONCE() and WRITE_ONCE() for runtime accesses to poll_evtchn. This marks the intentionally concurrent scalar accesses and prevents the compiler from splitting, merging, or inventing accesses. kvm_xen_schedop_poll() publishes the single port, or -1 for multiple ports, before setting poll_mask and halting the vCPU. Event delivery can call kvm_xen_check_poller() on another CPU while the vCPU thread publishes that value or resets the field to zero after returning from kvm_vcpu_halt(): vCPU thread event delivery thread ----------- --------------------- poll_evtchn = port set_bit(poll_mask) kvm_vcpu_halt() poll_evtchn = READ poll_evtchn = 0 clear_bit(poll_mask) The plain read and writes therefore race. KCSAN reported: BUG: KCSAN: data-race in kvm_xen_hypercall / kvm_xen_set_evtchn_fast read to 0xffff888112f55af0 of 4 bytes by task 98: kvm_xen_set_evtchn_fast+0x204/0x7c0 kvm_xen_hvm_evtchn_send+0xab/0x100 kvm_arch_vm_ioctl+0xb31/0xd90 kvm_vm_ioctl+0xf42/0x16c0 write to 0xffff888112f55af0 of 4 bytes by task 96: kvm_xen_hypercall+0xd8d/0xf50 kvm_emulate_hypercall+0x157/0x1d0 vmx_handle_exit+0x40f/0xae0 vcpu_run+0x137f/0x27d0 kvm_arch_vcpu_ioctl_run+0x5a5/0x970 The field is an aligned int on x86. Access annotations preserve the existing matching, callback, and poll-mask control flow while making the single-copy access requirement explicit. Fixes: 1a65105a5aba ("KVM: x86/xen: handle PV spinlocks slowpath") Signed-off-by: Chengfeng Ye Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-12-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 75225a3b1246..9eab212ea5a8 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1581,9 +1581,9 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, } if (sched_poll.nr_ports == 1) - vcpu->arch.xen.poll_evtchn = port; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, port); else - vcpu->arch.xen.poll_evtchn = -1; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, -1); set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask); @@ -1602,7 +1602,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); } - vcpu->arch.xen.poll_evtchn = 0; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, 0); *r = 0; out: /* Really, this is only needed in case of timeout */ @@ -1822,7 +1822,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) { - int poll_evtchn = vcpu->arch.xen.poll_evtchn; + int poll_evtchn = READ_ONCE(vcpu->arch.xen.poll_evtchn); if ((poll_evtchn == port || poll_evtchn == -1) && test_and_clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask)) { From da260bcb56feeb6489a83f310308453a0a7de25e Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Mon, 31 Aug 2026 22:26:43 +0100 Subject: [PATCH 012/506] KVM: pfncache: use a dedicated invalidation sequence for cache refresh The gfn_to_pfn_cache refresh path guards against mmu notifier invalidations which complete while it has dropped gpc->lock for the HVA->PFN lookup: hva_to_pfn_retry() samples kvm->mmu_invalidate_seq and retries if it changed, or if mn_active_invalidate_count is still elevated. That is insufficient for HVA-based caches. mmu_invalidate_seq is only advanced by kvm_mmu_invalidate_end() when the invalidated range overlaps a memslot, and an HVA-based cache (e.g. the Xen shared_info page mapped with KVM_XEN_ATTR_TYPE_SHARED_INFO_HVA) need not be backed by any memslot at all. An invalidation of the cached HVA which starts and ends entirely within the lookup window is thus invisible to the retry check: mn_active_invalidate_count is back to zero and the sequence never moved. The refresh then publishes a mapping of a page which has already been freed, and the next reader dereferences it: BUG: KASAN: use-after-free in kvm_xen_shared_info_init+0x3c6/0x440 Read of size 4 at addr ffff8880599c2900 by task syz.2.383/7257 Since gfn_to_pfn_cache_invalidate_start() deliberately skips caches which are not currently valid (including one whose refresh is in progress, as the refresh clears the valid flag before dropping the lock), the retry check is the only line of defence, and it must fire for *any* invalidation, not just those hitting a memslot. Add a dedicated kvm->gpc_invalidate_seq, incremented by every kvm_mmu_notifier_invalidate_range_end() under mn_invalidate_lock before mn_active_invalidate_count is decremented, and check it in hva_to_pfn_retry() instead of mmu_invalidate_seq. Incrementing in range_end() in the same critical section as the in-progress count also closes the variant where the cache is activated with the contested HVA only after invalidate_range_start() has run. The same bug is also reachable through the per-vCPU vcpu_info cache (KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO_HVA), where the stale mapping is then dereferenced by kvm_setup_guest_pvclock() on the next KVM_RUN: BUG: KASAN: use-after-free in kvm_setup_guest_pvclock+0x5bf/0x660 This intentionally makes refresh retry on *unrelated* mmu notifier events; restoring precision (and reworking the GPC locking more generally) is left for a subsequent series. Reproducers: https://david.woodhou.se/xen_shinfo_race.c https://david.woodhou.se/vcpu_info_race.c Suggested-by: Sean Christopherson Reported-by: syzbot+0948c82180d475ad24e2@syzkaller.appspotmail.com Closes: https://lore.kernel.org/all/6a0c5f2c.a00a0220.2c7954.0000.GAE@google.com/ Tested-by: syzbot+0948c82180d475ad24e2@syzkaller.appspotmail.com Reported-by: syzbot+fb7c2dd166d3ea63df2a@syzkaller.appspotmail.com Closes: https://lore.kernel.org/all/6a426dd2.854d4ab9.360e1d.0008.GAE@google.com/ Fixes: b9220d32799a ("KVM: x86/xen: allow shared_info to be mapped by fixed HVA") Cc: stable@vger.kernel.org Signed-off-by: David Woodhouse Assisted-by: Claude:claude-mythos-5 Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-13-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 2 ++ virt/kvm/kvm_main.c | 10 ++++++++++ virt/kvm/pfncache.c | 18 +++++++++--------- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..3dd04605f2e5 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -855,6 +855,8 @@ struct kvm { gfn_t mmu_invalidate_range_start; gfn_t mmu_invalidate_range_end; + unsigned long gpc_invalidate_seq; + struct list_head devices; u64 manual_dirty_log_protect; struct dentry *debugfs_dentry; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..108d42c5c1d6 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -813,6 +813,16 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, /* Pairs with the increment in range_start(). */ spin_lock(&kvm->mn_invalidate_lock); + kvm->gpc_invalidate_seq++; + + /* + * As with the MMU sequence counter and mmu_invalidate_in_progress, the + * GPC sequence increase must be visible before the invalidate count + * goes to zero. Pairs with the smp_rmb() in + * mmu_notifier_retry_cache(). + */ + smp_wmb(); + if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count)) --kvm->mn_active_invalidate_count; wake = !kvm->mn_active_invalidate_count; diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index 728d2c1b488a..3659686b97c2 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -124,7 +124,7 @@ static void gpc_unmap(kvm_pfn_t pfn, void *khva) #endif } -static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_seq) +static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long gpc_seq) { /* * mn_active_invalidate_count acts for all intents and purposes @@ -136,20 +136,20 @@ static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_s * Note, it does not matter that mn_active_invalidate_count * is not protected by gpc->lock. It is guaranteed to * be elevated before the mmu_notifier acquires gpc->lock, and - * isn't dropped until after mmu_invalidate_seq is updated. + * isn't dropped until after gpc_invalidate_seq is updated. */ if (kvm->mn_active_invalidate_count) return true; /* * Ensure mn_active_invalidate_count is read before - * mmu_invalidate_seq. This pairs with the smp_wmb() in - * mmu_notifier_invalidate_range_end() to guarantee either the + * gpc_invalidate_seq. This pairs with the smp_wmb() in + * kvm_mmu_notifier_invalidate_range_end() to guarantee either the * old (non-zero) value of mn_active_invalidate_count or the - * new (incremented) value of mmu_invalidate_seq is observed. + * new (incremented) value of gpc_invalidate_seq is observed. */ smp_rmb(); - return kvm->mmu_invalidate_seq != mmu_seq; + return kvm->gpc_invalidate_seq != gpc_seq; } static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) @@ -158,7 +158,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) void *old_khva = (void *)PAGE_ALIGN_DOWN((uintptr_t)gpc->khva); kvm_pfn_t new_pfn = KVM_PFN_ERR_FAULT; void *new_khva = NULL; - unsigned long mmu_seq; + unsigned long gpc_seq; struct page *page; struct kvm_follow_pfn kfp = { @@ -181,7 +181,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) gpc->valid = false; do { - mmu_seq = gpc->kvm->mmu_invalidate_seq; + gpc_seq = gpc->kvm->gpc_invalidate_seq; smp_rmb(); write_unlock_irq(&gpc->lock); @@ -232,7 +232,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) * attempting to refresh. */ WARN_ON_ONCE(gpc->valid); - } while (mmu_notifier_retry_cache(gpc->kvm, mmu_seq)); + } while (mmu_notifier_retry_cache(gpc->kvm, gpc_seq)); gpc->valid = true; gpc->pfn = new_pfn; From c02e72c8daa6c63bfadf69a1a86a64f6330bcf5f Mon Sep 17 00:00:00 2001 From: Furkan Caliskan Date: Mon, 31 Aug 2026 22:26:44 +0100 Subject: [PATCH 013/506] KVM: x86/xen: Convert evtchn_ports from IDR to XArray IDR is deprecated in favor of XArray: see Documentation/core-api/idr.rst. Convert evtchn_ports accordingly. kvm_xen_eventfd_assign()'s single-slot idr_alloc() becomes xa_insert(), since it was really an insert-at-index, not an allocation: -EBUSY replaces -ENOSPC, still mapped to -EEXIST. kvm_xen_hcall_evtchn_send() drops its explicit rcu_read_lock(), since xa_load() takes its own RCU read-side section internally. evtchnfd's lifetime is still guaranteed by kvm->srcu. xen_lock is left in place: it protects state beyond the map itself. Signed-off-by: Furkan Caliskan Reviewed-by: David Woodhouse Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260831213632.81023-14-dwmw2@infradead.org Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 ++- arch/x86/kvm/xen.c | 34 ++++++++++++++++----------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 683bb8bf43a9..3af7395c2430 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -1113,7 +1114,7 @@ struct kvm_xen { bool runstate_update_flag; u8 upcall_vector; struct gfn_to_pfn_cache shinfo_cache; - struct idr evtchn_ports; + struct xarray evtchn_ports; unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; struct kvm_xen_hvm_config hvm_config; diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 9eab212ea5a8..6aa00f746cba 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -2141,7 +2141,7 @@ static int kvm_xen_eventfd_update(struct kvm *kvm, /* Protect writes to evtchnfd as well as the idr lookup. */ mutex_lock(&kvm->arch.xen.xen_lock); - evtchnfd = idr_find(&kvm->arch.xen.evtchn_ports, port); + evtchnfd = xa_load(&kvm->arch.xen.evtchn_ports, port); ret = -ENOENT; if (!evtchnfd) @@ -2235,13 +2235,13 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, } mutex_lock(&kvm->arch.xen.xen_lock); - ret = idr_alloc(&kvm->arch.xen.evtchn_ports, evtchnfd, port, port + 1, + ret = xa_insert(&kvm->arch.xen.evtchn_ports, port, evtchnfd, GFP_KERNEL); mutex_unlock(&kvm->arch.xen.xen_lock); - if (ret >= 0) + if (!ret) return 0; - if (ret == -ENOSPC) + if (ret == -EBUSY) ret = -EEXIST; out: if (eventfd) @@ -2256,7 +2256,7 @@ static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port) struct evtchnfd *evtchnfd; mutex_lock(&kvm->arch.xen.xen_lock); - evtchnfd = idr_remove(&kvm->arch.xen.evtchn_ports, port); + evtchnfd = xa_erase(&kvm->arch.xen.evtchn_ports, port); mutex_unlock(&kvm->arch.xen.xen_lock); if (!evtchnfd) @@ -2272,7 +2272,7 @@ static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port) static int kvm_xen_eventfd_reset(struct kvm *kvm) { struct evtchnfd *evtchnfd, **all_evtchnfds; - int i; + unsigned long i; int n = 0; mutex_lock(&kvm->arch.xen.xen_lock); @@ -2282,7 +2282,7 @@ static int kvm_xen_eventfd_reset(struct kvm *kvm) * critical section, first collect all the evtchnfd objects * in an array as they are removed from evtchn_ports. */ - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) n++; all_evtchnfds = kmalloc_objs(struct evtchnfd *, n); @@ -2292,9 +2292,9 @@ static int kvm_xen_eventfd_reset(struct kvm *kvm) } n = 0; - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) { all_evtchnfds[n++] = evtchnfd; - idr_remove(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port); + xa_erase(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port); } mutex_unlock(&kvm->arch.xen.xen_lock); @@ -2345,12 +2345,10 @@ static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r) } /* - * evtchnfd is protected by kvm->srcu; the idr lookup instead - * is protected by RCU. + * evtchnfd is protected by kvm->srcu; the xa_load is RCU-safe + * internally, no explicit rcu_read_lock() needed. */ - rcu_read_lock(); - evtchnfd = idr_find(&vcpu->kvm->arch.xen.evtchn_ports, send.port); - rcu_read_unlock(); + evtchnfd = xa_load(&vcpu->kvm->arch.xen.evtchn_ports, send.port); if (!evtchnfd) return false; @@ -2397,23 +2395,23 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) void kvm_xen_init_vm(struct kvm *kvm) { mutex_init(&kvm->arch.xen.xen_lock); - idr_init(&kvm->arch.xen.evtchn_ports); + xa_init(&kvm->arch.xen.evtchn_ports); kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm); } void kvm_xen_destroy_vm(struct kvm *kvm) { struct evtchnfd *evtchnfd; - int i; + unsigned long i; kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache); - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) { if (!evtchnfd->deliver.port.port) eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx); kfree(evtchnfd); } - idr_destroy(&kvm->arch.xen.evtchn_ports); + xa_destroy(&kvm->arch.xen.evtchn_ports); if (kvm->arch.xen.hvm_config.msr) static_branch_slow_dec_deferred(&kvm_xen_enabled); From 5be2d20f981843faa8b754479102ac8e9e707ead Mon Sep 17 00:00:00 2001 From: Hisam Mehboob Date: Mon, 3 Aug 2026 22:08:28 +0500 Subject: [PATCH 014/506] KVM: selftests: Replace ulong with unsigned long KVM selftests use the non-standard ulong alias in several files. glibc exposes the alias transitively through its headers, whereas musl does not, causing musl builds to fail with ulong undeclared. Use unsigned long directly instead of depending on a libc-specific typedef. This also avoids adding a duplicate typedef to a shared tools header. Reported-by: Aqib Faruqui Closes: https://lore.kernel.org/r/20250829142556.72577-10-aqibaf@amazon.com Signed-off-by: Hisam Mehboob Link: https://patch.msgid.link/20260803170825.925561-4-hisamshar@gmail.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/arm64/hypercalls.c | 2 +- .../testing/selftests/kvm/memslot_perf_test.c | 2 +- tools/testing/selftests/kvm/steal_time.c | 20 +++++++++---------- tools/testing/selftests/kvm/x86/amx_test.c | 2 +- tools/testing/selftests/kvm/x86/cpuid_test.c | 2 +- .../testing/selftests/kvm/x86/hyperv_clock.c | 2 +- .../testing/selftests/kvm/x86/hyperv_evmcs.c | 4 ++-- .../selftests/kvm/x86/hyperv_svm_test.c | 2 +- .../selftests/kvm/x86/set_boot_cpu_id.c | 2 +- tools/testing/selftests/kvm/x86/state_test.c | 4 ++-- .../testing/selftests/kvm/x86/tsc_msrs_test.c | 2 +- .../kvm/x86/vmx_nested_la57_state_test.c | 2 +- .../kvm/x86/vmx_preemption_timer_test.c | 4 ++-- 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/hypercalls.c b/tools/testing/selftests/kvm/arm64/hypercalls.c index 5d96cdf382c4..90d22b2cb6cc 100644 --- a/tools/testing/selftests/kvm/arm64/hypercalls.c +++ b/tools/testing/selftests/kvm/arm64/hypercalls.c @@ -162,7 +162,7 @@ struct st_time { static void steal_time_init(struct kvm_vcpu *vcpu) { - u64 st_ipa = (ulong)ST_GPA_BASE; + u64 st_ipa = (unsigned long)ST_GPA_BASE; unsigned int gpages; gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE); diff --git a/tools/testing/selftests/kvm/memslot_perf_test.c b/tools/testing/selftests/kvm/memslot_perf_test.c index e1299611be32..bc184d4e1656 100644 --- a/tools/testing/selftests/kvm/memslot_perf_test.c +++ b/tools/testing/selftests/kvm/memslot_perf_test.c @@ -152,7 +152,7 @@ static void *vcpu_worker(void *__data) case UCALL_SYNC: TEST_ASSERT(uc.args[1] == 0, "Unexpected sync ucall, got %lx", - (ulong)uc.args[1]); + (unsigned long)uc.args[1]); sem_post(&vcpu_ready); continue; case UCALL_NONE: diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index bc3c62b72c58..e85e0f1e7fdb 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -73,12 +73,12 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); - vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_MSR_ENABLED); + vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (unsigned long)st_gva[i] | KVM_MSR_ENABLED); } static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct kvm_steal_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" steal: %lld\n", st->steal); @@ -102,7 +102,7 @@ static void check_steal_time_uapi(void) vm = vm_create_with_one_vcpu(&vcpu, NULL); ret = _vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, - (ulong)ST_GPA_BASE | KVM_STEAL_RESERVED_MASK); + (unsigned long)ST_GPA_BASE | KVM_STEAL_RESERVED_MASK); TEST_ASSERT(ret == 0, "Bad GPA didn't fail"); kvm_vm_free(vm); @@ -151,7 +151,7 @@ static void guest_code(int cpu) status = smccc(PV_TIME_ST, 0); GUEST_ASSERT_NE(status, -1); - GUEST_ASSERT_EQ(status, (ulong)st_gva[cpu]); + GUEST_ASSERT_EQ(status, (unsigned long)st_gva[cpu]); st = (struct st_time *)status; GUEST_SYNC(0); @@ -189,13 +189,13 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) /* ST_GPA_BASE is identity mapped */ WRITE_AND_SYNC_TO_GUEST(vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); - st_ipa = (ulong)st_gva[i]; + st_ipa = (unsigned long)st_gva[i]; vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); } static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct st_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct st_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" rev: %d\n", st->rev); @@ -222,11 +222,11 @@ static void check_steal_time_uapi(void) vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, 1, 0); virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, 1); - st_ipa = (ulong)ST_GPA_BASE | 1; + st_ipa = (unsigned long)ST_GPA_BASE | 1; ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); TEST_ASSERT(ret == -1 && errno == EINVAL, "Bad IPA didn't report EINVAL"); - st_ipa = (ulong)ST_GPA_BASE; + st_ipa = (unsigned long)ST_GPA_BASE; vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); @@ -315,7 +315,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct sta_struct *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct sta_struct *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); int i; pr_info("VCPU%d:\n", vcpu_idx); @@ -451,7 +451,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct kvm_steal_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" steal: %lld\n", st->steal); diff --git a/tools/testing/selftests/kvm/x86/amx_test.c b/tools/testing/selftests/kvm/x86/amx_test.c index 4e63da2b1889..66b6e442c41c 100644 --- a/tools/testing/selftests/kvm/x86/amx_test.c +++ b/tools/testing/selftests/kvm/x86/amx_test.c @@ -325,7 +325,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } break; case UCALL_DONE: diff --git a/tools/testing/selftests/kvm/x86/cpuid_test.c b/tools/testing/selftests/kvm/x86/cpuid_test.c index ef0ddd240887..dcd99a0501ee 100644 --- a/tools/testing/selftests/kvm/x86/cpuid_test.c +++ b/tools/testing/selftests/kvm/x86/cpuid_test.c @@ -128,7 +128,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage + 1, "Stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); return; case UCALL_DONE: return; diff --git a/tools/testing/selftests/kvm/x86/hyperv_clock.c b/tools/testing/selftests/kvm/x86/hyperv_clock.c index d5d779623cc6..7a1b3b5d1e62 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_clock.c +++ b/tools/testing/selftests/kvm/x86/hyperv_clock.c @@ -248,7 +248,7 @@ int main(void) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); /* Reset kvmclock triggering TSC page update */ if (stage == 7 || stage == 8 || stage == 10) { diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 63ea1533e4ea..981a3a6d72fe 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -221,7 +221,7 @@ static struct kvm_vcpu *save_restore_vm(struct kvm_vm *vm, vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); return vcpu; } @@ -277,7 +277,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); vcpu = save_restore_vm(vm, vcpu); diff --git a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c index 1f74b0fa9b83..3a3dc31072e8 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c +++ b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c @@ -186,7 +186,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); } diff --git a/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c b/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c index 8e3898646c69..909cd9e83fda 100644 --- a/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c +++ b/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c @@ -69,7 +69,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage + 1, "Stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); test_set_bsp_busy(vcpu, "while running vm"); break; case UCALL_DONE: diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 4a1056a6cb8d..e6f521a6e737 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -296,7 +296,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); state = vcpu_save_state(vcpu); memset(®s1, 0, sizeof(regs1)); @@ -343,7 +343,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } done: diff --git a/tools/testing/selftests/kvm/x86/tsc_msrs_test.c b/tools/testing/selftests/kvm/x86/tsc_msrs_test.c index 91583969a14f..55779773614b 100644 --- a/tools/testing/selftests/kvm/x86/tsc_msrs_test.c +++ b/tools/testing/selftests/kvm/x86/tsc_msrs_test.c @@ -78,7 +78,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage) else ksft_test_result_fail( "stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); return; case UCALL_DONE: ksft_test_result_pass("stage %d passed\n", stage + 1); diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c index 75073efa926d..448721b54181 100644 --- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c @@ -109,7 +109,7 @@ int main(int argc, char *argv[]) } TEST_ASSERT(uc.args[1] == stage, - "Expected stage %d, got stage %lu", stage, (ulong)uc.args[1]); + "Expected stage %d, got stage %lu", stage, (unsigned long)uc.args[1]); if (stage == 1) { pr_info("L2 is active; performing save/restore.\n"); state = vcpu_save_state(vcpu); diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index eb8021c33cd4..72fbf1b70e9f 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -193,7 +193,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); /* * If this stage 2 then we should verify the vmx pt expiry * is as expected. @@ -234,7 +234,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } done: From 73ac07325851f50b602bb2d009c9d0eb16762c42 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:04 -0700 Subject: [PATCH 015/506] KVM: selftests: Use MEM_REGION_PT memslot instead of '0' for s390 regions/segments Use MEM_REGION_PT to indirectly specify memslot '0' instead of hardcoding the memslot in s390's virtual memory allocator, as a first step towards taking the allocation type instead of the exact, raw memslot in the physical memory allocator. Note, the only tests that set MEM_REGION_PT to a memslot other than '0' are arm64-only, i.e. s390 always uses memslot '0' for page tables. No functional change intended. Tested-by: Itaru Kitayama Acked-by: Claudio Imbrenda Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-2-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/s390/processor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/lib/s390/processor.c b/tools/testing/selftests/kvm/lib/s390/processor.c index a9adb3782b35..fb9dab92d4d1 100644 --- a/tools/testing/selftests/kvm/lib/s390/processor.c +++ b/tools/testing/selftests/kvm/lib/s390/processor.c @@ -39,7 +39,8 @@ static u64 virt_alloc_region(struct kvm_vm *vm, int ri) u64 taddr; taddr = vm_phy_pages_alloc(vm, ri < 4 ? PAGES_PER_REGION : 1, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0); + KVM_GUEST_PAGE_TABLE_MIN_PADDR, + vm->memslots[MEM_REGION_PT]); memset(addr_gpa2hva(vm, taddr), 0xff, PAGES_PER_REGION * vm->page_size); return (taddr & REGION_ENTRY_ORIGIN) From 9886413e04667924c9b42ab8833712e2445302ad Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:05 -0700 Subject: [PATCH 016/506] KVM: selftests: Use the common minimum GPA for page tables on LoongArch Use the common minimum GPA for page tables (0x180000) on LoongArch instead of LoongArch's custom minimum of 0x200000, so that LoongArch can use vm_alloc_page_table() instead of open coding a rough equivalent. There are no known reasons why LoongArch can't use a lower, less-aligned minimum. The more-aligned value was originally selected to support huge pages for the guest, but that only requires the address of the final allocation to be aligned, the addresses of page table pages is irrelevant on that front. Reviewed-by: Bibo Mao Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-3-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/loongarch/processor.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/loongarch/processor.c b/tools/testing/selftests/kvm/lib/loongarch/processor.c index 47bbde3e205a..057a77713542 100644 --- a/tools/testing/selftests/kvm/lib/loongarch/processor.c +++ b/tools/testing/selftests/kvm/lib/loongarch/processor.c @@ -9,7 +9,6 @@ #include "processor.h" #include "ucall_common.h" -#define LOONGARCH_PAGE_TABLE_PHYS_MIN 0x200000 #define LOONGARCH_GUEST_STACK_VADDR_MIN 0x200000 static gpa_t invalid_pgtable[4]; @@ -57,7 +56,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) child = table = 0; for (i = 0; i < vm->mmu.pgtable_levels; i++) { invalid_pgtable[i] = child; - table = vm_phy_page_alloc(vm, LOONGARCH_PAGE_TABLE_PHYS_MIN, + table = vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, vm->memslots[MEM_REGION_PT]); TEST_ASSERT(table, "Fail to allocate page tale at level %d\n", i); virt_set_pgtable(vm, table, child); From befd94805906f986a564c16bab17467a114cfb81 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:06 -0700 Subject: [PATCH 017/506] KVM: selftests: Use vm_alloc_page_table() to allocate LoongArch page tables Now that LoongArch uses KVM_GUEST_PAGE_TABLE_MIN_PADDR for its page table page allocations, use the common vm_alloc_page_table() instead of open coding the same. Opportunistically drop the assert that the allocation succeeded, as the allocator itself guarantees success. No functional change intended. Reviewed-by: Bibo Mao Tested-by: Itaru Kitayama Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-4-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/loongarch/processor.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/loongarch/processor.c b/tools/testing/selftests/kvm/lib/loongarch/processor.c index 057a77713542..fcd41bc6fbda 100644 --- a/tools/testing/selftests/kvm/lib/loongarch/processor.c +++ b/tools/testing/selftests/kvm/lib/loongarch/processor.c @@ -56,9 +56,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) child = table = 0; for (i = 0; i < vm->mmu.pgtable_levels; i++) { invalid_pgtable[i] = child; - table = vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); - TEST_ASSERT(table, "Fail to allocate page tale at level %d\n", i); + table = vm_alloc_page_table(vm); virt_set_pgtable(vm, table, child); child = table; } From d220e33516fa3faa458b814d08dfa680f93ea2ae Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:07 -0700 Subject: [PATCH 018/506] KVM: selftests: Rename "num" param to "nr_pages" for physical page allocators Rename the "num" param in the physical page allocator APIs to "nr_pages" to make the unit and role of the param more obvious. Opportunistically rework the function comment to drop the superfluous kerneldoc style, and to very explicitly call out that success is guaranteed. No functional change intended. Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-5-seanjc@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 6 ++-- tools/testing/selftests/kvm/lib/kvm_util.c | 35 ++++++------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 5a7a455b5387..95bc0a78d2bc 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -991,11 +991,11 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); const char *exit_reason_str(unsigned int exit_reason); gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot); -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, gpa_t min_gpa, +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot, bool protected); gpa_t vm_alloc_page_table(struct kvm_vm *vm); -static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, +static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot) { /* @@ -1003,7 +1003,7 @@ static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, * protected memory, as the majority of memory for such VMs is * protected, i.e. using shared memory is effectively opt-in. */ - return __vm_phy_pages_alloc(vm, num, min_gpa, memslot, + return __vm_phy_pages_alloc(vm, nr_pages, min_gpa, memslot, vm_arch_has_protected_memory(vm)); } diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9ddc047d5c27..76d2598a9f90 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -2025,33 +2025,20 @@ const char *exit_reason_str(unsigned int exit_reason) } /* - * Physical Contiguous Page Allocator + * Allocate contiguous (guest) physical pages in a given memory region, at or + * above the minimum specified GPA. If the memory is protected/private, also + * add the allocated pages to the region's set of protected pages, e.g. so that + * arch code knows which pages need to be encrypted when launching the VM. * - * Input Args: - * vm - Virtual Machine - * num - number of pages - * min_gpa - Physical address minimum - * memslot - Memory region to allocate page from - * protected - True if the pages will be used as protected/private memory - * - * Output Args: None - * - * Return: - * Starting physical address - * - * Within the VM specified by vm, locates a range of available physical - * pages at or above min_gpa. If found, the pages are marked as in use - * and their base address is returned. A TEST_ASSERT failure occurs if - * not enough pages are available at or above min_gpa. + * Note, success is guaranteed! */ -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, - gpa_t min_gpa, u32 memslot, - bool protected) +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected) { struct userspace_mem_region *region; sparsebit_idx_t pg, base; - TEST_ASSERT(num > 0, "Must allocate at least one page"); + TEST_ASSERT(nr_pages, "Must allocate at least one page"); TEST_ASSERT((min_gpa % vm->page_size) == 0, "Min physical address " "not divisible by page size.\n" @@ -2064,13 +2051,13 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, base = pg = min_gpa >> vm->page_shift; do { - for (; pg < base + num; ++pg) { + for (; pg < base + nr_pages; ++pg) { if (!sparsebit_is_set(region->unused_phy_pages, pg)) { base = pg = sparsebit_next_set(region->unused_phy_pages, pg); break; } } - } while (pg && pg != base + num); + } while (pg && pg != base + nr_pages); if (pg == 0) { fprintf(stderr, "No guest physical page available, " @@ -2081,7 +2068,7 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, abort(); } - for (pg = base; pg < base + num; ++pg) { + for (pg = base; pg < base + nr_pages; ++pg) { sparsebit_clear(region->unused_phy_pages, pg); if (protected) sparsebit_set(region->protected_phy_pages, pg); From 2c0555c51e7ca615c2bba7a99efd720876239b05 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 2 Sep 2026 09:41:08 -0700 Subject: [PATCH 019/506] KVM: selftests: Use goto instead of do-while to retry finding unused physical pages Rework the core loop of the physical page allocator to use a goto instead of do-while loop when retrying with a new base address. The do-while loop obviously works, but it obfuscates the success vs. failure paths, and in general is convoluted. For all intents and purposes, no functional change intended. Signed-off-by: Nicholas Piggin [Rebased to latest mainline tree] Signed-off-by: Ritesh Harjani (IBM) [sean: separate from alignment change, use enomem label, write changelog] Tested-by: Itaru Kitayama Acked-by: Gautam Menghani Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-6-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/kvm_util.c | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 76d2598a9f90..87e747bf3e3e 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -2049,23 +2049,15 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, TEST_ASSERT(!protected || region->protected_phy_pages, "Region doesn't support protected memory"); - base = pg = min_gpa >> vm->page_shift; - do { - for (; pg < base + nr_pages; ++pg) { - if (!sparsebit_is_set(region->unused_phy_pages, pg)) { - base = pg = sparsebit_next_set(region->unused_phy_pages, pg); - break; - } + base = min_gpa >> vm->page_shift; +again: + for (pg = base; pg < base + nr_pages; ++pg) { + if (!sparsebit_is_set(region->unused_phy_pages, pg)) { + base = sparsebit_next_set(region->unused_phy_pages, pg); + if (!base) + goto enomem; + goto again; } - } while (pg && pg != base + nr_pages); - - if (pg == 0) { - fprintf(stderr, "No guest physical page available, " - "min_gpa: 0x%lx page_size: 0x%x memslot: %u\n", - min_gpa, vm->page_size, memslot); - fputs("---- vm dump ----\n", stderr); - vm_dump(stderr, vm, 2); - abort(); } for (pg = base; pg < base + nr_pages; ++pg) { @@ -2075,6 +2067,14 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, } return base * vm->page_size; + +enomem: + fprintf(stderr, "No guest physical page available, min_gpa: 0x%lx page_size: 0x%x memslot: %u\n", + min_gpa, vm->page_size, memslot); + fputs("---- vm dump ----\n", stderr); + vm_dump(stderr, vm, 2); + abort(); + __builtin_unreachable(); } gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot) From 2429ecf7ccd52cac5d49adba5e880a25f16314ea Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:09 -0700 Subject: [PATCH 020/506] KVM: selftests: Extend page allocator to support naturally aligned allocations Extend the page allocator to support naturally aligned allocations, as some flavors of PowerPC page tables require multiple guest pages per table, and those tables need to be naturally aligned. Take care to align the potential base address *inside* the retry loop, i.e. the base needs to be (re)aligned on every attempt. Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-7-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/kvm_util.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 95bc0a78d2bc..4732b9528812 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -991,6 +991,8 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); const char *exit_reason_str(unsigned int exit_reason); gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot); +gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected, bool naturally_aligned); gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot, bool protected); gpa_t vm_alloc_page_table(struct kvm_vm *vm); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 87e747bf3e3e..4c14b1c73239 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #define KVM_UTIL_MIN_PFN 2 @@ -2032,9 +2033,10 @@ const char *exit_reason_str(unsigned int exit_reason) * * Note, success is guaranteed! */ -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, - u32 memslot, bool protected) +gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected, bool naturally_aligned) { + size_t alignment = naturally_aligned ? nr_pages : 1; struct userspace_mem_region *region; sparsebit_idx_t pg, base; @@ -2051,6 +2053,7 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, base = min_gpa >> vm->page_shift; again: + base = ALIGN(base, alignment); for (pg = base; pg < base + nr_pages; ++pg) { if (!sparsebit_is_set(region->unused_phy_pages, pg)) { base = sparsebit_next_set(region->unused_phy_pages, pg); @@ -2077,6 +2080,12 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, __builtin_unreachable(); } +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected) +{ + return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, memslot, protected, false); +} + gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot) { return vm_phy_pages_alloc(vm, 1, min_gpa, memslot); From 54e83f729385f918a4a50259a5f32c63b87b99fd Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:10 -0700 Subject: [PATCH 021/506] KVM: selftests: Make the single-page allocator APIs static inline Move the single-page allocator APIs to kvm_util.h as inline functions, as the are trivial wrappers, and unlike __vm_phy_pages_alloc(), are expected to remain that way for the foreseeable future. No functional change intended. Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-8-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/kvm_util.h | 14 ++++++++++++-- tools/testing/selftests/kvm/lib/kvm_util.c | 11 ----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 4732b9528812..d9d39a8894e2 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -990,12 +990,10 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); const char *exit_reason_str(unsigned int exit_reason); -gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot); gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot, bool protected, bool naturally_aligned); gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot, bool protected); -gpa_t vm_alloc_page_table(struct kvm_vm *vm); static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot) @@ -1009,6 +1007,18 @@ static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, vm_arch_has_protected_memory(vm)); } +static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, + u32 memslot) +{ + return vm_phy_pages_alloc(vm, 1, min_gpa, memslot); +} + +static inline gpa_t vm_alloc_page_table(struct kvm_vm *vm) +{ + return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, + vm->memslots[MEM_REGION_PT]); +} + /* * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also * loads the test binary into guest memory and creates an IRQ chip (x86 only). diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 4c14b1c73239..bb73671770de 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -2086,17 +2086,6 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, memslot, protected, false); } -gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot) -{ - return vm_phy_pages_alloc(vm, 1, min_gpa, memslot); -} - -gpa_t vm_alloc_page_table(struct kvm_vm *vm) -{ - return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); -} - /* * Address Guest Virtual to Host Virtual * From b8d9e6ec747a7bb7c11dc6614db7bdd5c9e260f6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:11 -0700 Subject: [PATCH 022/506] KVM: selftests: Use the innermost page allocator API in the memslot perf test Use the quad-underscores physical page allocator API in the memslot perf test in anticipation of converting the outer versions to take the memory region type instead of the raw memslot. Stating the obvious, the memslot perf test creates and allocates from multiple memslots, and so can't use MEM_REGION_TEST_DATA to communicate which region/slot to allocate from. No functional change intended. Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-9-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/memslot_perf_test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/memslot_perf_test.c b/tools/testing/selftests/kvm/memslot_perf_test.c index bc184d4e1656..1e241390e11e 100644 --- a/tools/testing/selftests/kvm/memslot_perf_test.c +++ b/tools/testing/selftests/kvm/memslot_perf_test.c @@ -339,7 +339,8 @@ static bool prepare_vm(struct vm_data *data, int nslots, u64 *maxslots, if (slot == data->nslots) npages += rempages; - gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr, slot); + gpa = ____vm_phy_pages_alloc(data->vm, npages, guest_addr, slot, + false, false); TEST_ASSERT(gpa == guest_addr, "vm_phy_pages_alloc() failed"); From 9c0061d2e7cd25bea2adcf3aecf1aa4027e83343 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:12 -0700 Subject: [PATCH 023/506] KVM: selftests: Use the innermost page allocator API in s390's IRQ routing test Use the quad-underscores page allocator in s390's IRQ routing test in anticipation of dropping the @min_gpa param from the primary APIs. Note, it's not entirely clear that the test *needs* to use a very magical @min_gpa, i.e. it's possible a more elegant solution exists. But that's a problem for a different day. No functional change intended. Tested-by: Itaru Kitayama Acked-by: Claudio Imbrenda Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-10-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/s390/irq_routing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/s390/irq_routing.c b/tools/testing/selftests/kvm/s390/irq_routing.c index f3839284ac08..cd26cc1aed80 100644 --- a/tools/testing/selftests/kvm/s390/irq_routing.c +++ b/tools/testing/selftests/kvm/s390/irq_routing.c @@ -36,7 +36,7 @@ static void test(void) }; vm = vm_create_with_one_vcpu(&vcpu, guest_code); - mem = vm_phy_pages_alloc(vm, 2, 4096 * 42, 0); + mem = ____vm_phy_pages_alloc(vm, 2, 4096 * 42, 0, false, false); routing = kvm_gsi_routing_create(); routing->nr = 1; From 7f477ba0878e9db2cfe5da8b6d058f3a677fad91 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:13 -0700 Subject: [PATCH 024/506] KVM: selftests: Add a wrapper API to allocate multiple page table pages Add vm_alloc_page_table_pages() and use it throughout arch code to replace open coded equivalents. No functional change intended. Reviewed-by: Itaru Kitayama Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Acked-by: Claudio Imbrenda Acked-by: Anup Patel Acked-by: Gautam Menghani Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-11-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/kvm_util.h | 9 +++++++-- tools/testing/selftests/kvm/lib/arm64/processor.c | 4 +--- tools/testing/selftests/kvm/lib/riscv/processor.c | 4 +--- tools/testing/selftests/kvm/lib/s390/processor.c | 8 ++------ 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index d9d39a8894e2..c56f342b16fd 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -1013,10 +1013,15 @@ static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, return vm_phy_pages_alloc(vm, 1, min_gpa, memslot); } +static inline gpa_t vm_alloc_page_table_pages(struct kvm_vm *vm, size_t nr_pages) +{ + return vm_phy_pages_alloc(vm, nr_pages, KVM_GUEST_PAGE_TABLE_MIN_PADDR, + vm->memslots[MEM_REGION_PT]); +} + static inline gpa_t vm_alloc_page_table(struct kvm_vm *vm) { - return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + return vm_alloc_page_table_pages(vm, 1); } /* diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c index 01325bf4d36f..2dcaa2c4a038 100644 --- a/tools/testing/selftests/kvm/lib/arm64/processor.c +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c @@ -115,9 +115,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - vm->mmu.pgd = vm_phy_pages_alloc(vm, nr_pages, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + vm->mmu.pgd = vm_alloc_page_table_pages(vm, nr_pages); vm->mmu.pgd_created = true; } diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index ded5429f3448..c175dcd14114 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -69,9 +69,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - vm->mmu.pgd = vm_phy_pages_alloc(vm, nr_pages, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + vm->mmu.pgd = vm_alloc_page_table_pages(vm, nr_pages); vm->mmu.pgd_created = true; } diff --git a/tools/testing/selftests/kvm/lib/s390/processor.c b/tools/testing/selftests/kvm/lib/s390/processor.c index fb9dab92d4d1..33f49d743b5f 100644 --- a/tools/testing/selftests/kvm/lib/s390/processor.c +++ b/tools/testing/selftests/kvm/lib/s390/processor.c @@ -20,9 +20,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - gpa = vm_phy_pages_alloc(vm, PAGES_PER_REGION, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + gpa = vm_alloc_page_table_pages(vm, PAGES_PER_REGION); memset(addr_gpa2hva(vm, gpa), 0xff, PAGES_PER_REGION * vm->page_size); vm->mmu.pgd = gpa; @@ -38,9 +36,7 @@ static u64 virt_alloc_region(struct kvm_vm *vm, int ri) { u64 taddr; - taddr = vm_phy_pages_alloc(vm, ri < 4 ? PAGES_PER_REGION : 1, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + taddr = vm_alloc_page_table_pages(vm, ri < 4 ? PAGES_PER_REGION : 1); memset(addr_gpa2hva(vm, taddr), 0xff, PAGES_PER_REGION * vm->page_size); return (taddr & REGION_ENTRY_ORIGIN) From 9e4e2281df12260e4b3419101d7d4dc027ab5d2f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:14 -0700 Subject: [PATCH 025/506] KVM: selftests: Initialize vm->memslots[] with invalid memslots during creation Initialize vm->memslots[] with an invalid memslots value during very initial VM creation so that failure to configure the core memory regions generates an immediate assert instead of potential weirdness, e.g. if slot0 is coincidentally valid. This will also allow hardening the region APIs to guard overriding a region's memslot multiple times. Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Acked-by: Gautam Menghani Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-12-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/kvm_util.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 5 +++++ tools/testing/selftests/kvm/lib/x86/processor.c | 5 ++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index c56f342b16fd..304ef6af3268 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -33,6 +33,8 @@ #define NSEC_PER_SEC 1000000000L +#define KVM_INVALID_MEMSLOT UINT_MAX + struct userspace_mem_region { struct kvm_userspace_memory_region2 region; struct sparsebit *unused_phy_pages; diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index bb73671770de..ca900ed111d6 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -288,6 +288,7 @@ __weak void vm_populate_gva_bitmap(struct kvm_vm *vm) struct kvm_vm *____vm_create(struct vm_shape shape) { struct kvm_vm *vm; + int i; vm = calloc(1, sizeof(*vm)); TEST_ASSERT(vm != NULL, "Insufficient Memory"); @@ -296,6 +297,8 @@ struct kvm_vm *____vm_create(struct vm_shape shape) vm->regions.gpa_tree = RB_ROOT; vm->regions.hva_tree = RB_ROOT; hash_init(vm->regions.slot_hash); + for (i = 0; i < NR_MEM_REGIONS; i++) + vm->memslots[i] = KVM_INVALID_MEMSLOT; vm->mode = shape.mode; vm->type = shape.type; @@ -1190,6 +1193,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot) { struct userspace_mem_region *region; + TEST_ASSERT(memslot != KVM_INVALID_MEMSLOT, "vm->memslots[] unpopulated?"); + hash_for_each_possible(vm->regions.slot_hash, region, slot_node, memslot) if (region->region.slot == memslot) diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index d31fa81ea075..ea5fa59888af 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -529,7 +529,10 @@ void tdp_identity_map_default_memslots(struct kvm_vm *vm) /* Only memslot 0 is mapped here, ensure it's the only one being used */ for (s = 0; s < NR_MEM_REGIONS; s++) - TEST_ASSERT_EQ(vm->memslots[s], 0); + TEST_ASSERT(!vm->memslots[s] || + vm->memslots[s] == KVM_INVALID_MEMSLOT, + "Unhandled memslot '%u' for type '%u'", + vm->memslots[s], s); i = (region->region.guest_phys_addr >> vm->page_shift) - 1; last = i + (region->region.memory_size >> vm->page_shift); From d69b3e714265068778a4f0832897f85daa5b90d8 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:15 -0700 Subject: [PATCH 026/506] KVM: selftests: Add APIs to override memory region types with custom memslots Add and use APIs to override memory region types instead of open coding the logic in tests, partly to dedup test code, but mostly to add some amount of protection against conflicting overrides. This will allow converting more tests to override memory regions without taking on too much risk of bugs in the long term, e.g. due to silently overriding a region multiple times. Deliberately "unroll" the loops in __vm_create() and the CMMA test's create_main_memslot() to avoid filling all memslots in anticipation of adding an "extra" memslot that is for test use and *only* test use, i.e. shouldn't be configured by core library code. Converting tests to overriding memory regions will allow taking the region type instead of the raw memslot in the physical page allocator APIs. Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Acked-by: Claudio Imbrenda Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-13-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/arm64/page_fault_test.c | 18 +++++------ .../testing/selftests/kvm/include/kvm_util.h | 30 +++++++++++++++++++ tools/testing/selftests/kvm/lib/kvm_util.c | 8 +++-- tools/testing/selftests/kvm/s390/cmma_test.c | 19 ++++-------- 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/page_fault_test.c b/tools/testing/selftests/kvm/arm64/page_fault_test.c index 6bb3d82906b2..b0e3fc8cd76a 100644 --- a/tools/testing/selftests/kvm/arm64/page_fault_test.c +++ b/tools/testing/selftests/kvm/arm64/page_fault_test.c @@ -597,18 +597,16 @@ static void setup_memslots(struct kvm_vm *vm, struct test_params *p) vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, CODE_AND_DATA_MEMSLOT, code_npages, 0); - vm->memslots[MEM_REGION_CODE] = CODE_AND_DATA_MEMSLOT; - vm->memslots[MEM_REGION_DATA] = CODE_AND_DATA_MEMSLOT; + ____vm_override_mem_region(vm, MEM_REGION_CODE, CODE_AND_DATA_MEMSLOT); + ____vm_override_mem_region(vm, MEM_REGION_DATA, CODE_AND_DATA_MEMSLOT); - vm_userspace_mem_region_add(vm, p->src_type, data_gpa - pt_size, - PAGE_TABLE_MEMSLOT, pt_size / guest_page_size, - p->test_desc->pt_memslot_flags); - vm->memslots[MEM_REGION_PT] = PAGE_TABLE_MEMSLOT; + __vm_override_mem_region(vm, MEM_REGION_PT, p->src_type, data_gpa - pt_size, + PAGE_TABLE_MEMSLOT, pt_size / guest_page_size, + p->test_desc->pt_memslot_flags); - vm_userspace_mem_region_add(vm, p->src_type, data_gpa, TEST_DATA_MEMSLOT, - data_size / guest_page_size, - p->test_desc->data_memslot_flags); - vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT; + __vm_override_mem_region(vm, MEM_REGION_TEST_DATA, p->src_type, data_gpa, + TEST_DATA_MEMSLOT, data_size / guest_page_size, + p->test_desc->data_memslot_flags); } static void setup_ucall(struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 304ef6af3268..c8faa8a02b93 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -704,6 +704,36 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, int guest_memfd_fd, u64 guest_memfd_offset); + +static inline void ____vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + u32 slot) +{ + TEST_ASSERT(vm->memslots[type] == KVM_INVALID_MEMSLOT, + "Memory region type '%u' was already overridden with slot=%u", + type, vm->memslots[type]); + + vm->memslots[type] = slot; +} + +static inline void __vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + enum vm_mem_backing_src_type src_type, + gpa_t gpa, u32 slot, u64 npages, + u32 flags) +{ + ____vm_override_mem_region(vm, type, slot); + vm_userspace_mem_region_add(vm, src_type, gpa, slot, npages, flags); +} + +static inline void vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + enum vm_mem_backing_src_type src_type, + gpa_t gpa, u32 slot, u64 npages) +{ + __vm_override_mem_region(vm, type, src_type, gpa, slot, npages, 0); +} + #ifndef vm_arch_has_protected_memory static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) { diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index ca900ed111d6..00cdc5154ed4 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -495,7 +495,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, nr_extra_pages); struct userspace_mem_region *slot0; struct kvm_vm *vm; - int i, flags; + int flags; kvm_set_files_rlimit(nr_runnable_vcpus); @@ -513,8 +513,10 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, flags |= KVM_MEM_GUEST_MEMFD; vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags); - for (i = 0; i < NR_MEM_REGIONS; i++) - vm->memslots[i] = 0; + ____vm_override_mem_region(vm, MEM_REGION_CODE, 0); + ____vm_override_mem_region(vm, MEM_REGION_PT, 0); + ____vm_override_mem_region(vm, MEM_REGION_DATA, 0); + ____vm_override_mem_region(vm, MEM_REGION_TEST_DATA, 0); kvm_vm_elf_load(vm, program_invocation_name); diff --git a/tools/testing/selftests/kvm/s390/cmma_test.c b/tools/testing/selftests/kvm/s390/cmma_test.c index 15d81b2ed7ad..73387cc3bca7 100644 --- a/tools/testing/selftests/kvm/s390/cmma_test.c +++ b/tools/testing/selftests/kvm/s390/cmma_test.c @@ -102,24 +102,17 @@ static void guest_dirty_test_data(void) static void create_main_memslot(struct kvm_vm *vm) { - int i; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, MAIN_PAGE_COUNT, 0); - /* set the array of memslots to zero like __vm_create does */ - for (i = 0; i < NR_MEM_REGIONS; i++) - vm->memslots[i] = 0; + ____vm_override_mem_region(vm, MEM_REGION_CODE, 0); + ____vm_override_mem_region(vm, MEM_REGION_PT, 0); + ____vm_override_mem_region(vm, MEM_REGION_DATA, 0); } static void create_test_memslot(struct kvm_vm *vm) { - vm_userspace_mem_region_add(vm, - VM_MEM_SRC_ANONYMOUS, - TEST_DATA_START_GFN << vm->page_shift, - TEST_DATA_MEMSLOT, - TEST_DATA_PAGE_COUNT, - 0 - ); - vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT; + vm_override_mem_region(vm, MEM_REGION_TEST_DATA, VM_MEM_SRC_ANONYMOUS, + TEST_DATA_START_GFN << vm->page_shift, + TEST_DATA_MEMSLOT, TEST_DATA_PAGE_COUNT); } static void create_memslots(struct kvm_vm *vm) From 9f035aca8765d11f0f10a20422d5763835110096 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:16 -0700 Subject: [PATCH 027/506] KVM: selftests: Add TEST_EXTRA memory region type for "special" memslots And another memory region type to deal with extra, one-off memory regions, and use the new type to manage x86's SMRAM memslot, as another step towards taking the region type instead of the raw memslot in the physical page allocator APIs. Alternatively, SMRAM setup could simply use the quad-underscore API to continue passing in the memslot, but a surprising number of tests use an "extra" memslot for a variety of reasons. I.e. allocating memory from one (and exactly one) extra memslot isn't all that rare, and so should be treated as normal behavior, not as something extraordinary, as quad-underscore functions typically suggest. Opportunistically add comments to document the intended usage of the types, as the difference between DATA, TEST_DATA, and TEST_EXTRA in particular isn't exactly obvious. Tested-by: Itaru Kitayama Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-14-seanjc@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 39 ++++++++++++++++--- tools/testing/selftests/kvm/include/x86/smm.h | 2 +- .../testing/selftests/kvm/lib/x86/processor.c | 7 ++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index c8faa8a02b93..9b808bcf7c18 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -82,11 +82,45 @@ struct userspace_mem_regions { DECLARE_HASHTABLE(slot_hash, 9); }; +/* + * Memory region types are passed to various page allocators to communicate + * various properties and metadata related to the allocation. Note, the + * descriptions below described the primary usage of each type. Individual + * tests may allocate memory for other purposes. + * + * By default, all regions except TEST_EXTRA are mapped to memslot '0'. The + * TEST_EXTRA region is left unmapped as it's intended to be used only for test + * specific allocations, i.e. should never be used by core/infrastructure code. + * Tests can override the memslot for any or all types, e.g. so that all test + * data is allocated from a curated memslot. + */ enum kvm_mem_region_type { + /* + * The CODE region is used by lib/elf when loading the test's code into + * guest memory. + */ MEM_REGION_CODE, + /* + * The DATA region is used to allocate core data structures, e.g. vCPU + * stacks, VM exception tables, x86's TSS, etc. + */ MEM_REGION_DATA, + /* + * The PT region, a.k.a. Page Table region, is used to allocate page + * table pages. + */ MEM_REGION_PT, + /* + * The TEST_DATA region is used for allocating test data that is either + * test specific, and/or isn't considered a "core" data structure. + */ MEM_REGION_TEST_DATA, + /* + * The TEST_EXTRA region is for special snowflakes, where a test wants + * to create and use a one-off memslot, without impacting "normal" test + * data allocations. + */ + MEM_REGION_TEST_EXTRA, NR_MEM_REGIONS, }; @@ -129,11 +163,6 @@ struct kvm_vm { struct kvm_binary_stats stats; - /* - * KVM region slots. These are the default memslots used by page - * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] - * memslot. - */ u32 memslots[NR_MEM_REGIONS]; }; diff --git a/tools/testing/selftests/kvm/include/x86/smm.h b/tools/testing/selftests/kvm/include/x86/smm.h index 2d1afa09819b..15faaa060126 100644 --- a/tools/testing/selftests/kvm/include/x86/smm.h +++ b/tools/testing/selftests/kvm/include/x86/smm.h @@ -8,7 +8,7 @@ #define SMRAM_MEMSLOT ((1 << 16) | 1) #define SMRAM_PAGES (SMRAM_SIZE / PAGE_SIZE) -void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa, +void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, const void *smi_handler, size_t handler_size); void inject_smi(struct kvm_vcpu *vcpu); diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index ea5fa59888af..b988eea373ad 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -1468,11 +1468,12 @@ bool kvm_arch_has_default_irqchip(void) return true; } -void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa, +void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, const void *smi_handler, size_t handler_size) { - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, smram_gpa, - SMRAM_MEMSLOT, SMRAM_PAGES, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + smram_gpa, SMRAM_MEMSLOT, SMRAM_PAGES); + TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, smram_gpa, SMRAM_MEMSLOT) == smram_gpa, "Could not allocate guest physical addresses for SMRAM"); From 5d81014d38684068f2cbe50c33d32896fa1df30d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:17 -0700 Subject: [PATCH 028/506] KVM: selftests: Use TEST_EXTRA region in arm64's vGIC LPI stress test Use the TEST_EXTRA region/memslot for arm64's vGIC LPI stress test instead of a custom memslot, so that the primary vm_phy_pages_alloc() APIs can be reworked to take a region type instead of a raw memslot. For all intents and purposes, no functional change intended, as vm_override_mem_region() is barely more than a wrapper for vm_userspace_mem_region_add(). Tested-by: Itaru Kitayama Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-15-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c index 9a69c439a0f8..9865b204fdeb 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c +++ b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c @@ -164,8 +164,8 @@ static void setup_memslot(void) pages = sz / vm->page_size; gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base, - TEST_MEMSLOT_INDEX, pages, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + gpa_base, TEST_MEMSLOT_INDEX, pages); } #define LPI_PROP_DEFAULT_PRIO 0xa0 From 00c37bfc5b4e81f3b1a1a95749a17f933803d324 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:18 -0700 Subject: [PATCH 029/506] KVM: selftests: Use TEST_EXTRA region in x86's smaller MAXPHYADDR test Use the TEST_EXTRA region/memslot for x86's smaller MAXPHYADDR test instead of a custom memslot, so that the primary vm_phy_pages_alloc() APIs can be reworked to take a region type instead of a raw memslot. For all intents and purposes, no functional change intended, as vm_add_extra_mem_region() is barely more than a wrapper for vm_userspace_mem_region_add(). Tested-by: Itaru Kitayama Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-16-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c index 3dca85e95478..b21798a385ca 100644 --- a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c @@ -62,9 +62,10 @@ int main(int argc, char *argv[]) TEST_ASSERT(rc, "KVM_CAP_EXIT_ON_EMULATION_FAILURE is unavailable"); vm_enable_cap(vm, KVM_CAP_EXIT_ON_EMULATION_FAILURE, 1); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, - MEM_REGION_GPA, MEM_REGION_SLOT, - MEM_REGION_SIZE / PAGE_SIZE, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / PAGE_SIZE); + gpa = vm_phy_pages_alloc(vm, MEM_REGION_SIZE / PAGE_SIZE, MEM_REGION_GPA, MEM_REGION_SLOT); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc"); From 5ed16822bc4525974a48e580098cab5a8ce4d7e1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:19 -0700 Subject: [PATCH 030/506] KVM: selftests: Use TEST_EXTRA region in set memory region test Use the TEST_EXTRA region/memslot in the "set memory region" test instead of a custom memslot, so that the primary vm_phy_pages_alloc() APIs can be reworked to take a region type instead of a raw memslot. For all intents and purposes, no functional change intended, as vm_override_mem_region() is barely more than a wrapper for vm_userspace_mem_region_add(). Tested-by: Itaru Kitayama Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-17-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/set_memory_region_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index dffe9654f45b..9d21594e2f81 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -116,9 +116,9 @@ static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread, vm = vm_create_with_one_vcpu(vcpu, guest_code); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP, - MEM_REGION_GPA, MEM_REGION_SLOT, - MEM_REGION_SIZE / getpagesize(), 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS_THP, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / getpagesize()); /* * Allocate and map two pages so that the GPA accessed by guest_code() From 36a062568ea80fd8714b1c4efa58d48a798c4b9f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:20 -0700 Subject: [PATCH 031/506] KVM: selftests: Take the memory region type, not memslot, in page allocators Take the memory region type instead of the memslot in all page allocator APIs, except for the innermost quad-underscores function. This will allow automatically selecting the minimum GPA based on the allocation type, which can't be done using the memslot due to all regions sharing a single memslot by default. PowerPC support will also leverage the type to ensure that page table allocations are naturally aligned. No functional change intended. Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-18-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c | 12 ++++++------ tools/testing/selftests/kvm/include/kvm_util.h | 13 +++++++------ tools/testing/selftests/kvm/lib/kvm_util.c | 10 +++++++--- tools/testing/selftests/kvm/lib/x86/processor.c | 2 +- .../testing/selftests/kvm/set_memory_region_test.c | 2 +- .../kvm/x86/smaller_maxphyaddr_emulation_test.c | 2 +- 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c index 9865b204fdeb..a45c0849a47a 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c +++ b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c @@ -192,27 +192,27 @@ static void setup_test_data(void) test_data.device_table = vm_phy_pages_alloc(vm, pages_per_64k, gpa_base, - TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); test_data.collection_table = vm_phy_pages_alloc(vm, pages_per_64k, gpa_base, - TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, gpa_base, - TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); virt_map(vm, cmdq_base, cmdq_base, pages_per_64k); test_data.cmdq_base = cmdq_base; test_data.cmdq_base_va = (void *)cmdq_base; test_data.itt_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_devices, - gpa_base, TEST_MEMSLOT_INDEX); + gpa_base, MEM_REGION_TEST_EXTRA); test_data.lpi_prop_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, TEST_MEMSLOT_INDEX); + gpa_base, MEM_REGION_TEST_EXTRA); configure_lpis(); test_data.lpi_pend_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_cpus, - gpa_base, TEST_MEMSLOT_INDEX); + gpa_base, MEM_REGION_TEST_EXTRA); sync_global_to_guest(vm, test_data); } diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 9b808bcf7c18..92ab2f5001c9 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -1054,30 +1054,31 @@ const char *exit_reason_str(unsigned int exit_reason); gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot, bool protected, bool naturally_aligned); gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, - u32 memslot, bool protected); + enum kvm_mem_region_type type, bool protected); static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, - gpa_t min_gpa, u32 memslot) + gpa_t min_gpa, + enum kvm_mem_region_type type) { /* * By default, allocate memory as protected for VMs that support * protected memory, as the majority of memory for such VMs is * protected, i.e. using shared memory is effectively opt-in. */ - return __vm_phy_pages_alloc(vm, nr_pages, min_gpa, memslot, + return __vm_phy_pages_alloc(vm, nr_pages, min_gpa, type, vm_arch_has_protected_memory(vm)); } static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, - u32 memslot) + enum kvm_mem_region_type type) { - return vm_phy_pages_alloc(vm, 1, min_gpa, memslot); + return vm_phy_pages_alloc(vm, 1, min_gpa, type); } static inline gpa_t vm_alloc_page_table_pages(struct kvm_vm *vm, size_t nr_pages) { return vm_phy_pages_alloc(vm, nr_pages, KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + MEM_REGION_PT); } static inline gpa_t vm_alloc_page_table(struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 00cdc5154ed4..aae881c8b927 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1475,7 +1475,7 @@ static gva_t ____vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva, virt_pgd_alloc(vm); gpa_t gpa = __vm_phy_pages_alloc(vm, pages, KVM_UTIL_MIN_PFN * vm->page_size, - vm->memslots[type], protected); + type, protected); /* * Find an unused range of virtual page addresses of at least @@ -2088,9 +2088,13 @@ gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, } gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, - u32 memslot, bool protected) + enum kvm_mem_region_type type, bool protected) { - return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, memslot, protected, false); + TEST_ASSERT(type < NR_MEM_REGIONS, + "Invalid memory region type '%u'", type); + + return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, vm->memslots[type], + protected, false); } /* diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index b988eea373ad..39d9ca6ceb1d 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -1475,7 +1475,7 @@ void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, smram_gpa, SMRAM_MEMSLOT, SMRAM_PAGES); TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, smram_gpa, - SMRAM_MEMSLOT) == smram_gpa, + MEM_REGION_TEST_EXTRA) == smram_gpa, "Could not allocate guest physical addresses for SMRAM"); memset(addr_gpa2hva(vm, smram_gpa), 0x0, SMRAM_SIZE); diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index 9d21594e2f81..160bbe3d7203 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -124,7 +124,7 @@ static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread, * Allocate and map two pages so that the GPA accessed by guest_code() * stays valid across the memslot move. */ - gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT); + gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_TEST_EXTRA); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n"); virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2); diff --git a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c index b21798a385ca..4e125eb5e0cf 100644 --- a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c @@ -67,7 +67,7 @@ int main(int argc, char *argv[]) MEM_REGION_SIZE / PAGE_SIZE); gpa = vm_phy_pages_alloc(vm, MEM_REGION_SIZE / PAGE_SIZE, - MEM_REGION_GPA, MEM_REGION_SLOT); + MEM_REGION_GPA, MEM_REGION_TEST_EXTRA); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc"); virt_map(vm, MEM_REGION_GVA, MEM_REGION_GPA, 1); hva = addr_gpa2hva(vm, MEM_REGION_GPA); From 1856064afc9db977209d8e9150613e5cb1c3906c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:21 -0700 Subject: [PATCH 032/506] KVM: selftests: Use TEST_ASSERT(), not assert(), in vm_get_mem_region() Use a proper selftests TEST_ASSERT() in vm_get_mem_region() so that bad input is guaranteed to fail the test, and with the standard error message format. Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-19-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/kvm_util.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 92ab2f5001c9..de5ebd85b606 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -200,7 +200,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot); static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, enum kvm_mem_region_type type) { - assert(type < NR_MEM_REGIONS); + TEST_ASSERT(type < NR_MEM_REGIONS, + "Invalid memory region type '%u'", type); return memslot2region(vm, vm->memslots[type]); } From 6c28a0d7cf1254d0916fba5c387b4b56ed49e124 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:22 -0700 Subject: [PATCH 033/506] KVM: selftests: Automatically pick min_gpa for allocations based on region type Automatically choose the minimum GPA for physical page allocations based on the region type instead of sprinkling the logic over various wrappers and tests. All usage falls into three categories: (a) don't care, just use the bare minimum GPA, (b) page tables, use a slightly higher min to keep low memory available, (c) custom memslot, use the base of the memslot. I.e. there isn't a strong need to allow completely custom minimums. Opportunistically move the vGIC LPI stress test's "gpa_base" variable into setup_memslot() as the only reason it was a global variable was so that it could be passed as the minimum GPA for allocations from the extra memslot. Tested-by: Itaru Kitayama Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-20-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/arm64/vgic_lpi_stress.c | 13 +++---- .../testing/selftests/kvm/include/kvm_util.h | 12 +++--- tools/testing/selftests/kvm/lib/kvm_util.c | 38 ++++++++++++++++--- .../testing/selftests/kvm/lib/x86/processor.c | 3 +- .../selftests/kvm/set_memory_region_test.c | 2 +- .../x86/smaller_maxphyaddr_emulation_test.c | 2 +- 6 files changed, 45 insertions(+), 25 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c index a45c0849a47a..45a87ba5a737 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c +++ b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c @@ -23,7 +23,6 @@ #define GIC_LPI_OFFSET 8192 static size_t nr_iterations = 1000; -static gpa_t gpa_base; static struct kvm_vm *vm; static struct kvm_vcpu **vcpus; @@ -143,6 +142,7 @@ static void guest_code(size_t nr_lpis) static void setup_memslot(void) { + gpa_t gpa_base; size_t pages; size_t sz; @@ -191,28 +191,25 @@ static void setup_test_data(void) gpa_t cmdq_base; test_data.device_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, MEM_REGION_TEST_EXTRA); test_data.collection_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, MEM_REGION_TEST_EXTRA); - cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, gpa_base, - MEM_REGION_TEST_EXTRA); + cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, MEM_REGION_TEST_EXTRA); virt_map(vm, cmdq_base, cmdq_base, pages_per_64k); test_data.cmdq_base = cmdq_base; test_data.cmdq_base_va = (void *)cmdq_base; test_data.itt_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_devices, - gpa_base, MEM_REGION_TEST_EXTRA); + MEM_REGION_TEST_EXTRA); test_data.lpi_prop_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, MEM_REGION_TEST_EXTRA); + MEM_REGION_TEST_EXTRA); configure_lpis(); test_data.lpi_pend_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_cpus, - gpa_base, MEM_REGION_TEST_EXTRA); + MEM_REGION_TEST_EXTRA); sync_global_to_guest(vm, test_data); } diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index de5ebd85b606..e77842311919 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -1054,11 +1054,10 @@ const char *exit_reason_str(unsigned int exit_reason); gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot, bool protected, bool naturally_aligned); -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, enum kvm_mem_region_type type, bool protected); static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, - gpa_t min_gpa, enum kvm_mem_region_type type) { /* @@ -1066,20 +1065,19 @@ static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, * protected memory, as the majority of memory for such VMs is * protected, i.e. using shared memory is effectively opt-in. */ - return __vm_phy_pages_alloc(vm, nr_pages, min_gpa, type, + return __vm_phy_pages_alloc(vm, nr_pages, type, vm_arch_has_protected_memory(vm)); } -static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, +static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm, enum kvm_mem_region_type type) { - return vm_phy_pages_alloc(vm, 1, min_gpa, type); + return vm_phy_pages_alloc(vm, 1, type); } static inline gpa_t vm_alloc_page_table_pages(struct kvm_vm *vm, size_t nr_pages) { - return vm_phy_pages_alloc(vm, nr_pages, KVM_GUEST_PAGE_TABLE_MIN_PADDR, - MEM_REGION_PT); + return vm_phy_pages_alloc(vm, nr_pages, MEM_REGION_PT); } static inline gpa_t vm_alloc_page_table(struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index aae881c8b927..ae17642ebf7f 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1473,9 +1473,7 @@ static gva_t ____vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva, u64 pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0); virt_pgd_alloc(vm); - gpa_t gpa = __vm_phy_pages_alloc(vm, pages, - KVM_UTIL_MIN_PFN * vm->page_size, - type, protected); + gpa_t gpa = __vm_phy_pages_alloc(vm, pages, type, protected); /* * Find an unused range of virtual page addresses of at least @@ -2087,11 +2085,39 @@ gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, __builtin_unreachable(); } -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, enum kvm_mem_region_type type, bool protected) { - TEST_ASSERT(type < NR_MEM_REGIONS, - "Invalid memory region type '%u'", type); + struct userspace_mem_region *region = vm_get_mem_region(vm, type); + gpa_t min_gpa; + + TEST_ASSERT(region, "No region for type '%u', memslot '%u'", + type, vm->memslots[type]); + + switch (type) { + case MEM_REGION_CODE: + case MEM_REGION_DATA: + case MEM_REGION_TEST_DATA: + /* + * If the region is backed by the default memslot (id=0), use + * selftests' hardcoded minimum PFN, otherwise use the base of + * the custom memory slot that backs the region. + */ + if (!vm->memslots[type]) + min_gpa = KVM_UTIL_MIN_PFN * vm->page_size; + else + min_gpa = region->region.guest_phys_addr; + break; + case MEM_REGION_PT: + min_gpa = KVM_GUEST_PAGE_TABLE_MIN_PADDR; + break; + case MEM_REGION_TEST_EXTRA: + min_gpa = region->region.guest_phys_addr; + break; + default: + TEST_FAIL("Invalid memory region type '%u'", type); + break; + } return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, vm->memslots[type], protected, false); diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index 39d9ca6ceb1d..479e9a481687 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -1474,8 +1474,7 @@ void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, smram_gpa, SMRAM_MEMSLOT, SMRAM_PAGES); - TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, smram_gpa, - MEM_REGION_TEST_EXTRA) == smram_gpa, + TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, MEM_REGION_TEST_EXTRA) == smram_gpa, "Could not allocate guest physical addresses for SMRAM"); memset(addr_gpa2hva(vm, smram_gpa), 0x0, SMRAM_SIZE); diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index 160bbe3d7203..bfa5ac8ce029 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -124,7 +124,7 @@ static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread, * Allocate and map two pages so that the GPA accessed by guest_code() * stays valid across the memslot move. */ - gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_TEST_EXTRA); + gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_TEST_EXTRA); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n"); virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2); diff --git a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c index 4e125eb5e0cf..8d1822717abd 100644 --- a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c @@ -67,7 +67,7 @@ int main(int argc, char *argv[]) MEM_REGION_SIZE / PAGE_SIZE); gpa = vm_phy_pages_alloc(vm, MEM_REGION_SIZE / PAGE_SIZE, - MEM_REGION_GPA, MEM_REGION_TEST_EXTRA); + MEM_REGION_TEST_EXTRA); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc"); virt_map(vm, MEM_REGION_GVA, MEM_REGION_GPA, 1); hva = addr_gpa2hva(vm, MEM_REGION_GPA); From 35097cefedafcb6e9d1c8a9434844c487092ac07 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Thu, 3 Sep 2026 17:24:07 -0700 Subject: [PATCH 034/506] KVM: nVMX: Make VMPTRLD result in #UD when eVMCS is used VMPTRLD with active eVMCS is already forbidden, however, returning 1 without skipping the instruction or queuing an exception will likely result in L1 getting stuck. Genuine Hyper-V seems to inject #UD under similar circumstances, mimic the behavior in KVM. Reported-by: f734222792@gmail.com Link: https://bugzilla.kernel.org/show_bug.cgi?id=221841 Suggested-by: Sean Christopherson Signed-off-by: Vitaly Kuznetsov Link: https://patch.msgid.link/20260904002423.3154436-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 151873407abd..ad3726fd4fda 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5897,6 +5897,12 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; + /* Forbid normal VMPTRLD if Enlightened version was used */ + if (nested_vmx_is_evmptr12_valid(vmx)) { + kvm_queue_exception(vcpu, UD_VECTOR); + return 1; + } + if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) return r; @@ -5906,10 +5912,6 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) if (vmptr == vmx->nested.vmxon_ptr) return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); - /* Forbid normal VMPTRLD if Enlightened version was used */ - if (nested_vmx_is_evmptr12_valid(vmx)) - return 1; - if (vmx->nested.current_vmptr != vmptr) { struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; struct vmcs_hdr hdr; From 1e59e5a3bdbed560bb2acec3a117f193a646332a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 09:41:23 -0700 Subject: [PATCH 035/506] KVM: selftests: Add arch hook to force page tables to be naturally aligned Add an arch hook to communicate to the core physical page allocator that page table allocations need to be naturally aligned. The hook will be used for upcoming PowerPC support, which has MMU tables that are larger than the guest base page size and need to be naturally aligned. Bury the hook in the physical page allocator to avoid having to plumb a rarely used param into the main APIs. Cc: Ritesh Harjani (IBM) Tested-by: Itaru Kitayama Reviewed-by: Ritesh Harjani (IBM) Tested-by: Itaru Kitayam Link: https://patch.msgid.link/20260902164123.2546762-21-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/kvm_util.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index e77842311919..8fff5b40a592 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -1052,6 +1052,8 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); const char *exit_reason_str(unsigned int exit_reason); +bool kvm_arch_needs_naturally_aligned_page_tables(void); + gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot, bool protected, bool naturally_aligned); gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index ae17642ebf7f..cb2e7a98820a 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -2085,10 +2085,16 @@ gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, __builtin_unreachable(); } +__weak bool kvm_arch_needs_naturally_aligned_page_tables(void) +{ + return false; +} + gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, enum kvm_mem_region_type type, bool protected) { struct userspace_mem_region *region = vm_get_mem_region(vm, type); + bool naturally_aligned = false; gpa_t min_gpa; TEST_ASSERT(region, "No region for type '%u', memslot '%u'", @@ -2110,6 +2116,7 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, break; case MEM_REGION_PT: min_gpa = KVM_GUEST_PAGE_TABLE_MIN_PADDR; + naturally_aligned = kvm_arch_needs_naturally_aligned_page_tables(); break; case MEM_REGION_TEST_EXTRA: min_gpa = region->region.guest_phys_addr; @@ -2120,7 +2127,7 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, } return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, vm->memslots[type], - protected, false); + protected, naturally_aligned); } /* From 5b1acaadcdcbedee1ead455639df06f11ce33e6a Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Thu, 3 Sep 2026 17:24:08 -0700 Subject: [PATCH 036/506] KVM: nVMX: Make VMPTRST return eVMCS GPA when it is used VMPTRST with active eVMCS is currently forbidden, however, returning 1 without skipping the instruction will likely result in L1 getting stuck. While TLFS does not specify the expected behavior, genuine Hyper-V seems to be returning eVMCS GPA. Implement the same behavior in KVM. Reported-by: f734222792@gmail.com Link: https://bugzilla.kernel.org/show_bug.cgi?id=221841 Signed-off-by: Vitaly Kuznetsov Link: https://patch.msgid.link/20260904002423.3154436-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index ad3726fd4fda..3058efa65c82 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5964,7 +5964,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) { unsigned long exit_qual = vmx_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); - gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; + gpa_t current_vmptr; struct x86_exception e; gva_t gva; int r; @@ -5972,8 +5972,16 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; - if (unlikely(nested_vmx_is_evmptr12_valid(to_vmx(vcpu)))) - return 1; + /* + * Hyper-V TLFS does not specify the behavior of VMPTRST when eVMCS is used + * but genuine Hyper-V seems to be returning eVMCS GPA. + */ +#ifdef CONFIG_KVM_HYPERV + if (nested_vmx_is_evmptr12_valid(to_vmx(vcpu))) + current_vmptr = to_vmx(vcpu)->nested.hv_evmcs_vmptr; + else +#endif + current_vmptr = to_vmx(vcpu)->nested.current_vmptr; if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, sizeof(gpa_t), &gva)) From be7875808fcdec9e76eaf611b8fd1f1cb5121939 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:09 -0700 Subject: [PATCH 037/506] KVM: selftests: Don't clobber RFLAGS in happy path of __KVM_ASM_SAFE() Use a straight MOV instead of an XOR to zero R9 in the non-faulting path of __KVM_ASM_SAFE() so as not to clobber RFLAGS. XOR generates smaller code, but optimizing performance by minimizing the code footprint is just about the last thing that KVM selftests care about. The bug is relatively benign in the current code base, e.g. it marginally degrades the coverage in the fastops test, but otherwise does no harm. But the bug will become actively problematic if/when the "safe assembly" macros are used for instructions that communicate results via RFLAGS, e.g. for VMX instructions, which signal failure via RFLAGS.CF and RFLAGS.ZF. Opportunistically use R9D instead of the full R9, as bits 63:32 of GPRs are zeroed on writes to bit 31:0. Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260818165258.2613603-5-vkuznets@redhat.com Cc: Vitaly Kuznetsov Link: https://patch.msgid.link/20260904002423.3154436-4-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/processor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 6e6f70035508..282b8aaff799 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -1318,7 +1318,7 @@ gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages); "lea 1f(%%rip), %%r10\n\t" \ "lea 2f(%%rip), %%r11\n\t" \ fep "1: " insn "\n\t" \ - "xor %%r9, %%r9\n\t" \ + "mov $0, %%r9d\n\t" \ "2:\n\t" \ "mov %%r9b, %[vector]\n\t" \ "mov %%r10, %[error_code]\n\t" From 455919e7694500a6cf8884b65e7460d362c7663d Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Thu, 3 Sep 2026 17:24:10 -0700 Subject: [PATCH 038/506] KVM: selftests: Adapt to the updated VMPTRST behavior when eVMCS is used Previously, VMPTRST was forbidden with eVMCS and selftests were mocking the correct behavior in vmptrst() by returning enlightened vmptr directly. Since KVM's behavior has changed to match genuine Hyper-V, adjust evmcs test accordingly. Signed-off-by: Vitaly Kuznetsov Link: https://patch.msgid.link/20260904002423.3154436-5-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/evmcs.h | 8 -------- tools/testing/selftests/kvm/include/x86/vmx.h | 3 --- tools/testing/selftests/kvm/x86/hyperv_evmcs.c | 5 ++--- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/evmcs.h b/tools/testing/selftests/kvm/include/x86/evmcs.h index 82a8ea6b661f..c5b9d26f6cdb 100644 --- a/tools/testing/selftests/kvm/include/x86/evmcs.h +++ b/tools/testing/selftests/kvm/include/x86/evmcs.h @@ -265,14 +265,6 @@ static inline bool load_evmcs(struct hyperv_test_pages *hv) return true; } -static inline int evmcs_vmptrst(u64 *value) -{ - *value = current_vp_assist->current_nested_vmcs & - ~HV_X64_MSR_VP_ASSIST_PAGE_ENABLE; - - return 0; -} - static inline int evmcs_vmread(u64 encoding, u64 *value) { switch (encoding) { diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 04f5e34dea3a..ac7d48d1387e 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -358,9 +358,6 @@ static inline int vmptrst(u64 *value) u64 tmp; u8 ret; - if (enable_evmcs) - return evmcs_vmptrst(value); - __asm__ __volatile__("vmptrst %[value]; setna %[ret]" : [value]"=m"(tmp), [ret]"=rm"(ret) : : "cc", "memory"); diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 981a3a6d72fe..eecb06e120c2 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -92,15 +92,14 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_SYNC(3); GUEST_ASSERT(load_evmcs(hv_pages)); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); + /* VMPTRST returns -1 until VMLAUNCH with eVMCS ptr set */ + GUEST_ASSERT(vmptrstz() == -1); GUEST_SYNC(4); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); prepare_vmcs(vmx_pages, l2_guest_code); GUEST_SYNC(5); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); current_evmcs->revision_id = -1u; GUEST_ASSERT(vmlaunch()); current_evmcs->revision_id = EVMCS_VERSION; From 0d6e35f6ab7d6c8b9afa6882447793aad58701e4 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Thu, 3 Sep 2026 17:24:11 -0700 Subject: [PATCH 039/506] KVM: selftests: Check VMPTRLD with active eVMCS Check that VMPTRLD when eVMCS is active results in #UD. This matches genuine Hyper-V's behavior. Use KVM_ASM_SAFE framework to handle #UD from VMPTRLD. Unfortunately, the same trick cannot be applied to the existing #UD check on VMLAUNCH as VMLAUNCH clobbers all registers which KVM_ASM_SAFE depends on. Keep VMLAUNCH handling separately. Deliberately use a double-underscores prefix instead of a "safe" postfix for the helper, to communicate that the inner/outer helpers handle more than just exceptions, they also handle VM-Fail conditions. Assisted-by: Claude:claude-sonnet-4-6 Signed-off-by: Vitaly Kuznetsov [sean: use __ instead of _safe] Link: https://patch.msgid.link/20260904002423.3154436-6-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 19 +++++++++++--- .../testing/selftests/kvm/x86/hyperv_evmcs.c | 26 +++++++++++++++---- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index ac7d48d1387e..a161dbee7e04 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -342,9 +342,6 @@ static inline int vmptrld(u64 vmcs_pa) { u8 ret; - if (enable_evmcs) - return -1; - __asm__ __volatile__ ("vmptrld %[pa]; setna %[ret]" : [ret]"=rm"(ret) : [pa]"m"(vmcs_pa) @@ -353,6 +350,22 @@ static inline int vmptrld(u64 vmcs_pa) return ret; } +static inline int __vmptrld(u64 vmcs_pa) +{ + u64 error_code; + u8 vector; + u8 failed; + + asm volatile(KVM_ASM_SAFE("vmptrld %[pa]") + "\n\tsetna %[failed]" + : KVM_ASM_SAFE_OUTPUTS(vector, error_code), + [failed]"=qm"(failed) + : [pa]"m"(vmcs_pa) + : "cc", "memory", KVM_ASM_SAFE_CLOBBERS); + + return vector ? vector : failed ? -EINVAL : 0; +} + static inline int vmptrst(u64 *value) { u64 tmp; diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index eecb06e120c2..9cb823e20a68 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -23,7 +23,11 @@ static int ud_count; static void guest_ud_handler(struct ex_regs *regs) { ud_count++; - regs->rip += 3; /* VMLAUNCH */ + /* + * VMLAUNCH insn can't be easily covered by KVM_ASM_SAFE framework but + * luckily the instruction is always three bytes. + */ + regs->rip += 3; } static void guest_nmi_handler(struct ex_regs *regs) @@ -178,7 +182,15 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(11); - /* Try enlightened vmptrld with an incorrect GPA */ + /* VMPTRLD instruction causes #UD after enlightened VMLAUNCH */ + GUEST_ASSERT(__vmptrld(hv_pages->enlightened_vmcs_gpa) == UD_VECTOR); + + /* + * Try enlightened vmptrld with an incorrect GPA. GUEST_SYNC(12) signals + * the host to enable guest_ud_handler() which cannot be enabled beforehand + * to not override the default fixup handler from KVM_ASM_SAFE(). + */ + GUEST_SYNC(12); evmcs_vmptrld(0xdeadbeef, hv_pages->enlightened_vmcs); GUEST_ASSERT(vmlaunch()); GUEST_ASSERT(ud_count == 1); @@ -252,7 +264,6 @@ int main(int argc, char *argv[]) vcpu_args_set(vcpu, 3, vmx_pages_gva, hv_pages_gva, addr_gva2gpa(vm, hcall_page)); vcpu_set_msr(vcpu, HV_X64_MSR_VP_INDEX, vcpu->id); - vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); pr_info("Running L1 which uses EVMCS to run L2\n"); @@ -282,7 +293,7 @@ int main(int argc, char *argv[]) /* Force immediate L2->L1 exit before resuming */ if (stage == 8) { - pr_info("Injecting NMI into L1 before L2 had a chance to run after restore\n"); + pr_debug("Injecting NMI into L1 before L2 had a chance to run after restore\n"); inject_nmi(vcpu); } @@ -292,9 +303,14 @@ int main(int argc, char *argv[]) * KVM_STATE_NESTED_EVMCS is not lost. */ if (stage == 9) { - pr_info("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n"); + pr_debug("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n"); vcpu = save_restore_vm(vm, vcpu); } + + if (stage == 12) { + pr_debug("Trying enlightened VMLAUNCH with an invalid PTR\n"); + vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); + } } done: From f1d4e357a69559e47e7f34fe8dc9d156d0de7588 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:12 -0700 Subject: [PATCH 040/506] KVM: selftests: Assert success in vmptrst(), kill off vmptrstz() Assert success in vmptrst() instead of punting to the caller, as literally every user expects VMPTRST to succeed (and because '0' is a valid physical address). Link: https://patch.msgid.link/20260904002423.3154436-7-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 19 ++++--------------- .../testing/selftests/kvm/x86/hyperv_evmcs.c | 4 ++-- tools/testing/selftests/kvm/x86/state_test.c | 8 ++++---- .../kvm/x86/vmx_preemption_timer_test.c | 2 +- 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index a161dbee7e04..04f638fcd3ba 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -366,27 +366,16 @@ static inline int __vmptrld(u64 vmcs_pa) return vector ? vector : failed ? -EINVAL : 0; } -static inline int vmptrst(u64 *value) +static inline u64 vmptrst(void) { - u64 tmp; + u64 value = 0; u8 ret; __asm__ __volatile__("vmptrst %[value]; setna %[ret]" - : [value]"=m"(tmp), [ret]"=rm"(ret) + : [value]"=m"(value), [ret]"=rm"(ret) : : "cc", "memory"); - *value = tmp; - return ret; -} - -/* - * A wrapper around vmptrst that ignores errors and returns zero if the - * vmptrst instruction fails. - */ -static inline u64 vmptrstz(void) -{ - u64 value = 0; - vmptrst(&value); + __GUEST_ASSERT(!ret, "vmptrst failed"); return value; } diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 9cb823e20a68..7923229c2dfc 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -97,7 +97,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, GUEST_SYNC(3); GUEST_ASSERT(load_evmcs(hv_pages)); /* VMPTRST returns -1 until VMLAUNCH with eVMCS ptr set */ - GUEST_ASSERT(vmptrstz() == -1); + GUEST_ASSERT(vmptrst() == -1); GUEST_SYNC(4); @@ -123,7 +123,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, GUEST_ASSERT(!vmlaunch()); GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); + GUEST_ASSERT(vmptrst() == hv_pages->enlightened_vmcs_gpa); /* * NMI forces L2->L1 exit, resuming L2 and hope that EVMCS is diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index e6f521a6e737..354ff3cda597 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -78,17 +78,17 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_SYNC(3); GUEST_ASSERT(load_vmcs(vmx_pages)); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); GUEST_SYNC(4); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); prepare_vmcs(vmx_pages, vmx_l2_guest_code); GUEST_SYNC(5); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* Check that the launched state is preserved. */ diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index 72fbf1b70e9f..3f77afeaad1b 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -73,7 +73,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(vmx_pages->vmcs_gpa); GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); prepare_vmcs(vmx_pages, l2_guest_code); From 154b098cada9d673058c3911cb1a3864de1ae05b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:13 -0700 Subject: [PATCH 041/506] KVM: selftests: Always assert that vmxon() and prepare_for_vmx_operation() succeed Assert that vmxon() succeeds, and by extension prepare_for_vmx_operation(). Forcing every caller to assert that a core preparation helper succeeds is absurd and generates hard-to-read code. For all intents and purposes, no functional change intended. Link: https://patch.msgid.link/20260904002423.3154436-8-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 6 +++--- tools/testing/selftests/kvm/lib/x86/memstress.c | 2 +- tools/testing/selftests/kvm/lib/x86/vmx.c | 7 ++----- tools/testing/selftests/kvm/x86/aperfmperf_test.c | 2 +- tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c | 2 +- tools/testing/selftests/kvm/x86/hyperv_evmcs.c | 2 +- tools/testing/selftests/kvm/x86/kvm_buslock_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_close_kvm_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_dirty_log_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_emulation_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_exceptions_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c | 2 +- .../selftests/kvm/x86/save_restore_pf_stress_test.c | 2 +- tools/testing/selftests/kvm/x86/smm_test.c | 2 +- tools/testing/selftests/kvm/x86/state_test.c | 2 +- tools/testing/selftests/kvm/x86/triple_fault_event_test.c | 2 +- tools/testing/selftests/kvm/x86/vmx_apic_access_test.c | 2 +- tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c | 2 +- .../selftests/kvm/x86/vmx_invalid_nested_guest_state.c | 2 +- .../testing/selftests/kvm/x86/vmx_nested_la57_state_test.c | 2 +- .../testing/selftests/kvm/x86/vmx_preemption_timer_test.c | 2 +- 24 files changed, 27 insertions(+), 30 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 04f638fcd3ba..f7ca6646dfbe 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -309,7 +309,7 @@ struct vmx_msr_entry { #include "evmcs.h" -static inline int vmxon(u64 phys) +static inline void vmxon(u64 phys) { u8 ret; @@ -318,7 +318,7 @@ static inline int vmxon(u64 phys) : [pa]"m"(phys) : "cc", "memory"); - return ret; + __GUEST_ASSERT(!ret, "vmxon [0x%lx] failed", phys); } static inline void vmxoff(void) @@ -549,7 +549,7 @@ union vmx_ctrl_msr { }; struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva); -bool prepare_for_vmx_operation(struct vmx_pages *vmx); +void prepare_for_vmx_operation(struct vmx_pages *vmx); void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip); bool load_vmcs(struct vmx_pages *vmx); diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c index e19e8b5a09c5..2fa39470c1b3 100644 --- a/tools/testing/selftests/kvm/lib/x86/memstress.c +++ b/tools/testing/selftests/kvm/lib/x86/memstress.c @@ -33,7 +33,7 @@ __asm__( static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id) { GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); + prepare_for_vmx_operation(vmx); GUEST_ASSERT(load_vmcs(vmx)); GUEST_ASSERT(ept_1g_pages_supported()); diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 089e1a8af53f..03d61b1159f8 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -125,7 +125,7 @@ vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva) return vmx; } -bool prepare_for_vmx_operation(struct vmx_pages *vmx) +void prepare_for_vmx_operation(struct vmx_pages *vmx) { u64 feature_control; u64 required; @@ -163,10 +163,7 @@ bool prepare_for_vmx_operation(struct vmx_pages *vmx) /* Enter VMX root operation. */ *(u32 *)(vmx->vmxon) = vmcs_revision(); - if (vmxon(vmx->vmxon_gpa)) - return false; - - return true; + vmxon(vmx->vmxon_gpa); } bool load_vmcs(struct vmx_pages *vmx) diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c index 845cb685f174..e32131cd6ed1 100644 --- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c +++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c @@ -70,7 +70,7 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + prepare_for_vmx_operation(vmx); GUEST_ASSERT_EQ(load_vmcs(vmx), true); prepare_vmcs(vmx, NULL); diff --git a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c index aa7f3b405fd3..cafb58a37023 100644 --- a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c +++ b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c @@ -55,7 +55,7 @@ static void guest_code(struct vmx_pages *vmx_pages, enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist); evmcs_enable(); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_evmcs(hv_pages)); prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 7923229c2dfc..849d66c8470d 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -93,7 +93,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist); evmcs_enable(); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_SYNC(3); GUEST_ASSERT(load_evmcs(hv_pages)); /* VMPTRST returns -1 until VMLAUNCH with eVMCS ptr set */ diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c index 25a182be00a9..2e006ce95896 100644 --- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c +++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c @@ -42,7 +42,7 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + prepare_for_vmx_operation(vmx); GUEST_ASSERT_EQ(load_vmcs(vmx), true); prepare_vmcs(vmx, NULL); diff --git a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c index b974cfb347d6..479e9adcc20b 100644 --- a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c +++ b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c @@ -30,7 +30,7 @@ static void l2_guest_code(void) static void l1_vmx_code(struct vmx_pages *vmx_pages) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); /* Prepare the VMCS for L2 execution. */ diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c index 26b474bf1353..57f49471f4aa 100644 --- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c @@ -93,7 +93,7 @@ void l1_vmx_code(struct vmx_pages *vmx) void *l2_rip; GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); + prepare_for_vmx_operation(vmx); GUEST_ASSERT(load_vmcs(vmx)); if (vmx->eptp_gpa) diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c index e08c6b0697e5..6f27930bb6b5 100644 --- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c @@ -66,7 +66,7 @@ static void guest_code(void *test_data) BIT_ULL(INTERCEPT_HLT); vmcb->control.intercept_exceptions = 0; } else { - GUEST_ASSERT(prepare_for_vmx_operation(test_data)); + prepare_for_vmx_operation(test_data); GUEST_ASSERT(load_vmcs(test_data)); prepare_vmcs(test_data, NULL); diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index aeec3121c8e8..6a36cf4d1eab 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c @@ -125,7 +125,7 @@ static void vmx_run_l2(void *l2_code, int vector, u32 error_code) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + prepare_for_vmx_operation(vmx); GUEST_ASSERT_EQ(load_vmcs(vmx), true); diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c index 8c2ba9674558..749a93c318fe 100644 --- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c +++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c @@ -40,7 +40,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) { uintptr_t save_cr3; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c index 2e04563790ff..b143ebe710e7 100644 --- a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c @@ -55,7 +55,7 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, u64 exit_qual; GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); + prepare_for_vmx_operation(vmx); GUEST_ASSERT(load_vmcs(vmx)); prepare_vmcs(vmx, l2_entry); diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c index cb79d7b9619c..3d2eae848ba6 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c @@ -86,7 +86,7 @@ static void l1_guest_code(void *data) struct vmx_pages *vmx_pages = data; u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 18f765835bf4..584071146b69 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -105,7 +105,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) /* check that L1's frequency looks alright before launching L2 */ check_tsc_freq(UCHECK_L1); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); /* prepare the VMCS for L2 execution */ diff --git a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c index 507391ab2c93..6ab73a0bd34a 100644 --- a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c +++ b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c @@ -98,7 +98,7 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); + prepare_for_vmx_operation(vmx); GUEST_ASSERT(load_vmcs(vmx)); prepare_vmcs(vmx, guest_access_memory); diff --git a/tools/testing/selftests/kvm/x86/smm_test.c b/tools/testing/selftests/kvm/x86/smm_test.c index d1edafd5af75..be5662792ec8 100644 --- a/tools/testing/selftests/kvm/x86/smm_test.c +++ b/tools/testing/selftests/kvm/x86/smm_test.c @@ -78,7 +78,7 @@ static void guest_code(void *arg) if (this_cpu_has(X86_FEATURE_SVM)) { generic_svm_setup(svm, l2_guest_code); } else { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); prepare_vmcs(vmx_pages, l2_guest_code); } diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 354ff3cda597..d29a3e113460 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -75,7 +75,7 @@ void vmx_l2_guest_code(void) static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) { GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_SYNC(3); GUEST_ASSERT(load_vmcs(vmx_pages)); GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c index 0d83516f4bd0..57e79922b16b 100644 --- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c +++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c @@ -25,7 +25,7 @@ void l1_guest_code_vmx(struct vmx_pages *vmx) { GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); + prepare_for_vmx_operation(vmx); GUEST_ASSERT(load_vmcs(vmx)); prepare_vmcs(vmx, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c index 463f73aa9159..7df9ab833cb2 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c @@ -38,7 +38,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa) { u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); /* Prepare the VMCS for L2 execution. */ diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c index f9b88a6f6113..8e4dc304533a 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c @@ -33,7 +33,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) { u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); /* Prepare the VMCS for L2 execution. */ diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c index c8379124b317..8999f0458aa1 100644 --- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c @@ -41,7 +41,7 @@ static void l2_guest_code(void) static void l1_guest_code(struct vmx_pages *vmx_pages) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); /* Prepare the VMCS for L2 execution. */ diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c index 448721b54181..ab9f38daef17 100644 --- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c @@ -37,7 +37,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(rdmsr(MSR_GS_BASE) == LA57_GS_BASE); GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index 3f77afeaad1b..9d2b2ce11a65 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -71,7 +71,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) u64 l1_tsc_deadline, l2_tsc_deadline; GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_ASSERT(load_vmcs(vmx_pages)); GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); From 5f73777a27b16f5778a9da12fc82f60d9e54a0f9 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:14 -0700 Subject: [PATCH 042/506] KVM: selftests: Always assert that vmclear() succeeds Continue removing dead error "handling", and assert that VMCLEAR succeeds instead of forcing callers of load_vmcs() to assert success. For all intents and purposes, no functional change intended. Link: https://patch.msgid.link/20260904002423.3154436-9-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 4 ++-- tools/testing/selftests/kvm/lib/x86/vmx.c | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index f7ca6646dfbe..cf37a03a93b9 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -326,7 +326,7 @@ static inline void vmxoff(void) __asm__ __volatile__("vmxoff"); } -static inline int vmclear(u64 vmcs_pa) +static inline void vmclear(u64 vmcs_pa) { u8 ret; @@ -335,7 +335,7 @@ static inline int vmclear(u64 vmcs_pa) : [pa]"m"(vmcs_pa) : "cc", "memory"); - return ret; + __GUEST_ASSERT(!ret, "vmclear [0x%lx] failed\n", vmcs_pa); } static inline int vmptrld(u64 vmcs_pa) diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 03d61b1159f8..57a8287de0d2 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -170,16 +170,14 @@ bool load_vmcs(struct vmx_pages *vmx) { /* Load a VMCS. */ *(u32 *)(vmx->vmcs) = vmcs_revision(); - if (vmclear(vmx->vmcs_gpa)) - return false; + vmclear(vmx->vmcs_gpa); if (vmptrld(vmx->vmcs_gpa)) return false; /* Setup shadow VMCS, do not load it yet. */ *(u32 *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul; - if (vmclear(vmx->shadow_vmcs_gpa)) - return false; + vmclear(vmx->shadow_vmcs_gpa); return true; } From 9007b746aeb5073d03c722990080e4a8ec691478 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:15 -0700 Subject: [PATCH 043/506] KVM: selftests: Always assert that vmptrld() succeeds Assert that vmptrld() succeeds instead of forcing all callers to do the same, and opportunistically provide a macro to print (hopefully) useful information on VMX instruction failures. Leave load_vmcs() as-is for the moment to minimize the amount of churn since the vmptrld() changes aren't completely trivial. Link: https://patch.msgid.link/20260904002423.3154436-10-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 23 +++++++++---------- tools/testing/selftests/kvm/lib/x86/vmx.c | 3 +-- tools/testing/selftests/kvm/x86/state_test.c | 6 ++--- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index cf37a03a93b9..8f004b946826 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -309,6 +309,10 @@ struct vmx_msr_entry { #include "evmcs.h" +#define GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, __r, __pa) \ + __GUEST_ASSERT(!__r, __stringify(insn) "[0x%lx] hit %s", \ + __pa, __r < 0 ? "VM-Fail" : ex_str(__r)) + static inline void vmxon(u64 phys) { u8 ret; @@ -338,18 +342,6 @@ static inline void vmclear(u64 vmcs_pa) __GUEST_ASSERT(!ret, "vmclear [0x%lx] failed\n", vmcs_pa); } -static inline int vmptrld(u64 vmcs_pa) -{ - u8 ret; - - __asm__ __volatile__ ("vmptrld %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(vmcs_pa) - : "cc", "memory"); - - return ret; -} - static inline int __vmptrld(u64 vmcs_pa) { u64 error_code; @@ -366,6 +358,13 @@ static inline int __vmptrld(u64 vmcs_pa) return vector ? vector : failed ? -EINVAL : 0; } +static inline void vmptrld(u64 vmcs_pa) +{ + int ret = __vmptrld(vmcs_pa); + + GUEST_ASSERT_VMX_INSN_SUCCEEDED(vmptrld, ret, vmcs_pa); +} + static inline u64 vmptrst(void) { u64 value = 0; diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 57a8287de0d2..e65024b9ea0d 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -172,8 +172,7 @@ bool load_vmcs(struct vmx_pages *vmx) *(u32 *)(vmx->vmcs) = vmcs_revision(); vmclear(vmx->vmcs_gpa); - if (vmptrld(vmx->vmcs_gpa)) - return false; + vmptrld(vmx->vmcs_gpa); /* Setup shadow VMCS, do not load it yet. */ *(u32 *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul; diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index d29a3e113460..2d5fc603929b 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -108,7 +108,7 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) vmwrite(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); vmwrite(VMCS_LINK_POINTER, vmx_pages->shadow_vmcs_gpa); - GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa)); + vmptrld(vmx_pages->shadow_vmcs_gpa); GUEST_ASSERT(vmlaunch()); GUEST_SYNC(8); GUEST_ASSERT(vmlaunch()); @@ -118,11 +118,11 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) GUEST_SYNC(9); GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); - GUEST_ASSERT(!vmptrld(vmx_pages->vmcs_gpa)); + vmptrld(vmx_pages->vmcs_gpa); GUEST_ASSERT(!vmresume()); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa)); + vmptrld(vmx_pages->shadow_vmcs_gpa); GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); GUEST_ASSERT(vmlaunch()); GUEST_ASSERT(vmresume()); From f2bad303f4033e9cb516ce456c98cd8b2495c24b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:16 -0700 Subject: [PATCH 044/506] KVM: selftests: Drop useless return code from load_vmcs() Now that load_vmcs() either asserts or returns success, drop the return code and the many asserts in its callers. Link: https://patch.msgid.link/20260904002423.3154436-11-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 2 +- tools/testing/selftests/kvm/lib/x86/memstress.c | 2 +- tools/testing/selftests/kvm/lib/x86/vmx.c | 4 +--- tools/testing/selftests/kvm/x86/aperfmperf_test.c | 2 +- tools/testing/selftests/kvm/x86/kvm_buslock_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_close_kvm_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_dirty_log_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_emulation_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_exceptions_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c | 2 +- tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c | 2 +- tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c | 2 +- tools/testing/selftests/kvm/x86/smm_test.c | 2 +- tools/testing/selftests/kvm/x86/state_test.c | 2 +- tools/testing/selftests/kvm/x86/triple_fault_event_test.c | 2 +- tools/testing/selftests/kvm/x86/vmx_apic_access_test.c | 2 +- tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c | 2 +- .../selftests/kvm/x86/vmx_invalid_nested_guest_state.c | 2 +- tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c | 2 +- tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c | 2 +- 22 files changed, 22 insertions(+), 24 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 8f004b946826..5c46ae95c4c3 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -550,7 +550,7 @@ union vmx_ctrl_msr { struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva); void prepare_for_vmx_operation(struct vmx_pages *vmx); void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip); -bool load_vmcs(struct vmx_pages *vmx); +void load_vmcs(struct vmx_pages *vmx); bool ept_1g_pages_supported(void); diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c index 2fa39470c1b3..207350ab6114 100644 --- a/tools/testing/selftests/kvm/lib/x86/memstress.c +++ b/tools/testing/selftests/kvm/lib/x86/memstress.c @@ -34,7 +34,7 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id) { GUEST_ASSERT(vmx->vmcs_gpa); prepare_for_vmx_operation(vmx); - GUEST_ASSERT(load_vmcs(vmx)); + load_vmcs(vmx); GUEST_ASSERT(ept_1g_pages_supported()); *(u64 *)vmx->stack = vcpu_id; diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index e65024b9ea0d..5c6cadc08405 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -166,7 +166,7 @@ void prepare_for_vmx_operation(struct vmx_pages *vmx) vmxon(vmx->vmxon_gpa); } -bool load_vmcs(struct vmx_pages *vmx) +void load_vmcs(struct vmx_pages *vmx) { /* Load a VMCS. */ *(u32 *)(vmx->vmcs) = vmcs_revision(); @@ -177,8 +177,6 @@ bool load_vmcs(struct vmx_pages *vmx) /* Setup shadow VMCS, do not load it yet. */ *(u32 *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul; vmclear(vmx->shadow_vmcs_gpa); - - return true; } static bool ept_vpid_cap_supported(u64 mask) diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c index e32131cd6ed1..11f5894d8ac7 100644 --- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c +++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c @@ -71,7 +71,7 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { prepare_for_vmx_operation(vmx); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c index 2e006ce95896..22993ec697fa 100644 --- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c +++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c @@ -43,7 +43,7 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { prepare_for_vmx_operation(vmx); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); diff --git a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c index 479e9adcc20b..309cab009929 100644 --- a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c +++ b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c @@ -31,7 +31,7 @@ static void l2_guest_code(void) static void l1_vmx_code(struct vmx_pages *vmx_pages) { prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c index 57f49471f4aa..dca2bfc8b158 100644 --- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c @@ -94,7 +94,7 @@ void l1_vmx_code(struct vmx_pages *vmx) GUEST_ASSERT(vmx->vmcs_gpa); prepare_for_vmx_operation(vmx); - GUEST_ASSERT(load_vmcs(vmx)); + load_vmcs(vmx); if (vmx->eptp_gpa) l2_rip = l2_guest_code_tdp_enabled; diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c index 6f27930bb6b5..2b38eff4f516 100644 --- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c @@ -67,7 +67,7 @@ static void guest_code(void *test_data) vmcb->control.intercept_exceptions = 0; } else { prepare_for_vmx_operation(test_data); - GUEST_ASSERT(load_vmcs(test_data)); + load_vmcs(test_data); prepare_vmcs(test_data, NULL); GUEST_ASSERT(!vmwrite(GUEST_IDTR_LIMIT, 0)); diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index 6a36cf4d1eab..50c271a03692 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c @@ -127,7 +127,7 @@ static void l1_vmx_code(struct vmx_pages *vmx) { prepare_for_vmx_operation(vmx); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); GUEST_ASSERT_EQ(vmwrite(GUEST_IDTR_LIMIT, 0), 0); diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c index 749a93c318fe..43deb3b358ea 100644 --- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c +++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c @@ -41,7 +41,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) uintptr_t save_cr3; prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c index b143ebe710e7..52df467e9c44 100644 --- a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c @@ -56,7 +56,7 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, GUEST_ASSERT(vmx->vmcs_gpa); prepare_for_vmx_operation(vmx); - GUEST_ASSERT(load_vmcs(vmx)); + load_vmcs(vmx); prepare_vmcs(vmx, l2_entry); diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c index 3d2eae848ba6..05194762ed33 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c @@ -87,7 +87,7 @@ static void l1_guest_code(void *data) u32 control; prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 584071146b69..b9fae268a9ec 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -106,7 +106,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) check_tsc_freq(UCHECK_L1); prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); /* prepare the VMCS for L2 execution */ prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c index 6ab73a0bd34a..c0e4d6e156ad 100644 --- a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c +++ b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c @@ -99,7 +99,7 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { prepare_for_vmx_operation(vmx); - GUEST_ASSERT(load_vmcs(vmx)); + load_vmcs(vmx); prepare_vmcs(vmx, guest_access_memory); GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR))); diff --git a/tools/testing/selftests/kvm/x86/smm_test.c b/tools/testing/selftests/kvm/x86/smm_test.c index be5662792ec8..377df09e582a 100644 --- a/tools/testing/selftests/kvm/x86/smm_test.c +++ b/tools/testing/selftests/kvm/x86/smm_test.c @@ -79,7 +79,7 @@ static void guest_code(void *arg) generic_svm_setup(svm, l2_guest_code); } else { prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); } diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 2d5fc603929b..6880db3442e5 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -77,7 +77,7 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(vmx_pages->vmcs_gpa); prepare_for_vmx_operation(vmx_pages); GUEST_SYNC(3); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); GUEST_SYNC(4); diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c index 57e79922b16b..a9fb7046ea4f 100644 --- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c +++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c @@ -26,7 +26,7 @@ void l1_guest_code_vmx(struct vmx_pages *vmx) GUEST_ASSERT(vmx->vmcs_gpa); prepare_for_vmx_operation(vmx); - GUEST_ASSERT(load_vmcs(vmx)); + load_vmcs(vmx); prepare_vmcs(vmx, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c index 7df9ab833cb2..1793f7b02219 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c @@ -39,7 +39,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa) u32 control; prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c index 8e4dc304533a..f7ebc9b15629 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c @@ -34,7 +34,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) u32 control; prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c index 8999f0458aa1..4a6445958cf4 100644 --- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c @@ -42,7 +42,7 @@ static void l2_guest_code(void) static void l1_guest_code(struct vmx_pages *vmx_pages) { prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c index ab9f38daef17..608bc0ed27db 100644 --- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c @@ -38,7 +38,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(vmx_pages->vmcs_gpa); prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index 9d2b2ce11a65..abbe5aedd35f 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -72,7 +72,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(vmx_pages->vmcs_gpa); prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_vmcs(vmx_pages)); + load_vmcs(vmx_pages); GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); prepare_vmcs(vmx_pages, l2_guest_code); From b2f44e713c6d1c27bac17e376490e6ed9c334191 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:17 -0700 Subject: [PATCH 045/506] KVM: selftests: Add macros to handle simple VMX instructions Add macros to build helpers for VMXON, VMPTRLD, and VMCLEAR, i.e. the VMX instructions that take a VMCS address as input, and don't have outputs. Getting the params right is annoying, especially when using KVM_ASM_SAFE(). As a bonus, this adds non-asserting double-underscores versions of VMXON and VMCLEAR for "free". Link: https://patch.msgid.link/20260904002423.3154436-12-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 71 +++++++------------ 1 file changed, 26 insertions(+), 45 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 5c46ae95c4c3..3def8df902a2 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -313,58 +313,39 @@ struct vmx_msr_entry { __GUEST_ASSERT(!__r, __stringify(insn) "[0x%lx] hit %s", \ __pa, __r < 0 ? "VM-Fail" : ex_str(__r)) -static inline void vmxon(u64 phys) -{ - u8 ret; - - __asm__ __volatile__ ("vmxon %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(phys) - : "cc", "memory"); - - __GUEST_ASSERT(!ret, "vmxon [0x%lx] failed", phys); +#define BUILD_VMCS_ASM_HELPERS(insn) \ +static inline int __##insn(u64 vmcs_pa) \ +{ \ + u64 error_code; \ + u8 vector; \ + u8 failed; \ + \ + asm volatile(KVM_ASM_SAFE(__stringify(insn) " %[pa]") \ + "\n\tsetna %[failed]" \ + : KVM_ASM_SAFE_OUTPUTS(vector, error_code), \ + [failed]"=qm"(failed) \ + : [pa]"m"(vmcs_pa) \ + : "cc", "memory", KVM_ASM_SAFE_CLOBBERS); \ + \ + return vector ? vector : failed ? -EINVAL : 0; \ +} \ + \ +static inline void insn(u64 vmcs_pa) \ +{ \ + int ret = __##insn(vmcs_pa); \ + \ + GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, ret, vmcs_pa); \ } +BUILD_VMCS_ASM_HELPERS(vmxon) +BUILD_VMCS_ASM_HELPERS(vmptrld) +BUILD_VMCS_ASM_HELPERS(vmclear) + static inline void vmxoff(void) { __asm__ __volatile__("vmxoff"); } -static inline void vmclear(u64 vmcs_pa) -{ - u8 ret; - - __asm__ __volatile__ ("vmclear %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(vmcs_pa) - : "cc", "memory"); - - __GUEST_ASSERT(!ret, "vmclear [0x%lx] failed\n", vmcs_pa); -} - -static inline int __vmptrld(u64 vmcs_pa) -{ - u64 error_code; - u8 vector; - u8 failed; - - asm volatile(KVM_ASM_SAFE("vmptrld %[pa]") - "\n\tsetna %[failed]" - : KVM_ASM_SAFE_OUTPUTS(vector, error_code), - [failed]"=qm"(failed) - : [pa]"m"(vmcs_pa) - : "cc", "memory", KVM_ASM_SAFE_CLOBBERS); - - return vector ? vector : failed ? -EINVAL : 0; -} - -static inline void vmptrld(u64 vmcs_pa) -{ - int ret = __vmptrld(vmcs_pa); - - GUEST_ASSERT_VMX_INSN_SUCCEEDED(vmptrld, ret, vmcs_pa); -} - static inline u64 vmptrst(void) { u64 value = 0; From 7763344704509dfe89a9394550ac85ba6f7f76f3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:18 -0700 Subject: [PATCH 046/506] KVM: selftests: Drop dead return code from evmcs_vmptrld() and load_evmcs() Drop the return codes from evmcs_vmptrld() and load_evmcs() as they are hardcoded to return '0' and "true" respectively. No functional change intended. Link: https://patch.msgid.link/20260904002423.3154436-13-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/evmcs.h | 12 +++--------- .../selftests/kvm/x86/evmcs_smm_controls_test.c | 2 +- tools/testing/selftests/kvm/x86/hyperv_evmcs.c | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/evmcs.h b/tools/testing/selftests/kvm/include/x86/evmcs.h index c5b9d26f6cdb..f18ebfb10ee0 100644 --- a/tools/testing/selftests/kvm/include/x86/evmcs.h +++ b/tools/testing/selftests/kvm/include/x86/evmcs.h @@ -245,24 +245,18 @@ static inline void evmcs_enable(void) enable_evmcs = true; } -static inline int evmcs_vmptrld(u64 vmcs_pa, void *vmcs) +static inline void evmcs_vmptrld(u64 vmcs_pa, void *vmcs) { current_vp_assist->current_nested_vmcs = vmcs_pa; current_vp_assist->enlighten_vmentry = 1; current_evmcs = vmcs; - - return 0; } -static inline bool load_evmcs(struct hyperv_test_pages *hv) +static inline void load_evmcs(struct hyperv_test_pages *hv) { - if (evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs)) - return false; - + evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs); current_evmcs->revision_id = EVMCS_VERSION; - - return true; } static inline int evmcs_vmread(u64 encoding, u64 *value) diff --git a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c index cafb58a37023..5c0391f8a38d 100644 --- a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c +++ b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c @@ -56,7 +56,7 @@ static void guest_code(struct vmx_pages *vmx_pages, evmcs_enable(); prepare_for_vmx_operation(vmx_pages); - GUEST_ASSERT(load_evmcs(hv_pages)); + load_evmcs(hv_pages); prepare_vmcs(vmx_pages, l2_guest_code); GUEST_ASSERT(!vmlaunch()); diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 849d66c8470d..4c0e76d64c83 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -95,7 +95,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, prepare_for_vmx_operation(vmx_pages); GUEST_SYNC(3); - GUEST_ASSERT(load_evmcs(hv_pages)); + load_evmcs(hv_pages); /* VMPTRST returns -1 until VMLAUNCH with eVMCS ptr set */ GUEST_ASSERT(vmptrst() == -1); From 978c4df4b2458ec5a26bc6fe8c8ac57bfaf6c9db Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:19 -0700 Subject: [PATCH 047/506] KVM: selftests: Dedup assembly code for VMLAUNCH and VMRESUME Dedup the assembly code for VMLAUNCH vs. VMRESUME, for both architectural VMCS and eVMCS, as the difference is literally only the actual VM-Enter instruction for VMLAUNCH vs. VMRESUME, and VMWRITE vs. MOV for VMCS vs. eVMCS. Opportunistically place the implementations in vmx.c so that the lengthy sequence (~580 bytes of code) aren't inlined for every single VM-Enter. 000000000041b910 <__vmlaunch>: 41b910: 80 3d 61 5e 10 00 00 cmpb $0x0,0x105e61(%rip) # 521778 41b917: 0f 85 13 01 00 00 jne 41ba30 <__vmlaunch+0x120> 41b91d: ba 14 6c 00 00 mov $0x6c14,%edx 41b922: b9 16 6c 00 00 mov $0x6c16,%ecx 41b927: 6a 00 push $0x0 41b929: 0f 79 d4 vmwrite %rsp,%rdx 41b92c: 48 8d 05 82 00 00 00 lea 0x82(%rip),%rax # 41b9b5 <__vmlaunch+0xa5> 41b933: 0f 79 c8 vmwrite %rax,%rcx 41b9ae: 0f 01 c2 vmlaunch 41b9b1: 48 ff 04 24 incq (%rsp) 41ba2d: 58 pop %rax 41ba2e: c3 ret 41ba2f: 90 nop 41ba30: 48 8b 35 39 5d 10 00 mov 0x105d39(%rip),%rsi # 521770 41ba37: c7 86 38 03 00 00 00 00 00 00 movl $0x0,0x338(%rsi) 41ba41: 6a 00 push $0x0 41ba43: 48 89 a6 68 02 00 00 mov %rsp,0x268(%rsi) 41ba4a: 48 8d 05 83 00 00 00 lea 0x83(%rip),%rax # 41bad4 <__vmlaunch+0x1c4> 41ba51: 48 89 46 50 mov %rax,0x50(%rsi) 41bacd: 0f 01 c2 vmlaunch 41bad0: 48 ff 04 24 incq (%rsp) 41bb4c: 58 pop %rax 41bb4d: c3 ret No functional change intended (though the code generation will be quite different). Cc: Yosry Ahmed Link: https://patch.msgid.link/20260904002423.3154436-14-seanjc@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/x86/evmcs.h | 58 ------------------- tools/testing/selftests/kvm/include/x86/vmx.h | 47 ++------------- tools/testing/selftests/kvm/lib/x86/vmx.c | 55 ++++++++++++++++++ 3 files changed, 60 insertions(+), 100 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/evmcs.h b/tools/testing/selftests/kvm/include/x86/evmcs.h index f18ebfb10ee0..491fb8107473 100644 --- a/tools/testing/selftests/kvm/include/x86/evmcs.h +++ b/tools/testing/selftests/kvm/include/x86/evmcs.h @@ -1187,62 +1187,4 @@ static inline int evmcs_vmwrite(u64 encoding, u64 value) return 0; } -static inline int evmcs_vmlaunch(void) -{ - int ret; - - current_evmcs->hv_clean_fields = 0; - - __asm__ __volatile__("push $0;" - "mov %%rsp, (%[host_rsp]);" - "lea 1f(%%rip), %%rax;" - "mov %%rax, (%[host_rip]);" - VMX_SWITCH_GPRS_ASM - "vmlaunch;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r" - ((u64)¤t_evmcs->host_rsp), - [host_rip]"r" - ((u64)¤t_evmcs->host_rip), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; -} - -/* - * No guest state (e.g. GPRs) is established by this vmresume. - */ -static inline int evmcs_vmresume(void) -{ - int ret; - - /* HOST_RIP */ - current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1; - /* HOST_RSP */ - current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER; - - __asm__ __volatile__("push $0;" - "mov %%rsp, (%[host_rsp]);" - "lea 1f(%%rip), %%rax;" - "mov %%rax, (%[host_rip]);" - VMX_SWITCH_GPRS_ASM - "vmresume;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r" - ((u64)¤t_evmcs->host_rsp), - [host_rip]"r" - ((u64)¤t_evmcs->host_rip), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; -} - #endif /* !SELFTEST_KVM_EVMCS_H */ diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 3def8df902a2..d0ac8e128d98 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -359,54 +359,17 @@ static inline u64 vmptrst(void) return value; } +int __vmlaunch(void); +int __vmresume(void); + static inline int vmlaunch(void) { - int ret; - - if (enable_evmcs) - return evmcs_vmlaunch(); - - __asm__ __volatile__("push $0;" - "vmwrite %%rsp, %[host_rsp];" - "lea 1f(%%rip), %%rax;" - "vmwrite %%rax, %[host_rip];" - VMX_SWITCH_GPRS_ASM - "vmlaunch;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r"((u64)HOST_RSP), - [host_rip]"r"((u64)HOST_RIP), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; + return __vmlaunch(); } static inline int vmresume(void) { - int ret; - - if (enable_evmcs) - return evmcs_vmresume(); - - __asm__ __volatile__("push $0;" - "vmwrite %%rsp, %[host_rsp];" - "lea 1f(%%rip), %%rax;" - "vmwrite %%rax, %[host_rip];" - VMX_SWITCH_GPRS_ASM - "vmresume;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r"((u64)HOST_RSP), - [host_rip]"r"((u64)HOST_RIP), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; + return __vmresume(); } static inline void vmcall(void) diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 5c6cadc08405..1a8515de42b0 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -179,6 +179,61 @@ void load_vmcs(struct vmx_pages *vmx) vmclear(vmx->shadow_vmcs_gpa); } +#define __BUILD_VMX_VM_ENTRY_HELPER(insn, prefix, vmwrite_insn, vmwrite_operand, \ + __host_rsp, __host_rip) \ +static int __##prefix##_##insn(void) \ +{ \ + int ret; \ + \ + __asm__ __volatile__("push $0;" \ + __stringify(vmwrite_insn) " %%rsp, %[host_rsp];" \ + "lea 1f(%%rip), %%rax;" \ + __stringify(vmwrite_insn) " %%rax, %[host_rip];" \ + VMX_SWITCH_GPRS_ASM \ + __stringify(insn)";" \ + "incq (%%rsp);" \ + "1: ;" \ + VMX_SWITCH_GPRS_ASM \ + "pop %%rax;" \ + : [ret]"=&a"(ret) \ + : [host_rsp]__stringify(vmwrite_operand)(__host_rsp), \ + [host_rip]__stringify(vmwrite_operand)(__host_rip), \ + GUEST_REGS_OFFSETS \ + : "memory", "cc"); \ + return ret; \ +} + +#define BUILD_VMX_VM_ENTRY_HELPER(insn) \ + __BUILD_VMX_VM_ENTRY_HELPER(insn, _, vmwrite, r, (u64)HOST_RSP, (u64)HOST_RIP) \ + __BUILD_VMX_VM_ENTRY_HELPER(insn, __evmcs, mov, m, \ + current_evmcs->host_rsp, current_evmcs->host_rip) + +BUILD_VMX_VM_ENTRY_HELPER(vmlaunch) +BUILD_VMX_VM_ENTRY_HELPER(vmresume) + +int __vmlaunch(void) +{ + if (enable_evmcs) { + current_evmcs->hv_clean_fields = 0; + return ____evmcs_vmlaunch(); + } + + return ____vmlaunch(); +} + +int __vmresume(void) +{ + if (enable_evmcs) { + /* HOST_RIP */ + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1; + /* HOST_RSP */ + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER; + return ____evmcs_vmresume(); + } + + return ____vmresume(); +} + static bool ept_vpid_cap_supported(u64 mask) { return rdmsr(MSR_IA32_VMX_EPT_VPID_CAP) & mask; From 5adf7087cfd280970beeb2b8b46393787003ea51 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:20 -0700 Subject: [PATCH 048/506] KVM: selftests: Assert success in vmlaunch() and vmresume() Assert success in the "outer" APIs for VMLAUNCH and VMRESUME to dedup a pile of code, and switch to the double-underscores versions as necessary, to make it more obvious which paths expect VM-Enter *failure*. Link: https://patch.msgid.link/20260904002423.3154436-15-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 8 +++---- .../testing/selftests/kvm/lib/x86/memstress.c | 2 +- .../selftests/kvm/x86/aperfmperf_test.c | 2 +- .../kvm/x86/evmcs_smm_controls_test.c | 2 +- .../testing/selftests/kvm/x86/hyperv_evmcs.c | 22 ++++++++--------- .../selftests/kvm/x86/kvm_buslock_test.c | 2 +- .../selftests/kvm/x86/nested_close_kvm_test.c | 2 +- .../selftests/kvm/x86/nested_dirty_log_test.c | 2 +- .../selftests/kvm/x86/nested_emulation_test.c | 5 +++- .../kvm/x86/nested_exceptions_test.c | 5 +++- .../kvm/x86/nested_invalid_cr3_test.c | 4 ++-- .../selftests/kvm/x86/nested_tdp_fault_test.c | 2 +- .../kvm/x86/nested_tsc_adjust_test.c | 2 +- .../kvm/x86/nested_tsc_scaling_test.c | 2 +- .../kvm/x86/save_restore_pf_stress_test.c | 4 ++-- tools/testing/selftests/kvm/x86/state_test.c | 24 +++++++++---------- .../kvm/x86/triple_fault_event_test.c | 2 +- .../selftests/kvm/x86/vmx_apic_access_test.c | 4 ++-- .../kvm/x86/vmx_apicv_updates_test.c | 4 ++-- .../kvm/x86/vmx_invalid_nested_guest_state.c | 2 +- .../kvm/x86/vmx_nested_la57_state_test.c | 2 +- .../kvm/x86/vmx_preemption_timer_test.c | 4 ++-- 22 files changed, 57 insertions(+), 51 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index d0ac8e128d98..1419043b41dd 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -362,14 +362,14 @@ static inline u64 vmptrst(void) int __vmlaunch(void); int __vmresume(void); -static inline int vmlaunch(void) +static inline void vmlaunch(void) { - return __vmlaunch(); + __GUEST_ASSERT(!__vmlaunch(), "vmlaunch hit VM-Fail"); } -static inline int vmresume(void) +static inline void vmresume(void) { - return __vmresume(); + __GUEST_ASSERT(!__vmresume(), "vmresume hit VM-Fail"); } static inline void vmcall(void) diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c index 207350ab6114..6a1815fb97e9 100644 --- a/tools/testing/selftests/kvm/lib/x86/memstress.c +++ b/tools/testing/selftests/kvm/lib/x86/memstress.c @@ -40,7 +40,7 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id) *(u64 *)vmx->stack = vcpu_id; prepare_vmcs(vmx, memstress_l2_guest_entry); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c index 11f5894d8ac7..a082658c1e48 100644 --- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c +++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c @@ -83,7 +83,7 @@ static void l1_vmx_code(struct vmx_pages *vmx) vmreadz(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); } static void guest_code(void *nested_test_data) diff --git a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c index 5c0391f8a38d..4a65eb02349c 100644 --- a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c +++ b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c @@ -59,7 +59,7 @@ static void guest_code(struct vmx_pages *vmx_pages, load_evmcs(hv_pages); prepare_vmcs(vmx_pages, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 exits via vmcall if test fails */ sync_with_host(2); diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 4c0e76d64c83..cbacb114db1b 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -105,7 +105,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, GUEST_SYNC(5); current_evmcs->revision_id = -1u; - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); current_evmcs->revision_id = EVMCS_VERSION; GUEST_SYNC(6); @@ -120,7 +120,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, current_vp_assist->nested_control.features.directhypercall = 1; *(u32 *)(hv_pages->partition_assist) = 0; - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR); GUEST_ASSERT(vmptrst() == hv_pages->enlightened_vmcs_gpa); @@ -130,7 +130,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, * up-to-date (RIP points where it should and not at the beginning * of l2_guest_code(). GUEST_SYNC(9) checks that. */ - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_SYNC(10); @@ -141,13 +141,13 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); __set_bit(MSR_FS_BASE & 0x1fff, vmx_pages->msr + 0x400); - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Enable enlightened MSR bitmap */ current_evmcs->hv_enlightenments_control.msr_bitmap = 1; - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ @@ -155,14 +155,14 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, __set_bit(MSR_GS_BASE & 0x1fff, vmx_pages->msr + 0x400); /* Make sure HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP is set */ current_evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; - GUEST_ASSERT(!vmresume()); + vmresume(); /* Make sure we don't see EXIT_REASON_MSR_READ here so eMSR bitmap works */ GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); current_evmcs->guest_rip += 3; /* vmcall */ /* Now tell KVM we've changed MSR-Bitmap */ current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ @@ -170,15 +170,15 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, * L2 TLB flush test. First VMCALL should be handled directly by L0, * no VMCALL exit expected. */ - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Enable synthetic vmexit */ *(u32 *)(hv_pages->partition_assist) = 1; - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH); - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(11); @@ -192,7 +192,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, */ GUEST_SYNC(12); evmcs_vmptrld(0xdeadbeef, hv_pages->enlightened_vmcs); - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); GUEST_ASSERT(ud_count == 1); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c index 22993ec697fa..337d3c28bc6d 100644 --- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c +++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c @@ -48,7 +48,7 @@ static void l1_vmx_code(struct vmx_pages *vmx) prepare_vmcs(vmx, NULL); GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); } static void guest_code(void *test_data) diff --git a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c index 309cab009929..1f4a5e44a6ef 100644 --- a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c +++ b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c @@ -36,7 +36,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(0); } diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c index dca2bfc8b158..7987ce50c382 100644 --- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c @@ -104,7 +104,7 @@ void l1_vmx_code(struct vmx_pages *vmx) prepare_vmcs(vmx, l2_rip); GUEST_SYNC(TEST_SYNC_NO_FAULT); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_SYNC(TEST_SYNC_NO_FAULT); GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); GUEST_DONE(); diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c index 2b38eff4f516..51e2f7e386c1 100644 --- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c @@ -102,7 +102,10 @@ static void guest_code(void *test_data) exit_insn_len = vmcb->control.next_rip - vmcb->save.rip; GUEST_ASSERT_EQ(vmcb->save.rip, (u64)l2_instruction); } else { - GUEST_ASSERT_EQ(i ? vmresume() : vmlaunch(), 0); + if (!i) + vmlaunch(); + else + vmresume(); exit_reason = vmreadz(VM_EXIT_REASON); exit_insn_len = vmreadz(VM_EXIT_INSTRUCTION_LEN); GUEST_ASSERT_EQ(vmreadz(GUEST_RIP), (u64)l2_instruction); diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index 50c271a03692..031eb84229dc 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c @@ -112,7 +112,10 @@ static void vmx_run_l2(void *l2_code, int vector, u32 error_code) { GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_code)); - GUEST_ASSERT_EQ(vector == SS_VECTOR ? vmlaunch() : vmresume(), 0); + if (vector == SS_VECTOR) + vmlaunch(); + else + vmresume(); if (vector == FAKE_TRIPLE_FAULT_VECTOR) return; diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c index 43deb3b358ea..101c1fe79eeb 100644 --- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c +++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c @@ -48,13 +48,13 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) /* Try to run L2 with invalid CR3 and make sure it fails */ save_cr3 = vmreadz(GUEST_CR3); vmwrite(GUEST_CR3, -1ull); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == (EXIT_REASON_FAILED_VMENTRY | EXIT_REASON_INVALID_STATE)); /* Now restore CR3 and make sure L2 runs successfully */ vmwrite(GUEST_CR3, save_cr3); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_DONE(); diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c index 52df467e9c44..d6c443e3d147 100644 --- a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c @@ -60,7 +60,7 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, prepare_vmcs(vmx, l2_entry); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* Verify we got an EPT violation exit */ __GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION, diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c index 05194762ed33..c60dea899624 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c @@ -95,7 +95,7 @@ static void l1_guest_code(void *data) vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); vmwrite(TSC_OFFSET, TSC_OFFSET_VALUE); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); } else { struct svm_test_data *svm = data; diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index b9fae268a9ec..b2fc7f0b5b3c 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -125,7 +125,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) vmwrite(TSC_MULTIPLIER_HIGH, TSC_MULTIPLIER_L2 >> 32); /* launch L2 */ - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* check that L1's frequency still looks good */ diff --git a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c index c0e4d6e156ad..f88427e925fc 100644 --- a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c +++ b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c @@ -104,11 +104,11 @@ static void l1_vmx_code(struct vmx_pages *vmx) GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR))); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); while (1) { GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR); - GUEST_ASSERT(!vmresume()); + vmresume(); } } diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 6880db3442e5..4dadae350dca 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -87,20 +87,20 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) GUEST_SYNC(5); GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* Check that the launched state is preserved. */ - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(7); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + 3); @@ -109,27 +109,27 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) vmwrite(VMCS_LINK_POINTER, vmx_pages->shadow_vmcs_gpa); vmptrld(vmx_pages->shadow_vmcs_gpa); - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); GUEST_SYNC(8); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); vmwrite(GUEST_RIP, 0xc0ffee); GUEST_SYNC(9); GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); vmptrld(vmx_pages->vmcs_gpa); - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmptrld(vmx_pages->shadow_vmcs_gpa); GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); GUEST_SYNC(13); GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); } static void __attribute__((__flatten__)) guest_code(void *arg) diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c index a9fb7046ea4f..bb31a1ba8932 100644 --- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c +++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c @@ -30,7 +30,7 @@ void l1_guest_code_vmx(struct vmx_pages *vmx) prepare_vmcs(vmx, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 should triple fault after a triple fault event injected. */ GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c index 1793f7b02219..fe29c869a235 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c @@ -53,14 +53,14 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa) /* Try to launch L2 with the memory-backed APIC-access address. */ GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR)); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmwrite(APIC_ACCESS_ADDR, high_gpa); /* Try to resume L2 with the unbacked APIC-access address. */ GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR)); - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_DONE(); diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c index f7ebc9b15629..e578221da247 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c @@ -62,7 +62,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * Run L2 to switch to x2APIC mode, which in turn will uninhibit APICv, * as KVM should force the APIC ID back to its default. */ - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); GUEST_ASSERT(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD); @@ -90,7 +90,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * handles the x2APIC => xAPIC transition and inhibits APICv while L2 * is active. */ - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_ASSERT(!(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD)); diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c index 4a6445958cf4..3017938fa993 100644 --- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c @@ -55,7 +55,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(!(vmreadz(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) || !(vmreadz(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST)); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 should triple fault after main() stuffs invalid guest state. */ GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c index 608bc0ed27db..46e5ca997e74 100644 --- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c @@ -56,7 +56,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) guest_cr4 &= ~X86_CR4_LA57; vmwrite(GUEST_CR4, guest_cr4); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); exit_reason = vmreadz(VM_EXIT_REASON); GUEST_ASSERT(exit_reason == EXIT_REASON_VMCALL); diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index abbe5aedd35f..97c84849f83e 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -90,7 +90,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) !(ctrl_exit_rev.clr & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) return; - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); @@ -110,7 +110,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) l1_vmx_pt_start = (rdtsc() >> vmx_pt_rate) << vmx_pt_rate; - GUEST_ASSERT(!vmresume()); + vmresume(); l1_vmx_pt_finish = rdtsc(); From 68b94d08f9dcff3954c4f5baa7f2c5faa828b625 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:21 -0700 Subject: [PATCH 049/506] KVM: selftests: Add __vmwrite() and use it when initializing optional fields. Add __vmwrite() in anticipation of having vmwrite() assert success, and use the "failure allowed" version in the common VMCS setup code, which is the only code that wants to write fields that may or may not exist (although arguably the code should check if the field exists instead of blindly writing). E.g. When doing initial configuration of a VMCS, the core library doesn't want to assume secondary controls are supported, and so blindly does a VMWRITE and pivots on success/failure. All usage of vmwrite() outside of common setup already asserts success, or 100% relies on VMWRITE succeeding in order for the test to function correctly. No functional change intended. Link: https://patch.msgid.link/20260904002423.3154436-16-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 7 ++++- tools/testing/selftests/kvm/lib/x86/vmx.c | 28 +++++++++---------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 1419043b41dd..40111db580a8 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -414,7 +414,7 @@ static inline u64 vmreadz(u64 encoding) return value; } -static inline int vmwrite(u64 encoding, u64 value) +static __always_inline int __vmwrite(u64 encoding, u64 value) { u8 ret; @@ -429,6 +429,11 @@ static inline int vmwrite(u64 encoding, u64 value) return ret; } +static inline int vmwrite(u64 encoding, u64 value) +{ + return __vmwrite(encoding, value); +} + static inline u32 vmcs_revision(void) { return rdmsr(MSR_IA32_VMX_BASIC); diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 1a8515de42b0..14ccfbd9c32d 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -251,8 +251,8 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) { u32 sec_exec_ctl = 0; - vmwrite(VIRTUAL_PROCESSOR_ID, 0); - vmwrite(POSTED_INTR_NV, 0); + __vmwrite(VIRTUAL_PROCESSOR_ID, 0); + __vmwrite(POSTED_INTR_NV, 0); vmwrite(PIN_BASED_VM_EXEC_CONTROL, rdmsr(MSR_IA32_VMX_TRUE_PINBASED_CTLS)); @@ -269,7 +269,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) sec_exec_ctl |= SECONDARY_EXEC_ENABLE_EPT; } - if (!vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_exec_ctl)) + if (!__vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_exec_ctl)) vmwrite(CPU_BASED_VM_EXEC_CONTROL, rdmsr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); else { @@ -289,16 +289,16 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) VM_ENTRY_IA32E_MODE); /* 64-bit guest */ vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0); vmwrite(VM_ENTRY_INTR_INFO_FIELD, 0); - vmwrite(TPR_THRESHOLD, 0); + __vmwrite(TPR_THRESHOLD, 0); vmwrite(CR0_GUEST_HOST_MASK, 0); vmwrite(CR4_GUEST_HOST_MASK, 0); vmwrite(CR0_READ_SHADOW, get_cr0()); vmwrite(CR4_READ_SHADOW, get_cr4()); - vmwrite(MSR_BITMAP, vmx->msr_gpa); - vmwrite(VMREAD_BITMAP, vmx->vmread_gpa); - vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa); + __vmwrite(MSR_BITMAP, vmx->msr_gpa); + __vmwrite(VMREAD_BITMAP, vmx->vmread_gpa); + __vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa); } /* @@ -356,15 +356,15 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp) vmwrite(GUEST_GS_SELECTOR, vmreadz(HOST_GS_SELECTOR)); vmwrite(GUEST_LDTR_SELECTOR, 0); vmwrite(GUEST_TR_SELECTOR, vmreadz(HOST_TR_SELECTOR)); - vmwrite(GUEST_INTR_STATUS, 0); - vmwrite(GUEST_PML_INDEX, 0); + __vmwrite(GUEST_INTR_STATUS, 0); + __vmwrite(GUEST_PML_INDEX, 0); vmwrite(VMCS_LINK_POINTER, -1ll); vmwrite(GUEST_IA32_DEBUGCTL, 0); - vmwrite(GUEST_IA32_PAT, vmreadz(HOST_IA32_PAT)); - vmwrite(GUEST_IA32_EFER, vmreadz(HOST_IA32_EFER)); - vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL, - vmreadz(HOST_IA32_PERF_GLOBAL_CTRL)); + __vmwrite(GUEST_IA32_PAT, vmreadz(HOST_IA32_PAT)); + __vmwrite(GUEST_IA32_EFER, vmreadz(HOST_IA32_EFER)); + __vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL, + vmreadz(HOST_IA32_PERF_GLOBAL_CTRL)); vmwrite(GUEST_ES_LIMIT, -1); vmwrite(GUEST_CS_LIMIT, -1); @@ -391,7 +391,7 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp) vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0); vmwrite(GUEST_ACTIVITY_STATE, 0); vmwrite(GUEST_SYSENTER_CS, vmreadz(HOST_IA32_SYSENTER_CS)); - vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0); + __vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0); vmwrite(GUEST_CR0, vmreadz(HOST_CR0)); vmwrite(GUEST_CR3, vmreadz(HOST_CR3)); From 6ecd351628f3493060fdd02d8044712ba7af094a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:22 -0700 Subject: [PATCH 050/506] KVM: selftests: Always assert that vmwrite() succeeds Assert that vmwrite() succeeds instead of forcing all callers to do the same (or as is much more common, ignoring VM-Fail and likely letting the test fail in weird ways), opportunistically provide a more verbose error message on failure. All usage of vmwrite() "guarantees" success, either because the field in question is unconditionally emulated by KVM, by virtue of earlier support checks, or by falling over (later in the test) if the VMWRITE fails. Cc: Yosry Ahmed Link: https://patch.msgid.link/20260904002423.3154436-17-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 5 +++-- tools/testing/selftests/kvm/x86/aperfmperf_test.c | 2 +- tools/testing/selftests/kvm/x86/kvm_buslock_test.c | 2 +- .../testing/selftests/kvm/x86/nested_emulation_test.c | 6 +++--- .../testing/selftests/kvm/x86/nested_exceptions_test.c | 10 +++++----- .../selftests/kvm/x86/save_restore_pf_stress_test.c | 2 +- tools/testing/selftests/kvm/x86/state_test.c | 4 ++-- .../selftests/kvm/x86/vmx_preemption_timer_test.c | 8 +++----- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 40111db580a8..be10028c51b9 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -429,9 +429,10 @@ static __always_inline int __vmwrite(u64 encoding, u64 value) return ret; } -static inline int vmwrite(u64 encoding, u64 value) +static inline void vmwrite(u64 encoding, u64 value) { - return __vmwrite(encoding, value); + __GUEST_ASSERT(!__vmwrite(encoding, value), + "vmwrite[0x%lx] = 0x%lx hit VM-Fail", encoding, value); } static inline u32 vmcs_revision(void) diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c index a082658c1e48..7761a8bdf5f6 100644 --- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c +++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c @@ -82,7 +82,7 @@ static void l1_vmx_code(struct vmx_pages *vmx) vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); + vmwrite(GUEST_RIP, (u64)l2_guest_code); vmlaunch(); } diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c index 337d3c28bc6d..e1d2730b16ba 100644 --- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c +++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c @@ -47,7 +47,7 @@ static void l1_vmx_code(struct vmx_pages *vmx) prepare_vmcs(vmx, NULL); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); + vmwrite(GUEST_RIP, (u64)l2_guest_code); vmlaunch(); } diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c index 51e2f7e386c1..009932b1604e 100644 --- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c @@ -70,9 +70,9 @@ static void guest_code(void *test_data) load_vmcs(test_data); prepare_vmcs(test_data, NULL); - GUEST_ASSERT(!vmwrite(GUEST_IDTR_LIMIT, 0)); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, 0)); + vmwrite(GUEST_IDTR_LIMIT, 0); + vmwrite(GUEST_RIP, (u64)l2_guest_code); + vmwrite(EXCEPTION_BITMAP, 0); vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_PAUSE_EXITING | diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index 031eb84229dc..d0beaa7b280b 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c @@ -110,7 +110,7 @@ static void l1_svm_code(struct svm_test_data *svm) static void vmx_run_l2(void *l2_code, int vector, u32 error_code) { - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_code)); + vmwrite(GUEST_RIP, (u64)l2_code); if (vector == SS_VECTOR) vmlaunch(); @@ -133,21 +133,21 @@ static void l1_vmx_code(struct vmx_pages *vmx) load_vmcs(vmx); prepare_vmcs(vmx, NULL); - GUEST_ASSERT_EQ(vmwrite(GUEST_IDTR_LIMIT, 0), 0); + vmwrite(GUEST_IDTR_LIMIT, 0); /* * VMX disallows injecting an exception with error_code[31:16] != 0, * and hardware will never generate a VM-Exit with bits 31:16 set. * KVM should likewise truncate the "bad" userspace value. */ - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF); vmx_run_l2(l2_ss_pending_test, SS_VECTOR, (u16)SS_ERROR_CODE); vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, GP_ERROR_CODE_INTEL); - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF); vmx_run_l2(l2_ss_injected_df_test, DF_VECTOR, DF_ERROR_CODE); - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS); vmx_run_l2(l2_ss_injected_tf_test, FAKE_TRIPLE_FAULT_VECTOR, 0); GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT); diff --git a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c index f88427e925fc..e728f040916f 100644 --- a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c +++ b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c @@ -102,7 +102,7 @@ static void l1_vmx_code(struct vmx_pages *vmx) load_vmcs(vmx); prepare_vmcs(vmx, guest_access_memory); - GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR))); + vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR)); vmlaunch(); while (1) { diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 4dadae350dca..8de6eb183670 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -62,10 +62,10 @@ void vmx_l2_guest_code(void) GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); GUEST_SYNC(10); GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); - GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0fffee)); + vmwrite(GUEST_RIP, 0xc0fffee); GUEST_SYNC(11); GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0fffee); - GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0ffffee)); + vmwrite(GUEST_RIP, 0xc0ffffee); GUEST_SYNC(12); /* Done, exit to L1 and never come back. */ diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index 97c84849f83e..06e74c72fe65 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -97,12 +97,10 @@ void l1_guest_code(struct vmx_pages *vmx_pages) /* * Turn on PIN control and resume the guest */ - GUEST_ASSERT(!vmwrite(PIN_BASED_VM_EXEC_CONTROL, - vmreadz(PIN_BASED_VM_EXEC_CONTROL) | - PIN_BASED_VMX_PREEMPTION_TIMER)); + vmwrite(PIN_BASED_VM_EXEC_CONTROL, + vmreadz(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_VMX_PREEMPTION_TIMER); - GUEST_ASSERT(!vmwrite(VMX_PREEMPTION_TIMER_VALUE, - PREEMPTION_TIMER_VALUE)); + vmwrite(VMX_PREEMPTION_TIMER_VALUE, PREEMPTION_TIMER_VALUE); vmx_pt_rate = rdmsr(MSR_IA32_VMX_MISC) & 0x1F; From 4e824a04d4fad3f2754d0089fb150c043daab732 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 17:24:23 -0700 Subject: [PATCH 051/506] KVM: selftests: Always assert that vmreadz() succeeds Rename vmreadz() => vmread() (and vmread() => __vmread()) and have it assert success instead of returning garbage on failure. While the eVMCS path avoids clobbering *value on failure, the VMREAD path unconditionally stores tmp to *value, without ever zeroing tmp. I.e. if VMREAD did fail, *value would be loaded with random stack data, not '0'. An audit of the code confirms that nothing is actually relying on vmreadz() to zero the result on failure, as all paths either check for a non-zero value, propagate the result into vmwrite(), or guard the vmread() with a pre-check that the field exists. Opportunistically fix the bug in what is now __vmread(), so that the code isn't consuming uninitialized stack data. Begrudgingly keep zeroing value in vmread(), as gcc-15 at least isn't smart enough to figure out that value won't be used uninitialized (though to be fair, strictly speaking it isn't actually unreachable code). Link: https://patch.msgid.link/20260904002423.3154436-18-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 15 +++--- .../testing/selftests/kvm/lib/x86/memstress.c | 2 +- tools/testing/selftests/kvm/lib/x86/vmx.c | 52 +++++++++---------- .../selftests/kvm/x86/aperfmperf_test.c | 2 +- .../testing/selftests/kvm/x86/hyperv_evmcs.c | 24 ++++----- .../selftests/kvm/x86/nested_dirty_log_test.c | 2 +- .../selftests/kvm/x86/nested_emulation_test.c | 8 +-- .../kvm/x86/nested_exceptions_test.c | 10 ++-- .../kvm/x86/nested_invalid_cr3_test.c | 6 +-- .../selftests/kvm/x86/nested_tdp_fault_test.c | 10 ++-- .../kvm/x86/nested_tsc_adjust_test.c | 4 +- .../kvm/x86/nested_tsc_scaling_test.c | 6 +-- .../kvm/x86/save_restore_pf_stress_test.c | 4 +- tools/testing/selftests/kvm/x86/state_test.c | 24 ++++----- .../kvm/x86/triple_fault_event_test.c | 2 +- .../selftests/kvm/x86/vmx_apic_access_test.c | 12 ++--- .../kvm/x86/vmx_apicv_updates_test.c | 8 +-- .../kvm/x86/vmx_invalid_nested_guest_state.c | 6 +-- .../kvm/x86/vmx_nested_la57_state_test.c | 4 +- .../kvm/x86/vmx_preemption_timer_test.c | 8 +-- 20 files changed, 104 insertions(+), 105 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index be10028c51b9..ca67a618b68f 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -386,7 +386,7 @@ static inline void vmcall(void) "r10", "r11", "r12", "r13", "r14", "r15"); } -static inline int vmread(u64 encoding, u64 *value) +static inline int __vmread(u64 encoding, u64 *value) { u64 tmp; u8 ret; @@ -399,18 +399,17 @@ static inline int vmread(u64 encoding, u64 *value) : [encoding]"r"(encoding) : "cc", "memory"); - *value = tmp; + if (!ret) + *value = tmp; return ret; } -/* - * A wrapper around vmread that ignores errors and returns zero if the - * vmread instruction fails. - */ -static inline u64 vmreadz(u64 encoding) +static inline u64 vmread(u64 encoding) { u64 value = 0; - vmread(encoding, &value); + + __GUEST_ASSERT(!__vmread(encoding, &value), + "vmread[0x%lx] hit VM-Fail", encoding); return value; } diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c index 6a1815fb97e9..9bfea36144a7 100644 --- a/tools/testing/selftests/kvm/lib/x86/memstress.c +++ b/tools/testing/selftests/kvm/lib/x86/memstress.c @@ -41,7 +41,7 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id) prepare_vmcs(vmx, memstress_l2_guest_entry); vmlaunch(); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_VMCALL); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 14ccfbd9c32d..6a53c8ce3b3c 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -308,7 +308,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) */ static inline void init_vmcs_host_state(void) { - u32 exit_controls = vmreadz(VM_EXIT_CONTROLS); + u32 exit_controls = vmread(VM_EXIT_CONTROLS); vmwrite(HOST_ES_SELECTOR, get_es()); vmwrite(HOST_CS_SELECTOR, get_cs()); @@ -348,23 +348,23 @@ static inline void init_vmcs_host_state(void) */ static inline void init_vmcs_guest_state(void *rip, void *rsp) { - vmwrite(GUEST_ES_SELECTOR, vmreadz(HOST_ES_SELECTOR)); - vmwrite(GUEST_CS_SELECTOR, vmreadz(HOST_CS_SELECTOR)); - vmwrite(GUEST_SS_SELECTOR, vmreadz(HOST_SS_SELECTOR)); - vmwrite(GUEST_DS_SELECTOR, vmreadz(HOST_DS_SELECTOR)); - vmwrite(GUEST_FS_SELECTOR, vmreadz(HOST_FS_SELECTOR)); - vmwrite(GUEST_GS_SELECTOR, vmreadz(HOST_GS_SELECTOR)); + vmwrite(GUEST_ES_SELECTOR, vmread(HOST_ES_SELECTOR)); + vmwrite(GUEST_CS_SELECTOR, vmread(HOST_CS_SELECTOR)); + vmwrite(GUEST_SS_SELECTOR, vmread(HOST_SS_SELECTOR)); + vmwrite(GUEST_DS_SELECTOR, vmread(HOST_DS_SELECTOR)); + vmwrite(GUEST_FS_SELECTOR, vmread(HOST_FS_SELECTOR)); + vmwrite(GUEST_GS_SELECTOR, vmread(HOST_GS_SELECTOR)); vmwrite(GUEST_LDTR_SELECTOR, 0); - vmwrite(GUEST_TR_SELECTOR, vmreadz(HOST_TR_SELECTOR)); + vmwrite(GUEST_TR_SELECTOR, vmread(HOST_TR_SELECTOR)); __vmwrite(GUEST_INTR_STATUS, 0); __vmwrite(GUEST_PML_INDEX, 0); vmwrite(VMCS_LINK_POINTER, -1ll); vmwrite(GUEST_IA32_DEBUGCTL, 0); - __vmwrite(GUEST_IA32_PAT, vmreadz(HOST_IA32_PAT)); - __vmwrite(GUEST_IA32_EFER, vmreadz(HOST_IA32_EFER)); + __vmwrite(GUEST_IA32_PAT, vmread(HOST_IA32_PAT)); + __vmwrite(GUEST_IA32_EFER, vmread(HOST_IA32_EFER)); __vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL, - vmreadz(HOST_IA32_PERF_GLOBAL_CTRL)); + vmread(HOST_IA32_PERF_GLOBAL_CTRL)); vmwrite(GUEST_ES_LIMIT, -1); vmwrite(GUEST_CS_LIMIT, -1); @@ -377,42 +377,42 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp) vmwrite(GUEST_GDTR_LIMIT, 0xffff); vmwrite(GUEST_IDTR_LIMIT, 0xffff); vmwrite(GUEST_ES_AR_BYTES, - vmreadz(GUEST_ES_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_ES_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_CS_AR_BYTES, 0xa09b); vmwrite(GUEST_SS_AR_BYTES, 0xc093); vmwrite(GUEST_DS_AR_BYTES, - vmreadz(GUEST_DS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_DS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_FS_AR_BYTES, - vmreadz(GUEST_FS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_FS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_GS_AR_BYTES, - vmreadz(GUEST_GS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_GS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_LDTR_AR_BYTES, 0x10000); vmwrite(GUEST_TR_AR_BYTES, 0x8b); vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0); vmwrite(GUEST_ACTIVITY_STATE, 0); - vmwrite(GUEST_SYSENTER_CS, vmreadz(HOST_IA32_SYSENTER_CS)); + vmwrite(GUEST_SYSENTER_CS, vmread(HOST_IA32_SYSENTER_CS)); __vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0); - vmwrite(GUEST_CR0, vmreadz(HOST_CR0)); - vmwrite(GUEST_CR3, vmreadz(HOST_CR3)); - vmwrite(GUEST_CR4, vmreadz(HOST_CR4)); + vmwrite(GUEST_CR0, vmread(HOST_CR0)); + vmwrite(GUEST_CR3, vmread(HOST_CR3)); + vmwrite(GUEST_CR4, vmread(HOST_CR4)); vmwrite(GUEST_ES_BASE, 0); vmwrite(GUEST_CS_BASE, 0); vmwrite(GUEST_SS_BASE, 0); vmwrite(GUEST_DS_BASE, 0); - vmwrite(GUEST_FS_BASE, vmreadz(HOST_FS_BASE)); - vmwrite(GUEST_GS_BASE, vmreadz(HOST_GS_BASE)); + vmwrite(GUEST_FS_BASE, vmread(HOST_FS_BASE)); + vmwrite(GUEST_GS_BASE, vmread(HOST_GS_BASE)); vmwrite(GUEST_LDTR_BASE, 0); - vmwrite(GUEST_TR_BASE, vmreadz(HOST_TR_BASE)); - vmwrite(GUEST_GDTR_BASE, vmreadz(HOST_GDTR_BASE)); - vmwrite(GUEST_IDTR_BASE, vmreadz(HOST_IDTR_BASE)); + vmwrite(GUEST_TR_BASE, vmread(HOST_TR_BASE)); + vmwrite(GUEST_GDTR_BASE, vmread(HOST_GDTR_BASE)); + vmwrite(GUEST_IDTR_BASE, vmread(HOST_IDTR_BASE)); vmwrite(GUEST_DR7, 0x400); vmwrite(GUEST_RSP, (u64)rsp); vmwrite(GUEST_RIP, (u64)rip); vmwrite(GUEST_RFLAGS, X86_EFLAGS_FIXED); vmwrite(GUEST_PENDING_DBG_EXCEPTIONS, 0); - vmwrite(GUEST_SYSENTER_ESP, vmreadz(HOST_IA32_SYSENTER_ESP)); - vmwrite(GUEST_SYSENTER_EIP, vmreadz(HOST_IA32_SYSENTER_EIP)); + vmwrite(GUEST_SYSENTER_ESP, vmread(HOST_IA32_SYSENTER_ESP)); + vmwrite(GUEST_SYSENTER_EIP, vmread(HOST_IA32_SYSENTER_EIP)); } void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip) diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c index 7761a8bdf5f6..1ca6495a446f 100644 --- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c +++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c @@ -80,7 +80,7 @@ static void l1_vmx_code(struct vmx_pages *vmx) * in the VMCS by prepare_vmcs()), as MSR exiting mandatory on Intel. */ vmwrite(CPU_BASED_VM_EXEC_CONTROL, - vmreadz(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); + vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); vmwrite(GUEST_RIP, (u64)l2_guest_code); vmlaunch(); diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index cbacb114db1b..c81f2dd4f2a1 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -109,7 +109,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, current_evmcs->revision_id = EVMCS_VERSION; GUEST_SYNC(6); - vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmreadz(PIN_BASED_VM_EXEC_CONTROL) | + vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmread(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_NMI_EXITING); /* L2 TLB flush setup */ @@ -121,8 +121,8 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, *(u32 *)(hv_pages->partition_assist) = 0; vmlaunch(); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ((vmread(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR); GUEST_ASSERT(vmptrst() == hv_pages->enlightened_vmcs_gpa); /* @@ -134,21 +134,21 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, GUEST_SYNC(10); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); current_evmcs->guest_rip += 3; /* vmcall */ /* Intercept RDMSR 0xc0000100 */ - vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | + vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); __set_bit(MSR_FS_BASE & 0x1fff, vmx_pages->msr + 0x400); vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Enable enlightened MSR bitmap */ current_evmcs->hv_enlightenments_control.msr_bitmap = 1; vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Intercept RDMSR 0xc0000101 without telling KVM about it */ @@ -157,13 +157,13 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, current_evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; vmresume(); /* Make sure we don't see EXIT_REASON_MSR_READ here so eMSR bitmap works */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); current_evmcs->guest_rip += 3; /* vmcall */ /* Now tell KVM we've changed MSR-Bitmap */ current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* @@ -171,15 +171,15 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, * no VMCALL exit expected. */ vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Enable synthetic vmexit */ *(u32 *)(hv_pages->partition_assist) = 1; vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH); vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(11); /* VMPTRLD instruction causes #UD after enlightened VMLAUNCH */ diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c index 7987ce50c382..7e5f9ae1cb00 100644 --- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c @@ -106,7 +106,7 @@ void l1_vmx_code(struct vmx_pages *vmx) GUEST_SYNC(TEST_SYNC_NO_FAULT); vmlaunch(); GUEST_SYNC(TEST_SYNC_NO_FAULT); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_VMCALL); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c index 009932b1604e..68d9ce9dc6ba 100644 --- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c @@ -74,7 +74,7 @@ static void guest_code(void *test_data) vmwrite(GUEST_RIP, (u64)l2_guest_code); vmwrite(EXCEPTION_BITMAP, 0); - vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | + vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_PAUSE_EXITING | CPU_BASED_HLT_EXITING); } @@ -106,9 +106,9 @@ static void guest_code(void *test_data) vmlaunch(); else vmresume(); - exit_reason = vmreadz(VM_EXIT_REASON); - exit_insn_len = vmreadz(VM_EXIT_INSTRUCTION_LEN); - GUEST_ASSERT_EQ(vmreadz(GUEST_RIP), (u64)l2_instruction); + exit_reason = vmread(VM_EXIT_REASON); + exit_insn_len = vmread(VM_EXIT_INSTRUCTION_LEN); + GUEST_ASSERT_EQ(vmread(GUEST_RIP), (u64)l2_instruction); } __GUEST_ASSERT(exit_reason == insn->exit_reason[f], diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index d0beaa7b280b..d23e24f35790 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c @@ -120,10 +120,10 @@ static void vmx_run_l2(void *l2_code, int vector, u32 error_code) if (vector == FAKE_TRIPLE_FAULT_VECTOR) return; - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), vector); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_ERROR_CODE), error_code); - GUEST_ASSERT(!vmreadz(GUEST_INTERRUPTIBILITY_INFO)); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ((vmread(VM_EXIT_INTR_INFO) & 0xff), vector); + GUEST_ASSERT_EQ(vmread(VM_EXIT_INTR_ERROR_CODE), error_code); + GUEST_ASSERT(!vmread(GUEST_INTERRUPTIBILITY_INFO)); } static void l1_vmx_code(struct vmx_pages *vmx) @@ -149,7 +149,7 @@ static void l1_vmx_code(struct vmx_pages *vmx) vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS); vmx_run_l2(l2_ss_injected_tf_test, FAKE_TRIPLE_FAULT_VECTOR, 0); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c index 101c1fe79eeb..6e147826bc90 100644 --- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c +++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c @@ -46,16 +46,16 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) prepare_vmcs(vmx_pages, l2_guest_code); /* Try to run L2 with invalid CR3 and make sure it fails */ - save_cr3 = vmreadz(GUEST_CR3); + save_cr3 = vmread(GUEST_CR3); vmwrite(GUEST_CR3, -1ull); vmlaunch(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == + GUEST_ASSERT(vmread(VM_EXIT_REASON) == (EXIT_REASON_FAILED_VMENTRY | EXIT_REASON_INVALID_STATE)); /* Now restore CR3 and make sure L2 runs successfully */ vmwrite(GUEST_CR3, save_cr3); vmlaunch(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c index d6c443e3d147..fae77d843b13 100644 --- a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c @@ -63,17 +63,17 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, vmlaunch(); /* Verify we got an EPT violation exit */ - __GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION, + __GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION, "Expected EPT violation (0x%x), got 0x%lx", EXIT_REASON_EPT_VIOLATION, - vmreadz(VM_EXIT_REASON)); + vmread(VM_EXIT_REASON)); - __GUEST_ASSERT(vmreadz(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa, + __GUEST_ASSERT(vmread(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa, "Expected guest_physical_address = 0x%lx, got 0x%lx", expected_fault_gpa, - vmreadz(GUEST_PHYSICAL_ADDRESS)); + vmread(GUEST_PHYSICAL_ADDRESS)); - exit_qual = vmreadz(EXIT_QUALIFICATION); + exit_qual = vmread(EXIT_QUALIFICATION); /* * Note, EPT page table accesses are always read+write, e.g. so that diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c index c60dea899624..6a1a88e926f8 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c @@ -90,13 +90,13 @@ static void l1_guest_code(void *data) load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); vmwrite(TSC_OFFSET, TSC_OFFSET_VALUE); vmlaunch(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); } else { struct svm_test_data *svm = data; diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index b2fc7f0b5b3c..c763f4cf0f62 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -112,11 +112,11 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) prepare_vmcs(vmx_pages, l2_guest_code); /* enable TSC offsetting and TSC scaling for L2 */ - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control |= SECONDARY_EXEC_TSC_SCALING; vmwrite(SECONDARY_VM_EXEC_CONTROL, control); @@ -126,7 +126,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) /* launch L2 */ vmlaunch(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* check that L1's frequency still looks good */ check_tsc_freq(UCHECK_L1); diff --git a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c index e728f040916f..24056921110c 100644 --- a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c +++ b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c @@ -106,8 +106,8 @@ static void l1_vmx_code(struct vmx_pages *vmx) vmlaunch(); while (1) { - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ(vmread(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR); vmresume(); } } diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 8de6eb183670..500b09762555 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -59,12 +59,12 @@ void vmx_l2_guest_code(void) vmcall(); /* L1 has now set up a shadow VMCS for us. */ - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); GUEST_SYNC(10); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); vmwrite(GUEST_RIP, 0xc0fffee); GUEST_SYNC(11); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0fffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0fffee); vmwrite(GUEST_RIP, 0xc0ffffee); GUEST_SYNC(12); @@ -89,21 +89,21 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); vmlaunch(); GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* Check that the launched state is preserved. */ GUEST_ASSERT(__vmlaunch()); vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(7); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + 3); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + 3); vmwrite(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); vmwrite(VMCS_LINK_POINTER, vmx_pages->shadow_vmcs_gpa); @@ -116,18 +116,18 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) vmwrite(GUEST_RIP, 0xc0ffee); GUEST_SYNC(9); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); vmptrld(vmx_pages->vmcs_gpa); vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmptrld(vmx_pages->shadow_vmcs_gpa); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffffee); GUEST_ASSERT(__vmlaunch()); GUEST_ASSERT(__vmresume()); GUEST_SYNC(13); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffffee); GUEST_ASSERT(__vmlaunch()); GUEST_ASSERT(__vmresume()); } diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c index bb31a1ba8932..b2b796fd9ffb 100644 --- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c +++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c @@ -32,7 +32,7 @@ void l1_guest_code_vmx(struct vmx_pages *vmx) vmlaunch(); /* L2 should triple fault after a triple fault event injected. */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c index fe29c869a235..d6373c414d18 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c @@ -43,25 +43,25 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa) /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; vmwrite(SECONDARY_VM_EXEC_CONTROL, control); vmwrite(APIC_ACCESS_ADDR, vmx_pages->apic_access_gpa); /* Try to launch L2 with the memory-backed APIC-access address. */ - GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR)); + GUEST_SYNC(vmread(APIC_ACCESS_ADDR)); vmlaunch(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmwrite(APIC_ACCESS_ADDR, high_gpa); /* Try to resume L2 with the unbacked APIC-access address. */ - GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR)); + GUEST_SYNC(vmread(APIC_ACCESS_ADDR)); vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c index e578221da247..cda9e43e8599 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c @@ -38,7 +38,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); @@ -63,8 +63,8 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * as KVM should force the APIC ID back to its default. */ vmlaunch(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); GUEST_ASSERT(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD); /* @@ -91,7 +91,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * is active. */ vmresume(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_ASSERT(!(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD)); xapic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_DM_FIXED | GOOD_IPI_VECTOR); diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c index 3017938fa993..bbd188949412 100644 --- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c @@ -52,13 +52,13 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * library hasn't enabled it. Because KVM selftests jump directly to * 64-bit mode, unrestricted guest support isn't required. */ - GUEST_ASSERT(!(vmreadz(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) || - !(vmreadz(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST)); + GUEST_ASSERT(!(vmread(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) || + !(vmread(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST)); vmlaunch(); /* L2 should triple fault after main() stuffs invalid guest state. */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c index 46e5ca997e74..7dcee11877ea 100644 --- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c @@ -52,13 +52,13 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) pml4_pa = pml5[0] & PHYSICAL_PAGE_MASK; vmwrite(GUEST_CR3, pml4_pa); - guest_cr4 = vmreadz(GUEST_CR4); + guest_cr4 = vmread(GUEST_CR4); guest_cr4 &= ~X86_CR4_LA57; vmwrite(GUEST_CR4, guest_cr4); vmlaunch(); - exit_reason = vmreadz(VM_EXIT_REASON); + exit_reason = vmread(VM_EXIT_REASON); GUEST_ASSERT(exit_reason == EXIT_REASON_VMCALL); } diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index 06e74c72fe65..b9ee9d3c1da4 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -91,14 +91,14 @@ void l1_guest_code(struct vmx_pages *vmx_pages) return; vmlaunch(); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); /* * Turn on PIN control and resume the guest */ vmwrite(PIN_BASED_VM_EXEC_CONTROL, - vmreadz(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_VMX_PREEMPTION_TIMER); + vmread(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_VMX_PREEMPTION_TIMER); vmwrite(VMX_PREEMPTION_TIMER_VALUE, PREEMPTION_TIMER_VALUE); @@ -121,7 +121,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) /* * Ensure the exit from L2 is due to preemption timer expiry */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER); l1_tsc_deadline = l1_vmx_pt_start + (PREEMPTION_TIMER_VALUE << vmx_pt_rate); From 25158cac2d15ae2a3ac1f8791094df9c6522efeb Mon Sep 17 00:00:00 2001 From: Chaithanya Lagisetty Date: Mon, 7 Sep 2026 04:52:46 +0000 Subject: [PATCH 052/506] KVM: selftests: Fix the never-true negative UFFD delay check demand_paging_test parses the -d option with strtoul() and then asserts that the result is not negative: p.uffd_delay = strtoul(optarg, NULL, 0); TEST_ASSERT(p.uffd_delay >= 0, "A negative UFFD delay is not supported."); p.uffd_delay is a useconds_t, which is an unsigned type, so the comparison is always true and the assertion can never fire. GCC points this out with -Wtype-limits, which is enabled by -Wextra. As a result, "-d -1" is accepted and converted to a very large unsigned delay, causing each demand paging fault to sleep for an unexpectedly long time in usleep() instead of rejecting the argument up front. The return value of strtoul() is not validated either, so a non-numeric argument such as "-d abc" is silently treated as a zero delay. Use atoi_non_negative() instead. It rejects negative values, unparsable input, and trailing garbage. It is already used a few lines below for -v, and hexadecimal input keeps working because atoi_paranoid() also passes a base of 0 to strtol(). Fixes: 0119cb365c93 ("KVM: selftests: Add configurable demand paging delay") Signed-off-by: Chaithanya Lagisetty Tested-by: Gautam Menghani Reviewed-by: Gautam Menghani Link: https://patch.msgid.link/20260907045246.3783304-1-nagachaithanya9911@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/demand_paging_test.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c index f8b3d0b68830..619a0b2be45c 100644 --- a/tools/testing/selftests/kvm/demand_paging_test.c +++ b/tools/testing/selftests/kvm/demand_paging_test.c @@ -297,8 +297,7 @@ int main(int argc, char *argv[]) p.single_uffd = true; break; case 'd': - p.uffd_delay = strtoul(optarg, NULL, 0); - TEST_ASSERT(p.uffd_delay >= 0, "A negative UFFD delay is not supported."); + p.uffd_delay = atoi_non_negative("UFFD delay", optarg); break; case 'b': guest_percpu_mem_size = parse_size(optarg); From 2023782431927239c61330792904eff96ab9c1dc Mon Sep 17 00:00:00 2001 From: Chaithanya Lagisetty Date: Fri, 4 Sep 2026 05:20:33 +0000 Subject: [PATCH 053/506] KVM: selftests: Fix transposed calloc() arguments in the uffd helper uffd_setup_demand_paging() passes the element size as the first argument to calloc() and the element count as the second: uffd_desc->pipefds = calloc(sizeof(int), num_readers); calloc() expects the number of elements first and the size of each element second. The product, and therefore the allocation size, is the same either way, so this is not a functional change, but the arguments are misleading and GCC 14 diagnoses them with -Wcalloc-transposed-args, which is enabled by -Wextra. Swap the arguments to match the calloc() prototype. Signed-off-by: Chaithanya Lagisetty Reviewed-by: Gautam Menghani Link: https://patch.msgid.link/20260904052033.3648259-1-nagachaithanya9911@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/userfaultfd_util.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/userfaultfd_util.c b/tools/testing/selftests/kvm/lib/userfaultfd_util.c index f7ce5a6ddcc2..49eff05b7d39 100644 --- a/tools/testing/selftests/kvm/lib/userfaultfd_util.c +++ b/tools/testing/selftests/kvm/lib/userfaultfd_util.c @@ -119,13 +119,13 @@ struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay, uffd_desc = malloc(sizeof(struct uffd_desc)); TEST_ASSERT(uffd_desc, "Failed to malloc uffd descriptor"); - uffd_desc->pipefds = calloc(sizeof(int), num_readers); + uffd_desc->pipefds = calloc(num_readers, sizeof(int)); TEST_ASSERT(uffd_desc->pipefds, "Failed to alloc pipes"); - uffd_desc->readers = calloc(sizeof(pthread_t), num_readers); + uffd_desc->readers = calloc(num_readers, sizeof(pthread_t)); TEST_ASSERT(uffd_desc->readers, "Failed to alloc reader threads"); - uffd_desc->reader_args = calloc(sizeof(struct uffd_reader_args), num_readers); + uffd_desc->reader_args = calloc(num_readers, sizeof(struct uffd_reader_args)); TEST_ASSERT(uffd_desc->reader_args, "Failed to alloc reader_args"); uffd_desc->num_readers = num_readers; From b57917fc2b26561a409894188ee55e7b6568941a Mon Sep 17 00:00:00 2001 From: Chaithanya Lagisetty Date: Tue, 1 Sep 2026 09:08:38 +0000 Subject: [PATCH 054/506] KVM: selftests: Check calloc() return in vcpu_get_reg_list() vcpu_get_reg_list() allocates reg_list with calloc() and immediately dereferences it via reg_list->n without checking for allocation failure, unlike every other allocation in this file which is guarded by TEST_ASSERT(). If calloc() returns NULL the test crashes with a NULL pointer dereference instead of a clean failure message. Add the missing TEST_ASSERT() check. Fixes: fd02029a9e01 ("KVM: selftests: Add aarch64 get-reg-list test") Signed-off-by: Chaithanya Lagisetty Link: https://patch.msgid.link/20260901090839.3338955-1-nagachaithanya9911@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/kvm_util.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index cb2e7a98820a..980764e34208 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1733,6 +1733,7 @@ struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu) TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0"); reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64)); + TEST_ASSERT(reg_list, "Failed to allocate reg_list"); reg_list->n = reg_list_n.n; vcpu_ioctl(vcpu, KVM_GET_REG_LIST, reg_list); return reg_list; From 622a09f6deb53f8dd95d6b6482cf080997a0836a Mon Sep 17 00:00:00 2001 From: Gokul K Date: Sat, 8 Aug 2026 11:49:34 +0530 Subject: [PATCH 055/506] KVM: selftests: Fix the bounds of the SEV debug dst buffer check validate_dst() is passed a starting offset and a byte count, but bounds its loop with "i < nr_bytes", comparing the offset against the count rather than against offset + nr_bytes. Only calls that pass offset 0 check the region they were asked to check. Every other call checks too few bytes, and any call whose offset is >= the count returns without checking anything at all. Of the 1990 validate_dst() calls a full run makes per VM type, 740 (37%) check zero bytes, only the 29 zero-offset calls are correct, and 62% of the intended bytes are checked overall. The unaligned offsets the sweep deliberately picks to exercise partial chunks are precisely the ones that end up checking nothing. The bug is latent today: validate_buffers() runs first and compares all of src[] against dst[], so a genuine {en,de}crypt corruption is still caught. That is presumably why this went unnoticed. Bound the loop with offset + nr_bytes, and rename the parameter to offset so it cannot be mistaken for a length again. The caller already guarantees offset + nr_bytes <= BUFFER_SIZE, so the corrected loop stays in bounds. Fixes: 6edd35e77a42 ("KVM: selftests: Add a test to verify SEV {en,de}crypt debug ioctls") Signed-off-by: Gokul K Link: https://patch.msgid.link/20260808061934.400892-1-gokul02k@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/sev_dbg_test.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/sev_dbg_test.c b/tools/testing/selftests/kvm/x86/sev_dbg_test.c index eaa8201b937d..0277e38e3556 100644 --- a/tools/testing/selftests/kvm/x86/sev_dbg_test.c +++ b/tools/testing/selftests/kvm/x86/sev_dbg_test.c @@ -14,9 +14,11 @@ static u8 *data; static u8 src[BUFFER_SIZE] __aligned(PAGE_SIZE); static u8 dst[BUFFER_SIZE] __aligned(PAGE_SIZE); -static void validate_dst(int i, int nr_bytes, u8 pattern) +static void validate_dst(int offset, int nr_bytes, u8 pattern) { - for ( ; i < nr_bytes; i++) + int i; + + for (i = offset; i < offset + nr_bytes; i++) TEST_ASSERT(dst[i] == pattern, "Expected 0x%x at byte %u, got 0x%x", pattern, i, dst[i]); From 434c50e703c5aa24b0409a06f2ec1fc4040788b0 Mon Sep 17 00:00:00 2001 From: Gokul K Date: Tue, 11 Aug 2026 10:30:28 +0530 Subject: [PATCH 056/506] KVM: selftests: Free the VM when the SEV smoke test's guest completes test_sev() returns directly out of the UCALL_DONE case rather than leaving the loop, so the plain-SEV path skips the kvm_vm_free() at the end of the function. The SEV-ES path is unaffected; it breaks out of the loop and frees the VM correctly. main() invokes test_sev() once per supported SEV VM type, so a full run leaks a VM and its file descriptors. Nothing fails today because the process exits shortly afterwards, which is presumably why this was not noticed, but the leak also means the plain-SEV path never exercises VM teardown. Use a goto so UCALL_DONE joins the existing exit path. A plain break would only leave the switch statement and spin the loop again. Fixes: be250ff437fa ("KVM: selftests: Add a basic SEV smoke test") Signed-off-by: Gokul K Link: https://patch.msgid.link/20260811050029.530698-2-gokul02k@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/sev_smoke_test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/sev_smoke_test.c b/tools/testing/selftests/kvm/x86/sev_smoke_test.c index bf27b6187afa..19f76f7d8ca6 100644 --- a/tools/testing/selftests/kvm/x86/sev_smoke_test.c +++ b/tools/testing/selftests/kvm/x86/sev_smoke_test.c @@ -178,7 +178,7 @@ static void test_sev(void *guest_code, u32 type, u64 policy) case UCALL_SYNC: continue; case UCALL_DONE: - return; + goto done; case UCALL_ABORT: REPORT_GUEST_ASSERT(uc); default: @@ -187,6 +187,7 @@ static void test_sev(void *guest_code, u32 type, u64 policy) } } +done: kvm_vm_free(vm); } From ebb7027078965527d920360e7835a14eac6f835e Mon Sep 17 00:00:00 2001 From: Gokul K Date: Tue, 11 Aug 2026 10:30:29 +0530 Subject: [PATCH 057/506] KVM: selftests: Free the VM when NX hugepage disabling is denied run_test() returns as soon as it has confirmed that disabling NX huge pages fails with -EPERM, without freeing the VM created a few lines earlier. Jump to the kvm_vm_free() at the end of the function instead, matching the pattern used for the SEV smoke test in the previous patch. Fixes: b774da3f2e57 ("KVM: selftests: Test disabling NX hugepages on a VM") Signed-off-by: Gokul K Link: https://patch.msgid.link/20260811050029.530698-3-gokul02k@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/nx_huge_pages_test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c b/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c index 70950067b989..e3b20033a884 100644 --- a/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c +++ b/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c @@ -120,7 +120,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages, } else { TEST_ASSERT(r == -1 && errno == EPERM, "This process should not have permission to disable NX huge pages"); - return; + goto done; } } @@ -213,6 +213,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages, check_2m_page_count(vm, disable_nx_huge_pages ? 3 : 2); check_split_count(vm, 0); +done: kvm_vm_free(vm); } From be3c6888e1ab29d1d2a36af224eaf7739985c414 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 17:16:19 -0700 Subject: [PATCH 058/506] KVM: selftests: Account for kernel's off-by-one bug in NUMA node syscalls Add and use MAXNODE_FOR_MASK() to compute the "correct" maxnode value that is passed to various NUMA-related syscalls, e.g. get_mempolicy(), mbind(), migrate_pages(), etc. In quotes, because the kernel has an undocumented, longstanding off-by-one bug that requires userspace to specify the number of bits plus one, i.e. the max node plus two. The kernel bug has been known since 2007[*]: : And this is I think the reason why we can't change this now. I assume : numactl allocates 1024 bits (0 to 1023) and passes 1025 to make sure all : 1024 bits are processed. If we change it now, kernel will process 1025 : bits (0 to 1024) and overflow the allocated bitmask. If it happens to be : at the border of mmaped vma, it's a segfault... But unfortunately the manpages haven't yet been updated, e.g. : The maxnode argument is the maximum node number in the bit mask plus one Link: https://lore.kernel.org/all/63ccc890-fd57-118b-5997-e0259f507d28@suse.cz[*] Reviewed-by: Shivank Garg Link: https://patch.msgid.link/20260903001625.2792367-2-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/guest_memfd_test.c | 2 +- tools/testing/selftests/kvm/include/numaif.h | 11 +++++++++++ tools/testing/selftests/kvm/x86/xapic_ipi_test.c | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index 2233d871a38f..cd5df88bc642 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -80,7 +80,7 @@ static void test_mbind(int fd, size_t total_size) { const unsigned long nodemask_0 = 1; /* nid: 0 */ unsigned long nodemask = 0; - unsigned long maxnode = BITS_PER_TYPE(nodemask); + unsigned long maxnode = MAXNODE_FOR_MASK(nodemask); int policy; char *mem; int ret; diff --git a/tools/testing/selftests/kvm/include/numaif.h b/tools/testing/selftests/kvm/include/numaif.h index 29572a6d789c..71f261eafc90 100644 --- a/tools/testing/selftests/kvm/include/numaif.h +++ b/tools/testing/selftests/kvm/include/numaif.h @@ -6,6 +6,7 @@ #include +#include #include #include "kvm_syscalls.h" @@ -30,6 +31,16 @@ KVM_SYSCALL_DEFINE(mbind, 6, void *, addr, unsigned long, size, int, mode, const unsigned long *, nodemask, unsigned long, maxnode, unsigned int, flags); +/* + * Calculate the @maxnode param for the above syscalls given the mask that will + * be passed to the kernel, to account for a longstanding off-by-one bug in the + * kernel that isn't properly documented in the manpages. The manpages say + * that @maxnode is "the maximum node ID plus one", but the kernel's actual + * behavior is "the number of bits in the mask plus one", i.e. "the maximum + * node ID plus two". + */ +#define MAXNODE_FOR_MASK(mask) (BITS_PER_TYPE(mask) + 1) + static inline int get_max_numa_node(void) { struct dirent *de; diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 469e3ab16460..1ddcf95d7fe4 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -248,7 +248,7 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, delay_usecs); /* Get set of first 64 numa nodes available */ - kvm_get_mempolicy(NULL, &nodemask, sizeof(nodemask) * 8, + kvm_get_mempolicy(NULL, &nodemask, MAXNODE_FOR_MASK(nodemask), 0, MPOL_F_MEMS_ALLOWED); fprintf(stderr, "Numa nodes found amongst first %lu possible nodes " From 8883d2410c45bb2882b7ba44b38a70cfef0b84f9 Mon Sep 17 00:00:00 2001 From: Shivank Garg Date: Wed, 2 Sep 2026 17:16:20 -0700 Subject: [PATCH 059/506] KVM: selftests: Fix maxnode argument to migrate_pages() in xapic_ipi_test Use MAXNODE_FOR_MASK() to compute the @maxnode argument when migrating pages in the xAPIC IPI test, as the current code incorrectly passes the number of bytes, not the number of bits, in the mask, and also fails to account for the kernel's off-by-one bug. I.e. the test sets maxnode to 8, and so the kernel only checks node IDs 0-6 even though the nodemask covers node IDs 0-63. Fixes: 678e90a349a4 ("KVM: selftests: Test IPI to halted vCPU in xAPIC while backing page moves") Signed-off-by: Shivank Garg [sean: use MAXNODE_FOR_MASK(), rewrite changelog] Link: https://patch.msgid.link/20260903001625.2792367-3-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/xapic_ipi_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 1ddcf95d7fe4..6be11c1c725c 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -291,7 +291,7 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, * KVM_CREATE_VCPU ioctl. If that assumption ever changes this * test may break or give a false positive signal. */ - pages_not_moved = migrate_pages(0, sizeof(nodemasks[from]), + pages_not_moved = migrate_pages(0, MAXNODE_FOR_MASK(nodemasks[from]), &nodemasks[from], &nodemasks[to]); if (pages_not_moved < 0) From d2dde5c387fc9ae1b9364c1bc096bac5d419756f Mon Sep 17 00:00:00 2001 From: Shivank Garg Date: Wed, 2 Sep 2026 17:16:21 -0700 Subject: [PATCH 060/506] KVM: selftests: use BITS_PER_TYPE() for NUMA masks Replace the open-coded sizeof() * 8 calculations in do_migrations() with BITS_PER_TYPE(). No functional change intended. Signed-off-by: Shivank Garg Link: https://patch.msgid.link/20260903001625.2792367-4-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/xapic_ipi_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 6be11c1c725c..9d1dfad4efa6 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -233,7 +233,7 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, { long pages_not_moved; unsigned long nodemask = 0; - unsigned long nodemasks[sizeof(nodemask) * 8]; + unsigned long nodemasks[BITS_PER_TYPE(nodemask)]; int nodes = 0; time_t start_time, last_update, now; time_t interval_secs = 1; @@ -253,13 +253,13 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, fprintf(stderr, "Numa nodes found amongst first %lu possible nodes " "(each 1-bit indicates node is present): %#lx\n", - sizeof(nodemask) * 8, nodemask); + BITS_PER_TYPE(nodemask), nodemask); /* Init array of masks containing a single-bit in each, one for each * available node. migrate_pages called below requires specifying nodes * as bit masks. */ - for (i = 0, bit = 1; i < sizeof(nodemask) * 8; i++, bit <<= 1) { + for (i = 0, bit = 1; i < BITS_PER_TYPE(nodemask); i++, bit <<= 1) { if (nodemask & bit) { nodemasks[nodes] = nodemask & bit; nodes++; From 3a2a37d2d48e6844dde61032faf130400fad5312 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 17:16:22 -0700 Subject: [PATCH 061/506] KVM: selftests: Compute node masks on-demand in xAPIC IPI test Compute the node masks for the source (from) and destination (to) NUMA nodes in the xAPIC IPI test instead of pre-filling an array of masks with node per mask. Computing the mask on-demand is technically slower, but doesn't require a large-ish on-stack array, and more importantly allows the test to use a generic "get next NUMA node" API without having to commit all of KVM selftests to using a large array of single-bit nodemasks. Implement said API as a common KVM NUMA API so that it can be used by other tests, e.g. in guest_memfd tests. Deliberately make @from "exclusive" as the anticipated usage in KVM selftests is to select the next, *different* node, i.e. so that users don't have to copy+paste code to assert that the found node is different than the starting node. The obvious downside is that implementing the exclusive logic forces callers to pass -1 instead of 0 when the goal is to find the first node in the mask, but that arguably yields more intuitive code anyways. From an overall test functionality/coverage perspective, no functional change intended (the walking pattern of node migration should be unchanged). Reviewed-by: Shivank Garg Link: https://patch.msgid.link/20260903001625.2792367-5-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/numaif.h | 21 +++++++++++- .../selftests/kvm/x86/xapic_ipi_test.c | 32 ++++++------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/tools/testing/selftests/kvm/include/numaif.h b/tools/testing/selftests/kvm/include/numaif.h index 71f261eafc90..299dddff2729 100644 --- a/tools/testing/selftests/kvm/include/numaif.h +++ b/tools/testing/selftests/kvm/include/numaif.h @@ -6,7 +6,7 @@ #include -#include +#include #include #include "kvm_syscalls.h" @@ -41,6 +41,25 @@ KVM_SYSCALL_DEFINE(mbind, 6, void *, addr, unsigned long, size, int, mode, */ #define MAXNODE_FOR_MASK(mask) (BITS_PER_TYPE(mask) + 1) +/* + * Return the node ID of the next NUMA node in the mask, starting at @from+1. + * Guarantees a node is found, and that the found node is not @from. Pass -1 + * to find the first node in the mask. + */ +static inline int kvm_get_next_numa_node(unsigned long nodemask, int from) +{ + const unsigned long nr_bits = BITS_PER_TYPE(nodemask); + int to; + + to = find_next_bit(&nodemask, nr_bits, from + 1); + if (to == nr_bits) + to = find_next_bit(&nodemask, nr_bits, 0); + + TEST_ASSERT(to != nr_bits && to != from, + "Unabled to find second NUMA node (from = %d, to = %d)", from, to); + return to; +} + static inline int get_max_numa_node(void) { struct dirent *de; diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 9d1dfad4efa6..7144ad833ae0 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -233,13 +233,10 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, { long pages_not_moved; unsigned long nodemask = 0; - unsigned long nodemasks[BITS_PER_TYPE(nodemask)]; int nodes = 0; time_t start_time, last_update, now; time_t interval_secs = 1; - int i; int from, to; - unsigned long bit; u64 hlt_count; u64 wake_count; u64 ipis_sent; @@ -255,24 +252,15 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, "(each 1-bit indicates node is present): %#lx\n", BITS_PER_TYPE(nodemask), nodemask); - /* Init array of masks containing a single-bit in each, one for each - * available node. migrate_pages called below requires specifying nodes - * as bit masks. - */ - for (i = 0, bit = 1; i < BITS_PER_TYPE(nodemask); i++, bit <<= 1) { - if (nodemask & bit) { - nodemasks[nodes] = nodemask & bit; - nodes++; - } - } - + nodes = __builtin_popcountl(nodemask); TEST_ASSERT(nodes > 1, "Did not find at least 2 numa nodes. Can't do migration"); fprintf(stderr, "Migrating amongst %d nodes found\n", nodes); - from = 0; - to = 1; + from = kvm_get_next_numa_node(nodemask, -1); + to = kvm_get_next_numa_node(nodemask, from); + start_time = time(NULL); last_update = start_time; @@ -281,6 +269,9 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, wake_count = data->wake_count; while ((int)(time(NULL) - start_time) < run_secs) { + unsigned long from_mask = BIT(from); + unsigned long to_mask = BIT(to); + data->migrations_attempted++; /* @@ -291,9 +282,8 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, * KVM_CREATE_VCPU ioctl. If that assumption ever changes this * test may break or give a false positive signal. */ - pages_not_moved = migrate_pages(0, MAXNODE_FOR_MASK(nodemasks[from]), - &nodemasks[from], - &nodemasks[to]); + pages_not_moved = migrate_pages(0, MAXNODE_FOR_MASK(from_mask), + &from_mask, &to_mask); if (pages_not_moved < 0) fprintf(stderr, "migrate_pages failed, errno=%d\n", errno); @@ -305,9 +295,7 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, data->migrations_completed++; from = to; - to++; - if (to == nodes) - to = 0; + to = kvm_get_next_numa_node(nodemask, from); now = time(NULL); if (((now - start_time) % interval_secs == 0) && From 0be096bbc9cc7734b9aa82f62b4a5b44bd884540 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 17:16:23 -0700 Subject: [PATCH 062/506] KVM: selftests: Add common helper to get mask+number of usable memory NUMA nodes Extract and slightly adopt the xAPIC IPI test's logic for getting the mask and number of usable memory NUMA nodes into a common helper. To allow for friendlier behavior when the underlying kernel doesn't support NUMA, or the test was run without sufficient permissions, zero out the mask and return "zero nodes" if get_mempolicy() fails with ENOSYS or EPERM respectively. I.e. allow tests to skip (sub)tests instead of throwing an assert. Suggested-by: Shivank Garg Reviewed-by: Shivank Garg Link: https://patch.msgid.link/20260903001625.2792367-6-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/numaif.h | 13 +++++++++++++ tools/testing/selftests/kvm/x86/xapic_ipi_test.c | 5 +---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/kvm/include/numaif.h b/tools/testing/selftests/kvm/include/numaif.h index 299dddff2729..0945500c6322 100644 --- a/tools/testing/selftests/kvm/include/numaif.h +++ b/tools/testing/selftests/kvm/include/numaif.h @@ -41,6 +41,19 @@ KVM_SYSCALL_DEFINE(mbind, 6, void *, addr, unsigned long, size, int, mode, */ #define MAXNODE_FOR_MASK(mask) (BITS_PER_TYPE(mask) + 1) +static inline int kvm_get_numa_memory_nodes(unsigned long *nodemask) +{ + int r; + + *nodemask = 0; + + r = get_mempolicy(NULL, nodemask, MAXNODE_FOR_MASK(*nodemask), 0, + MPOL_F_MEMS_ALLOWED); + TEST_ASSERT(!r || errno == ENOSYS || errno == EPERM, + "Unexpected get_mempolicy() failure"); + return __builtin_popcountl(*nodemask); +} + /* * Return the node ID of the next NUMA node in the mask, starting at @from+1. * Guarantees a node is found, and that the found node is not @from. Pass -1 diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 7144ad833ae0..42c601617f5a 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -244,15 +244,12 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, fprintf(stderr, "Calling migrate_pages every %d microseconds\n", delay_usecs); - /* Get set of first 64 numa nodes available */ - kvm_get_mempolicy(NULL, &nodemask, MAXNODE_FOR_MASK(nodemask), - 0, MPOL_F_MEMS_ALLOWED); + nodes = kvm_get_numa_memory_nodes(&nodemask); fprintf(stderr, "Numa nodes found amongst first %lu possible nodes " "(each 1-bit indicates node is present): %#lx\n", BITS_PER_TYPE(nodemask), nodemask); - nodes = __builtin_popcountl(nodemask); TEST_ASSERT(nodes > 1, "Did not find at least 2 numa nodes. Can't do migration"); From 92ed4d5c78fdfcbec3f6b64b194b5202f868ef4c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 17:16:24 -0700 Subject: [PATCH 063/506] KVM: selftests: Automatically run xAPIC IPI migration test when possible Rework the interface to the xAPIC IPI test to automatically run the migration testcase if at least two NUMA nodes are found. The test exists specifically to validate KVM's handling of migration of the APIC backing page, i.e. forcing end users to opt-in to running the test in migration mode largely defeats the purpose of the test. Run both the "sleeping" and "migration" testcases by default, e.g. so that the more basic testcase will fail if KVM completely breaks IPI delivery. Keep the -m / migration parameter so that infrastructure that wants to specifically test the migration case can do so and not get false passes. Reviewed-by: Shivank Garg Link: https://patch.msgid.link/20260903001625.2792367-7-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/x86/xapic_ipi_test.c | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 42c601617f5a..c5b506eb5e7f 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -245,14 +245,13 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, delay_usecs); nodes = kvm_get_numa_memory_nodes(&nodemask); + TEST_ASSERT(nodes > 1, + "NUMA nodes disappeared? nodemask = 0x%lx", nodemask); fprintf(stderr, "Numa nodes found amongst first %lu possible nodes " "(each 1-bit indicates node is present): %#lx\n", BITS_PER_TYPE(nodemask), nodemask); - TEST_ASSERT(nodes > 1, - "Did not find at least 2 numa nodes. Can't do migration"); - fprintf(stderr, "Migrating amongst %d nodes found\n", nodes); from = kvm_get_next_numa_node(nodemask, -1); @@ -351,26 +350,17 @@ void get_cmdline_args(int argc, char *argv[], int *run_secs, } } -int main(int argc, char *argv[]) +static void test_xapic_ipi(int run_secs, int delay_usecs, bool migrate) { int wait_secs; const int max_halter_wait = 10; - int run_secs = 0; - int delay_usecs = 0; struct test_data_page *data; gva_t test_data_page_gva; - bool migrate = false; pthread_t threads[2]; struct thread_params params[2]; struct kvm_vm *vm; u64 *pipis_rcvd; - get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs); - if (run_secs <= 0) - run_secs = DEFAULT_RUN_SECS; - if (delay_usecs <= 0) - delay_usecs = DEFAULT_DELAY_USECS; - vm = vm_create_with_one_vcpu(¶ms[0].vcpu, halter_guest_code); vm_install_exception_handler(vm, IPI_VECTOR, guest_ipi_handler); @@ -458,5 +448,29 @@ int main(int argc, char *argv[]) kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + bool force_migrate = false; + unsigned long nodemask; + int run_secs = 0; + int delay_usecs = 0; + + get_cmdline_args(argc, argv, &run_secs, &force_migrate, &delay_usecs); + if (run_secs <= 0) + run_secs = DEFAULT_RUN_SECS; + if (delay_usecs <= 0) + delay_usecs = DEFAULT_DELAY_USECS; + + if (!force_migrate) + test_xapic_ipi(run_secs, delay_usecs, false); + + if (kvm_get_numa_memory_nodes(&nodemask) > 1) + test_xapic_ipi(run_secs, delay_usecs, true); + else + TEST_ASSERT(!force_migrate, + "Did not find at least 2 numa nodes. Can't do migration"); + return 0; } From d5aa06fbfec28a2acbb03bbba361ad9c1a90eb73 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 17:16:25 -0700 Subject: [PATCH 064/506] KVM: selftests: Skip xAPIC IPI migration test when forced but unsupported Skip the xAPIC IPI migration test instead of failing outright if migration is forced by the user, but is unsupported due to lack of NUMA support. Failing the test is arguably "fine" for opt-in behavior, but in practice doesn't allow running the test in forced-migration mode in any kind of automated test environment. Reviewed-by: Shivank Garg Link: https://patch.msgid.link/20260903001625.2792367-8-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/xapic_ipi_test.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index c5b506eb5e7f..2362bf272e80 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -469,8 +469,9 @@ int main(int argc, char *argv[]) if (kvm_get_numa_memory_nodes(&nodemask) > 1) test_xapic_ipi(run_secs, delay_usecs, true); else - TEST_ASSERT(!force_migrate, - "Did not find at least 2 numa nodes. Can't do migration"); + __TEST_REQUIRE(!force_migrate, + "Need at least 2 NUMA nodes to do migration (nodemask = 0x%lx)", + nodemask); return 0; } From d80ddf303f3a31440de93572407c68d0d7493283 Mon Sep 17 00:00:00 2001 From: Frank Zhu Date: Thu, 10 Sep 2026 19:54:06 +0800 Subject: [PATCH 065/506] KVM: selftests: Add Zhaoxin CPU detection support Add helpers to detect Zhaoxin CPUs using the "CentaurHauls" and " Shanghai " vendor strings. KVM handles Zhaoxin CPUs with its VMX backend. Add host_cpu_is_intel_compatible for selftests that exercise behavior shared by Intel and Zhaoxin CPUs, while retaining host_cpu_is_zhaoxin for Zhaoxin-specific behavior. Signed-off-by: Frank Zhu Link: https://patch.msgid.link/20260910115414.3015260-2-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/processor.h | 8 ++++++++ tools/testing/selftests/kvm/lib/x86/processor.c | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 282b8aaff799..a86bde201ed8 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -23,6 +23,8 @@ extern bool host_cpu_is_intel; extern bool host_cpu_is_amd; extern bool host_cpu_is_hygon; extern bool host_cpu_is_amd_compatible; +extern bool host_cpu_is_zhaoxin; +extern bool host_cpu_is_intel_compatible; extern u64 guest_tsc_khz; extern struct kvm_mmu guest_mmu; @@ -782,6 +784,12 @@ static inline bool this_cpu_is_hygon(void) return this_cpu_vendor_string_is("HygonGenuine"); } +static inline bool this_cpu_is_zhaoxin(void) +{ + return this_cpu_vendor_string_is("CentaurHauls") || + this_cpu_vendor_string_is(" Shanghai "); +} + static inline u32 __this_cpu_has(u32 function, u32 index, u8 reg, u8 lo, u8 hi) { u32 gprs[4]; diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index 479e9a481687..51d891131f3e 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -26,6 +26,8 @@ bool host_cpu_is_amd; bool host_cpu_is_intel; bool host_cpu_is_hygon; bool host_cpu_is_amd_compatible; +bool host_cpu_is_zhaoxin; +bool host_cpu_is_intel_compatible; bool is_forced_emulation_enabled; u64 guest_tsc_khz; struct kvm_mmu guest_mmu; @@ -822,6 +824,8 @@ void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) sync_global_to_guest(vm, host_cpu_is_amd); sync_global_to_guest(vm, host_cpu_is_hygon); sync_global_to_guest(vm, host_cpu_is_amd_compatible); + sync_global_to_guest(vm, host_cpu_is_zhaoxin); + sync_global_to_guest(vm, host_cpu_is_intel_compatible); sync_global_to_guest(vm, is_forced_emulation_enabled); sync_global_to_guest(vm, pmu_errata_mask); @@ -1447,6 +1451,8 @@ void kvm_selftest_arch_init(void) host_cpu_is_amd = this_cpu_is_amd(); host_cpu_is_hygon = this_cpu_is_hygon(); host_cpu_is_amd_compatible = host_cpu_is_amd || host_cpu_is_hygon; + host_cpu_is_zhaoxin = this_cpu_is_zhaoxin(); + host_cpu_is_intel_compatible = host_cpu_is_intel || host_cpu_is_zhaoxin; is_forced_emulation_enabled = kvm_is_forced_emulation_enabled(); kvm_init_pmu_errata(); From 9bb23b9e85ea7e8f8ef65d31cee15aa44173cb97 Mon Sep 17 00:00:00 2001 From: Frank Zhu Date: Thu, 10 Sep 2026 19:54:07 +0800 Subject: [PATCH 066/506] KVM: selftests: Use host_cpu_is_intel_compatible for mediated PMU kvm_is_mediated_pmu_enabled() returns kvm_amd's enable_mediated_pmu module parameter unless the host CPU is Intel. Zhaoxin CPUs implement VMX-compatible virtualization and are handled by KVM's VMX backend, so on Zhaoxin hosts the helper wrongly reports kvm_amd's parameter. Gate the check on host_cpu_is_intel_compatible so that Zhaoxin CPUs, like Intel CPUs, query kvm_intel's enable_mediated_pmu instead. This does not change behavior on Intel, and AMD and other non-Intel- compatible CPUs still fall back to kvm_amd's parameter. Signed-off-by: Frank Zhu Link: https://patch.msgid.link/20260910115414.3015260-3-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/processor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index a86bde201ed8..7579e692bc2b 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -1454,7 +1454,7 @@ static inline bool kvm_is_pmu_enabled(void) static inline bool kvm_is_mediated_pmu_enabled(void) { - if (host_cpu_is_intel) + if (host_cpu_is_intel_compatible) return get_kvm_intel_param_bool("enable_mediated_pmu"); return get_kvm_amd_param_bool("enable_mediated_pmu"); From 54edcc375a46613bc6b14fa5c310be68ea3446b4 Mon Sep 17 00:00:00 2001 From: Frank Zhu Date: Thu, 10 Sep 2026 19:54:08 +0800 Subject: [PATCH 067/506] KVM: selftests: Add TDP detection for Zhaoxin CPUs Update the kvm_is_tdp_enabled() function to include Zhaoxin CPUs by querying checking "Intel compatible" when checking for EPT (Extended Page Tables) support. This ensures that Zhaoxin CPUs use the appropriate TDP (Two-Dimensional Paging) parameter check, similar to Intel CPUs. Signed-off-by: Frank Zhu Link: https://patch.msgid.link/20260910115414.3015260-4-ewanhai-oc@zhaoxin.com [sean: massage changelog] Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/x86/processor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index 51d891131f3e..8374afd08baf 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -160,7 +160,7 @@ static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, u8 indent) bool kvm_is_tdp_enabled(void) { - if (host_cpu_is_intel) + if (host_cpu_is_intel_compatible) return get_kvm_intel_param_bool("ept"); else return get_kvm_amd_param_bool("npt"); From 7610139e1ab7cc143491fb4b176a477e0001d134 Mon Sep 17 00:00:00 2001 From: Frank Zhu Date: Thu, 10 Sep 2026 19:54:09 +0800 Subject: [PATCH 068/506] KVM: selftests: Allow running invalid guest state test on Zhaoxin CPUs The vmx_exception_with_invalid_guest_state test currently requires an Intel CPU. However, Zhaoxin CPUs also support VMX and the relevant invalid guest state handling under KVM. Update the test requirement to allow execution on Zhaoxin platforms in addition to Intel, enabling broader coverage of VMX-related error paths. Signed-off-by: Frank Zhu Link: https://patch.msgid.link/20260910115414.3015260-5-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c index 2cae86d9d5e2..08a0caa9c52d 100644 --- a/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c @@ -109,7 +109,7 @@ int main(int argc, char *argv[]) struct kvm_vcpu *vcpu; struct kvm_vm *vm; - TEST_REQUIRE(host_cpu_is_intel); + TEST_REQUIRE(host_cpu_is_intel_compatible); TEST_REQUIRE(!kvm_is_unrestricted_guest_enabled()); vm = vm_create_with_one_vcpu(&vcpu, guest_code); From 2625a21954c46d0a9c8a926127c95a8178c80a54 Mon Sep 17 00:00:00 2001 From: Frank Zhu Date: Thu, 10 Sep 2026 19:54:10 +0800 Subject: [PATCH 069/506] KVM: selftests: Add Zhaoxin CPU support in x86's "fix" hypercall test Zhaoxin CPUs use VMCALL as the native hypercall instruction. Use host_cpu_is_intel_compatible so that fix_hypercall_test selects the VMX hypercall path on Zhaoxin CPUs. Signed-off-by: Frank Zhu Link: https://patch.msgid.link/20260910115414.3015260-6-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/fix_hypercall_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/fix_hypercall_test.c b/tools/testing/selftests/kvm/x86/fix_hypercall_test.c index 4931ec22768e..f11fcc520f24 100644 --- a/tools/testing/selftests/kvm/x86/fix_hypercall_test.c +++ b/tools/testing/selftests/kvm/x86/fix_hypercall_test.c @@ -48,7 +48,7 @@ static void guest_main(void) const u8 *other_hypercall_insn; u64 ret; - if (host_cpu_is_intel) { + if (host_cpu_is_intel_compatible) { native_hypercall_insn = vmx_vmcall; other_hypercall_insn = svm_vmmcall; } else if (host_cpu_is_amd_compatible) { From 658d82c07e78eaa153991ca72326637a6cc6ce92 Mon Sep 17 00:00:00 2001 From: Frank Zhu Date: Thu, 10 Sep 2026 19:54:12 +0800 Subject: [PATCH 070/506] KVM: selftests: Extend Intel MSR_IA32_UCODE_REV quirk to Zhaoxin Zhaoxin CPUs use KVM's VMX backend and therefore observe the same KVM-defined initial value as Intel vCPUs. Update feature_msrs_test to expect 0x100000000ULL for Zhaoxin vCPUs. Signed-off-by: Frank Zhu Link: https://patch.msgid.link/20260910115414.3015260-8-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/feature_msrs_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/feature_msrs_test.c b/tools/testing/selftests/kvm/x86/feature_msrs_test.c index 158550701771..d56d0ddb3a57 100644 --- a/tools/testing/selftests/kvm/x86/feature_msrs_test.c +++ b/tools/testing/selftests/kvm/x86/feature_msrs_test.c @@ -59,7 +59,7 @@ static void test_feature_msr(u32 msr) * but initializes the vCPU's revision ID to an arbitrary value. */ if (msr == MSR_IA32_UCODE_REV) - reset_value = host_cpu_is_intel ? 0x100000000ULL : 0x01000065; + reset_value = host_cpu_is_intel_compatible ? 0x100000000ULL : 0x01000065; /* * For quirked MSRs, KVM's ABI is to initialize the vCPU's value to the From 1a0abfad6c8096dfb00aa5a764d92819ff749103 Mon Sep 17 00:00:00 2001 From: Frank Zhu Date: Thu, 10 Sep 2026 19:54:13 +0800 Subject: [PATCH 071/506] KVM: selftests: Add Zhaoxin CPU support to nested exceptions test When vectoring an injected #SS through the empty L2 IDT, Zhaoxin CPUs report the resulting #GP with the IDT bit set and EXT clear. Zhaoxin sets EXT only for interrupt delivery and selected guest-mode events; #SS delivery is not one of those cases. Expect error code 0x62 when running the test on Zhaoxin CPUs. Signed-off-by: Frank Zhu Link: https://patch.msgid.link/20260910115414.3015260-9-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/nested_exceptions_test.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index d23e24f35790..f47432526ea1 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c @@ -38,6 +38,7 @@ */ #define GP_ERROR_CODE_AMD ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG) #define GP_ERROR_CODE_INTEL ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG | ERROR_CODE_EXT_FLAG) +#define GP_ERROR_CODE_ZHAOXIN ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG) /* * Intel and AMD both shove '0' into the error code on #DF, regardless of what @@ -142,7 +143,8 @@ static void l1_vmx_code(struct vmx_pages *vmx) */ vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF); vmx_run_l2(l2_ss_pending_test, SS_VECTOR, (u16)SS_ERROR_CODE); - vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, GP_ERROR_CODE_INTEL); + vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, + host_cpu_is_zhaoxin ? GP_ERROR_CODE_ZHAOXIN : GP_ERROR_CODE_INTEL); vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF); vmx_run_l2(l2_ss_injected_df_test, DF_VECTOR, DF_ERROR_CODE); From 0d5c14800de007ee45fd76008e4a115f170fa88d Mon Sep 17 00:00:00 2001 From: Frank Zhu Date: Thu, 10 Sep 2026 19:54:14 +0800 Subject: [PATCH 072/506] KVM: selftests: Check for APIC access virtualization support in said test Not all CPUs that support VMX necessarily support the "virtualize APIC accesses" secondary processor-based VM-execution control. For example, some Zhaoxin CPUs lack this feature. Add a capability check before running the test to properly skip on unsupported hardware instead of failing. Carve out the guts of kvm_cpu_has_ept() into a generic helper and use said helper to provide kvm_cpu_has_vmx_apic_access_virt(). Signed-off-by: Frank Zhu Link: https://patch.msgid.link/20260910115414.3015260-10-ewanhai-oc@zhaoxin.com [sean: add a helper instead of copy+pasting kvm_cpu_has_ept()] Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/vmx.h | 13 ++++++++++++- tools/testing/selftests/kvm/lib/x86/vmx.c | 12 ++++++------ .../selftests/kvm/x86/vmx_apic_access_test.c | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index ca67a618b68f..8fd97434eee5 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -503,7 +503,18 @@ void load_vmcs(struct vmx_pages *vmx); bool ept_1g_pages_supported(void); -bool kvm_cpu_has_ept(void); +bool kvm_cpu_has_secondary_exec_control(u32 ctrl); + +static inline bool kvm_cpu_has_ept(void) +{ + return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_ENABLE_EPT); +} + +static inline bool kvm_cpu_has_vmx_virtualize_apic_accesses(void) +{ + return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); +} + void vm_enable_ept(struct kvm_vm *vm); void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm); diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 6a53c8ce3b3c..5a06746ebea4 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -422,19 +422,19 @@ void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip) init_vmcs_guest_state(guest_rip, vmx->stack); } -bool kvm_cpu_has_ept(void) +bool kvm_cpu_has_secondary_exec_control(u32 ctrl) { - u64 ctrl; + u64 ctrl_msr; if (!kvm_cpu_has(X86_FEATURE_VMX)) return false; - ctrl = kvm_get_feature_msr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32; - if (!(ctrl & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) + ctrl_msr = kvm_get_feature_msr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32; + if (!(ctrl_msr & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) return false; - ctrl = kvm_get_feature_msr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32; - return ctrl & SECONDARY_EXEC_ENABLE_EPT; + ctrl_msr = kvm_get_feature_msr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32; + return ctrl_msr & ctrl; } void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c index d6373c414d18..1d7a529c8297 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c @@ -77,7 +77,7 @@ int main(int argc, char *argv[]) struct kvm_vcpu *vcpu; struct kvm_vm *vm; - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX)); + TEST_REQUIRE(kvm_cpu_has_vmx_virtualize_apic_accesses()); vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); From ba78382f34c10dcb04f352b959b16b469fba417a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 14:32:41 -0700 Subject: [PATCH 073/506] KVM: x86: Update "last guest TSC" snapshot prior to enabling IRQs/preemption When refreshing the last observed guest TSC during a guest time update, write the snapshot before enabling IRQs, i.e. before enabling preemption. If the task is migrated between updating the local tsc_timestamp, e.g. to account for catch-up mode, and setting last_guest_tsc, kvm_arch_vcpu_load() would set the vCPU's TSC offset using the old last_guest_tsc. In practice, the bug is largely benign as it's not even strictly necessary for KVM to refresh last_guest_tsc when updating guest time, as KVM's goal is purely to prevent the guest from observing time jump backwards, i.e. super duper strictly speaking, KVM only *needs* to update last_guest_tsc in the VM-Exit path. In fact, the update kvm_guest_time_update() in wasn't even added to play nice with kvm_arch_vcpu_load(), it was added by commit 28e4639adf0c ("KVM: x86: Fix kvmclock bug") to fix code that no longer exists. As of commit 28e4639adf0c, kvm_guest_time_update() also consumed last_guest_tsc, to try and prevent guest time from jumping backwards. That code was eventually removed by commit f25e656d31ad ("KVM: x86: fix tsc catchup issue with tsc scaling"), but the last_guest_tsc update hung around. Keep the update even though it's technically ok to drop the update, e.g. so that the tsc_catchup updates aren't lost, and so that the guest won't see a PV clock timestamp that appears to be in the future. Link: https://patch.msgid.link/20260826213303.914988-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..a359606b7655 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1820,6 +1820,12 @@ int kvm_guest_time_update(struct kvm_vcpu *v) } } + /* + * Refresh L1's last "observed" TSC to match the PV clock's timestamp, + * e.g. so that the guest can't see a TSC that's behind the reference. + */ + vcpu->last_guest_tsc = tsc_timestamp; + local_irq_restore(flags); /* With all the info we got, fill in the values */ @@ -1841,7 +1847,6 @@ int kvm_guest_time_update(struct kvm_vcpu *v) hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul; hv_clock.tsc_timestamp = tsc_timestamp; hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; - vcpu->last_guest_tsc = tsc_timestamp; /* If the host uses TSC clocksource, then it is stable */ hv_clock.flags = 0; From d4f54ed48e0043371f8b81aac3450ea86ce30a3e Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:42 -0700 Subject: [PATCH 074/506] KVM: x86: Improve accuracy of KVM clock when TSC scaling is in force The kvm_guest_time_update() function scales the host TSC frequency to the guest's using kvm_scale_tsc() and the v->arch.l1_tsc_scaling_ratio scaling ratio previously calculated for that vCPU. Then calculates the scaling factors for the KVM clock itself based on that guest TSC frequency. However, it uses kHz as the unit when scaling, and then multiplies by 1000 only at the end. With a host TSC frequency of 3000MHz and a guest set to 2500MHz, the result of kvm_scale_tsc() will actually come out at 2,499,999kHz. So the KVM clock advertised to the guest is based on a frequency of 2,499,999,000 Hz. By using Hz as the unit from the beginning, the KVM clock would be based on a more accurate frequency of 2,499,999,999 Hz in this example. Use u64 for the hw_tsc_hz field since an unsigned int would overflow for TSC frequencies above 4GHz. Fixes: 78db6a503796 ("KVM: x86: rewrite handling of scaled TSC for kvmclock") Reviewed-by: Paul Durrant Signed-off-by: David Woodhouse Tested-by: Dongli Zhang Link: https://patch.msgid.link/20260826213303.914988-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/cpuid.c | 1 + arch/x86/kvm/x86.c | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3af7395c2430..a384e25793ad 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -865,7 +865,7 @@ struct kvm_vcpu_arch { gpa_t time; s8 pvclock_tsc_shift; u32 pvclock_tsc_mul; - unsigned int hw_tsc_khz; + u64 hw_tsc_hz; struct gfn_to_pfn_cache pv_time; /* set guest stopped flag in pvclock flags field */ bool pvclock_set_guest_stopped_request; diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 851f151efb35..38450e9fca9a 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index a359606b7655..b38aafac7226 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -61,6 +61,7 @@ #include #include #include +#include #include #include @@ -1763,7 +1764,8 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, int kvm_guest_time_update(struct kvm_vcpu *v) { struct pvclock_vcpu_time_info hv_clock = {}; - unsigned long flags, tgt_tsc_khz; + unsigned long flags; + u64 tgt_tsc_hz; unsigned seq; struct kvm_vcpu_arch *vcpu = &v->arch; struct kvm_arch *ka = &v->kvm->arch; @@ -1789,8 +1791,8 @@ int kvm_guest_time_update(struct kvm_vcpu *v) /* Keep irq disabled to prevent changes to the clock */ local_irq_save(flags); - tgt_tsc_khz = get_cpu_tsc_khz(); - if (unlikely(tgt_tsc_khz == 0)) { + tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + if (unlikely(tgt_tsc_hz == 0)) { local_irq_restore(flags); kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); return 1; @@ -1831,16 +1833,16 @@ int kvm_guest_time_update(struct kvm_vcpu *v) /* With all the info we got, fill in the values */ if (kvm_caps.has_tsc_control) { - tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz, + tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz, v->arch.l1_tsc_scaling_ratio); - tgt_tsc_khz = tgt_tsc_khz ? : 1; + tgt_tsc_hz = tgt_tsc_hz ? : 1; } - if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) { - kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL, + if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) { + kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz, &vcpu->pvclock_tsc_shift, &vcpu->pvclock_tsc_mul); - vcpu->hw_tsc_khz = tgt_tsc_khz; + vcpu->hw_tsc_hz = tgt_tsc_hz; } hv_clock.tsc_shift = vcpu->pvclock_tsc_shift; From dd0a8675f4ba65c208b12ea3d9bd45093c7aa2cf Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:43 -0700 Subject: [PATCH 075/506] KVM: x86: Explicitly disable TSC scaling without CONSTANT_TSC KVM does make an attempt to cope with non-constant TSC, and has notifiers to handle host TSC frequency changes. However, it *only* adjusts the KVM clock, and doesn't adjust TSC frequency scaling when the host changes. This is presumably because non-constant TSCs were fixed in hardware long before TSC scaling was implemented, so there should never be real CPUs which have TSC scaling but *not* CONSTANT_TSC. Such a combination could potentially happen in some odd L1 nesting environment, but it isn't worth trying to support it. Just make the dependency explicit. Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260826213303.914988-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 3 ++- arch/x86/kvm/vmx/vmx.c | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7d59d301e1e5..f5aa3d7d3a10 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5620,7 +5620,8 @@ static __init int svm_hardware_setup(void) XFEATURE_MASK_BNDCSR); if (tsc_scaling) { - if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { + if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR) || + !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { tsc_scaling = false; } else { pr_info("TSC scaling supported\n"); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..54d93167e342 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2843,6 +2843,16 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, if (!cpu_has_sgx()) _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; + /* + * KVM doesn't re-derive the TSC scaling ratio when the host TSC + * frequency changes, so TSC scaling is only usable with a constant + * TSC. Clear the control here rather than in vmx_hardware_setup() so + * that the per-CPU configs recomputed by vmx_check_processor_compat() + * stay consistent with the golden vmcs_config. + */ + if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) + _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_TSC_SCALING; + if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) _cpu_based_3rd_exec_control = adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, From 7254e09bb5e90089d82c7ee65ce180f49c38c43e Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:44 -0700 Subject: [PATCH 076/506] KVM: x86: Activate master clock immediately on vCPU creation Previously, the master clock was only activated when the first vCPU processed KVM_REQ_MASTERCLOCK_UPDATE during KVM_RUN. This meant that KVM_GET_CLOCK could not return the host_tsc field until after the first KVM_RUN, making it impossible for userspace to follow the documented TSC migration procedure without a dummy vCPU run. Fix this by calling kvm_update_masterclock() directly from kvm_arch_vcpu_postcreate(), after kvm_synchronize_tsc() has already set all_vcpus_matched_freq. This ensures the master clock is active immediately, and KVM_GET_CLOCK returns a valid {host_tsc, realtime} pair as soon as a vCPU exists. Signed-off-by: David Woodhouse Link: https://patch.msgid.link/20260826213303.914988-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index b38aafac7226..a4f4b072d2ac 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9447,6 +9447,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) return; vcpu_load(vcpu); kvm_synchronize_tsc(vcpu, NULL); + if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) + kvm_update_masterclock(vcpu->kvm); vcpu_put(vcpu); /* poll control enabled by default */ From 7ffdb9d111c3e0ce881f037a7834e107f1e4b361 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:45 -0700 Subject: [PATCH 077/506] KVM: x86: Compute kvmclock base without pvclock_gtod_data get_kvmclock_base_ns() needs CLOCK_MONOTONIC_RAW + offs_boot. Compute this directly rather than reading offs_boot from the pvclock_gtod_data private copy. offs_boot only changes at suspend/resume so does not need to be atomically paired with the raw clock read. Signed-off-by: David Woodhouse Assisted-by: Kiro:claude-opus-4.6-1m Link: https://patch.msgid.link/20260826213303.914988-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index a4f4b072d2ac..043d6d18fb4a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -930,7 +930,7 @@ static void update_pvclock_gtod(struct timekeeper *tk) static s64 get_kvmclock_base_ns(void) { /* Count up from boot time, but with the frequency of the raw clock. */ - return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot)); + return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT)); } #else static s64 get_kvmclock_base_ns(void) From 4c710e8182f4fe31889663c5d893e3be692ae3c6 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:46 -0700 Subject: [PATCH 078/506] KVM: x86: Avoid NTP frequency skew for KVM clock on 32-bit host Now that 64-bit KVM doesn't consume pvclock_gtod_data when getting the kvmclock base, use the ktime_mono_to_any() version on 32-bit hosts so that they too are unaffected by NPT sync. Note, even if KVM still used pvclock_gtod_data for getting kvmclock base, it would still be very doable and reasonable to avoid skew on 32-bit hosts, it's just much simpler now that KVM uses ktime_mono_to_any(). Specifically, commit 53fafdbb8b21 ("KVM: x86: switch KVMCLOCK base to monotonic raw clock") did so only for 64-bit hosts, by capturing the boot offset from within the existing clocksource notifier update_pvclock_gtod(). That notifier was added in commit 16e8d74d2da9 ("KVM: x86: notifier for clocksource changes") but only on x86_64, because its original purpose was just to disable the "master clock" mode which is only supported on x86_64. Now that the notifier is used for more than disabling master clock mode, enabling it would be trivial for 32-bit builds as well, but as above, that's a moot point since get_kvmclock_base_ns() no longer relies on the notifier. Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant [sean: rebase on top of ktime_mono_to_any() usage] Link: https://patch.msgid.link/20260826213303.914988-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 043d6d18fb4a..113c9c564ad3 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -926,19 +926,13 @@ static void update_pvclock_gtod(struct timekeeper *tk) write_seqcount_end(&vdata->seq); } +#endif static s64 get_kvmclock_base_ns(void) { /* Count up from boot time, but with the frequency of the raw clock. */ return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT)); } -#else -static s64 get_kvmclock_base_ns(void) -{ - /* Master clock not used, so we can just use CLOCK_BOOTTIME. */ - return ktime_get_boottime_ns(); -} -#endif static uint32_t div_frac(uint32_t dividend, uint32_t divisor) { From 796b73bba57728f8a67744916459afbed75f629a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 14:32:47 -0700 Subject: [PATCH 079/506] KVM: x86: Drop unnecessary CPU pinning when computing/getting kvmclock When computing the current kvmclock value, don't pin the task to the current CPU for the entire duration of the master clock path, as the CPU pinning was never about ensuring rdtsc() and cpu_tsc_khz would agree. As pointed out by David, ka->use_master_clock can only be true when the host clocksource is TSC based, which in turn requires a stable, constant and synchronised TSC across all CPUs. The CPU pinning was added in commit e2c2206a1899 ("KVM: x86: Fix potential preemption when get the current kvmclock timestamp") purely in response to a CONFIG_DEBUG_PREEMPT=y bug due to accessing a per-CPU variable with preemption enabled. Despite what the comment would suggest, including rdtsc() in the {get,put}_cpu() section was opportunistic. In fact, Paolo even said exactly that when suggesting that KVM guarantee the rdtsc() would execute on the same CPU[*]: : Also, rdtsc() should really be on the same CPU as __this_cpu_read. We : know it's not really really necessary because the master clock is : active, but since we need a get_cpu/put_cpu pair, better be clean. Nothing has changed in the last ~9 years, i.e. the rdtsc() still *should* be on the same CPU, but super strictly speaking, all will be fine if the task is migrated between grabbing the frequency and doing rdtsc(). Dropping the CPU pinning will allow dropping the rdtsc() entirely without having to resort to a large "rewrite get_kvmclock()" patch. Opportunistically add a comment to explain why KVM needs to snapshot the frequency, because that _is_ a hard requirement to avoid reintroducing the bug fixed by commit e70b57a6ce4e ("KVM: X86: Fix softlockup when get the current kvmclock") Link: https://lore.kernel.org/all/ae8de642-8f14-a70a-1fab-57e2c4093cd5@redhat.com [*] Suggested-by: David Woodhouse Link: https://patch.msgid.link/20260826213303.914988-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 113c9c564ad3..b26699cf302d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1657,13 +1657,18 @@ static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) { struct kvm_arch *ka = &kvm->arch; struct pvclock_vcpu_time_info hv_clock; + u64 tsc_hz; - /* both __this_cpu_read() and rdtsc() should be on the same cpu */ + /* + * Snapshot and validate the TSC frequency as kvmclock_cpu_down_prep() + * zeros the per-CPU value when a CPU is going offline. + */ get_cpu(); + tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + put_cpu(); data->flags = 0; - if (ka->use_master_clock && - (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) { + if (ka->use_master_clock && tsc_hz) { #ifdef CONFIG_X86_64 struct timespec64 ts; @@ -1677,15 +1682,13 @@ static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) data->flags |= KVM_CLOCK_TSC_STABLE; hv_clock.tsc_timestamp = ka->master_cycle_now; hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL, + kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, &hv_clock.tsc_shift, &hv_clock.tsc_to_system_mul); data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); } else { data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; } - - put_cpu(); } static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) From 1a926fb39304ca8aaa4355ecda3052aa53018317 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 14:32:48 -0700 Subject: [PATCH 080/506] KVM: x86: Move "no master clock" fallback from __get_kvmclock() to get_kvmclock() Move the fallback logic for getting the current kvmclock when not in master clock mode out of __get_kvmclock() and into its sole caller, get_kvmclock(). This will allow using early-return logic in the master clock code, without having to resort to a do-while() loop and/or gotos. No functional change intended. Link: https://patch.msgid.link/20260826213303.914988-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index b26699cf302d..789d96991026 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1653,12 +1653,16 @@ static unsigned long get_cpu_tsc_khz(void) } /* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */ -static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) +static bool __get_kvmclock_master_clock(struct kvm *kvm, + struct kvm_clock_data *data) { struct kvm_arch *ka = &kvm->arch; struct pvclock_vcpu_time_info hv_clock; u64 tsc_hz; + if (!ka->use_master_clock) + return false; + /* * Snapshot and validate the TSC frequency as kvmclock_cpu_down_prep() * zeros the per-CPU value when a CPU is going offline. @@ -1667,8 +1671,10 @@ static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; put_cpu(); - data->flags = 0; - if (ka->use_master_clock && tsc_hz) { + if (!tsc_hz) + return false; + + { #ifdef CONFIG_X86_64 struct timespec64 ts; @@ -1686,9 +1692,9 @@ static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) &hv_clock.tsc_shift, &hv_clock.tsc_to_system_mul); data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); - } else { - data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; } + + return true; } static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) @@ -1697,8 +1703,11 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) unsigned seq; do { + data->flags = 0; + seq = read_seqcount_begin(&ka->pvclock_sc); - __get_kvmclock(kvm, data); + if (!__get_kvmclock_master_clock(kvm, data)) + data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; } while (read_seqcount_retry(&ka->pvclock_sc, seq)); } From 8e7e24b88e26be5214a22995bb585065f4065c25 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:49 -0700 Subject: [PATCH 081/506] KVM: x86: Wrap all of __get_kvmclock_master_clock() with CONFIG_X86_64=y Wrap the entire use_master_clock block in #ifdef CONFIG_X86_64, since use_master_clock is never true on 32-bit (host_tsc_clocksource is only set under CONFIG_X86_64), and declare hv_clock inside the block so it is not left as an unused variable on 32-bit. Signed-off-by: David Woodhouse [sean: keep only the CONFIG_X86_64 ifdef, update changelog accordingly] Link: https://patch.msgid.link/20260826213303.914988-10-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 789d96991026..e23bbb3b6f82 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1656,8 +1656,10 @@ static unsigned long get_cpu_tsc_khz(void) static bool __get_kvmclock_master_clock(struct kvm *kvm, struct kvm_clock_data *data) { +#ifdef CONFIG_X86_64 struct kvm_arch *ka = &kvm->arch; struct pvclock_vcpu_time_info hv_clock; + struct timespec64 ts; u64 tsc_hz; if (!ka->use_master_clock) @@ -1674,27 +1676,24 @@ static bool __get_kvmclock_master_clock(struct kvm *kvm, if (!tsc_hz) return false; - { -#ifdef CONFIG_X86_64 - struct timespec64 ts; - - if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) { - data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; - data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC; - } else -#endif + if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) { + data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; + data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC; + } else { data->host_tsc = rdtsc(); - - data->flags |= KVM_CLOCK_TSC_STABLE; - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); } + data->flags |= KVM_CLOCK_TSC_STABLE; + hv_clock.tsc_timestamp = ka->master_cycle_now; + hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; + kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, + &hv_clock.tsc_shift, + &hv_clock.tsc_to_system_mul); + data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); return true; +#else + return false; +#endif } static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) From 25f75fb43fd53138c248869334fdff4555a7a1e4 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:50 -0700 Subject: [PATCH 082/506] KVM: x86: Fall back to non-master-clock if clockread fails in get_kvmclock() When computing kvmclock and it's currently in master-clock mode, fall back to the non-master-clock path if the clock read fails, e.g. if the kernel's clocksource is transitioning away from TSC but ka->use_master_clock hasn't been updated yet. The rdtsc() fallback was added (well, kept) in commit c68dc1b577ea ("KVM: x86: Report host tsc and realtime values in KVM_GET_CLOCK") purely to avoid uninitialized variables and compilation problems on 32-bit kernels (already addressed). In hindsight, keeping the rdtsc() was a hack and a mistake. Link: https://lore.kernel.org/all/CAOQ_QsgVqS_PuJo8F10Gg5Xw+tKt+5gDx+kJf1j3CiPO4MAOqg@mail.gmail.com Signed-off-by: David Woodhouse [sean: isolate from refactoring changes, write changelog] Link: https://patch.msgid.link/20260826213303.914988-11-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e23bbb3b6f82..872d49a2eeb0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1676,14 +1676,13 @@ static bool __get_kvmclock_master_clock(struct kvm *kvm, if (!tsc_hz) return false; - if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) { - data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; - data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC; - } else { - data->host_tsc = rdtsc(); - } + if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) + return false; + + data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; + data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC | + KVM_CLOCK_TSC_STABLE; - data->flags |= KVM_CLOCK_TSC_STABLE; hv_clock.tsc_timestamp = ka->master_cycle_now; hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, From abdef42276d88417f6a0c1f9fae8190accc3bc38 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:51 -0700 Subject: [PATCH 083/506] KVM: x86: Fix KVM clock precision in get_kvmclock() with TSC scaling When in master clock mode, the KVM clock is defined in terms of the guest TSC. But get_kvmclock() was computing it from the host TSC without applying TSC scaling, leading to a systemic drift from the values the guest computes from its own TSC. Store the VM's TSC scaling ratio in kvm_arch and precompute the guest-TSC-based mul/shift in pvclock_update_vm_gtod_copy(). Use these in get_kvmclock() to scale the host TSC delta to guest TSC before converting to nanoseconds. This avoids "definition C" of the KVM clock described in commit 633d7652f80f ("KVM: x86/xen: Do not corrupt KVM clock in kvm_xen_shared_info_init()"). Signed-off-by: David Woodhouse Link: https://patch.msgid.link/20260826213303.914988-12-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 4 +++ arch/x86/kvm/x86.c | 61 ++++++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a384e25793ad..1e3b53e1d2bb 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1237,6 +1237,7 @@ struct kvm_arch { u64 last_tsc_write; u32 last_tsc_khz; u64 last_tsc_offset; + u64 last_tsc_scaling_ratio; u64 cur_tsc_nsec; u64 cur_tsc_write; u64 cur_tsc_offset; @@ -1252,6 +1253,9 @@ struct kvm_arch { u64 master_kernel_ns; u64 master_cycle_now; struct ratelimit_state kvmclock_update_rs; + u64 master_tsc_scaling_ratio; + s8 master_tsc_shift; + u32 master_tsc_mul; #ifdef CONFIG_KVM_HYPERV struct kvm_hv hyperv; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 872d49a2eeb0..4567896b7cf7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1249,6 +1249,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, kvm->arch.last_tsc_write = tsc; kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; kvm->arch.last_tsc_offset = offset; + kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; vcpu->arch.last_guest_tsc = tsc; @@ -1561,6 +1562,8 @@ static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, * */ +static unsigned long get_cpu_tsc_khz(void); + static void pvclock_update_vm_gtod_copy(struct kvm *kvm) { #ifdef CONFIG_X86_64 @@ -1584,9 +1587,30 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm) && !ka->backwards_tsc_observed && !ka->boot_vcpu_runs_old_kvmclock; - if (ka->use_master_clock) + if (ka->use_master_clock) { + u64 tsc_hz; + atomic_set(&kvm_guest_has_master_clock, 1); + /* + * Copy the scaling ratio and precompute the mul/shift for + * converting guest TSC to nanoseconds. These are used by + * get_kvmclock() to compute kvmclock from the host TSC + * without needing a vCPU reference. + */ + ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio; + tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + if (tsc_hz && kvm_caps.has_tsc_control) + tsc_hz = kvm_scale_tsc(tsc_hz, + ka->master_tsc_scaling_ratio); + if (tsc_hz) + kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, + &ka->master_tsc_shift, + &ka->master_tsc_mul); + else + ka->use_master_clock = false; + } + vclock_mode = pvclock_gtod_data.clock.vclock_mode; trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, vcpus_matched); @@ -1660,22 +1684,10 @@ static bool __get_kvmclock_master_clock(struct kvm *kvm, struct kvm_arch *ka = &kvm->arch; struct pvclock_vcpu_time_info hv_clock; struct timespec64 ts; - u64 tsc_hz; if (!ka->use_master_clock) return false; - /* - * Snapshot and validate the TSC frequency as kvmclock_cpu_down_prep() - * zeros the per-CPU value when a CPU is going offline. - */ - get_cpu(); - tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; - put_cpu(); - - if (!tsc_hz) - return false; - if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) return false; @@ -1685,10 +1697,25 @@ static bool __get_kvmclock_master_clock(struct kvm *kvm, hv_clock.tsc_timestamp = ka->master_cycle_now; hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); + + /* + * Use the precomputed guest-TSC-based mul/shift so that the kvmclock + * value matches what the guest computes from its own TSC. + */ + hv_clock.tsc_shift = ka->master_tsc_shift; + hv_clock.tsc_to_system_mul = ka->master_tsc_mul; + + if (kvm_caps.has_tsc_control) { + u64 tsc_delta = data->host_tsc - ka->master_cycle_now; + + tsc_delta = kvm_scale_tsc(tsc_delta, ka->master_tsc_scaling_ratio); + data->clock = hv_clock.system_time + + pvclock_scale_delta(tsc_delta, + hv_clock.tsc_to_system_mul, + hv_clock.tsc_shift); + } else { + data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); + } return true; #else return false; From c56419275caad2d99264b7c50b0701552ed3ef75 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:52 -0700 Subject: [PATCH 084/506] KVM: x86: Use get_kvmclock() in kvm_get_wall_clock_epoch() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that get_kvmclock() correctly handles TSC scaling and captures both wallclock and kvmclock from the same TSC reading, kvm_get_wall_clock_epoch() can simply call it instead of duplicating the pvclock computation. This eliminates the last instance of the "definition C" kvmclock calculation — as described in commit 633d7652f80f ("KVM: x86/xen: Do not corrupt KVM clock in kvm_xen_shared_info_init()") — which computed nanoseconds directly from the host TSC without accounting for guest TSC scaling. Signed-off-by: David Woodhouse Link: https://patch.msgid.link/20260826213303.914988-13-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 59 +++++++--------------------------------------- 1 file changed, 9 insertions(+), 50 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 4567896b7cf7..e0e13235a69e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1941,63 +1941,22 @@ int kvm_guest_time_update(struct kvm_vcpu *v) * wallclock and kvmclock times, and subtracting one from the other. * * Fall back to using their values at slightly different moments by - * calling ktime_get_real_ns() and get_kvmclock_ns() separately. + * calling ktime_get_real_ns() and get_kvmclock() separately. */ uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm) { -#ifdef CONFIG_X86_64 - struct pvclock_vcpu_time_info hv_clock; - struct kvm_arch *ka = &kvm->arch; - unsigned long seq, local_tsc_khz; - struct timespec64 ts; - uint64_t host_tsc; + struct kvm_clock_data data; - do { - seq = read_seqcount_begin(&ka->pvclock_sc); - - local_tsc_khz = 0; - if (!ka->use_master_clock) - break; - - /* - * The TSC read and the call to get_cpu_tsc_khz() must happen - * on the same CPU. - */ - get_cpu(); - - local_tsc_khz = get_cpu_tsc_khz(); - - if (local_tsc_khz && - !kvm_get_walltime_and_clockread(&ts, &host_tsc)) - local_tsc_khz = 0; /* Fall back to old method */ - - put_cpu(); - - /* - * These values must be snapshotted within the seqcount loop. - * After that, it's just mathematics which can happen on any - * CPU at any time. - */ - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - - } while (read_seqcount_retry(&ka->pvclock_sc, seq)); + get_kvmclock(kvm, &data); /* - * If the conditions were right, and obtaining the wallclock+TSC was - * successful, calculate the KVM clock at the corresponding time and - * subtract one from the other to get the guest's epoch in nanoseconds - * since 1970-01-01. + * If get_kvmclock() captured both wallclock and kvmclock from the + * same TSC reading, use them for a precise epoch calculation. */ - if (local_tsc_khz) { - kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec - - __pvclock_read_cycles(&hv_clock, host_tsc); - } -#endif - return ktime_get_real_ns() - get_kvmclock_ns(kvm); + if (data.flags & KVM_CLOCK_REALTIME) + return data.realtime - data.clock; + + return ktime_get_real_ns() - data.clock; } /* From d9173f59df3a625b0e8b7df9873b7358996acc6a Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:53 -0700 Subject: [PATCH 085/506] KVM: x86: Fix compute_guest_tsc() to handle negative time deltas The compute_guest_tsc() function computes the guest TSC at a given kernel_ns timestamp. When the master clock reference point (master_kernel_ns) is earlier than vcpu->arch.this_tsc_nsec, the delta is negative. Since pvclock_scale_delta() takes a u64, the negative value wraps to a huge positive number, producing a wildly wrong result. Handle negative deltas explicitly by scaling the absolute value of the delta and applying it to this_tsc_write with the appropriate sign. This is believed to be unreachable in practice; no path has been identified which invokes compute_guest_tsc() with a timestamp from before the vCPU's TSC generation was established. Fix it for robustness, in the spirit of defence in depth. Signed-off-by: David Woodhouse Link: https://patch.msgid.link/20260826213303.914988-14-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e0e13235a69e..f918db1c8127 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1056,11 +1056,15 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) { - u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec, - vcpu->arch.virtual_tsc_mult, - vcpu->arch.virtual_tsc_shift); - tsc += vcpu->arch.this_tsc_write; - return tsc; + s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec; + u64 tsc; + + /* Handle negative deltas gracefully (master clock ref may be earlier) */ + tsc = pvclock_scale_delta(abs(delta_ns), + vcpu->arch.virtual_tsc_mult, + vcpu->arch.virtual_tsc_shift); + + return vcpu->arch.this_tsc_write + (delta_ns >= 0 ? tsc : -tsc); } #ifdef CONFIG_X86_64 From 917a53a4b461df5a848856e0715aa2d1fa3783fe Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 14:32:54 -0700 Subject: [PATCH 086/506] KVM: x86: Disable preemption, not IRQs, when getting TSC+freq pair Disable "just" preemption, not IRQs, when reading the TSC+frequency pair to update guest time, as disabling IRQs to protect against task migration is overkill (though it's *extremely* hard to see that it's overkill). Disabling IRQs was added by commit 18068523d3a0 ("KVM: paravirtualized clocksource: host part") before there was any coordination with timekeeping (presumably disabling IRQs prevented the kernel from completing a software- induced frequency change). After the coordination and locking was added, commit c09664bb4418 ("KVM: x86: fix deadlock in clock-in-progress request handling") moved the locking and coordination out of IRQ protection, and thus made disabling IRQs pointless, except for protecting get_cpu_tsc_khz(). And while cpu_tsc_khz is written only from IRQ context, and the *extremely* confusing double IPIs sent by __kvmclock_cpufreq_notifier() to update the per-CPU frequency make it seem like they would require readers to disable IRQs, it is safe to read and consume cpu_tsc_khz (via get_cpu_tsc_khz()) with IRQs enabled. The per-CPU variable is specifically written only in IRQ context to ensure hotplugging a CPU wouldn't write cpu_tsc_khz with a stale value (because apparently disabling IRQs would be too simple?!?). As for the double IPIs in the frequency notifier, both IPIs are red herrings. The actual sequence that ensures KVM updates guest time with the new frequency is that the first write is completed *before* the notifier sets KVM_REQ_CLOCK_UPDATE for all vCPUs that last ran on the target pCPU. The first write is done via IPI to adhere to the above rules, and the second IPI is sent purely to kick any vCPU that happens to be running on the target CPU out of the guest. I.e. the second IPI writes cpu_tsc_khz out of pure KVM laziness: it saves having to define another IPI callback. In fact prior to commit 8cfdc0008542 ("KVM: x86: Make cpu_tsc_khz updates use local CPU"), KVM did indeed use an empty callback to ack the IPI. As for why it was deemed cleaner to abuse tsc_khz_changed()... Link: https://patch.msgid.link/20260826213303.914988-15-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f918db1c8127..4a1ddb49c6c2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1799,7 +1799,6 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, int kvm_guest_time_update(struct kvm_vcpu *v) { struct pvclock_vcpu_time_info hv_clock = {}; - unsigned long flags; u64 tgt_tsc_hz; unsigned seq; struct kvm_vcpu_arch *vcpu = &v->arch; @@ -1824,11 +1823,14 @@ int kvm_guest_time_update(struct kvm_vcpu *v) } } while (read_seqcount_retry(&ka->pvclock_sc, seq)); - /* Keep irq disabled to prevent changes to the clock */ - local_irq_save(flags); + /* + * Ensure reading the TSC+frequency pair is done on the same CPU. When + * NOT using the master clock, the TSC frequency may vary between CPUs. + */ + preempt_disable(); tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; if (unlikely(tgt_tsc_hz == 0)) { - local_irq_restore(flags); + preempt_enable(); kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); return 1; } @@ -1863,7 +1865,7 @@ int kvm_guest_time_update(struct kvm_vcpu *v) */ vcpu->last_guest_tsc = tsc_timestamp; - local_irq_restore(flags); + preempt_enable(); /* With all the info we got, fill in the values */ From f21b04302404fa87826caa7f5fd3f7e07ab6782e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 14:32:55 -0700 Subject: [PATCH 087/506] KVM: x86: Make master clock logic in guest PV clock updates 64-bit only Wrap kvm_guest_time_update()'s entire use_master_clock block in #ifdef CONFIG_X86_64 and provide a hardcoded-to-false variable for the 32-bit path, as use_master_clock is never true on 32-bit (host_tsc_clocksource is only set under CONFIG_X86_64) Keep the local "ka" variable outside of the 64-bit-only code as it's also used for Xen code, but tag it as potentially unused since it's not used on 32-bit kernels without Xen emulation. For all intents and purposes, no functional change intended. Link: https://patch.msgid.link/20260826213303.914988-16-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 4a1ddb49c6c2..14e7efdd74ed 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1798,22 +1798,21 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, int kvm_guest_time_update(struct kvm_vcpu *v) { + struct kvm_arch *ka __maybe_unused = &v->kvm->arch; struct pvclock_vcpu_time_info hv_clock = {}; u64 tgt_tsc_hz; - unsigned seq; struct kvm_vcpu_arch *vcpu = &v->arch; - struct kvm_arch *ka = &v->kvm->arch; s64 kernel_ns; u64 tsc_timestamp, host_tsc; - bool use_master_clock; - - kernel_ns = 0; - host_tsc = 0; /* * If the host uses TSC clock, then passthrough TSC as stable * to the guest. */ +#ifdef CONFIG_X86_64 + bool use_master_clock; + unsigned int seq; + do { seq = read_seqcount_begin(&ka->pvclock_sc); use_master_clock = ka->use_master_clock; @@ -1822,7 +1821,9 @@ int kvm_guest_time_update(struct kvm_vcpu *v) kernel_ns = ka->master_kernel_ns; } } while (read_seqcount_retry(&ka->pvclock_sc, seq)); - +#else + const bool use_master_clock = false; +#endif /* * Ensure reading the TSC+frequency pair is done on the same CPU. When * NOT using the master clock, the TSC frequency may vary between CPUs. From 92fac63cc73920175f712cec792f9ab698923ffb Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:56 -0700 Subject: [PATCH 088/506] KVM: x86: Upscale TSC to "now", not master clock when updating PV clocks When doing TSC upscaling in software, e.g. when the guest TSC frequency is configured to run faster than the host without hardware support, always advance guest TSC to "now", not to the master clock's reference values, which represents some time in the past. In master clock mode, there are two points of time which need to be taken into account. First there is the master clock reference point, stored in kvm->arch.master_kernel_ns (and associated host TSC ->master_cycle_now). Secondly, there is the time *now*, at the point kvm_update_guest_time() is being called. With software TSC upscaling, the guest TSC is getting further and further ahead of the host TSC as time elapses. So at time "now", the guest TSC should be further ahead of the host, than it was at master_kernel_ns. Not taking that into account means KVM was advancing the guest TSC only by the amount appropriate for the snapshot taken in the past, *not* to the current time (which is what RDTSC is supposed to return). Signed-off-by: David Woodhouse Co-developed-by: Sean Christopherson Link: https://patch.msgid.link/20260826213303.914988-17-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 14e7efdd74ed..4180e18e0cbb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1798,12 +1798,11 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, int kvm_guest_time_update(struct kvm_vcpu *v) { + u64 tgt_tsc_hz, tsc_timestamp, host_tsc, master_tsc, master_ns; struct kvm_arch *ka __maybe_unused = &v->kvm->arch; struct pvclock_vcpu_time_info hv_clock = {}; - u64 tgt_tsc_hz; struct kvm_vcpu_arch *vcpu = &v->arch; s64 kernel_ns; - u64 tsc_timestamp, host_tsc; /* * If the host uses TSC clock, then passthrough TSC as stable @@ -1816,10 +1815,16 @@ int kvm_guest_time_update(struct kvm_vcpu *v) do { seq = read_seqcount_begin(&ka->pvclock_sc); use_master_clock = ka->use_master_clock; - if (use_master_clock) { - host_tsc = ka->master_cycle_now; - kernel_ns = ka->master_kernel_ns; + if (!use_master_clock) + continue; + + if (!kvm_get_time_and_clockread(&kernel_ns, &host_tsc)) { + use_master_clock = false; + continue; } + + master_tsc = ka->master_cycle_now; + master_ns = ka->master_kernel_ns; } while (read_seqcount_retry(&ka->pvclock_sc, seq)); #else const bool use_master_clock = false; @@ -1885,8 +1890,18 @@ int kvm_guest_time_update(struct kvm_vcpu *v) hv_clock.tsc_shift = vcpu->pvclock_tsc_shift; hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul; - hv_clock.tsc_timestamp = tsc_timestamp; - hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; + /* + * If the master clock is NOT in use, the reference time placed in the + * hv_clock is "now". If master clock is in use, the reference time is + * the master clock's snapshot from some time in the past, not "now". + */ + if (use_master_clock) { + hv_clock.tsc_timestamp = kvm_read_l1_tsc(v, master_tsc); + hv_clock.system_time = master_ns + v->kvm->arch.kvmclock_offset; + } else { + hv_clock.tsc_timestamp = tsc_timestamp; + hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; + } /* If the host uses TSC clocksource, then it is stable */ hv_clock.flags = 0; From 2918af628d0dae8476f89abeb76fe9fe3fd029cb Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:57 -0700 Subject: [PATCH 089/506] KVM: x86: Simplify and comment kvm_get_time_scale() The kvm_get_time_scale() function was entirely opaque. Add comments explaining what it does: compute a fixed-point multiplier and shift for converting TSC ticks to nanoseconds via pvclock_scale_delta(). Rename the local variables from the cryptic tps64/tps32/scaled64 to base_hz_u64/base32/scaled_hz_u64 to make the code self-documenting. The "tps32" name stood for "Ticks Per Second" but was misleading since it held the shifted base frequency, not a tick count. No functional change. Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260826213303.914988-18-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 55 +++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 4180e18e0cbb..fb585470746e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -940,32 +940,57 @@ static uint32_t div_frac(uint32_t dividend, uint32_t divisor) return dividend; } -static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz, +static void kvm_get_time_scale(u64 scaled_hz, u64 base_hz, s8 *pshift, u32 *pmultiplier) { - uint64_t scaled64; - int32_t shift = 0; - uint64_t tps64; - uint32_t tps32; + u64 scaled_hz_u64 = scaled_hz; + s32 shift = 0; + u64 base_hz_u64; + u32 base32; - tps64 = base_hz; - scaled64 = scaled_hz; - while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) { - tps64 >>= 1; + /* + * This function calculates a fixed-point multiplier and shift such + * that: + * time_ns = (tsc_cycles << shift) * multiplier >> 32 + * + * Where tsc_cycles tick at base_hz, and time_ns should count at + * scaled_hz (typically NSEC_PER_SEC for a TSC→nanoseconds conversion). + * + * The multiplier is: (scaled_hz << 32) / base_hz, adjusted by shift + * to keep everything in range. + */ + + base_hz_u64 = base_hz; + + /* + * Start by shifting base_hz right until it fits in 32 bits, and + * is lower than double the target rate. This introduces a negative + * shift value which would result in pvclock_scale_delta() shifting + * the actual tick count right before performing the multiplication. + */ + while (base_hz_u64 > scaled_hz_u64 * 2 || base_hz_u64 >> 32) { + base_hz_u64 >>= 1; shift--; } - tps32 = (uint32_t)tps64; - while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) { - if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000) - scaled64 >>= 1; + /* Now the shifted base_hz fits in 32 bits. */ + base32 = (u32)base_hz_u64; + + /* + * Next, shift scaled_hz right until it fits in 32 bits, and ensure + * that the shifted base_hz is strictly larger (so that the result of the + * final division also fits in 32 bits). + */ + while (base32 <= scaled_hz_u64 || scaled_hz_u64 >> 32) { + if (scaled_hz_u64 >> 32 || base32 & BIT(31)) + scaled_hz_u64 >>= 1; else - tps32 <<= 1; + base32 <<= 1; shift++; } *pshift = shift; - *pmultiplier = div_frac(scaled64, tps32); + *pmultiplier = div_frac(scaled_hz_u64, base32); } #ifdef CONFIG_X86_64 From d4bbf2a8eb82ec8dfcdace0c56d024d3e83c21ad Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:58 -0700 Subject: [PATCH 090/506] KVM: x86: Remove implicit rdtsc() from kvm_compute_l1_tsc_offset() Let the callers pass the host TSC value in as an explicit parameter. This leaves some fairly obviously stupid code, which is using this function to compare the guest TSC at some *other* time, with the newly-minted TSC value from rdtsc(). Unless it's being used to measure *elapsed* time, that isn't very sensible. In this case, "obviously stupid" is an improvement over being non-obviously so. No functional change intended. Signed-off-by: David Woodhouse Reviewed-by: Paul Durrant Link: https://patch.msgid.link/20260826213303.914988-19-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/msrs.c | 3 ++- arch/x86/kvm/x86.c | 11 ++++++----- arch/x86/kvm/x86.h | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index dd3bb04878ca..37d5bed3529f 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -1631,7 +1631,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (msr_info->host_initiated) { kvm_synchronize_tsc(vcpu, &data); } else if (!vcpu->arch.guest_tsc_protected) { - u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; + u64 adj = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data) - + vcpu->arch.l1_tsc_offset; adjust_tsc_offset_guest(vcpu, adj); vcpu->arch.ia32_tsc_adjust_msr += adj; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index fb585470746e..eecc3aa28f28 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1155,11 +1155,12 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio) return _tsc; } -u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) +u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, + u64 target_tsc) { u64 tsc; - tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio); + tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); return target_tsc - tsc; } @@ -1321,7 +1322,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) bool synchronizing = false; raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); - offset = kvm_compute_l1_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); ns = get_kvmclock_base_ns(); elapsed = ns - kvm->arch.last_tsc_nsec; @@ -1370,7 +1371,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) } else { u64 delta = nsec_to_cycles(vcpu, elapsed); data += delta; - offset = kvm_compute_l1_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); } matched = true; } @@ -2627,7 +2628,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) mark_tsc_unstable("KVM discovered backwards TSC"); if (kvm_check_tsc_unstable()) { - u64 offset = kvm_compute_l1_tsc_offset(vcpu, + u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), vcpu->arch.last_guest_tsc); kvm_vcpu_write_tsc_offset(vcpu, offset); if (!vcpu->arch.guest_tsc_protected) diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 0f5919b092e4..1d9a66048b01 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -335,7 +335,8 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio); u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc); u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier); u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier); -u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc); +u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, + u64 target_tsc); void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset); static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, From ef7d809987c9c36d2b50ae0f0feb07251ff98cf8 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 26 Aug 2026 14:32:59 -0700 Subject: [PATCH 091/506] KVM: x86: Use kernel timekeeping snapshots for getting kvmclock time since boot Replace the KVM-private vgettsc()+do_kvmclock_base() timekeeping reimplementation with calls to the recently crafted, generic ktime_get_snapshot_id() interface. This is the first step towards dropping KVM's homebrewed implementation entirely (do_monotonic() and do_realtime() will be converted in the near future). As with KVM's implementation, the snapshot provides both the system time and the raw_cycles (TSC), atomically paired using a sequence counter. The equivalents to vgettsc()'s TSC and HVCLOCK modes respectively are if the clocksource itself is TSC (cs_id == CSID_X86_TSC) and if the underlying hardware clocksource is TSC (hw_csid == CSID_X86_TSC). In the Hyper-V case, i.e. hw_csid == CSID_X86_TSC, if the clocksource couldn't provide a raw hardware counter value, treat the clock not being based on TSC, which which is equivalent to vgettsc() returning VDSO_CLOCKMODE_NONE. Unlike KVM's current implementation, don't include offs_boot in the atomically-acquired tuple as there's simply no need to do so: the time since boot only changes at boot (duh) and at suspend/resume boundaries. Unless processes aren't being frozen/thawed before/after suspend/resume, which would completely break suspend/resume, TK_OFFS_BOOT can't change while kvm_get_time_and_clockread() is running. And if KVM does somehow try to take a snapshot during suspend, timekeeping core will WARN and refuse to provide the snapshot. This is a step towards eliminating the pvclock_gtod_data private copy of timekeeping state and the associated notifier callback. Signed-off-by: David Woodhouse [sean: separate from other conversions, massage changelog accordingly] Link: https://patch.msgid.link/20260826213303.914988-20-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 57 ++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index eecc3aa28f28..5746a77942de 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -35,6 +35,7 @@ #include "smm.h" #include +#include #include #include #include @@ -1437,29 +1438,6 @@ static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, return v * clock->mult; } -/* - * As with get_kvmclock_base_ns(), this counts from boot time, at the - * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot). - */ -static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ns = gtod->raw_clock.base_cycles; - ns += vgettsc(>od->raw_clock, tsc_timestamp, &mode); - ns >>= gtod->raw_clock.shift; - ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot)); - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - *t = ns; - - return mode; -} - /* * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with * no boot time offset. @@ -1504,6 +1482,29 @@ static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) return mode; } +static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap, + u64 *tsc_timestamp) +{ + /* + * ktime_get_snapshot_id() cannot fail for standard clock IDs + * (only for invalid/aux clocks or during suspend, with a WARN). + */ + if (!snap->valid) + return false; + + if (snap->cs_id == CSID_X86_TSC) { + *tsc_timestamp = snap->cycles; + return true; + } + + if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) { + *tsc_timestamp = snap->hw_cycles; + return true; + } + + return false; +} + /* * Calculates the kvmclock_base_ns (CLOCK_MONOTONIC_RAW + boot time) and * reports the TSC value from which it do so. Returns true if host is @@ -1511,12 +1512,14 @@ static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) */ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns, - tsc_timestamp)); + *kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT)); + return true; } /* From 0a0ab23b05f031ee849357fe0a5e1f667b46277e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 14:33:00 -0700 Subject: [PATCH 092/506] KVM: x86: Use kernel timekeeping snapshot for monotonic clock Replace the KVM-private vgettsc()+/do_monotonic() timekeeping reimplementation with calls to the recently crafted, generic ktime_get_snapshot_id() interface. As noted previously, the snapshot provides both the system time and the raw_cycles (TSC), atomically paired using a sequence counter. Signed-off-by: David Woodhouse [sean: separate from other conversions] Link: https://patch.msgid.link/20260826213303.914988-21-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5746a77942de..f0955f7baf6a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1438,29 +1438,6 @@ static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, return v * clock->mult; } -/* - * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with - * no boot time offset. - */ -static int do_monotonic(s64 *t, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ns = gtod->clock.base_cycles; - ns += vgettsc(>od->clock, tsc_timestamp, &mode); - ns >>= gtod->clock.shift; - ns += ktime_to_ns(gtod->clock.offset); - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - *t = ns; - - return mode; -} - static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) { struct pvclock_gtod_data *gtod = &pvclock_gtod_data; @@ -1528,12 +1505,14 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) */ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_monotonic(kernel_ns, - tsc_timestamp)); + *kernel_ns = ktime_to_ns(snap.systime); + return true; } /* From d7841275e1fb4ad1e1877e56725ebda5b6e3143d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 14:33:01 -0700 Subject: [PATCH 093/506] KVM: x86: Use kernel timekeeping snapshot to get walltime+TSC Replace the KVM-private vgettsc()+/do_do_realtime() timekeeping reimplementation with calls to the recently crafted, generic ktime_get_snapshot_id() interface. As noted previously, the snapshot provides both the system time and the raw_cycles (TSC), atomically paired using a sequence counter. With great pleasure, delete the now unused read_tsc() and vgettsc() Signed-off-by: David Woodhouse [sean: separate from other conversions, express joy at vgettsc()'s demise] Link: https://patch.msgid.link/20260826213303.914988-22-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 85 ++++------------------------------------------ 1 file changed, 6 insertions(+), 79 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f0955f7baf6a..4d612231bcf2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1383,82 +1383,6 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) #ifdef CONFIG_X86_64 -static u64 read_tsc(void) -{ - u64 ret = (u64)rdtsc_ordered(); - u64 last = pvclock_gtod_data.clock.cycle_last; - - if (likely(ret >= last)) - return ret; - - /* - * GCC likes to generate cmov here, but this branch is extremely - * predictable (it's just a function of time and the likely is - * very likely) and there's a data dependence, so force GCC - * to generate a branch instead. I don't barrier() because - * we don't actually need a barrier, and if this function - * ever gets inlined it will generate worse code. - */ - asm volatile (""); - return last; -} - -static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, - int *mode) -{ - u64 tsc_pg_val; - long v; - - switch (clock->vclock_mode) { - case VDSO_CLOCKMODE_HVCLOCK: - if (hv_read_tsc_page_tsc(hv_get_tsc_page(), - tsc_timestamp, &tsc_pg_val)) { - /* TSC page valid */ - *mode = VDSO_CLOCKMODE_HVCLOCK; - v = (tsc_pg_val - clock->cycle_last) & - clock->mask; - } else { - /* TSC page invalid */ - *mode = VDSO_CLOCKMODE_NONE; - } - break; - case VDSO_CLOCKMODE_TSC: - *mode = VDSO_CLOCKMODE_TSC; - *tsc_timestamp = read_tsc(); - v = (*tsc_timestamp - clock->cycle_last) & - clock->mask; - break; - default: - *mode = VDSO_CLOCKMODE_NONE; - } - - if (*mode == VDSO_CLOCKMODE_NONE) - *tsc_timestamp = v = 0; - - return v * clock->mult; -} - -static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ts->tv_sec = gtod->wall_time_sec; - ns = gtod->clock.base_cycles; - ns += vgettsc(>od->clock, tsc_timestamp, &mode); - ns >>= gtod->clock.shift; - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return mode; -} - static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap, u64 *tsc_timestamp) { @@ -1525,11 +1449,14 @@ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_REALTIME, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp)); + *ts = ktime_to_timespec64(snap.systime); + return true; } #endif From b59a87c1ba3c347e8aec5f932eb7abf13064fd62 Mon Sep 17 00:00:00 2001 From: Chengfeng Ye Date: Wed, 26 Aug 2026 02:39:24 +0800 Subject: [PATCH 094/506] KVM: x86: Take SRCU in kvm_zap_gfn_range() kvm_zap_gfn_range() walks memslots and rmaps and may drop mmu_lock to reschedule. Callers that do not already hold kvm->srcu (or slots_lock) can race with memslot deletion: synchronize_srcu_expedited() does not wait, kvm_free_memslot() frees the old slot and its rmap, and the zap resumes on freed memory. The VFIO noncoherent-DMA path hits this by zapping the entire GPA space without SRCU. KASAN reported: BUG: KASAN: vmalloc-out-of-bounds in slot_rmap_walk_next+0x82/0x1c0 Read of size 8 at addr ffffc900005c1008 Call Trace: slot_rmap_walk_next+0x82/0x1c0 __kvm_rmap_zap_gfn_range+0x17a/0x280 kvm_zap_gfn_range+0x2a6/0x6a0 kvm_vfio_set_attr+0x576/0x770 kvm_device_ioctl+0x1ff/0x3b0 Take SRCU inside kvm_zap_gfn_range() so every caller is covered. Nesting with an existing kvm->srcu critical section is fine; the helper uses a local index. Drop the now-redundant SRCU pair from __kvm_set_or_clear_apicv_inhibit(). Fixes: 362ff6dca541 ("KVM: x86/mmu: Zap KVM TDP when noncoherent DMA assignment starts/stops") Cc: stable@vger.kernel.org Signed-off-by: Chengfeng Ye Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260825183924.237727-1-nicoyip.dev@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 2 ++ arch/x86/kvm/x86.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..87518f238e5e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7117,6 +7117,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) if (WARN_ON_ONCE(gfn_end <= gfn_start)) return; + guard(srcu)(&kvm->srcu); + write_lock(&kvm->mmu_lock); kvm_mmu_invalidate_start(kvm); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..15721ce659bb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7925,10 +7925,8 @@ void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, kvm->arch.apicv_inhibit_reasons = new; if (new) { unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); - int idx = srcu_read_lock(&kvm->srcu); kvm_zap_gfn_range(kvm, gfn, gfn+1); - srcu_read_unlock(&kvm->srcu, idx); } } else { kvm->arch.apicv_inhibit_reasons = new; From 355323a4dc945d0176f54c324aa44943820f932f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:42:11 -0700 Subject: [PATCH 095/506] KVM: x86/mmu: Reload MMU on *every* page pre-fault attempt/iteration Reload the MMU (which is a nop if the MMU doesn't need to be reloaded) on every attempt to pre-fault a guest page, i.e. when the page fault path signals that the caller should retry. If the synchronize_srcu_expedited() in kvm_invalidate_memslot() completes before kvm_vcpu_pre_fault_memory() grabs SRCU, but kvm_mmu_reload() in the pre-fault path completes before kvm_invalidate_memslot() triggers x86's "fast zap all", then the pre-fault task will reach kvm_tdp_page_prefault() with an invalid root. Attempting to fault-in memory with an invalid root ultimately puts kvm_tdp_page_prefault() into an infinite (breakable) retry loop, which manifests most obviously as a hang in the pre_fault_memory_test selftest, but also eventually causes RCU (SRCU?) to complain. INFO: rcu_tasks detected stalls on tasks: 000000000cda47bd: .. nvcsw: 6/6 holdout: 1 idle_cpu: -1/25 task:pre_fault_memor state:R running task stack:12696 pid:95588 tgid:95588 ppid:95584 task_flags:0x400000 flags:0x00080801 Call Trace: lock_release+0x4e/0x320 __get_user_pages+0x546/0xcd0 up_read+0x1b/0x30 get_user_pages_unlocked+0xee/0x350 hva_to_pfn+0xd3/0x3d0 [kvm] lock_release+0x4e/0x320 xa_load+0x5c/0x170 xa_load+0x14c/0x170 __kvm_faultin_pfn+0xd9/0x130 [kvm] lock_acquire+0x65/0x2b0 lock_release+0x4e/0x320 kvm_mmu_faultin_pfn+0x1e1/0x690 [kvm] gup_fast_fallback+0x63e/0xdf0 kvm_tdp_page_fault+0xeb/0x140 [kvm] kvm_mmu_do_page_fault+0x12e/0x200 [kvm] kvm_arch_vcpu_pre_fault_memory+0x16e/0x200 [kvm] kvm_vcpu_pre_fault_memory+0xc1/0x1f0 [kvm] kvm_vcpu_pre_fault_memory+0x116/0x1f0 [kvm] kvm_vcpu_ioctl+0x3a4/0x6b0 [kvm] clockevents_program_event+0x5d/0x170 __se_sys_ioctl+0x6d/0xb0 entry_SYSCALL_64_after_hwframe+0x4b/0x53 do_syscall_64+0x10a/0x480 __irq_exit_rcu+0x8e/0x140 entry_SYSCALL_64_after_hwframe+0x4b/0x53 Fixes: 6e01b7601dfe ("KVM: x86: Implement kvm_arch_vcpu_pre_fault_memory()") Reviewed-by: Rick Edgecombe Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260826164214.756512-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 87518f238e5e..786ddbc30dd5 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5061,6 +5061,10 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; + r = kvm_mmu_reload(vcpu); + if (r) + return r; + cond_resched(); r = kvm_mmu_do_page_fault(vcpu, gpa, error_code, true, NULL, level); } while (r == RET_PF_RETRY); @@ -5101,14 +5105,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa))) return -EINVAL; - /* - * reload is efficient when called repeatedly, so we can do it on - * every iteration. - */ - r = kvm_mmu_reload(vcpu); - if (r) - return r; - direct_bits = 0; if (kvm_arch_has_private_mem(vcpu->kvm) && kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa))) From 698b8c88e15914a88f686b552c14c0c5caff3cc7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:42:12 -0700 Subject: [PATCH 096/506] KVM: x86/mmu: Harden "map private PFN" against unexpected root invalidation Move kvm_tdp_mmu_map_private_pfn()'s reload of the MMU into its tight loop so that an unexpected root invalidation has a better chance of being handled gracefully, even though it should be impossible for the vCPU's root to be invalidated after the initial reload. As is, encountering an invalid root is *guaranteed* to put the task into an infinite loop (albeit a breakable loop that honors NEED_RESCHED). Add a WARN to try and detect bugs that break KVM's expectations, along with a comment to explain why it should be impossible for the root to be invalidated. Note, the loop in question doesn't actually check for a stale page fault, i.e. likely won't detect an invalid loop in the first place. That bug will be addressed shortly. Cc: Kai Huang Cc: Yan Zhao Cc: Rick Edgecombe Link: https://patch.msgid.link/20260826164214.756512-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 786ddbc30dd5..f8d29fa2e538 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5209,10 +5209,6 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn)) return -EPERM; - r = kvm_mmu_reload(vcpu); - if (r) - return r; - r = mmu_topup_memory_caches(vcpu, false); if (r) return r; @@ -5224,10 +5220,22 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; + r = kvm_mmu_reload(vcpu); + if (r) + return r; + cond_resched(); guard(read_lock)(&kvm->mmu_lock); + /* + * Because slots_lock is held, it should be impossible for *any* + * roots to be invalidated after the initial MMU reload. WARN, + * but continue on; the above MMU reload will do the right thing + * if the current root is actually invalid. + */ + WARN_ON_ONCE(kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)); + r = kvm_tdp_mmu_map(vcpu, &fault); } while (r == RET_PF_RETRY); From 6415d9c4ee19be3816c89974562d67e2c00c8638 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:42:13 -0700 Subject: [PATCH 097/506] KVM: x86/mmu: Top-up memory caches when retrying "map private PFN" When mapping a private PFN in TDX's post-populate callback, top-up the memory caches on every attempt to map the PFN to harden against bugs in the map flow that could consume cache entries even if mapping ultimately fails. E.g. as pointed out by Sashiko, the in-progress Dynamic PAMT support could consume PAMT cache entries on TDX-Module lock contention. Harden KVM even though consuming an entry on failure is considered a KVM bug. Retry should only be encountered if KVM is buggy (the locks held by the sole call path will prevent retries from being needed due to TDX-specific details, and memory can be faulted in only once the VM is TD_STATE_RUNNABLE, and KVM_TDX_INIT_MEM_REGION is only usable if the VM is *not* TD_STATE_RUNNABLE), top-up is "free" if there's no work to be done, and populating a TDX guest's memory is a slow path, i.e. there's no meaningful downside to the hardening. Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260718061050.E17B01F000E9@smtp.kernel.org Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260826164214.756512-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f8d29fa2e538..45522105a472 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5209,10 +5209,6 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn)) return -EPERM; - r = mmu_topup_memory_caches(vcpu, false); - if (r) - return r; - do { if (signal_pending(current)) return -EINTR; @@ -5224,6 +5220,10 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (r) return r; + r = mmu_topup_memory_caches(vcpu, false); + if (r) + return r; + cond_resched(); guard(read_lock)(&kvm->mmu_lock); From f7a3be22761953f4a1f754786b7a35b4c5ea6824 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:42:14 -0700 Subject: [PATCH 098/506] KVM: x86/mmu: Add sanity check to detect stale page faults in "map private PFN" Harden the "map private PFN" flow against potentially-fatal bugs or future KVM changes by checking for a stale "fault" prior to actually mapping the PFN into the guest. While it should be impossible for the "page fault" to become stale, the sanity check is cheap, whereas a broken assumption would have a high probability of leading to a guest-expoitable use-after-free. Snapshot the invalidation sequence after acquiring mmu_lock to avoid false positives, even though doing so completely voids anys and all protection against unexpected invalidations. Pretty much the entire point of kvm_tdp_mmu_map_private_pfn() is that it allows mapping a PFN that was gifted by the caller, i.e. the caller would have to mess up its one and only responsibility. Link: https://patch.msgid.link/20260826164214.756512-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 45522105a472..319125faf9fa 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5236,6 +5236,18 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) */ WARN_ON_ONCE(kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)); + /* + * Snapshot the invalidation sequence counter after acquiring + * mmu_lock, as guest_memfd guarantees the validity of the pfn, + * i.e. any concurrent invalidations are guaranteed to be + * irrelevant. + */ + fault.mmu_seq = vcpu->kvm->mmu_invalidate_seq; + if (is_page_fault_stale(vcpu, &fault)) { + r = RET_PF_RETRY; + continue; + } + r = kvm_tdp_mmu_map(vcpu, &fault); } while (r == RET_PF_RETRY); From df269c8e097f306b09d9f371a60a936e2530c10d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 14:17:59 -0700 Subject: [PATCH 099/506] KVM: x86/mmu: Always guard rmaps with mmu_lock on PREEMPT_RT=y kernels For all intents and purposes, revert KVM's ability to walk rmaps outside of mmu_lock when running on a realtime (PREEMPT_RT=y) kernel. I.e. don't use a non-sleepable bit-spinlock to protect rmap entries, as realtime kernels are highly unlikely to benefit from increased aging throughput and reduced jitter for memory-overcommitted nested VMs, whereas using a non-sleepable lock is currently buggy and goes against the spirit of realtime kernels. Because KVM's rmap locks are hand-crafted bit-spinlocks, preemption must be disabled before acquiring the lock, otherwise a preempted lock holder will result in all other walkers of the locked rmap to spin and wait, with no tracked owner for PI to boost. For non-RT kernels, acquiring mmu_lock suffices, as mmu_lock is a non-sleepable rwlock. But on RT, where mmu_lock becomes sleepable, preemption is left enabled for rmap writers: WARNING: arch/x86/kvm/mmu/mmu.c:920 at __kvm_rmap_lock+0x1a7/0x1e0 [kvm], CPU#16: vmx_apic_update/3708 CPU: 16 UID: 0 PID: 3708 Comm: vmx_apic_update Not tainted 7.2.0-rc7 #52 PREEMPT_{RT,LAZY} RIP: 0010:__kvm_rmap_lock+0x1a7/0x1e0 [kvm] Call Trace: pte_list_add+0x67/0x4d0 [kvm] __link_shadow_page+0x249/0x480 [kvm] ept_fetch+0x4d5/0x1220 [kvm] ept_page_fault+0x60b/0x850 [kvm] kvm_mmu_do_page_fault+0x252/0x690 [kvm] Alternatively, KVM could manually disable preemption when grabbing an rmap lock, but as above, that isn't what RT kernels generally want, and it's actually more complex to implement (cleanly). To not completely lose the scaling advantage of per-rmap locks, take mmu_lock for read in the aging path, i.e. allow multiple concurrent aging tasks, as the aging code needs to use atomic SPTE accesses no matter what, i.e. no extra code/work is required to guard against concurrent aging of SPTEs. Reported-by: David Woodhouse Closes: https://lore.kernel.org/all/8d47b43e1829ac92703723e6a1a4afc7a2eaacb5.camel@infradead.org Fixes: 4834eaded91e ("KVM: x86/mmu: Add infrastructure to allow walking rmaps outside of mmu_lock") Reviewed-by: David Woodhouse Tested-by: David Woodhouse Reviewed-by: James Houghton Link: https://patch.msgid.link/20260902211759.2700289-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 48 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 319125faf9fa..855c4f030f4c 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -895,6 +895,7 @@ static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu */ #define KVM_RMAP_MANY BIT(0) +#ifndef CONFIG_PREEMPT_RT /* * rmaps and PTE lists are mostly protected by mmu_lock (the shadow MMU always * operates with mmu_lock held for write), but rmaps can be walked without @@ -1008,7 +1009,8 @@ static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head) * actual locking is the same, but the caller is disallowed from modifying the * rmap, and so the unlock flow is a nop if the rmap is/was empty. */ -static unsigned long kvm_rmap_lock_readonly(struct kvm_rmap_head *rmap_head) +static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) { unsigned long rmap_val; @@ -1032,6 +1034,35 @@ static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head, __kvm_rmap_unlock(rmap_head, old_val); preempt_enable(); } +#else +static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head) +{ + return atomic_long_read(&rmap_head->val); +} +static unsigned long kvm_rmap_lock(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) +{ + lockdep_assert_held_write(&kvm->mmu_lock); + return kvm_rmap_get(rmap_head); +} + +static void kvm_rmap_unlock(struct kvm *kvm, + struct kvm_rmap_head *rmap_head, + unsigned long new_val) +{ + atomic_long_set_release(&rmap_head->val, new_val); +} + +static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) +{ + lockdep_assert_held_read(&kvm->mmu_lock); + return kvm_rmap_get(rmap_head); +} + +static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head, + unsigned long old_val) { } +#endif /* * Returns the number of pointers in the rmap chain, not counting the new one. @@ -1745,11 +1776,24 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, gfn_t gfn; int level; + BUILD_BUG_ON(!IS_ENABLED(CONFIG_KVM_MMU_LOCKLESS_AGING)); + + /* + * For realtime kernels, do aging under mmu_lock (per-rmap locking is + * compiled out), as realtime deployments are unlikely to benefit from + * increased aging throughput and reduced jitter for memory-overcommitted + * nested VMs, whereas keeping preemption enabled is extremely valuable + * (mmu_lock becomes a sleepable lock on realtime kernels). + */ +#ifdef CONFIG_PREEMPT_RT + guard(read_lock)(&kvm->mmu_lock); +#endif + for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { for (gfn = range->start; gfn < range->end; gfn += KVM_PAGES_PER_HPAGE(level)) { rmap_head = gfn_to_rmap(gfn, level, range->slot); - rmap_val = kvm_rmap_lock_readonly(rmap_head); + rmap_val = kvm_rmap_lock_readonly(kvm, rmap_head); for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) { if (!is_accessed_spte(old_spte)) From 27f02d851410ece239f4024aae87143ed00f4354 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Aug 2026 12:19:45 -0700 Subject: [PATCH 100/506] KVM: x86/mmu: Bug the VM if KVM attempts to unsync an upper-level shadow page Bug the VM and skip marking a shadow page unsync if KVM attempts to unsync an upper-level shadow page, as marking an upper-level SP unsync is all but guaranteed to be far worse than terminating the VM. E.g. sync_spte() (rightly) assumes SPTEs in the SP are leaf SPTEs and so calls drop_spte() instead of drop_parent_pte(), which can effectively crash the host via when running with CONFIG_BUG_ON_DATA_CORRUPTION=y due to there not being an rmap entry for the SPTE. Simply not marking the SP unsync, i.e. letting the VM continue on, is equally dangerous for the guest as it means KVM's shadow MMU is using stale information, e.g. could unintentionally corrupt the guest. Cc: Jinu Kim Link: https://patch.msgid.link/20260806191945.4192732-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 855c4f030f4c..2bf6ed878589 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3065,6 +3065,9 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, if (prefetch) return -EEXIST; + if (KVM_BUG_ON(sp->role.level != PG_LEVEL_4K, kvm)) + continue; + /* * TDP MMU page faults require an additional spinlock as they * run with mmu_lock held for read, not write, and the unsync @@ -3088,7 +3091,6 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, continue; } - WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K); kvm_unsync_page(kvm, sp); } if (locked) From c5991239b148d40d4981901968b3a6a838ff517b Mon Sep 17 00:00:00 2001 From: Chaithanya Lagisetty Date: Tue, 8 Sep 2026 05:37:22 +0000 Subject: [PATCH 101/506] x86/svm: Avoid sign extension of SVM_EVTINJ_VALID in 64-bit expressions SVM_EVTINJ_VALID is defined as (1 << 31), i.e. as an int with the sign bit set. Whenever it is used in a 64-bit expression the int is sign extended, so bits 63:32 end up set as well. Two 64-bit users are affected. verify_exception_info() uses the macro as a mask against a 64-bit value when it sanity checks hypervisor supplied exception information: u64 info = ghcb->save.sw_exit_info_2; ... if ((info & SVM_EVTINJ_VALID) && The mask is 0xffffffff80000000, so the check also passes when any bit of the error code in 63:32 is set while the valid bit itself is clear. svm_vmgexit_inject_exception() builds the value that KVM hands to an SEV-ES guest in GHCB SW_EXITINFO2: u64 data = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT | vector; For vector == X86_TRAP_GP this yields 0xffffffff8000030d instead of 0x000000008000030d. The field uses the EVENTINJ format, which holds the error code in bits 63:32, so KVM asks the guest to inject #GP with an error code of 0xffffffff. Linux guests only consume the error code when SVM_EVTINJ_VALID_ERR is set, which it is not here, so they are not affected in practice, but the value KVM writes is not the one it intends. Use BIT() instead, as is already done for most single bit definitions in this header. Every other user applies the macro to a u32, where the resulting bits are unchanged. Opportunistically give SVM_EVTINJ_VALID_ERR the same treatment, there's zero reason to open code BIT(). Fixes: 597cfe48212a ("x86/boot/compressed/64: Setup a GHCB-based VC Exception handler") Fixes: c3392d0ab714 ("KVM: SVM: Provide helpers to set the error code") Signed-off-by: Chaithanya Lagisetty Link: https://patch.msgid.link/20260908053722.3917934-1-nagachaithanya9911@gmail.com [sean: use BIT() for SVM_EVTINJ_VALID_ERR as well] Signed-off-by: Sean Christopherson --- arch/x86/include/asm/svm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index aa63431ba92c..e883241b5e1e 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -637,8 +637,8 @@ static inline void __unused_size_checks(void) #define SVM_EVTINJ_TYPE_EXEPT (3 << SVM_EVTINJ_TYPE_SHIFT) #define SVM_EVTINJ_TYPE_SOFT (4 << SVM_EVTINJ_TYPE_SHIFT) -#define SVM_EVTINJ_VALID (1 << 31) -#define SVM_EVTINJ_VALID_ERR (1 << 11) +#define SVM_EVTINJ_VALID BIT(31) +#define SVM_EVTINJ_VALID_ERR BIT(11) #define SVM_EVTINJ_RESERVED_BITS ~(SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | \ SVM_EVTINJ_VALID_ERR | SVM_EVTINJ_VALID) From 7d2a727f44d833c5f5771e61fc721359f01a0b6f Mon Sep 17 00:00:00 2001 From: Zeng Chi Date: Thu, 27 Aug 2026 19:22:00 +0800 Subject: [PATCH 102/506] KVM: Mask off the address space ID in kvm_set_internal_memslot() check kvm_set_internal_memslot() is supposed to reject memslot IDs that belong to userspace, i.e. IDs below KVM_USER_MEM_SLOTS, but it compares the raw "slot" field, which packs both the address space ID and the memslot ID (as_id << 16 | id). For any address space other than 0, the packed value is always >= 65536, which is larger than KVM_USER_MEM_SLOTS, and so the sanity check is a nop regardless of the actual memslot ID. E.g. x86 sets internal memslots for every address space, including SMM, so the check would silently accept a userspace memslot ID for the SMM address space while rejecting the same ID for the normal address space. Extract the memslot ID the same way kvm_vm_ioctl_set_memory_region() and kvm_set_memory_region() do, so that the check covers all address spaces. No in-tree caller trips the check; this only makes the sanity check effective for all address spaces. Fixes: 156bffdb2b49 ("KVM: Add a dedicated API for setting KVM-internal memslots") Signed-off-by: Zeng Chi Link: https://patch.msgid.link/20260827112200.866476-1-zeng_chi911@163.com Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 108d42c5c1d6..d9da8b51614a 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2138,7 +2138,7 @@ static int kvm_set_memory_region(struct kvm *kvm, int kvm_set_internal_memslot(struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem) { - if (WARN_ON_ONCE(mem->slot < KVM_USER_MEM_SLOTS)) + if (WARN_ON_ONCE((u16)mem->slot < KVM_USER_MEM_SLOTS)) return -EINVAL; if (WARN_ON_ONCE(mem->flags)) From 2616f497506a772db53e7bb1024cde50b4c165e0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 11:44:49 -0700 Subject: [PATCH 103/506] KVM: x86: Zero reserved synic flags when delivering message to avoid stack leak Ensure all reserved flags are zeroed when setting the msg_pending flag as part of SynIC message delivery, as setting only msg_pending can leak seven bits of kernel stack data to the guest. E.g. gcc typically uses a bitwise-OR to set the msg_pending flag, without initializing the on-stack variable. Precisely zero the flags, e.g. as opposed to zeroing the entire structure, to make it somewhat more obvious that the flags *need* to be zeroed. E.g. it would be quite easy to misread the initial read of guest memory and think that zeroing the entire structure is completely superfluous. Fixes: 3a0e7731724f ("x86: kvm: hyperv: simplify SynIC message delivery") Cc: stable@vger.kernel.org Reported-by: Stefan Teodorescu Reviewed-by: Vitaly Kuznetsov Link: https://patch.msgid.link/20260826184449.814130-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 604651cb2739..8d2669d8ef34 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -802,7 +802,9 @@ static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, if (no_retry) return 0; + hv_hdr.message_flags.asu8 = 0; hv_hdr.message_flags.msg_pending = 1; + r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_flags, msg_off + From 926a9ea9bdf0b09c1cd8e896b251152d892f2566 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Aug 2026 10:14:00 -0700 Subject: [PATCH 104/506] KVM: x86: Track kvm_vcpu_arch.pending_ioapic_eoi as a u8 to drop dead BUG_ON() Track the vector associated with KVM_EXIT_IOAPIC_EOI as a u8, as the vector is zero-initialized and is only ever set to a legal vector. I.e. unlike many of the other vector variables in KVM, it can't be -1 (and can't be greater than 255, though that should hold true for all vectors in KVM). Drop the now fully dead BUG_ON(), which aside from being completely unnecessary, was also useless, as it wouldn't guard against a negative value, i.e. wouldn't detect KVM's magic -1 value. For all intents and purposes, no functional change intended. Suggested-by: Paolo Bonzini Reviewed-by: Kai Huang Reviewed-by: Paolo Bonzini Link: https://patch.msgid.link/20260806171400.4119301-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/lapic.c | 4 ++-- arch/x86/kvm/lapic.h | 2 +- arch/x86/kvm/vmx/vmx.c | 3 +-- arch/x86/kvm/x86.c | 1 - 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3af7395c2430..14865d36fa49 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -974,7 +974,7 @@ struct kvm_vcpu_arch { bool pv_unhalted; } pv; - int pending_ioapic_eoi; + u8 pending_ioapic_eoi; int pending_external_vector; int highest_stale_pending_ioapic_eoi; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index e1f3cea14765..ecf611845fb1 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1552,7 +1552,7 @@ static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); } -static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) +static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, u8 vector) { int __maybe_unused trigger_mode; @@ -1622,7 +1622,7 @@ static int apic_set_eoi(struct kvm_lapic *apic) * this interface assumes a trap-like exit, which has already finished * desired side effect including vISR and vPPR update. */ -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector) { struct kvm_lapic *apic = vcpu->arch.apic; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index bd1098c89d99..db071d0b172d 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -177,7 +177,7 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data); void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset); -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector); +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector); int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr); void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..58078e50c4de 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5888,10 +5888,9 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { unsigned long exit_qualification = vmx_get_exit_qual(vcpu); - int vector = exit_qualification & 0xff; /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ - kvm_apic_set_eoi_accelerated(vcpu, vector); + kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff); return 1; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..28172424c502 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8151,7 +8151,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_NMI, vcpu)) process_nmi(vcpu); if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { - BUG_ON(vcpu->arch.pending_ioapic_eoi > 255); if (test_bit(vcpu->arch.pending_ioapic_eoi, vcpu->arch.ioapic_handled_vectors)) { vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; From 7ed619909773d1cc2d026b63b003010693ef0c5f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:09:30 -0700 Subject: [PATCH 105/506] KVM: x86/mmu: Use KVM's max TDP level to determine need for 32-bit TDP root When checking to see if KVM needs to allocate a TDP PAE root that's 32-bit addressable, query KVM's overall max TDP level, not the vCPU-specific TDP level, as the logic is specific to using NPT on 32-bit hosts. Querying the vCPU's alleged TDP level is flawed and confusing, as KVM selects between 4-level vs. 5-level based on the guest's MAXPHYADDR, and MAXPHYADDR isn't yet configured (via CPUID) when the vCPU is being created. I.e. as is, it would *appear* that KVM is violating its own rules with respect to changing guest CPUID (see kvm_mmu_after_set_cpuid()). In practice, the flaw is benign as the goal is purely to see if KVM needs to use PAE-paging; whether KVM will use 4-level vs. 5-level is irrelevant. More importantly, avoiding kvm_mmu_get_tdp_level() during vCPU creation will allow hardening KVM's handling of S-EPT mirror root level. For all intents and purposes, no functional change intended. Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260902230932.2760127-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..c9a684151420 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6834,7 +6834,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, struct k * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit * KVM; that horror is handled on-demand by mmu_alloc_special_roots(). */ - if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) + if (tdp_enabled && kvm_mmu_get_max_tdp_level() > PT32E_ROOT_LEVEL) return 0; page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); From cc8a8f2a90b45cd4c8509db7bc61c2ac9dbd2cf2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:09:31 -0700 Subject: [PATCH 106/506] KVM: VMX: Explicitly track TDX VMs' root level instead of guessing it from CPUID Explicitly track the root level for TDX VMs instead of trying to infer the depth of the paging tree based on an individual vCPU's CPUID information. Applying KVM's existing logic to select the root level to TDX is flawed as nothing *requires* userspace to fill in the correct guest.MAXPHYADDR for a vCPU's CPUID. Guessing at the correct root level is also ridiculous given that userspace has already told KVM the root level during TD initialization. Relying on userspace to set the expected/correct CPUID lets a misbehaving userspace trip the KVM_BUG_ON() in tdx_load_mmu_pgd() by configuring guest CPUID to use an "incorrect" guest.MAXPHYADDR. Don't use kvm_gfn_direct_bits() to infer the mirror root level, as the connection between TDX's one and only "direct" bit and the predetermined root level is a TDX implementation detail. I.e. avoid baking in the assumption that there is exactly one "direct bits", that the one bit is a pivot between normal and mirror roots, and that the pivot bit is the most significant bit of the effective GPA space. For the same reason, set the root level and direct bits in TDX code, i.e. don't provide a helper in the MMU, because from the MMU's perspective, they are two separate concepts. Keep gfn_direct_bits even though it can be trivially derived from mirror_root_level as saving a whole eight bytes per VM is meaningless, keeping the TDX details buried in TDX would require a kvm_x86_ops hook, and the value is queried fairly often and in hot paths. And for the moment, keep the S-bit sanity check in tdx_load_mmu_pgd(), even though it really only needs to ensure the incoming level matches the preconfigured mirror root level. Because KVM manually configures the S-bit location, there's technically a risk that the S-bit location and mirror root level could get out of sync. That can be addressed by more programmatically computing the S-bit, but that doesn't need to be done now. Cc: Rick Edgecombe Cc: Xiaoyao Li Cc: Kai Huang Cc: Yan Zhao Fixes: 20d913729c11 ("KVM: x86/mmu: Taking guest pa into consideration when calculate tdp level") Reviewed-by: Rick Edgecombe Tested-by: Yan Zhao Tested-by: Binbin Wu Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260902230932.2760127-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/cpuid.c | 14 -------------- arch/x86/kvm/cpuid.h | 1 - arch/x86/kvm/mmu/mmu.c | 19 +++++++++++-------- arch/x86/kvm/vmx/tdx.c | 14 ++++++++++++-- 5 files changed, 24 insertions(+), 25 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 14865d36fa49..e369faf3c285 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1407,6 +1407,7 @@ struct kvm_arch { struct kvm_mmu_memory_cache split_desc_cache; gfn_t gfn_direct_bits; + int mirror_root_level; /* * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 851f151efb35..bb68a021cfaf 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -483,20 +483,6 @@ int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu) return 36; } -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu) -{ - struct kvm_cpuid_entry2 *best; - - best = kvm_find_cpuid_entry(vcpu, 0x80000000); - if (!best || best->eax < 0x80000008) - goto not_found; - best = kvm_find_cpuid_entry(vcpu, 0x80000008); - if (best) - return (best->eax >> 16) & 0xff; -not_found: - return 0; -} - /* * This "raw" version returns the reserved GPA bits without any adjustments for * encryption technologies that usurp bits. The raw mask should be used if and diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 8d863f45585d..46bfe8699e67 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -68,7 +68,6 @@ void __init kvm_init_xstate_sizes(void); u32 xstate_required_size(u64 xstate_bv, bool compacted); int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu); u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu); static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index c9a684151420..d43b8b6c3b17 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5953,19 +5953,22 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu, static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) { - int maxpa; - - if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM) - maxpa = cpuid_query_maxguestphyaddr(vcpu); - else - maxpa = cpuid_maxphyaddr(vcpu); - /* tdp_root_level is architecture forced level, use it if nonzero */ if (tdp_root_level) return tdp_root_level; + /* + * If the VM has mirror roots, then the root level is predefined as the + * mirror root (and by extension the normal root) needs to match the + * root level that was configured for the external page tables that are + * being mirrored by KVM. + */ + if (kvm_has_mirrored_tdp(vcpu->kvm) && + !WARN_ON_ONCE(!vcpu->kvm->arch.mirror_root_level)) + return vcpu->kvm->arch.mirror_root_level; + /* Use 5-level TDP if and only if it's useful/necessary. */ - if (max_tdp_level == 5 && maxpa <= 48) + if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) return 4; return max_tdp_level; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..e6b7da616817 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2760,6 +2760,13 @@ DEFINE_CLASS(tdx_vm_state_guard, tdx_vm_state_guard_t, if (!IS_ERR(_T)) tdx_release_vm_state_locks(_T), tdx_acquire_vm_state_locks(kvm), struct kvm *kvm); +static __always_inline void tdx_set_mirror_root_level(struct kvm *kvm, int level) +{ + BUILD_BUG_ON(level != 4 && level != 5); + + kvm->arch.mirror_root_level = level; +} + static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) { struct kvm_tdx_init_vm __user *user_data = u64_to_user_ptr(cmd->data); @@ -2822,10 +2829,13 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) kvm_tdx->attributes = td_params->attributes; kvm_tdx->xfam = td_params->xfam; - if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) + if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) { kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5; - else + tdx_set_mirror_root_level(kvm, 5); + } else { kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4; + tdx_set_mirror_root_level(kvm, 4); + } kvm_tdx->state = TD_STATE_INITIALIZED; out: From 37e3197c50c824720c3af3118942bda3678f832a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:09:32 -0700 Subject: [PATCH 107/506] KVM: VMX: Drop TDX_SHARED_BIT_PWL_{4,5} and dedup related code Fold the GPA => GFN conversion and bitshift logic for identifying the S-bit given the EPT root level into tdx_set_mirror_root_level() to dedup the math and drop TDX_SHARED_BIT_PWL_{4,5} in the process. In addition to deduping a small amount of code, using the level to compute the S-bit position more or less eliminates the risk of the mirror_root_level and gfn_direct_bits getting out of sync. Opportunistically switch the sanity check in tdx_load_mmu_pgd() to check the mirror root level, not the S-Bit location, now that it's all but impossible for the two things to get out of sync. For all intents and purposes, no functional change intended. Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260902230932.2760127-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index e6b7da616817..a0bc9f818f43 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -56,9 +56,6 @@ bool enable_tdx __ro_after_init; module_param_named(tdx, enable_tdx, bool, 0444); -#define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51)) -#define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47)) - static const struct tdx_sys_info *tdx_sysinfo; void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err) @@ -1609,10 +1606,7 @@ static int handle_tdvmcall(struct kvm_vcpu *vcpu) void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level) { - u64 shared_bit = (pgd_level == 5) ? TDX_SHARED_BIT_PWL_5 : - TDX_SHARED_BIT_PWL_4; - - if (KVM_BUG_ON(shared_bit != kvm_gfn_direct_bits(vcpu->kvm), vcpu->kvm)) + if (KVM_BUG_ON(pgd_level != vcpu->kvm->arch.mirror_root_level, vcpu->kvm)) return; td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa); @@ -2765,6 +2759,7 @@ static __always_inline void tdx_set_mirror_root_level(struct kvm *kvm, int level BUILD_BUG_ON(level != 4 && level != 5); kvm->arch.mirror_root_level = level; + kvm->arch.gfn_direct_bits = gpa_to_gfn(BIT_ULL(level == 4 ? 47 : 51)); } static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) @@ -2829,13 +2824,10 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) kvm_tdx->attributes = td_params->attributes; kvm_tdx->xfam = td_params->xfam; - if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) { - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5; + if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) tdx_set_mirror_root_level(kvm, 5); - } else { - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4; + else tdx_set_mirror_root_level(kvm, 4); - } kvm_tdx->state = TD_STATE_INITIALIZED; out: From 5dc500a354cc8e109df678076ae3488094649468 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:24 -0700 Subject: [PATCH 108/506] KVM: nSVM: Reject KVM_SET_NESTED_STATE if L1 has EFER.LMA=1 && EFER.LME=0 Reject KVM_SET_NESTED_STATE if the incoming L1 host state has what is effectively an impossible EFER combination of LMA=1 but LME=0, i.e. if the state says long mode is active but not enabled. Unlike VMX, SVM doesn't have an explicit consistent check for the illegal combination; presumably hardware simply ignores EFER.LMA if EFER.LME=0. Unfortunately, KVM doesn't ignore EFER.LMA in this case and consumes the illegal state when constructing the shadow MMU for L2. E.g. if userspace also clears CR4.PAE, then kvm_calc_cpu_role() will compute a role with 4 or 5 levels of paging, but shadow_mmu_init_context() will wire up the MMU to use the paging32 template, which maxes out its levels at 2. Note, the "real badness" is effectively the same as what happened with the nVMX bug fixed by commit 112e66017bff ("KVM: nVMX: add missing consistency checks for CR0 and CR4"). Unfortunately, the sanity check added by commit 72e2fb24a0b0 ("KVM: x86/mmu: Bug the VM if a vCPU ends up in long mode without PAE enabled") doesn't work for this case, since L2 state is active at the time of the page fault, but it's L1 that has the bad state. Fixes: cc440cdad5b7 ("KVM: nSVM: implement KVM_GET_NESTED_STATE and KVM_SET_NESTED_STATE") Cc: stable@vger.kernel.org Cc: Yosry Ahmed Reported-by: Stefan Teodorescu Link: https://patch.msgid.link/20260902232028.2767071-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 73f37b050d0a..49fb10ad1f9f 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -2028,6 +2028,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, if (!(save->cr0 & X86_CR0_PG) || !(save->cr0 & X86_CR0_PE) || (save->rflags & X86_EFLAGS_VM) || + ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) || !nested_vmcb_check_save(vcpu, &save_cached, false)) goto out_free; From 1c4c84a643b704e473cb171f7b61359ff29b12a4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:25 -0700 Subject: [PATCH 109/506] KVM: nSVM: Ignore EFER.LMA if EFER.LME=0 when preparing L2 state Force EFER.LMA=0 if EFER.LME=0 when preparing L2 state for VMRUN, i.e. mimic real hardware's behavior of ignoring EFER.LMA if EFER.LME=0. VMRUN unfortunately allows the nonsensical combination, i.e. doesn't fail, but KVM itself has an invariant EFER.LMA can be set et if and only if EFER.LME is set. Breaking that invariant can lead to a variety of issue, particularly in MMU code that keys off EFER.LMA when determining whether to emulate/virtualization 4/5-level paging versus PAE paging. Cc: stable@vger.kernel.org Cc: Yosry Ahmed Link: https://patch.msgid.link/20260902232028.2767071-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 49fb10ad1f9f..23d29597d6bf 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -789,6 +789,10 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED); + /* SVM ignores EFER.LMA if EFER.LME=0 (instead of failing VMRUN). */ + if (!(svm->nested.save.efer & EFER_LME)) + svm->nested.save.efer &= ~EFER_LMA; + svm_set_efer(vcpu, svm->nested.save.efer); svm_set_cr0(vcpu, svm->nested.save.cr0); From 115f9405a2ed6d78ee7ad8436bdd638743f6d1d4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:26 -0700 Subject: [PATCH 110/506] KVM: x86/mmu: Bug the VM if KVM attempts to walk more levels than the MMU has Extend the "EFER.LMA && !CR4.PAE" check, which exists largely to guard against KVM configuring a paging32 MMU with more than 2 levels of paging, with a very explicit check for exactly that: that KVM isn't trying to walk more levels of paging than the MMU template provides. I.e. harden KVM against all bugs that would cause KVM to generates accesses beyond the bounds of guest_walker's arrays, regardless of how KVM ended up with the misconfigured MMU. Note, don't use w->cpu_role.base.level directly as the paging64 template only provides two levels of page tables for PAE paging on 32-bit hosts, and handles the third level by manually emulating the PDPTR access. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260902232028.2767071-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/paging_tmpl.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 27427e7f22fa..f925b11d76dd 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -368,13 +368,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, pte_access = ~0; /* - * Queue a page fault for injection if this assertion fails, as callers - * assume that walker.fault contains sane info on a walk failure. I.e. - * avoid making the situation worse by inducing even worse badness - * between when the assertion fails and when KVM kicks the vCPU out to - * userspace (because the VM is bugged). + * Queue a page fault for injection if any of the below assertions fail, + * as callers assume that walker.fault contains sane info on a walk + * failure. I.e. avoid making the situation worse by inducing even + * worse badness between when the assertion fails and when KVM kicks + * the vCPU out to userspace (because the VM is bugged). */ - if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm)) + if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm) || + KVM_BUG_ON(walker->max_level > PT_MAX_FULL_LEVELS, vcpu->kvm)) goto error; ++walker->level; From 34dacbe7ae93f2d3c26cd0bf3e08200b1fa2c180 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:27 -0700 Subject: [PATCH 111/506] KVM: x86/mmu: Bug the VM if KVM calcs a CPU role with EFER.LMA=1 && CR4.PAE=0 Bug the VM if KVM attempts to construct a CPU role with the should-be- impossible combination of long mode being active without PAE paging being enabled. KVM's MMU construction assumes that EFER.LMA can be set if and only CR4.PAE is set, and will create a completely invalid MMU if that assumption fails. FNAME(walk_addr_generic) already has sanity checks to try and mitigate the fallout, but attempt to catch such bugs earlier, as this is (at least) the second time KVM has had bugs that escaped into FNAME(walk_addr_generic), and it's entirely possible the bad state could cause problems elsewhere. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260902232028.2767071-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index d43b8b6c3b17..0601964934a1 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5910,6 +5910,9 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu, return role; } + if (KVM_BUG_ON(____is_efer_lma(regs) && !____is_cr4_pae(regs), vcpu->kvm)) + *(u64 *)®s->efer &= ~EFER_LMA; + role.base.efer_nx = ____is_efer_nx(regs); role.base.cr0_wp = ____is_cr0_wp(regs); role.base.cr4_smep = ____is_cr4_smep(regs); From 47b437b2d903137d547365391c7a5153b01dee6a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Sep 2026 16:20:28 -0700 Subject: [PATCH 112/506] KVM: x86/mmu: Convert MMU walker's bounds check from BUG_ON() to KVM_BUG_ON() Bug the VM, not the host, if KVM's sanity check that walking guest PTEs doesn't underflow the walker's level fires. Bugging the host while holding mmu_lock is all but guaranteed to panic the host, KVM hasn't _yet_ consumed the out-of-bounds level (i.e. hasn't corrupted memory), and KVM is already committed to bugging the VM and synthesizing a guest page fault if a fatal MMU error occurs while walking guest PTEs. I.e. there's no reason to keep the BUG_ON() at this point. Link: https://patch.msgid.link/20260902232028.2767071-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/paging_tmpl.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index f925b11d76dd..c8ec47b09264 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -392,7 +392,9 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, offset = index * sizeof(pt_element_t); pte_gpa = gfn_to_gpa(table_gfn) + offset; - BUG_ON(walker->level < 1); + if (KVM_BUG_ON(walker->level < 1, vcpu->kvm)) + goto error; + walker->table_gfn[walker->level - 1] = table_gfn; walker->pte_gpa[walker->level - 1] = pte_gpa; From 674717509d9ecfd3362c3fbdde1fe477fe117fe0 Mon Sep 17 00:00:00 2001 From: Hao Xiang Date: Tue, 8 Sep 2026 19:54:51 +0800 Subject: [PATCH 113/506] KVM: x86: Rename CPUID feature bit AVX10_VNNI_INT to AVX10_V1_AUX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename AVX10_VNNI_INT is to AVX10_V1_AUX to match the most recent version of the AVX10.2 spec (revision 7.0) [*]. Intel renamed the CPUID flag to apply to a variety of "convert" instructions as well as VNNI instructions. [*] Intel® Advanced Vector Extensions 10.2 Architecture Specification (rev 7.0) Signed-off-by: Hao Xiang Reviewed-by: Zhao Liu Link: https://patch.msgid.link/047d5e3a-b64d-434c-9a40-140468441de2.hao.xiang@linux.alibaba.com [sean: massage changelog] Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/reverse_cpuid.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index bb68a021cfaf..cbf1fe4ae01f 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1081,7 +1081,7 @@ void kvm_initialize_cpu_caps(void) ); kvm_cpu_cap_init(CPUID_24_1_ECX, - F(AVX10_VNNI_INT), + F(AVX10_V1_AUX), ); kvm_cpu_cap_init(CPUID_8000_0001_ECX, diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index 2ad25781cefb..89345af0f7ad 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -66,7 +66,7 @@ #define X86_FEATURE_AVX10_512 KVM_X86_FEATURE(CPUID_24_0_EBX, 18) /* Intel-defined sub-features, CPUID level 0x00000024:1 (ECX) */ -#define X86_FEATURE_AVX10_VNNI_INT KVM_X86_FEATURE(CPUID_24_1_ECX, 2) +#define X86_FEATURE_AVX10_V1_AUX KVM_X86_FEATURE(CPUID_24_1_ECX, 2) /* CPUID level 0x80000007 (EDX). */ #define KVM_X86_FEATURE_CONSTANT_TSC KVM_X86_FEATURE(CPUID_8000_0007_EDX, 8) From 3e7a842fe53301e4d0c2717108b9cd41df009a65 Mon Sep 17 00:00:00 2001 From: Tharit Tangkijwanichakul Date: Tue, 8 Sep 2026 23:04:25 +0700 Subject: [PATCH 114/506] KVM: x86: Reject reserved CR8 bits in KVM_SET_SREGS kvm_is_valid_sregs() validates the incoming CR0, CR4, and efer values but never checks CR8. When userspace passes a CR8 value with any of the reserved bits [63:4] set, __set_sregs_common() forwards it to kvm_set_cr8(), which rejects the reserved bits and returns early. That return value is not checked, so the ioctl reports success while the requested value is silently dropped. A subsequent KVM_GET_SREGS then returns a CR8 different from the one userspace believed it had written. Factor the reserved-bit check out into kvm_is_valid_cr8() and use it both in kvm_set_cr8() and in kvm_is_valid_sregs(). Note, prior to commit 7017fc3d1a12 ("KVM: Define and use cr8 access functions"), which unintentionally introduced the silent dropping behavior, KVM didn't check the incoming CR8 at all, i.e. allowed illegal values and likely generated failed VM-Entries. Fixes: 6aa8b732ca01 ("[PATCH] kvm: userspace interface") Fixes: 7017fc3d1a12 ("KVM: Define and use cr8 access functions") Signed-off-by: Tharit Tangkijwanichakul Link: https://patch.msgid.link/20260908160426.6547-2-tharitt97@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 8f66438989e4..fad31b59c622 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -440,9 +440,14 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); +static bool kvm_is_valid_cr8(unsigned long cr8) +{ + return !(cr8 & CR8_RESERVED_BITS); +} + int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { - if (cr8 & CR8_RESERVED_BITS) + if (!kvm_is_valid_cr8(cr8)) return 1; if (lapic_in_kernel(vcpu)) kvm_lapic_set_tpr(vcpu, cr8); @@ -565,6 +570,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) return kvm_is_valid_cr4(vcpu, sregs->cr4) && kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_is_valid_cr8(sregs->cr8) && kvm_valid_efer(vcpu, sregs->efer); } From 2b0e0aa54fa27af7c4afd502ef805e61f80c4498 Mon Sep 17 00:00:00 2001 From: Tharit Tangkijwanichakul Date: Tue, 8 Sep 2026 23:04:26 +0700 Subject: [PATCH 115/506] KVM: selftests: Add CR8 reserved-bit checks to set_sregs_test Extend set_sregs_test to cover CR8. Verify that the architecturally allowed bits [3:0] can be set and read back, that every reserved bit [63:4] is rejected by KVM_SET_SREGS, and that a successful set is observable via KVM_GET_SREGS. Without the accompanying fix to kvm_is_valid_sregs(), setting a reserved CR8 bit is silently accepted and the read-back value diverges from the value written, so this test fails on an unpatched kernel and passes once the fix is applied. Signed-off-by: Tharit Tangkijwanichakul Link: https://patch.msgid.link/20260908160426.6547-3-tharitt97@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/set_sregs_test.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/set_sregs_test.c b/tools/testing/selftests/kvm/x86/set_sregs_test.c index 603226ffe437..5afc4187f5eb 100644 --- a/tools/testing/selftests/kvm/x86/set_sregs_test.c +++ b/tools/testing/selftests/kvm/x86/set_sregs_test.c @@ -135,6 +135,17 @@ static void test_cr_bits(struct kvm_vcpu *vcpu, u64 cr4) /* NW without CD is illegal, as is PG without PE. */ TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_NW); TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_PG); + + /* CR8 bits 3:0 are writable; bits 63:4 are reserved. */ + vcpu_sregs_get(vcpu, &sregs); + sregs.cr8 = 0xf; + vcpu_sregs_set(vcpu, &sregs); + + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT_EQ(sregs.cr8, 0xf); + + for (i = 4; i < 64; i++) + TEST_INVALID_SREG_BIT(vcpu, cr8, sregs, BIT_ULL(i)); } static void test_efer_bits(struct kvm_vcpu *vcpu, u64 efer) From 59df0510c197bca35c4b11adaa7c8afa8f5898b9 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:34:46 +0000 Subject: [PATCH 116/506] irqchip/gic-v5: Allow KVM setup without a maintenance IRQ GICv5 does not require a virtual CPU interface maintenance interrupt for native GCIE operation. The interrupt is only needed when FEAT_GCIE_LEGACY is present, as the legacy GICv3 interface still relies on maintenance IRQ delivery. Stop rejecting KVM setup solely because the maintenance interrupt is absent. Parse the interrupt if present, but if none is described and the system does not advertise FEAT_GCIE_LEGACY, tell KVM that no maintenance interrupt is required. This lets native GICv5 KVM support be registered on systems that do not provide a maintenance interrupt, while requiring a maintenance interrupt for GICv3-capable systems. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-2-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic-v5.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index 5f2551cf077d..65d70998ab36 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -1139,10 +1139,18 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) /* GIC Virtual CPU interface maintenance interrupt */ gic_v5_kvm_info.no_maint_irq_mask = false; gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); - if (!gic_v5_kvm_info.maint_irq) { - pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n"); - return; - } + + /* + * We require an MI if we have legacy support, but don't, otherwise. + * Given that there's an existing flag to convey that an MI isn't + * needed, we (ab)use it to tell KVM that the MI isn't needed if we + * don't support legacy. + * + * The check for ARM64_HAS_GICV5_LEGACY explicitly doesn't use + * cpus_have_final_cap() here as we run too early. + */ + if (!cpus_have_cap(ARM64_HAS_GICV5_LEGACY) && !gic_v5_kvm_info.maint_irq) + gic_v5_kvm_info.no_maint_irq_mask = true; vgic_set_kvm_info(&gic_v5_kvm_info); } From e6418104fbc661d5c9b280e5b9f3ff83709cf0c9 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:35:17 +0000 Subject: [PATCH 117/506] irqchip/gic-v5: Provide OF IRS config frame attrs to KVM KVM needs to interact with the host IRS in order to, for example, make VMs or VPEs valid. There are two potential approaches here. Either the host irqchip driver can provide an interface, or KVM can interact directly with the host IRS. The latter of these two is chosen as the set of MMIO registers that KVM needs to interact with is orthogonal to the set used by the host irqchip driver (with the exception of some of the read-only IRS_IDRx registers). Pass KVM a pointer to an IRS config frame - the config frame belonging to ANY IRS is fine as long as one IRS's config frame is used consistently - in struct gic_kvm_info. Additionally, include a flag telling KVM whether the IRS is coherent or non-coherent in order to make sure that KVM can do the correct cache state management, if required. Only OF (Device Tree) is supported with this change. ACPI is not. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-3-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic-v5-irs.c | 7 +++++-- drivers/irqchip/irq-gic-v5.c | 5 +++++ include/linux/irqchip/arm-gic-v5.h | 3 +++ include/linux/irqchip/arm-vgic-info.h | 5 +++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c index 4486645a4b6f..01b032a06c2b 100644 --- a/drivers/irqchip/irq-gic-v5-irs.c +++ b/drivers/irqchip/irq-gic-v5-irs.c @@ -21,8 +21,6 @@ */ #define LPI_ID_BITS_LINEAR 12 -#define IRS_FLAGS_NON_COHERENT BIT(0) - static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data); static LIST_HEAD(irs_nodes); @@ -50,6 +48,11 @@ static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data, writeq_relaxed(val, irs_data->irs_base + reg_offset); } +struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void) +{ + return per_cpu(per_cpu_irs_data, 0); +} + /* * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register * provides the memory barriers (through MMIO accessors) diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index 65d70998ab36..ee2f4b790f40 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -1124,6 +1124,8 @@ static struct gic_kvm_info gic_v5_kvm_info __initdata; static void __init gic_of_setup_kvm_info(struct device_node *node) { + struct gicv5_irs_chip_data *irs_data = gicv5_irs_get_chip_data(); + /* * If we don't have native GICv5 virtualisation support, then * we also don't have FEAT_GCIE_LEGACY - the architecture @@ -1136,6 +1138,9 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) gic_v5_kvm_info.type = GIC_V5; + gic_v5_kvm_info.gicv5_irs.base = irs_data->irs_base; + gic_v5_kvm_info.gicv5_irs.non_coherent = !!(irs_data->flags & IRS_FLAGS_NON_COHERENT); + /* GIC Virtual CPU interface maintenance interrupt */ gic_v5_kvm_info.no_maint_irq_mask = false; gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 2c2fb39f049c..59a587585431 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -332,6 +332,8 @@ struct gicv5_irs_chip_data { raw_spinlock_t spi_config_lock; }; +#define IRS_FLAGS_NON_COHERENT BIT(0) + static inline int gicv5_wait_for_op_s_atomic(void __iomem *addr, u32 offset, const char *reg_s, u32 mask, u32 *val) @@ -379,6 +381,7 @@ void __init gicv5_free_lpi_domain(void); int gicv5_irs_of_probe(struct device_node *parent); int gicv5_irs_acpi_probe(void); +struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void); void gicv5_irs_remove(void); int gicv5_irs_enable(void); void gicv5_irs_its_probe(void); diff --git a/include/linux/irqchip/arm-vgic-info.h b/include/linux/irqchip/arm-vgic-info.h index 67d9d960273b..f05370e2debf 100644 --- a/include/linux/irqchip/arm-vgic-info.h +++ b/include/linux/irqchip/arm-vgic-info.h @@ -38,6 +38,11 @@ struct gic_kvm_info { bool has_v4_1; /* Deactivation impared, subpar stuff */ bool no_hw_deactivation; + /* GICv5 IRS base */ + struct { + void __iomem *base; + bool non_coherent; + } gicv5_irs; }; #ifdef CONFIG_KVM From 6055910efe9d5666eb096f5262e3501dbbca9aa2 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:35:48 +0000 Subject: [PATCH 118/506] irqchip/gic-v5: Set up gic_kvm_info on ACPI hosts Device-tree based GICv5 probing already passes the IRS details and maintenance interrupt to KVM, but the ACPI path only initialises the irqchip and installs the ACPI IRQ model. As a result, KVM never sees the GICv5 host information required to probe the vGIC on ACPI systems. Add the ACPI equivalent of the DT KVM setup. Parse the MADT GICC entries for the maintenance interrupt, retaining the value from the first relevant entry and warning if later entries disagree. Warn if firmware describes the interrupt as edge-triggered, but register it as a level-sensitive, active-high GSI. Pass the resulting IRQ together with the IRS base and coherency information to KVM. Native GICv5 does not require a maintenance interrupt unless the legacy GICv3-compatible CPU interface is present, so preserve the existing no-maintenance-IRQ handling for that case. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-4-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic-v5.c | 94 ++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index ee2f4b790f40..0341b05c888b 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -1122,7 +1122,7 @@ static void gicv5_set_cpuif_idbits(void) #ifdef CONFIG_KVM static struct gic_kvm_info gic_v5_kvm_info __initdata; -static void __init gic_of_setup_kvm_info(struct device_node *node) +static int __init gic_setup_kvm_info(unsigned int maint_irq) { struct gicv5_irs_chip_data *irs_data = gicv5_irs_get_chip_data(); @@ -1133,17 +1133,19 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) */ if (!gicv5_global_data.virt_capable) { pr_info("GIC implementation is not virtualization capable\n"); - return; + return -ENODEV; } - gic_v5_kvm_info.type = GIC_V5; + if (WARN_ON(!irs_data)) + return -ENODEV; + gic_v5_kvm_info.type = GIC_V5; gic_v5_kvm_info.gicv5_irs.base = irs_data->irs_base; gic_v5_kvm_info.gicv5_irs.non_coherent = !!(irs_data->flags & IRS_FLAGS_NON_COHERENT); /* GIC Virtual CPU interface maintenance interrupt */ gic_v5_kvm_info.no_maint_irq_mask = false; - gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); + gic_v5_kvm_info.maint_irq = maint_irq; /* * We require an MI if we have legacy support, but don't, otherwise. @@ -1158,11 +1160,93 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) gic_v5_kvm_info.no_maint_irq_mask = true; vgic_set_kvm_info(&gic_v5_kvm_info); + return 0; } + +static void __init gic_of_setup_kvm_info(struct device_node *node) +{ + unsigned int maint_irq; + + /* GIC Virtual CPU interface maintenance interrupt */ + maint_irq = irq_of_parse_and_map(node, 0); + if (gic_setup_kvm_info(maint_irq)) + irq_dispose_mapping(maint_irq); +} + +#ifdef CONFIG_ACPI +struct gicv5_acpi_kvm_info { + u32 maint_irq; + bool valid; +}; + +static struct gicv5_acpi_kvm_info acpi_v5_kvm_info __initdata; + +static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_generic_interrupt *gicc = + (struct acpi_madt_generic_interrupt *)header; + + if (!(gicc->flags & + (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) + return 0; + + if (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) + pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n"); + + if (!acpi_v5_kvm_info.valid) { + acpi_v5_kvm_info.valid = true; + acpi_v5_kvm_info.maint_irq = gicc->vgic_interrupt; + return 0; + } + + if (acpi_v5_kvm_info.maint_irq != gicc->vgic_interrupt) + pr_warn_once(FW_BUG "MI not the same on all CPUs\n"); + + return 0; +} + +static bool __init gic_acpi_collect_virt_info(void) +{ + acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, + gic_acpi_parse_virt_madt_gicc, 0); + + return acpi_v5_kvm_info.valid; +} + +static void __init gic_acpi_setup_kvm_info(void) +{ + int irq = 0; + + if (!gic_acpi_collect_virt_info()) { + pr_warn("Unable to get hardware information used for virtualization\n"); + return; + } + + if (acpi_v5_kvm_info.maint_irq) { + irq = acpi_register_gsi(NULL, acpi_v5_kvm_info.maint_irq, + ACPI_LEVEL_SENSITIVE, + ACPI_ACTIVE_HIGH); + if (irq <= 0) { + pr_warn("Failed to register GSI for GICv5 maintenance IRQ\n"); + irq = 0; + } + } + + if (gic_setup_kvm_info(irq) && irq) + acpi_unregister_gsi(acpi_v5_kvm_info.maint_irq); +} +#endif // CONFIG_ACPI #else static inline void __init gic_of_setup_kvm_info(struct device_node *node) { } + +#ifdef CONFIG_ACPI +static inline void __init gic_acpi_setup_kvm_info(void) +{ +} +#endif #endif // CONFIG_KVM static int __init gicv5_init_common(struct fwnode_handle *parent_domain) @@ -1272,6 +1356,8 @@ static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsig acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id, gic_v5_get_gsi_handle); + gic_acpi_setup_kvm_info(); + return 0; out_irs: From 5ae1ecf679afefadccea64ff393675aebedd3253 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:36:18 +0000 Subject: [PATCH 119/506] KVM: arm64: gic-v5: Define remaining IRS MMIO registers Complete the set of defined IRS MMIO registers in the GICv5 header file. Up until now, the set of defined IRS MMIO registers has been driven by code requirements. However, in order to properly emulate the IRS MMIO interface in KVM, the full set of IRS MMIO registers needs to be added. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-5-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- include/linux/irqchip/arm-gic-v5.h | 237 ++++++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 8 deletions(-) diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 59a587585431..8dc5847a646e 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -63,20 +63,34 @@ #define GICV5_OUTER_SHARE 0b10 #define GICV5_INNER_SHARE 0b11 +#define GICV5_AIDR_COMPONENT_IRS 0b00 +#define GICV5_AIDR_COMPONENT_ITS 0b01 +#define GICV5_AIDR_COMPONENT_IWB 0b10 + +#define GICV5_AIDR_ARCH_MAJ_REV_V5 0 +#define GICV5_AIDR_ARCH_MIN_REV_V0 0 + /* * IRS registers and tables structures */ #define GICV5_IRS_IDR0 0x0000 #define GICV5_IRS_IDR1 0x0004 #define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 #define GICV5_IRS_IDR5 0x0014 #define GICV5_IRS_IDR6 0x0018 #define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 #define GICV5_IRS_CR0 0x0080 #define GICV5_IRS_CR1 0x0084 #define GICV5_IRS_SYNCR 0x00c0 #define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 #define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 #define GICV5_IRS_SPI_CFGR 0x0114 #define GICV5_IRS_SPI_STATUSR 0x0118 #define GICV5_IRS_PE_SELR 0x0140 @@ -86,11 +100,51 @@ #define GICV5_IRS_IST_CFGR 0x0190 #define GICV5_IRS_IST_STATUSR 0x0194 #define GICV5_IRS_MAP_L2_ISTR 0x01c0 +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 +#define GICV5_IRS_IDR0_IRSID GENMASK(31, 16) +#define GICV5_IRS_IDR0_SWE BIT(12) +#define GICV5_IRS_IDR0_MPAM BIT(11) +#define GICV5_IRS_IDR0_MEC BIT(10) +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_VIRT_ONE_N BIT(8) +#define GICV5_IRS_IDR0_ONE_N BIT(7) #define GICV5_IRS_IDR0_VIRT BIT(6) +#define GICV5_IRS_IDR0_PA_RANGE GENMASK(5, 2) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) + +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 +#define GICV5_IRS_IDR0_INT_DOM_NON_SECURE 0b01 +#define GICV5_IRS_IDR0_INT_DOM_EL3 0b10 +#define GICV5_IRS_IDR0_INT_DOM_REALM 0b11 #define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) #define GICV5_IRS_IDR1_IAFFID_BITS GENMASK(19, 16) +#define GICV5_IRS_IDR1_PE_CNT GENMASK(15, 0) #define GICV5_IRS_IDR1_PRIORITY_BITS_1BITS 0b000 #define GICV5_IRS_IDR1_PRIORITY_BITS_2BITS 0b001 @@ -106,13 +160,30 @@ #define GICV5_IRS_IDR2_LPI BIT(5) #define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) + +#define GICV5_IRS_IDR3_VMT_LEVELS BIT(10) +#define GICV5_IRS_IDR3_VM_ID_BITS GENMASK(9, 5) +#define GICV5_IRS_IDR3_VMD_SZ GENMASK(4, 1) +#define GICV5_IRS_IDR3_VMD BIT(0) + +#define GICV5_IRS_IDR4_VPE_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR4_VPED_SZ GENMASK(5, 0) + #define GICV5_IRS_IDR5_SPI_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR6_SPI_IRS_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR7_SPI_BASE GENMASK(23, 0) -#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) +#define GICV5_IRS_IIDR_PRODUCT_ID GENMASK(31, 20) +#define GICV5_IRS_IIDR_VARIANT GENMASK(19, 16) +#define GICV5_IRS_IIDR_REVISION GENMASK(15, 12) +#define GICV5_IRS_IIDR_IMPLEMENTER GENMASK(11, 0) + +#define GICV5_IRS_AIDR_COMPONENT GENMASK(11, 8) +#define GICV5_IRS_AIDR_ARCHMAJORREV GENMASK(7, 4) +#define GICV5_IRS_AIDR_ARCHMINORREV GENMASK(3, 0) #define GICV5_IRS_CR0_IDLE BIT(1) #define GICV5_IRS_CR0_IRSEN BIT(0) @@ -135,21 +206,39 @@ #define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) -#define GICV5_IRS_SPI_STATUSR_V BIT(1) -#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_VMR_VIRT BIT_ULL(63) +#define GICV5_IRS_SPI_VMR_VM_ID GENMASK_ULL(15, 0) #define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_DOMAINR_DOMAIN GENMASK(1, 0) + +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_SECURE 0b00 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE 0b01 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_EL3 0b10 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_REALM 0b11 + +#define GICV5_IRS_SPI_RESAMPLER_ID GENMASK(23, 0) + #define GICV5_IRS_SPI_CFGR_TM BIT(0) +#define GICV5_IRS_SPI_CFGR_TM_EDGE 0b0 +#define GICV5_IRS_SPI_CFGR_TM_LEVEL 0b1 + +#define GICV5_IRS_SPI_STATUSR_V BIT(1) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) + #define GICV5_IRS_PE_SELR_IAFFID GENMASK(15, 0) +#define GICV5_IRS_PE_STATUSR_ONLINE BIT(2) #define GICV5_IRS_PE_STATUSR_V BIT(1) #define GICV5_IRS_PE_STATUSR_IDLE BIT(0) #define GICV5_IRS_PE_CR0_DPS BIT(0) -#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL #define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) #define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) @@ -167,15 +256,111 @@ #define GICV5_IRS_IST_CFGR_L2SZ_16K 0b01 #define GICV5_IRS_IST_CFGR_L2SZ_64K 0b10 -#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) -#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) #define GICV5_IRS_MAP_L2_ISTR_ID GENMASK(23, 0) +#define GICV5_IRS_VMT_BASER_ADDR GENMASK_ULL(55, 3) +#define GICV5_IRS_VMT_BASER_ADDR_SHIFT 3ULL +#define GICV5_IRS_VMT_BASER_VALID BIT_ULL(0) + +#define GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL 0b1 +#define GICV5_IRS_VMT_CFGR_STRUCTURE_LINEAR 0b0 + +#define GICV5_IRS_VMT_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_VMT_CFGR_VM_ID_BITS GENMASK(4, 0) + +#define GICV5_IRS_VMT_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VPE_SELR_S BIT_ULL(63) +#define GICV5_IRS_VPE_SELR_VPE_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VPE_SELR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VPE_DBR_DBV BIT_ULL(63) +#define GICV5_IRS_VPE_DBR_REQ_DB BIT_ULL(62) +#define GICV5_IRS_VPE_DBR_DBPM GENMASK_ULL(36, 32) +#define GICV5_IRS_VPE_DBR_INTID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_HPPIR_HPPIV BIT_ULL(32) +#define GICV5_IRS_VPE_HPPIR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VPE_HPPIR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_CR0_DPS BIT(0) + +#define GICV5_IRS_VPE_STATUSR_V BIT(1) +#define GICV5_IRS_VPE_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VM_DBR_EN BIT_ULL(63) +#define GICV5_IRS_VM_DBR_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VM_SELR_VM_ID GENMASK(15, 0) + +#define GICV5_IRS_VM_STATUSR_V BIT(1) +#define GICV5_IRS_VM_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VMAP_L2_VMTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VMTR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VMR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VMR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VMR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VISTR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VISTR_TYPE GENMASK_ULL(31, 29) + +#define GICV5_IRS_VMAP_L2_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_L2_VISTR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VMAP_L2_VISTR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VMAP_VPER_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VPER_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VPER_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SAVE_VMR_VM_ID GENMASK_ULL(15, 0) +#define GICV5_IRS_SAVE_VMR_Q BIT_ULL(62) +#define GICV5_IRS_SAVE_VMR_S BIT_ULL(63) + +#define GICV5_IRS_SAVE_VM_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SAVE_VM_STATUSR_Q BIT(1) + +#define GICV5_IRS_MEC_IDR_MECIDSIZE GENMASK(3, 0) + +#define GICV5_IRS_MEC_MECID_R_MECID GENMASK(15, 0) + +#define GICV5_IRS_MPAM_IDR_HAS_MPAM_SP BIT(24) +#define GICV5_IRS_MPAM_IDR_PMG_MAX GENMASK(23, 16) +#define GICV5_IRS_MPAM_IDR_PARTID_MAX GENMASK(15, 0) + +#define GICV5_IRS_MPAM_PARTID_R_IDLE BIT(31) +#define GICV5_IRS_MPAM_PARTID_R_MPAM_SP GENMASK(25, 24) +#define GICV5_IRS_MPAM_PARTID_R_PMG GENMASK(23, 16) +#define GICV5_IRS_MPAM_PARTID_R_PARTID GENMASK(15, 0) + +#define GICV5_IRS_SWERR_STATUSR_IMP_EC GENMASK_ULL(31, 24) +#define GICV5_IRS_SWERR_STATUSR_EC GENMASK_ULL(23, 16) +#define GICV5_IRS_SWERR_STATUSR_OF BIT_ULL(3) +#define GICV5_IRS_SWERR_STATUSR_S1V BIT_ULL(2) +#define GICV5_IRS_SWERR_STATUSR_S0V BIT_ULL(1) +#define GICV5_IRS_SWERR_STATUSR_V BIT_ULL(0) + +#define GICV5_IRS_SWERR_SYNDROMER0_VIRTUAL BIT_ULL(63) +#define GICV5_IRS_SWERR_SYNDROMER0_TYPE GENMASK_ULL(62, 60) +#define GICV5_IRS_SWERR_SYNDROMER0_ID GENMASK_ULL(55, 32) +#define GICV5_IRS_SWERR_SYNDROMER0_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SWERR_SYNDROMER1_ADDR GENMASK_ULL(55, 3) + #define GICV5_ISTL1E_VALID BIT_ULL(0) +#define GICV5_IRS_ISTL1E_SIZE 8UL #define GICV5_ISTL1E_L2_ADDR_MASK GENMASK_ULL(55, 12) +#define GICV5_IRS_SETLPIR 0x0000 +#define GICV5_IRS_SETLPIR_ID GENMASK(23, 0) + /* * ITS registers and tables structures */ @@ -297,6 +482,42 @@ #define GICV5_GSI_IWB_FRAME_ID GENMASK(28, 16) #define GICV5_GSI_IWB_WIRE GENMASK(15, 0) +/* + * CoreSight identification registers - ordered by increasing offset. + */ +#define GICV5_CORESIGHT_DEVARCH 0xffbc +#define GICV5_CORESIGHT_PIDR4 0xffd0 +#define GICV5_CORESIGHT_PIDR5 0xffd4 +#define GICV5_CORESIGHT_PIDR6 0xffd8 +#define GICV5_CORESIGHT_PIDR7 0xffdc +#define GICV5_CORESIGHT_PIDR0 0xffe0 +#define GICV5_CORESIGHT_PIDR1 0xffe4 +#define GICV5_CORESIGHT_PIDR2 0xffe8 +#define GICV5_CORESIGHT_PIDR3 0xffec +#define GICV5_CORESIGHT_CIDR0 0xfff0 +#define GICV5_CORESIGHT_CIDR1 0xfff4 +#define GICV5_CORESIGHT_CIDR2 0xfff8 +#define GICV5_CORESIGHT_CIDR3 0xfffc + +#define GICV5_CORESIGHT_DEVARCH_VAL \ + (FIELD_PREP(GENMASK(31, 21), 0x23b) | \ + BIT(20) | \ + 0x5a19) + +#define GICV5_CORESIGHT_PIDR4_JEP106_CONT 0x04 +#define GICV5_CORESIGHT_PIDR5_RES0 0x00 +#define GICV5_CORESIGHT_PIDR6_RES0 0x00 +#define GICV5_CORESIGHT_PIDR7_RES0 0x00 +#define GICV5_CORESIGHT_PIDR0_PART_0 0x4b +#define GICV5_CORESIGHT_PIDR1_DES_0_PART_1 0xb0 +#define GICV5_CORESIGHT_PIDR2_DES_1 0x0b +#define GICV5_CORESIGHT_PIDR3_REVAND_CMOD 0x00 + +#define GICV5_CORESIGHT_CIDR0_VAL 0x0d +#define GICV5_CORESIGHT_CIDR1_VAL 0xf0 +#define GICV5_CORESIGHT_CIDR2_VAL 0x05 +#define GICV5_CORESIGHT_CIDR3_VAL 0xb1 + /* * Global Data structures and functions */ From be0b3fd6345369e1fca0b52df52815a589629cb2 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:36:49 +0000 Subject: [PATCH 120/506] arm64/sysreg: Add GICv5 GIC VDPEND encoding Add the encoding for the GIC VDPEND system instruction. This operates on the virtual interrupt domain, and is used to make interrupts pending for a VM. This is part of enabling GICv5 KVM support, and is required for injection of SPIs and LPIs. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-6-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/sysreg.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 7aa08d59d494..c840b37adb2f 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -1040,7 +1040,7 @@ #define GCS_CAP(x) ((((unsigned long)x) & GCS_CAP_ADDR_MASK) | \ GCS_CAP_VALID_TOKEN) /* - * Definitions for GICv5 instructions + * Definitions for GICv5 instructions for the Current Domain */ #define GICV5_OP_GIC_CDAFF sys_insn(1, 0, 12, 1, 3) #define GICV5_OP_GIC_CDDI sys_insn(1, 0, 12, 2, 0) @@ -1105,6 +1105,17 @@ #define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29) #define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0) +/* + * Definitions for GICv5 instructions for the Virtual Domain + */ +#define GICV5_OP_GIC_VDPEND sys_insn(1, 4, 12, 1, 4) + +/* Shift and mask definitions for GIC VDPEND */ +#define GICV5_GIC_VDPEND_PENDING_MASK BIT_ULL(63) +#define GICV5_GIC_VDPEND_VM_MASK GENMASK_ULL(47, 32) +#define GICV5_GIC_VDPEND_TYPE_MASK GENMASK_ULL(31, 29) +#define GICV5_GIC_VDPEND_ID_MASK GENMASK_ULL(23, 0) + #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) #define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn) From 896b6a10253c104eeb6a2bd98be297a7b51f6600 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:37:50 +0000 Subject: [PATCH 121/506] KVM: arm64: gic-v5: Cache host IRS ID registers KVM needs access to the host IRS configuration frame to configure virtual-machine resources. The frame ID registers describe the capabilities available to KVM. Cache IRS_IDR2, IRS_IDR3, and IRS_IDR4 rather than extracting their individual fields into kvm_vgic_global_state. Accessor helpers derive the required capabilities from the cached registers. Use the cached IRS_IDR4 VPE capacity to limit the maximum number of vCPUs, while retaining VGIC_V5_MAX_CPUS as KVM's own upper bound. Keeping the raw register values lets KVM adjust the exposed capabilities when needed, as is done for ICH_VTR_EL2, and avoids tying future EL2 code to the layout of kvm_vgic_global_state. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-8-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5.c | 22 +++++++++- include/kvm/arm_vgic.h | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 4e3e4db24e77..e75a66b3920a 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -11,6 +11,7 @@ #include "vgic.h" #define ppi_caps kvm_vgic_global_state.vgic_v5_ppi_caps +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps /* * Not all PPIs are guaranteed to be implemented for GICv5. Deterermine which @@ -34,6 +35,21 @@ static void vgic_v5_get_implemented_ppis(void) __assign_bit(GICV5_ARCH_PPI_PMUIRQ, ppi_caps.impl_ppi_mask, system_supports_pmuv3()); } +static u32 irs_readl_relaxed(const u32 reg_offset) +{ + return readl_relaxed(irs_caps.irs_base + reg_offset); +} + +static void vgic_v5_irs_cache_id_regs(const struct gic_kvm_info *info) +{ + irs_caps.irs_base = info->gicv5_irs.base; + irs_caps.non_coherent = info->gicv5_irs.non_coherent; + + irs_caps.idr2 = irs_readl_relaxed(GICV5_IRS_IDR2); + irs_caps.idr3 = irs_readl_relaxed(GICV5_IRS_IDR3); + irs_caps.idr4 = irs_readl_relaxed(GICV5_IRS_IDR4); +} + /* * Probe for a vGICv5 compatible interrupt controller, returning 0 on success. */ @@ -60,10 +76,12 @@ int vgic_v5_probe(const struct gic_kvm_info *info) goto skip_v5; } - kvm_vgic_global_state.max_gic_vcpus = VGIC_V5_MAX_CPUS; - + vgic_v5_irs_cache_id_regs(info); vgic_v5_get_implemented_ppis(); + kvm_vgic_global_state.max_gic_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps), + VGIC_V5_MAX_CPUS); + ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V5); if (ret) { kvm_err("Cannot register GICv5 KVM device.\n"); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 086b7578e5f3..c82aa7c9e3f3 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -132,6 +132,19 @@ enum vgic_type { VGIC_V5, /* Newer, fancier GICv5 */ }; +struct vgic_v5_irs_caps { + /* Base address of the host IRS's CONFIG_FRAME */ + void __iomem *irs_base; + + /* Raw host IRS ID registers */ + u32 idr2; + u32 idr3; + u32 idr4; + + /* Is the IRS coherent with us, or not? */ + bool non_coherent; +}; + /* same for all guests, as depending only on the _host's_ GIC model */ struct vgic_global { /* type of the host GIC */ @@ -182,6 +195,9 @@ struct vgic_global { struct { DECLARE_BITMAP(impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); } vgic_v5_ppi_caps; + + /* GICv5 IRS capabilities */ + struct vgic_v5_irs_caps vgic_v5_irs_caps; }; extern struct vgic_global kvm_vgic_global_state; @@ -627,6 +643,65 @@ int vgic_v4_load(struct kvm_vcpu *vcpu); void vgic_v4_commit(struct kvm_vcpu *vcpu); int vgic_v4_put(struct kvm_vcpu *vcpu); +static inline u8 vgic_v5_irs_ist_id_bits(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_ID_BITS, caps->idr2); +} + +static inline u8 vgic_v5_irs_min_lpi_id_bits(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, caps->idr2); +} + +static inline bool vgic_v5_irs_ist_levels(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr2 & GICV5_IRS_IDR2_IST_LEVELS); +} + +static inline u8 vgic_v5_irs_ist_l2sz(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, caps->idr2); +} + +static inline bool vgic_v5_irs_istmd(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr2 & GICV5_IRS_IDR2_ISTMD); +} + +static inline u8 vgic_v5_irs_istmd_sz(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, caps->idr2); +} + +static inline u32 vgic_v5_irs_max_vms(const struct vgic_v5_irs_caps *caps) +{ + return BIT(FIELD_GET(GICV5_IRS_IDR3_VM_ID_BITS, caps->idr3)); +} + +static inline bool vgic_v5_irs_two_level_vmt_support(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr3 & GICV5_IRS_IDR3_VMT_LEVELS); +} + +static inline u16 vgic_v5_irs_vmd_size(const struct vgic_v5_irs_caps *caps) +{ + if (!(caps->idr3 & GICV5_IRS_IDR3_VMD)) + return 0; + + return BIT(FIELD_GET(GICV5_IRS_IDR3_VMD_SZ, caps->idr3)); +} + +static inline u16 vgic_v5_irs_vped_size(const struct vgic_v5_irs_caps *caps) +{ + return BIT(FIELD_GET(GICV5_IRS_IDR4_VPED_SZ, caps->idr4)); +} + +static inline u32 vgic_v5_irs_max_vpes(const struct vgic_v5_irs_caps *caps) +{ + /* Field stores VPE_ID_BITS - 1 */ + return BIT(FIELD_GET(GICV5_IRS_IDR4_VPE_ID_BITS, caps->idr4) + 1); +} + int vgic_v5_finalize_ppi_state(struct kvm *kvm); bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, unsigned long flags); From b3a40e812322daaca26d40fde681dc3079b4f3b4 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:38:20 +0000 Subject: [PATCH 122/506] KVM: arm64: gic-v5: Add VPE doorbell domain VPE doorbells allow the GICv5 hardware to notify KVM when an SPI or LPI can be signalled to a non-resident VPE. This provides the mechanism used to wake blocked vcpus once the hardware determines that the interrupt is eligible to be delivered. Add support for a per-VM VPE doorbell irq domain. The domain is created under the GICv5 LPI domain, with one doorbell allocated per VPE. Store the allocated doorbell base in the VM's GICv5 state so that later patches can request per-vcpu doorbell IRQs and use them for IRS commands and wakeups. Add the per-VPE doorbell state to the GICv5 CPU interface state. The doorbell IRQ number is populated when the IRQs are requested, and the db_fired state is used by later patches once doorbell delivery is wired up. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-9-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-init.c | 24 +++-- arch/arm64/kvm/vgic/vgic-v5.c | 142 +++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.h | 1 + include/kvm/arm_vgic.h | 4 + include/linux/irqchip/arm-gic-v5.h | 2 + 5 files changed, 166 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 4012df6002ea..052e680b8b60 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -474,8 +474,12 @@ int vgic_init(struct kvm *kvm) kvm_vgic_vcpu_reset(vcpu); ret = kvm_vgic_setup_default_irq_routing(kvm); - if (ret) + if (ret) { + if (vgic_is_v5(kvm)) + vgic_v5_teardown(kvm); + return ret; + } vgic_debug_init(kvm); dist->initialized = true; @@ -496,16 +500,22 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) dist->nr_spis = 0; dist->vgic_dist_base = VGIC_ADDR_UNDEF; - if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V2: + dist->vgic_cpu_base = VGIC_ADDR_UNDEF; + break; + case KVM_DEV_TYPE_ARM_VGIC_V3: list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list) vgic_v3_free_redist_region(kvm, rdreg); INIT_LIST_HEAD(&dist->rd_regions); - } else { - dist->vgic_cpu_base = VGIC_ADDR_UNDEF; - } - if (vgic_supports_direct_irqs(kvm)) - vgic_v4_teardown(kvm); + if (vgic_supports_direct_irqs(kvm)) + vgic_v4_teardown(kvm); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + vgic_v5_teardown(kvm); + break; + } xa_destroy(&dist->lpi_xa); } diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index e75a66b3920a..60bb39421ad0 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -7,6 +7,7 @@ #include #include +#include #include "vgic.h" @@ -125,6 +126,137 @@ int vgic_v5_probe(const struct gic_kvm_info *info) return 0; } +/* + * This set of irq_chip functions is specific for doorbells. + */ +static const struct irq_chip vgic_v5_db_irq_chip = { + .name = "GICv5-DB", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static void vgic_v5_irq_db_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static int vgic_v5_irq_db_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *arg) +{ + const struct irq_chip *chip = &vgic_v5_db_irq_chip; + struct vgic_v5_vm *vm = arg; + struct irq_data *irqd; + int ret; + + if (!vm) { + kvm_err("invalid parameter for doorbell irq allocation\n"); + return -EINVAL; + } + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL); + if (ret) + return ret; + + for (int i = 0; i < nr_irqs; i++) { + irq_domain_set_hwirq_and_chip(domain, virq + i, i, chip, + domain->host_data); + irqd = irq_desc_get_irq_data(irq_to_desc(virq + i)); + irqd_set_single_target(irqd); + } + + return 0; +} + +static const struct irq_domain_ops vgic_v5_irq_db_domain_ops = { + .alloc = vgic_v5_irq_db_domain_alloc, + .free = vgic_v5_irq_db_domain_free, +}; + +static int vgic_v5_create_per_vm_domain(struct kvm *kvm) +{ + struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm; + int nr_vcpus = atomic_read(&kvm->online_vcpus); + int id = task_pid_nr(current); + int ret, db_virq = 0; + + if (!gicv5_global_data.lpi_domain) { + kvm_err("LPI domain uninitialized, can't set up KVM Doorbells\n"); + return -ENODEV; + } + + vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv5-vpe-db", id); + if (!vm->fwnode) + return -ENOMEM; + + /* + * KVM per-VM VPE DB domain; child of LPI domain; only ever handles + * doorbells. We know how many doorbells we have, and therefore we + * create a linear domain. + */ + vm->domain = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain, + 0, nr_vcpus, vm->fwnode, + &vgic_v5_irq_db_domain_ops, vm); + if (!vm->domain) { + ret = -ENOMEM; + goto err; + } + + db_virq = irq_domain_alloc_irqs(vm->domain, nr_vcpus, NUMA_NO_NODE, vm); + if (db_virq <= 0) { + ret = db_virq; + goto err; + } + + kvm->arch.vgic.gicv5_vm.vpe_db_base = db_virq; + + return 0; + +err: + if (vm->domain) + irq_domain_remove(vm->domain); + if (vm->fwnode) + irq_domain_free_fwnode(vm->fwnode); + + kvm->arch.vgic.gicv5_vm.vpe_db_base = 0; + vm->domain = NULL; + vm->fwnode = NULL; + + return ret; +} + +static void vgic_v5_teardown_per_vm_domain(struct vgic_v5_vm *vm) +{ + if (!vm->domain) + return; + + if (vm->vpe_db_base) { + irq_domain_free_irqs(vm->vpe_db_base, vm->domain->revmap_size); + vm->vpe_db_base = 0; + } + + irq_domain_remove(vm->domain); + irq_domain_free_fwnode(vm->fwnode); + vm->domain = NULL; + vm->fwnode = NULL; +} + void vgic_v5_reset(struct kvm_vcpu *vcpu) { /* @@ -140,10 +272,16 @@ void vgic_v5_reset(struct kvm_vcpu *vcpu) vcpu->arch.vgic_cpu.num_pri_bits = 5; } +void vgic_v5_teardown(struct kvm *kvm) +{ + vgic_v5_teardown_per_vm_domain(&kvm->arch.vgic.gicv5_vm); +} + int vgic_v5_init(struct kvm *kvm) { struct kvm_vcpu *vcpu; unsigned long idx; + int ret; if (vgic_initialized(kvm)) return 0; @@ -155,6 +293,10 @@ int vgic_v5_init(struct kvm *kvm) } } + ret = vgic_v5_create_per_vm_domain(kvm); + if (ret) + return ret; + /* We only allow userspace to drive the SW_PPI, if it is implemented. */ bitmap_zero(kvm->arch.vgic.gicv5_vm.userspace_ppis, VGIC_V5_NR_PRIVATE_IRQS); diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index b71d486ae514..42cb205554da 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -383,6 +383,7 @@ void vgic_debug_destroy(struct kvm *kvm); int vgic_v5_probe(const struct gic_kvm_info *info); void vgic_v5_reset(struct kvm_vcpu *vcpu); int vgic_v5_init(struct kvm *kvm); +void vgic_v5_teardown(struct kvm *kvm); int vgic_v5_map_resources(struct kvm *kvm); void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid); bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index c82aa7c9e3f3..78a442046ebc 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -379,6 +379,10 @@ struct vgic_v5_vm { * convenient way to do that). */ DECLARE_BITMAP(vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS); + + struct fwnode_handle *fwnode; + struct irq_domain *domain; + int vpe_db_base; }; struct vgic_dist { diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 8dc5847a646e..6ed886c09bec 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -615,6 +615,8 @@ void gicv5_irs_syncr(void); /* Embedded in kvm.arch */ struct gicv5_vpe { + int db; + bool db_fired; bool resident; }; From f45438c015291f26e99f7fe355c4c191b5750f8d Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:38:51 +0000 Subject: [PATCH 123/506] KVM: arm64: gic-v5: Create and manage VM and VPE tables GICv5 uses a set of in-memory tables to track and manage VM state. These must be allocated by the hypervisor and provided to the IRS. The VMT (Virtual Machine Table) is a linear or two-level table comprising VMT Entries (VMTEs). Each VMTE describes the state for a single VM. This state includes things such as the SPI and LPI IST configuration (coming in a future commit), an implementation-defined VM Descriptor, and a VPE Table (VPET). The VPET contains one entry per possible VPE ID belonging to a VM. It is used to mark a VPE as valid and provide the address of an implementation-defined VPE Descriptor (VPED), which the hardware uses to track and manage VPE state. Allocate each VM's VPEDs as a single dense array indexed by vcpu_idx, while the VPET remains indexed by the userspace-provided vcpu_id. This keeps VPED storage proportional to the number of vCPUs even when their IDs are sparse. The VMT and VPET are shared with the IRS. On systems with a non-coherent IRS, cache maintenance operates at cache-line granularity, while multiple entries can occupy the same cache line. Use a common lock for CPU accesses to these tables and IRS command processing so that writing back one entry cannot overwrite an IRS update to a neighbouring entry. The implementation-defined VMD and VPED storage is also visible to the IRS. Round these allocations up to whole cache lines to prevent cache maintenance from corrupting unrelated slab objects. Initialise the storage before publishing its addresses to the IRS. This commit adds support for allocating the VMT and its descriptor backing state, and for managing VMTEs. The VMTEs can be initialised or released for reuse. VM IDs are allocated with an IDA, while an XArray tracks the host-side allocations associated with populated VMTEs. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-10-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/Makefile | 2 +- arch/arm64/kvm/vgic/vgic-init.c | 2 + arch/arm64/kvm/vgic/vgic-v5-tables.c | 681 +++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v5-tables.h | 100 ++++ arch/arm64/kvm/vgic/vgic-v5.c | 19 + drivers/irqchip/irq-gic-v5-irs.c | 12 +- include/kvm/arm_vgic.h | 4 + include/linux/irqchip/arm-gic-v5.h | 14 +- 8 files changed, 824 insertions(+), 10 deletions(-) create mode 100644 arch/arm64/kvm/vgic/vgic-v5-tables.c create mode 100644 arch/arm64/kvm/vgic/vgic-v5-tables.h diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 59612d2f277c..431de9b145ca 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -24,7 +24,7 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \ vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \ - vgic/vgic-v5.o + vgic/vgic-v5.o vgic/vgic-v5-tables.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 052e680b8b60..d5e05d979407 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -154,6 +154,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) case KVM_DEV_TYPE_ARM_VGIC_V3: INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions); break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL; } /* diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c new file mode 100644 index 000000000000..7252d48431a5 --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vgic.h" +#include "vgic-v5-tables.h" + +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static struct vgic_v5_vmt *vmt_info; + +/* Serialises IRS MMIO interaction (commands) and accesses to IRS tables. */ +DEFINE_RAW_SPINLOCK(vgic_v5_irs_lock); + +/* Serialises lazy installation of shared second-level VMTs. */ +static DEFINE_MUTEX(vmt_l2_lock); + +static DEFINE_XARRAY(vm_info); + +/* Level 1 Virtual Machine Table Entry */ +#define GICV5_VMTEL1E_VALID BIT_ULL(0) +/* Note that there is no shift for the address by design */ +#define GICV5_VMTEL1E_L2_ADDR GENMASK(51, 12) + +#define GICV5_VMTEL2E_SIZE 32ULL +/* An L2 table (two-level VMT) is ALWAYS 4kB! */ +#define GICV5_VMT_L2_TABLE_SIZE 4096ULL +#define GICV5_VMT_L2_TABLE_ENTRIES (GICV5_VMT_L2_TABLE_SIZE / GICV5_VMTEL2E_SIZE) + +/* + * As the L2 VMTE is a large data structure, we are splitting it into 4 parts. + * We only mask and shift WITHIN each part for simplicity. + */ +/* First 64-bit chunk */ +#define GICV5_VMTEL2E_VALID BIT_ULL(0) +#define GICV5_VMTEL2E_VMD_ADDR_SHIFT 3ULL +#define GICV5_VMTEL2E_VMD_ADDR GENMASK_ULL(55, 3) +/* Second 64-bit chunk */ +#define GICV5_VMTEL2E_VPET_ADDR_SHIFT 3ULL +#define GICV5_VMTEL2E_VPET_ADDR GENMASK_ULL(55, 3) +#define GICV5_VMTEL2E_VPE_ID_BITS GENMASK_ULL(63, 59) +/* Third & fourth 64-bit chunks (the encodings are the same for each) */ +#define GICV5_VMTEL2E_IST_VALID BIT_ULL(0) +#define GICV5_VMTEL2E_IST_L2SZ GENMASK_ULL(2, 1) +#define GICV5_VMTEL2E_IST_ADDR_SHIFT 6ULL +#define GICV5_VMTEL2E_IST_ADDR GENMASK_ULL(55, 6) +#define GICV5_VMTEL2E_IST_ISTSZ GENMASK_ULL(57, 56) +#define GICV5_VMTEL2E_IST_STRUCTURE BIT_ULL(58) +#define GICV5_VMTEL2E_IST_ID_BITS GENMASK_ULL(63, 59) + +/* Virtual PE Table Entry */ +#define GICV5_VPE_VALID BIT_ULL(0) +/* Note that there is no shift for the address by design. */ +#define GICV5_VPED_ADDR_SHIFT 3ULL +#define GICV5_VPED_ADDR GENMASK_ULL(55, 3) + +/* + * Our IRS might be coherent or non-coherent. If coherent, we can just emit a + * DSB to ensure that we're in sync. However, when non-coherent, we need to + * manage our cached data explicitly. + * + * This helper is used to handle both coherent and non-coherent IRSes, and + * handles all combinations of cleaning and invalidating to the PoC. + */ +static void vgic_v5_clean_inval(void *va, size_t size) +{ + unsigned long base = (unsigned long)va; + + dsb(ishst); + + if (kvm_vgic_global_state.vgic_v5_irs_caps.non_coherent) + dcache_clean_inval_poc(base, base + size); +} + +/* + * Create a linear VM Table. Directly using the number of entries supplied as + * the size of an L2 VMTE (32 bytes) guarantees that our allocation is aligned per + * the GICv5 requirements for the IRS_VMT_BASER. + */ +static int vgic_v5_alloc_vmt_linear(unsigned int num_entries) +{ + vmt_info->linear.vmt_base = kzalloc_objs(*vmt_info->linear.vmt_base, + num_entries); + if (!vmt_info->linear.vmt_base) + return -ENOMEM; + + vgic_v5_clean_inval(vmt_info->linear.vmt_base, + num_entries * sizeof(struct vmtl2_entry)); + + return 0; +} + +/* + * Allocate the first level of a two-level VM table. The second-level VM tables + * are allocated on demand (by vgic_v5_alloc_l2_vmt()). + */ +static int vgic_v5_alloc_vmt_two_level(unsigned int num_entries) +{ + /* + * Each L2 VMT array is always 4k-sized (covering 128 VMs). This is + * mandated by the GICv5 specification (GICv5 EAC0 Specification rule + * D_LSPBK). Hence, round up the number of entries to be at least 128 + * (or the next highest power of two as we give the HW the number of VM + * ID bits). + */ + if (num_entries < GICV5_VMT_L2_TABLE_ENTRIES) + num_entries = GICV5_VMT_L2_TABLE_ENTRIES; + num_entries = roundup_pow_of_two(num_entries); + + vmt_info->l2.num_l1_ents = (num_entries / GICV5_VMT_L2_TABLE_ENTRIES); + vmt_info->l2.vmt_base = kzalloc_objs(*vmt_info->l2.vmt_base, + vmt_info->l2.num_l1_ents); + if (!vmt_info->l2.vmt_base) + return -ENOMEM; + + vmt_info->l2.l2ptrs = kzalloc_objs(*vmt_info->l2.l2ptrs, + vmt_info->l2.num_l1_ents, + GFP_KERNEL); + if (!vmt_info->l2.l2ptrs) { + kfree(vmt_info->l2.vmt_base); + return -ENOMEM; + } + + vgic_v5_clean_inval(vmt_info->l2.vmt_base, + vmt_info->l2.num_l1_ents * sizeof(vmtl1_entry)); + + return 0; +} + +/* + * Allocate a second level VMT, if required. This can be called eagerly, and + * will only perform the allocation if required. + */ +static int vgic_v5_alloc_l2_vmt(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + u32 vm_id = vgic_v5_vm_id(kvm); + enum gicv5_vcpu_cmd cmd = VMT_L2_MAP; + struct vmtl2_entry *l2_table; + unsigned int l1_index; + int ret; + + /* Nothing to do if we have linear tables! */ + if (!vmt_info->two_level) + return 0; + + if (vm_id == VGIC_V5_VM_ID_INVAL) + return -EINVAL; + + /* + * We have 4k-sized L2 tables - this is mandated by the spec for + * two-level VMTs (GICv5 EAC0 Specification rule D_LSPBK). This means + * that we have 128 entries per L1 VMTE. + */ + l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES; + + guard(mutex)(&vmt_l2_lock); + + /* Already valid? Great! */ + if (vmt_info->l2.l2ptrs[l1_index]) + return 0; + + l2_table = kzalloc_objs(*l2_table, GICV5_VMT_L2_TABLE_ENTRIES); + if (!l2_table) + return -ENOMEM; + + /* The VMT is shared between all VMs. */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(l2_table, GICV5_VMT_L2_TABLE_SIZE); + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + + WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], + cpu_to_le64(virt_to_phys(l2_table))); + + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + + } + + /* + * VMAP in the L2 VMT via the IRS. We use any of the VM's CPUs as a + * conduit for interacting with the host's IRS. In the current case, + * this lets us resolve the VM ID to pass to the hardware. + */ + ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + + /* We've failed to make the L2 VMT valid - things are very broken! */ + if (ret) { + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* Remove the pointer from L1 table */ + WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], 0); + + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + } + + kfree(l2_table); + return ret; + } + + vmt_info->l2.l2ptrs[l1_index] = l2_table; + + return 0; +} + +/* + * Allocate the top-level VMT. This can either be linear or two-level. + */ +int vgic_v5_vmt_allocate(unsigned int max_vpes) +{ + int ret; + + /* Allocate the tracking structure */ + vmt_info = kzalloc_obj(*vmt_info, GFP_KERNEL); + if (!vmt_info) + return -ENOMEM; + + ida_init(&vmt_info->vm_id_ida); + vmt_info->max_vpes = max_vpes; + vmt_info->vmd_size = vgic_v5_irs_vmd_size(&irs_caps); + vmt_info->vped_size = vgic_v5_irs_vped_size(&irs_caps); + vmt_info->two_level = vgic_v5_irs_two_level_vmt_support(&irs_caps); + vmt_info->num_entries = vgic_v5_irs_max_vms(&irs_caps); + + if (vmt_info->two_level) + ret = vgic_v5_alloc_vmt_two_level(vmt_info->num_entries); + else + ret = vgic_v5_alloc_vmt_linear(vmt_info->num_entries); + + /* If anything failed, free our tracking structure before returning */ + if (ret) { + kfree(vmt_info); + vmt_info = NULL; + } + + return ret; +} + +/* + * Free the VMT and associated tracking structures. This isn't strictly expected + * to be called in general operation, but instead exists for completeness. + */ +int vgic_v5_vmt_free(void) +{ + if (!vmt_info) + return 0; + + if (!vmt_info->two_level) { + kfree(vmt_info->linear.vmt_base); + } else { + /* Free the L2 tables; kfree(NULL) is safe */ + for (int i = 0; i < vmt_info->l2.num_l1_ents; ++i) + kfree(vmt_info->l2.l2ptrs[i]); + kfree(vmt_info->l2.l2ptrs); + + /* And now free the L1 table */ + kfree(vmt_info->l2.vmt_base); + } + + ida_destroy(&vmt_info->vm_id_ida); + kfree(vmt_info); + vmt_info = NULL; + + return 0; +} + +/* + * Look up a VMT Entry by VM ID. + */ +static struct vmtl2_entry *vgic_v5_get_l2_vmte(u32 vm_id) +{ + unsigned int l1_index, l2_index; + struct vmtl2_entry *l2_table; + + if (vm_id == VGIC_V5_VM_ID_INVAL) + return ERR_PTR(-EINVAL); + + if (!vmt_info->two_level) + return &vmt_info->linear.vmt_base[vm_id]; + + l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES; + l2_index = vm_id % GICV5_VMT_L2_TABLE_ENTRIES; + + if (l1_index >= vmt_info->l2.num_l1_ents) + return ERR_PTR(-E2BIG); + + if (!vmt_info->l2.l2ptrs[l1_index]) + return ERR_PTR(-EINVAL); + + l2_table = vmt_info->l2.l2ptrs[l1_index]; + return &l2_table[l2_index]; +} + +/* + * Zero a VMT Entry, and flush & invalidate to the PoC, if required. + */ +static int vgic_v5_reset_vmte(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* + * The VMT is normal memory shared with the IRS. Invalidate before + * rewriting the entry so that cacheline-granular maintenance cannot + * later push stale data for neighbouring IRS-visible state back to + * memory. + */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + + /* + * Prevent the compiler from eliding the individual VMTE + * stores. Ordering and visibility to the IRS are provided by the + * surrounding cache maintenance and command protocol, not by + * WRITE_ONCE(). + * + * The same compiler-access constraint applies to READ_ONCE() users in + * this file: when inspecting IRS-visible table entries, read the field + * exactly once and prevent the compiler from reusing, merging or + * tearing the access. Coherency and freshness for non-coherent IRSes + * still come from the surrounding cache maintenance. + */ + WRITE_ONCE(vmte->val[0], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[1], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[2], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[3], cpu_to_le64(0ULL)); + + /* And make our write visible to the IRS (if non-coherent) */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* + * Use the IDA to allocate a new VM ID, and track it in the gicv5_vm data + * structure. If we're out of VM IDs, the IDA catches that, and we return the + * error (-ENOSPC). If we've previously allocated a VM ID, we catch that too and + * return -EBUSY. + */ +int vgic_v5_allocate_vm_id(struct kvm *kvm) +{ + int id; + + if (kvm->arch.vgic.gicv5_vm.vm_id != VGIC_V5_VM_ID_INVAL) + return -EBUSY; + + id = ida_alloc_max(&vmt_info->vm_id_ida, vmt_info->num_entries - 1u, + GFP_KERNEL); + if (id < 0) + return id; + + kvm->arch.vgic.gicv5_vm.vm_id = id; + + return 0; +} + +/* + * Release the VM ID to allow it to be reallocated in the future. + */ +void vgic_v5_release_vm_id(struct kvm *kvm) +{ + if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL) + return; + + ida_free(&vmt_info->vm_id_ida, kvm->arch.vgic.gicv5_vm.vm_id); + kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL; +} + +/* + * Initialise an entry in the VMT based on the index of the VM. + * + * Note: We don't mark the VMTE as valid as this needs to be done by + * the hardware. + */ +int vgic_v5_vmte_init(struct kvm *kvm) +{ + size_t vmd_alloc_size, vpet_alloc_size, vped_alloc_size; + void *vped_base = NULL, *vmd = NULL; + struct vgic_v5_vm_info *vmi = NULL; + u64 tmp, vmte_val0 = 0, vmte_val1; + u32 vm_id = vgic_v5_vm_id(kvm); + int ret, nr_cpus, nr_vcpus; + bool vmi_inserted = false; + struct vmtl2_entry *vmte; + vpe_entry *vpet = NULL; + struct kvm_vcpu *vcpu; + u16 max_vpe_id = 0; + unsigned long i; + + nr_vcpus = atomic_read(&kvm->online_vcpus); + if (nr_vcpus > vmt_info->max_vpes) + return -E2BIG; + + /* + * If we're using two-level VMTs, L2 is allocated on demand. For linear + * VMTs, this is a NOP. + */ + ret = vgic_v5_alloc_l2_vmt(kvm); + if (ret) + return ret; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + /* If the entry is already valid, something went wrong */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EINVAL; + } + + ret = vgic_v5_reset_vmte(kvm); + if (ret) + return ret; + + vmi = kzalloc_obj(*vmi); + if (!vmi) { + ret = -ENOMEM; + goto out_fail; + } + + ret = xa_insert(&vm_info, vm_id, vmi, GFP_KERNEL); + if (ret) + goto out_fail; + vmi_inserted = true; + + /* Allocate and assign the VM Descriptor, if required. */ + if (vmt_info->vmd_size != 0) { + vmd_alloc_size = round_up(vmt_info->vmd_size, + dma_get_cache_alignment()); + vmd = kzalloc(vmd_alloc_size, GFP_KERNEL); + if (!vmd) { + ret = -ENOMEM; + goto out_fail; + } + + /* Stash the VA so we can free it later */ + vmi->vmd_base = vmd; + + tmp = FIELD_PREP(GICV5_VMTEL2E_VMD_ADDR, + virt_to_phys(vmd) >> GICV5_VMTEL2E_VMD_ADDR_SHIFT); + vmte_val0 = tmp; + } + + /* + * Allocate and assign the VPE Table. + * + * First of all, iterate over all vcpus to find the highest VPE ID we + * require - we need to ensure that we have enough storage for all + * vcpu_id values that userspace has picked and not just the total + * number of vcpus. This gives us the number of VPEs required for the + * VM. + * + * Round up the number of VPEs to a whole power of two as we cannot + * describe non-powers-of-two in the VMTE field as it conveys the number + * of ID bits used and not the number of vPEs. IRS_IDR1.IAFFID_BITS is + * encoded as N - 1, so expose at least one VPE ID bit even for a + * single-vCPU VM to keep the views consistent. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + u16 vpe_id = vgic_v5_vpe_id(vcpu); + + if (vpe_id > max_vpe_id) + max_vpe_id = vpe_id; + } + + nr_cpus = max(2UL, roundup_pow_of_two(max_vpe_id + 1)); + vmi->vpe_id_bits = fls(nr_cpus) - 1; + + vpet_alloc_size = round_up((size_t)nr_cpus * sizeof(*vpet), + dma_get_cache_alignment()); + vpet = kzalloc(vpet_alloc_size, GFP_KERNEL); + if (!vpet) { + ret = -ENOMEM; + goto out_fail; + } + + /* Stash the VA so we can free it later */ + vmi->vpet_base = vpet; + + tmp = FIELD_PREP(GICV5_VMTEL2E_VPET_ADDR, + virt_to_phys(vpet) >> GICV5_VMTEL2E_VPET_ADDR_SHIFT); + tmp |= FIELD_PREP(GICV5_VMTEL2E_VPE_ID_BITS, vmi->vpe_id_bits); + vmte_val1 = tmp; + + /* + * Allocate a dense VPED array indexed by vcpu_idx. The VPET is indexed + * by the potentially sparse vcpu_id, but using that ID here would waste + * memory. Given that this is not userspace visible, we can cheat a bit + * and use the dense index instead. This is the ONLY place that we do + * this. + * + * Round the requested size up to a whole cacheline. kzalloc() gurantees + * natural alignment, so we ensure that the cachelines cannot be shared + * with unrelated slab objects. VPED and cacheline sizes are powers of + * two, so this also preserves the required VPED alignment when a VPED + * is larger than a cacheline. + */ + vped_alloc_size = round_up((size_t)nr_vcpus * vmt_info->vped_size, + dma_get_cache_alignment()); + vped_base = kzalloc(vped_alloc_size, GFP_KERNEL); + if (!vped_base) { + ret = -ENOMEM; + goto out_fail; + } + vmi->vped_base = vped_base; + + if (vmd) + vgic_v5_clean_inval(vmd, vmd_alloc_size); + vgic_v5_clean_inval(vpet, vpet_alloc_size); + vgic_v5_clean_inval(vped_base, vped_alloc_size); + + /* Publish the VMTE while serialising access to the shared VMT. */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[0], cpu_to_le64(vmte_val0)); + WRITE_ONCE(vmte->val[1], cpu_to_le64(vmte_val1)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + kvm->arch.vgic.gicv5_vm.vmte_allocated = true; + + return 0; + +out_fail: + /* kfree(NULL) is safe so we can just kfree() at leisure */ + kfree(vmd); + kfree(vpet); + kfree(vped_base); + if (vmi_inserted) + xa_erase(&vm_info, vm_id); + kfree(vmi); + + vgic_v5_reset_vmte(kvm); + + return ret; +} + +/* + * Release the VMT Entry, freeing up any allocated data structures before + * zeroing the VMTE. + * + * The VMTE must be marked as invalid before it is released. + */ +int vgic_v5_vmte_release(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + int ret; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + /* Reject if the VMTE has not been marked as invalid! */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EINVAL; + } + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + goto no_vmi; + + kfree(vmi->vped_base); + kfree(vmi->vpet_base); + kfree(vmi->vmd_base); + + xa_erase(&vm_info, vm_id); + kfree(vmi); + +no_vmi: + /* + * If we didn't get far enough into allocating a VMTE to create the VM + * info structure, then we just zero the VMTE and move on. There's + * nothing else we can realistically do here. + */ + ret = vgic_v5_reset_vmte(kvm); + if (ret) + return ret; + + kvm->arch.vgic.gicv5_vm.vmte_allocated = false; + + return 0; +} + +/* Provide the preallocated VPE descriptor to the hardware via the VPE Table. */ +int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + u16 vpe_id = vgic_v5_vpe_id(vcpu); + struct vgic_v5_vm_info *vmi; + vpe_entry tmp, *vpet_base; + void *vped; + + /* Make sure we're not over what the hardware supports */ + if (vpe_id >= vmt_info->max_vpes) + return -E2BIG; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vpe_id >= 1 << vmi->vpe_id_bits) + return -E2BIG; + + vpet_base = vmi->vpet_base; + + /* If the VPETE for this CPU is already valid we've gone wrong */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(&vpet_base[vpe_id], sizeof(*vpet_base)); + if (le64_to_cpu(READ_ONCE(vpet_base[vpe_id])) & GICV5_VPE_VALID) + return -EBUSY; + } + + vped = (u8 *)vmi->vped_base + + (size_t)vcpu->vcpu_idx * vmt_info->vped_size; + + tmp = FIELD_PREP(GICV5_VPED_ADDR, virt_to_phys(vped) >> GICV5_VPED_ADDR_SHIFT); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vpet_base[vpe_id], cpu_to_le64(tmp)); + vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); + } + + return 0; +} + +/* Clear the VPE's table entry after the VMTE has been made invalid. */ +int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + u16 vpe_id = vgic_v5_vpe_id(vcpu); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + vpe_entry *vpet_base; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EBUSY; + } + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vpe_id >= 1 << vmi->vpe_id_bits) + return -E2BIG; + + vpet_base = vmi->vpet_base; + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vpet_base[vpe_id], 0ULL); + vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); + } + + return 0; +} diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h new file mode 100644 index 000000000000..962be0c7cd3f --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +#ifndef __KVM_ARM_VGICV5_TABLES_H__ +#define __KVM_ARM_VGICV5_TABLES_H__ + +#include +#include +#include + +/* Level 1 Virtual Machine Table Entry */ +typedef __le64 vmtl1_entry; + +/* Level 2 Virtual Machine Table Entry */ +struct vmtl2_entry { + __le64 val[4]; +}; + +/* Virtual PE Table Entry */ +typedef __le64 vpe_entry; + +struct vgic_v5_vm_info { + void __iomem *vmd_base; + vpe_entry __iomem *vpet_base; + void *vped_base; + u8 vpe_id_bits; +}; + +struct vgic_v5_vmt { + union { + struct { + struct vmtl2_entry *vmt_base; + unsigned int num_ents; + } linear; + struct { + vmtl1_entry *vmt_base; + struct vmtl2_entry **l2ptrs; + unsigned int num_l1_ents; + } l2; + }; + bool two_level; + unsigned int num_entries; + unsigned int max_vpes; + size_t vmd_size; + size_t vped_size; + struct ida vm_id_ida; +}; + +static inline u32 vgic_v5_vm_id(struct kvm *kvm) +{ + return kvm->arch.vgic.gicv5_vm.vm_id; +} + +/* + * For vGICv5, we need to consolidate two views: + * - The vcpu ID assigned by userspace, which it can assign however it + * sees fit. + * - The index into the VPET, which is also presented to the guest as + * the IAFFID for the VPE in question. + * + * Ideally, we don't want to intercept the guest setting an interrupt's affinity + * - this would somewhat defeat the point of allowing the hardware to manage the + * interrupt directly. Therefore, we need to use vcpu->vcpu_id as our VPE ID, + * which ensures that both userspace and the GICv5 hardware have a consistent + * view. + * + * This means two things: + * - Our VPET might be sparse, depending on the IDs picked by userspace. + * - We need to check the vcpu ID provided by userspace, and reject anything + * we cannot handle. + * + * In order to at least ensure consistent usage in the places that we need to + * use the VPE ID, we provide this helper. + */ +static inline u16 vgic_v5_vpe_id(struct kvm_vcpu *vcpu) +{ + return vcpu->vcpu_id; +} + +static inline int vgic_v5_vpe_db(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db; +} + +extern raw_spinlock_t vgic_v5_irs_lock; + +int vgic_v5_vmt_allocate(unsigned int max_vpes); +int vgic_v5_vmt_free(void); + +int vgic_v5_allocate_vm_id(struct kvm *kvm); +void vgic_v5_release_vm_id(struct kvm *kvm); + +int vgic_v5_vmte_init(struct kvm *kvm); +int vgic_v5_vmte_release(struct kvm *kvm); +int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu); +int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu); + +#endif diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 60bb39421ad0..efe11a5a28f9 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -5,10 +5,12 @@ #include +#include #include #include #include +#include "vgic-v5-tables.h" #include "vgic.h" #define ppi_caps kvm_vgic_global_state.vgic_v5_ppi_caps @@ -126,6 +128,22 @@ int vgic_v5_probe(const struct gic_kvm_info *info) return 0; } +static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) +{ + enum gicv5_vcpu_cmd *cmd = vcpu_info; + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + switch (*cmd) { + case VMT_L2_MAP: + case VMTE_MAKE_VALID: + case VMTE_MAKE_INVALID: + /* Not yet implemented */ + default: + return -EINVAL; + } +} + /* * This set of irq_chip functions is specific for doorbells. */ @@ -137,6 +155,7 @@ static const struct irq_chip vgic_v5_db_irq_chip = { .irq_set_affinity = irq_chip_set_affinity_parent, .irq_get_irqchip_state = irq_chip_get_parent_state, .irq_set_irqchip_state = irq_chip_set_parent_state, + .irq_set_vcpu_affinity = vgic_v5_db_set_vcpu_affinity, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c index 01b032a06c2b..abf92b1c619d 100644 --- a/drivers/irqchip/irq-gic-v5-irs.c +++ b/drivers/irqchip/irq-gic-v5-irs.c @@ -269,24 +269,24 @@ int gicv5_irs_iste_alloc(const u32 lpi) * itself is not supported) again serves to make it easier to find physically * contiguous blocks of memory. */ -static unsigned int gicv5_irs_l2_sz(u32 idr2) +unsigned int gicv5_irs_l2_sz(u32 l2sz) { switch (PAGE_SIZE) { case SZ_64K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_64K; fallthrough; case SZ_4K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_4K; fallthrough; case SZ_16K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_16K; break; } - if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_4K; return GICV5_IRS_IST_CFGR_L2SZ_64K; @@ -334,7 +334,7 @@ static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); if (two_levels) - l2sz = gicv5_irs_l2_sz(idr2); + l2sz = gicv5_irs_l2_sz(FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, idr2)); istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 78a442046ebc..3ec3f7e1b88b 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -361,6 +361,8 @@ struct vgic_redist_region { struct list_head list; }; +#define VGIC_V5_VM_ID_INVAL (-1) + struct vgic_v5_vm { /* * We only expose a subset of PPIs to the guest. This subset is a @@ -383,6 +385,8 @@ struct vgic_v5_vm { struct fwnode_handle *fwnode; struct irq_domain *domain; int vpe_db_base; + u32 vm_id; + bool vmte_allocated; }; struct vgic_dist { diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 6ed886c09bec..fae3af230953 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -160,9 +160,9 @@ #define GICV5_IRS_IDR2_LPI BIT(5) #define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) -#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(0), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(1), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(2), (r)) #define GICV5_IRS_IDR3_VMT_LEVELS BIT(10) #define GICV5_IRS_IDR3_VM_ID_BITS GENMASK(9, 5) @@ -611,6 +611,7 @@ int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid); struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id); int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type); int gicv5_irs_iste_alloc(u32 lpi); +unsigned int gicv5_irs_l2_sz(u32 l2sz); void gicv5_irs_syncr(void); /* Embedded in kvm.arch */ @@ -655,4 +656,11 @@ void gicv5_deinit_lpis(void); void __init gicv5_its_of_probe(struct device_node *parent); void __init gicv5_its_acpi_probe(void); + +enum gicv5_vcpu_cmd { + VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ + VMTE_MAKE_VALID, /* Make the VMTE valid */ + VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ +}; + #endif From 77f7a9282fee67180134e75b107ff09d7e1a5876 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:39:21 +0000 Subject: [PATCH 124/506] KVM: arm64: gic-v5: Introduce guest IST alloc and management GICv5 guests use Interrupt State Tables (ISTs) to track and manage the interrupt state for SPIs and LPIs. These ISTs are provided to the host's IRS via the VMTE. On a host GICv5 system, SPIs do not require any up-front memory allocation prior to their use, unlike LPIs which require the OS to allocate an IST. For a GICv5 guest, the same holds from the guest's point of view: SPIs should require no explicit memory allocation by the guest. This means that KVM must provision the memory passed to the IRS for managing a guest's SPI state. Introduce the machinery needed to allocate a linear SPI IST. A later commit will allocate it as part of VM creation, before running the guest for the first time. The host is responsible for freeing this memory on guest teardown. For LPIs, the guest provisions memory for its LPI IST. KVM does not pass that memory directly to the host IRS. Instead, add support for allocating a shadow LPI IST and passing that to the IRS through the VMTE. The LPI IST may be allocated as a two-level structure when supported and required by the configured LPI ID space, as many more LPIs are expected than SPIs. The host frees this memory on guest teardown. Assigning an IST to a VM requires an IRS operation rather than writing the Valid bit directly in the VMTE. Extend the doorbell command interface with the operations needed to make SPI and LPI ISTs valid or invalid. Together with the existing VMT commands, the command set is: VMT_L2_MAP - Make a second level VM table valid VMTE_MAKE_VALID - Make a single VMTE, and hence VM, valid VMTE_MAKE_INVALID - Make a single VMTE, and hence VM, invalid SPI_VIST_MAKE_VALID - Make the SPI IST valid LPI_VIST_MAKE_VALID - Make the LPI IST valid LPI_VIST_MAKE_INVALID - Make the LPI IST invalid Once implemented, irq_set_vcpu_affinity() will act as an ioctl-like interface for issuing commands specific to either a VM or the particular VPE that the doorbell belongs to. None of these commands are plumbed through to the host IRS at this stage. There is intentionally no SPI_VIST_MAKE_INVALID command. The SPI IST will be allocated as part of VM creation and will remain valid for the lifetime of the VM. It can be freed after the VMTE has been made invalid during teardown. The LPI IST, on the other hand, is driven by the guest, which is free to invalidate and free its LPI IST at any point. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-11-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5-tables.c | 551 +++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v5-tables.h | 22 ++ arch/arm64/kvm/vgic/vgic-v5.c | 3 + include/linux/irqchip/arm-gic-v5.h | 3 + 4 files changed, 579 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index 7252d48431a5..f088006f3967 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -65,6 +65,14 @@ static DEFINE_XARRAY(vm_info); #define GICV5_VPED_ADDR_SHIFT 3ULL #define GICV5_VPED_ADDR GENMASK_ULL(55, 3) +/* + * The LPI and SPI configuration is stored in the 2nd and 3rd 64-bit chunks of + * the VMTE (0-based). We call this a section here in an attempt to simplify the + * code. + */ +#define GICV5_VMTEL2_LPI_SECTION 2 +#define GICV5_VMTEL2_SPI_SECTION 3 + /* * Our IRS might be coherent or non-coherent. If coherent, we can just emit a * DSB to ensure that we're in sync. However, when non-coherent, we need to @@ -553,6 +561,25 @@ int vgic_v5_vmte_init(struct kvm *kvm) return ret; } +/* + * The following set of forward declarations makes the code layout a *little* + * clearer as it lets us keep the IST-related code together. + */ +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, + unsigned int istsz); +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2_split); +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_spi_ist_free(struct kvm *kvm); + /* * Release the VMT Entry, freeing up any allocated data structures before * zeroing the VMTE. @@ -581,6 +608,22 @@ int vgic_v5_vmte_release(struct kvm *kvm) if (!vmi) goto no_vmi; + /* If we have an LPI IST, free it */ + if (vmi->h_lpi_ist) { + ret = vgic_v5_lpi_ist_free(kvm); + if (ret) + return ret; + } + vmi->h_lpi_ist = NULL; + + /* If we have an SPI IST, free it */ + if (vmi->h_spi_ist) { + ret = vgic_v5_spi_ist_free(kvm); + if (ret) + return ret; + } + vmi->h_spi_ist = NULL; + kfree(vmi->vped_base); kfree(vmi->vpet_base); kfree(vmi->vmd_base); @@ -679,3 +722,511 @@ int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu) return 0; } + +/* + * Assign an already allocated IST to the VM by populating the fields in the + * corresponding VMTE. We re-use this code for both an SPI IST and LPI IST, even + * if the paths to reach it might be vastly different. + */ +static int vgic_v5_vmte_assign_ist(struct kvm *kvm, phys_addr_t ist_base, + bool two_level, unsigned int id_bits, + unsigned int l2sz, unsigned int istsz, + bool spi_ist) +{ + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + u32 vm_id = vgic_v5_vm_id(kvm); + enum gicv5_vcpu_cmd cmd; + struct vmtl2_entry *vmte; + unsigned int section; + u64 tmp; + int ret; + + /* + * The L2 VMTE comprises four 64-bit "sections", where sections 2 & 3 + * describe the LPI and SPI ISTs, respectively. Both the LPI and SPI + * sections have the same layout, and as we are either operating on SPIs + * or LPIs we pick a section of the VMTE to modify up-front. + * + * See the GICv5 EAC0 Specification 11.2.2 for more details about the + * VMTE layout. + */ + section = spi_ist ? GICV5_VMTEL2_SPI_SECTION : GICV5_VMTEL2_LPI_SECTION; + + if (ist_base & ~GICV5_VMTEL2E_IST_ADDR) { + pr_err_ratelimited("kvm [%i]: IST misaligned: address 0x%llx, mask 0x%llx\n", + task_pid_nr(current), ist_base, + GICV5_VMTEL2E_IST_ADDR); + return -EINVAL; + } + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + tmp = FIELD_PREP(GICV5_VMTEL2E_IST_L2SZ, l2sz); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ADDR, + ist_base >> GICV5_VMTEL2E_IST_ADDR_SHIFT); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ISTSZ, istsz); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ID_BITS, id_bits); + if (two_level) + tmp |= GICV5_VMTEL2E_IST_STRUCTURE; + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* Bail if already allocated */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[section])) & + GICV5_VMTEL2E_IST_VALID) + return -EINVAL; + + WRITE_ONCE(vmte->val[section], cpu_to_le64(tmp)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + /* Finally, mark the entry as valid */ + cmd = spi_ist ? SPI_VIST_MAKE_VALID : LPI_VIST_MAKE_VALID; + ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + + return ret; +} + +/* + * Allocate a Linear IST - always used for SPIs and potentially LPIs. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. + * + * NOTE: istsz is the FIELD used by GICv5, not the actual size (or log2() of the + * size). + */ +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, unsigned int istsz) +{ + const size_t n = max(5, id_bits + 1 + istsz); + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + __le64 *ist; + u32 l1sz; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + /* + * Allocate the IST. We only have one level, so we just use the L2 ISTE. + */ + l1sz = BIT(n + 1); + ist = kzalloc(l1sz, GFP_KERNEL_ACCOUNT); + if (!ist) + return -ENOMEM; + + if (spi_ist) { + vmi->h_spi_ist = ist; + } else { + vmi->h_lpi_ist_structure = false; + vmi->h_lpi_ist = ist; + } + + vgic_v5_clean_inval(ist, l1sz); + + return 0; +} + +/* + * Allocate the first level of a two-level IST - LPI, only. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. + * + * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or + * log2() of the sizes). + */ +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + u32 vm_id = vgic_v5_vm_id(kvm); + const u32 l1_size = BIT(n + 1); + struct vgic_v5_vm_info *vmi; + __le64 *ist; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + ist = kzalloc(l1_size, GFP_KERNEL_ACCOUNT); + if (!ist) + return -ENOMEM; + + vmi->h_lpi_ist_structure = true; + vmi->h_lpi_ist = ist; + + vgic_v5_clean_inval(ist, l1_size); + + return 0; +} + +/* + * Allocate ALL of the second level ISTs for a two-level IST - LPI, only. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. The l2_size calculation is from section 11.2.3 of the same document. + * + * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or + * log2() of the sizes). + */ +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + const size_t l2_size = BIT(11 + (2 * l2sz) + 1); + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + __le64 *l2ist; + __le64 *l1ist; + int index; + u64 val; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + l1ist = vmi->h_lpi_ist; + + /* + * Allocate the storage for the pointers to the L2 ISTs (used when + * freeing later). + */ + vmi->h_lpi_l2_ists = kzalloc_objs(*vmi->h_lpi_l2_ists, l1_entries, + GFP_KERNEL_ACCOUNT); + if (!vmi->h_lpi_l2_ists) + return -ENOMEM; + + /* Allocate the L2 IST for each L1 IST entry */ + for (index = 0; index < l1_entries; ++index) { + l2ist = kzalloc(l2_size, GFP_KERNEL_ACCOUNT); + if (!l2ist) { + while (--index >= 0) + kfree(vmi->h_lpi_l2_ists[index]); + + kfree(vmi->h_lpi_l2_ists); + vmi->h_lpi_l2_ists = NULL; + + return -ENOMEM; + } + + /* + * We are not doing on-demand allocation of the L2 ISTs, and are + * instead provisioning the whole IST up front. This means that + * we are able to mark the L2 ISTs as valid in the L1 ISTEs as + * the overall IST is not yet valid. + */ + val = (virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK) | + GICV5_ISTL1E_VALID; + l1ist[index] = cpu_to_le64(val); + + vmi->h_lpi_l2_ists[index] = l2ist; + + vgic_v5_clean_inval(l2ist, l2_size); + } + + /* Handle CMOs for the whole L1 IST in one go */ + vgic_v5_clean_inval(l1ist, l1_entries * sizeof(*l1ist)); + + return 0; +} + +/* Allocate a two-level IST - LPIs, only */ +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + int ret; + + /* + * Allocate the L1 IST first, then all of the L2s. Everything + * is preallocated and we do no on-demand IST allocation. This + * is to avoid needing to track if and when the guest is doing + * on-demand IST allocation. + */ + ret = vgic_v5_alloc_l1_ist(kvm, id_bits, istsz, l2sz); + if (ret) + return ret; + + ret = vgic_v5_alloc_l2_ists(kvm, id_bits, istsz, l2sz); + if (ret) { + /* Free the L1 IST again */ + vmi = xa_load(&vm_info, vm_id); + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = 0; + + return ret; + } + + return 0; +} + +static void vgic_v5_free_allocated_lpi_ist(struct vgic_v5_vm_info *vmi, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2sz) +{ + if (!vmi->h_lpi_ist_structure) { + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; + return; + } + + if (vmi->h_lpi_l2_ists) { + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + int index; + + for (index = 0; index < l1_entries; ++index) + kfree(vmi->h_lpi_l2_ists[index]); + + kfree(vmi->h_lpi_l2_ists); + vmi->h_lpi_l2_ists = NULL; + } + + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; +} + +static void vgic_v5_free_allocated_spi_ist(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return; + + kfree(vmi->h_spi_ist); + vmi->h_spi_ist = NULL; +} + +/* + * Free a Linear IST. Can only happen once the VM is dead. + */ +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + struct vgic_v5_vm_info *vmi; + int section; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + if (spi) { + section = GICV5_VMTEL2_SPI_SECTION; + vgic_v5_free_allocated_spi_ist(kvm); + } else { + section = GICV5_VMTEL2_LPI_SECTION; + vgic_v5_free_allocated_lpi_ist(vmi, 0, 0, 0); + } + + /* The VM should be dead here, so we can just zero the VMT section */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], cpu_to_le64(0)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* + * Free a Two-Level IST. Can only happen once the VM is dead. + */ +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi) +{ + unsigned int id_bits, istsz, l2sz; + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + u64 tmp; + int section; + + /* We don't create two-level SPI ISTs, so freeing is a bad idea! */ + if (spi) + return -EINVAL; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + section = GICV5_VMTEL2_LPI_SECTION; + + if (!vmi->h_lpi_ist_structure) + return -EINVAL; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + tmp = le64_to_cpu(READ_ONCE(vmte->val[section])); + } + + id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp); + istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp); + l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp); + + vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); + + /* The VM must be dead, so we can just zero the VMT section */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], cpu_to_le64(0)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* Helper to determine ISTE size based on metadata requirements */ +static unsigned int vgic_v5_ist_istsz(unsigned int id_bits) +{ + if (!vgic_v5_irs_istmd(&irs_caps)) + return GICV5_IRS_IST_CFGR_ISTSZ_4; + + if (id_bits >= vgic_v5_irs_istmd_sz(&irs_caps)) + return GICV5_IRS_IST_CFGR_ISTSZ_16; + + return GICV5_IRS_IST_CFGR_ISTSZ_8; +} + +/* + * Allocate an IST for SPIs. + * + * We don't anticipate a large number of SPIs being allocated. Therefore, we + * always allocate a Linear IST for SPIs. This will need to be revisited should + * that assumption no longer hold. + */ +int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + phys_addr_t base_addr; + unsigned int istsz; + int ret; + + lockdep_assert_held(&kvm->arch.config_lock); + + istsz = vgic_v5_ist_istsz(id_bits); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vmi->h_spi_ist) + return -EBUSY; + + ret = vgic_v5_alloc_linear_ist(kvm, true, id_bits, istsz); + if (ret) + return ret; + base_addr = virt_to_phys(vmi->h_spi_ist); + + ret = vgic_v5_vmte_assign_ist(kvm, base_addr, false, id_bits, 0, istsz, + true); + if (ret) { + vgic_v5_free_allocated_spi_ist(kvm); + return ret; + } + + return 0; +} + +/* + * Free the IST for SPIs. Should only happen once the VM is dead. + */ +static int vgic_v5_spi_ist_free(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->arch.config_lock); + + return vgic_v5_linear_ist_free(kvm, true); +} + +/* + * Allocate an IST for LPIs. + * + * Unlike with SPIs, we anticipate that the guest will allocate a relatively + * large number of LPIs. Therefore, while we support doing a linear LPI IST, it + * is expected that LPI ISTs will be two-level. + */ +int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + unsigned int istsz, l2sz; + phys_addr_t phys_addr; + bool two_level; + int ret; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vmi->h_lpi_ist) + return -EBUSY; + + istsz = vgic_v5_ist_istsz(id_bits); + l2sz = gicv5_irs_l2_sz(vgic_v5_irs_ist_l2sz(&irs_caps)); + + /* + * Determine if we want to create a Linear or a Two-Level IST. + * + * A two-level IST is only required when a single L2 IST cannot cover + * the requested ID space. This depends on the L2 IST size selected for + * the IRS, not PAGE_SIZE. Using PAGE_SIZE here would switch to + * two-level too early when the selected L2 IST is larger than a page, + * and the allocation sizing arithmetic would underflow. + */ + two_level = vgic_v5_irs_ist_levels(&irs_caps) && + id_bits > ((10 - istsz) + (2 * l2sz)); + + if (!two_level) + ret = vgic_v5_alloc_linear_ist(kvm, false /* LPIs, not SPIs */, + id_bits, istsz); + else + ret = vgic_v5_alloc_two_level_lpi_ist(kvm, id_bits, istsz, + l2sz); + + if (ret) + return ret; + + phys_addr = virt_to_phys(vmi->h_lpi_ist); + ret = vgic_v5_vmte_assign_ist(kvm, phys_addr, two_level, id_bits, l2sz, + istsz, false); + if (ret) + vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); + + return ret; +} + +/* Free the LPI IST again */ +int vgic_v5_lpi_ist_free(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + if (!vmi->h_lpi_ist_structure) + return vgic_v5_linear_ist_free(kvm, false); + else + return vgic_v5_two_level_ist_free(kvm, false); +} diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h index 962be0c7cd3f..a711a247cc6b 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.h +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -26,6 +26,24 @@ struct vgic_v5_vm_info { vpe_entry __iomem *vpet_base; void *vped_base; u8 vpe_id_bits; + + /* + * Both the LPI and SPI ISTs are allocated by the hypervisor. While it + * would be possible to track and access them by iterating over the ISTs + * themselves, it makes more sense to store pointers to the ISTs. + * + * The LPI IST can either be two-level or linear. Hence, we keep track + * of the structure. If it is two-level, we retain pointers to the L1 + * IST and to each L2 IST array. If it is linear, we just store the base + * address of the IST array. + * + * The SPI IST is linear, and therefore we just store the base address + * of the SPI IST array. + */ + bool h_lpi_ist_structure; + __le64 *h_lpi_ist; + __le64 **h_lpi_l2_ists; + __le64 *h_spi_ist; }; struct vgic_v5_vmt { @@ -97,4 +115,8 @@ int vgic_v5_vmte_release(struct kvm *kvm); int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu); int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu); +int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_free(struct kvm *kvm); + #endif diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index efe11a5a28f9..c65355483f64 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -138,6 +138,9 @@ static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) case VMT_L2_MAP: case VMTE_MAKE_VALID: case VMTE_MAKE_INVALID: + case SPI_VIST_MAKE_VALID: + case LPI_VIST_MAKE_VALID: + case LPI_VIST_MAKE_INVALID: /* Not yet implemented */ default: return -EINVAL; diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index fae3af230953..24949b604531 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -661,6 +661,9 @@ enum gicv5_vcpu_cmd { VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ VMTE_MAKE_VALID, /* Make the VMTE valid */ VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ + SPI_VIST_MAKE_VALID, /* No corresponding invalid */ + LPI_VIST_MAKE_VALID, /* Triggered by a guest */ + LPI_VIST_MAKE_INVALID, /* Triggered by a guest */ }; #endif From 884d518cef62c5f4beb940709bf1920021b773b8 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:39:52 +0000 Subject: [PATCH 125/506] KVM: arm64: gic-v5: Implement VMT/vIST IRS MMIO Ops GICv5 has rules about which fields of a VMTE (or L1 VMT) may be directly written by the host once the table is valid. This ensures that no stale state is cached by the hardware, and provides a clear interface for making VMs, ISTs, etc. valid. The hypervisor is responsible for populating the VMTE for a VM. However, it is not permitted to write the Valid bit, as the VM table is already valid. Instead, the VM is made valid via an IRS MMIO operation. The same applies to the ISTs: they must be made valid via the host IRS. This commit adds support for: * Making level 1 VMTs valid, allowing for dynamic level 2 array allocation * Making VMTEs, and hence VMs, valid or invalid * Making SPI ISTs valid, and LPI ISTs valid or invalid, for a specific VM As part of this commit, the following vcpu_affinity-based commands are plumbed in: VMT_L2_MAP - Make a second level VM table valid VMTE_MAKE_VALID - Make a single VMTE, and hence VM, valid VMTE_MAKE_INVALID - Make a single VMTE, and hence VM, invalid SPI_VIST_MAKE_VALID - Make the SPI IST valid LPI_VIST_MAKE_VALID - Make the LPI IST valid LPI_VIST_MAKE_INVALID - Make the LPI IST invalid The lack of SPI_VIST_MAKE_INVALID is intentional. An IRS command timeout does not indicate whether the command was consumed. If VIST assignment times out, retain both the VMTE and IST allocation and mark the VM dead. This prevents KVM from freeing memory which may still be accessible to hardware. When successfully probing for a GICv5, allocate the VMT and make it valid through the IRS MMIO interface. Treat failures while allocating or assigning the VMT as hard GICv5 probe failures. At that point the IRS VM table state is a prerequisite for vGICv5 operation, and falling back to the legacy path would leave the host without a valid GICv5 VM table setup. Conversely, if vGICv5 registration succeeds but registration of the optional legacy vGICv3 device fails, retain the working vGICv5 device rather than failing the entire probe. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-12-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5-tables.c | 75 ++++++++--- arch/arm64/kvm/vgic/vgic-v5-tables.h | 1 + arch/arm64/kvm/vgic/vgic-v5.c | 192 ++++++++++++++++++++++++++- 3 files changed, 242 insertions(+), 26 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index f088006f3967..c028c4fb95e0 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -73,6 +73,21 @@ static DEFINE_XARRAY(vm_info); #define GICV5_VMTEL2_LPI_SECTION 2 #define GICV5_VMTEL2_SPI_SECTION 3 +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, + unsigned int istsz); +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2_split); +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_spi_ist_free(struct kvm *kvm); + /* * Our IRS might be coherent or non-coherent. If coherent, we can just emit a * DSB to ensure that we're in sync. However, when non-coherent, we need to @@ -561,25 +576,6 @@ int vgic_v5_vmte_init(struct kvm *kvm) return ret; } -/* - * The following set of forward declarations makes the code layout a *little* - * clearer as it lets us keep the IST-related code together. - */ -static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, - unsigned int id_bits, - unsigned int istsz); -static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, - unsigned int istsz, unsigned int l2_split); -static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, - unsigned int istsz, unsigned int l2_split); -static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, - unsigned int id_bits, - unsigned int istsz, - unsigned int l2_split); -static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi); -static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi); -static int vgic_v5_spi_ist_free(struct kvm *kvm); - /* * Release the VMT Entry, freeing up any allocated data structures before * zeroing the VMTE. @@ -723,6 +719,18 @@ int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu) return 0; } +phys_addr_t vgic_v5_get_vmt_base(void) +{ + phys_addr_t vmt_base; + + if (!vmt_info->two_level) + vmt_base = virt_to_phys(vmt_info->linear.vmt_base); + else + vmt_base = virt_to_phys(vmt_info->l2.vmt_base); + + return vmt_base; +} + /* * Assign an already allocated IST to the VM by populating the fields in the * corresponding VMTE. We re-use this code for both an SPI IST and LPI IST, even @@ -785,8 +793,30 @@ static int vgic_v5_vmte_assign_ist(struct kvm *kvm, phys_addr_t ist_base, /* Finally, mark the entry as valid */ cmd = spi_ist ? SPI_VIST_MAKE_VALID : LPI_VIST_MAKE_VALID; ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + if (ret) { + /* + * A timeout does not tell us whether the IRS consumed the + * command, so we're in some indeterminate and potentially + * transient state. It may still make the VIST valid and access + * the backing memory, so leave both the VMTE and allocation + * intact. Killing the VM ensures that teardown is the only path + * that can reclaim them, after it has successfully made the + * complete VMTE invalid. + */ + if (ret == -ETIMEDOUT) { + kvm_vm_dead(kvm); + return ret; + } - return ret; + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], 0ULL); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return ret; + } + + return 0; } /* @@ -1137,7 +1167,8 @@ int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits) ret = vgic_v5_vmte_assign_ist(kvm, base_addr, false, id_bits, 0, istsz, true); if (ret) { - vgic_v5_free_allocated_spi_ist(kvm); + if (ret != -ETIMEDOUT) + vgic_v5_free_allocated_spi_ist(kvm); return ret; } @@ -1207,7 +1238,7 @@ int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits) phys_addr = virt_to_phys(vmi->h_lpi_ist); ret = vgic_v5_vmte_assign_ist(kvm, phys_addr, two_level, id_bits, l2sz, istsz, false); - if (ret) + if (ret && ret != -ETIMEDOUT) vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); return ret; diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h index a711a247cc6b..38b6f5269a2a 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.h +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -106,6 +106,7 @@ extern raw_spinlock_t vgic_v5_irs_lock; int vgic_v5_vmt_allocate(unsigned int max_vpes); int vgic_v5_vmt_free(void); +phys_addr_t vgic_v5_get_vmt_base(void); int vgic_v5_allocate_vm_id(struct kvm *kvm); void vgic_v5_release_vm_id(struct kvm *kvm); diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index c65355483f64..3d8f17301567 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -16,6 +16,9 @@ #define ppi_caps kvm_vgic_global_state.vgic_v5_ppi_caps #define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps +static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, phys_addr_t vmt_base); +static int vgic_v5_irs_clear_vmt(void); + /* * Not all PPIs are guaranteed to be implemented for GICv5. Deterermine which * ones are, and generate a mask. @@ -43,6 +46,21 @@ static u32 irs_readl_relaxed(const u32 reg_offset) return readl_relaxed(irs_caps.irs_base + reg_offset); } +static void irs_writel_relaxed(const u32 val, const u32 reg_offset) +{ + writel_relaxed(val, irs_caps.irs_base + reg_offset); +} + +static u64 irs_readq_relaxed(const u32 reg_offset) +{ + return readq_relaxed(irs_caps.irs_base + reg_offset); +} + +static void irs_writeq_relaxed(const u64 val, const u32 reg_offset) +{ + writeq_relaxed(val, irs_caps.irs_base + reg_offset); +} + static void vgic_v5_irs_cache_id_regs(const struct gic_kvm_info *info) { irs_caps.irs_base = info->gicv5_irs.base; @@ -62,6 +80,7 @@ int vgic_v5_probe(const struct gic_kvm_info *info) int ret; kvm_vgic_global_state.type = VGIC_V5; + kvm_vgic_global_state.max_gic_vcpus = VGIC_V5_MAX_CPUS; kvm_vgic_global_state.vcpu_base = 0; kvm_vgic_global_state.vctrl_base = NULL; @@ -82,13 +101,52 @@ int vgic_v5_probe(const struct gic_kvm_info *info) vgic_v5_irs_cache_id_regs(info); vgic_v5_get_implemented_ppis(); + /* + * Even if the HW supports more per-VM vCPUs, artificially cap as we + * can't use them all. + */ kvm_vgic_global_state.max_gic_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps), VGIC_V5_MAX_CPUS); + /* + * GICv5 requires a set of tables to be allocated in order to manage + * VMs. We allocate them in advance here, which alas means that we + * already have to make a decisions regarding the maximum number of VMs + * we want to run. For now, we match the maximum number offered by the + * hardware, but this might not be a wise choice in the long term. + */ + ret = vgic_v5_vmt_allocate(kvm_vgic_global_state.max_gic_vcpus); + if (ret) { + kvm_err("Failed to allocate the GICv5 VM tables; no GICv5 support\n"); + return -ENODEV; + } + + /* + * We've now allocated the VM table, but the host's IRS doesn't know + * about it yet. Provide the base address of the VMT to the IRS, as well + * as the number of ID bits that it covers and the structure used + * (linear/two-level). + */ + ret = vgic_v5_irs_assign_vmt(vgic_v5_irs_two_level_vmt_support(&irs_caps), + ilog2(vgic_v5_irs_max_vms(&irs_caps)), + vgic_v5_get_vmt_base()); + if (ret) { + kvm_err("Failed to assign the GICv5 VM tables to the IRS; no GICv5 support\n"); + if (!vgic_v5_irs_clear_vmt()) + vgic_v5_vmt_free(); + return -ENODEV; + } + ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V5); if (ret) { kvm_err("Cannot register GICv5 KVM device.\n"); - goto skip_v5; + /* + * Don't free the VMT itself if the hardware still has a valid + * pointer to it. + */ + if (!vgic_v5_irs_clear_vmt()) + vgic_v5_vmt_free(); + return -ENODEV; } v5_registered = true; @@ -113,12 +171,13 @@ int vgic_v5_probe(const struct gic_kvm_info *info) ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3); if (ret) { kvm_err("Cannot register GICv3-legacy KVM device.\n"); - return ret; + /* vGICv5 should still work */ + return v5_registered ? 0 : ret; } /* We potentially limit the max VCPUs further than we need to here */ kvm_vgic_global_state.max_gic_vcpus = min(VGIC_V3_MAX_CPUS, - VGIC_V5_MAX_CPUS); + kvm_vgic_global_state.max_gic_vcpus); static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif); kvm_info("GCIE legacy system register CPU interface\n"); @@ -128,20 +187,145 @@ int vgic_v5_probe(const struct gic_kvm_info *info) return 0; } +/* + * Wait for completion of a change in any of IRS_VMT_BASER, IRS_VMAP_L2_VMTR, + * IRS_VMAP_VMR, IRS_VMAP_VPER, IRS_VMAP_VISTR, IRS_VMAP_L2_VISTR. + */ +static int vgic_v5_irs_wait_for_vm_op(void) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_VMT_STATUSR, + GICV5_IRS_VMT_STATUSR_IDLE, + NULL); +} + +static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset) +{ + int ret; + + lockdep_assert_held(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + irs_writeq_relaxed(val, offset); + + return vgic_v5_irs_wait_for_vm_op(); +} + +static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, + phys_addr_t vmt_base) +{ + u64 vmt_baser; + u32 vmt_cfgr; + int ret; + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + vmt_baser = irs_readq_relaxed(GICV5_IRS_VMT_BASER); + if (!!FIELD_GET(GICV5_IRS_VMT_BASER_VALID, vmt_baser)) + return -EBUSY; + + vmt_cfgr = FIELD_PREP(GICV5_IRS_VMT_CFGR_VM_ID_BITS, vm_id_bits); + if (two_level) + vmt_cfgr |= FIELD_PREP(GICV5_IRS_VMT_CFGR_STRUCTURE, + GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL); + + irs_writel_relaxed(vmt_cfgr, GICV5_IRS_VMT_CFGR); + + /* The base address is intentionally only masked and not shifted */ + vmt_baser = FIELD_PREP(GICV5_IRS_VMT_BASER_VALID, true) | + (vmt_base & GICV5_IRS_VMT_BASER_ADDR); + irs_writeq_relaxed(vmt_baser, GICV5_IRS_VMT_BASER); + + return vgic_v5_irs_wait_for_vm_op(); +} + +static int vgic_v5_irs_clear_vmt(void) +{ + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + return vgic_v5_irs_write_vm_mmio_reg(0, GICV5_IRS_VMT_BASER); +} + +static int vgic_v5_irs_vmap_l2_vmt(u16 vm_id) +{ + u64 val = FIELD_PREP(GICV5_IRS_VMAP_L2_VMTR_VM_ID, vm_id) | + GICV5_IRS_VMAP_L2_VMTR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_L2_VMTR); +} + +static int __vgic_v5_irs_vmap_vm(u16 vm_id, bool unmap) +{ + u64 val = FIELD_PREP(GICV5_IRS_VMAP_VMR_VM_ID, vm_id) | + FIELD_PREP(GICV5_IRS_VMAP_VMR_U, unmap) | + GICV5_IRS_VMAP_VMR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VMR); +} + +static int vgic_v5_irs_set_vm_valid(u16 vm_id) +{ + return __vgic_v5_irs_vmap_vm(vm_id, false); +} + +static int vgic_v5_irs_set_vm_invalid(u16 vm_id) +{ + return __vgic_v5_irs_vmap_vm(vm_id, true); +} + +static int __vgic_v5_irs_update_vist_validity(u16 vm_id, bool spi_ist, bool unmap) +{ + u8 type = spi_ist ? 0b011 : 0b010; + u64 val = FIELD_PREP(GICV5_IRS_VMAP_VISTR_TYPE, type) | + FIELD_PREP(GICV5_IRS_VMAP_VISTR_VM_ID, vm_id) | + FIELD_PREP(GICV5_IRS_VMAP_VISTR_U, unmap) | + GICV5_IRS_VMAP_VISTR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VISTR); +} + +static int vgic_v5_irs_set_vist_valid(u16 vm_id, bool spi_ist) +{ + return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, false); +} + +/* + * LPI ISTs can be invalidated explicitly. SPI ISTs are invalidated by making + * the VMTE invalid during teardown. + */ +static int vgic_v5_irs_set_vist_invalid(u16 vm_id, bool spi_ist) +{ + return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, true); +} + static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) { + struct vgic_v5_vm *vm = data->domain->host_data; enum gicv5_vcpu_cmd *cmd = vcpu_info; guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); switch (*cmd) { case VMT_L2_MAP: + return vgic_v5_irs_vmap_l2_vmt(vm->vm_id); case VMTE_MAKE_VALID: + return vgic_v5_irs_set_vm_valid(vm->vm_id); case VMTE_MAKE_INVALID: + return vgic_v5_irs_set_vm_invalid(vm->vm_id); case SPI_VIST_MAKE_VALID: + return vgic_v5_irs_set_vist_valid(vm->vm_id, true); case LPI_VIST_MAKE_VALID: + return vgic_v5_irs_set_vist_valid(vm->vm_id, false); case LPI_VIST_MAKE_INVALID: - /* Not yet implemented */ + return vgic_v5_irs_set_vist_invalid(vm->vm_id, false); default: return -EINVAL; } From 21fd40005fb36c24ce0c28758b48da1238ffc909 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:40:22 +0000 Subject: [PATCH 126/506] KVM: arm64: vgic: Enforce model-specific vCPU limits A GICv5 host with FEAT_GCIE_LEGACY can expose either a native vGICv5 or a vGICv3 device. These models do not necessarily have the same vCPU limit: the native GICv5 limit is probed from the IRS VPE capacity, while the GICv3 limit remains the fixed KVM vGICv3 limit. Keep the IRS-derived limit separately for vGICv5 creation. The pre-VGIC KVM_CAP_MAX_VCPUS value continues to expose the largest limit among the still-selectable models, and kvm_vgic_create() clamps the VM to the limit of the VGIC model userspace actually selected. Userspace can create vCPUs before creating a VGIC. Hence, checking only the number of existing vCPUs after reducing the limit is not sufficient. For example, a single vCPU with ID 500 passes the GICv2 count limit even though GICv2 can represent only eight target CPUs. The GICv2 code subsequently uses the vCPU ID in target and SGI source masks, leading to shifts beyond the width of the operand. GICv5 has a similar requirement because the userspace-provided vcpu_id is used as the index into the VPET. This means that the vcpu_id also represents the IAFFID for a VPE, and therefore is visible to the guest. After selecting the model-specific limit, validate every existing vCPU ID against it. KVM_CREATE_VCPU already enforces the same limit for vCPUs created after the VGIC, making the result independent of creation order. Link: https://lore.kernel.org/r/20260807133051.15D381F000E9@smtp.kernel.org Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-13-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-init.c | 49 ++++++++++++++++++++++++++------- arch/arm64/kvm/vgic/vgic-v5.c | 16 ++++++----- include/kvm/arm_vgic.h | 16 ++++++++--- 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index d5e05d979407..ccbeb2f783c4 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -129,13 +129,30 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) } ret = 0; - if (type == KVM_DEV_TYPE_ARM_VGIC_V2) + switch (type) { + case KVM_DEV_TYPE_ARM_VGIC_V2: kvm->max_vcpus = VGIC_V2_MAX_CPUS; - else if (type == KVM_DEV_TYPE_ARM_VGIC_V3) + break; + case KVM_DEV_TYPE_ARM_VGIC_V3: kvm->max_vcpus = VGIC_V3_MAX_CPUS; - else if (type == KVM_DEV_TYPE_ARM_VGIC_V5) - kvm->max_vcpus = min(VGIC_V5_MAX_CPUS, - kvm_vgic_global_state.max_gic_vcpus); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + kvm->max_vcpus = kvm_vgic_global_state.max_gicv5_vcpus; + break; + } + + /* + * KVM_CREATE_VCPU enforces the model-specific limit if the VGIC has + * already been created. Apply the same limit to any existing vCPUs so + * that the result does not depend on the order in which userspace + * creates them. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vcpu->vcpu_id >= kvm->max_vcpus) { + ret = -E2BIG; + goto out_unlock; + } + } if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) { ret = -E2BIG; @@ -399,15 +416,27 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) if (ret) return ret; - /* - * If we are creating a VCPU with a GICv3 we must also register the - * KVM io device for the redistributor that belongs to this VCPU. - */ - if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V3: + /* + * If we are creating a VCPU with a GICv3 we must also register + * the KVM io device for the redistributor that belongs to this + * VCPU. + */ mutex_lock(&vcpu->kvm->slots_lock); ret = vgic_register_redist_iodev(vcpu); mutex_unlock(&vcpu->kvm->slots_lock); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + /* + * Ensure that it is possible to represent the + * userspace-allocated vcpu_id in the hardware-limited (or + * KVM-capped) VPE table used by GICv5. + */ + if (vcpu->vcpu_id >= kvm_vgic_global_state.max_gicv5_vcpus) + return -EINVAL; } + return ret; } diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 3d8f17301567..e1571d1788a6 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -80,7 +80,8 @@ int vgic_v5_probe(const struct gic_kvm_info *info) int ret; kvm_vgic_global_state.type = VGIC_V5; - kvm_vgic_global_state.max_gic_vcpus = VGIC_V5_MAX_CPUS; + kvm_vgic_global_state.max_gic_vcpus = 0; + kvm_vgic_global_state.max_gicv5_vcpus = 0; kvm_vgic_global_state.vcpu_base = 0; kvm_vgic_global_state.vctrl_base = NULL; @@ -105,8 +106,8 @@ int vgic_v5_probe(const struct gic_kvm_info *info) * Even if the HW supports more per-VM vCPUs, artificially cap as we * can't use them all. */ - kvm_vgic_global_state.max_gic_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps), - VGIC_V5_MAX_CPUS); + kvm_vgic_global_state.max_gicv5_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps), + VGIC_V5_MAX_CPUS); /* * GICv5 requires a set of tables to be allocated in order to manage @@ -115,7 +116,7 @@ int vgic_v5_probe(const struct gic_kvm_info *info) * we want to run. For now, we match the maximum number offered by the * hardware, but this might not be a wise choice in the long term. */ - ret = vgic_v5_vmt_allocate(kvm_vgic_global_state.max_gic_vcpus); + ret = vgic_v5_vmt_allocate(kvm_vgic_global_state.max_gicv5_vcpus); if (ret) { kvm_err("Failed to allocate the GICv5 VM tables; no GICv5 support\n"); return -ENODEV; @@ -150,6 +151,8 @@ int vgic_v5_probe(const struct gic_kvm_info *info) } v5_registered = true; + kvm_vgic_global_state.max_gic_vcpus = + kvm_vgic_global_state.max_gicv5_vcpus; kvm_info("GCIE system register CPU interface\n"); skip_v5: @@ -175,9 +178,8 @@ int vgic_v5_probe(const struct gic_kvm_info *info) return v5_registered ? 0 : ret; } - /* We potentially limit the max VCPUs further than we need to here */ - kvm_vgic_global_state.max_gic_vcpus = min(VGIC_V3_MAX_CPUS, - kvm_vgic_global_state.max_gic_vcpus); + kvm_vgic_global_state.max_gic_vcpus = max(kvm_vgic_global_state.max_gic_vcpus, + VGIC_V3_MAX_CPUS); static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif); kvm_info("GCIE legacy system register CPU interface\n"); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 3ec3f7e1b88b..593145760812 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -172,9 +172,16 @@ struct vgic_global { /* Maintenance IRQ number */ unsigned int maint_irq; - /* maximum number of VCPUs allowed (GICv2 limits us to 8) */ + /* + * Maximum number of VCPUs exposed before userspace has selected a + * VGIC model. Individual VGIC models can impose a lower limit + * (GICv2 limits us to 8). + */ int max_gic_vcpus; + /* Maximum number of VCPUs allowed for a GICv5 VM. */ + int max_gicv5_vcpus; + /* Only needed for the legacy KVM_CREATE_IRQCHIP */ bool can_emulate_gicv2; @@ -622,10 +629,11 @@ void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu); void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1); /** - * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW + * kvm_vgic_get_max_vcpus - Get the pre-VGIC-selection VCPU limit * - * The host's GIC naturally limits the maximum amount of VCPUs a guest - * can use. + * Userspace can query KVM_CAP_MAX_VCPUS before selecting a VGIC model, so + * expose the highest model-specific limit and let kvm_vgic_create() enforce + * the selected model's actual limit. */ static inline int kvm_vgic_get_max_vcpus(void) { From d749533c006d4e6bd4413a8c99768cdd4f4e706d Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:40:53 +0000 Subject: [PATCH 127/506] KVM: arm64: gic-v5: Implement VPE IRS MMIO Ops Introduce interfaces to make VPEs valid, and to configure them, via the host's IRS. As with the other valid bits in the GICv5 VM tables, VPEs cannot be made valid directly, and instead are made valid via an IRS MMIO Op. Additionally, some of the VPE configuration takes place via the IRS MMIO interface too (via the IRS_VPE_CR0, IRS_VPE_DBR). VPE doorbells are, for example, configured via this interface. The existing VPE-doorbell-based commands are extended with: VPE_MAKE_VALID - Make the VPE valid in the VPET Note: There is no VPE_MAKE_INVALID as VPEs are only made invalid on teardown, at which point the whole VMTE is marked as invalid. Hence, it is not required. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-14-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5.c | 84 ++++++++++++++++++++++++++++++ include/linux/irqchip/arm-gic-v5.h | 1 + 2 files changed, 85 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index e1571d1788a6..455161e52f81 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -201,6 +201,18 @@ static int vgic_v5_irs_wait_for_vm_op(void) NULL); } +/* + * Wait for completion of a change in any of IRS_VPE_SELR, IRS_VPE_DBR, + * IRS_VPE_CR0. + */ +static int vgic_v5_irs_wait_for_vpe_op(void) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_VPE_STATUSR, + GICV5_IRS_VPE_STATUSR_IDLE, + NULL); +} + static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset) { int ret; @@ -308,10 +320,73 @@ static int vgic_v5_irs_set_vist_invalid(u16 vm_id, bool spi_ist) return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, true); } +static int vgic_v5_irs_set_up_vpe(u16 vm_id, u16 vpe_id, + irq_hw_number_t db_hwirq) +{ + u64 vmap_vper, dbr, selr; + u32 statusr, cr0; + int ret; + + lockdep_assert_held(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + /* Mark the VPE as valid */ + vmap_vper = FIELD_PREP(GICV5_IRS_VMAP_VPER_VPE_ID, vpe_id) | + FIELD_PREP(GICV5_IRS_VMAP_VPER_VM_ID, vm_id) | + GICV5_IRS_VMAP_VPER_M; + irs_writeq_relaxed(vmap_vper, GICV5_IRS_VMAP_VPER); + + /* Wait for the VPE to be marked valid in the VPET */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + selr = FIELD_PREP(GICV5_IRS_VPE_SELR_VPE_ID, vpe_id) | + FIELD_PREP(GICV5_IRS_VPE_SELR_VM_ID, vm_id) | + GICV5_IRS_VPE_SELR_S; + irs_writeq_relaxed(selr, GICV5_IRS_VPE_SELR); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + statusr = irs_readl_relaxed(GICV5_IRS_VPE_STATUSR); + if (!FIELD_GET(GICV5_IRS_VPE_STATUSR_V, statusr)) + return -EINVAL; + + /* Set targeted only routing (disable 1ofN vPE selection) */ + cr0 = GICV5_IRS_VPE_CR0_DPS; + irs_writel_relaxed(cr0, GICV5_IRS_VPE_CR0); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + /* + * The VPE has not yet run. Therefore, make sure that all interrupts + * will generate a doorbell. + */ + dbr = FIELD_PREP(GICV5_IRS_VPE_DBR_INTID, db_hwirq) | + GICV5_IRS_VPE_DBR_DBV; + irs_writeq_relaxed(dbr, GICV5_IRS_VPE_DBR); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + return 0; +} + static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) { struct vgic_v5_vm *vm = data->domain->host_data; enum gicv5_vcpu_cmd *cmd = vcpu_info; + /* Our VPE ID is the index within the doorbell domain */ + u16 vpe_id = data->hwirq; guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); @@ -322,6 +397,15 @@ static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) return vgic_v5_irs_set_vm_valid(vm->vm_id); case VMTE_MAKE_INVALID: return vgic_v5_irs_set_vm_invalid(vm->vm_id); + case VPE_MAKE_VALID: + /* + * We need the actual LPI ID which lives in the top-most parent + * domain. This hwirq won't include the type (LPI) but that's + * not required for the IRS_VPE_DBR. + */ + while (data->parent_data) + data = data->parent_data; + return vgic_v5_irs_set_up_vpe(vm->vm_id, vpe_id, data->hwirq); case SPI_VIST_MAKE_VALID: return vgic_v5_irs_set_vist_valid(vm->vm_id, true); case LPI_VIST_MAKE_VALID: diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 24949b604531..299e75bc9525 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -661,6 +661,7 @@ enum gicv5_vcpu_cmd { VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ VMTE_MAKE_VALID, /* Make the VMTE valid */ VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ + VPE_MAKE_VALID, /* No corresponding invalid */ SPI_VIST_MAKE_VALID, /* No corresponding invalid */ LPI_VIST_MAKE_VALID, /* Triggered by a guest */ LPI_VIST_MAKE_INVALID, /* Triggered by a guest */ From 5c3966b2ecebcaef1c40f54e2c0b87b1deafb051 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:41:23 +0000 Subject: [PATCH 128/506] KVM: arm64: gic-v5: Set up VMTEs and VPE doorbells A GICv5 VM needs a VM table entry before it can use SPIs and LPIs, which are backed by the host IRS. The VM table itself is created at probe time, but each VM still needs to claim and populate one VMTE before it can use those interrupts. VPE doorbells are allocated from the host LPI irq domain. Without that domain, KVM cannot issue IRS commands or receive doorbell wakeups. Fail the GICv5 KVM probe if the host driver did not create an LPI domain. Allocate a VM ID during vgic_v5_init(). The VM ID is also the index into the VM table, so allocating it selects the VMTE slot that will be used for the lifetime of the VM. Create a per-VM VPE doorbell irq domain, allocate one doorbell interrupt per vCPU, request the interrupts, and keep the doorbell IRQ number in the vCPU's GICv5 state. The doorbell handler marks the VPE doorbell as fired, raises KVM_REQ_IRQ_PENDING, and kicks the target vCPU so that KVM can re-evaluate pending interrupt state. The doorbell domain indexes its interrupts using the dense vcpu_idx, while GICv5 uses the userspace-provided vcpu_id as the VPE ID. Store a backpointer to the VM and use it to resolve the vCPU before issuing VPE-specific IRS commands. This preserves the userspace VPE ID when vCPU IDs are sparse. With the VM ID and doorbells in place, initialise the VMTE backing state, including the VM descriptor, VPE table, and preallocated VPED storage. The doorbells have to exist before making the VMTE valid, as they provide the IRQ-side conduit used by the IRS commands. Make the VMTE valid via the IRS, then populate the VPETE for each vCPU. Add vgic_v5_teardown() to unwind the state in the reverse order. Make the VMTE invalid, clear the per-vCPU VPETEs, release the VMTE backing state, free the doorbell IRQs and irq domain, and finally release the VM ID so that the VMTE slot can be reused by a later VM. If invalidating the VMTE, clearing a VPETE, or releasing the VMTE fails, still free the software doorbells and domain, but keep the VM ID allocated so that the VMTE slot is not reused while hardware-visible state may remain. On init failure, call the same teardown path so that partially created state is unwound consistently. As part of resetting vCPUs, mark them as valid in the VM's VPE table. This informs the IRS that a specific VPE may be made resident. Without this, the IRS will treat the VPE as invalid. Also introduce vgic_v5_send_command(), a wrapper around the VPE doorbells. It takes a struct kvm_vcpu pointer and the command to run, and invokes the function bound to that command through the vCPU's doorbell. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-15-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5.c | 229 +++++++++++++++++++++++++++++----- include/kvm/arm_vgic.h | 2 + 2 files changed, 198 insertions(+), 33 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 455161e52f81..5b599821b2da 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -99,6 +99,11 @@ int vgic_v5_probe(const struct gic_kvm_info *info) goto skip_v5; } + if (!gicv5_global_data.lpi_domain) { + kvm_err("GICv5 LPI domain unavailable\n"); + return -ENODEV; + } + vgic_v5_irs_cache_id_regs(info); vgic_v5_get_implemented_ppis(); @@ -381,12 +386,35 @@ static int vgic_v5_irs_set_up_vpe(u16 vm_id, u16 vpe_id, return 0; } +static irqreturn_t db_handler(int irq, void *data) +{ + struct kvm_vcpu *vcpu = data; + + WRITE_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired, true); + + kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); + kvm_vcpu_kick(vcpu); + + return IRQ_HANDLED; +} + +static int vgic_v5_send_command(struct kvm_vcpu *vcpu, enum gicv5_vcpu_cmd cmd) +{ + int irq = vgic_v5_vpe_db(vcpu); + + if (!irq) + return -ENXIO; + + return irq_set_vcpu_affinity(irq, &cmd); +} + static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) { struct vgic_v5_vm *vm = data->domain->host_data; enum gicv5_vcpu_cmd *cmd = vcpu_info; - /* Our VPE ID is the index within the doorbell domain */ - u16 vpe_id = data->hwirq; + unsigned int vcpu_idx = data->hwirq; + struct kvm_vcpu *vcpu; + u16 vpe_id; guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); @@ -398,6 +426,17 @@ static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) case VMTE_MAKE_INVALID: return vgic_v5_irs_set_vm_invalid(vm->vm_id); case VPE_MAKE_VALID: + /* + * The index in the doorbell domain aligns with our flat + * vcpu_idx index. However, we need the actual VPE ID, which + * means we first need to resolve the actual vcpu. + */ + vcpu = kvm_get_vcpu(vm->kvm, vcpu_idx); + if (!vcpu) + return -EINVAL; + + vpe_id = vgic_v5_vpe_id(vcpu); + /* * We need the actual LPI ID which lives in the top-most parent * domain. This hwirq won't include the type (LPI) but that's @@ -453,15 +492,9 @@ static int vgic_v5_irq_db_domain_alloc(struct irq_domain *domain, void *arg) { const struct irq_chip *chip = &vgic_v5_db_irq_chip; - struct vgic_v5_vm *vm = arg; struct irq_data *irqd; int ret; - if (!vm) { - kvm_err("invalid parameter for doorbell irq allocation\n"); - return -EINVAL; - } - ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL); if (ret) return ret; @@ -488,20 +521,10 @@ static int vgic_v5_create_per_vm_domain(struct kvm *kvm) int id = task_pid_nr(current); int ret, db_virq = 0; - if (!gicv5_global_data.lpi_domain) { - kvm_err("LPI domain uninitialized, can't set up KVM Doorbells\n"); - return -ENODEV; - } - vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv5-vpe-db", id); if (!vm->fwnode) return -ENOMEM; - /* - * KVM per-VM VPE DB domain; child of LPI domain; only ever handles - * doorbells. We know how many doorbells we have, and therefore we - * create a linear domain. - */ vm->domain = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain, 0, nr_vcpus, vm->fwnode, &vgic_v5_irq_db_domain_ops, vm); @@ -538,11 +561,6 @@ static void vgic_v5_teardown_per_vm_domain(struct vgic_v5_vm *vm) if (!vm->domain) return; - if (vm->vpe_db_base) { - irq_domain_free_irqs(vm->vpe_db_base, vm->domain->revmap_size); - vm->vpe_db_base = 0; - } - irq_domain_remove(vm->domain); irq_domain_free_fwnode(vm->fwnode); vm->domain = NULL; @@ -562,33 +580,122 @@ void vgic_v5_reset(struct kvm_vcpu *vcpu) * CPUIF (but potentially fewer in the IRS). */ vcpu->arch.vgic_cpu.num_pri_bits = 5; + + /* Make the VPE valid in the VPET */ + if (WARN_ON(vgic_v5_send_command(vcpu, VPE_MAKE_VALID))) + return; +} + +static void vgic_v5_free_doorbells(struct kvm *kvm, unsigned int nr_dbs) +{ + struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm; + struct kvm_vcpu *vcpu; + unsigned long i; + int db; + + for (i = 0; i < nr_dbs; i++) { + vcpu = kvm_get_vcpu(kvm, i); + db = vgic_v5_vpe_db(vcpu); + if (!db) + continue; + + free_irq(db, vcpu); + vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = 0; + } + + if (vm->vpe_db_base) { + irq_domain_free_irqs(vm->vpe_db_base, + atomic_read(&kvm->online_vcpus)); + vm->vpe_db_base = 0; + } } void vgic_v5_teardown(struct kvm *kvm) { + struct vgic_dist *dist = &kvm->arch.vgic; + struct kvm_vcpu *vcpu, *vcpu0; + bool release_vm_id = true; + unsigned long i; + int rc; + + lockdep_assert_held(&kvm->arch.config_lock); + + /* + * If the VM's ID isn't valid, then we either failed init very early or + * we've been called a second time. Nothing to do here in either case. + */ + if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL) + return; + + if (kvm->arch.vgic.gicv5_vm.vmte_allocated) { + /* Make the VM invalid */ + vcpu0 = kvm_get_vcpu(kvm, 0); + rc = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID); + if (rc) { + kvm_err("could not make VMTE invalid\n"); + release_vm_id = false; + goto out_free_doorbells; + } + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vgic_v5_vmte_free_vpe(vcpu)) { + kvm_err("Failed to free VPE\n"); + release_vm_id = false; + goto out_free_doorbells; + } + } + + if (vgic_v5_vmte_release(kvm)) { + kvm_err("Failed to release VM 0x%x\n", dist->gicv5_vm.vm_id); + release_vm_id = false; + } + } + +out_free_doorbells: + vgic_v5_free_doorbells(kvm, atomic_read(&kvm->online_vcpus)); vgic_v5_teardown_per_vm_domain(&kvm->arch.vgic.gicv5_vm); + + /* + * We only release the VM ID itself if we didn't fail earlier. It does + * mean that we might lose the VM ID (and associated VMTE, etc), but + * given that we've failed to tear them down correctly there's no way to + * safely reuse them. The VM ID allocating IDA will make sure we don't + * accidentally reuse this partially torn down state. + */ + if (release_vm_id) + vgic_v5_release_vm_id(kvm); } +/* + * Claim and populate a VMTE (optionally making a new L2 VMT valid), create VPE + * doorbells, allocate VPET and populate for each VPE. + * + * Note: We do need to put the cart before the horse here. The VPE doorbells are + * our conduit for communication with the IRS, which means we need to have those + * before making the VMTE valid. + * + * On failure, we clean up in the teardown path (vgic_v5_teardown()). + */ int vgic_v5_init(struct kvm *kvm) { - struct kvm_vcpu *vcpu; - unsigned long idx; - int ret; + struct kvm_vcpu *vcpu, *vcpu0; + int nr_vcpus, ret = 0; + unsigned int db_virq; + unsigned long i; - if (vgic_initialized(kvm)) - return 0; + lockdep_assert_held(&kvm->arch.config_lock); - kvm_for_each_vcpu(idx, vcpu, kvm) { + nr_vcpus = atomic_read(&kvm->online_vcpus); + if (nr_vcpus == 0) + return -ENODEV; + + kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu_has_nv(vcpu)) { kvm_err("Nested GICv5 VMs are currently unsupported\n"); return -EINVAL; } } - ret = vgic_v5_create_per_vm_domain(kvm); - if (ret) - return ret; - /* We only allow userspace to drive the SW_PPI, if it is implemented. */ bitmap_zero(kvm->arch.vgic.gicv5_vm.userspace_ppis, VGIC_V5_NR_PRIVATE_IRQS); @@ -597,7 +704,63 @@ int vgic_v5_init(struct kvm *kvm) kvm->arch.vgic.gicv5_vm.userspace_ppis, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + ret = vgic_v5_allocate_vm_id(kvm); + if (ret) + return ret; + + /* + * Stash a backpointer to struct kvm. It is required to resolve the VPE + * ID from the doorbell index, which matches the flat vcpu_idx and not + * the vcpu_id. + */ + kvm->arch.vgic.gicv5_vm.kvm = kvm; + + ret = vgic_v5_create_per_vm_domain(kvm); + if (ret) + goto err; + + db_virq = kvm->arch.vgic.gicv5_vm.vpe_db_base; + kvm_for_each_vcpu(i, vcpu, kvm) { + ret = request_irq(db_virq + i, db_handler, 0, "vcpu", vcpu); + if (ret) + goto err; + + /* Stash it with the VCPU for easy retrieval */ + vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = db_virq + i; + } + + /* Populate VMTE (with VPET and VM descriptor) */ + ret = vgic_v5_vmte_init(kvm); + if (ret) + goto err; + + /* We pick the first vcpu to make the VMTE valid - any would do */ + vcpu0 = kvm_get_vcpu(kvm, 0); + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID); + if (ret) + goto err; + + /* Populate the VPETE for each VPE. */ + kvm_for_each_vcpu(i, vcpu, kvm) { + ret = vgic_v5_vmte_alloc_vpe(vcpu); + if (ret) + goto err; + } + return 0; + +err: + /* + * Explicitly tear everything down on failure. The teardown function is + * written to handle any partial state we might have, so we don't need + * to do any clean-up first. Teardown will be called a second time on VM + * destruction, but that's fine - it is better to leave things in a + * clean state now, and doubly so because userspace could actually go + * and retry init. + */ + vgic_v5_teardown(kvm); + + return ret; } int vgic_v5_map_resources(struct kvm *kvm) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 593145760812..a00d5151536b 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -371,6 +371,8 @@ struct vgic_redist_region { #define VGIC_V5_VM_ID_INVAL (-1) struct vgic_v5_vm { + struct kvm *kvm; + /* * We only expose a subset of PPIs to the guest. This subset is a * combination of the PPIs that are actually implemented and what we From dc5e201aba78c8977336f71b9eb4eae9f72654ba Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:41:53 +0000 Subject: [PATCH 129/506] KVM: arm64: gic-v5: Add resident/non-resident hyp calls GICv5 introduces the concept of VPE residency - a VPE can be either resident or non-resident. When the VPE is resident, the IRS is allowed to select interrupts that target that VPE (or the VM) as the HPPI (Highest Priority Pending Interrupt). As the IRS handles both SPIs and LPIs, these will only be picked as the IRS's HPPI when a VPE is resident. A GICv5 VPE is made resident by writing ICH_CONTEXTR_EL2 with ICH_CONTEXTR_EL2.V set, together with valid VM and VPE IDs. This informs the IRS that a specific VPE is running, and that it can begin HPPI selection for that VPE. Making a VPE non-resident (by making the ICH_CONTEXTR_EL2 invalid) informs the IRS that the VPE is no longer running, and it stops HPPI selection for it. This change introduces two new hyp calls - one to make a VPE resident and its counterpart to make a VPE non-resident. As part of making a VPE resident, the resulting ICH_CONTEXTR_EL2.F bit is checked to catch residency faults. Such a fault indicates a broken VM/VPE setup, so warn and mark the VM dead. Both of these new hypercalls are explicitly no-ops with pKVM as we currently don't support the combination of GICv5 and pKVM. Furthermore, this change extends vgic_v5_load() and vgic_v5_put() to make the VPEs resident and non-resident, respectively. Hence, the VPE is considered resident for the entire load-to-put interval. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-16-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 2 ++ arch/arm64/include/asm/kvm_hyp.h | 2 ++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 22 ++++++++++++++++++++++ arch/arm64/kvm/hyp/vgic-v5-sr.c | 26 ++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v5.c | 16 ++++++++++++++-- include/kvm/arm_vgic.h | 3 +++ 6 files changed, 69 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index e5b92ac09e69..55f7c5c74d1a 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -87,6 +87,8 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___tracing_write_event, __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, + __KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident, + __KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident, __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 4974492744cc..8fcbbff719cc 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -88,6 +88,8 @@ void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if); int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu); /* GICv5 */ +void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if); +void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_vmcr_apr(struct vgic_v5_cpu_if *cpu_if); /* No hypercalls for the following */ diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9a3b92e626ad..cd709eb426b5 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -885,6 +885,26 @@ static void handle___tracing_write_event(struct kvm_cpu_context *host_ctxt) trace_selftest(id); } +static void handle___vgic_v5_make_resident(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); + + if (unlikely(is_protected_kvm_enabled())) + return; + + __vgic_v5_make_resident(kern_hyp_va(cpu_if)); +} + +static void handle___vgic_v5_make_non_resident(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); + + if (unlikely(is_protected_kvm_enabled())) + return; + + __vgic_v5_make_non_resident(kern_hyp_va(cpu_if)); +} + static void handle___vgic_v5_save_apr(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); @@ -932,6 +952,8 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__tracing_write_event), HANDLE_FUNC(__vgic_v3_save_aprs), HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), + HANDLE_FUNC(__vgic_v5_make_resident), + HANDLE_FUNC(__vgic_v5_make_non_resident), HANDLE_FUNC(__vgic_v5_save_apr), HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), diff --git a/arch/arm64/kvm/hyp/vgic-v5-sr.c b/arch/arm64/kvm/hyp/vgic-v5-sr.c index 6d69dfe89a96..f064045a31ae 100644 --- a/arch/arm64/kvm/hyp/vgic-v5-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v5-sr.c @@ -7,6 +7,32 @@ #include +void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if) +{ + write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2); + isb(); + + /* Catch any faults */ + cpu_if->vgic_contextr = read_sysreg_s(SYS_ICH_CONTEXTR_EL2); + if (!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr)) + return; + + cpu_if->gicv5_vpe.resident = true; +} + +void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if) +{ + /* + * Make as non-resident before actually making non-resident. Avoids race + * with doorbell arriving. + */ + cpu_if->gicv5_vpe.resident = false; + dsb(st); + + write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2); + isb(); +} + void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if) { cpu_if->vgic_apr = read_sysreg_s(SYS_ICH_APR_EL2); diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 5b599821b2da..d7d6db501426 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -1054,6 +1054,8 @@ void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu) void vgic_v5_load(struct kvm_vcpu *vcpu) { struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + u16 vm = vgic_v5_vm_id(vcpu->kvm); + u16 vpe = vgic_v5_vpe_id(vcpu); /* * On the WFI path, vgic_load is called a second time. The first is when @@ -1066,7 +1068,15 @@ void vgic_v5_load(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_restore_vmcr_apr, cpu_if); - cpu_if->gicv5_vpe.resident = true; + cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_V, true) | + FIELD_PREP(ICH_CONTEXTR_EL2_VPE, vpe) | + FIELD_PREP(ICH_CONTEXTR_EL2_VM, vm); + + kvm_call_hyp(__vgic_v5_make_resident, cpu_if); + + /* Failed to make the VPE resident? Bang! */ + if (WARN_ON(!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr))) + kvm_vm_dead(vcpu->kvm); } void vgic_v5_put(struct kvm_vcpu *vcpu) @@ -1084,7 +1094,9 @@ void vgic_v5_put(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_save_apr, cpu_if); - cpu_if->gicv5_vpe.resident = false; + cpu_if->vgic_contextr = 0; + + kvm_call_hyp(__vgic_v5_make_non_resident, cpu_if); /* The shadow priority is only updated on entering WFI */ if (vcpu_get_flag(vcpu, IN_WFI)) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index a00d5151536b..aadd47c959b9 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -525,6 +525,9 @@ struct vgic_v5_cpu_if { */ u64 vgic_icsr; + /* The contextr used to make VPEs resident and non-resident */ + u64 vgic_contextr; + struct gicv5_vpe gicv5_vpe; }; From 2111261ac1c8cdd6d271c8a10860943720504241 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:42:24 +0000 Subject: [PATCH 130/506] KVM: arm64: gic-v5: Request doorbells when VPEs enter WFI When a GICv5 VPE is made non-resident as part of the vcpu entering WFI, request a VPE doorbell so that KVM can be notified when a suitable SPI or LPI becomes pending for that VPE. Program the doorbell priority mask, DBPM, from the effective virtual priority mask before making the VPE non-resident. DBPM is the priority threshold used by the GICv5 hardware to decide whether a pending SPI or LPI is allowed to signal the VPE doorbell. This allows hardware to signal the doorbell only for interrupts that the vcpu can actually take, and avoids waking it for interrupts masked by the guest priority state. If no interrupt can be signalled to the vcpu, leave the doorbell request clear. Attempt to make the doorbell interrupt affine to the current CPU before requesting it. Treat affinity failures as non-fatal and continue using the existing placement. A successful move nudges the wakeup back towards the CPU that last ran the vcpu, where the relevant state is more likely to be cache-hot, while also spreading doorbell interrupts across host PEs as different vcpus enter WFI on different CPUs. Clear stale db_fired state before making the VPE non-resident. Any previous doorbell notification has already been consumed by this point, and clearing it before the non-resident transition ensures that a newly fired doorbell is observed. Finally, teach kvm_vgic_vcpu_pending_irq() to report pending work for a GICv5 vcpu when its VPE doorbell has fired, in addition to the existing pending-PPI check. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-17-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/vgic-v5-sr.c | 9 +++++++ arch/arm64/kvm/vgic/vgic-v5.c | 46 +++++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.c | 6 ++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/vgic-v5-sr.c b/arch/arm64/kvm/hyp/vgic-v5-sr.c index f064045a31ae..46992a6c2cac 100644 --- a/arch/arm64/kvm/hyp/vgic-v5-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v5-sr.c @@ -22,6 +22,15 @@ void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if) void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if) { + /* + * Clear the db_fired state to ensure that we're ready for the next + * doorbell when it is requested. If a doorbell firing caused us to + * enter the guest, then we've already consumed that state at this + * point, so this is safe to clear. Use WRITE_ONCE() to ensure we're not + * racing with the doorbell firing and setting the state true again. + */ + WRITE_ONCE(cpu_if->gicv5_vpe.db_fired, false); + /* * Make as non-resident before actually making non-resident. Avoids race * with doorbell arriving. diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index d7d6db501426..5a058ef8fe70 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -1095,6 +1095,52 @@ void vgic_v5_put(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_save_apr, cpu_if); cpu_if->vgic_contextr = 0; + if (vcpu_get_flag(vcpu, IN_WFI)) { + u32 priority_mask; + int dbpm; + + /* + * Find the virtual running priority and use this to calculate + * the doorbell priority mask. We combine the highest active + * priority and the CPU's priority mask. The guest can't handle + * interrupts with priorities less than or equal to the virtual + * running priority, so there's literally no point in waking the + * guest for these. + * + * The priority needs to be higher than the mask to signal, so + * pick the next higher priority (subtract 1). + */ + priority_mask = vgic_v5_get_effective_priority_mask(vcpu); + + /* + * Request a doorbell *unless* the priority is 0, indicating + * that no interrupt can wake the CPU up. + */ + if (priority_mask) { + int db_irq = vgic_v5_vpe_db(vcpu); + struct irq_data *d = irq_get_irq_data(db_irq); + const struct cpumask *aff = irq_data_get_effective_affinity_mask(d); + int cpu = smp_processor_id(); + + dbpm = priority_mask - 1; + cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_DB, 1) | + FIELD_PREP(ICH_CONTEXTR_EL2_DBPM, dbpm); + + /* + * Make the doorbell affine to this CPU, if it isn't + * already. Actively check the cpumask first as it is + * cheaper than changing the affinity every time. + */ + if (!cpumask_test_cpu(cpu, aff)) { + int ret; + + ret = irq_set_affinity(db_irq, cpumask_of(cpu)); + if (ret) + kvm_debug_ratelimited("Failed to move doorbell IRQ %d to CPU %d: %d\n", + db_irq, cpu, ret); + } + } + } kvm_call_hyp(__vgic_v5_make_non_resident, cpu_if); diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index b25303d9919f..ca4fb45f7d50 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -1236,8 +1236,12 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu) unsigned long flags; struct vgic_vmcr vmcr; - if (vgic_is_v5(vcpu->kvm)) + if (vgic_is_v5(vcpu->kvm)) { + if (READ_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired)) + return true; + return vgic_v5_has_pending_ppi(vcpu); + } if (!vcpu->kvm->arch.vgic.enabled) return false; From 297e838887a56700098f69891970f2b0949cde11 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:42:54 +0000 Subject: [PATCH 131/506] KVM: arm64: gic-v5: Introduce struct vgic_v5_irs and IRS base address In order to properly emulate the operation of the IRS from KVM, we require storage for the MMIO register state. This change introduces struct vgic_v5_irs, and adds a pointer to it to the struct vgic_dist. This new data structure contains the storage for IRS MMIO state that is required for emulating the MMIO interface in KVM. This provides persistent storage, and a way to track data across MMIO writes, e.g., selecting an SPI and updating the configuration of it is two MMIO writes. Note that only a pointer to the data structure is added to struct vgic_dist as this new structure is very large, and hence it makes sense to dynamically allocate it and just provide a pointer to retrieve it in struct vgic_dist. In addition to adding a structure to store the MMIO state for the IRS, we add the base address in GPA space to struct vgic_v5_irs. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-18-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- include/kvm/arm_vgic.h | 86 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index aadd47c959b9..b62ae14c6a06 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -398,6 +398,87 @@ struct vgic_v5_vm { bool vmte_allocated; }; +/*** GICv5 ***/ +struct vgic_v5_irs { + /* base addresses in guest physical address space: */ + gpa_t vgic_v5_irs_base; + + struct vgic_io_device iodev; + struct kvm_device *dev; + + /* IRS state - used for registers etc */ + struct { + u8 domain; + u8 pa_range; + bool virt; + bool setlpi; + bool mec; + bool mpam; + bool swe; + u16 irs_id; + } idr0; + + struct { + /* PE_CNT is populated from online_vcpus at runtime */ + u8 priority_bits; + } idr1; + + struct { + u8 id_bits; + u8 min_lpi_id_bits; + bool ist_levels; + u8 ist_l2sz; + bool istmd; + u8 istmd_sz; + } idr2; + + struct { + u32 spi_range; + } idr5; + + struct { + u32 spi_irs_range; + } idr6; + + struct { + u32 spi_base; + } idr7; + + struct { + u8 sh; + u8 oc; + u8 ic; + bool ist_ra; + bool ist_wa; + bool vmt_ra; + bool vpet_ra; + bool vmd_ra; + bool vmd_wa; + bool vped_ra; + bool vped_wa; + } cr1; + + struct { + u32 id; + } spi_selr; + + struct { + u32 iaffid; + } pe_selr; + + struct { + u8 lpi_id_bits; + u8 l2sz; + u8 istsz; + bool structure; + } ist_cfgr; + + struct { + bool valid; + u64 addr; + } ist_baser; +}; + struct vgic_dist { bool in_kernel; bool ready; @@ -475,6 +556,11 @@ struct vgic_dist { * GICv5 per-VM data. */ struct vgic_v5_vm gicv5_vm; + + /* + * GICv5 IRS data. Dynamically allocated due to the size. + */ + struct vgic_v5_irs *vgic_v5_irs_data; }; struct vgic_v2_cpu_if { From 11070de575623735ca1040c9b3c42086fd5eaba9 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:43:25 +0000 Subject: [PATCH 132/506] KVM: arm64: gic-v5: Add IRS IODEV support to MMIO handlers In order to support proper VMs (that support more than just PPIs) for GICv5, it is important to emulate the GICv5 IRS too. The IRS includes an MMIO interface which is used to interact with and configure the IRS. As part of providing the emulated IRS MMIO interface in KVM, extend enum iodev_type to include a GICv5 IRS device, and extend the MMIO code to handle reads and writes to that type of IO device. This will allow the creation of a GICv5 IRS IO Device in KVM. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-19-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-mmio.c | 6 ++++++ arch/arm64/kvm/vgic/vgic-mmio.h | 2 ++ include/kvm/arm_vgic.h | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index 74d76dec9730..fddb9da0403d 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c @@ -1065,6 +1065,9 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, case IODEV_ITS: data = region->its_read(vcpu->kvm, iodev->its, addr, len); break; + case IODEV_GICV5_IRS: + data = region->read(vcpu, addr, len); + break; } vgic_data_host_to_mmio_bus(val, len, data); @@ -1095,6 +1098,9 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, case IODEV_ITS: region->its_write(vcpu->kvm, iodev->its, addr, len, data); break; + case IODEV_GICV5_IRS: + region->write(vcpu, addr, len, data); + break; } return 0; diff --git a/arch/arm64/kvm/vgic/vgic-mmio.h b/arch/arm64/kvm/vgic/vgic-mmio.h index 50dc80220b0f..38ed730d68ac 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.h +++ b/arch/arm64/kvm/vgic/vgic-mmio.h @@ -217,6 +217,8 @@ unsigned int vgic_v2_init_cpuif_iodev(struct vgic_io_device *dev); unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev); +unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev); + u64 vgic_sanitise_outer_cacheability(u64 reg); u64 vgic_sanitise_inner_cacheability(u64 reg); u64 vgic_sanitise_shareability(u64 reg); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index b62ae14c6a06..d894c32d0929 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -310,7 +310,8 @@ enum iodev_type { IODEV_CPUIF, IODEV_DIST, IODEV_REDIST, - IODEV_ITS + IODEV_ITS, + IODEV_GICV5_IRS }; struct vgic_io_device { From f6732cd67a025d8d216ee591ba5705a99ab974a1 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:43:55 +0000 Subject: [PATCH 133/506] KVM: arm64: gic-v5: Add KVM_VGIC_V5_ADDR_TYPE_IRS to UAPI Define the UAPI address type used by userspace to describe the location of the emulated IRS in guest physical address space, together with the size reserved for that region. As per the GICv5 specification, the IRS has one CONFIG_FRAME and optionally one SETLPI_FRAME per interrupt domain. Within a KVM VM we are only concerned with one interrupt domain. Each of these frames is 64kB in size, so reserve 2x64kB of contiguous memory in the GPA space for a GICv5 IRS. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-20-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/uapi/asm/kvm.h | 5 +++++ tools/arch/arm64/include/uapi/asm/kvm.h | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 019e5e3d892e..00cafa118a73 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -97,6 +97,11 @@ struct kvm_regs { #define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) #define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K) +/* Supported VGICv5 address types */ +#define KVM_VGIC_V5_ADDR_TYPE_IRS 6 + +#define KVM_VGIC_V5_IRS_SIZE (2 * SZ_64K) + #define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ #define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ #define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */ diff --git a/tools/arch/arm64/include/uapi/asm/kvm.h b/tools/arch/arm64/include/uapi/asm/kvm.h index 1c13bfa2d38a..d1b2ca317f58 100644 --- a/tools/arch/arm64/include/uapi/asm/kvm.h +++ b/tools/arch/arm64/include/uapi/asm/kvm.h @@ -97,6 +97,11 @@ struct kvm_regs { #define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) #define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K) +/* Supported VGICv5 address types */ +#define KVM_VGIC_V5_ADDR_TYPE_IRS 6 + +#define KVM_VGIC_V5_IRS_SIZE (2 * SZ_64K) + #define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ #define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ #define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */ From f116c96e451b66831e05807d8a75259db041d2a5 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:44:25 +0000 Subject: [PATCH 134/506] KVM: arm64: gic-v5: Add GICv5 IRS IODEV and MMIO emulation In order to properly support GICv5-based VMs in KVM, emulate the CONFIG_FRAME for a virtual IRS. This emulation needs to handle guest accesses to the MMIO region and mimic the behaviour of a real IRS. Introduce an IODEV for the GICv5 IRS and an associated initialisation function that sets up the SPIs and initial IRS state. The MMIO emulation allows the guest to query the IRS_IDx registers, manipulate SPIs, configure ISTs, and so forth. Allow 32-bit accesses to the 64-bit IRS registers in addition to 64-bit accesses. Reconstruct IRS_IST_BASER for reads and merge partial writes so that updating either word preserves the other half. The emulation tracks selector state across MMIO accesses. For example, a guest writes IRS_PE_SELR to select a PE by IAFFID. This is the VPE ID for a VM, but the guest does not know this. If the guest reads IRS_PE_STATUSR, KVM checks whether that IAFFID selects a valid VPE and sets the V bit accordingly. IRS_PE_CR0 is accepted as write-ignored because KVM does not support 1-of-N routing. The same selector and status register model is exposed for SPIs. Track the state of IRS_CR0.IRSEN and introduce KVM_REQ_RELOAD_GICv5 to reload the GICv5 context of running vCPUs when it changes. Only make the request when the enable state changes, avoiding unnecessary IPIs for writes that leave it unchanged. The LPI IST requires KVM to perform actions on behalf of the guest. Treat changes to IRS_IST_BASER.Valid as the lifetime of the guest's IST. On an Invalid-to-Valid transition, validate IRS_IST_CFGR, allocate a shadow host IST, and assign it to the physical IRS through the VMTE. On a Valid-to-Invalid transition, invalidate and free the host IST. Ignore guest address changes while the BASER remains valid, and prevent changes to IRS_IST_CFGR during that time. As far as the guest is concerned, the IST memory it provided is being used by the hardware, but the physical IRS uses the host shadow IST instead. This change provides the core IRS IODEV and MMIO emulation, but does not plumb the device into the rest of KVM yet. The CoreSight identification registers are added separately. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-21-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/Makefile | 2 +- arch/arm64/kvm/arm.c | 8 + arch/arm64/kvm/vgic/vgic-irs-v5.c | 793 +++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v5-tables.c | 16 + arch/arm64/kvm/vgic/vgic-v5-tables.h | 1 + arch/arm64/kvm/vgic/vgic.h | 2 + include/kvm/arm_vgic.h | 1 + 8 files changed, 823 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kvm/vgic/vgic-irs-v5.c diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..1afaf88f21db 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -55,6 +55,7 @@ #define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9) #define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10) #define KVM_REQ_VGIC_PROCESS_UPDATE KVM_ARCH_REQ(11) +#define KVM_REQ_RELOAD_GICv5 KVM_ARCH_REQ(12) #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 431de9b145ca..92dda57c0876 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -24,7 +24,7 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \ vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \ - vgic/vgic-v5.o vgic/vgic-v5-tables.o + vgic/vgic-v5.o vgic/vgic-v5-tables.o vgic/vgic-irs-v5.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..0bce2b4cb7cc 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1159,6 +1159,14 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) preempt_enable(); } + if (kvm_check_request(KVM_REQ_RELOAD_GICv5, vcpu)) { + /* The IRS enable bit was changed */ + preempt_disable(); + vgic_v5_put(vcpu); + vgic_v5_load(vcpu); + preempt_enable(); + } + if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu)) kvm_vcpu_reload_pmu(vcpu); diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c new file mode 100644 index 000000000000..76f5f40ecc1c --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -0,0 +1,793 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 ARM Limited, All Rights Reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vgic.h" +#include "vgic-mmio.h" +#include "vgic-v5-tables.h" + +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static struct vgic_dist *vgic_v5_get_vgic(struct kvm_vcpu *vcpu) +{ + return &vcpu->kvm->arch.vgic; +} + +static struct vgic_v5_irs *vgic_v5_get_irs(struct kvm_vcpu *vcpu) +{ + return vcpu->kvm->arch.vgic.vgic_v5_irs_data; +} + +static unsigned long vgic_v5_mmio_read_irs_misc(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct kvm_vcpu *target_vcpu; + u8 vpe_id_bits; + u64 value = 0; + + switch (offset) { + case GICV5_IRS_IDR0: + value = FIELD_PREP(GICV5_IRS_IDR0_INT_DOM, irs->idr0.domain); + value |= FIELD_PREP(GICV5_IRS_IDR0_PA_RANGE, irs->idr0.pa_range); + if (irs->idr0.virt) + value |= GICV5_IRS_IDR0_VIRT; + if (irs->idr0.setlpi) + value |= GICV5_IRS_IDR0_SETLPI; + if (irs->idr0.mec) + value |= GICV5_IRS_IDR0_MEC; + if (irs->idr0.mpam) + value |= GICV5_IRS_IDR0_MPAM; + if (irs->idr0.swe) + value |= GICV5_IRS_IDR0_SWE; + value |= FIELD_PREP(GICV5_IRS_IDR0_IRSID, irs->idr0.irs_id); + break; + case GICV5_IRS_IDR1: + value = FIELD_PREP(GICV5_IRS_IDR1_PE_CNT, + atomic_read(&vcpu->kvm->online_vcpus)); + /* + * IRS_IDR1 encodes IAFFID_BITS as N - 1. + */ + vpe_id_bits = vgic_v5_vmte_vpe_id_bits(vcpu); + value |= FIELD_PREP(GICV5_IRS_IDR1_IAFFID_BITS, vpe_id_bits - 1); + value |= FIELD_PREP(GICV5_IRS_IDR1_PRIORITY_BITS, irs->idr1.priority_bits); + break; + case GICV5_IRS_IDR2: + value = FIELD_PREP(GICV5_IRS_IDR2_ISTMD_SZ, irs->idr2.istmd_sz); + if (irs->idr2.istmd) + value |= GICV5_IRS_IDR2_ISTMD; + value |= FIELD_PREP(GICV5_IRS_IDR2_IST_L2SZ, irs->idr2.ist_l2sz); + if (irs->idr2.ist_levels) + value |= GICV5_IRS_IDR2_IST_LEVELS; + value |= FIELD_PREP(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, irs->idr2.min_lpi_id_bits); + value |= GICV5_IRS_IDR2_LPI; + value |= FIELD_PREP(GICV5_IRS_IDR2_ID_BITS, irs->idr2.id_bits); + break; + case GICV5_IRS_IDR5: + value = FIELD_PREP(GICV5_IRS_IDR5_SPI_RANGE, irs->idr5.spi_range); + break; + case GICV5_IRS_IDR6: + value = FIELD_PREP(GICV5_IRS_IDR6_SPI_IRS_RANGE, irs->idr6.spi_irs_range); + break; + case GICV5_IRS_IDR7: + value = FIELD_PREP(GICV5_IRS_IDR7_SPI_BASE, irs->idr7.spi_base); + break; + case GICV5_IRS_IIDR: + /* Revision, Variant, ProductID are implementation defined */ + value = FIELD_PREP(GICV5_IRS_IIDR_PRODUCT_ID, PRODUCT_ID_KVM); + value |= FIELD_PREP(GICV5_IRS_IIDR_VARIANT, 0); + value |= FIELD_PREP(GICV5_IRS_IIDR_REVISION, 0); + value |= FIELD_PREP(GICV5_IRS_IIDR_IMPLEMENTER, IMPLEMENTER_ARM); + break; + case GICV5_IRS_AIDR: + value = FIELD_PREP(GICV5_IRS_AIDR_COMPONENT, + GICV5_AIDR_COMPONENT_IRS); + value |= FIELD_PREP(GICV5_IRS_AIDR_ARCHMAJORREV, + GICV5_AIDR_ARCH_MAJ_REV_V5); + value |= FIELD_PREP(GICV5_IRS_AIDR_ARCHMINORREV, + GICV5_AIDR_ARCH_MIN_REV_V0); + break; + case GICV5_IRS_CR0: + /* + * The IRS is ALWAYS idle as we handle things instantaneously + * from a guest's viewpoint. + */ + value = GICV5_IRS_CR0_IDLE; + if (READ_ONCE(vcpu->kvm->arch.vgic.enabled)) + value |= GICV5_IRS_CR0_IRSEN; + break; + case GICV5_IRS_CR1: + if (irs->cr1.vped_wa) + value |= GICV5_IRS_CR1_VPED_WA; + if (irs->cr1.vped_ra) + value |= GICV5_IRS_CR1_VPED_RA; + if (irs->cr1.vmd_wa) + value |= GICV5_IRS_CR1_VMD_WA; + if (irs->cr1.vmd_ra) + value |= GICV5_IRS_CR1_VMD_RA; + if (irs->cr1.vpet_ra) + value |= GICV5_IRS_CR1_VPET_RA; + if (irs->cr1.vmt_ra) + value |= GICV5_IRS_CR1_VMT_RA; + if (irs->cr1.ist_wa) + value |= GICV5_IRS_CR1_IST_WA; + if (irs->cr1.ist_ra) + value |= GICV5_IRS_CR1_IST_RA; + value |= FIELD_PREP(GICV5_IRS_CR1_IC, irs->cr1.ic); + value |= FIELD_PREP(GICV5_IRS_CR1_OC, irs->cr1.oc); + value |= FIELD_PREP(GICV5_IRS_CR1_SH, irs->cr1.sh); + break; + case GICV5_IRS_SYNC_STATUSR: + value = GICV5_IRS_SYNC_STATUSR_IDLE; + break; + case GICV5_IRS_PE_SELR: + value = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, + READ_ONCE(irs->pe_selr.iaffid)); + break; + case GICV5_IRS_PE_STATUSR: + /* We assume that the PE is Online if present. Always IDLE too */ + value = GICV5_IRS_PE_STATUSR_IDLE; + + /* Set ONLINE and V if IAFFID selects a present PE */ + if (kvm_get_vcpu_by_id(vcpu->kvm, + READ_ONCE(irs->pe_selr.iaffid))) { + value |= GICV5_IRS_PE_STATUSR_ONLINE; + value |= GICV5_IRS_PE_STATUSR_V; + } + break; + case GICV5_IRS_PE_CR0: { + u32 iaffid = READ_ONCE(irs->pe_selr.iaffid); + + /* + * Make sure that we are doing something reasonable first. + * Remember, the IAFFID is the same as the VPE_ID + */ + target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, iaffid); + if (!target_vcpu) { + kvm_debug("Guest programmed invalid IAFFID (0x%x) into the IRS_PE_SELR\n", + iaffid); + break; + } + + value = GICV5_IRS_PE_CR0_DPS; + break; + } + default: + return 0; + } + + return value; +} + +static void vgic_v5_mmio_write_irs_misc(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + struct vgic_dist *vgic = vgic_v5_get_vgic(vcpu); + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_IRS_CR0: + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + bool was_enabled = vgic->enabled; + + WRITE_ONCE(vgic->enabled, !!(val & GICV5_IRS_CR0_IRSEN)); + + /* Reload other vCPUs on an edge */ + if (was_enabled != !!(val & GICV5_IRS_CR0_IRSEN)) + kvm_make_all_cpus_request(vcpu->kvm, + KVM_REQ_RELOAD_GICv5); + } + return; + case GICV5_IRS_CR1: + irs->cr1.sh = FIELD_GET(GICV5_IRS_CR1_SH, val); + irs->cr1.oc = FIELD_GET(GICV5_IRS_CR1_OC, val); + irs->cr1.ic = FIELD_GET(GICV5_IRS_CR1_IC, val); + irs->cr1.ist_ra = !!(val & GICV5_IRS_CR1_IST_RA); + irs->cr1.ist_wa = !!(val & GICV5_IRS_CR1_IST_WA); + irs->cr1.vmt_ra = !!(val & GICV5_IRS_CR1_VMT_RA); + irs->cr1.vpet_ra = !!(val & GICV5_IRS_CR1_VPET_RA); + irs->cr1.vmd_ra = !!(val & GICV5_IRS_CR1_VMD_RA); + irs->cr1.vmd_wa = !!(val & GICV5_IRS_CR1_VMD_WA); + irs->cr1.vped_ra = !!(val & GICV5_IRS_CR1_VPED_RA); + irs->cr1.vped_wa = !!(val & GICV5_IRS_CR1_VPED_WA); + return; + case GICV5_IRS_PE_SELR: + WRITE_ONCE(irs->pe_selr.iaffid, + FIELD_GET(GICV5_IRS_PE_SELR_IAFFID, val)); + return; + case GICV5_IRS_PE_CR0: + /* + * We actually have nothing to do here as we don't support + * 1-of-N routing. The only thing that the guest can correctly + * write here is 0x1. However, there's no way to fault if it + * writes something else. This is effectively a WI in our case, + * but we keep it here for the purposes of documenting it. + */ + return; + default: + return; + } +} + +static bool vgic_v5_is_spi_selr_valid(struct vgic_v5_irs *irs, u32 id) +{ + /* Invalid - we don't have any SPIs at all */ + if (irs->idr5.spi_range == 0) + return false; + + /* Invalid - we don't have any on this IRS */ + if (irs->idr6.spi_irs_range == 0) + return false; + + /* Invalid - ID is less than min */ + if (id < irs->idr7.spi_base) + return false; + + /* Invalid - ID is greater than max */ + if (id >= (irs->idr7.spi_base + irs->idr6.spi_irs_range)) + return false; + + return true; +} + +static unsigned long vgic_v5_mmio_read_irs_spi(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct vgic_irq *irq; + u64 value = 0; + + switch (offset) { + case GICV5_IRS_SPI_SELR: + /* Return whatever was last written */ + value = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, READ_ONCE(irs->spi_selr.id)); + break; + case GICV5_IRS_SPI_STATUSR: + /* We assume that we can always claim to be idle */ + value = GICV5_IRS_SPI_STATUSR_IDLE; + if (vgic_v5_is_spi_selr_valid(irs, READ_ONCE(irs->spi_selr.id))) + value |= GICV5_IRS_SPI_STATUSR_V; + break; + case GICV5_IRS_SPI_DOMAINR: + value = FIELD_PREP(GICV5_IRS_SPI_DOMAINR_DOMAIN, + GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE); + break; + case GICV5_IRS_SPI_CFGR: { + u32 id = READ_ONCE(irs->spi_selr.id); + + if (!vgic_v5_is_spi_selr_valid(irs, id)) { + /* Fault with IRS_SPI_SELR; return 0*/ + value = 0; + break; + } + + irq = vgic_get_irq(vcpu->kvm, vgic_v5_make_spi(id)); + if (!irq) { + kvm_debug_ratelimited("Guest trying to access SPI not backed by KVM\n"); + value = 0; + break; + } + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (irq->config == VGIC_CONFIG_LEVEL) + value = GICV5_IRS_SPI_CFGR_TM; + } + + vgic_put_irq(vcpu->kvm, irq); + + break; + } + default: + return 0; + } + + return value; +} + +static void vgic_v5_mmio_write_irs_spi(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct vgic_irq *irq; + + switch (offset) { + case GICV5_IRS_SPI_SELR: + WRITE_ONCE(irs->spi_selr.id, + FIELD_GET(GICV5_IRS_SPI_SELR_ID, val)); + return; + case GICV5_IRS_SPI_CFGR: { + u32 id = READ_ONCE(irs->spi_selr.id); + + if (!vgic_v5_is_spi_selr_valid(irs, id)) + return; + + /* + * Find KVM's representation of the interrupt - we need to make + * sure that KVM's view agrees with the guest's, else interrupt + * injection won't work properly for level-triggered interrupts + * (we fail to handle the clearing of the pending state if KVM + * thinks that the interrupt is edge-triggered, which is the + * default.) + */ + irq = vgic_get_irq(vcpu->kvm, vgic_v5_make_spi(id)); + if (!irq) + return; + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (val & GICV5_IRS_SPI_CFGR_TM) + irq->config = VGIC_CONFIG_LEVEL; + else + irq->config = VGIC_CONFIG_EDGE; + } + + vgic_put_irq(vcpu->kvm, irq); + + return; + } + default: + return; + } +} + +static bool vgic_v5_ist_cfgr_valid(struct vgic_v5_irs *irs) +{ + unsigned int expected_istsz; + + if (irs->ist_cfgr.lpi_id_bits < irs->idr2.min_lpi_id_bits || + irs->ist_cfgr.lpi_id_bits > irs->idr2.id_bits) + return false; + + if (!irs->idr2.istmd) + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_4; + else if (irs->ist_cfgr.lpi_id_bits >= irs->idr2.istmd_sz) + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_16; + else + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_8; + + if (irs->ist_cfgr.istsz != expected_istsz) + return false; + + if (irs->ist_cfgr.structure && !irs->idr2.ist_levels) + return false; + + if (!irs->ist_cfgr.structure) + return true; + + return irs->ist_cfgr.l2sz == irs->idr2.ist_l2sz; +} + +static u64 vgic_v5_read_irs_ist_baser(struct vgic_v5_irs *irs) +{ + u64 value; + + value = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + irs->ist_baser.addr >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + if (irs->ist_baser.valid) + value |= GICV5_IRS_IST_BASER_VALID; + + return value; +} + +static unsigned long vgic_v5_mmio_read_irs_ist(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + u64 value = 0; + + switch (offset) { + case GICV5_IRS_IST_STATUSR: + return GICV5_IRS_IST_STATUSR_IDLE; + case GICV5_IRS_IST_CFGR: + if (irs->ist_cfgr.structure) + value |= GICV5_IRS_IST_CFGR_STRUCTURE; + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, irs->ist_cfgr.istsz); + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, irs->ist_cfgr.l2sz); + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, irs->ist_cfgr.lpi_id_bits); + break; + case GICV5_IRS_IST_BASER: + case GICV5_IRS_IST_BASER + 4: + value = vgic_v5_read_irs_ist_baser(irs); + return extract_bytes(value, offset & 7, len); + default: + return 0; + } + + return value; +} + +static void vgic_v5_mmio_write_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + enum gicv5_vcpu_cmd cmd = LPI_VIST_MAKE_INVALID; + + switch (offset) { + case GICV5_IRS_IST_CFGR: + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + if (irs->ist_baser.valid) + return; + + irs->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + irs->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + irs->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + irs->ist_cfgr.structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + } + return; + case GICV5_IRS_IST_BASER: + case GICV5_IRS_IST_BASER + 4: { + u64 baser; + bool valid; + + guard(mutex)(&vcpu->kvm->arch.config_lock); + + baser = update_64bit_reg(vgic_v5_read_irs_ist_baser(irs), + offset & 4, len, val); + valid = !!(baser & GICV5_IRS_IST_BASER_VALID); + + /* The address cannot be changed while the IST is valid. */ + if (irs->ist_baser.valid && valid) + return; + + /* Valid -> Invalid */ + if (irs->ist_baser.valid && !valid) { + /* Make the LPI IST invalid and then ... */ + if (irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu), &cmd)) + break; + + /* + * ... free the host IST if we successfully marked the + * IST as invalid. Frankly, if we failed to make the + * guest's IST as invalid, we're cooked because it means + * that the IRS may still be using the memory that we + * want to free. Hence, we leave it allocated and skip + * the clearing of valid bit in the baser. + */ + vgic_v5_lpi_ist_free(vcpu->kvm); + } else if (!irs->ist_baser.valid && valid) { /* Invalid -> Valid */ + if (!vgic_v5_ist_cfgr_valid(irs)) + break; + + if (vgic_v5_lpi_ist_alloc(vcpu->kvm, irs->ist_cfgr.lpi_id_bits)) + break; + } + + /* Now that we've handled the edges, update the valid bit and addr */ + irs->ist_baser.valid = valid; + irs->ist_baser.addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, baser) + << GICV5_IRS_IST_BASER_ADDR_SHIFT; + + return; + } + default: + return; + } +} + +static const struct vgic_register_region vgic_v5_irs_registers[] = { + /* + * This is the IRS_CONFIG_FRAME. + */ + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR0, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR1, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR2, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR3, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR4, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR5, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR6, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR7, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IIDR, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_AIDR, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_CR0, vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_CR1, vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SYNCR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SYNC_STATUSR, + vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_SELR, vgic_v5_mmio_read_irs_spi, + vgic_v5_mmio_write_irs_spi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_DOMAINR, vgic_v5_mmio_read_irs_spi, + vgic_v5_mmio_write_irs_spi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_RESAMPLER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_CFGR, vgic_v5_mmio_read_irs_spi, + vgic_v5_mmio_write_irs_spi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_STATUSR, + vgic_v5_mmio_read_irs_spi, vgic_mmio_write_wi, + 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_SELR, vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_STATUSR, + vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_CR0, vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IST_BASER, vgic_v5_mmio_read_irs_ist, + vgic_v5_mmio_write_irs_ist, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IST_CFGR, vgic_v5_mmio_read_irs_ist, + vgic_v5_mmio_write_irs_ist, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IST_STATUSR, + vgic_v5_mmio_read_irs_ist, vgic_mmio_write_wi, + 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MAP_L2_ISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + + /* + * The following registers are only for running VMs. They are not yet + * supported as we don't currently support nested, so expose them as + * read-as-zero/write-ignored. + */ + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_BASER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_CFGR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_SELR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_DBR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_HPPIR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_CR0, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_DBR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_SELR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_L2_VMTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_L2_VISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VPER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SAVE_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SAVE_VM_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + + /* MEC, MPAM, SWERR - all unimplemented */ + + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MEC_IDR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MEC_MECID_R, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MPAM_IDR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MPAM_PARTID_R, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER0, + vgic_mmio_read_raz, vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER1, + vgic_mmio_read_raz, vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), +}; + +unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev) +{ + dev->regions = vgic_v5_irs_registers; + dev->nr_regions = ARRAY_SIZE(vgic_v5_irs_registers); + + kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops); + + /* We represent both of the IRS frames back to back, so this is 128K */ + return KVM_VGIC_V5_IRS_SIZE; +} + +int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address) +{ + struct vgic_io_device *io_device = &kvm->arch.vgic.vgic_v5_irs_data->iodev; + unsigned int len; + + /* + * Design choice: Force MMIO region to be 64k aligned. Simplifies + * pulling out registers. + */ + if (!IS_ALIGNED(irs_base_address, SZ_64K)) + return -EINVAL; + + len = vgic_v5_init_irs_iodev(io_device); + + io_device->base_addr = irs_base_address; + io_device->iodev_type = IODEV_GICV5_IRS; + io_device->redist_vcpu = NULL; + + return kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, irs_base_address, len, + &io_device->dev); +} + +/** + * kvm_vgic_v5_irs_init: initialize the IRS data structures + * @kvm: kvm struct pointer + * @nr_spis: number of spis, frozen by caller + */ +int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct vgic_v5_irs *irs = dist->vgic_v5_irs_data; + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + size_t nr_spi_bits; + u64 mmfr0; + int ret, i; + + /* + * We (KVM) allocate an Interrupt State Table (IST) for SPIs. The + * hardware mandates that lower 6 bits of the address are 0. Each ISTE + * is 4 bytes in size (or larger if metadata storage is required), so 16 + * entries would be enough for alignment. We require userspace to + * provide at least 32 SPIs, so we always meet alignment requirements. + */ + if (nr_spis) { + dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), + GFP_KERNEL_ACCOUNT); + if (!dist->spis) + return -ENOMEM; + + /* + * In the following code we do not take the irq struct lock since + * no other action on irq structs can happen while the VGIC is + * not initialized yet. + */ + for (i = 0; i < nr_spis; i++) { + struct vgic_irq *irq = &dist->spis[i]; + + irq->intid = vgic_v5_make_spi(i); + INIT_LIST_HEAD(&irq->ap_list); + raw_spin_lock_init(&irq->irq_lock); + irq->vcpu = NULL; + irq->target_vcpu = vcpu0; + refcount_set(&irq->refcount, 0); + /* + * The guest controls the enable state, and again it is + * directly handled by the hardware. From our point of + * view it is always enabled. + */ + irq->enabled = 1; + } + + nr_spi_bits = fls(roundup_pow_of_two(nr_spis)) - 1; + + ret = vgic_v5_spi_ist_alloc(kvm, nr_spi_bits); + if (ret) { + kfree(dist->spis); + dist->spis = NULL; + return ret; + } + } + + /* Set sane initial state for the IRS MMIO registers */ + + irs->idr0.domain = GICV5_IRS_IDR0_INT_DOM_NON_SECURE; + + mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); + irs->idr0.pa_range = cpuid_feature_extract_unsigned_field(mmfr0, + ID_AA64MMFR0_EL1_PARANGE_SHIFT); + + irs->idr0.virt = 0; + irs->idr0.setlpi = 0; + irs->idr0.mec = 0; + irs->idr0.mpam = 0; + irs->idr0.swe = 0; + irs->idr0.irs_id = 0; + + irs->idr1.priority_bits = gicv5_global_data.irs_pri_bits - 1; + + /* + * Support 16-bits of ID space for the IRS. This should be sufficient + * for most applications, and the CPUIF is guaranteed to have at least + * 16-bits of ID space support (we actually present 16-bits there, even + * if the hardware supports more). Warn if the hardware doesn't support + * 16 bits, and use the smaller value. YMMV! + * + * As for the minimum number of ID bits, we match the hardware's + * capability. + */ + if (vgic_v5_irs_ist_id_bits(&irs_caps) < 16) + pr_warn("Host IRS supports fewer than 16 ID bits for ISTs (%u)\n", + vgic_v5_irs_ist_id_bits(&irs_caps)); + + irs->idr2.id_bits = min(16, vgic_v5_irs_ist_id_bits(&irs_caps)); + irs->idr2.min_lpi_id_bits = vgic_v5_irs_min_lpi_id_bits(&irs_caps); + + /* Only allow the guest to create Linear ISTs - simplifies Save/Restore */ + irs->idr2.ist_levels = 0; + irs->idr2.ist_l2sz = GICV5_IRS_IST_CFGR_L2SZ_4K; + irs->idr2.istmd = 0; + irs->idr2.istmd_sz = 0; + + /* We have a single IRS, only. All SPIs reside here! */ + irs->idr5.spi_range = nr_spis; + irs->idr6.spi_irs_range = nr_spis; + irs->idr7.spi_base = 0; + + irs->cr1.sh = 0; + irs->cr1.oc = 0; + irs->cr1.ic = 0; + irs->cr1.ist_ra = 0; + irs->cr1.ist_wa = 0; + irs->cr1.vmt_ra = 0; + irs->cr1.vpet_ra = 0; + irs->cr1.vmd_ra = 0; + irs->cr1.vmd_wa = 0; + irs->cr1.vped_ra = 0; + irs->cr1.vped_wa = 0; + + irs->spi_selr.id = -1; + + irs->pe_selr.iaffid = -1; + + irs->ist_cfgr.lpi_id_bits = 0; + irs->ist_cfgr.l2sz = 0; + irs->ist_cfgr.istsz = 0; + irs->ist_cfgr.structure = 0; + + irs->ist_baser.valid = 0; + irs->ist_baser.addr = 0; + + return 0; +} diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index c028c4fb95e0..c1c43be0add2 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -642,6 +642,22 @@ int vgic_v5_vmte_release(struct kvm *kvm) return 0; } +/* + * Provide a way for the IRS MMIO emulation to correctly populate the number of + * IAFFID bits (which correspond to our vpe_id_bits. + */ +u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return 0; + + return vmi->vpe_id_bits; +} + /* Provide the preallocated VPE descriptor to the hardware via the VPE Table. */ int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu) { diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h index 38b6f5269a2a..e28d39d59f7f 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.h +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -113,6 +113,7 @@ void vgic_v5_release_vm_id(struct kvm *kvm); int vgic_v5_vmte_init(struct kvm *kvm); int vgic_v5_vmte_release(struct kvm *kvm); +u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu); int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu); int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index 42cb205554da..da682092c658 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -383,6 +383,7 @@ void vgic_debug_destroy(struct kvm *kvm); int vgic_v5_probe(const struct gic_kvm_info *info); void vgic_v5_reset(struct kvm_vcpu *vcpu); int vgic_v5_init(struct kvm *kvm); +int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis); void vgic_v5_teardown(struct kvm *kvm); int vgic_v5_map_resources(struct kvm *kvm); void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid); @@ -395,6 +396,7 @@ void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); void vgic_v5_restore_state(struct kvm_vcpu *vcpu); void vgic_v5_save_state(struct kvm_vcpu *vcpu); +int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address); #define for_each_visible_v5_ppi(__i, __k) \ for_each_set_bit(__i, (__k)->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index d894c32d0929..ecb0e3526d31 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -40,6 +40,7 @@ * in KVM for now. At a future stage, this can be bumped up to 128, if required. */ #define VGIC_V5_NR_PRIVATE_IRQS 64 +#define VGIC_V5_DEFAULT_NR_SPIS 32 #define is_v5_type(t, i) (FIELD_GET(GICV5_HWIRQ_TYPE, (i)) == (t)) From c7f62066f2cf452043b43fbb7af179411029eff6 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:44:56 +0000 Subject: [PATCH 135/506] KVM: arm64: gic-v5: Initialise per-VM IRS state A virtual GICv5 needs an emulated IRS in addition to the host IRS state used to back VMTEs, VPEs, and ISTs. Without this, KVM can only provide the CPU-local PPI state and cannot expose the IRS-backed SPI and LPI configuration expected by a GICv5 guest. Allocate the per-VM emulated IRS state when creating a virtual GICv5, and initialise it from vgic_v5_init(). If userspace has not provided a number of SPIs, use the GICv5 default of 32. The IRS init path allocates the SPI state, initialises the virtual IRS register state, and creates the backing SPI IST when SPIs are present. Keep the per-VM IRS object alive for the lifetime of the virtual GICv5. vgic_v5_teardown() only unwinds resources allocated by vgic_v5_init(), so failed initialisation can be retried, while kvm_vgic_dist_destroy() frees the IRS object during final VGIC destruction. This gives virtual GICv5s the IRS backing required for SPIs and LPIs, rather than being limited to PPIs only. Further patches add support for SPI injection and lifecycle tracking. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-22-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-init.c | 61 ++++++++++++++++++++++--------- arch/arm64/kvm/vgic/vgic-irs-v5.c | 4 +- arch/arm64/kvm/vgic/vgic-v5.c | 8 +++- 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index ccbeb2f783c4..9b504b02aae0 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -187,29 +187,49 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) break; } - if (ret) { - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; - kfree(vgic_cpu->private_irqs); - vgic_cpu->private_irqs = NULL; - } - - kvm->arch.vgic.vgic_model = 0; - kvm->arch.vgic.in_kernel = false; - goto out_unlock; - } + if (ret) + goto out_free_private_irqs; if (type == KVM_DEV_TYPE_ARM_VGIC_V3) kvm->arch.vgic.nassgicap = system_supports_direct_sgis(); - /* - * We now know that we have a GICv5. The Arch Timer PPI interrupts may - * have been initialised at this stage, but will have done so assuming - * that we have an older GIC, meaning that the IntIDs won't be - * correct. We init them again, and this time they will be correct. - */ - if (type == KVM_DEV_TYPE_ARM_VGIC_V5) + if (type == KVM_DEV_TYPE_ARM_VGIC_V5) { + /* Allocate a vIRS for GICv5 systems */ + kvm->arch.vgic.vgic_v5_irs_data = kzalloc_obj(struct vgic_v5_irs, + GFP_KERNEL_ACCOUNT); + if (!kvm->arch.vgic.vgic_v5_irs_data) { + ret = -ENOMEM; + goto out_free_private_irqs; + } + + /* + * Initialization happens later, for now just explicitly + * disable the device and undef its base address. + */ + kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base = VGIC_ADDR_UNDEF; + + /* + * We now know that we have a GICv5. The Arch Timer PPI + * interrupts may have been initialised at this stage, but will + * have done so assuming that we have an older GIC, meaning that + * the IntIDs won't be correct. We init them again, and this + * time they will be correct. + */ kvm_timer_init_vm(kvm); + } + + goto out_unlock; + +out_free_private_irqs: + kvm_for_each_vcpu(i, vcpu, kvm) { + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + + kfree(vgic_cpu->private_irqs); + vgic_cpu->private_irqs = NULL; + } + + kvm->arch.vgic.in_kernel = false; + kvm->arch.vgic.vgic_model = 0; out_unlock: mutex_unlock(&kvm->arch.config_lock); @@ -496,6 +516,9 @@ int vgic_init(struct kvm *kvm) return ret; } } else { + if (!dist->nr_spis) + dist->nr_spis = VGIC_V5_DEFAULT_NR_SPIS; + ret = vgic_v5_init(kvm); if (ret) return ret; @@ -545,6 +568,8 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) break; case KVM_DEV_TYPE_ARM_VGIC_V5: vgic_v5_teardown(kvm); + kfree(dist->vgic_v5_irs_data); + dist->vgic_v5_irs_data = NULL; break; } diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c index 76f5f40ecc1c..35caf51d8831 100644 --- a/arch/arm64/kvm/vgic/vgic-irs-v5.c +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -748,8 +748,8 @@ int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis) * capability. */ if (vgic_v5_irs_ist_id_bits(&irs_caps) < 16) - pr_warn("Host IRS supports fewer than 16 ID bits for ISTs (%u)\n", - vgic_v5_irs_ist_id_bits(&irs_caps)); + pr_warn_ratelimited("Host IRS supports fewer than 16 ID bits for ISTs (%u)\n", + vgic_v5_irs_ist_id_bits(&irs_caps)); irs->idr2.id_bits = min(16, vgic_v5_irs_ist_id_bits(&irs_caps)); irs->idr2.min_lpi_id_bits = vgic_v5_irs_min_lpi_id_bits(&irs_caps); diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 5a058ef8fe70..bdf06dbff423 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -668,7 +668,8 @@ void vgic_v5_teardown(struct kvm *kvm) /* * Claim and populate a VMTE (optionally making a new L2 VMT valid), create VPE - * doorbells, allocate VPET and populate for each VPE. + * doorbells, allocate VPET and populate for each VPE. Finally, we also init the + * vIRS, which means allocating and making the virtual SPI IST valid. * * Note: We do need to put the cart before the horse here. The VPE doorbells are * our conduit for communication with the IRS, which means we need to have those @@ -747,6 +748,11 @@ int vgic_v5_init(struct kvm *kvm) goto err; } + /* Init IRS (and alloc SPI IST) */ + ret = kvm_vgic_v5_irs_init(kvm, kvm->arch.vgic.nr_spis); + if (ret) + goto err; + return 0; err: From e3db95b1bc289fe3a4433561ad15275b13eb6b8f Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:45:26 +0000 Subject: [PATCH 136/506] KVM: arm64: gic-v5: Register the IRS IODEV Now that we have an emulated IRS, it needs to be registered, which ensures that guest accesses to the MMIO regions handled by the device are handled appropriately in KVM. Therefore, as part of vgic_map_resources, the GICv5 IRS IODEV is registered. If the address for the IRS is not provided, bail out reporting an error - this is not a supported config. As part of this change, expose setting the address of the emulated IRS via KVM_VGIC_V5_ADDR_TYPE_IRS to userspace. Also allow userspace to set the number of SPIs handled by the emulated GICv5 implementation, using a GICv5-specific SPI count rather than the legacy total interrupt count. Limit the configurable range to 32 through 1024 SPIs, in multiples of 32. KVM keeps one struct vgic_irq per SPI in a physically contiguous allocation, and allowing the full 16-bit KVM_IRQ_LINE SPI namespace would make that allocation exceed KMALLOC_MAX_SIZE on common arm64 configurations. The default routing has one IRQCHIP route per configured SPI, so size the common IRQ routing table for the largest GICv5 configuration. The model-specific routing validation retains the 988-pin limit for GICv2 and GICv3. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-23-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-init.c | 18 ++++- arch/arm64/kvm/vgic/vgic-kvm-device.c | 108 ++++++++++++++++++-------- include/kvm/arm_vgic.h | 8 +- 3 files changed, 98 insertions(+), 36 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 9b504b02aae0..1537f4d20318 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -690,9 +690,8 @@ int vgic_lazy_init(struct kvm *kvm) int kvm_vgic_map_resources(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; - bool needs_dist = true; enum vgic_type type; - gpa_t dist_base; + gpa_t dist_base, irs_base; int ret = 0; if (likely(smp_load_acquire(&dist->ready))) @@ -715,13 +714,12 @@ int kvm_vgic_map_resources(struct kvm *kvm) } else { ret = vgic_v5_map_resources(kvm); type = VGIC_V5; - needs_dist = false; } if (ret) goto out; - if (needs_dist) { + if (type != VGIC_V5) { dist_base = dist->vgic_dist_base; mutex_unlock(&kvm->arch.config_lock); @@ -731,7 +729,19 @@ int kvm_vgic_map_resources(struct kvm *kvm) goto out_slots; } } else { + irs_base = dist->vgic_v5_irs_data->vgic_v5_irs_base; mutex_unlock(&kvm->arch.config_lock); + + if (IS_VGIC_ADDR_UNDEF(irs_base)) { + ret = -ENXIO; + goto out_slots; + } + + ret = vgic_v5_register_irs_iodev(kvm, irs_base); + if (ret) { + kvm_err("Unable to register VGIC IRS MMIO regions\n"); + goto out_slots; + } } smp_store_release(&dist->ready, true); diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c index 90be99443df3..8c2c6886af4e 100644 --- a/arch/arm64/kvm/vgic/vgic-kvm-device.c +++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c @@ -181,6 +181,14 @@ static int kvm_vgic_addr(struct kvm *kvm, struct kvm_device_attr *attr, bool wri addr |= (u64)rdreg->count << KVM_VGIC_V3_RDIST_COUNT_SHIFT; goto out; } + case KVM_VGIC_V5_ADDR_TYPE_IRS: + r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V5); + if (r) + break; + addr_ptr = &vgic->vgic_v5_irs_data->vgic_v5_irs_base; + alignment = SZ_64K; + size = KVM_VGIC_V5_IRS_SIZE; + break; default: r = -ENODEV; } @@ -224,31 +232,48 @@ static int vgic_set_common_attr(struct kvm_device *dev, if (get_user(val, uaddr)) return -EFAULT; - /* - * We require: - * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs - * - at most 1024 interrupts - * - a multiple of 32 interrupts - */ - if (val < (VGIC_NR_PRIVATE_IRQS + 32) || - val > VGIC_MAX_RESERVED || - (val & 31)) - return -EINVAL; + if (!vgic_is_v5(dev->kvm)) { + /* + * We require: + * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs + * - at most 1024 interrupts + * - a multiple of 32 interrupts + */ + if (val < (VGIC_NR_PRIVATE_IRQS + 32) || + val > VGIC_MAX_RESERVED || (val & 31)) + return -EINVAL; - mutex_lock(&dev->kvm->arch.config_lock); + mutex_lock(&dev->kvm->arch.config_lock); - /* - * Either userspace has already configured NR_IRQS or - * the vgic has already been initialized and vgic_init() - * supplied a default amount of SPIs. - */ - if (dev->kvm->arch.vgic.nr_spis) - ret = -EBUSY; - else - dev->kvm->arch.vgic.nr_spis = - val - VGIC_NR_PRIVATE_IRQS; + /* + * Either userspace has already configured NR_IRQS or + * the vgic has already been initialized and vgic_init() + * supplied a default amount of SPIs. + */ + if (dev->kvm->arch.vgic.nr_spis) + ret = -EBUSY; + else + dev->kvm->arch.vgic.nr_spis = + val - VGIC_NR_PRIVATE_IRQS; - mutex_unlock(&dev->kvm->arch.config_lock); + mutex_unlock(&dev->kvm->arch.config_lock); + } else { + /* + * GICv5 reports a number of SPIs, not a total number of + * interrupts. Require a multiple of 32 SPIs. + */ + if (val < VGIC_V5_DEFAULT_NR_SPIS || + val > VGIC_V5_MAX_NR_SPIS || + (val & 31)) + return -EINVAL; + + mutex_lock(&dev->kvm->arch.config_lock); + if (vgic_initialized(dev->kvm) || dev->kvm->arch.vgic.nr_spis) + ret = -EBUSY; + else + dev->kvm->arch.vgic.nr_spis = val; + mutex_unlock(&dev->kvm->arch.config_lock); + } return ret; } @@ -299,9 +324,14 @@ static int vgic_get_common_attr(struct kvm_device *dev, return (r == -ENODEV) ? -ENXIO : r; case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: { u32 __user *uaddr = (u32 __user *)(long)attr->addr; - - r = put_user(dev->kvm->arch.vgic.nr_spis + - VGIC_NR_PRIVATE_IRQS, uaddr); + /* Older GICs */ + if (!vgic_is_v5(dev->kvm)) { + r = put_user(dev->kvm->arch.vgic.nr_spis + + VGIC_NR_PRIVATE_IRQS, + uaddr); + } else { + r = put_user(dev->kvm->arch.vgic.nr_spis, uaddr); + } break; } } @@ -748,21 +778,25 @@ static int vgic_v5_set_attr(struct kvm_device *dev, { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: + break; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: - case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: return -ENXIO; + case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: + break; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: - return vgic_set_common_attr(dev, attr); + break; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: default: return -ENXIO; } + break; default: return -ENXIO; } + return vgic_set_common_attr(dev, attr); } static int vgic_v5_get_attr(struct kvm_device *dev, @@ -770,21 +804,26 @@ static int vgic_v5_get_attr(struct kvm_device *dev, { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: + break; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: - case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: return -ENXIO; + case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: + break; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: - return vgic_get_common_attr(dev, attr); + break; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: return vgic_v5_get_userspace_ppis(dev, attr); default: return -ENXIO; } + break; default: return -ENXIO; } + + return vgic_get_common_attr(dev, attr); } static int vgic_v5_has_attr(struct kvm_device *dev, @@ -792,15 +831,22 @@ static int vgic_v5_has_attr(struct kvm_device *dev, { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: - case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: - case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: + switch (attr->attr) { + case KVM_VGIC_V5_ADDR_TYPE_IRS: + return 0; + } return -ENXIO; + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + return -ENXIO; + case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: + return 0; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: return 0; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: return 0; + case KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES: default: return -ENXIO; } diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index ecb0e3526d31..1edc7b378f5a 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -31,7 +31,12 @@ #define VGIC_MAX_SPI 1019 #define VGIC_MAX_RESERVED 1023 #define VGIC_MIN_LPI 8192 -#define KVM_IRQCHIP_NUM_PINS (1020 - 32) + +/* + * The common IRQ routing table must accommodate all GIC types. GICv2 and + * GICv3 need 988 pins, while GICv5 needs 1024 pins. + */ +#define KVM_IRQCHIP_NUM_PINS 1024 /* * GICv5 supports 128 PPIs, but only the first 64 are architected. We only @@ -41,6 +46,7 @@ */ #define VGIC_V5_NR_PRIVATE_IRQS 64 #define VGIC_V5_DEFAULT_NR_SPIS 32 +#define VGIC_V5_MAX_NR_SPIS BIT(10) #define is_v5_type(t, i) (FIELD_GET(GICV5_HWIRQ_TYPE, (i)) == (t)) From 0829416d816edb01bab8b68bf3c2c278b3665515 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:45:57 +0000 Subject: [PATCH 137/506] KVM: arm64: gic-v5: Set IRICHPPIDIS based on IRS enable state The GICv5 ICH_CONTEXTR_EL2 register has the IRICHPPIDIS field, which allows the hypervisor to disable HPPI selection for SPIs and LPIs. Use this to emulate the guest enabling and disabling the IRS. Program IRICHPPIDIS from the virtual IRS enable state when loading a VPE. This prevents SPIs and LPIs from being delivered until the guest enables the emulated IRS, matching the behaviour of real hardware. Likewise, when making a VPE non-resident as it enters WFI, only calculate the doorbell priority mask and request a doorbell if the virtual IRS is enabled. A disabled IRS cannot select an SPI or LPI to wake the VPE, so neither the priority calculation nor a doorbell request serves any purpose in that state. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-24-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index bdf06dbff423..5a5abf51d4dd 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -1059,6 +1059,7 @@ void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu) void vgic_v5_load(struct kvm_vcpu *vcpu) { + bool irichppidis = !READ_ONCE(vcpu->kvm->arch.vgic.enabled); struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; u16 vm = vgic_v5_vm_id(vcpu->kvm); u16 vpe = vgic_v5_vpe_id(vcpu); @@ -1075,6 +1076,7 @@ void vgic_v5_load(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_restore_vmcr_apr, cpu_if); cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_V, true) | + FIELD_PREP(ICH_CONTEXTR_EL2_IRICHPPIDIS, irichppidis) | FIELD_PREP(ICH_CONTEXTR_EL2_VPE, vpe) | FIELD_PREP(ICH_CONTEXTR_EL2_VM, vm); @@ -1101,7 +1103,10 @@ void vgic_v5_put(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_save_apr, cpu_if); cpu_if->vgic_contextr = 0; - if (vcpu_get_flag(vcpu, IN_WFI)) { + + /* Request a doorbell if entering WFI, unless the IRS is disabled */ + if (vcpu_get_flag(vcpu, IN_WFI) && + READ_ONCE(vcpu->kvm->arch.vgic.enabled)) { u32 priority_mask; int dbpm; From 414e17f21cfba084c6beb805c2b1d4bce15ed390 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:46:27 +0000 Subject: [PATCH 138/506] KVM: arm64: selftests: Update vGICv5 selftest to set IRS address This selftest was added before the GICv5 IRS was supported in KVM. Therefore, there was no address to set, and the specific UAPI didn't even exist. Now that the IRS is supported, and setting its address is mandatory before VGIC resources are mapped, set the emulated IRS GPA before initialising the VGIC. Running a GICv5 VM will fail if userspace has not provided the IRS address before the first vCPU run. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-25-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 6 ++++++ tools/testing/selftests/kvm/include/arm64/gic_v5.h | 3 +++ 2 files changed, 9 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index 96cfd6bb32f6..19039a894056 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -100,6 +100,7 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) struct ucall uc; u64 user_ppis[2]; struct vm_gic v; + uint64_t attr; int ret, i; v.gic_dev_type = gic_dev_type; @@ -116,6 +117,11 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) for (i = 0; i < NR_VCPUS; i++) vcpu_init_descriptor_tables(vcpus[i]); + /* Set the address of the IRS before initialising the GIC */ + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v5.h b/tools/testing/selftests/kvm/include/arm64/gic_v5.h index eb523d9277cf..c388df8f2a2b 100644 --- a/tools/testing/selftests/kvm/include/arm64/gic_v5.h +++ b/tools/testing/selftests/kvm/include/arm64/gic_v5.h @@ -10,6 +10,9 @@ #include "processor.h" +/* GIC component base address is guest PA space */ +#define GICV5_IRS_CONFIG_BASE_GPA 0x8000000ULL + /* * Definitions for GICv5 instructions for the Current Domain */ From 32b253d3049a9f7c8d08ae11420d9166a9b06bed Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:46:57 +0000 Subject: [PATCH 139/506] KVM: arm64: gic-v5: Add GIC VDPEND hyp call With PPIs, their state is injected via the ICH_PPI_x_EL2 system registers. For SPIs and LPIs, there are no such registers as these would limit the number of interrupts significantly. Instead, SPI and LPI pending state can be managed from the hypervisor using the GIC VDPEND instruction. This provides a way to set an SPI or LPI for a VM as pending or non-pending, i.e., to inject interrupts into a guest. This system instruction is only executable from EL2, and therefore must be wrapped in a hypercall for NVHE/hVHE configurations. With pKVM this hypercall is explicitly a no-op as we currently don't support the combination of GICv5 and pKVM. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-26-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_hyp.h | 1 + arch/arm64/kvm/hyp/nvhe/hyp-main.c | 13 +++++++++++++ arch/arm64/kvm/hyp/vgic-v5-sr.c | 10 ++++++++++ 4 files changed, 25 insertions(+) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 55f7c5c74d1a..bffdeb36da1d 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -89,6 +89,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident, __KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident, + __KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend, __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 8fcbbff719cc..8e2f0280c0cc 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -97,6 +97,7 @@ void __vgic_v5_save_ppi_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_ppi_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_save_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if); +void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm); #ifdef __KVM_NVHE_HYPERVISOR__ void __timer_enable_traps(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index cd709eb426b5..bb18c174b247 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -919,6 +919,18 @@ static void handle___vgic_v5_restore_vmcr_apr(struct kvm_cpu_context *host_ctxt) __vgic_v5_restore_vmcr_apr(kern_hyp_va(cpu_if)); } +static void handle___vgic_v5_vdpend(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(u32, intid, host_ctxt, 1); + DECLARE_REG(bool, pending, host_ctxt, 2); + DECLARE_REG(u16, vm, host_ctxt, 3); + + if (unlikely(is_protected_kvm_enabled())) + return; + + __vgic_v5_vdpend(intid, pending, vm); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -954,6 +966,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), HANDLE_FUNC(__vgic_v5_make_resident), HANDLE_FUNC(__vgic_v5_make_non_resident), + HANDLE_FUNC(__vgic_v5_vdpend), HANDLE_FUNC(__vgic_v5_save_apr), HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), diff --git a/arch/arm64/kvm/hyp/vgic-v5-sr.c b/arch/arm64/kvm/hyp/vgic-v5-sr.c index 46992a6c2cac..94746317ca79 100644 --- a/arch/arm64/kvm/hyp/vgic-v5-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v5-sr.c @@ -149,3 +149,13 @@ void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if) { write_sysreg_s(cpu_if->vgic_icsr, SYS_ICC_ICSR_EL1); } + +void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm) +{ + u64 value; + + value = intid & (GICV5_GIC_VDPEND_ID_MASK | GICV5_GIC_VDPEND_TYPE_MASK); + value |= FIELD_PREP(GICV5_GIC_VDPEND_PENDING_MASK, pending); + value |= FIELD_PREP(GICV5_GIC_VDPEND_VM_MASK, vm); + gic_insn(value, VDPEND); +} From 7e9e2aaca2f84e7a549737edf4f9fa5262c10b84 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:47:28 +0000 Subject: [PATCH 140/506] KVM: arm64: gic: Introduce set_pending_state() to irq_ops There are times, such as with GICv5 SPIs and LPIs, where the hardware itself manages parts of the interrupt lifecycle. This means that pending state can be directly communicated to the hardware instead of being represented only in the VGIC shadow state. In order to accommodate cases where the hardware handles pending state directly, add a new set_pending_state() function pointer to irq_ops. The intent is for this to be used after the VGIC shadow pending state has changed, allowing the backend to mirror the updated state into hardware. This new function is plumbed into kvm_vgic_inject_irq(), and is only called if irq_ops are provided and this function pointer is explicitly set. In the general case, this has no effect. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-27-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic.c | 3 +++ include/kvm/arm_vgic.h | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index ca4fb45f7d50..fc88024ca3a5 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -565,6 +565,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, else irq->pending_latch = true; + if (irq->ops && irq->ops->set_pending_state) + WARN_ON_ONCE(!irq->ops->set_pending_state(vcpu, irq)); + vgic_queue_irq_unlock(kvm, irq, flags); vgic_put_irq(kvm, irq); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 1edc7b378f5a..684e8324a617 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -246,6 +246,12 @@ struct irq_ops { */ bool (*get_input_level)(int vintid); + /* + * Function pointer to directly update hardware pending state after the + * VGIC shadow pending state has changed. + */ + bool (*set_pending_state)(struct kvm_vcpu *vcpu, struct vgic_irq *irq); + /* * Function pointer to override the queuing of an IRQ. */ From c9e4010b4b222df9f73142a60454ee1d0c0d29aa Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:47:58 +0000 Subject: [PATCH 141/506] KVM: arm64: gic-v5: Support SPI injection GICv5 SPI lifecycle is handled by the GICv5 hardware once the pending state has been injected. This change adds support for injecting SPIs to the core VGIC code and GICv5 code. First of all, allow GICv5 SPIs to be looked up by ID via vgic_get_irq(). Previously, only PPIs were supported. Two irq_ops are used for GICv5 SPI injection. The set_pending_state() irq_op is used to inject the SPI's pending state into the guest. The queue_irq_unlock() irq_op releases the IRQ lock without queueing the SPI on a per-vCPU AP list, as GICv5 does not track in-flight SPI state in KVM. Tracking the affinity and lifecycle of individual SPIs in software would negate much of the benefit of their lifecycle being hardware managed. Limit injection to the configured per-VM SPI range. Since GICv5 SPI IDs start at zero, KVM_IRQ_LINE accepts IDs from zero through one less than the configured number of SPIs, up to 1023. Update the KVM_IRQ_LINE documentation accordingly. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-28-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- Documentation/virt/kvm/api.rst | 3 ++- arch/arm64/kvm/vgic/vgic-irs-v5.c | 1 + arch/arm64/kvm/vgic/vgic-v5.c | 41 +++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.c | 30 +++++++++++++++------- arch/arm64/kvm/vgic/vgic.h | 2 ++ 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc750c9..f216a2e6d687 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -918,7 +918,8 @@ The irq_type field has the following values: - KVM_ARM_IRQ_TYPE_SPI: in-kernel GICv2/GICv3: SPI, irq_id between 32 and 1019 (incl.) (the vcpu_index field is ignored) - in-kernel GICv5: SPI, irq_id between 0 and 65535 (incl.) + in-kernel GICv5: SPI, irq_id between 0 and the configured + number of SPIs minus one (1023 maximum) - KVM_ARM_IRQ_TYPE_PPI: in-kernel GICv2/GICv3: PPI, irq_id between 16 and 31 (incl.) in-kernel GICv5: PPI, irq_id between 0 and 127 (incl.) diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c index 35caf51d8831..06d2deb770a6 100644 --- a/arch/arm64/kvm/vgic/vgic-irs-v5.c +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -708,6 +708,7 @@ int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis) * view it is always enabled. */ irq->enabled = 1; + vgic_v5_set_spi_ops(irq); } nr_spi_bits = fls(roundup_pow_of_two(nr_spis)) - 1; diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 5a5abf51d4dd..d8f106384c6e 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -1057,6 +1057,47 @@ void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu) VGIC_V5_NR_PRIVATE_IRQS); } +static bool vgic_v5_set_spi_pending_state(struct kvm_vcpu *vcpu, + struct vgic_irq *irq) +{ + vgic_v5_set_irq_pend(irq->target_vcpu, irq); + return true; +} + +static bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, + struct vgic_irq *irq, + unsigned long flags) + __releases(&irq->irq_lock) +{ + lockdep_assert_held(&irq->irq_lock); + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + return true; +} + +static const struct irq_ops vgic_v5_spi_irq_ops = { + .set_pending_state = vgic_v5_set_spi_pending_state, + .queue_irq_unlock = vgic_v5_spi_queue_irq_unlock, +}; + +void vgic_v5_set_spi_ops(struct vgic_irq *irq) +{ + if (WARN_ON(!irq) || WARN_ON(irq->ops)) + return; + + irq->ops = &vgic_v5_spi_irq_ops; +} + +/* Set the pending state for GICv5 SPIs and LPIs */ +void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq) +{ + if (WARN_ON(__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, irq->intid))) + return; + + kvm_call_hyp(__vgic_v5_vdpend, irq->intid, irq_is_pending(irq), + vcpu->kvm->arch.vgic.gicv5_vm.vm_id); +} + void vgic_v5_load(struct kvm_vcpu *vcpu) { bool irichppidis = !READ_ONCE(vcpu->kvm->arch.vgic.enabled); diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index fc88024ca3a5..62d273946c31 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -86,20 +86,32 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid) */ struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid) { - /* Non-private IRQs are not yet implemented for GICv5 */ - if (vgic_is_v5(kvm)) - return NULL; + enum kvm_device_type type = kvm->arch.vgic.vgic_model; /* SPIs */ - if (intid >= VGIC_NR_PRIVATE_IRQS && - intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) { - intid -= VGIC_NR_PRIVATE_IRQS; - intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); - return &kvm->arch.vgic.spis[intid]; + if (__irq_is_spi(type, intid)) { + switch (type) { + case KVM_DEV_TYPE_ARM_VGIC_V5: + intid = vgic_v5_get_hwirq_id(intid); + + if (intid >= kvm->arch.vgic.nr_spis) + return NULL; + + intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); + return &kvm->arch.vgic.spis[intid]; + default: { + u32 max_intid = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; + + if (intid < max_intid) { + intid -= VGIC_NR_PRIVATE_IRQS; + intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); + return &kvm->arch.vgic.spis[intid]; + } + }} } /* LPIs */ - if (irq_is_lpi(kvm, intid)) + if (__irq_is_lpi(type, intid)) return vgic_get_lpi(kvm, intid); return NULL; diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index da682092c658..7b4ec72bcd02 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -387,6 +387,8 @@ int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis); void vgic_v5_teardown(struct kvm *kvm); int vgic_v5_map_resources(struct kvm *kvm); void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid); +void vgic_v5_set_spi_ops(struct vgic_irq *irq); +void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq); bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu); void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu); void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu); From f549eb53494f096b1b39b58954244b6c17ea58ef Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:48:30 +0000 Subject: [PATCH 142/506] Documentation: KVM: Extend VGICv5 device attribute docs The VGICv5 documentation still describes the device as supporting PPIs only. Update the overview to cover PPI, SPI, and LPI interrupts. PPIs and SPIs can be injected by in-kernel devices or through KVM_IRQ_LINE, while LPI pending state is driven by the guest and handled in hardware through the LPI IST. Document the GICv5 IRS address attribute and its GPA layout requirements. The IRS region must be 64K aligned and covers a 128K range. Also document the GICv5-specific interpretation of KVM_DEV_ARM_VGIC_GRP_NR_IRQS. Unlike GICv2 and GICv3, the value describes only the number of SPIs exposed by the IRS and does not include SGIs or PPIs. The supported range is 32 through 1024 SPIs in multiples of 32, and defaults to 32 when userspace leaves it unset. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-29-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- .../virt/kvm/devices/arm-vgic-v5.rst | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst index 70b9162755c7..c782ea4d4609 100644 --- a/Documentation/virt/kvm/devices/arm-vgic-v5.rst +++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst @@ -12,12 +12,39 @@ Only one VGIC instance may be instantiated through this API. The created VGIC will act as the VM interrupt controller, requiring emulated user-space devices to inject interrupts to the VGIC instead of directly to CPUs. -Creating a guest GICv5 device requires a GICv5 host. The current VGICv5 device -only supports PPI interrupts. These can either be injected from emulated -in-kernel devices (such as the Arch Timer, or PMU), or via the KVM_IRQ_LINE -ioctl. +Creating a guest GICv5 device requires a GICv5 host. The VGICv5 device supports +PPI, SPI, and LPI interrupts. The PPI and SPI interrupts can either be injected +from emulated in-kernel devices (such as the Arch Timer, or PMU), or via the +KVM_IRQ_LINE ioctl. LPIs are not externally injected, but are handled in +hardware via the LPI IST. Their pending state is driven directly by the guest. Groups: + KVM_DEV_ARM_VGIC_GRP_ADDR + Attributes: + + KVM_VGIC_V5_ADDR_TYPE_IRS (rw, 64-bit) + Base address in the guest physical address space of the GICv5 IRS + (Interrupt Routing Service) register mappings. Only valid for + KVM_DEV_TYPE_ARM_VGIC_V5. This address needs to be 64K aligned and the + region covers 128 KByte - the IRS has a CONFIG_FRAME and a SETLPI_FRAME, + each of which is 64 KBytes in size. + + Setting the address of the IRS in GPA space is mandatory before VGIC + resources are mapped, as the IRS is responsible for handling SPIs and + LPIs. Failure to set the IRS address before the first vCPU run results in + an error. + + KVM_DEV_ARM_VGIC_GRP_NR_IRQS + Attributes: + + A value describing the number of SPIs for this GIC instance. This is + GICv5-specific: unlike GICv2/v3, the value does not include SGIs or PPIs. + The value ranges from 32 to KVM's VGICv5 maximum of 1024 SPIs, in + increments of 32. If userspace does not set this attribute, KVM uses 32 + SPIs by default. + + kvm_device_attr.addr points to a __u32 value. + KVM_DEV_ARM_VGIC_GRP_CTRL Attributes: From af4d5229bf674b2554b825c79a71d1caa0088219 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:49:01 +0000 Subject: [PATCH 143/506] KVM: arm64: gic-v5: Add GICv5 SPI injection to irqfd Now that there is support for GICv5 SPIs in KVM, update vgic_irqfd_set_irq() to translate irqchip pins into GICv5 SPI IntIDs before injecting them. Validate IRQCHIP route pins against the GIC model. GICv2 and GICv3 only support 988 pins for SPIs 32 through 1019, while KVM supports up to 1024 GICv5 SPI pins. The common routing table is sized for the largest supported range. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-30-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-irqfd.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-irqfd.c b/arch/arm64/kvm/vgic/vgic-irqfd.c index 19a1094536e6..ee1a1b214f03 100644 --- a/arch/arm64/kvm/vgic/vgic-irqfd.c +++ b/arch/arm64/kvm/vgic/vgic-irqfd.c @@ -19,9 +19,14 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) { - unsigned int spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS; + unsigned int spi_id; int ret; + if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5) + spi_id = vgic_v5_make_spi(e->irqchip.pin); + else + spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS; + if (!vgic_valid_spi(kvm, spi_id)) return -EINVAL; @@ -45,6 +50,13 @@ int kvm_set_routing_entry(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) { + /* + * The common routing table reserves enough entries for all GICv5 SPI + * pins. KVM limits GICv5 to 1024 SPI pins, while GICv2 and GICv3 + * only support SPIs 32..1019, leaving 988 pins. + */ + unsigned int max_irqchip_pins = vgic_is_v5(kvm) ? + VGIC_V5_MAX_NR_SPIS : VGIC_MAX_SPI - VGIC_NR_PRIVATE_IRQS + 1; int r = -EINVAL; switch (ue->type) { @@ -52,8 +64,8 @@ int kvm_set_routing_entry(struct kvm *kvm, e->set = vgic_irqfd_set_irq; e->irqchip.irqchip = ue->u.irqchip.irqchip; e->irqchip.pin = ue->u.irqchip.pin; - if ((e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) || - (e->irqchip.irqchip >= KVM_NR_IRQCHIPS)) + if (e->irqchip.pin >= max_irqchip_pins || + e->irqchip.irqchip >= KVM_NR_IRQCHIPS) goto out; break; case KVM_IRQ_ROUTING_MSI: From 589d6072f25d0690ab9bace88b0d77a06e6ff22d Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:49:31 +0000 Subject: [PATCH 144/506] KVM: arm64: gic-v5: Mask per-vCPU PPI state in vgic_v5_finalize_ppi_state() Only a subset of the possible PPIs are exposed to a guest when running with a vGICv5. First of all, only the architected PPIs are considered by KVM. Secondly, only a set of those is exposed to a guest: those corresponding to devices that KVM emulates, such as the timers and PMU, and the GICv5 SW_PPI. The finalisation of exposed PPIs happens on first vCPU run, as this is the first time when the full set of exposed devices is known. At this stage a mask is calculated, and this mask is applied both to hide non-exposed PPI state from the guest and to reduce overhead when iterating over the PPIs. While preparing userspace access to the GICv5 system registers, it became apparent that restoring the GICv5 PPI registers can result in a mismatch between the state supplied by userspace and the state KVM intends to expose. Userspace can provide Enable, Active, and Pending state for PPIs that KVM has chosen to hide from the guest. Userspace must restore PPI state before any vCPU runs. The userspace access path added subsequently enforces this ordering. Rework vgic_v5_finalize_ppi_state() to calculate the mask of exposed PPIs and clear any state belonging to non-exposed PPIs. This ensures that only the state KVM intends to expose is visible to the guest. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-31-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 2 +- arch/arm64/kvm/vgic/vgic-v5.c | 69 +++++++++++++++++++++++++---------- include/kvm/arm_vgic.h | 2 +- 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 0bce2b4cb7cc..b0de4ea7dc39 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -979,7 +979,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; } - ret = vgic_v5_finalize_ppi_state(kvm); + ret = vgic_v5_finalize_ppi_state(vcpu); if (ret) return ret; diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index d8f106384c6e..b940f7beeaa5 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -777,9 +777,10 @@ int vgic_v5_map_resources(struct kvm *kvm) return 0; } -int vgic_v5_finalize_ppi_state(struct kvm *kvm) +int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu) { - struct kvm_vcpu *vcpu0; + struct kvm *kvm = vcpu->kvm; + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; int i; if (!vgic_is_v5(kvm)) @@ -788,35 +789,65 @@ int vgic_v5_finalize_ppi_state(struct kvm *kvm) guard(mutex)(&kvm->arch.config_lock); /* - * If SW_PPI has been advertised, then we know we already - * initialised the whole thing, and we can return early. Yes, - * this is pretty hackish as far as state tracking goes... + * Discover the set of PPIs that are exposed to the guest once per VM. + * Once known, apply that mask to each VCPU's restored PPI state as the + * VCPUs are first run. */ - if (test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask)) - return 0; + if (!test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask)) { + bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, + VGIC_V5_NR_PRIVATE_IRQS); + bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, + VGIC_V5_NR_PRIVATE_IRQS); - /* The PPI state for all VCPUs should be the same. Pick the first. */ - vcpu0 = kvm_get_vcpu(kvm, 0); + for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) { + const u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; - bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); - bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS); + irq = vgic_get_vcpu_irq(vcpu, intid); - for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) { + /* Expose PPIs with an owner or the SW_PPI, only */ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) { + __set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); + __assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, + irq->config == VGIC_CONFIG_LEVEL); + } + } + + vgic_put_irq(kvm, irq); + } + } + + /* + * Apply the mask to Enable, Active. Skip pending as that's calculated + * on guest entry. + */ + bitmap_and(cpu_if->vgic_ppi_enabler, cpu_if->vgic_ppi_enabler, + kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + bitmap_and(cpu_if->vgic_ppi_activer, cpu_if->vgic_ppi_activer, + kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + + /* Also update the vgic_irqs */ + for (i = 0; i < VGIC_V5_NR_PRIVATE_IRQS; i++) { + bool visible = test_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); const u32 intid = vgic_v5_make_ppi(i); struct vgic_irq *irq; - irq = vgic_get_vcpu_irq(vcpu0, intid); + irq = vgic_get_vcpu_irq(vcpu, intid); - /* Expose PPIs with an owner or the SW_PPI, only */ scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { - if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) { - __set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); - __assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, - irq->config == VGIC_CONFIG_LEVEL); + if (!visible) { + irq->enabled = false; + irq->active = false; + irq->pending_latch = false; + irq->line_level = false; + } else { + irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler); + irq->active = test_bit(i, cpu_if->vgic_ppi_activer); } } - vgic_put_irq(vcpu0->kvm, irq); + vgic_put_irq(kvm, irq); } return 0; diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 684e8324a617..fd8d938f4650 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -823,7 +823,7 @@ static inline u32 vgic_v5_irs_max_vpes(const struct vgic_v5_irs_caps *caps) return BIT(FIELD_GET(GICV5_IRS_IDR4_VPE_ID_BITS, caps->idr4) + 1); } -int vgic_v5_finalize_ppi_state(struct kvm *kvm); +int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu); bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, unsigned long flags); void vgic_v5_set_ppi_dvi(struct kvm_vcpu *vcpu, struct vgic_irq *irq, bool dvi); From 6479d16be163b2e3ce9cfdc6791e16aa42480f8a Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:50:02 +0000 Subject: [PATCH 145/506] KVM: arm64: gic-v5: Add GICv5 EL1 sysreg userspace accessors Now that KVM is able to run meaningful VMs with GICv5, it is important to be able to save and restore the GICv5 state in order to support VM migration. Add functions to handle userspace reads and writes of the GICv5 EL1 system registers. Some architected register values are reconstructed from KVM's internal representation rather than stored as raw register state. For example, ICC_CR0_EL1.En and ICC_PCR_EL1.Priority are backed by fields in the saved virtual VMCR, while the PPI Enable, Active, and Pending registers are assembled from per-interrupt vgic_irq state. Userspace writes are translated back into the corresponding internal representation. Wire the accessors into KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS for the GICv5 device, reusing the existing GICv3 CPU sysreg UAPI encoding. Require the VGIC to be initialised before accessing the register state. Allow userspace to read the state after a vCPU has run, but reject writes once any vCPU has run. This ensures that userspace restores the GICv5 CPU interface state before KVM finalises it for guest execution. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-32-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/Makefile | 3 +- arch/arm64/kvm/sys_regs.c | 6 +- arch/arm64/kvm/vgic-sys-reg-v5.c | 519 ++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-kvm-device.c | 124 +++++- arch/arm64/kvm/vgic/vgic.h | 7 + 5 files changed, 651 insertions(+), 8 deletions(-) create mode 100644 arch/arm64/kvm/vgic-sys-reg-v5.c diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 92dda57c0876..7aaeeb84e788 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -24,7 +24,8 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \ vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \ - vgic/vgic-v5.o vgic/vgic-v5-tables.o vgic/vgic-irs-v5.o + vgic/vgic-v5.o vgic/vgic-v5-tables.o vgic/vgic-irs-v5.o \ + vgic-sys-reg-v5.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 44aae52c473d..40927a4d4409 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -5933,7 +5933,7 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) int __init kvm_sys_reg_table_init(void) { - const struct sys_reg_desc *gicv3_regs; + const struct sys_reg_desc *gicv3_regs, *gicv5_regs; bool valid = true; unsigned int i, sz; int ret = 0; @@ -5946,8 +5946,12 @@ int __init kvm_sys_reg_table_init(void) valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), false); valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false); + /* The GICv3 system registers... */ gicv3_regs = vgic_v3_get_sysreg_table(&sz); valid &= check_sysreg_table(gicv3_regs, sz, false); + /* ...and the GICv5 system registers. */ + gicv5_regs = vgic_v5_get_sysreg_table(&sz); + valid &= check_sysreg_table(gicv5_regs, sz, false); if (!valid) return -EINVAL; diff --git a/arch/arm64/kvm/vgic-sys-reg-v5.c b/arch/arm64/kvm/vgic-sys-reg-v5.c new file mode 100644 index 000000000000..6b255eacbf60 --- /dev/null +++ b/arch/arm64/kvm/vgic-sys-reg-v5.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +/* + * VGICv5 system registers handling functions for AArch64 mode + */ + +#include + +#include +#include +#include + +#include + +#include "vgic/vgic.h" +#include "sys_regs.h" + +#define ICC_PPI_PRIORITYR_PRIORITY_MASK REPEAT_BYTE(0x1f) + +static int set_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* The upper 32 bits are RES0 */ + cpu_if->vgic_apr = val & ~ICC_APR_EL1_RES0; + + return 0; +} + +static int get_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = cpu_if->vgic_apr; + + return 0; +} + +static int set_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* + * We only support setting the ICC_CR0_EL1.En bit, which is actually + * stored in the VMCR. + */ + FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_EN, &cpu_if->vgic_vmcr, + FIELD_GET(ICC_CR0_EL1_EN, val)); + + return 0; +} + +static int get_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* + * PID only applies if EL3 is present. Same applies to IPPT. Hence, + * those fields are always presented as 0. + * + * We always present the link as connected and idle: + * (LINK = 1, LINK_IDLE = 1). + */ + *val = FIELD_PREP(ICC_CR0_EL1_EN, + FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, cpu_if->vgic_vmcr)); + *val |= ICC_CR0_EL1_LINK_MASK; + *val |= ICC_CR0_EL1_LINK_IDLE_MASK; + + return 0; +} + +static int set_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* Set the VPMR field in the VMCR */ + FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_VPMR, &cpu_if->vgic_vmcr, + FIELD_GET(ICC_PCR_EL1_PRIORITY, val)); + + return 0; +} + +static int get_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = FIELD_PREP(ICC_PCR_EL1_PRIORITY, + FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, cpu_if->vgic_vmcr)); + + return 0; +} + +static int set_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + cpu_if->vgic_icsr = val & ~ICC_ICSR_EL1_RES0; + + return 0; +} + +static int get_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = cpu_if->vgic_icsr; + + return 0; +} + +/* + * Helper macro to iterate over a range of PPIs and execute some code (to either + * extract or set the vgic_irq state). This is used when `get`-ing the PPI + * ENABLER, ACTIVER, PENDR and when setting the PRIORITYR state. + * + * vcpu: Pointer to struct kvm_vcpu (to which these PPIs belong) + * r: The register index. 0 or 1 for all except PRIORITYR (which is 0-15) + * nr: The number of PPIs iterated over. 64 for all but PRIORITYR (which is 8) + * code: The code snippet to execute for each vgic_irq + */ +#define for_ppi_state(vcpu, r, nr, code) \ + do { \ + struct kvm_vcpu *__vcpu = (vcpu); \ + int __r = (r); \ + int __nr = (nr); \ + \ + for (int i = 0; i < __nr; i++) { \ + u32 id = vgic_v5_make_ppi(__r * __nr + i); \ + struct vgic_irq *irq; \ + \ + irq = vgic_get_vcpu_irq(__vcpu, id); \ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { \ + code; \ + } \ + vgic_put_irq(__vcpu->kvm, irq); \ + } \ + } while (0) + +static int set_gic_ppi_enabler(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int i, start, end, reg = r->Op2 % 2; + + /* + * If we're only handling architected PPIs and the guest writes to the + * enable for the non-architected PPIs, we just return as there's + * nothing to do at all. We don't even allocate the storage for them in + * this case. + */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Merge the raw guest write into our bitmap at an offset of either 0 or + * 64. + * + * Note that there is *NO* masking applied - the enable state is written + * unfiltered. The assumption is that userspace uses this interface to + * set initial state before the guest runs, and then the exposed PPI + * mask is applied later, when vgic_v5_finalize_ppi_state() runs on + * first entry to each vCPU. If userspace chooses to set the enabler + * state later, it is fully capable of breaking the illusion we provided + * to the guest by exposing register state (and PPIs) to the guest that + * were not initially exposed. Good luck! + */ + bitmap_write(cpu_if->vgic_ppi_enabler, val, 64 * reg, 64); + + /* + * Sync the change in enable states to the vgic_irqs for the written + * register slice. + */ + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) + irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler); + + vgic_put_irq(vcpu->kvm, irq); + } + + return 0; +} + +static int get_gic_ppi_enabler(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long enabler = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the ENABLER value. */ + for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &enabler, irq->enabled)); + + *val = enabler; + + return 0; +} + +static int set_gic_ppi_activer(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int i, start, end, reg = r->Op2 % 2; + + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Store the raw guest write. The exposed PPI mask is applied later, + * when vgic_v5_finalize_ppi_state() runs on first entry to each + * vCPU. See comment on set_gic_ppi_enabler() for details. + */ + bitmap_write(cpu_if->vgic_ppi_activer, val, 64 * reg, 64); + + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) + irq->active = test_bit(i, cpu_if->vgic_ppi_activer); + + vgic_put_irq(vcpu->kvm, irq); + } + + return 0; +} + +static int get_gic_ppi_activer(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long activer = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the ACTIVER value. */ + for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &activer, irq->active)); + + *val = activer; + + return 0; +} + +static int set_gic_ppi_pendr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + int i, start, end, reg = r->Op2 % 2; + + /* If we only support architected PPIs, return */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Update each struct vgic_irq with the pending state, treating Level + * and Edge interrupts differently. The exposed PPI mask is applied + * later, when vgic_v5_finalize_ppi_state() runs on first entry to each + * vCPU. See comment on set_gic_ppi_enabler() for details. + */ + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + bool level = !!(val & BIT_ULL(i - start)); + + if (irq->config == VGIC_CONFIG_LEVEL) + irq->line_level = level; + else + irq->pending_latch = level; + } + + vgic_put_irq(vcpu->kvm, irq); + } + + /* + * The pending state is generated from the vgic_irqs on each guest + * entry. Therefore, we don't store the raw value written anywhere in + * the case of userspace PPI_PENDRx_EL1 writes. + */ + + return 0; +} + +static int get_gic_ppi_pendr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long pendr = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the PENDR value. */ + for_ppi_state(vcpu, reg, 64, { + if (irq_is_pending(irq)) + __assign_bit(i % 64, &pendr, 1); + }); + + *val = pendr; + + return 0; +} + +static int set_gic_ppi_priorityr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int reg = ((r->CRm & 0x1) << 3) + r->Op2; + + /* If we only support architected PPIs, return */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7) + return 0; + + val &= ICC_PPI_PRIORITYR_PRIORITY_MASK; + + /* + * Although priorities are not regularly synced back to the vgic_irq + * state, they are explicitly synced back here. This is to ensure that + * any pending PPIs are evaluated correctly when first running the guest + * after setting the state. + */ + for_ppi_state(vcpu, reg, 8, + irq->priority = (u8)(val >> (8 * i)); + ); + + /* + * Update the state that will be written to the ICH_PPI_PRIORITYRx_EL2 + * on next guest entry. + */ + cpu_if->vgic_ppi_priorityr[reg] = val; + + return 0; +} + +static int get_gic_ppi_priorityr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int reg = ((r->CRm & 0x1) << 3) + r->Op2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7) { + *val = 0; + return 0; + } + + /* + * The priorities are only synced back to the vgic_irq state when the + * vcpu is entering WFI (KVM only needs to know the priorities when + * evaluating if there are pending PPI interrupts for a vcpu). The raw + * register ICH_PPI_PRIORITYRx_EL1 state is simply saved and restored + * blindly. This state is just returned as it contains the most recent + * priorities written by the guest. + */ + *val = cpu_if->vgic_ppi_priorityr[reg]; + + return 0; +} + +/* + * The following registers are NOT supported: + * + * - ICC_HAPR_EL1 + * The value of this is directly generated by the GICv5 hardware based on + * the ICC_APR_EL1 when the guest is running. + * - ICC_IAFFIDR_EL1 + * The IAFFID for a GICv5 VPE is the same as the VPE ID, which is the index + * into the in-memory VPE Table. This is not configurable, and instead we + * rely on userspace recreating the VPEs in the same order prior to + * restoring guest state. + * - ICC_PPI_CACTIVER_EL1 + * Only raw state writes are supported via the S(et) variant. + * - ICC_PPI_CPENDR_EL1 + * Only raw state writes are supported via the S(et) variant. + */ +static const struct sys_reg_desc gic_v5_icc_reg_descs[] = { + { SYS_DESC(SYS_ICC_ICSR_EL1), + .set_user = set_gic_icsr, .get_user = get_gic_icsr, }, + { SYS_DESC(SYS_ICC_PPI_ENABLER0_EL1), + .set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, }, + { SYS_DESC(SYS_ICC_PPI_ENABLER1_EL1), + .set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, }, + /* + * Only ICC_SACTIVER_EL1 is exposed to the guest. This is treated as + * a *RAW* write of register state for writes. + */ + { SYS_DESC(SYS_ICC_PPI_SACTIVER0_EL1), + .set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, }, + { SYS_DESC(SYS_ICC_PPI_SACTIVER1_EL1), + .set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, }, + /* + * Only ICC_SPENDR_EL1 is exposed to the guest. This is treated as + * a *RAW* write of register state for writes. + */ + { SYS_DESC(SYS_ICC_PPI_SPENDR0_EL1), + .set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, }, + { SYS_DESC(SYS_ICC_PPI_SPENDR1_EL1), + .set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR0_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR1_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR2_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR3_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR4_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR5_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR6_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR7_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR8_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR9_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR10_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR11_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR12_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR13_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR14_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR15_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_APR_EL1), + .set_user = set_gic_apr, .get_user = get_gic_apr, }, + { SYS_DESC(SYS_ICC_CR0_EL1), + .set_user = set_gic_cr0, .get_user = get_gic_cr0, }, + { SYS_DESC(SYS_ICC_PCR_EL1), + .set_user = set_gic_pcr, .get_user = get_gic_pcr, }, +}; + +const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz) +{ + *sz = ARRAY_SIZE(gic_v5_icc_reg_descs); + return gic_v5_icc_reg_descs; +} + +static u64 attr_to_id(u64 attr) +{ + return ARM64_SYS_REG(FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP0_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP1_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRN_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRM_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP2_MASK, attr)); +} + +int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) +{ + const struct sys_reg_desc *r; + + r = get_reg_by_id(attr_to_id(attr->attr), gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); + + if (r && !sysreg_hidden(vcpu, r)) + return 0; + + return -ENXIO; +} + +int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr, + bool is_write) +{ + struct kvm_one_reg reg = { + .id = attr_to_id(attr->attr), + .addr = attr->addr, + }; + + if (is_write) + return kvm_sys_reg_set_user(vcpu, ®, gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); + else + return kvm_sys_reg_get_user(vcpu, ®, gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); +} diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c index 8c2c6886af4e..599ab4efc1db 100644 --- a/arch/arm64/kvm/vgic/vgic-kvm-device.c +++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c @@ -542,7 +542,7 @@ int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, * Allow access to certain ID-like registers prior to VGIC initialization, * thereby allowing the VMM to provision the features / sizing of the VGIC. */ -static bool reg_allowed_pre_init(struct kvm_device_attr *attr) +static bool v3_reg_allowed_pre_init(struct kvm_device_attr *attr) { if (attr->group != KVM_DEV_ARM_VGIC_GRP_DIST_REGS) return false; @@ -605,7 +605,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev, mutex_lock(&dev->kvm->arch.config_lock); - if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) { + if (!(vgic_initialized(dev->kvm) || v3_reg_allowed_pre_init(attr))) { ret = -EBUSY; goto out; } @@ -773,6 +773,110 @@ static int vgic_v5_get_userspace_ppis(struct kvm_device *dev, return ret; } +int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, + struct vgic_reg_attr *reg_attr) +{ + unsigned long vgic_mpidr, mpidr_reg; + + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + vgic_mpidr = (attr->attr & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) >> + KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT; + + mpidr_reg = VGIC_TO_MPIDR(vgic_mpidr); + reg_attr->vcpu = kvm_mpidr_to_vcpu(dev->kvm, mpidr_reg); + break; + default: + return -EINVAL; + } + + if (!reg_attr->vcpu) + return -EINVAL; + + reg_attr->addr = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK; + + return 0; +} + +/* + * Some registers can potentially be read before the core GIC & IRS has been + * initialised. Right now, everything is required to be post-init. + */ +static bool v5_reg_allowed_pre_init(struct kvm_device_attr *attr) +{ + return false; +} + +static bool vgic_v5_vm_has_run_once(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vcpu_has_run_once(vcpu)) + return true; + } + + return false; +} + +/* + * vgic_v5_attr_regs_access - allows user space to access VGIC v5 state + * + * @dev: kvm device handle + * @attr: kvm device attribute + * @is_write: true if userspace is writing a register + */ +static int vgic_v5_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + bool is_write) +{ + struct vgic_reg_attr reg_attr; + struct kvm_vcpu *vcpu; + int ret; + + ret = vgic_v5_parse_attr(dev, attr, ®_attr); + if (ret) + return ret; + + vcpu = reg_attr.vcpu; + + mutex_lock(&dev->kvm->lock); + + if (kvm_trylock_all_vcpus(dev->kvm)) { + mutex_unlock(&dev->kvm->lock); + return -EBUSY; + } + + mutex_lock(&dev->kvm->arch.config_lock); + + if (!(vgic_initialized(dev->kvm) || v5_reg_allowed_pre_init(attr))) { + ret = -EBUSY; + goto out; + } + + if (is_write && vgic_v5_vm_has_run_once(dev->kvm)) { + ret = -EBUSY; + goto out; + } + + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + ret = vgic_v5_cpu_sysregs_uaccess(vcpu, attr, is_write); + break; + default: + ret = -EINVAL; + break; + } + +out: + mutex_unlock(&dev->kvm->arch.config_lock); + kvm_unlock_all_vcpus(dev->kvm); + mutex_unlock(&dev->kvm->lock); + + return ret; +} + static int vgic_v5_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { @@ -780,7 +884,7 @@ static int vgic_v5_set_attr(struct kvm_device *dev, case KVM_DEV_ARM_VGIC_GRP_ADDR: break; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: - return -ENXIO; + return vgic_v5_attr_regs_access(dev, attr, true); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: break; case KVM_DEV_ARM_VGIC_GRP_CTRL: @@ -806,7 +910,7 @@ static int vgic_v5_get_attr(struct kvm_device *dev, case KVM_DEV_ARM_VGIC_GRP_ADDR: break; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: - return -ENXIO; + return vgic_v5_attr_regs_access(dev, attr, false); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: break; case KVM_DEV_ARM_VGIC_GRP_CTRL: @@ -836,8 +940,16 @@ static int vgic_v5_has_attr(struct kvm_device *dev, return 0; } return -ENXIO; - case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: - return -ENXIO; + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: { + struct vgic_reg_attr reg_attr; + int ret; + + ret = vgic_v5_parse_attr(dev, attr, ®_attr); + if (ret) + return ret; + + return vgic_v5_has_cpu_sysregs_attr(reg_attr.vcpu, attr); + } case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: return 0; case KVM_DEV_ARM_VGIC_GRP_CTRL: diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index 7b4ec72bcd02..d5e041e1ad75 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -269,6 +269,8 @@ struct ap_list_summary { #define irqs_active_outside_lrs(s) \ ((s)->nr_act && irqs_outside_lrs(s)) +int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, + struct vgic_reg_attr *reg_attr); int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, struct vgic_reg_attr *reg_attr); int vgic_v2_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, @@ -400,6 +402,11 @@ void vgic_v5_restore_state(struct kvm_vcpu *vcpu); void vgic_v5_save_state(struct kvm_vcpu *vcpu); int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address); +int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr, bool is_write); +int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); +const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz); + #define for_each_visible_v5_ppi(__i, __k) \ for_each_set_bit(__i, (__k)->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) From 71091136d1452b6125579b210d29efec3e79681a Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:50:32 +0000 Subject: [PATCH 146/506] KVM: arm64: gic-v5: Handle userspace accesses to IRS MMIO region As part of saving and restoring the state of a GICv5-based system, userspace must save and restore the IRS MMIO registers. These include important information such as the guest IST configuration, and KVM must present consistent state to the guest after migration. Introduce KVM_DEV_ARM_VGIC_GRP_IRS_REGS and provide accessors to read and write the virtual IRS register state. This is modelled on the GICv3 ITS register interface, as the migration requirements are broadly the same. Reuse the guest MMIO handlers where userspace and guest accesses have the same semantics. Add userspace-specific handling where restoring a register image must not trigger the operation associated with a guest MMIO write. Validate restored ID register fields against the capabilities KVM and the host can support. Restore the emulated IST configuration without allocating or freeing a host IST, and report operation status registers as idle. Accept writes to IRS_SPI_CFGR, IRS_IIDR, and IRS_AIDR without changing their state, allowing userspace to replay the values it previously read. Restoring IRS_IST_BASER.Valid recreates the guest-visible configuration, but the host LPI IST cannot be allocated until userspace supplies its contents. Track this as a pending LPI IST restore and reject guest entry until userspace restores the IST through KVM_DEV_ARM_VGIC_GRP_IST. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-33-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/uapi/asm/kvm.h | 1 + arch/arm64/kvm/vgic/vgic-irs-v5.c | 468 +++++++++++++++++++----- arch/arm64/kvm/vgic/vgic-kvm-device.c | 55 ++- arch/arm64/kvm/vgic/vgic-v5.c | 2 + arch/arm64/kvm/vgic/vgic.h | 4 + include/kvm/arm_vgic.h | 3 + tools/arch/arm64/include/uapi/asm/kvm.h | 1 + 7 files changed, 441 insertions(+), 93 deletions(-) diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 00cafa118a73..8a023ab54bff 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -423,6 +423,7 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 +#define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT) diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c index 06d2deb770a6..9e9853b88ce2 100644 --- a/arch/arm64/kvm/vgic/vgic-irs-v5.c +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -410,12 +410,70 @@ static unsigned long vgic_v5_mmio_read_irs_ist(struct kvm_vcpu *vcpu, return value; } +static bool vgic_v5_ist_baser_matches(const struct vgic_v5_irs *irs, + unsigned long val) +{ + u64 addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, val) + << GICV5_IRS_IST_BASER_ADDR_SHIFT; + bool valid = !!(val & GICV5_IRS_IST_BASER_VALID); + + return irs->ist_baser.addr == addr && irs->ist_baser.valid == valid; +} + +static void vgic_v5_update_irs_ist_baser(struct vgic_v5_irs *irs, + unsigned long val) +{ + irs->ist_baser.valid = !!(val & GICV5_IRS_IST_BASER_VALID); + irs->ist_baser.addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, val) + << GICV5_IRS_IST_BASER_ADDR_SHIFT; +} + +static int vgic_v5_write_irs_ist_baser(struct kvm_vcpu *vcpu, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + enum gicv5_vcpu_cmd cmd = LPI_VIST_MAKE_INVALID; + bool valid = !!(val & GICV5_IRS_IST_BASER_VALID); + int rc; + + /* The address cannot be changed while the IST is valid. */ + if (irs->ist_baser.valid && valid) + return 0; + + /* Valid -> Invalid */ + if (irs->ist_baser.valid && !valid) { + /* Make the LPI IST invalid and then ... */ + rc = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu), &cmd); + if (rc) + return rc; + + vgic_v5_update_irs_ist_baser(irs, val); + + /* + * ... reflect that in the emulated BASER before freeing the + * host IST. If the free fails, the guest-visible valid bit + * still matches the hardware state. + */ + return vgic_v5_lpi_ist_free(vcpu->kvm); + } else if (!irs->ist_baser.valid && valid) { /* Invalid -> Valid */ + if (!vgic_v5_ist_cfgr_valid(irs)) + return -EINVAL; + + rc = vgic_v5_lpi_ist_alloc(vcpu->kvm, irs->ist_cfgr.lpi_id_bits); + if (rc) + return rc; + } + + /* Now that we've handled the edges, update the valid bit and addr */ + vgic_v5_update_irs_ist_baser(irs, val); + + return 0; +} + static void vgic_v5_mmio_write_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); const size_t offset = addr & (SZ_64K - 1); - enum gicv5_vcpu_cmd cmd = LPI_VIST_MAKE_INVALID; switch (offset) { case GICV5_IRS_IST_CFGR: @@ -431,47 +489,13 @@ static void vgic_v5_mmio_write_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr, return; case GICV5_IRS_IST_BASER: case GICV5_IRS_IST_BASER + 4: { - u64 baser; - bool valid; + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + u64 baser; - guard(mutex)(&vcpu->kvm->arch.config_lock); - - baser = update_64bit_reg(vgic_v5_read_irs_ist_baser(irs), - offset & 4, len, val); - valid = !!(baser & GICV5_IRS_IST_BASER_VALID); - - /* The address cannot be changed while the IST is valid. */ - if (irs->ist_baser.valid && valid) - return; - - /* Valid -> Invalid */ - if (irs->ist_baser.valid && !valid) { - /* Make the LPI IST invalid and then ... */ - if (irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu), &cmd)) - break; - - /* - * ... free the host IST if we successfully marked the - * IST as invalid. Frankly, if we failed to make the - * guest's IST as invalid, we're cooked because it means - * that the IRS may still be using the memory that we - * want to free. Hence, we leave it allocated and skip - * the clearing of valid bit in the baser. - */ - vgic_v5_lpi_ist_free(vcpu->kvm); - } else if (!irs->ist_baser.valid && valid) { /* Invalid -> Valid */ - if (!vgic_v5_ist_cfgr_valid(irs)) - break; - - if (vgic_v5_lpi_ist_alloc(vcpu->kvm, irs->ist_cfgr.lpi_id_bits)) - break; + baser = update_64bit_reg(vgic_v5_read_irs_ist_baser(irs), + offset & 4, len, val); + vgic_v5_write_irs_ist_baser(vcpu, baser); } - - /* Now that we've handled the edges, update the valid bit and addr */ - irs->ist_baser.valid = valid; - irs->ist_baser.addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, baser) - << GICV5_IRS_IST_BASER_ADDR_SHIFT; - return; } default: @@ -479,42 +503,217 @@ static void vgic_v5_mmio_write_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr, } } +static unsigned long vgic_v5_mmio_uaccess_read_irs_status(struct kvm_vcpu *vcpu, + gpa_t addr, + unsigned int len) +{ + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_IRS_SYNC_STATUSR: + return GICV5_IRS_SYNC_STATUSR_IDLE; + case GICV5_IRS_SPI_STATUSR: + return GICV5_IRS_SPI_STATUSR_IDLE; + case GICV5_IRS_PE_STATUSR: + return GICV5_IRS_PE_STATUSR_IDLE; + case GICV5_IRS_IST_STATUSR: + return GICV5_IRS_IST_STATUSR_IDLE; + default: + return 0; + } +} + +static int vgic_v5_mmio_uaccess_write_irs(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_dist *vgic = &vcpu->kvm->arch.vgic; + struct vgic_v5_irs *irs_data = vgic->vgic_v5_irs_data; + size_t offset = addr & (SZ_64K - 1); + + /* + * The following registers are ONLY settable via uaccesses. The guest + * cannot write them! + */ + + switch (offset) { + case GICV5_IRS_IDR0: + if (FIELD_GET(GICV5_IRS_IDR0_INT_DOM, val) != + GICV5_IRS_IDR0_INT_DOM_NON_SECURE) + return -EINVAL; + + if ((val & GICV5_IRS_IDR0_VIRT) || + (val & GICV5_IRS_IDR0_ONE_N) || + (val & GICV5_IRS_IDR0_VIRT_ONE_N) || + (val & GICV5_IRS_IDR0_SETLPI) || + (val & GICV5_IRS_IDR0_MEC) || + (val & GICV5_IRS_IDR0_MPAM) || + (val & GICV5_IRS_IDR0_SWE)) + return -EINVAL; + + irs_data->idr0.domain = FIELD_GET(GICV5_IRS_IDR0_INT_DOM, val); + irs_data->idr0.pa_range = FIELD_GET(GICV5_IRS_IDR0_PA_RANGE, val); + irs_data->idr0.virt = !!(val & GICV5_IRS_IDR0_VIRT); + irs_data->idr0.setlpi = !!(val & GICV5_IRS_IDR0_SETLPI); + irs_data->idr0.mec = !!(val & GICV5_IRS_IDR0_MEC); + irs_data->idr0.mpam = !!(val & GICV5_IRS_IDR0_MPAM); + irs_data->idr0.swe = !!(val & GICV5_IRS_IDR0_SWE); + irs_data->idr0.irs_id = FIELD_GET(GICV5_IRS_IDR0_IRSID, val); + break; + case GICV5_IRS_IDR1: { + unsigned int iaffid_bits, priority_bits; + u8 vpe_id_bits; + + /* Ignore writes to PE_CNT as this is populated from num vcpus */ + iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, val); + priority_bits = FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, val); + + /* IAFFID_BITS is derived from the VMTE and encoded as N - 1. */ + vpe_id_bits = vgic_v5_vmte_vpe_id_bits(vcpu); + if (iaffid_bits != vpe_id_bits - 1) + return -EINVAL; + + if (priority_bits > gicv5_global_data.irs_pri_bits - 1) + return -EINVAL; + + irs_data->idr1.priority_bits = priority_bits; + break; + } + case GICV5_IRS_IDR2: + /* We always support LPIs */ + if (!(val & GICV5_IRS_IDR2_LPI)) + return -EINVAL; + + /* We only support LPIs with linear, non-metadata guest ISTs */ + if (val & GICV5_IRS_IDR2_IST_LEVELS) + return -EINVAL; + + if ((val & GICV5_IRS_IDR2_ISTMD) || + FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, val)) + return -EINVAL; + + /* We can't present more bits than we have support for in HW */ + if (FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val) > + vgic_v5_irs_ist_id_bits(&irs_caps)) + return -EINVAL; + + /* Min LPI ID bits must be greater than or equal to the HW */ + if (FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val) < + vgic_v5_irs_min_lpi_id_bits(&irs_caps)) + return -EINVAL; + + if (FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val) > + FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val)) + return -EINVAL; + + irs_data->idr2.istmd_sz = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, val); + irs_data->idr2.istmd = !!(val & GICV5_IRS_IDR2_ISTMD); + irs_data->idr2.ist_l2sz = FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, val); + irs_data->idr2.ist_levels = !!(val & GICV5_IRS_IDR2_IST_LEVELS); + irs_data->idr2.min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val); + irs_data->idr2.id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val); + break; + case GICV5_IRS_IDR5: + if (FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, val) != irs_data->idr5.spi_range) + return -EINVAL; + break; + case GICV5_IRS_IDR6: + if (FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, val) != irs_data->idr6.spi_irs_range) + return -EINVAL; + break; + case GICV5_IRS_IDR7: + if (FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, val) != irs_data->idr7.spi_base) + return -EINVAL; + break; + case GICV5_IRS_IST_BASER: + if (irs_data->ist_baser.valid && + !vgic_v5_ist_baser_matches(irs_data, val)) + return -EINVAL; + + if (!irs_data->ist_baser.valid && + (val & GICV5_IRS_IST_BASER_VALID)) + irs_data->lpi_ist_restore_pending = true; + + vgic_v5_update_irs_ist_baser(irs_data, val); + break; + case GICV5_IRS_IST_CFGR: + irs_data->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + irs_data->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + irs_data->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + irs_data->ist_cfgr.structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + break; + case GICV5_IRS_CR0: + vgic->enabled = !!(val & GICV5_IRS_CR0_IRSEN); + break; + case GICV5_IRS_SPI_CFGR: + break; + case GICV5_IRS_IIDR: + fallthrough; + case GICV5_IRS_AIDR: + break; + default: + return -EINVAL; + } + + return 0; +} + static const struct vgic_register_region vgic_v5_irs_registers[] = { /* * This is the IRS_CONFIG_FRAME. */ - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR0, vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR1, vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR2, vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR0, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR1, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR2, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR3, vgic_mmio_read_raz, vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR4, vgic_mmio_read_raz, vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR5, vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR6, vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR7, vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IIDR, vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_AIDR, vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_CR0, vgic_v5_mmio_read_irs_misc, - vgic_v5_mmio_write_irs_misc, 4, - VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR5, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR6, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR7, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IIDR, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_AIDR, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_CR0, + vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_CR1, vgic_v5_mmio_read_irs_misc, vgic_v5_mmio_write_irs_misc, 4, VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SYNCR, vgic_mmio_read_raz, vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SYNC_STATUSR, - vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SYNC_STATUSR, + vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_VMR, vgic_mmio_read_raz, vgic_mmio_write_wi, 8, VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), @@ -522,35 +721,50 @@ static const struct vgic_register_region vgic_v5_irs_registers[] = { vgic_v5_mmio_write_irs_spi, 4, VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_DOMAINR, vgic_v5_mmio_read_irs_spi, - vgic_v5_mmio_write_irs_spi, 4, - VGIC_ACCESS_32bit), + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_RESAMPLER, vgic_mmio_read_raz, vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_CFGR, vgic_v5_mmio_read_irs_spi, - vgic_v5_mmio_write_irs_spi, 4, - VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_STATUSR, - vgic_v5_mmio_read_irs_spi, vgic_mmio_write_wi, - 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_SELR, vgic_v5_mmio_read_irs_misc, + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SPI_CFGR, + vgic_v5_mmio_read_irs_spi, + vgic_v5_mmio_write_irs_spi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SPI_STATUSR, + vgic_v5_mmio_read_irs_spi, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_SELR, + vgic_v5_mmio_read_irs_misc, vgic_v5_mmio_write_irs_misc, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_STATUSR, - vgic_v5_mmio_read_irs_misc, - vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_PE_STATUSR, + vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_CR0, vgic_v5_mmio_read_irs_misc, vgic_v5_mmio_write_irs_misc, 4, VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IST_BASER, vgic_v5_mmio_read_irs_ist, - vgic_v5_mmio_write_irs_ist, 8, - VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IST_CFGR, vgic_v5_mmio_read_irs_ist, - vgic_v5_mmio_write_irs_ist, 4, - VGIC_ACCESS_32bit), - REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IST_STATUSR, - vgic_v5_mmio_read_irs_ist, vgic_mmio_write_wi, - 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_BASER, + vgic_v5_mmio_read_irs_ist, + vgic_v5_mmio_write_irs_ist, NULL, + vgic_v5_mmio_uaccess_write_irs, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_CFGR, + vgic_v5_mmio_read_irs_ist, + vgic_v5_mmio_write_irs_ist, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_STATUSR, + vgic_v5_mmio_read_irs_ist, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MAP_L2_ISTR, vgic_mmio_read_raz, vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), @@ -789,6 +1003,94 @@ int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis) irs->ist_baser.valid = 0; irs->ist_baser.addr = 0; + irs->lpi_ist_restore_pending = false; return 0; } + +int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + const struct vgic_register_region *region; + struct vgic_reg_attr reg_attr; + struct kvm_vcpu *vcpu; + gpa_t addr, offset; + int ret, align; + + ret = vgic_v5_parse_attr(dev, attr, ®_attr); + if (ret) + return ret; + + vcpu = reg_attr.vcpu; + addr = reg_attr.addr; + + if (attr->group == KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS) + return vgic_v5_has_cpu_sysregs_attr(vcpu, attr); + + offset = attr->attr; + + region = vgic_find_mmio_region(vgic_v5_irs_registers, + ARRAY_SIZE(vgic_v5_irs_registers), + offset); + if (!region) + return -ENXIO; + + align = region->access_flags & VGIC_ACCESS_64bit ? 0x7 : 0x3; + if (offset & align) + return -EINVAL; + + return 0; +} + +/* + * Access the IRS MMIO Regs. Relevant locks have been taken by the calling code. + */ +int vgic_v5_irs_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + u64 *reg, bool is_write) +{ + const struct vgic_register_region *region; + gpa_t addr, offset; + unsigned int len; + int align, ret = 0; + + offset = attr->attr; + + if (IS_VGIC_ADDR_UNDEF(dev->kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base)) + return -ENXIO; + + region = vgic_find_mmio_region(vgic_v5_irs_registers, + ARRAY_SIZE(vgic_v5_irs_registers), + offset); + if (!region) + return -ENXIO; + + /* + * Although the spec supports upper/lower 32-bit accesses to + * 64-bit IRS registers, the userspace ABI requires 64-bit + * accesses to all 64-bit wide registers. We therefore only + * support 32-bit accesses to 32-bit-wide registers. + */ + align = region->access_flags & VGIC_ACCESS_64bit ? 0x7 : 0x3; + len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4; + + if (offset & align) + return -EINVAL; + + addr = dev->kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base + offset; + + if (is_write) { + if (region->uaccess_write) + ret = region->uaccess_write(kvm_get_vcpu(dev->kvm, 0), + addr, len, *reg); + else + region->write(kvm_get_vcpu(dev->kvm, 0), addr, len, *reg); + } else { + if (region->uaccess_read) + *reg = region->uaccess_read(kvm_get_vcpu(dev->kvm, 0), + addr, len); + else + *reg = region->read(kvm_get_vcpu(dev->kvm, 0), addr, len); + } + + return ret; +} diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c index 599ab4efc1db..f1f1fcb08161 100644 --- a/arch/arm64/kvm/vgic/vgic-kvm-device.c +++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c @@ -786,6 +786,9 @@ int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, mpidr_reg = VGIC_TO_MPIDR(vgic_mpidr); reg_attr->vcpu = kvm_mpidr_to_vcpu(dev->kvm, mpidr_reg); break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + reg_attr->vcpu = kvm_get_vcpu(dev->kvm, 0); + break; default: return -EINVAL; } @@ -831,8 +834,11 @@ static int vgic_v5_attr_regs_access(struct kvm_device *dev, struct kvm_device_attr *attr, bool is_write) { + u64 __user *uaddr = (u64 __user *)(unsigned long)attr->addr; struct vgic_reg_attr reg_attr; struct kvm_vcpu *vcpu; + bool uaccess; + u64 val; int ret; ret = vgic_v5_parse_attr(dev, attr, ®_attr); @@ -841,6 +847,22 @@ static int vgic_v5_attr_regs_access(struct kvm_device *dev, vcpu = reg_attr.vcpu; + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + /* Sysregs uaccess is performed by the sysreg handling code */ + uaccess = false; + break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; + default: + uaccess = true; + } + + if (uaccess && is_write) { + if (get_user(val, uaddr)) + return -EFAULT; + } + mutex_lock(&dev->kvm->lock); if (kvm_trylock_all_vcpus(dev->kvm)) { @@ -864,6 +886,18 @@ static int vgic_v5_attr_regs_access(struct kvm_device *dev, case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: ret = vgic_v5_cpu_sysregs_uaccess(vcpu, attr, is_write); break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + /* + * The IRS registers are a mixture of 32-bit and 64-bit + * registers. Internally, we always perform the correctly sized + * access, but the UAPI is defined in such a way that we are + * always provided a __u64 by userspace. When userspace writes, + * the upper 32-bits are ignored for 32-bit accesses, and on a + * read any 32-bit accesses are written back to user memory + * using the full 64-bits. + */ + ret = vgic_v5_irs_attr_regs_access(dev, attr, &val, is_write); + break; default: ret = -EINVAL; break; @@ -874,6 +908,9 @@ static int vgic_v5_attr_regs_access(struct kvm_device *dev, kvm_unlock_all_vcpus(dev->kvm); mutex_unlock(&dev->kvm->lock); + if (!ret && uaccess && !is_write) + ret = put_user(val, uaddr); + return ret; } @@ -883,6 +920,8 @@ static int vgic_v5_set_attr(struct kvm_device *dev, switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: return vgic_v5_attr_regs_access(dev, attr, true); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: @@ -909,6 +948,8 @@ static int vgic_v5_get_attr(struct kvm_device *dev, switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: return vgic_v5_attr_regs_access(dev, attr, false); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: @@ -940,16 +981,10 @@ static int vgic_v5_has_attr(struct kvm_device *dev, return 0; } return -ENXIO; - case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: { - struct vgic_reg_attr reg_attr; - int ret; - - ret = vgic_v5_parse_attr(dev, attr, ®_attr); - if (ret) - return ret; - - return vgic_v5_has_cpu_sysregs_attr(reg_attr.vcpu, attr); - } + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + return vgic_v5_has_attr_regs(dev, attr); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: return 0; case KVM_DEV_ARM_VGIC_GRP_CTRL: diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index b940f7beeaa5..1552ca2da350 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -773,6 +773,8 @@ int vgic_v5_map_resources(struct kvm *kvm) { if (!vgic_initialized(kvm)) return -EBUSY; + if (kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending) + return -EINVAL; return 0; } diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index d5e041e1ad75..76d90558273a 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -406,6 +406,10 @@ int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr, bool is_write); int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz); +int vgic_v5_irs_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + u64 *reg, bool is_write); +int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr); #define for_each_visible_v5_ppi(__i, __k) \ for_each_set_bit(__i, (__k)->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index fd8d938f4650..7be81b936bf1 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -491,6 +491,9 @@ struct vgic_v5_irs { bool valid; u64 addr; } ist_baser; + + /* A valid userspace-restored BASER still needs its host LPI IST. */ + bool lpi_ist_restore_pending; }; struct vgic_dist { diff --git a/tools/arch/arm64/include/uapi/asm/kvm.h b/tools/arch/arm64/include/uapi/asm/kvm.h index d1b2ca317f58..710a0d267347 100644 --- a/tools/arch/arm64/include/uapi/asm/kvm.h +++ b/tools/arch/arm64/include/uapi/asm/kvm.h @@ -422,6 +422,7 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 +#define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT) From 9264a0c0bf0e1fa4e2be2c146aef6c3bedceb72d Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:51:03 +0000 Subject: [PATCH 147/506] KVM: arm64: gic-v5: Add CoreSight MMIO regs to IRS Expose the CoreSight identification registers in the emulated GICv5 IRS MMIO region. These registers are read-only from the guest's point of view, so return the architectural identification values and ignore writes. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-34-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-irs-v5.c | 78 +++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c index 9e9853b88ce2..2de03bbf0762 100644 --- a/arch/arm64/kvm/vgic/vgic-irs-v5.c +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -503,6 +503,43 @@ static void vgic_v5_mmio_write_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr, } } +static unsigned long vgic_v5_coresight_read(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_CORESIGHT_DEVARCH: + return GICV5_CORESIGHT_DEVARCH_VAL; + case GICV5_CORESIGHT_PIDR4: + return GICV5_CORESIGHT_PIDR4_JEP106_CONT; + case GICV5_CORESIGHT_PIDR5: + return GICV5_CORESIGHT_PIDR5_RES0; + case GICV5_CORESIGHT_PIDR6: + return GICV5_CORESIGHT_PIDR6_RES0; + case GICV5_CORESIGHT_PIDR7: + return GICV5_CORESIGHT_PIDR7_RES0; + case GICV5_CORESIGHT_PIDR0: + return GICV5_CORESIGHT_PIDR0_PART_0; + case GICV5_CORESIGHT_PIDR1: + return GICV5_CORESIGHT_PIDR1_DES_0_PART_1; + case GICV5_CORESIGHT_PIDR2: + return GICV5_CORESIGHT_PIDR2_DES_1; + case GICV5_CORESIGHT_PIDR3: + return GICV5_CORESIGHT_PIDR3_REVAND_CMOD; + case GICV5_CORESIGHT_CIDR0: + return GICV5_CORESIGHT_CIDR0_VAL; + case GICV5_CORESIGHT_CIDR1: + return GICV5_CORESIGHT_CIDR1_VAL; + case GICV5_CORESIGHT_CIDR2: + return GICV5_CORESIGHT_CIDR2_VAL; + case GICV5_CORESIGHT_CIDR3: + return GICV5_CORESIGHT_CIDR3_VAL; + default: + return 0; + } +} + static unsigned long vgic_v5_mmio_uaccess_read_irs_status(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len) @@ -840,6 +877,47 @@ static const struct vgic_register_region vgic_v5_irs_registers[] = { REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER1, vgic_mmio_read_raz, vgic_mmio_write_wi, 8, VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + + /* CoreSight identification registers */ + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_DEVARCH, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR4, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR5, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR6, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR7, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR0, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR1, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR2, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR3, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR0, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR1, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR2, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR3, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), }; unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev) From 8bcdcbb57a46a5f72ae55e8d69517ac5df8c05bd Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:51:33 +0000 Subject: [PATCH 148/506] KVM: arm64: gic-v5: Add VGICv5 IST save/restore UAPI Introduce KVM_DEV_ARM_VGIC_GRP_IST as the VGICv5 device attribute group used to transfer SPI and LPI IST state between KVM and userspace during migration. The group uses struct kvm_vgic_v5_ist as a descriptor for the userspace-owned SPI and LPI IST buffers. Each buffer contains the architected 32-bit ISTE state for the corresponding interrupt number space, while the host IST layout remains private to KVM. The descriptor is architecture-specific, as it is only the payload of an ARM VGIC device attribute. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-35-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/uapi/asm/kvm.h | 9 +++++++++ tools/arch/arm64/include/uapi/asm/kvm.h | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 8a023ab54bff..65ee09bd1463 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -424,6 +424,15 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 #define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10 +#define KVM_DEV_ARM_VGIC_GRP_IST 11 + +struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; +}; + #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT) diff --git a/tools/arch/arm64/include/uapi/asm/kvm.h b/tools/arch/arm64/include/uapi/asm/kvm.h index 710a0d267347..9998252ea8ec 100644 --- a/tools/arch/arm64/include/uapi/asm/kvm.h +++ b/tools/arch/arm64/include/uapi/asm/kvm.h @@ -423,6 +423,15 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 #define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10 +#define KVM_DEV_ARM_VGIC_GRP_IST 11 + +struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; +}; + #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT) From f9846846c4df6818d323f4a86c168bd2d63c6cdb Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:52:03 +0000 Subject: [PATCH 149/506] KVM: arm64: gic-v5: Implement save/restore mechanisms for ISTs When running a GICv5 VM, there are up to two ISTs that must be saved or restored when migrating a VM. The SPI IST is allocated by the hypervisor, as the guest presumes the memory for the SPI state is allocated by the hardware. The LPI IST is also shadowed in KVM when the guest enables LPIs, so the guest's LPI IST memory is not used directly by the physical GICv5 hardware. As both in-use ISTs are backed by host allocations, userspace provides migration storage for both tables through KVM_DEV_ARM_VGIC_GRP_IST. The userspace descriptor supplies separate SPI and LPI buffers, each containing the architected 32-bit ISTE state for the corresponding interrupt number space. If the guest has not configured an LPI IST, userspace must omit the LPI buffer. On save, acquire every vCPU mutex, returning -EBUSY if any vCPU is already running. Holding these locks blocks KVM_RUN while the IST state is exported. Use IRS_SAVE_VMR to write the IRS's internal state back to the ISTs and check that the VM remained quiescent. After copying each IST, issue a Q-only IRS_SAVE_VMR operation to update IRS_SAVE_VM_STATUSR.Q and repeat the check. If the VM has not remained quiescent since the save began, propagate an error to userspace so that the save can be retried without losing incoming interrupt state. On restore, reject the operation if any vCPU has already run. Validate the userspace buffers and, if the restored IRS state describes an LPI IST, allocate the shadow host IST while the VMTE is still valid. This allows the IRS operation that assigns the IST to update the VMTE. Then make the VMTE invalid before copying the SPI and LPI IST state from the userspace-provided buffers, and make it valid again once the copy is complete. Once restoration completes successfully, clear the pending LPI IST restore state recorded when IRS_IST_BASER.Valid was restored, allowing the guest to run. As part of restoring the ISTs, track pending interrupts and clear their pending state from the restored host ISTs. Once the VM is valid again, make those interrupts pending through the GIC VDPEND system instruction. Once a host LPI IST has been allocated, the guest-visible IRS_IST_BASER and IRS_IST_CFGR state describes that allocation. Userspace may replay values with the same defined fields, but KVM rejects changes while the host IST exists. During save, also require the VMTE IST_ID_BITS value to match IRS_IST_CFGR.LPI_ID_BITS so that userspace buffer validation and the host IST walk use the same number of entries. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-36-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-irs-v5.c | 47 +- arch/arm64/kvm/vgic/vgic-kvm-device.c | 13 + arch/arm64/kvm/vgic/vgic-v5-tables.c | 596 ++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v5-tables.h | 16 + arch/arm64/kvm/vgic/vgic-v5.c | 338 ++++++++++++++- arch/arm64/kvm/vgic/vgic.h | 5 + 6 files changed, 1009 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c index 2de03bbf0762..1874c3c289a6 100644 --- a/arch/arm64/kvm/vgic/vgic-irs-v5.c +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -382,6 +382,20 @@ static u64 vgic_v5_read_irs_ist_baser(struct vgic_v5_irs *irs) return value; } +static bool vgic_v5_ist_cfgr_matches(const struct vgic_v5_irs *irs, + unsigned long val) +{ + u8 lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + u8 l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + u8 istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + bool structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + + return irs->ist_cfgr.lpi_id_bits == lpi_id_bits && + irs->ist_cfgr.l2sz == l2sz && + irs->ist_cfgr.istsz == istsz && + irs->ist_cfgr.structure == structure; +} + static unsigned long vgic_v5_mmio_read_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len) { @@ -566,6 +580,7 @@ static int vgic_v5_mmio_uaccess_write_irs(struct kvm_vcpu *vcpu, gpa_t addr, struct vgic_dist *vgic = &vcpu->kvm->arch.vgic; struct vgic_v5_irs *irs_data = vgic->vgic_v5_irs_data; size_t offset = addr & (SZ_64K - 1); + int ret; /* * The following registers are ONLY settable via uaccesses. The guest @@ -662,8 +677,11 @@ static int vgic_v5_mmio_uaccess_write_irs(struct kvm_vcpu *vcpu, gpa_t addr, return -EINVAL; break; case GICV5_IRS_IST_BASER: - if (irs_data->ist_baser.valid && - !vgic_v5_ist_baser_matches(irs_data, val)) + ret = vgic_v5_lpi_ist_exists(vcpu->kvm); + if (ret < 0) + return ret; + + if (ret && !vgic_v5_ist_baser_matches(irs_data, val)) return -EINVAL; if (!irs_data->ist_baser.valid && @@ -673,6 +691,13 @@ static int vgic_v5_mmio_uaccess_write_irs(struct kvm_vcpu *vcpu, gpa_t addr, vgic_v5_update_irs_ist_baser(irs_data, val); break; case GICV5_IRS_IST_CFGR: + ret = vgic_v5_lpi_ist_exists(vcpu->kvm); + if (ret < 0) + return ret; + + if (ret && !vgic_v5_ist_cfgr_matches(irs_data, val)) + return -EINVAL; + irs_data->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); irs_data->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); irs_data->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); @@ -1086,6 +1111,24 @@ int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis) return 0; } +int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits) +{ + struct vgic_v5_irs *irs = kvm->arch.vgic.vgic_v5_irs_data; + + if (!irs) + return -ENXIO; + + if (!irs->ist_baser.valid) + return 0; + + if (!vgic_v5_ist_cfgr_valid(irs)) + return -EINVAL; + + *id_bits = irs->ist_cfgr.lpi_id_bits; + + return 1; +} + int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) { const struct vgic_register_region *region; diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c index f1f1fcb08161..1c205cb1361f 100644 --- a/arch/arm64/kvm/vgic/vgic-kvm-device.c +++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c @@ -920,6 +920,11 @@ static int vgic_v5_set_attr(struct kvm_device *dev, switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: break; + case KVM_DEV_ARM_VGIC_GRP_IST: + if (attr->attr) + return -ENXIO; + + return vgic_v5_irs_restore_ists(dev->kvm, attr); case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: fallthrough; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: @@ -948,6 +953,11 @@ static int vgic_v5_get_attr(struct kvm_device *dev, switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: break; + case KVM_DEV_ARM_VGIC_GRP_IST: + if (attr->attr) + return -ENXIO; + + return vgic_v5_irs_save_ists(dev->kvm, attr); case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: fallthrough; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: @@ -997,6 +1007,9 @@ static int vgic_v5_has_attr(struct kvm_device *dev, default: return -ENXIO; } + break; + case KVM_DEV_ARM_VGIC_GRP_IST: + return attr->attr ? -ENXIO : 0; default: return -ENXIO; } diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index c1c43be0add2..ba8a8d5fcc2a 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -65,6 +66,20 @@ static DEFINE_XARRAY(vm_info); #define GICV5_VPED_ADDR_SHIFT 3ULL #define GICV5_VPED_ADDR GENMASK_ULL(55, 3) +/* L2 Interrupt State Table Entry */ +#define GICV5_ISTL2E_PENDING BIT(0) +#define GICV5_ISTL2E_ACTIVE BIT(1) +#define GICV5_ISTL2E_HM BIT(2) +#define GICV5_ISTL2E_ENABLE BIT(3) +#define GICV5_ISTL2E_IRM BIT(4) +#define GICV5_ISTL2E_HWU GENMASK(10, 9) +#define GICV5_ISTL2E_PRIORITY GENMASK(15, 11) +#define GICV5_ISTL2E_IAFFID GENMASK(31, 16) + +#define GICV5_ISTE_SIZE(istsz) BIT((istsz) + 2) +#define GICV5_LINEAR_IST_SIZE(id_bits, istsz) \ + (BIT(id_bits) * GICV5_ISTE_SIZE(istsz)) + /* * The LPI and SPI configuration is stored in the 2nd and 3rd 64-bit chunks of * the VMTE (0-based). We call this a section here in an attempt to simplify the @@ -73,6 +88,26 @@ static DEFINE_XARRAY(vm_info); #define GICV5_VMTEL2_LPI_SECTION 2 #define GICV5_VMTEL2_SPI_SECTION 3 +struct vgic_v5_ist_desc { + struct vgic_v5_vm_info *vmi; + void *base; + unsigned int id_bits; + unsigned int istsz; + unsigned int l2sz; + size_t iste_size; + bool present; +}; + +struct vgic_v5_two_level_ist_shape { + size_t l1_entries; + size_t l2_entries; +}; + +struct vgic_v5_pending_irq { + u32 irq; + struct list_head next; +}; + static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, unsigned int id_bits, unsigned int istsz); @@ -106,6 +141,22 @@ static void vgic_v5_clean_inval(void *va, size_t size) dcache_clean_inval_poc(base, base + size); } +static void vgic_v5_drain_pending_irqs(struct kvm *kvm, + struct vgic_v5_vm_info *vmi, + bool reinject) +{ + struct vgic_v5_pending_irq *pirq, *tmp; + + list_for_each_entry_safe(pirq, tmp, &vmi->pending_irqs, next) { + if (reinject) + kvm_call_hyp(__vgic_v5_vdpend, pirq->irq, true, + vgic_v5_vm_id(kvm)); + + list_del(&pirq->next); + kfree(pirq); + } +} + /* * Create a linear VM Table. Directly using the number of entries supplied as * the size of an L2 VMTE (32 bytes) guarantees that our allocation is aligned per @@ -465,6 +516,13 @@ int vgic_v5_vmte_init(struct kvm *kvm) goto out_fail; vmi_inserted = true; + /* + * If we are restoring the state of a guest, we need to re-inject any + * IRQs that were pending when the state of the guest was originally + * saved. We use the pending_irqs list for this. + */ + INIT_LIST_HEAD(&vmi->pending_irqs); + /* Allocate and assign the VM Descriptor, if required. */ if (vmt_info->vmd_size != 0) { vmd_alloc_size = round_up(vmt_info->vmd_size, @@ -604,6 +662,9 @@ int vgic_v5_vmte_release(struct kvm *kvm) if (!vmi) goto no_vmi; + /* Unlikely, but possible. Avoid leaking the memory. */ + vgic_v5_drain_pending_irqs(kvm, vmi, false); + /* If we have an LPI IST, free it */ if (vmi->h_lpi_ist) { ret = vgic_v5_lpi_ist_free(kvm); @@ -1201,6 +1262,18 @@ static int vgic_v5_spi_ist_free(struct kvm *kvm) return vgic_v5_linear_ist_free(kvm, true); } +int vgic_v5_lpi_ist_exists(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + return !!vmi->h_lpi_ist; +} + /* * Allocate an IST for LPIs. * @@ -1277,3 +1350,526 @@ int vgic_v5_lpi_ist_free(struct kvm *kvm) else return vgic_v5_two_level_ist_free(kvm, false); } + +static struct vgic_v5_two_level_ist_shape +vgic_v5_two_level_ist_shape(const struct vgic_v5_ist_desc *ist) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t l2bits, n; + + l2bits = (10 - ist->istsz) + (2 * ist->l2sz); + n = max(2, ist->id_bits - l2bits + 3 - 1); + + shape.l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + shape.l2_entries = BIT(l2bits); + + return shape; +} + +static int vgic_v5_read_vm_ist_desc(struct kvm *kvm, unsigned int section, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + u64 vmte_ist_section; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + vmte_ist_section = le64_to_cpu(READ_ONCE(vmte->val[section])); + } + + ist->id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, vmte_ist_section); + ist->istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, vmte_ist_section); + ist->l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, vmte_ist_section); + ist->iste_size = GICV5_ISTE_SIZE(ist->istsz); + + return !!(vmte_ist_section & GICV5_VMTEL2E_IST_VALID); +} + +static int vgic_v5_get_spi_ist_desc(struct kvm *kvm, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + int ret; + + memset(ist, 0, sizeof(*ist)); + + ist->vmi = xa_load(&vm_info, vm_id); + if (!ist->vmi) + return -ENXIO; + + ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_SPI_SECTION, ist); + if (ret < 0) + return ret; + + ist->base = ist->vmi->h_spi_ist; + if (!ret || !ist->base) + return -ENXIO; + + ist->present = true; + return 0; +} + +static int vgic_v5_get_lpi_ist_desc(struct kvm *kvm, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + bool guest_valid, host_valid; + int ret; + + memset(ist, 0, sizeof(*ist)); + + ist->vmi = xa_load(&vm_info, vm_id); + if (WARN_ON_ONCE(!ist->vmi)) + return -ENXIO; + + ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_LPI_SECTION, ist); + if (ret < 0) + return ret; + + host_valid = ret; + guest_valid = kvm->arch.vgic.vgic_v5_irs_data->ist_baser.valid; + ist->base = ist->vmi->h_lpi_ist; + + /* If there is no IST to save/restore, return without error. */ + if (!guest_valid && !host_valid && !ist->base) + return 0; + + /* Mismatched combination of valid state */ + if (!guest_valid || !host_valid || !ist->base) + return -ENXIO; + + if (ist->vmi->h_lpi_ist_structure && !ist->vmi->h_lpi_l2_ists) + return -ENXIO; + + ist->present = true; + return 0; +} + +/* + * Save a linear host IST to userspace memory. + * + * Only the architected 32-bit ISTE state is stored. Metadata is skipped when + * striding through the host IST. + */ +static int vgic_v5_save_linear_ist(const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, size_t nr_entries) +{ + __le32 h_iste; + size_t index; + int ret; + + vgic_v5_clean_inval(ist->base, + GICV5_LINEAR_IST_SIZE(ist->id_bits, ist->istsz)); + + for (index = 0; index < nr_entries; index++) { + __le32 *h_iste_addr = ist->base + index * ist->iste_size; + + h_iste = READ_ONCE(*h_iste_addr); + ret = put_user(h_iste, uaddr); + if (ret) + return ret; + + uaddr++; + } + + return 0; +} + +/* + * Save a two-level host IST to userspace memory. + * + * Only the architected 32-bit ISTE state is stored. Metadata is skipped when + * striding through the host IST. + */ +static int vgic_v5_save_two_level_ist(const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t h_l1_index, h_l2_index; + void *h_l2_ist_base; + __le32 h_iste; + int ret; + + shape = vgic_v5_two_level_ist_shape(ist); + + vgic_v5_clean_inval(ist->base, + shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist)); + + for (h_l1_index = 0; h_l1_index < shape.l1_entries; h_l1_index++) { + u64 l1_iste; + + /* + * Host L2 ISTs are preallocated. Any invalid L1 entry means the + * host IST state is inconsistent. + */ + l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index])); + if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste)) + return -ENXIO; + + h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index]; + if (!h_l2_ist_base) + return -ENXIO; + + vgic_v5_clean_inval(h_l2_ist_base, + shape.l2_entries * ist->iste_size); + + for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) { + h_iste = *(__le32 *)(h_l2_ist_base + + h_l2_index * ist->iste_size); + + ret = put_user(h_iste, uaddr); + if (ret) + return ret; + + uaddr++; + } + } + + return 0; +} + +/* + * Save the SPI IST to userspace-provided memory. + */ +int vgic_v5_save_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + struct vgic_v5_ist_desc ist; + u32 __user *uaddr; + int ret; + + ret = vgic_v5_get_spi_ist_desc(kvm, &ist); + if (ret) + return ret; + + uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr; + + /* The host SPI IST is always linear. */ + return vgic_v5_save_linear_ist(&ist, uaddr, + kvm->arch.vgic.nr_spis); +} + +/* + * Save the LPI IST to userspace memory. + * + * The LPI IST may be linear or two-level, so host iteration depends on the + * allocated host shape. + */ +int vgic_v5_save_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + struct vgic_v5_ist_desc ist; + u32 __user *uaddr; + int ret; + + ret = vgic_v5_get_lpi_ist_desc(kvm, &ist); + if (ret) + return ret; + + if (!ist.present) + return 0; + + /* + * Userspace sized the buffer from the guest-visible configuration. + * Refuse to walk a host IST with a different number of entries. + */ + if (ist.id_bits != + kvm->arch.vgic.vgic_v5_irs_data->ist_cfgr.lpi_id_bits) + return -EINVAL; + + uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr; + + if (!ist.vmi->h_lpi_ist_structure) + return vgic_v5_save_linear_ist(&ist, uaddr, + BIT(ist.id_bits)); + + return vgic_v5_save_two_level_ist(&ist, uaddr); +} + +/* + * Track any SPIs and LPIs that were marked as pending at the point where the + * IST was restored. + * + * Restored pending state is cleared from the host ISTE and replayed with VDPEND + * before the VM first runs. + */ +static int vgic_v5_track_pending_irq(struct list_head *pending_irqs, u32 intid, + u32 type) +{ + struct vgic_v5_pending_irq *pirq; + + pirq = kzalloc_obj(*pirq, GFP_KERNEL); + if (!pirq) + return -ENOMEM; + + /* Encode the interrupt as a GICv5 IntID. */ + pirq->irq = FIELD_PREP(GICV5_HWIRQ_TYPE, type) | + FIELD_PREP(GICV5_HWIRQ_ID, intid); + + INIT_LIST_HEAD(&pirq->next); + list_add_tail(&pirq->next, pending_irqs); + + return 0; +} + +/* + * Process and sanitise each restored ISTE. + * + * HWU is for hardware use and must not survive migration. Pending state is + * tracked, cleared from the ISTE, and replayed before the VM first runs. + */ +static int vgic_v5_process_iste(__le32 *iste, struct list_head *pending_irqs, + u32 intid, u32 type) +{ + u32 iste_data = le32_to_cpu(READ_ONCE(*iste)); + int ret; + + /* Pending state is replayed later with VDPEND. */ + if (iste_data & GICV5_ISTL2E_PENDING) { + ret = vgic_v5_track_pending_irq(pending_irqs, intid, type); + if (ret) + return ret; + } + + iste_data &= ~GICV5_ISTL2E_PENDING; + iste_data &= ~GICV5_ISTL2E_HWU; + + WRITE_ONCE(*iste, cpu_to_le32(iste_data)); + + return 0; +} + +static void vgic_v5_restore_spi_config(struct kvm *kvm, __le32 iste, u32 spi) +{ + u32 iste_data = le32_to_cpu(iste); + bool pending = iste_data & GICV5_ISTL2E_PENDING; + struct vgic_irq *irq; + unsigned long flags; + + irq = vgic_get_irq(kvm, vgic_v5_make_spi(spi)); + if (WARN_ON_ONCE(!irq)) + return; + + raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (iste_data & GICV5_ISTL2E_HM) + irq->config = VGIC_CONFIG_LEVEL; + else + irq->config = VGIC_CONFIG_EDGE; + + if (irq->config == VGIC_CONFIG_EDGE) + irq->pending_latch = pending; + else if (pending) + irq->pending_latch = true; + else if (!irq->active) + irq->pending_latch = false; + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(kvm, irq); +} + +static int vgic_v5_restore_ist_entry(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + void *h_iste_addr, __le32 h_iste, + u32 intid, u32 intid_type) +{ + __le32 raw_iste = h_iste; + int ret; + + /* + * Sanitise the IST, clearing HWU & pending fields. Pending state is + * later replayed via GIC VDPEND. + */ + ret = vgic_v5_process_iste(&h_iste, &ist->vmi->pending_irqs, + intid, intid_type); + if (ret) + return ret; + + if (intid_type == GICV5_HWIRQ_TYPE_SPI) + vgic_v5_restore_spi_config(kvm, raw_iste, intid); + + /* + * Zero the full ISTE (incl metadata), and write back the non-metadata + * region, only. + */ + memset(h_iste_addr, 0, ist->iste_size); + WRITE_ONCE(*(__le32 *)h_iste_addr, h_iste); + vgic_v5_clean_inval(h_iste_addr, ist->iste_size); + + return 0; +} + +/* + * Restore a userspace IST image to a linear host IST. + * + * The userspace IST image is a linear array of 32-bit ISTEs. + */ +static int vgic_v5_restore_linear_ist(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, size_t nr_entries, + u32 intid_type) +{ + __le32 h_iste; + size_t index; + int ret; + + for (index = 0; index < nr_entries; index++) { + void *h_iste_addr = ist->base + index * ist->iste_size; + + ret = get_user(h_iste, uaddr); + if (ret) + return ret; + + ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr, + h_iste, index, intid_type); + if (ret) + return ret; + + uaddr++; + } + + return 0; +} + +/* + * Restore a userspace IST image to a two-level host IST. + * + * The userspace IST image is a linear array of 32-bit ISTEs. + */ +static int vgic_v5_restore_two_level_ist(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, u32 intid_type) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t h_l1_index, h_l2_index; + void *h_l2_ist_base; + __le32 h_iste; + int ret; + + shape = vgic_v5_two_level_ist_shape(ist); + + vgic_v5_clean_inval(ist->vmi->h_lpi_ist, + shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist)); + + for (h_l1_index = 0; h_l1_index < shape.l1_entries; ++h_l1_index) { + u64 l1_iste; + + /* + * Host L2 ISTs are preallocated. Any invalid L1 entry means the + * host IST state is inconsistent. + */ + l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index])); + if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste)) + return -ENXIO; + + h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index]; + if (!h_l2_ist_base) + return -ENXIO; + + for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) { + void *h_iste_addr = h_l2_ist_base + + h_l2_index * ist->iste_size; + u32 intid = h_l1_index * shape.l2_entries + h_l2_index; + + ret = get_user(h_iste, uaddr); + if (ret) + return ret; + + ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr, + h_iste, intid, + intid_type); + if (ret) + return ret; + + uaddr++; + } + } + + return 0; +} + +/* + * Restore the SPI IST from userspace-provided buffer to the host-allocated IST. + */ +int vgic_v5_restore_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + u32 __user *uaddr; + struct vgic_v5_ist_desc ist; + int ret; + + ret = vgic_v5_get_spi_ist_desc(kvm, &ist); + if (ret) + return ret; + + uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr; + + /* The host SPI IST is always linear. */ + return vgic_v5_restore_linear_ist(kvm, &ist, uaddr, + kvm->arch.vgic.nr_spis, + GICV5_HWIRQ_TYPE_SPI); +} + +/* + * Restore the LPI IST from userspace memory to the host-allocated LPI IST. + * + * The host LPI IST may be linear or two-level, so host iteration depends on the + * host IST's shape. + */ +int vgic_v5_restore_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + u32 __user *uaddr; + struct vgic_v5_ist_desc ist; + int ret; + + ret = vgic_v5_get_lpi_ist_desc(kvm, &ist); + if (ret) + return ret; + + if (!ist.present) + return 0; + + uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr; + + if (!ist.vmi->h_lpi_ist_structure) + return vgic_v5_restore_linear_ist(kvm, &ist, uaddr, + BIT(ist.id_bits), + GICV5_HWIRQ_TYPE_LPI); + + return vgic_v5_restore_two_level_ist(kvm, &ist, uaddr, + GICV5_HWIRQ_TYPE_LPI); +} + +/* + * Process the pending IRQs removing them from the list and optionally injecting + * them. + */ +static int vgic_v5_process_pending_irqs(struct kvm *kvm, bool inject) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + vgic_v5_drain_pending_irqs(kvm, vmi, inject); + + return 0; +} + +/* Replay pending state that was cleared while restoring guest IST state. */ +int vgic_v5_restore_pending_irqs(struct kvm *kvm) +{ + return vgic_v5_process_pending_irqs(kvm, true); +} + +/* Drop pending state collected by a failed IST restore. */ +void vgic_v5_discard_pending_irqs(struct kvm *kvm) +{ + vgic_v5_process_pending_irqs(kvm, false); +} diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h index e28d39d59f7f..b2ffbe68c05b 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.h +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -9,6 +9,7 @@ #include #include #include +#include /* Level 1 Virtual Machine Table Entry */ typedef __le64 vmtl1_entry; @@ -44,6 +45,9 @@ struct vgic_v5_vm_info { __le64 *h_lpi_ist; __le64 **h_lpi_l2_ists; __le64 *h_spi_ist; + + /* Tracking of pending interrupts as part of IST restore */ + struct list_head pending_irqs; }; struct vgic_v5_vmt { @@ -118,7 +122,19 @@ int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu); int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu); int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_exists(struct kvm *kvm); int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits); int vgic_v5_lpi_ist_free(struct kvm *kvm); +int vgic_v5_save_spi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_save_lpi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_spi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_lpi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_pending_irqs(struct kvm *kvm); +void vgic_v5_discard_pending_irqs(struct kvm *kvm); + #endif diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 1552ca2da350..91810fbd99af 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -5,10 +5,11 @@ #include -#include #include #include #include +#include +#include #include "vgic-v5-tables.h" #include "vgic.h" @@ -218,6 +219,17 @@ static int vgic_v5_irs_wait_for_vpe_op(void) NULL); } +/* + * Wait for a write to IRS_SAVE_VMR to complete. + */ +static int vgic_v5_irs_wait_for_save_vm_op(u32 *statusr) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_SAVE_VM_STATUSR, + GICV5_IRS_SAVE_VM_STATUSR_IDLE, + statusr); +} + static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset) { int ret; @@ -386,6 +398,27 @@ static int vgic_v5_irs_set_up_vpe(u16 vm_id, u16 vpe_id, return 0; } +static int vgic_v5_irs_save_vm_op(u16 vm_id, bool save, u32 *statusr) +{ + u64 save_vmr; + int ret; + + save_vmr = FIELD_PREP(GICV5_IRS_SAVE_VMR_VM_ID, vm_id); + save_vmr |= GICV5_IRS_SAVE_VMR_Q; + save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_S, save); + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with. */ + ret = vgic_v5_irs_wait_for_save_vm_op(NULL); + if (ret) + return ret; + + irs_writeq_relaxed(save_vmr, GICV5_IRS_SAVE_VMR); + + return vgic_v5_irs_wait_for_save_vm_op(statusr); +} + static irqreturn_t db_handler(int irq, void *data) { struct kvm_vcpu *vcpu = data; @@ -1097,9 +1130,9 @@ static bool vgic_v5_set_spi_pending_state(struct kvm_vcpu *vcpu, return true; } -static bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, - struct vgic_irq *irq, - unsigned long flags) +bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, + struct vgic_irq *irq, + unsigned long flags) __releases(&irq->irq_lock) { lockdep_assert_held(&irq->irq_lock); @@ -1271,3 +1304,300 @@ void vgic_v5_save_state(struct kvm_vcpu *vcpu) __vgic_v5_save_ppi_state(cpu_if); dsb(sy); } + +static int vgic_v5_irs_status_is_quiesced(u32 statusr) +{ + if (statusr & GICV5_IRS_SAVE_VM_STATUSR_Q) + return 0; + + return -EBUSY; +} + +static int vgic_v5_irs_is_quiesced(u16 vm_id) +{ + u32 statusr; + int ret; + + /* + * Ensure that the IST snapshot is complete before asking the IRS + * whether the VM remained quiescent while it was copied. + */ + mb(); + + ret = vgic_v5_irs_save_vm_op(vm_id, false, &statusr); + if (ret) + return ret; + + return vgic_v5_irs_status_is_quiesced(statusr); +} + +static int vgic_v5_copy_ist_attr(struct kvm_device_attr *attr, + struct kvm_vgic_v5_ist *ist_attr) +{ + void __user *uaddr = (void __user *)(unsigned long)attr->addr; + + if (!uaddr) + return -EINVAL; + + if (copy_from_user(ist_attr, uaddr, sizeof(*ist_attr))) + return -EFAULT; + + return 0; +} + +static int vgic_v5_validate_ist_user_buffer(__u64 addr, __u64 size, + size_t expected) +{ + if (!addr || size != expected) + return -EINVAL; + + return 0; +} + +static int vgic_v5_validate_ist_attr(struct kvm *kvm, + const struct kvm_vgic_v5_ist *ist_attr) +{ + unsigned int id_bits; + int ret; + + /* We always have SPIs to save */ + ret = vgic_v5_validate_ist_user_buffer(ist_attr->spi_ist_addr, + ist_attr->spi_ist_size, + kvm->arch.vgic.nr_spis * sizeof(__u32)); + if (ret) + return ret; + + /* We don't always have LPIs to save */ + ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits); + if (ret < 0) + return ret; + + /* No LPI IST */ + if (!ret) { + if (ist_attr->lpi_ist_addr || ist_attr->lpi_ist_size) + return -EINVAL; + + return 0; + } + + return vgic_v5_validate_ist_user_buffer(ist_attr->lpi_ist_addr, + ist_attr->lpi_ist_size, + BIT(id_bits) * sizeof(__u32)); +} + +int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr) +{ + struct kvm_vgic_v5_ist ist_attr; + u16 vm_id = vgic_v5_vm_id(kvm); + u32 statusr; + int ret = 0; + + mutex_lock(&kvm->lock); + + if (kvm_trylock_all_vcpus(kvm)) { + mutex_unlock(&kvm->lock); + return -EBUSY; + } + + mutex_lock(&kvm->arch.config_lock); + + if (!vgic_initialized(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + ret = vgic_v5_copy_ist_attr(attr, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_validate_ist_attr(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_save_vm_op(vm_id, true, &statusr); + if (ret) { + kvm_err("Failed to save GICv5 IRS VM state: %d\n", ret); + goto out_unlock; + } + + ret = vgic_v5_irs_status_is_quiesced(statusr); + if (ret) + goto out_unlock; + + /* Save the SPI IST to the userspace buffer. */ + ret = vgic_v5_save_spi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_is_quiesced(vm_id); + if (ret) + goto out_unlock; + + /* Save the LPI IST to the userspace buffer. */ + ret = vgic_v5_save_lpi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_is_quiesced(vm_id); + if (ret) + goto out_unlock; + +out_unlock: + mutex_unlock(&kvm->arch.config_lock); + kvm_unlock_all_vcpus(kvm); + mutex_unlock(&kvm->lock); + + return ret; +} + +/* Allocate the LPI IST to restore into */ +static int vgic_v5_restore_lpi_ist_alloc(struct kvm *kvm, bool *allocated) +{ + unsigned int id_bits; + int ret; + + *allocated = false; + + ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits); + if (ret <= 0) + return ret; + + ret = vgic_v5_lpi_ist_alloc(kvm, id_bits); + if (ret) + return ret; + + *allocated = true; + + return 0; +} + +/* + * Clean up the LPI IST if we allocated it, and restore the VMTE to the + * original, valid state. + */ +static void vgic_v5_restore_cleanup(struct kvm *kvm, + struct kvm_vcpu *vcpu, + bool lpi_ist_allocated) +{ + /* + * We are on the restore failure path, so we do a best-effort + * cleanup. These commands might fail, but at this stage this is the + * best we can realistically do. + */ + if (lpi_ist_allocated) { + if (!vgic_v5_send_command(vcpu, VMTE_MAKE_INVALID)) + vgic_v5_lpi_ist_free(kvm); + } + + vgic_v5_send_command(vcpu, VMTE_MAKE_VALID); +} + +int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr) +{ + bool lpi_ist_allocated = false, vmte_invalid = false; + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + struct kvm_vgic_v5_ist ist_attr; + int ret = 0; + + mutex_lock(&kvm->lock); + + if (kvm_trylock_all_vcpus(kvm)) { + mutex_unlock(&kvm->lock); + return -EBUSY; + } + + mutex_lock(&kvm->arch.config_lock); + + if (!vgic_initialized(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + if (kvm_vm_has_ran_once(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + ret = vgic_v5_copy_ist_attr(attr, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_validate_ist_attr(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_lpi_ist_exists(kvm); + if (ret) { + if (ret > 0) + ret = -EBUSY; + goto out_unlock; + } + + /* + * If the guest has previously allocated an IST (which we check based on + * the IRS_IST_BASER), extract the number of LPI ID bits from the + * IRS_IST_CFGR. Else, do nothing. + * + * We do this before making the VMTE invalid as we rely on + * IRS_VMAP_VISTR to mark the IST as valid in the VMTE. This can only + * happen while the VMTE is valid. + */ + ret = vgic_v5_restore_lpi_ist_alloc(kvm, &lpi_ist_allocated); + if (ret) + goto out_unlock; + + /* + * Host ISTs are updated while the VMTE is invalid, so the GIC cannot + * observe partially restored state. + */ + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID); + if (ret) { + /* + * If invalidation fails, the restore cannot safely update host + * IST state. + */ + goto out_unlock; + } + vmte_invalid = true; + + /* Restore the SPI IST from the userspace buffer. */ + ret = vgic_v5_restore_spi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + /* Restore the LPI IST from the userspace buffer. */ + if (lpi_ist_allocated) { + ret = vgic_v5_restore_lpi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + } + + /* And make the VM Valid again */ + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID); + if (ret) + goto out_unlock; + vmte_invalid = false; + + /* + * As part of restoring the ISTs, and previously pending interrupts have + * been tracked and made non-pending. Now that the ISTs have been + * restored, and the VM is valid again, restore the pending interrupts. + */ + ret = vgic_v5_restore_pending_irqs(kvm); + if (ret) + goto out_unlock; + + kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending = false; + +out_unlock: + if (ret && (vmte_invalid || lpi_ist_allocated)) { + vgic_v5_discard_pending_irqs(kvm); + vgic_v5_restore_cleanup(kvm, vcpu0, lpi_ist_allocated); + } + + mutex_unlock(&kvm->arch.config_lock); + kvm_unlock_all_vcpus(kvm); + mutex_unlock(&kvm->lock); + + return ret; +} diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index 76d90558273a..97fc3bdf520e 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -390,6 +390,8 @@ void vgic_v5_teardown(struct kvm *kvm); int vgic_v5_map_resources(struct kvm *kvm); void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid); void vgic_v5_set_spi_ops(struct vgic_irq *irq); +bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, + unsigned long flags); void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq); bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu); void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu); @@ -401,11 +403,14 @@ void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); void vgic_v5_restore_state(struct kvm_vcpu *vcpu); void vgic_v5_save_state(struct kvm_vcpu *vcpu); int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address); +int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits); int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr, bool is_write); int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz); +int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr); +int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr); int vgic_v5_irs_attr_regs_access(struct kvm_device *dev, struct kvm_device_attr *attr, u64 *reg, bool is_write); From fc3a33a6f388bf0a3645d019bc7a0cab3a6ead92 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:52:34 +0000 Subject: [PATCH 150/506] Documentation: KVM: Document KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS for VGICv5 The virtual GICv5 adopts the same mechanism as GICv3 for userspace reads and writes of the system registers, albeit operating on a different set of registers, of course. Document KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS for GICv5 in the VGICv5 documentation, explicitly calling out the registers it operates on. The main body of documentation has been directly copied from the VGICv3 documentation as it has identical operation. One key thing to note is that for two sets of GICv5 registers - those pertaining to Active and Pending state - the operation of the interface is different to how the actual registers operate. Both of these registers have C and S variants (to set and clear bits) in hardware. However for this interface, we ONLY implement the S variant, AND treat it as a raw write. This simplifies the act of reading or writing the state. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-37-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- .../virt/kvm/devices/arm-vgic-v5.rst | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst index c782ea4d4609..7e41d6699538 100644 --- a/Documentation/virt/kvm/devices/arm-vgic-v5.rst +++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst @@ -75,3 +75,77 @@ Groups: -EFAULT Invalid guest ram access -EBUSY One or more VCPUS are running ======= ======================================================== + + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS + Attributes: + + The attr field of kvm_device_attr encodes two values:: + + bits: | 63 .... 32 | 31 .... 16 | 15 .... 0 | + values: | mpidr | RES | instr | + + The mpidr field encodes the CPU ID based on the affinity information in the + architecture defined MPIDR, and the field is encoded as follows:: + + | 63 .... 56 | 55 .... 48 | 47 .... 40 | 39 .... 32 | + | Aff3 | Aff2 | Aff1 | Aff0 | + + The instr field encodes the system register to access based on the fields + defined in the A64 instruction set encoding for system register access + (RES means the bits are reserved for future use and should be zero):: + + | 15 ... 14 | 13 ... 11 | 10 ... 7 | 6 ... 3 | 2 ... 0 | + | Op 0 | Op1 | CRn | CRm | Op2 | + + All system regs accessed through this API are (rw, 64-bit) and + kvm_device_attr.addr points to a __u64 value. + + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS accesses the CPU interface registers for the + CPU specified by the mpidr field. + + The available registers are: + + ======================= =================================================== + ICC_ICSR_EL1 + ICC_PPI_ENABLER0_EL1 + ICC_PPI_ENABLER1_EL1 + ICC_PPI_SACTIVER0_EL1 ICC_PPI_CACTIVER0_EL1 is not supported. Writes to + ICC_PPI_SACTIVER0_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SACTIVER1_EL1 ICC_PPI_CACTIVER1_EL1 is not supported. Writes to + ICC_PPI_SACTIVER1_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SPENDR0_EL1 ICC_PPI_CPENDR0_EL1 is not supported. Writes to + ICC_PPI_SPENDR0_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SPENDR1_EL1 ICC_PPI_CPENDR1_EL1 is not supported. Writes to + ICC_PPI_SPENDR1_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_PRIORITYR0_EL1 + ICC_PPI_PRIORITYR1_EL1 + ICC_PPI_PRIORITYR2_EL1 + ICC_PPI_PRIORITYR3_EL1 + ICC_PPI_PRIORITYR4_EL1 + ICC_PPI_PRIORITYR5_EL1 + ICC_PPI_PRIORITYR6_EL1 + ICC_PPI_PRIORITYR7_EL1 + ICC_PPI_PRIORITYR8_EL1 + ICC_PPI_PRIORITYR9_EL1 + ICC_PPI_PRIORITYR10_EL1 + ICC_PPI_PRIORITYR11_EL1 + ICC_PPI_PRIORITYR12_EL1 + ICC_PPI_PRIORITYR13_EL1 + ICC_PPI_PRIORITYR14_EL1 + ICC_PPI_PRIORITYR15_EL1 + ICC_APR_EL1 + ICC_CR0_EL1 + ICC_PCR_EL1 + ======================= =================================================== + + Errors: + + ======= ============================================================= + -ENXIO Getting or setting this register is not supported + -EBUSY VCPU is running, or write attempted after a VCPU has run + -EINVAL Invalid mpidr or register value supplied + ======= ============================================================= From 99055cdfb2102c096a7b90c677d292a4528578e4 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:53:04 +0000 Subject: [PATCH 151/506] Documentation: KVM: Add KVM_DEV_ARM_VGIC_GRP_IRS_REGS to VGICv5 docs Document the KVM_DEV_ARM_VGIC_GRP_IRS_REGS attribute group used to read and write the virtual IRS's MMIO register state. This provides a GICv5-specific interface for state that is conceptually similar to the VGICv3 ITS register interface, but uses IRS terminology instead of ITS. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-38-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- .../virt/kvm/devices/arm-vgic-v5.rst | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst index 7e41d6699538..4f475d14f83d 100644 --- a/Documentation/virt/kvm/devices/arm-vgic-v5.rst +++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst @@ -149,3 +149,50 @@ Groups: -EBUSY VCPU is running, or write attempted after a VCPU has run -EINVAL Invalid mpidr or register value supplied ======= ============================================================= + + + KVM_DEV_ARM_VGIC_GRP_IRS_REGS + Attributes: + The attr field of kvm_device_attr encodes the offset of the IRS register, + relative to the IRS CONFIG_FRAME base address. This is the address that + was provided via KVM_VGIC_V5_ADDR_TYPE_IRS when creating VGICv5 in the + first place. + + kvm_device_attr.addr points to a __u64 value whatever the width + of the addressed register (32/64 bits). 64 bit registers can only + be accessed with full length. + + Writes to read-only registers are ignored by the kernel except for + IRS_IDR0 - IRS_IDR2 and IRS_IDR5 - IRS_IDR7, which are sanity checked to + ensure that they match a sane config. + + IRS_IDR3 and IRS_IDR4 are RAZ/WI, as nested virtualization is not + supported. + + For registers without dedicated userspace accessors, getting or setting a + register uses the same emulated MMIO handlers as guest reads/writes. + Dedicated userspace accessors may instead save or restore migration state + without triggering guest-visible side effects. For example, restoring + IRS_IST_BASER only restores the emulated register state; any host LPI IST + allocation based on the restored IRS_IST_CFGR and IRS_IST_BASER state + happens when KVM_DEV_ARM_VGIC_GRP_IST is restored. + + Once an LPI IST has been allocated, IRS_IST_CFGR is immutable. Userspace + may write back its current value, but setting a different value is + rejected with -EINVAL. + + Once an LPI IST has been allocated, IRS_IST_BASER is immutable. + Userspace may write back its current value, but setting a different value + is rejected with -EINVAL. + + Errors: + + ======= ================================================================= + -ENXIO Offset does not correspond to any supported register + -EFAULT Invalid user pointer for attr->addr + -EINVAL Offset is not 32-bit aligned for 32-bit MMIO registers, not + 64-bit aligned for 64-bit registers, or the supplied register + value is not compatible with the configured VGICv5 IRS state + -EBUSY VGIC is not initialized, one or more VCPUs are running, or a + write is attempted after a VCPU has run + ======= ================================================================= From a7f442da926bae29563f456e5df6e59d44ff77ee Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:53:34 +0000 Subject: [PATCH 152/506] Documentation: KVM: Add docs for KVM_DEV_ARM_VGIC_GRP_IST Document the IST save/restore userspace interface for the VGICv5 device, KVM_DEV_ARM_VGIC_GRP_IST. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-39-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- .../virt/kvm/devices/arm-vgic-v5.rst | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst index 4f475d14f83d..61db2f93373c 100644 --- a/Documentation/virt/kvm/devices/arm-vgic-v5.rst +++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst @@ -196,3 +196,69 @@ Groups: -EBUSY VGIC is not initialized, one or more VCPUs are running, or a write is attempted after a VCPU has run ======= ================================================================= + + KVM_DEV_ARM_VGIC_GRP_IST + Attributes: + This interface is used to either save the state of the IRS's Interrupt + State Tables (ISTs), or to restore them. A get operation saves IST state, + and a set operation restores IST state. kvm_device_attr.attr is reserved + and must be zero. + + The VGIC must be initialized before using this interface. Restore must be + performed before the VM has run. For restore, userspace must have already + restored the IRS state needed to describe any guest LPI IST. + + Saving first asks the IRS to save and quiesce the VM so that interrupt + state has been written back to the ISTs. KVM checks that the VM remains + quiesced while copying out the SPI and LPI IST state. + + kvm_device_attr.addr points to a struct kvm_vgic_v5_ist:: + + struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; + }; + + The SPI and LPI IST buffers contain one little-endian 32-bit IST entry per + interrupt, in interrupt number order. Only the architected 32-bit ISTE + state is exposed to userspace; host IST layout and metadata are private to + KVM. + + The SPI IST buffer is required and its size must be: + + nr_spis * sizeof(__u32) + + where nr_spis is the value returned by KVM_DEV_ARM_VGIC_GRP_NR_IRQS for + the VGICv5 device. For VGICv5 this value is the number of SPIs, not the + total number of interrupts. + + The LPI IST buffer is required if the guest's IRS_IST_CFGR and + IRS_IST_BASER state describes a valid LPI IST, and its size must be: + + BIT(lpi_id_bits) * sizeof(__u32) + + where lpi_id_bits is the LPI ID width configured in IRS_IST_CFGR. If no + LPI IST is configured, lpi_ist_addr and lpi_ist_size must both be zero. + + Errors: + + =========== ============================================================ + -EBUSY One or more VCPUs are running, the VGIC is not initialized, + restore was requested after the VM has run, an LPI IST + already exists, or the save operation completed but the VM + did not remain quiesced + -EINVAL attr->addr is NULL, the userspace IST descriptor is missing + a required buffer, supplies a non-zero LPI buffer when no + LPI IST is configured, has a buffer size that does not match + the configured VGICv5 IST state, or an internal VM table + operation rejected the VM state + -EFAULT Invalid user pointer for attr->addr, spi_ist_addr, or + lpi_ist_addr + -ENXIO Required per-VM VGICv5/IST backing state is missing or + inconsistent + -ENOMEM Restoring IST state failed while allocating the host LPI IST + or tracking pending interrupts + -ETIMEDOUT An IRS save/VM operation timed out + =========== ============================================================ From 53e9639e758524ab1f92b836d0fc4f023e762903 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:54:05 +0000 Subject: [PATCH 153/506] Documentation: KVM: Add the VGICv5 IRS save/restore sequences Document the ordering requirements for saving and restoring the GICv5 IRS state. The IST state is exposed through KVM_DEV_ARM_VGIC_GRP_IST and is copied to or from userspace-provided buffers. Saving the ISTs therefore does not update guest memory, so userspace does not need to serialize guest memory after saving the IST state. On restore, userspace must create the vCPUs, configure the IRS address, restore the number of SPIs, and initialize the VGIC before restoring IRS state. The IRS MMIO registers may be restored in any order, but the final IRS_IDR* state and the IRS_IST_CFGR/IRS_IST_BASER state that describes any LPI IST must be restored before KVM_DEV_ARM_VGIC_GRP_IST. KVM uses that state to validate the userspace IST buffers and allocate any host LPI IST. The IRS_*_STATUSR registers are observational state in KVM. Userspace may save them for validation or debugging, but they are not required as restore input and do not need to be replayed. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-40-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- .../virt/kvm/devices/arm-vgic-v5.rst | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst index 61db2f93373c..8a691c8938b6 100644 --- a/Documentation/virt/kvm/devices/arm-vgic-v5.rst +++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst @@ -262,3 +262,52 @@ Groups: or tracking pending interrupts -ETIMEDOUT An IRS save/VM operation timed out =========== ============================================================ + +IRS Save Sequence: +------------------ + +The following operations are required when saving the virtual GICv5 IRS: + +a) Save the ISTs by issuing KVM_GET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_IST. +b) Save the IRS MMIO register state by issuing KVM_GET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IRS_REGS. + +These two steps may be performed in either order. Saving the ISTs writes all +IST state to userspace-provided buffers and does not update guest memory. + +IRS Restore Sequence: +--------------------- + +The following ordering must be followed when restoring the virtual GICv5 and +IRS: + +a) Create vCPUs. +b) Provide the IRS base address by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_ADDR +c) Restore the number of SPIs by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_NR_IRQS. +d) Initialise the GIC - this sets up the default state and creates the SPI + IST - by issuing KVM_SET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_CTRL with + KVM_DEV_ARM_VGIC_CTRL_INIT +e) Restore the IRS MMIO register state by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IRS_REGS. The IRS registers may be restored in any + order, but any IRS_IDR* state and the IRS_IST_CFGR/IRS_IST_BASER state + describing an LPI IST must be restored before restoring the ISTs. KVM uses + the restored IRS_IST_CFGR and IRS_IST_BASER state to allocate the LPI IST + during the following step. +f) Restore the ISTs by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IST. + +The number of SPIs must be restored before VGIC initialization because +initialization allocates the SPI state and fixes the SPI range exposed by the +IRS ID registers. + +The IRS's MMIO registers must be restored prior to restoring the ISTs as these +are used to convey the number of LPIs the guest has configured to KVM. + +The various ``*_STATUSR`` registers are observational state in the current KVM +implementation. Userspace may save them for validation or debugging purposes, +but they are not required as restore input and do not need to be replayed during +restore. + +Then vCPUs can be started. From e9d07ebc7eb41ebbc83d5eb52f43a970917830e9 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:54:35 +0000 Subject: [PATCH 154/506] KVM: selftests: Add VGICv5 IRS address attribute tests Add coverage for the VGICv5 IRS address attribute. The address is the base of the guest-visible MMIO region used to configure the virtual IRS, so userspace must provide a valid base address before the VM can run. Verify that KVM reports the attribute, returns VGIC_ADDR_UNDEF before it is configured, rejects misaligned and out-of-IPA-range addresses, accepts a valid base, rejects a second base, and fails vCPU entry if the VGIC is initialized without an IRS address. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-41-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 121 ++++++++++++++++-- .../selftests/kvm/include/arm64/gic_v5.h | 3 + 2 files changed, 110 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index 19039a894056..51ea2fe7141a 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -14,6 +14,8 @@ #define NR_VCPUS 1 +static u64 max_phys_size; + struct vm_gic { struct kvm_vm *vm; int gic_fd; @@ -25,6 +27,104 @@ struct vm_gic { #define GUEST_CMD_IS_AWAKE 12 #define GUEST_CMD_IS_READY 13 +/* we don't want to assert on run execution, hence that helper */ +static int run_vcpu(struct kvm_vcpu *vcpu) +{ + return __vcpu_run(vcpu) ? -errno : 0; +} + +static void vm_gic_destroy(struct vm_gic *v) +{ + close(v->gic_fd); + kvm_vm_free(v->vm); +} + +struct vgic_region_attr { + u64 attr; + u64 size; + u64 alignment; +}; + +static const struct vgic_region_attr gic_v5_irs_region = { + .attr = KVM_VGIC_V5_ADDR_TYPE_IRS, + .size = GICV5_IRS_SIZE, + .alignment = GICV5_IRS_ALIGN, +}; + +static void test_vgic_v5_create(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 addr; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + + /* Check existing group/attributes */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, gic_v5_irs_region.attr); + + /* check non existing attribute */ + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, -1); + TEST_ASSERT(ret && errno == ENXIO, "attribute not supported"); + + /* get IRS base address before setting*/ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret && addr == (-1ULL), "GICv5 IRS returns VGIC_ADDR_UNDEF"); + + /* misaligned IRS address settings */ + addr = gic_v5_irs_region.alignment / 0x10; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == EINVAL, "GIC IRS base not aligned"); + + /* out of range address */ + addr = max_phys_size; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == E2BIG, "IRS address beyond IPA limit"); + + /* Space for half an IRS (an IRS is: 2 * irs.alignment). */ + addr = max_phys_size - gic_v5_irs_region.alignment; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == E2BIG, + "half of the IRS is beyond IPA limit"); + + /* set IRS base address @0x0*/ + addr = 0x00000; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret, "GICv5 IRS base correctly set"); + + /* get IRS base address */ + addr = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret && addr == 0, "GICv5 IRS base correctly set"); + + /* Attempt to create a second IRS region */ + addr = 0xE0000; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == EEXIST, "GICv5 IRS base set again"); + + vm_gic_destroy(&v); + + /* Try running a VM without ever setting the IRS base addr */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + ret = run_vcpu(vcpu); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS base not set"); + + vm_gic_destroy(&v); +} + static void guest_irq_handler(struct ex_regs *regs) { bool valid; @@ -81,19 +181,6 @@ static void guest_code(void) while (1); } - -/* we don't want to assert on run execution, hence that helper */ -static int run_vcpu(struct kvm_vcpu *vcpu) -{ - return __vcpu_run(vcpu) ? -errno : 0; -} - -static void vm_gic_destroy(struct vm_gic *v) -{ - close(v->gic_fd); - kvm_vm_free(v->vm); -} - static void test_vgic_v5_ppis(u32 gic_dev_type) { struct kvm_vcpu *vcpus[NR_VCPUS]; @@ -207,13 +294,19 @@ int test_kvm_device(u32 gic_dev_type) void run_tests(u32 gic_dev_type) { + pr_info("Test VGICv5 Creation & Setup\n"); + test_vgic_v5_create(); + pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); } int main(int ac, char **av) { - int ret; + int pa_bits, ret; + + pa_bits = vm_guest_mode_params[VM_MODE_DEFAULT].pa_bits; + max_phys_size = 1ULL << pa_bits; test_disable_default_vgic(); diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v5.h b/tools/testing/selftests/kvm/include/arm64/gic_v5.h index c388df8f2a2b..41b698b3952f 100644 --- a/tools/testing/selftests/kvm/include/arm64/gic_v5.h +++ b/tools/testing/selftests/kvm/include/arm64/gic_v5.h @@ -13,6 +13,9 @@ /* GIC component base address is guest PA space */ #define GICV5_IRS_CONFIG_BASE_GPA 0x8000000ULL +#define GICV5_IRS_SIZE 0x20000 +#define GICV5_IRS_ALIGN 0x10000 + /* * Definitions for GICv5 instructions for the Current Domain */ From 45895c459225150082427576582f887c9f80d720 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:55:07 +0000 Subject: [PATCH 155/506] KVM: selftests: Add VGICv5 NR_IRQS attribute tests Add coverage for the VGICv5 NR_IRQS attribute. VGICv5 exposes a userspace-selectable SPI count, and KVM must reject values outside its supported range or values that would change the interrupt layout after it has been fixed. Verify that the attribute defaults to zero before initialisation, rejects too few SPIs, non-32-aligned counts, and a 32-aligned count beyond the 1024-SPI maximum. Also verify that the maximum count is accepted and can be initialised, that the selected count is reported, that initialisation selects the default count, and that subsequent changes are rejected. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-42-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 175 +++++++++++++++----- 1 file changed, 134 insertions(+), 41 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index 51ea2fe7141a..bbcf0b5d2b22 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -13,6 +13,8 @@ #include "vgic.h" #define NR_VCPUS 1 +#define VGIC_V5_DEFAULT_NR_SPIS 32 +#define VGIC_V5_MAX_NR_SPIS BIT(10) static u64 max_phys_size; @@ -27,6 +29,63 @@ struct vm_gic { #define GUEST_CMD_IS_AWAKE 12 #define GUEST_CMD_IS_READY 13 +static void guest_irq_handler(struct ex_regs *regs) +{ + bool valid; + u32 hwirq; + u64 ia; + static int count; + + /* + * We have pending interrupts. Should never actually enter WFI + * here! + */ + wfi(); + GUEST_SYNC(GUEST_CMD_IS_AWAKE); + + ia = gicr_insn(CDIA); + valid = GICV5_GICR_CDIA_VALID(ia); + + GUEST_SYNC(GUEST_CMD_IRQ_CDIA); + + if (!valid) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + + GUEST_SYNC(GUEST_CMD_IRQ_DIEOI); + + if (++count >= 2) + GUEST_DONE(); + + /* Ask for the next interrupt to be injected */ + GUEST_SYNC(GUEST_CMD_IS_READY); +} + +static void guest_code(void) +{ + local_irq_disable(); + + gicv5_cpu_enable_interrupts(); + local_irq_enable(); + + /* Enable the SW_PPI (3) */ + write_sysreg_s(BIT_ULL(3), SYS_ICC_PPI_ENABLER0_EL1); + + /* Ask for the first interrupt to be injected */ + GUEST_SYNC(GUEST_CMD_IS_READY); + + /* Loop forever waiting for interrupts */ + for (;;) + cpu_relax(); +} + /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) { @@ -51,7 +110,7 @@ static const struct vgic_region_attr gic_v5_irs_region = { .alignment = GICV5_IRS_ALIGN, }; -static void test_vgic_v5_create(void) +static void test_vgic_v5_addr_attrs(void) { struct kvm_vcpu *vcpu; struct vm_gic v; @@ -125,60 +184,91 @@ static void test_vgic_v5_create(void) vm_gic_destroy(&v); } -static void guest_irq_handler(struct ex_regs *regs) +static void test_vgic_v5_nr_irqs_attrs(void) { - bool valid; - u32 hwirq; - u64 ia; - static int count; + struct kvm_vcpu *vcpu; + struct vm_gic v; + u32 nr_irqs; + int ret; - /* - * We have pending interrupts. Should never actually enter WFI - * here! - */ - wfi(); - GUEST_SYNC(GUEST_CMD_IS_AWAKE); + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); - ia = gicr_insn(CDIA); - valid = GICV5_GICR_CDIA_VALID(ia); + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0); - GUEST_SYNC(GUEST_CMD_IRQ_CDIA); + /* Before userspace sets NR_IRQS, no SPI count has been selected. */ + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == 0, "GICv5 NR_IRQS defaults to 0 before init"); - if (!valid) - return; + /* Too few SPIs */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS - 1; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS below minimum"); - gsb_ack(); - isb(); + /* Not a multiple of 32 */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS + 1; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS not 32-aligned"); - hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + /* Larger than KVM's supported VGICv5 SPI count */ + nr_irqs = VGIC_V5_MAX_NR_SPIS + 32; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS above maximum"); - gic_insn(hwirq, CDDI); - gic_insn(0, CDEOI); + /* Valid custom SPI count */ + nr_irqs = VGIC_V5_MAX_NR_SPIS; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret, "GICv5 NR_IRQS accepts valid custom SPI count"); - GUEST_SYNC(GUEST_CMD_IRQ_DIEOI); + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == VGIC_V5_MAX_NR_SPIS, + "GICv5 NR_IRQS returns SPI count only"); - if (++count >= 2) - GUEST_DONE(); + /* A second successful configuration attempt must be rejected. */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 NR_IRQS set twice"); - /* Ask for the next interrupt to be injected */ - GUEST_SYNC(GUEST_CMD_IS_READY); -} + /* The maximum supported count must also initialize successfully. */ + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); -static void guest_code(void) -{ - local_irq_disable(); + vm_gic_destroy(&v); - gicv5_cpu_enable_interrupts(); - local_irq_enable(); + /* If userspace does not set NR_IRQS, init selects the default. */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); - /* Enable the SW_PPI (3) */ - write_sysreg_s(BIT_ULL(3), SYS_ICC_PPI_ENABLER0_EL1); + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == VGIC_V5_DEFAULT_NR_SPIS, + "GICv5 NR_IRQS defaults to 32 SPIs after init"); - /* Ask for the first interrupt to be injected */ - GUEST_SYNC(GUEST_CMD_IS_READY); + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS * 2; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 NR_IRQS set after init"); + + vm_gic_destroy(&v); - /* Loop forever waiting for interrupts */ - while (1); } static void test_vgic_v5_ppis(u32 gic_dev_type) @@ -294,8 +384,11 @@ int test_kvm_device(u32 gic_dev_type) void run_tests(u32 gic_dev_type) { - pr_info("Test VGICv5 Creation & Setup\n"); - test_vgic_v5_create(); + pr_info("Test VGICv5 address attrs\n"); + test_vgic_v5_addr_attrs(); + + pr_info("Test VGICv5 NR_IRQS attrs\n"); + test_vgic_v5_nr_irqs_attrs(); pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); From fe9c6aa09239ea7a99ca1657afb4522c36972dc1 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:55:37 +0000 Subject: [PATCH 156/506] KVM: selftests: Add VGICv5 IRS_REGS attribute tests Add coverage for the VGICv5 IRS_REGS attribute group. Migration needs a userspace-visible view of the virtual IRS register state, but KVM must expose only valid registers and reject state that cannot be restored safely. Verify that KVM_HAS_DEVICE_ATTR reports the IRS_REGS group before the IRS base is configured, while register reads and writes are rejected before VGIC initialisation. Once the VGIC is initialised, verify that all exposed registers can be read and their values written back. Check that bad offsets, incorrect alignment, and bad userspace pointers are rejected. Also check ID register restore validation, idle status register values, and IRS_IST_BASER restore handling. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-43-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 260 ++++++++++++++++++ .../selftests/kvm/include/arm64/gic_v5.h | 72 +++++ 2 files changed, 332 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index bbcf0b5d2b22..1166766dc7c9 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -18,6 +18,8 @@ static u64 max_phys_size; +#define VGIC_V5_TEST_IST_BASER_GPA 0x10000000ULL + struct vm_gic { struct kvm_vm *vm; int gic_fd; @@ -110,6 +112,72 @@ static const struct vgic_region_attr gic_v5_irs_region = { .alignment = GICV5_IRS_ALIGN, }; +struct vgic_irs_reg_attr { + const char *name; + u64 attr; +}; + +#define IRS_REG(r) \ + { \ + .name = #r, \ + .attr = r, \ + } + +static const struct vgic_irs_reg_attr gic_v5_irs_regs[] = { + IRS_REG(GICV5_IRS_IDR0), + IRS_REG(GICV5_IRS_IDR1), + IRS_REG(GICV5_IRS_IDR2), + IRS_REG(GICV5_IRS_IDR3), + IRS_REG(GICV5_IRS_IDR4), + IRS_REG(GICV5_IRS_IDR5), + IRS_REG(GICV5_IRS_IDR6), + IRS_REG(GICV5_IRS_IDR7), + IRS_REG(GICV5_IRS_IIDR), + IRS_REG(GICV5_IRS_AIDR), + IRS_REG(GICV5_IRS_CR0), + IRS_REG(GICV5_IRS_CR1), + IRS_REG(GICV5_IRS_SYNCR), + IRS_REG(GICV5_IRS_SYNC_STATUSR), + IRS_REG(GICV5_IRS_SPI_VMR), + IRS_REG(GICV5_IRS_SPI_SELR), + IRS_REG(GICV5_IRS_SPI_DOMAINR), + IRS_REG(GICV5_IRS_SPI_RESAMPLER), + IRS_REG(GICV5_IRS_SPI_CFGR), + IRS_REG(GICV5_IRS_SPI_STATUSR), + IRS_REG(GICV5_IRS_PE_SELR), + IRS_REG(GICV5_IRS_PE_STATUSR), + IRS_REG(GICV5_IRS_PE_CR0), + IRS_REG(GICV5_IRS_IST_BASER), + IRS_REG(GICV5_IRS_IST_CFGR), + IRS_REG(GICV5_IRS_IST_STATUSR), + IRS_REG(GICV5_IRS_MAP_L2_ISTR), + IRS_REG(GICV5_IRS_VMT_BASER), + IRS_REG(GICV5_IRS_VMT_CFGR), + IRS_REG(GICV5_IRS_VMT_STATUSR), + IRS_REG(GICV5_IRS_VPE_SELR), + IRS_REG(GICV5_IRS_VPE_DBR), + IRS_REG(GICV5_IRS_VPE_HPPIR), + IRS_REG(GICV5_IRS_VPE_CR0), + IRS_REG(GICV5_IRS_VPE_STATUSR), + IRS_REG(GICV5_IRS_VM_DBR), + IRS_REG(GICV5_IRS_VM_SELR), + IRS_REG(GICV5_IRS_VM_STATUSR), + IRS_REG(GICV5_IRS_VMAP_L2_VMTR), + IRS_REG(GICV5_IRS_VMAP_VMR), + IRS_REG(GICV5_IRS_VMAP_VISTR), + IRS_REG(GICV5_IRS_VMAP_L2_VISTR), + IRS_REG(GICV5_IRS_VMAP_VPER), + IRS_REG(GICV5_IRS_SAVE_VMR), + IRS_REG(GICV5_IRS_SAVE_VM_STATUSR), + IRS_REG(GICV5_IRS_MEC_IDR), + IRS_REG(GICV5_IRS_MEC_MECID_R), + IRS_REG(GICV5_IRS_MPAM_IDR), + IRS_REG(GICV5_IRS_MPAM_PARTID_R), + IRS_REG(GICV5_IRS_SWERR_STATUSR), + IRS_REG(GICV5_IRS_SWERR_SYNDROMER0), + IRS_REG(GICV5_IRS_SWERR_SYNDROMER1), +}; + static void test_vgic_v5_addr_attrs(void) { struct kvm_vcpu *vcpu; @@ -271,6 +339,194 @@ static void test_vgic_v5_nr_irqs_attrs(void) } +static void test_vgic_v5_irs_regs_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 attr, val; + int ret, i; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* IRS_REGS attributes can be probed before the IRS base is set. */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0); + + /* IRS_REGS are not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IRS_REGS get before init"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IRS_REGS set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Read all supported IRS regs and write the value back. */ + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + attr = gic_v5_irs_regs[i].attr; + ret = __kvm_has_device_attr(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(!ret, "GICv5 IRS_REGS missing %s", + gic_v5_irs_regs[i].name); + + val = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get failed for %s", + gic_v5_irs_regs[i].name); + + ret = __kvm_device_attr_set(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS set failed for %s", + gic_v5_irs_regs[i].name); + } + + /* Check bad offsets */ + attr = 0x10000; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS_REGS accepted bad offset"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS_REGS get bad offset"); + + /* Check alignment for 32-bit and 64-bit IRS regs. */ + attr = GICV5_IRS_IDR0 + 2; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unaligned 32-bit attr"); + + attr = GICV5_IRS_IST_BASER + 4; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unaligned 64-bit attr"); + + /* Check bad user pointers */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IRS_REGS get with bad pointer"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IRS_REGS set with bad pointer"); + + /* ID restore validation rejects unsupported values. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR0 failed"); + val &= ~GICV5_IRS_IDR0_INT_DOM; + val |= FIELD_PREP(GICV5_IRS_IDR0_INT_DOM, + GICV5_IRS_IDR0_INT_DOM_SECURE); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR0 domain"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR0 failed"); + val |= GICV5_IRS_IDR0_SETLPI; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unsupported IDR0"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR1, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR1 failed"); + val |= FIELD_PREP(GICV5_IRS_IDR1_PRIORITY_BITS, 0x7); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR1, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR1"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR2, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR2"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR5, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR5 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR5, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR5"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR6, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR6 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR6, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR6"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR7, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR7 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR7, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR7"); + + /* Status registers read as idle through userspace accessors. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_SYNC_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_SYNC_STATUSR_IDLE, + "GICv5 IRS_REGS SYNC_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_SPI_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_SPI_STATUSR_IDLE, + "GICv5 IRS_REGS SPI_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_PE_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_PE_STATUSR_IDLE, + "GICv5 IRS_REGS PE_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_IST_STATUSR_IDLE, + "GICv5 IRS_REGS IST_STATUSR is not idle"); + + /* + * Userspace restores IST_BASER without allocating or accessing an IST. + * Supply a synthetic, aligned guest address only to verify the register + * round trip. + */ + val = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + VGIC_V5_TEST_IST_BASER_GPA >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + val |= GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS failed to restore IST_BASER"); + + attr = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &attr); + TEST_ASSERT(!ret && attr == val, "GICv5 IRS_REGS IST_BASER restore mismatch"); + + vm_gic_destroy(&v); +} + static void test_vgic_v5_ppis(u32 gic_dev_type) { struct kvm_vcpu *vcpus[NR_VCPUS]; @@ -390,6 +646,10 @@ void run_tests(u32 gic_dev_type) pr_info("Test VGICv5 NR_IRQS attrs\n"); test_vgic_v5_nr_irqs_attrs(); + pr_info("Test VGICv5 IRS_REGS attrs\n"); + test_vgic_v5_irs_regs_attrs(); + + pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); } diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v5.h b/tools/testing/selftests/kvm/include/arm64/gic_v5.h index 41b698b3952f..0ad6dcc3d510 100644 --- a/tools/testing/selftests/kvm/include/arm64/gic_v5.h +++ b/tools/testing/selftests/kvm/include/arm64/gic_v5.h @@ -16,6 +16,78 @@ #define GICV5_IRS_SIZE 0x20000 #define GICV5_IRS_ALIGN 0x10000 +/* + * IRS register offsets. Keep this selftest copy scoped to the offsets and + * fields exercised by the userspace API tests. + */ +#define GICV5_IRS_IDR0 0x0000 +#define GICV5_IRS_IDR1 0x0004 +#define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 +#define GICV5_IRS_IDR5 0x0014 +#define GICV5_IRS_IDR6 0x0018 +#define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 +#define GICV5_IRS_CR0 0x0080 +#define GICV5_IRS_CR1 0x0084 +#define GICV5_IRS_SYNCR 0x00c0 +#define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 +#define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 +#define GICV5_IRS_SPI_CFGR 0x0114 +#define GICV5_IRS_SPI_STATUSR 0x0118 +#define GICV5_IRS_PE_SELR 0x0140 +#define GICV5_IRS_PE_STATUSR 0x0144 +#define GICV5_IRS_PE_CR0 0x0148 +#define GICV5_IRS_IST_BASER 0x0180 +#define GICV5_IRS_IST_CFGR 0x0190 +#define GICV5_IRS_IST_STATUSR 0x0194 +#define GICV5_IRS_MAP_L2_ISTR 0x01c0 +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 + +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 + +#define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) + +#define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_PE_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL + /* * Definitions for GICv5 instructions for the Current Domain */ From aca2eb44d850d0c6fa16ba87334414cc2fa271c5 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:56:08 +0000 Subject: [PATCH 157/506] KVM: selftests: Add VGICv5 IST attribute tests Add API coverage for the VGICv5 IST attribute group. This does not test that IST save/restore preserves interrupt state; it only checks the attribute plumbing and the cases KVM must reject before the end-to-end restore path can rely on the interface. Verify that KVM reports the IST attribute group, rejects unknown attributes, rejects save/restore before initialization, rejects bad userspace pointers, and rejects IST restore after the VM has run. Also cover the userspace IST descriptor validation: a VM with SPIs must provide a SPI IST buffer of the expected size, and LPI storage must not be supplied when no LPI IST is configured. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-44-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 131 ++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index 1166766dc7c9..a50775da4345 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -31,6 +31,17 @@ struct vm_gic { #define GUEST_CMD_IS_AWAKE 12 #define GUEST_CMD_IS_READY 13 +static struct kvm_vgic_v5_ist vgic_v5_ist_attr(void *spi_ist, size_t spi_size, + void *lpi_ist, size_t lpi_size) +{ + return (struct kvm_vgic_v5_ist) { + .spi_ist_addr = (uintptr_t)spi_ist, + .spi_ist_size = spi_size, + .lpi_ist_addr = (uintptr_t)lpi_ist, + .lpi_ist_size = lpi_size, + }; +} + static void guest_irq_handler(struct ex_regs *regs) { bool valid; @@ -527,6 +538,123 @@ static void test_vgic_v5_irs_regs_attrs(void) vm_gic_destroy(&v); } +static void test_vgic_v5_ist_attrs(void) +{ + struct kvm_vgic_v5_ist ist_attr; + struct kvm_vcpu *vcpu; + struct vm_gic v; + u32 spi_ist[VGIC_V5_DEFAULT_NR_SPIS]; + u64 attr; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, 0); + + /* Check non-existing attribute */ + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, 1); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST accepted bad attr"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 1, &ist_attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST get accepted bad attr"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 1, &ist_attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST set accepted bad attr"); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + + /* IST save/restore is not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST get before init"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* A VM with SPIs must provide a userspace IST descriptor. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, NULL); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted NULL descriptor"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, NULL); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted NULL descriptor"); + + /* Check bad userspace IST descriptors. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, (void *)1); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IST get with bad descriptor"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, (void *)1); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IST set with bad descriptor"); + + /* Check missing and incorrectly sized SPI IST buffers. */ + ist_attr = vgic_v5_ist_attr(NULL, 0, NULL, 0); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted missing SPI buffer"); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted missing SPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist) - sizeof(__u32), + NULL, 0); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted bad SPI size"); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted bad SPI size"); + + /* LPI storage must be absent when no LPI IST is configured. */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), spi_ist, + sizeof(__u32)); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted unexpected LPI buffer"); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted unexpected LPI buffer"); + + vm_gic_destroy(&v); + + /* IST restore is rejected after the VM has run. */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, guest_code); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 VM before IST restore test"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST restore after run"); + + vm_gic_destroy(&v); +} + static void test_vgic_v5_ppis(u32 gic_dev_type) { struct kvm_vcpu *vcpus[NR_VCPUS]; @@ -649,6 +777,9 @@ void run_tests(u32 gic_dev_type) pr_info("Test VGICv5 IRS_REGS attrs\n"); test_vgic_v5_irs_regs_attrs(); + pr_info("Test VGICv5 IST attrs\n"); + test_vgic_v5_ist_attrs(); + pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); From 2899ec9a2e6a2694423efb46a408a37e66283744 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:56:38 +0000 Subject: [PATCH 158/506] KVM: selftests: Add VGICv5 USERSPACE_PPIS tests Add coverage for the VGICv5 USERSPACE_PPIS control attribute. VGICv5 only allows userspace to drive PPIs that KVM explicitly advertises, so the selftest should verify the discovery path and the corresponding KVM_IRQ_LINE rejection cases. Verify that KVM reports the control attribute and that the returned bitmap contains SW_PPI. Also check that the upper 64 bits are zero, as KVM only exposes architected PPIs (the lower 64 entries). Finally, verify that writes, bad userspace pointers, PPIs outside the advertised bitmap, and the first out-of-range PPI number are rejected. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-45-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 70 +++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index a50775da4345..ca574bd45ff0 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -13,6 +13,7 @@ #include "vgic.h" #define NR_VCPUS 1 +#define VGIC_V5_NR_PRIVATE_IRQS 64 #define VGIC_V5_DEFAULT_NR_SPIS 32 #define VGIC_V5_MAX_NR_SPIS BIT(10) @@ -111,6 +112,17 @@ static void vm_gic_destroy(struct vm_gic *v) kvm_vm_free(v->vm); } +static u32 vgic_v5_irq_line_payload(u32 type, u32 num) +{ + return (type << KVM_ARM_IRQ_TYPE_SHIFT) | + FIELD_PREP(KVM_ARM_IRQ_NUM_MASK, num); +} + +static int __vgic_v5_irq_line(struct kvm_vm *vm, u32 type, u32 num, int level) +{ + return _kvm_irq_line(vm, vgic_v5_irq_line_payload(type, num), level); +} + struct vgic_region_attr { u64 attr; u64 size; @@ -655,6 +667,47 @@ static void test_vgic_v5_ist_attrs(void) vm_gic_destroy(&v); } +static void test_vgic_v5_userspace_ppis_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 user_ppis[2]; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT); + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + user_ppis[0] = 0; + user_ppis[1] = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, user_ppis); + TEST_ASSERT(!ret, "GICv5 USERSPACE_PPIS get failed"); + TEST_ASSERT(user_ppis[0] & BIT(GICV5_ARCH_PPI_SW_PPI), + "GICv5 USERSPACE_PPIS does not expose SW_PPI"); + TEST_ASSERT(!user_ppis[1], "GICv5 USERSPACE_PPIS upper word is not zero"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, user_ppis); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 USERSPACE_PPIS set accepted"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 USERSPACE_PPIS get with bad pointer"); + + vm_gic_destroy(&v); +} + static void test_vgic_v5_ppis(u32 gic_dev_type) { struct kvm_vcpu *vcpus[NR_VCPUS]; @@ -694,6 +747,20 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) TEST_ASSERT(user_ppis[0] & BIT(GICV5_ARCH_PPI_SW_PPI), "SW_PPI is not drivable by userspace"); + /* PPIs not explicitly exposed to userspace must be rejected. */ + for (i = 0; i < VGIC_V5_NR_PRIVATE_IRQS; i++) { + if (user_ppis[i / 64] & BIT_ULL(i % 64)) + continue; + + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_PPI, i, 1); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted non-userspace PPI %d", i); + } + + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_PPI, + VGIC_V5_NR_PRIVATE_IRQS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted out-of-range PPI"); + while (1) { ret = run_vcpu(vcpus[0]); if (ret) @@ -780,6 +847,9 @@ void run_tests(u32 gic_dev_type) pr_info("Test VGICv5 IST attrs\n"); test_vgic_v5_ist_attrs(); + pr_info("Test VGICv5 userspace PPI attrs\n"); + test_vgic_v5_userspace_ppis_attrs(); + pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); From 44607858563390e7f124ee343abda200d823755b Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:57:08 +0000 Subject: [PATCH 159/506] KVM: selftests: Add VGICv5 CPU sysreg attribute tests Add coverage for the VGICv5 CPU_SYSREGS attribute group. The vCPU-local CPU interface registers are part of the VGICv5 userspace ABI, so KVM must expose the supported registers and reject invalid register or vCPU selectors. Verify that CPU_SYSREGS access is rejected before VGIC initialization, that all exposed VGICv5 CPU interface registers can be read and written back, and that unsupported sysregs, invalid MPIDRs and bad userspace pointers are rejected. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-46-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 137 ++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index ca574bd45ff0..f294e58f6b50 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -135,11 +135,29 @@ static const struct vgic_region_attr gic_v5_irs_region = { .alignment = GICV5_IRS_ALIGN, }; +struct vgic_sysreg_attr { + const char *name; + u32 encoding; +}; + struct vgic_irs_reg_attr { const char *name; u64 attr; }; +#define PACK_SR(r) \ + ((sys_reg_Op0(r) << 14) | \ + (sys_reg_Op1(r) << 11) | \ + (sys_reg_CRn(r) << 7) | \ + (sys_reg_CRm(r) << 3) | \ + (sys_reg_Op2(r))) + +#define SR(r) \ + { \ + .name = #r, \ + .encoding = r, \ + } + #define IRS_REG(r) \ { \ .name = #r, \ @@ -201,6 +219,35 @@ static const struct vgic_irs_reg_attr gic_v5_irs_regs[] = { IRS_REG(GICV5_IRS_SWERR_SYNDROMER1), }; +static const struct vgic_sysreg_attr gic_v5_cpu_sysregs[] = { + SR(SYS_ICC_ICSR_EL1), + SR(SYS_ICC_PPI_ENABLER0_EL1), + SR(SYS_ICC_PPI_ENABLER1_EL1), + SR(SYS_ICC_PPI_SACTIVER0_EL1), + SR(SYS_ICC_PPI_SACTIVER1_EL1), + SR(SYS_ICC_PPI_SPENDR0_EL1), + SR(SYS_ICC_PPI_SPENDR1_EL1), + SR(SYS_ICC_PPI_PRIORITYR0_EL1), + SR(SYS_ICC_PPI_PRIORITYR1_EL1), + SR(SYS_ICC_PPI_PRIORITYR2_EL1), + SR(SYS_ICC_PPI_PRIORITYR3_EL1), + SR(SYS_ICC_PPI_PRIORITYR4_EL1), + SR(SYS_ICC_PPI_PRIORITYR5_EL1), + SR(SYS_ICC_PPI_PRIORITYR6_EL1), + SR(SYS_ICC_PPI_PRIORITYR7_EL1), + SR(SYS_ICC_PPI_PRIORITYR8_EL1), + SR(SYS_ICC_PPI_PRIORITYR9_EL1), + SR(SYS_ICC_PPI_PRIORITYR10_EL1), + SR(SYS_ICC_PPI_PRIORITYR11_EL1), + SR(SYS_ICC_PPI_PRIORITYR12_EL1), + SR(SYS_ICC_PPI_PRIORITYR13_EL1), + SR(SYS_ICC_PPI_PRIORITYR14_EL1), + SR(SYS_ICC_PPI_PRIORITYR15_EL1), + SR(SYS_ICC_APR_EL1), + SR(SYS_ICC_CR0_EL1), + SR(SYS_ICC_PCR_EL1), +}; + static void test_vgic_v5_addr_attrs(void) { struct kvm_vcpu *vcpu; @@ -708,6 +755,93 @@ static void test_vgic_v5_userspace_ppis_attrs(void) vm_gic_destroy(&v); } +static void test_vgic_v5_cpu_sysreg_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 attr, val; + int ret, i; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* Check existing group/attribute */ + attr = PACK_SR(SYS_ICC_CR0_EL1); + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, attr); + + /* CPU sysregs are not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 CPU_SYSREGS get before init"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 CPU_SYSREGS set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Read all exposed CPU sysregs and write the value back. */ + for (i = 0; i < ARRAY_SIZE(gic_v5_cpu_sysregs); i++) { + attr = PACK_SR(gic_v5_cpu_sysregs[i].encoding); + ret = __kvm_has_device_attr(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS missing %s", + gic_v5_cpu_sysregs[i].name); + + val = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS get failed for %s", + gic_v5_cpu_sysregs[i].name); + + ret = __kvm_device_attr_set(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS set failed for %s", + gic_v5_cpu_sysregs[i].name); + } + + /* Check non existent GICv3 sysreg */ + attr = PACK_SR(SYS_ICC_CTLR_EL1); + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 CPU_SYSREGS accepted bad sysreg"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == ENOENT, "GICv5 CPU_SYSREGS get bad sysreg"); + + /* Check non existing vCPU */ + attr = PACK_SR(SYS_ICC_CR0_EL1) | + (1ULL << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT); + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 CPU_SYSREGS accepted bad MPIDR"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 CPU_SYSREGS get bad MPIDR"); + + /* Check bad user pointers */ + attr = PACK_SR(SYS_ICC_CR0_EL1); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 CPU_SYSREGS get with bad pointer"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 CPU_SYSREGS set with bad pointer"); + + vm_gic_destroy(&v); +} + static void test_vgic_v5_ppis(u32 gic_dev_type) { struct kvm_vcpu *vcpus[NR_VCPUS]; @@ -850,6 +984,9 @@ void run_tests(u32 gic_dev_type) pr_info("Test VGICv5 userspace PPI attrs\n"); test_vgic_v5_userspace_ppis_attrs(); + pr_info("Test VGICv5 CPU sysreg attrs\n"); + test_vgic_v5_cpu_sysreg_attrs(); + pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); From 96e48f2f5322cecebabcda953b96efc254919c49 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:57:39 +0000 Subject: [PATCH 160/506] KVM: selftests: Add VGICv5 SPI injection tests Add VGICv5 SPI delivery coverage through KVM_IRQ_LINE. SPIs are configured through the virtual IRS and delivered through the GICv5 CPU interface, so the test needs to exercise both the userspace injection path and the guest-side IRS programming path. Map the virtual IRS into the guest, enable the IRS, configure SPI trigger mode and target VPE from the guest, and check delivery of a default SPI and SPI 1023, the last SPI in the maximum configured range. This also exercises VGIC initialization and the IRS selector at KVM's upper bound. Verify that invalid SPI numbers are rejected, that lowering an edge-triggered SPI after it is pending does not recall it, and that level-sensitive SPIs are delivered. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-47-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 258 +++++++++++++++++- .../selftests/kvm/include/arm64/gic_v5.h | 11 + 2 files changed, 264 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index f294e58f6b50..ea3451d850c2 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -16,6 +16,8 @@ #define VGIC_V5_NR_PRIVATE_IRQS 64 #define VGIC_V5_DEFAULT_NR_SPIS 32 #define VGIC_V5_MAX_NR_SPIS BIT(10) +#define VGIC_V5_IRS_SIZE 0x20000 +#define VGIC_V5_IRS_WAIT_RETRIES 1000000 static u64 max_phys_size; @@ -43,7 +45,71 @@ static struct kvm_vgic_v5_ist vgic_v5_ist_attr(void *spi_ist, size_t spi_size, }; } -static void guest_irq_handler(struct ex_regs *regs) +static u32 spi_line_expected; +static bool spi_line_level_sensitive; + +static u64 gicv5_spi_hwirq(u32 spi) +{ + return FIELD_PREP(GICV5_HWIRQ_TYPE, GICV5_HWIRQ_TYPE_SPI) | + FIELD_PREP(GICV5_HWIRQ_ID, spi); +} + +static void gicv5_setup_and_enable_hwirq(u64 hwirq, u32 target_vpe) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, + GICV5_IRQ_DEFAULT_PRI); + gic_insn(val, CDPRI); + + val = hwirq | FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, target_vpe); + gic_insn(val, CDAFF); + + gic_insn(hwirq, CDEN); +} + +static void gicv5_enable_spi(u32 spi, u32 target_vpe) +{ + gicv5_setup_and_enable_hwirq(gicv5_spi_hwirq(spi), target_vpe); +} + +static void gicv5_wait_for_irs_idle(u32 reg, u32 idle) +{ + int i; + + for (i = 0; i < VGIC_V5_IRS_WAIT_RETRIES; i++) { + if (readl(GICV5_IRS_CONFIG_BASE_GVA + reg) & idle) + return; + + cpu_relax(); + } + + GUEST_FAIL("IRS operation did not become idle"); +} + +static void gicv5_configure_spi(u32 spi, bool level) +{ + u32 val; + + val = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, spi); + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_SPI_SELR); + gicv5_wait_for_irs_idle(GICV5_IRS_SPI_STATUSR, + GICV5_IRS_SPI_STATUSR_IDLE); + + val = level ? GICV5_IRS_SPI_CFGR_TM : 0; + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_SPI_CFGR); + gicv5_wait_for_irs_idle(GICV5_IRS_SPI_STATUSR, + GICV5_IRS_SPI_STATUSR_IDLE); +} + +static void gicv5_enable_irs(void) +{ + writel(GICV5_IRS_CR0_IRSEN, + GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_CR0); + gicv5_wait_for_irs_idle(GICV5_IRS_CR0, GICV5_IRS_CR0_IDLE); +} + +static void guest_ppi_irq_handler(struct ex_regs *regs) { bool valid; u32 hwirq; @@ -100,6 +166,49 @@ static void guest_code(void) cpu_relax(); } +static void guest_spi_irq_handler(struct ex_regs *regs) +{ + bool valid; + u32 hwirq; + u64 ia; + + ia = gicr_insn(CDIA); + valid = GICV5_GICR_CDIA_VALID(ia); + + if (!valid) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + + GUEST_ASSERT_EQ(hwirq, gicv5_spi_hwirq(READ_ONCE(spi_line_expected))); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + + GUEST_DONE(); +} + +static void guest_spi_line_code(void) +{ + local_irq_disable(); + + gicv5_enable_irs(); + gicv5_cpu_enable_interrupts(); + gicv5_configure_spi(READ_ONCE(spi_line_expected), + READ_ONCE(spi_line_level_sensitive)); + gicv5_enable_spi(READ_ONCE(spi_line_expected), 0); + + local_irq_enable(); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + while (1) + wfi(); +} + /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) { @@ -112,6 +221,17 @@ static void vm_gic_destroy(struct vm_gic *v) kvm_vm_free(v->vm); } +static void vgic_v5_map_irs(struct kvm_vm *vm) +{ + unsigned int nr_irs_pages; + + nr_irs_pages = vm_calc_num_guest_pages(vm->mode, VGIC_V5_IRS_SIZE); + + /* Map the IRS at VA == IPA so guest MMIO writes hit the IRS IODEV. */ + virt_map(vm, GICV5_IRS_CONFIG_BASE_GPA, + GICV5_IRS_CONFIG_BASE_GPA, nr_irs_pages); +} + static u32 vgic_v5_irq_line_payload(u32 type, u32 num) { return (type << KVM_ARM_IRQ_TYPE_SHIFT) | @@ -123,6 +243,93 @@ static int __vgic_v5_irq_line(struct kvm_vm *vm, u32 type, u32 num, int level) return _kvm_irq_line(vm, vgic_v5_irq_line_payload(type, num), level); } +static void vgic_v5_spi_line(struct kvm_vm *vm, u32 spi, int level) +{ + int ret = __vgic_v5_irq_line(vm, KVM_ARM_IRQ_TYPE_SPI, spi, level); + + TEST_ASSERT(!ret, "KVM_IRQ_LINE failed for SPI %u level %d", + spi, level); +} + +static void vgic_v5_spi_line_vm_create(struct vm_gic *v, + struct kvm_vcpu **vcpu, + u32 nr_spis) +{ + u64 attr; + + v->gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v->vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v->gic_fd = kvm_create_device(v->vm, v->gic_dev_type); + *vcpu = vm_vcpu_add(v->vm, 0, guest_spi_line_code); + TEST_ASSERT(*vcpu, "Failed to create vCPU"); + + vm_init_descriptor_tables(v->vm); + vm_install_exception_handler(v->vm, VECTOR_IRQ_CURRENT, + guest_spi_irq_handler); + vcpu_init_descriptor_tables(*vcpu); + + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0, + &nr_spis); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v->vm); + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); +} + +static void vgic_v5_run_spi_line_test(u32 nr_spis, u32 expected_spi, + bool level_sensitive, + bool lower_before_run) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + struct ucall uc; + int ret; + + spi_line_expected = expected_spi; + spi_line_level_sensitive = level_sensitive; + + vgic_v5_spi_line_vm_create(&v, &vcpu, nr_spis); + + sync_global_to_guest(v.vm, spi_line_expected); + sync_global_to_guest(v.vm, spi_line_level_sensitive); + + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 IRQ_LINE VM"); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 IRQ_LINE guest did not become ready"); + + vgic_v5_spi_line(v.vm, expected_spi, 1); + /* + * For edge-triggered SPIs, a following low transition must be ignored: + * once the edge has made the SPI pending, it cannot be recalled. This + * allows us to test that. + */ + if (lower_before_run) + vgic_v5_spi_line(v.vm, expected_spi, 0); + + while (1) { + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 IRQ_LINE VM"); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_DONE: + goto done; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } + +done: + vm_gic_destroy(&v); +} + struct vgic_region_attr { u64 attr; u64 size; @@ -860,7 +1067,8 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) vcpus[i] = vm_vcpu_add(v.vm, i, guest_code); vm_init_descriptor_tables(v.vm); - vm_install_exception_handler(v.vm, VECTOR_IRQ_CURRENT, guest_irq_handler); + vm_install_exception_handler(v.vm, VECTOR_IRQ_CURRENT, + guest_ppi_irq_handler); for (i = 0; i < NR_VCPUS; i++) vcpu_init_descriptor_tables(vcpus[i]); @@ -916,9 +1124,9 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) kvm_irq_line(v.vm, irq, level); } else if (uc.args[1] == GUEST_CMD_IS_AWAKE) { - pr_info("Guest skipping WFI due to pending IRQ\n"); + pr_debug("Guest skipping WFI due to pending IRQ\n"); } else if (uc.args[1] == GUEST_CMD_IRQ_CDIA) { - pr_info("Guest acknowledged IRQ\n"); + pr_debug("Guest acknowledged IRQ\n"); } continue; @@ -938,6 +1146,44 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) vm_gic_destroy(&v); } +static void test_vgic_v5_spis(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + int ret; + + /* Default NR_IRQS exposes 32 SPIs, numbered 0..31 in KVM_IRQ_LINE. */ + vgic_v5_spi_line_vm_create(&v, &vcpu, VGIC_V5_DEFAULT_NR_SPIS); + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_SPI, + VGIC_V5_DEFAULT_NR_SPIS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted first invalid SPI"); + + vm_gic_destroy(&v); + + /* Basic SPI injection through KVM_IRQ_LINE. */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 0, false, false); + + /* The last valid SPI in the maximum configured range is injectable. */ + vgic_v5_run_spi_line_test(VGIC_V5_MAX_NR_SPIS, + VGIC_V5_MAX_NR_SPIS - 1, false, false); + + vgic_v5_spi_line_vm_create(&v, &vcpu, VGIC_V5_MAX_NR_SPIS); + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_SPI, + VGIC_V5_MAX_NR_SPIS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted configured invalid SPI"); + vm_gic_destroy(&v); + + /* + * Edge SPIs remain pending after the line is lowered. The low + * transition is injected before the guest runs and must be ignored: + * once an edge interrupt is pending, lowering the line cannot recall it. + */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 1, false, true); + + /* Level SPIs can be raised and delivered through KVM_IRQ_LINE. */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 2, true, false); +} + /* * Returns 0 if it's possible to create GIC device of a given type (V5). */ @@ -987,9 +1233,11 @@ void run_tests(u32 gic_dev_type) pr_info("Test VGICv5 CPU sysreg attrs\n"); test_vgic_v5_cpu_sysreg_attrs(); - pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); + + pr_info("Test VGICv5 SPIs\n"); + test_vgic_v5_spis(); } int main(int ac, char **av) diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v5.h b/tools/testing/selftests/kvm/include/arm64/gic_v5.h index 0ad6dcc3d510..091d9dbd3c01 100644 --- a/tools/testing/selftests/kvm/include/arm64/gic_v5.h +++ b/tools/testing/selftests/kvm/include/arm64/gic_v5.h @@ -12,6 +12,7 @@ /* GIC component base address is guest PA space */ #define GICV5_IRS_CONFIG_BASE_GPA 0x8000000ULL +#define GICV5_IRS_CONFIG_BASE_GVA ((volatile void *)GICV5_IRS_CONFIG_BASE_GPA) #define GICV5_IRS_SIZE 0x20000 #define GICV5_IRS_ALIGN 0x10000 @@ -79,6 +80,12 @@ #define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) +#define GICV5_IRS_CR0_IDLE BIT(1) +#define GICV5_IRS_CR0_IRSEN BIT(0) + +#define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_CFGR_TM BIT(0) + #define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) #define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) #define GICV5_IRS_PE_STATUSR_IDLE BIT(0) @@ -155,6 +162,10 @@ #define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29) #define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE GENMASK_ULL(31, 29) +#define GICV5_HWIRQ_ID GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE_SPI 0x3 + #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) #define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn) From fc8ab69c081b99feddf3c873846068013b3a8a78 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:58:10 +0000 Subject: [PATCH 161/506] KVM: selftests: Add VGICv5 LPI delivery tests Add a two-VPE VGICv5 LPI delivery test. LPIs depend on an IST whose base is provided by the guest through IRS_IST_BASER, which causes KVM to allocate and provide an LPI IST to the hardware. Exercise the virtual IRS ID registers and IST base configuration path before checking interrupt delivery. Back the guest's LPI IST with a dedicated userspace memslot and identity-map it into the guest at the same IPA and VA. This lets the guest provide that address to the emulated IRS through IRS_IST_BASER while the host test still allocates and owns the backing memory. Have VPE0 enable the IRS, derive a valid linear LPI IST configuration from the virtual IRS ID registers, provide the IST base through IRS_IST_BASER, configure one LPI for each VPE, and send an LPI to VPE1. While the BASER is valid, attempt to change its address and verify that the write is ignored. VPE1 consumes the first LPI and replies with a second, which VPE0 consumes before completing the test. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-48-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 247 +++++++++++++++++- .../selftests/kvm/include/arm64/gic_v5.h | 16 ++ 2 files changed, 261 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index ea3451d850c2..c4c932f4c1d4 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -13,11 +13,17 @@ #include "vgic.h" #define NR_VCPUS 1 +#define VGIC_V5_LPI_NR_VCPUS 2 +#define VGIC_V5_LPI_MEMSLOT 1 #define VGIC_V5_NR_PRIVATE_IRQS 64 #define VGIC_V5_DEFAULT_NR_SPIS 32 #define VGIC_V5_MAX_NR_SPIS BIT(10) #define VGIC_V5_IRS_SIZE 0x20000 #define VGIC_V5_IRS_WAIT_RETRIES 1000000 +#define VGIC_V5_LPI_IST_BASE_GPA (GICV5_IRS_CONFIG_BASE_GPA + VGIC_V5_IRS_SIZE) +#define VGIC_V5_LPI_IST_SIZE 0x10000 +#define LPI_TEST_TO_VPE1 0 +#define LPI_TEST_TO_VPE0 1 static u64 max_phys_size; @@ -33,6 +39,8 @@ struct vm_gic { #define GUEST_CMD_IRQ_DIEOI 11 #define GUEST_CMD_IS_AWAKE 12 #define GUEST_CMD_IS_READY 13 +#define GUEST_CMD_LPI_SENT 14 +#define GUEST_CMD_LPI_REPLIED 15 static struct kvm_vgic_v5_ist vgic_v5_ist_attr(void *spi_ist, size_t spi_size, void *lpi_ist, size_t lpi_size) @@ -47,11 +55,22 @@ static struct kvm_vgic_v5_ist vgic_v5_ist_attr(void *spi_ist, size_t spi_size, static u32 spi_line_expected; static bool spi_line_level_sensitive; +static bool lpi_ist_ready; + +static u64 gicv5_hwirq(u32 type, u32 intid) +{ + return FIELD_PREP(GICV5_HWIRQ_TYPE, type) | + FIELD_PREP(GICV5_HWIRQ_ID, intid); +} + +static u64 gicv5_lpi_hwirq(u32 lpi) +{ + return gicv5_hwirq(GICV5_HWIRQ_TYPE_LPI, lpi); +} static u64 gicv5_spi_hwirq(u32 spi) { - return FIELD_PREP(GICV5_HWIRQ_TYPE, GICV5_HWIRQ_TYPE_SPI) | - FIELD_PREP(GICV5_HWIRQ_ID, spi); + return gicv5_hwirq(GICV5_HWIRQ_TYPE_SPI, spi); } static void gicv5_setup_and_enable_hwirq(u64 hwirq, u32 target_vpe) @@ -73,6 +92,18 @@ static void gicv5_enable_spi(u32 spi, u32 target_vpe) gicv5_setup_and_enable_hwirq(gicv5_spi_hwirq(spi), target_vpe); } +static void gicv5_enable_lpi(u32 lpi, u32 target_vpe) +{ + gicv5_setup_and_enable_hwirq(gicv5_lpi_hwirq(lpi), target_vpe); +} + +static void gicv5_send_lpi(u32 lpi) +{ + u64 hwirq = gicv5_lpi_hwirq(lpi); + + gic_insn(hwirq | GICV5_GIC_CDPEND_PENDING_MASK, CDPEND); +} + static void gicv5_wait_for_irs_idle(u32 reg, u32 idle) { int i; @@ -102,6 +133,58 @@ static void gicv5_configure_spi(u32 spi, bool level) GICV5_IRS_SPI_STATUSR_IDLE); } +static u32 gicv5_lpi_istsz(u32 idr2, u32 lpi_id_bits) +{ + if (!(idr2 & GICV5_IRS_IDR2_ISTMD)) + return GICV5_IRS_IST_CFGR_ISTSZ_4; + + if (lpi_id_bits >= FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2)) + return GICV5_IRS_IST_CFGR_ISTSZ_16; + + return GICV5_IRS_IST_CFGR_ISTSZ_8; +} + +static void gicv5_configure_lpi_ist(void) +{ + u32 idr2, min_lpi_id_bits, lpi_id_bits, id_bits, istsz; + u64 val; + + idr2 = readl(GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IDR2); + GUEST_ASSERT(idr2 & GICV5_IRS_IDR2_LPI); + + min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2); + id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2); + /* Default to 32 LPIs, unless the min requirement is higher */ + lpi_id_bits = max(5U, min_lpi_id_bits); + GUEST_ASSERT(lpi_id_bits <= id_bits); + + istsz = gicv5_lpi_istsz(idr2, lpi_id_bits); + /* + * We allocate a fixed-size IST buffer on the host to avoid dynamic + * allocation from the guest. Make sure it fits the linear IST described + * by IDR2 before programming IRS_IST_BASER. + */ + GUEST_ASSERT((BIT_ULL(lpi_id_bits) << (istsz + 2)) <= VGIC_V5_LPI_IST_SIZE); + + val = FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits) | + FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz); + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_CFGR); + + val = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + VGIC_V5_LPI_IST_BASE_GPA >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + val |= GICV5_IRS_IST_BASER_VALID; + writeq(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_BASER); + + GUEST_ASSERT_EQ(readl(GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_STATUSR), + GICV5_IRS_IST_STATUSR_IDLE); + + /* The address is immutable while the BASER is valid. */ + writeq(val ^ FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1), + GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_BASER); + GUEST_ASSERT_EQ(readq(GICV5_IRS_CONFIG_BASE_GVA + + GICV5_IRS_IST_BASER), val); +} + static void gicv5_enable_irs(void) { writel(GICV5_IRS_CR0_IRSEN, @@ -109,6 +192,12 @@ static void gicv5_enable_irs(void) gicv5_wait_for_irs_idle(GICV5_IRS_CR0, GICV5_IRS_CR0_IDLE); } +static void gicv5_configure_test_lpis(void) +{ + gicv5_enable_lpi(LPI_TEST_TO_VPE1, 1); + gicv5_enable_lpi(LPI_TEST_TO_VPE0, 0); +} + static void guest_ppi_irq_handler(struct ex_regs *regs) { bool valid; @@ -209,6 +298,67 @@ static void guest_spi_line_code(void) wfi(); } +static void guest_lpi_irq_handler(struct ex_regs *regs) +{ + u32 vcpu_id = guest_get_vcpuid(); + u32 expected_lpi = vcpu_id ? LPI_TEST_TO_VPE1 : LPI_TEST_TO_VPE0; + u64 expected_hwirq = gicv5_lpi_hwirq(expected_lpi); + u32 hwirq; + u64 ia; + + ia = gicr_insn(CDIA); + if (!GICV5_GICR_CDIA_VALID(ia)) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + GUEST_ASSERT_EQ(hwirq, expected_hwirq); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + + if (vcpu_id) { + gicv5_send_lpi(LPI_TEST_TO_VPE0); + GUEST_SYNC(GUEST_CMD_LPI_REPLIED); + while (1) + wfi(); + } + + GUEST_DONE(); +} + +static void guest_lpi_code(void) +{ + u32 vcpu_id = guest_get_vcpuid(); + + local_irq_disable(); + + if (!vcpu_id) { + gicv5_enable_irs(); + gicv5_configure_lpi_ist(); + gicv5_configure_test_lpis(); + WRITE_ONCE(lpi_ist_ready, true); + } + + /* Go bang if we run VCPU1 before VCPU0 */ + GUEST_ASSERT(READ_ONCE(lpi_ist_ready)); + + gicv5_cpu_enable_interrupts(); + local_irq_enable(); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + if (!vcpu_id) { + gicv5_send_lpi(LPI_TEST_TO_VPE1); + GUEST_SYNC(GUEST_CMD_LPI_SENT); + } + + while (1) + wfi(); +} + /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) { @@ -279,6 +429,48 @@ static void vgic_v5_spi_line_vm_create(struct vm_gic *v, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); } +static void vgic_v5_lpi_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]) +{ + unsigned int nr_lpi_ist_pages; + u64 attr; + int i; + + v->gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v->vm = __vm_create(VM_SHAPE_DEFAULT, VGIC_V5_LPI_NR_VCPUS, 0); + v->gic_fd = kvm_create_device(v->vm, v->gic_dev_type); + + for (i = 0; i < VGIC_V5_LPI_NR_VCPUS; i++) { + vcpus[i] = vm_vcpu_add(v->vm, i, guest_lpi_code); + TEST_ASSERT(vcpus[i], "Failed to create vCPU"); + } + + vm_init_descriptor_tables(v->vm); + vm_install_exception_handler(v->vm, VECTOR_IRQ_CURRENT, + guest_lpi_irq_handler); + + for (i = 0; i < VGIC_V5_LPI_NR_VCPUS; i++) + vcpu_init_descriptor_tables(vcpus[i]); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v->vm); + + nr_lpi_ist_pages = vm_calc_num_guest_pages(v->vm->mode, + VGIC_V5_LPI_IST_SIZE); + vm_userspace_mem_region_add(v->vm, VM_MEM_SRC_ANONYMOUS, + VGIC_V5_LPI_IST_BASE_GPA, + VGIC_V5_LPI_MEMSLOT, + nr_lpi_ist_pages, 0); + /* Map the IST at VA == IPA so the guest can program the same BASER. */ + virt_map(v->vm, VGIC_V5_LPI_IST_BASE_GPA, + VGIC_V5_LPI_IST_BASE_GPA, nr_lpi_ist_pages); + + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); +} + static void vgic_v5_run_spi_line_test(u32 nr_spis, u32 expected_spi, bool level_sensitive, bool lower_before_run) @@ -1184,6 +1376,54 @@ static void test_vgic_v5_spis(void) vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 2, true, false); } +static void test_vgic_v5_lpis(void) +{ + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]; + struct ucall uc; + struct vm_gic v; + int ret; + + lpi_ist_ready = false; + vgic_v5_lpi_vm_create(&v, vcpus); + sync_global_to_guest(v.vm, lpi_ist_ready); + + /* VPE0 programs a linear LPI IST from the virtual IRS ID registers. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 LPI vCPU0 did not become ready"); + + /* VPE1 waits for the shared LPI setup and enables its CPU interface. */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU1"); + TEST_ASSERT(get_ucall(vcpus[1], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 LPI vCPU1 did not become ready"); + + /* VPE0 sends an LPI to VPE1. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0 sender"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_LPI_SENT, + "GICv5 LPI vCPU0 did not send LPI"); + + /* VPE1 consumes the LPI and replies with another LPI. */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU1 receiver"); + TEST_ASSERT(get_ucall(vcpus[1], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_LPI_REPLIED, + "GICv5 LPI vCPU1 did not reply"); + + /* VPE0 consumes the reply LPI and ends the test. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0 receiver"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_DONE, + "GICv5 LPI vCPU0 did not complete"); + + vm_gic_destroy(&v); +} + /* * Returns 0 if it's possible to create GIC device of a given type (V5). */ @@ -1238,6 +1478,9 @@ void run_tests(u32 gic_dev_type) pr_info("Test VGICv5 SPIs\n"); test_vgic_v5_spis(); + + pr_info("Test VGICv5 LPIs\n"); + test_vgic_v5_lpis(); } int main(int ac, char **av) diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v5.h b/tools/testing/selftests/kvm/include/arm64/gic_v5.h index 091d9dbd3c01..2d455515563e 100644 --- a/tools/testing/selftests/kvm/include/arm64/gic_v5.h +++ b/tools/testing/selftests/kvm/include/arm64/gic_v5.h @@ -80,6 +80,14 @@ #define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) +#define GICV5_IRS_IDR2_ISTMD_SZ GENMASK(19, 15) +#define GICV5_IRS_IDR2_ISTMD BIT(14) +#define GICV5_IRS_IDR2_IST_L2SZ GENMASK(13, 11) +#define GICV5_IRS_IDR2_IST_LEVELS BIT(10) +#define GICV5_IRS_IDR2_MIN_LPI_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR2_LPI BIT(5) +#define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) + #define GICV5_IRS_CR0_IDLE BIT(1) #define GICV5_IRS_CR0_IRSEN BIT(0) @@ -94,6 +102,13 @@ #define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) #define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) #define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL +#define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) +#define GICV5_IRS_IST_CFGR_L2SZ GENMASK(6, 5) +#define GICV5_IRS_IST_CFGR_LPI_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_CFGR_ISTSZ_4 0b00 +#define GICV5_IRS_IST_CFGR_ISTSZ_8 0b01 +#define GICV5_IRS_IST_CFGR_ISTSZ_16 0b10 /* * Definitions for GICv5 instructions for the Current Domain @@ -164,6 +179,7 @@ #define GICV5_HWIRQ_TYPE GENMASK_ULL(31, 29) #define GICV5_HWIRQ_ID GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE_LPI 0x2 #define GICV5_HWIRQ_TYPE_SPI 0x3 #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) From 7bebd949670bb5ab1bd14a67779fd308795e247b Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:58:41 +0000 Subject: [PATCH 162/506] KVM: selftests: Add VGICv5 IST save/restore coverage Add end-to-end VGICv5 IST save/restore coverage. The earlier IST attribute test checks only API rejection cases; this test verifies that interrupt state saved from the ISTs can be restored into a fresh VM before first run and then consumed by the guest. Build a source VM that configures SPI and LPI enable, priority, and affinity state from the guest. Inject selected SPIs from userspace, save the IRS register state, and save both the SPI and LPI IST images through KVM_DEV_ARM_VGIC_GRP_IST using userspace-provided buffers. Create a restore VM and verify that IST restore is rejected before the IRS state has been restored. Restore the IRS registers, validate bad or missing LPI storage, and restore the saved SPI and LPI IST images. Before the host LPI IST is allocated, verify that userspace can change or invalidate IRS_IST_BASER. Once the IST images have been restored and the host LPI IST exists, verify that matching IRS_IST_BASER and IRS_IST_CFGR values can be replayed, while changing either register or clearing IRS_IST_BASER.Valid is rejected. Before running any destination vCPU, save the ISTs again and compare the re-saved buffers against the original saved buffers so that dropped serialised state fails independently of interrupt delivery. Finally, run the restored guest and check pending and enabled SPI and LPI state across two VPEs, priority ordering through ICC_PCR_EL1, delivery of an interrupt that was pending but disabled at save time, and affinity for interrupts injected after restore. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-49-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 506 +++++++++++++++++++- 1 file changed, 501 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index c4c932f4c1d4..6caac6cf7c62 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -24,6 +24,14 @@ #define VGIC_V5_LPI_IST_SIZE 0x10000 #define LPI_TEST_TO_VPE1 0 #define LPI_TEST_TO_VPE0 1 +#define RESTORE_TEST_LPI_PENDING 0 +#define RESTORE_TEST_LPI_ENABLED 1 +#define RESTORE_TEST_SPI_PRIO_HIGH 0 +#define RESTORE_TEST_SPI_PRIO_LOW 1 +#define RESTORE_TEST_SPI_PENDING_DISABLED 2 +#define RESTORE_TEST_SPI_ENABLED 3 +#define VGIC_V5_IST_PRIO_HIGH 0x08 +#define VGIC_V5_IST_PRIO_LOW 0x10 static u64 max_phys_size; @@ -41,6 +49,9 @@ struct vm_gic { #define GUEST_CMD_IS_READY 13 #define GUEST_CMD_LPI_SENT 14 #define GUEST_CMD_LPI_REPLIED 15 +#define GUEST_CMD_IST_SOURCE_READY 16 +#define GUEST_CMD_IST_SPI_INJECT_READY 17 +#define GUEST_CMD_IST_LPI_INJECT_READY 18 static struct kvm_vgic_v5_ist vgic_v5_ist_attr(void *spi_ist, size_t spi_size, void *lpi_ist, size_t lpi_size) @@ -56,6 +67,7 @@ static struct kvm_vgic_v5_ist vgic_v5_ist_attr(void *spi_ist, size_t spi_size, static u32 spi_line_expected; static bool spi_line_level_sensitive; static bool lpi_ist_ready; +static u32 ist_restore_irq_count[VGIC_V5_LPI_NR_VCPUS]; static u64 gicv5_hwirq(u32 type, u32 intid) { @@ -87,6 +99,27 @@ static void gicv5_setup_and_enable_hwirq(u64 hwirq, u32 target_vpe) gic_insn(hwirq, CDEN); } +static void gicv5_set_hwirq_priority(u64 hwirq, u32 priority) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority); + gic_insn(val, CDPRI); +} + +static void gicv5_set_hwirq_affinity(u64 hwirq, u32 target_vpe) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, target_vpe); + gic_insn(val, CDAFF); +} + +static void gicv5_enable_hwirq(u64 hwirq) +{ + gic_insn(hwirq, CDEN); +} + static void gicv5_enable_spi(u32 spi, u32 target_vpe) { gicv5_setup_and_enable_hwirq(gicv5_spi_hwirq(spi), target_vpe); @@ -198,6 +231,15 @@ static void gicv5_configure_test_lpis(void) gicv5_enable_lpi(LPI_TEST_TO_VPE0, 0); } +static void gicv5_set_pcr(u32 priority) +{ + u64 pcr; + + pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, priority); + write_sysreg_s(pcr, SYS_ICC_PCR_EL1); + isb(); +} + static void guest_ppi_irq_handler(struct ex_regs *regs) { bool valid; @@ -329,6 +371,11 @@ static void guest_lpi_irq_handler(struct ex_regs *regs) GUEST_DONE(); } +static void guest_unexpected_irq_handler(struct ex_regs *regs) +{ + GUEST_FAIL("Unexpected IRQ"); +} + static void guest_lpi_code(void) { u32 vcpu_id = guest_get_vcpuid(); @@ -359,6 +406,165 @@ static void guest_lpi_code(void) wfi(); } +static void restore_test_setup_hwirq(u64 hwirq, u32 target_vpe, + u32 priority, bool enable) +{ + gicv5_set_hwirq_priority(hwirq, priority); + gicv5_set_hwirq_affinity(hwirq, target_vpe); + + if (enable) + gicv5_enable_hwirq(hwirq); +} + +/* + * The source VM configures SPI and LPI state for the IST save/restore test. + * It leaves interrupts disabled, allowing userspace to save the state before + * the restore VM enables interrupts and checks delivery. + */ +static void guest_ist_source_code(void) +{ + local_irq_disable(); + + gicv5_enable_irs(); + gicv5_configure_lpi_ist(); + + restore_test_setup_hwirq(gicv5_lpi_hwirq(RESTORE_TEST_LPI_PENDING), + 1, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_lpi_hwirq(RESTORE_TEST_LPI_ENABLED), + 1, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_HIGH), + 0, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_LOW), + 0, VGIC_V5_IST_PRIO_LOW, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_ENABLED), + 0, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED), + 0, VGIC_V5_IST_PRIO_HIGH, false); + + gicv5_send_lpi(RESTORE_TEST_LPI_PENDING); + + GUEST_SYNC(GUEST_CMD_IST_SOURCE_READY); + + while (1) + wfi(); +} + +/* + * This is the restore VM for the IST save/restore test. The IRS regs and ISTs + * have been restored before this code runs. Based on which VPE is running, a + * different set of interrupts is expected. We check that we get what we + * expected before terminating the test. + * + * The expected interrupts for VPE0 are: + * - High-priority SPI, pending and enabled before save. + * - Low-priority SPI, pending and enabled before save. This is only delivered + * after the guest lowers the masked priority via ICC_PCR_EL1. + * - SPI that was pending but disabled before save. This is only delivered + * after the guest enables it after restore. + * - SPI that was enabled but not pending before save. Userspace injects this + * after restore to check that the enable state and affinity were restored. + * + * The expected interrupts for VPE1 are: + * - LPI that was pending and enabled before save. + * - LPI that was enabled but not pending before save. The guest injects this + * after restore to check that the enable state and affinity were restored. + */ +static void guest_ist_restore_irq_handler(struct ex_regs *regs) +{ + u32 vcpu_id = guest_get_vcpuid(); + u32 count = ist_restore_irq_count[vcpu_id]; + u64 hwirq, expected_hwirq = 0; + u64 ia; + + ia = gicr_insn(CDIA); + if (!GICV5_GICR_CDIA_VALID(ia)) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + + if (!vcpu_id) { + switch (count) { + case 0: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_HIGH); + break; + case 1: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_LOW); + break; + case 2: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED); + break; + case 3: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_ENABLED); + break; + default: + GUEST_FAIL("Unexpected VPE0 IST restore interrupt"); + } + } else { + switch (count) { + case 0: + expected_hwirq = gicv5_lpi_hwirq(RESTORE_TEST_LPI_PENDING); + break; + case 1: + expected_hwirq = gicv5_lpi_hwirq(RESTORE_TEST_LPI_ENABLED); + break; + default: + GUEST_FAIL("Unexpected VPE1 IST restore interrupt"); + } + } + + GUEST_ASSERT_EQ(hwirq, expected_hwirq); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + ist_restore_irq_count[vcpu_id]++; + + if (!vcpu_id) { + switch (count) { + case 0: + gicv5_set_pcr(VGIC_V5_IST_PRIO_LOW); + break; + case 1: + gicv5_enable_hwirq( + gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED)); + break; + case 2: + GUEST_SYNC(GUEST_CMD_IST_SPI_INJECT_READY); + break; + case 3: + GUEST_DONE(); + } + } else { + switch (count) { + case 0: + gicv5_send_lpi(RESTORE_TEST_LPI_ENABLED); + GUEST_SYNC(GUEST_CMD_IST_LPI_INJECT_READY); + break; + case 1: + GUEST_DONE(); + } + } +} + +static void guest_ist_restore_code(void) +{ + local_irq_disable(); + + gicv5_cpu_enable_interrupts(); + + if (!guest_get_vcpuid()) + gicv5_set_pcr(VGIC_V5_IST_PRIO_HIGH); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + local_irq_enable(); + + while (1) + wfi(); +} + /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) { @@ -429,8 +635,10 @@ static void vgic_v5_spi_line_vm_create(struct vm_gic *v, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); } -static void vgic_v5_lpi_vm_create(struct vm_gic *v, - struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]) +static void vgic_v5_lpi_ist_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS], + void (*guest_code)(void), + void (*irq_handler)(struct ex_regs *)) { unsigned int nr_lpi_ist_pages; u64 attr; @@ -441,13 +649,13 @@ static void vgic_v5_lpi_vm_create(struct vm_gic *v, v->gic_fd = kvm_create_device(v->vm, v->gic_dev_type); for (i = 0; i < VGIC_V5_LPI_NR_VCPUS; i++) { - vcpus[i] = vm_vcpu_add(v->vm, i, guest_lpi_code); + vcpus[i] = vm_vcpu_add(v->vm, i, guest_code); TEST_ASSERT(vcpus[i], "Failed to create vCPU"); } vm_init_descriptor_tables(v->vm); vm_install_exception_handler(v->vm, VECTOR_IRQ_CURRENT, - guest_lpi_irq_handler); + irq_handler); for (i = 0; i < VGIC_V5_LPI_NR_VCPUS; i++) vcpu_init_descriptor_tables(vcpus[i]); @@ -463,7 +671,10 @@ static void vgic_v5_lpi_vm_create(struct vm_gic *v, VGIC_V5_LPI_IST_BASE_GPA, VGIC_V5_LPI_MEMSLOT, nr_lpi_ist_pages, 0); - /* Map the IST at VA == IPA so the guest can program the same BASER. */ + /* + * Map the IST at VA == IPA so the guest can program the same BASER + * address. + */ virt_map(v->vm, VGIC_V5_LPI_IST_BASE_GPA, VGIC_V5_LPI_IST_BASE_GPA, nr_lpi_ist_pages); @@ -471,6 +682,21 @@ static void vgic_v5_lpi_vm_create(struct vm_gic *v, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); } +static void vgic_v5_lpi_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]) +{ + vgic_v5_lpi_ist_vm_create(v, vcpus, guest_lpi_code, + guest_lpi_irq_handler); +} + +static void vgic_v5_ist_restore_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS], + void (*guest_code)(void), + void (*irq_handler)(struct ex_regs *)) +{ + vgic_v5_lpi_ist_vm_create(v, vcpus, guest_code, irq_handler); +} + static void vgic_v5_run_spi_line_test(u32 nr_spis, u32 expected_spi, bool level_sensitive, bool lower_before_run) @@ -522,6 +748,15 @@ static void vgic_v5_run_spi_line_test(u32 nr_spis, u32 expected_spi, vm_gic_destroy(&v); } +static void vgic_v5_expect_sync(struct kvm_vcpu *vcpu, u64 cmd, + const char *msg) +{ + struct ucall uc; + + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && uc.args[1] == cmd, + "%s", msg); +} + struct vgic_region_attr { u64 attr; u64 size; @@ -647,6 +882,52 @@ static const struct vgic_sysreg_attr gic_v5_cpu_sysregs[] = { SR(SYS_ICC_PCR_EL1), }; +static void vgic_v5_save_irs_regs(struct vm_gic *v, u64 *regs) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + ret = __kvm_device_attr_get(v->gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + gic_v5_irs_regs[i].attr, + ®s[i]); + TEST_ASSERT(!ret, "GICv5 IRS_REGS save failed for %s", + gic_v5_irs_regs[i].name); + } +} + +static void vgic_v5_restore_irs_regs(struct vm_gic *v, u64 *regs) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + ret = __kvm_device_attr_set(v->gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + gic_v5_irs_regs[i].attr, + ®s[i]); + TEST_ASSERT(!ret, "GICv5 IRS_REGS restore failed for %s", + gic_v5_irs_regs[i].name); + } +} + +static u64 vgic_v5_saved_irs_reg(u64 *regs, u64 attr) +{ + for (int i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + if (gic_v5_irs_regs[i].attr == attr) + return regs[i]; + } + + TEST_FAIL("GICv5 IRS register 0x%lx was not saved", attr); +} + +static size_t vgic_v5_lpi_ist_state_size(u64 *irs_regs) +{ + u64 cfgr = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_CFGR); + u32 lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, cfgr); + + return BIT(lpi_id_bits) * sizeof(__u32); +} + static void test_vgic_v5_addr_attrs(void) { struct kvm_vcpu *vcpu; @@ -1424,6 +1705,218 @@ static void test_vgic_v5_lpis(void) vm_gic_destroy(&v); } +static void test_vgic_v5_ist_save_restore(void) +{ + static u32 lpi_ist[VGIC_V5_LPI_IST_SIZE / sizeof(u32)]; + static u32 lpi_ist_resave[VGIC_V5_LPI_IST_SIZE / sizeof(u32)]; + static u32 spi_ist[VGIC_V5_DEFAULT_NR_SPIS]; + static u32 spi_ist_resave[VGIC_V5_DEFAULT_NR_SPIS]; + static u64 irs_regs[ARRAY_SIZE(gic_v5_irs_regs)]; + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]; + struct kvm_vgic_v5_ist ist_attr; + struct vm_gic src, dst; + u64 ist_baser, bad_ist_baser; + u64 ist_cfgr, bad_ist_cfgr; + size_t lpi_ist_size; + int ret; + + memset(ist_restore_irq_count, 0, sizeof(ist_restore_irq_count)); + + vgic_v5_ist_restore_vm_create(&src, vcpus, guest_ist_source_code, + guest_unexpected_irq_handler); + + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST save source VM"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IST_SOURCE_READY, + "GICv5 IST source guest did not become ready"); + + /* + * The guest configured enable/priority/affinity state. Make selected + * SPIs pending from userspace so the SPI IST save path sees them. + */ + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PRIO_HIGH, 1); + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PRIO_LOW, 1); + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PENDING_DISABLED, 1); + + vgic_v5_save_irs_regs(&src, irs_regs); + lpi_ist_size = vgic_v5_lpi_ist_state_size(irs_regs); + + /* + * VCPU0 has configured the initial state. KVM must reject missing or + * incorrectly sized LPI storage before accepting the complete IST image. + */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST save accepted missing LPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size - sizeof(__u32)); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST save accepted bad LPI size"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST save failed"); + + vm_gic_destroy(&src); + + /* Create the guest that we are restoring state into */ + memset(ist_restore_irq_count, 0, sizeof(ist_restore_irq_count)); + vgic_v5_ist_restore_vm_create(&dst, vcpus, guest_ist_restore_code, + guest_ist_restore_irq_handler); + sync_global_to_guest(dst.vm, ist_restore_irq_count); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore before IRS state"); + + vgic_v5_restore_irs_regs(&dst, irs_regs); + + /* BASER remains mutable until the host LPI IST has been allocated. */ + ist_baser = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_BASER); + bad_ist_baser = ist_baser ^ + FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected changed unallocated IST_BASER"); + + bad_ist_baser = ist_baser & ~GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected invalidated unallocated IST_BASER"); + + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &ist_baser); + TEST_ASSERT(!ret, "GICv5 failed to restore IST_BASER"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore accepted missing LPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size - sizeof(__u32)); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore accepted bad LPI size"); + + /* Restore the VMM-owned SPI and LPI IST images. */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST restore failed"); + + /* A live IST makes BASER and the guest-visible configuration immutable. */ + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected unchanged live IST_BASER"); + + bad_ist_baser = ist_baser ^ + FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted changed live IST_BASER"); + + bad_ist_baser = ist_baser & ~GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted invalidated live IST_BASER"); + + ist_cfgr = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_CFGR); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_CFGR, &ist_cfgr); + TEST_ASSERT(!ret, "GICv5 rejected unchanged live IST_CFGR"); + + bad_ist_cfgr = ist_cfgr ^ + FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_CFGR, &bad_ist_cfgr); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted changed live IST_CFGR"); + + /* + * Re-save the restored IST image before the destination VM runs, so a + * dropped serialized field fails independently of later interrupt + * delivery. + */ + ist_attr = vgic_v5_ist_attr(spi_ist_resave, sizeof(spi_ist_resave), + lpi_ist_resave, lpi_ist_size); + ret = __kvm_device_attr_get(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST re-save failed"); + TEST_ASSERT(!memcmp(spi_ist, spi_ist_resave, sizeof(spi_ist)), + "GICv5 SPI IST state changed across restore"); + TEST_ASSERT(!memcmp(lpi_ist, lpi_ist_resave, lpi_ist_size), + "GICv5 LPI IST state changed across restore"); + + /* + * At this stage, the guest's IST state should match what was saved. Run + * each vcpu once to let it enable interrupts, etc. + */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST restore vCPU0"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IS_READY, + "GICv5 IST restore vCPU0 did not become ready"); + + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST restore vCPU1"); + vgic_v5_expect_sync(vcpus[1], GUEST_CMD_IS_READY, + "GICv5 IST restore vCPU1 did not become ready"); + + /* + * VPE0 consumes two pending SPIs in restored priority order, then + * enables a restored pending-but-disabled SPI. + */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to consume restored GICv5 SPIs"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IST_SPI_INJECT_READY, + "GICv5 IST restore vCPU0 did not consume restored SPIs"); + + /* + * This SPI was enabled before save, but not pending. Inject it after + * restore to prove enable, priority and affinity were restored. + */ + vgic_v5_spi_line(dst.vm, RESTORE_TEST_SPI_ENABLED, 1); + + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to consume post-restore GICv5 SPI"); + TEST_ASSERT(get_ucall(vcpus[0], NULL) == UCALL_DONE, + "GICv5 IST restore vCPU0 did not complete"); + + /* + * VPE1 consumes a restored pending LPI, then pends another LPI that + * was enabled before save but not pending. + */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to consume restored GICv5 LPI"); + vgic_v5_expect_sync(vcpus[1], GUEST_CMD_IST_LPI_INJECT_READY, + "GICv5 IST restore vCPU1 did not consume restored LPI"); + + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to consume post-restore GICv5 LPI"); + TEST_ASSERT(get_ucall(vcpus[1], NULL) == UCALL_DONE, + "GICv5 IST restore vCPU1 did not complete"); + + vm_gic_destroy(&dst); +} + /* * Returns 0 if it's possible to create GIC device of a given type (V5). */ @@ -1481,6 +1974,9 @@ void run_tests(u32 gic_dev_type) pr_info("Test VGICv5 LPIs\n"); test_vgic_v5_lpis(); + + pr_info("Test VGICv5 IST save/restore\n"); + test_vgic_v5_ist_save_restore(); } int main(int ac, char **av) From 3ca5a03d139b7d3de02e9de824ba1caebff97b8f Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Fri, 4 Sep 2026 11:59:11 +0000 Subject: [PATCH 163/506] KVM: selftests: Add VGICv5 sparse vCPU IDs test GICv5 uses the userspace-provided vCPU ID as the VPE ID. This must work when vCPU IDs are sparse and differ from the dense internal vCPU index. Add a test that creates vCPUs with IDs 17 and 3 and runs both vCPUs to check that ICC_IAFFIDR_EL1 reports the correct VPE ID on each. Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20260904113404.4051341-50-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 60 +++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index 6caac6cf7c62..d5df742b2300 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -14,6 +14,9 @@ #define NR_VCPUS 1 #define VGIC_V5_LPI_NR_VCPUS 2 +#define VGIC_V5_SPARSE_NR_VCPUS 2 +#define VGIC_V5_SPARSE_VCPU0_ID 17 +#define VGIC_V5_SPARSE_VCPU1_ID 3 #define VGIC_V5_LPI_MEMSLOT 1 #define VGIC_V5_NR_PRIVATE_IRQS 64 #define VGIC_V5_DEFAULT_NR_SPIS 32 @@ -297,6 +300,13 @@ static void guest_code(void) cpu_relax(); } +static void guest_iaffid_code(void) +{ + GUEST_SYNC(FIELD_GET(ICC_IAFFIDR_EL1_IAFFID, + read_sysreg_s(SYS_ICC_IAFFIDR_EL1))); + GUEST_DONE(); +} + static void guest_spi_irq_handler(struct ex_regs *regs) { bool valid; @@ -1522,6 +1532,53 @@ static void test_vgic_v5_cpu_sysreg_attrs(void) vm_gic_destroy(&v); } +static void test_vgic_v5_sparse_vcpu_ids(void) +{ + struct kvm_vcpu *vcpus[VGIC_V5_SPARSE_NR_VCPUS]; + const u32 vcpu_ids[VGIC_V5_SPARSE_NR_VCPUS] = { + VGIC_V5_SPARSE_VCPU0_ID, + VGIC_V5_SPARSE_VCPU1_ID, + }; + int ret, i; + struct ucall uc; + struct vm_gic v; + u64 attr; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, VGIC_V5_SPARSE_NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + + /* Create two vCPUs with legal, albeit sparse, IDs. */ + for (i = 0; i < ARRAY_SIZE(vcpus); i++) { + vcpus[i] = vm_vcpu_add(v.vm, vcpu_ids[i], guest_iaffid_code); + TEST_ASSERT(vcpus[i], "Failed to create vCPU %u", vcpu_ids[i]); + } + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v.vm); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Check both are runnable, and read back their correct IAFFIDs */ + for (i = 0; i < ARRAY_SIZE(vcpus); i++) { + /* IAFFID is the VPE ID and must match the userspace vCPU ID. */ + ret = run_vcpu(vcpus[i]); + TEST_ASSERT(!ret, "Failed to run GICv5 vCPU %u", vcpu_ids[i]); + TEST_ASSERT(get_ucall(vcpus[i], &uc) == UCALL_SYNC && + uc.args[1] == vcpu_ids[i], + "GICv5 vCPU %u IAFFID mismatch", vcpu_ids[i]); + + ret = run_vcpu(vcpus[i]); + TEST_ASSERT(!ret, "Failed to complete GICv5 vCPU %u", vcpu_ids[i]); + TEST_ASSERT(get_ucall(vcpus[i], NULL) == UCALL_DONE, + "GICv5 vCPU %u did not complete", vcpu_ids[i]); + } + + vm_gic_destroy(&v); +} + static void test_vgic_v5_ppis(u32 gic_dev_type) { struct kvm_vcpu *vcpus[NR_VCPUS]; @@ -1966,6 +2023,9 @@ void run_tests(u32 gic_dev_type) pr_info("Test VGICv5 CPU sysreg attrs\n"); test_vgic_v5_cpu_sysreg_attrs(); + pr_info("Test VGICv5 sparse vCPU IDs\n"); + test_vgic_v5_sparse_vcpu_ids(); + pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); From 5224e6f027ee8ba0ebc825ce80c45c0a7b716ff3 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Mon, 7 Sep 2026 17:19:23 +0000 Subject: [PATCH 164/506] KVM: arm64: Forward FFA_NOTIFICATION_BITMAP calls to Trustzone Allow FF-A notification bitmap messages to be forwarded to Trustzone from the host kernel driver and enforce the host vmid check. Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260907171930.4037166-2-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index a327c2bbb6b6..aa82c58380a8 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -692,8 +692,6 @@ static bool ffa_call_supported(u64 func_id) case FFA_MEM_DONATE: case FFA_MEM_RETRIEVE_REQ: /* Optional notification interfaces added in FF-A 1.1 */ - case FFA_NOTIFICATION_BITMAP_CREATE: - case FFA_NOTIFICATION_BITMAP_DESTROY: case FFA_NOTIFICATION_BIND: case FFA_NOTIFICATION_UNBIND: case FFA_NOTIFICATION_SET: @@ -878,6 +876,21 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, hyp_spin_unlock(&host_buffers.lock); } +static void do_ffa_notif_bitmap(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, vmid, ctxt, 1); + struct arm_smccc_1_2_regs *args; + + if (vmid != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_1_2_regs res = {0}; @@ -936,6 +949,10 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) case FFA_PARTITION_INFO_GET: do_ffa_part_get(&res, host_ctxt); goto out_handled; + case FFA_NOTIFICATION_BITMAP_CREATE: + case FFA_NOTIFICATION_BITMAP_DESTROY: + do_ffa_notif_bitmap(&res, host_ctxt); + goto out_handled; } if (ffa_call_supported(func_id)) From 85e44f7eeaec6c303fd6cb2d775cbfecebeb6802 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Mon, 7 Sep 2026 17:19:24 +0000 Subject: [PATCH 165/506] KVM: arm64: Support FFA_NOTIFICATION_BIND in host handler Verify the arguments of the FF-A notification bind call and forward the message to Trustzone. Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260907171930.4037166-3-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index aa82c58380a8..89872e12f9d4 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -42,6 +42,8 @@ */ #define HOST_FFA_ID 0 +#define FFA_NOTIF_RECEIVER_ENDP_MASK GENMASK(15, 0) + /* * A buffer to hold the maximum descriptor size we can see from the host, * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP @@ -692,7 +694,6 @@ static bool ffa_call_supported(u64 func_id) case FFA_MEM_DONATE: case FFA_MEM_RETRIEVE_REQ: /* Optional notification interfaces added in FF-A 1.1 */ - case FFA_NOTIFICATION_BIND: case FFA_NOTIFICATION_UNBIND: case FFA_NOTIFICATION_SET: case FFA_NOTIFICATION_GET: @@ -891,6 +892,27 @@ static void do_ffa_notif_bitmap(struct arm_smccc_1_2_regs *res, hyp_smccc_1_2_smc(args, res); } +static void do_ffa_notif_bind(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags > 1) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_1_2_regs res = {0}; @@ -953,6 +975,9 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) case FFA_NOTIFICATION_BITMAP_DESTROY: do_ffa_notif_bitmap(&res, host_ctxt); goto out_handled; + case FFA_NOTIFICATION_BIND: + do_ffa_notif_bind(&res, host_ctxt); + goto out_handled; } if (ffa_call_supported(func_id)) From 69fde06ec15936551f97e67c7bdc93427e8b08a6 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Mon, 7 Sep 2026 17:19:25 +0000 Subject: [PATCH 166/506] KVM: arm64: Support FFA_NOTIFICATION_UNBIND in host handler Verify the arguments of the FF-A notification unbind call and forward the message to Trustzone. Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260907171930.4037166-4-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 89872e12f9d4..a1ba4463d1ef 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -694,7 +694,6 @@ static bool ffa_call_supported(u64 func_id) case FFA_MEM_DONATE: case FFA_MEM_RETRIEVE_REQ: /* Optional notification interfaces added in FF-A 1.1 */ - case FFA_NOTIFICATION_UNBIND: case FFA_NOTIFICATION_SET: case FFA_NOTIFICATION_GET: case FFA_NOTIFICATION_INFO_GET: @@ -913,6 +912,27 @@ static void do_ffa_notif_bind(struct arm_smccc_1_2_regs *res, hyp_smccc_1_2_smc(args, res); } +static void do_ffa_notif_unbind(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, reserved, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (reserved) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_1_2_regs res = {0}; @@ -978,6 +998,9 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) case FFA_NOTIFICATION_BIND: do_ffa_notif_bind(&res, host_ctxt); goto out_handled; + case FFA_NOTIFICATION_UNBIND: + do_ffa_notif_unbind(&res, host_ctxt); + goto out_handled; } if (ffa_call_supported(func_id)) From e3ba9aec202ae2911ae1166654e21bbbd41ff29d Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Mon, 7 Sep 2026 17:19:26 +0000 Subject: [PATCH 167/506] KVM: arm64: Support FFA_NOTIFICATION_SET in host handler Allow FF-A notification SET messages to be proxied from the pKVM hypervisor to Trustzone and verify the arguments. Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260907171930.4037166-5-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index a1ba4463d1ef..923f32caa04b 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -43,6 +43,7 @@ #define HOST_FFA_ID 0 #define FFA_NOTIF_RECEIVER_ENDP_MASK GENMASK(15, 0) +#define FFA_NOTIF_SENDER_ENDP_MASK GENMASK(31, 16) /* * A buffer to hold the maximum descriptor size we can see from the host, @@ -694,7 +695,6 @@ static bool ffa_call_supported(u64 func_id) case FFA_MEM_DONATE: case FFA_MEM_RETRIEVE_REQ: /* Optional notification interfaces added in FF-A 1.1 */ - case FFA_NOTIFICATION_SET: case FFA_NOTIFICATION_GET: case FFA_NOTIFICATION_INFO_GET: /* Optional interfaces added in FF-A 1.2 */ @@ -933,6 +933,27 @@ static void do_ffa_notif_unbind(struct arm_smccc_1_2_regs *res, hyp_smccc_1_2_smc(args, res); } +static void do_ffa_notif_set(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (FIELD_GET(FFA_NOTIF_SENDER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags & GENMASK(15, 2)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_1_2_regs res = {0}; @@ -1001,6 +1022,9 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) case FFA_NOTIFICATION_UNBIND: do_ffa_notif_unbind(&res, host_ctxt); goto out_handled; + case FFA_NOTIFICATION_SET: + do_ffa_notif_set(&res, host_ctxt); + goto out_handled; } if (ffa_call_supported(func_id)) From 88180874ee3474ce866a00297aa238c64fc2ea85 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Mon, 7 Sep 2026 17:19:27 +0000 Subject: [PATCH 168/506] KVM: arm64: Support FFA_NOTIFICATION_GET in host handler Allow FF-A notification GET messages to be proxied from the pKVM hypervisor to Trustzone and verify the arguments sent from the host driver. Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260907171930.4037166-6-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 923f32caa04b..e11fb06df63d 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -695,7 +695,6 @@ static bool ffa_call_supported(u64 func_id) case FFA_MEM_DONATE: case FFA_MEM_RETRIEVE_REQ: /* Optional notification interfaces added in FF-A 1.1 */ - case FFA_NOTIFICATION_GET: case FFA_NOTIFICATION_INFO_GET: /* Optional interfaces added in FF-A 1.2 */ case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */ @@ -954,6 +953,27 @@ static void do_ffa_notif_set(struct arm_smccc_1_2_regs *res, hyp_smccc_1_2_smc(args, res); } +static void do_ffa_notif_get(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags & GENMASK(31, 4)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_1_2_regs res = {0}; @@ -1025,6 +1045,9 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) case FFA_NOTIFICATION_SET: do_ffa_notif_set(&res, host_ctxt); goto out_handled; + case FFA_NOTIFICATION_GET: + do_ffa_notif_get(&res, host_ctxt); + goto out_handled; } if (ffa_call_supported(func_id)) From 0aa3b20bbb2bf165d32fd802204a558fa461b130 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Mon, 7 Sep 2026 17:19:28 +0000 Subject: [PATCH 169/506] KVM: arm64: Support FFA_NOTIFICATION_INFO_GET in host handler Allow the host to send FF-A notification queries to Trustzone and proxy these messages from pKVM. Reviewed-by: Vincent Donnefort Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260907171930.4037166-7-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index e11fb06df63d..2281ac2e42fc 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -694,8 +694,6 @@ static bool ffa_call_supported(u64 func_id) case FFA_RXTX_MAP: case FFA_MEM_DONATE: case FFA_MEM_RETRIEVE_REQ: - /* Optional notification interfaces added in FF-A 1.1 */ - case FFA_NOTIFICATION_INFO_GET: /* Optional interfaces added in FF-A 1.2 */ case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */ case FFA_MSG_SEND_DIRECT_RESP2: /* Optional per 7.5.1 */ From 3e9df0b0ebd57e42b43842c996271275588660a2 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Mon, 7 Sep 2026 17:19:29 +0000 Subject: [PATCH 170/506] KVM: arm64: Enforce strict SBZ checks in the FF-A proxy Introduce a helper method ffa_check_unused_args_sbz to enforce strict arguments checking when the hypervisor acts as a relayer between the host and Trustzone. Reviewed-by: Vincent Donnefort Acked-by: Will Deacon Signed-off-by: Sebastian Ene Link: https://patch.msgid.link/20260907171930.4037166-8-sebastianene@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/ffa.c | 96 ++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 2281ac2e42fc..5ad74ab265b5 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -74,6 +74,21 @@ static u32 hyp_ffa_version; static bool has_version_negotiated; static hyp_spinlock_t version_lock; +static bool ffa_check_unused_args_sbz(struct kvm_cpu_context *ctxt, int first_reg) +{ + DECLARE_REG(u32, func_id, ctxt, 0); + int reg, end_reg = 7; + + if (hyp_ffa_version >= FFA_VERSION_1_2 && ARM_SMCCC_IS_64(func_id)) + end_reg = 17; + for (reg = first_reg; reg <= end_reg; reg++) { + if (cpu_reg(ctxt, reg)) + return true; + } + + return false; +} + static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno) { *res = (struct arm_smccc_1_2_regs) { @@ -242,6 +257,11 @@ static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res, int ret = 0; void *rx_virt, *tx_virt; + if (ffa_check_unused_args_sbz(ctxt, 4)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) { ret = FFA_RET_INVALID_PARAMETERS; goto out; @@ -318,6 +338,11 @@ static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, id, ctxt, 1); int ret = 0; + if (ffa_check_unused_args_sbz(ctxt, 2)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (id != HOST_FFA_ID) { ret = FFA_RET_INVALID_PARAMETERS; goto out; @@ -424,6 +449,11 @@ static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, int ret = FFA_RET_INVALID_PARAMETERS; u32 nr_ranges; + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) goto out; @@ -486,6 +516,11 @@ static void __do_ffa_mem_xfer(const u64 func_id, int ret = 0; size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version); + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (addr_mbz || npages_mbz || fraglen > len || fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { ret = FFA_RET_INVALID_PARAMETERS; @@ -589,6 +624,11 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, int ret = 0; u64 handle; + if (ffa_check_unused_args_sbz(ctxt, 4)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + handle = PACK_HANDLE(handle_lo, handle_hi); hyp_spin_lock(&host_buffers.lock); @@ -780,6 +820,11 @@ static void do_ffa_version(struct arm_smccc_1_2_regs *res, { DECLARE_REG(u32, ffa_req_version, ctxt, 1); + if (ffa_check_unused_args_sbz(ctxt, 2)) { + res->a0 = FFA_RET_NOT_SUPPORTED; + return; + } + if (FFA_MAJOR_VERSION(ffa_req_version) != 1) { res->a0 = FFA_RET_NOT_SUPPORTED; return; @@ -829,6 +874,11 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, flags, ctxt, 5); u32 count, partition_sz, copy_sz; + if (ffa_check_unused_args_sbz(ctxt, 6)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + hyp_spin_lock(&host_buffers.lock); if (!host_buffers.rx) { ffa_to_smccc_res(res, FFA_RET_BUSY); @@ -876,9 +926,15 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, static void do_ffa_notif_bitmap(struct arm_smccc_1_2_regs *res, struct kvm_cpu_context *ctxt) { + DECLARE_REG(u32, func_id, ctxt, 0); DECLARE_REG(u32, vmid, ctxt, 1); struct arm_smccc_1_2_regs *args; + if (ffa_check_unused_args_sbz(ctxt, func_id == FFA_NOTIFICATION_BITMAP_CREATE ? 3 : 2)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + if (vmid != HOST_FFA_ID) { ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); return; @@ -895,6 +951,11 @@ static void do_ffa_notif_bind(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, flags, ctxt, 2); struct arm_smccc_1_2_regs *args; + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); return; @@ -916,7 +977,7 @@ static void do_ffa_notif_unbind(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, reserved, ctxt, 2); struct arm_smccc_1_2_regs *args; - if (reserved) { + if (ffa_check_unused_args_sbz(ctxt, 5) || reserved) { ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); return; } @@ -942,6 +1003,11 @@ static void do_ffa_notif_set(struct arm_smccc_1_2_regs *res, return; } + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + if (flags & GENMASK(15, 2)) { ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); return; @@ -963,6 +1029,11 @@ static void do_ffa_notif_get(struct arm_smccc_1_2_regs *res, return; } + if (ffa_check_unused_args_sbz(ctxt, 3)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + if (flags & GENMASK(31, 4)) { ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); return; @@ -972,6 +1043,20 @@ static void do_ffa_notif_get(struct arm_smccc_1_2_regs *res, hyp_smccc_1_2_smc(args, res); } +static void do_ffa_notif_info_get(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, 1)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_1_2_regs res = {0}; @@ -1000,6 +1085,11 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) switch (func_id) { case FFA_FEATURES: + if (ffa_check_unused_args_sbz(host_ctxt, 3)) { + ffa_to_smccc_res(&res, FFA_RET_INVALID_PARAMETERS); + goto out_handled; + } + if (!do_ffa_features(&res, host_ctxt)) return false; goto out_handled; @@ -1046,6 +1136,10 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) case FFA_NOTIFICATION_GET: do_ffa_notif_get(&res, host_ctxt); goto out_handled; + case FFA_NOTIFICATION_INFO_GET: + case FFA_FN64_NOTIFICATION_INFO_GET: + do_ffa_notif_info_get(&res, host_ctxt); + goto out_handled; } if (ffa_call_supported(func_id)) From e9fac0fc75abadefbbfbcf514f5d3b6e3d8c1196 Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 10:14:32 +0530 Subject: [PATCH 171/506] KVM: arm64: Fix typo "synchonized" in comment Correct "synchonized" to "synchronized", reported by scripts/checkpatch.pl using the misspelling list in scripts/spelling.txt. Only touches comments, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Link: https://patch.msgid.link/20260907044434.4682-2-hemanth.selam@gmail.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/entry.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 308100ed25de..9709905f21b5 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -46,7 +46,7 @@ alternative_else_nop_endif // Ensure that __guest_enter() always provides a context // synchronization event so that callers don't need ISBs for anything - // that would usually be synchonized by the ERET. + // that would usually be synchronized by the ERET. isb mov x0, #ARM_EXCEPTION_IRQ ret From acf96ecc5fb6a802928a0b71bcded2addc3e58d4 Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 10:14:33 +0530 Subject: [PATCH 172/506] KVM: arm64: selftests: Fix typos in comments Fix typos in comments, reported by scripts/checkpatch.pl using the misspelling list in scripts/spelling.txt. Only touches comments, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Link: https://patch.msgid.link/20260907044434.4682-3-hemanth.selam@gmail.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/debug-exceptions.c | 2 +- tools/testing/selftests/kvm/arm64/hypercalls.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/debug-exceptions.c b/tools/testing/selftests/kvm/arm64/debug-exceptions.c index 7dc5f0b4f6ad..9da6dc5bcfe6 100644 --- a/tools/testing/selftests/kvm/arm64/debug-exceptions.c +++ b/tools/testing/selftests/kvm/arm64/debug-exceptions.c @@ -387,7 +387,7 @@ static void guest_code_ss(int test_cnt) * Enable Single Step execution. Note! This _must_ be a bare * ucall as the ucall() path uses atomic operations to manage * the ucall structures, and the built-in "atomics" are usually - * implemented via exclusive access instructions. The exlusive + * implemented via exclusive access instructions. The exclusive * monitor is cleared on ERET, and so taking debug exceptions * during a LDREX=>STREX sequence will prevent forward progress * and hang the guest/test. diff --git a/tools/testing/selftests/kvm/arm64/hypercalls.c b/tools/testing/selftests/kvm/arm64/hypercalls.c index 5d96cdf382c4..a24becda9e62 100644 --- a/tools/testing/selftests/kvm/arm64/hypercalls.c +++ b/tools/testing/selftests/kvm/arm64/hypercalls.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only -/* hypercalls: Check the ARM64's psuedo-firmware bitmap register interface. +/* hypercalls: Check the ARM64's pseudo-firmware bitmap register interface. * * The test validates the basic hypercall functionalities that are exposed - * via the psuedo-firmware bitmap register. This includes the registers' + * via the pseudo-firmware bitmap register. This includes the registers' * read/write behavior before and after the VM has started, and if the * hypercalls are properly masked or unmasked to the guest when disabled or * enabled from the KVM userspace, respectively. From dacfc40976411dfa97efbe1cc21650049a391eb7 Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 10:14:34 +0530 Subject: [PATCH 173/506] KVM: arm64: Fix typo in pauth.c comment The comment reads "This code assumes that is is run from EL2". The first "is" is not a repetition, it should be "it". Only touches a comment, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Link: https://patch.msgid.link/20260907044434.4682-4-hemanth.selam@gmail.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/pauth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/pauth.c b/arch/arm64/kvm/pauth.c index d5eb3ae876be..bb6c42e78058 100644 --- a/arch/arm64/kvm/pauth.c +++ b/arch/arm64/kvm/pauth.c @@ -5,7 +5,7 @@ * * Primitive PAuth emulation for ERETAA/ERETAB. * - * This code assumes that is is run from EL2, and that it is part of + * This code assumes that it is run from EL2, and that it is part of * the emulation of ERETAx for a guest hypervisor. That's a lot of * baked-in assumptions and shortcuts. * From 57a7d86e2aa7ae6508ee8e08f32b42eaa314a2f5 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Thu, 10 Sep 2026 21:05:28 +0200 Subject: [PATCH 174/506] KVM: arm64: Fix kvm_get_vtcr() kernel-doc kvm_get_vtcr() takes a parameter named phys_shift, but its kernel-doc describes phys_shfit. This leaves the real parameter undescribed and adds an excess parameter description to the generated documentation. Correct the spelling and describe phys_shift as the IPA size in bits. The value written to VTCR_EL2.T0SZ is 64 - phys_shift, rather than phys_shift itself. Fixes: bcb25a2b86b4 ("KVM: arm64: Refactor kvm_arm_setup_stage2()") Suggested-by: Fuad Tabba Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260910190528.4483-1-kmehltretter@gmail.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pgtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 41a8687938eb..20d12da3d28e 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -532,7 +532,7 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size); * kvm_get_vtcr() - Helper to construct VTCR_EL2 * @mmfr0: Sanitized value of SYS_ID_AA64MMFR0_EL1 register. * @mmfr1: Sanitized value of SYS_ID_AA64MMFR1_EL1 register. - * @phys_shfit: Value to set in VTCR_EL2.T0SZ. + * @phys_shift: IPA size, in bits. * * The VTCR value is common across all the physical CPUs on the system. * We use system wide sanitised values to fill in different fields, From 1e3b8f0af9ae1de02d3bb42c6bb699af62510c10 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 10:58:49 +0200 Subject: [PATCH 175/506] Documentation: KVM: Fix the event_source sysfs path in the PMU attribute description The PMU device directories live under /sys/bus/event_source/devices, the document spells the bus even_source. Fix the spelling. Fixes: 6ee7fca2a4a0 ("KVM: arm64: Add KVM_ARM_VCPU_PMU_V3_SET_PMU attribute") Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: Randy Dunlap Link: https://patch.msgid.link/20260905085849.40479-1-kmehltretter@gmail.com Signed-off-by: Marc Zyngier --- Documentation/virt/kvm/devices/vcpu.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst index deb5c51bc00c..a3f20a4ee4b1 100644 --- a/Documentation/virt/kvm/devices/vcpu.rst +++ b/Documentation/virt/kvm/devices/vcpu.rst @@ -135,7 +135,7 @@ after selecting a hardware PMU. Request that the VCPU uses the specified hardware PMU when creating guest events for the purpose of PMU emulation. The PMU identifier can be read from the "type" file for the desired PMU instance under /sys/devices (or, equivalent, -/sys/bus/even_source). This attribute is particularly useful on heterogeneous +/sys/bus/event_source). This attribute is particularly useful on heterogeneous systems where there are at least two CPU PMUs on the system. The PMU that is set for one VCPU will be used by all the other VCPUs. It isn't possible to set a PMU if a PMU event filter is already present. From 068df88a7d8900fedda1621435518c0f2db9e53a Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 11:06:55 +0200 Subject: [PATCH 176/506] Documentation: KVM: Fix the struct kvm_arm_device_addr name KVM_ARM_SET_DEVICE_ADDR takes struct kvm_arm_device_addr; the document names it struct kvm_arm_device_address, which does not exist. Use the real name. Fixes: 3401d54696f9 ("KVM: ARM: Introduce KVM_ARM_SET_DEVICE_ADDR ioctl") Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: Randy Dunlap Link: https://patch.msgid.link/20260905090655.40683-1-kmehltretter@gmail.com Signed-off-by: Marc Zyngier --- Documentation/virt/kvm/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc750c9..7a2de8640226 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -3676,7 +3676,7 @@ type KVM_X86_REG_TYPE_MSR, but are NOT enumerated via KVM_GET_REG_LIST. :Capability: KVM_CAP_ARM_SET_DEVICE_ADDR :Architectures: arm64 :Type: vm ioctl -:Parameters: struct kvm_arm_device_address (in) +:Parameters: struct kvm_arm_device_addr (in) :Returns: 0 on success, -1 on error Errors: From f3435cb7df9b8fe242bbf776a9ac496bdd2184e9 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 31 Aug 2026 17:28:15 +0100 Subject: [PATCH 177/506] KVM: arm64: Fix the KVM_ARM_PREFERRED_TARGET documentation KVM_ARM_PREFERRED_TARGET has never set a feature bit: the original arm64 implementation zeroed the bitmap, and kvm_vcpu_preferred_target(), which only ever zeroed it, was removed by commit 5346f7e13e5e ("KVM: arm64: Always return generic v8 as the preferred target"). The ENODEV error had gone earlier still, with commit 08e873cb70f3 ("KVM: arm64: Change the return type of kvm_vcpu_preferred_target()"). Document what it returns, that userspace selects vCPU features itself, and drop the error. No functional change intended. Signed-off-by: Fuad Tabba Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260831162815.269851-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- Documentation/virt/kvm/api.rst | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 7a2de8640226..f66805c1a450 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -3600,20 +3600,14 @@ Possible features: :Parameters: struct kvm_vcpu_init (out) :Returns: 0 on success; -1 on error -Errors: - - ====== ========================================== - ENODEV no preferred target available for the host - ====== ========================================== - This queries KVM for preferred CPU target type which can be emulated by KVM on underlying host. -The ioctl returns struct kvm_vcpu_init instance containing information -about preferred CPU target type and recommended features for it. The -kvm_vcpu_init->features bitmap returned will have feature bits set if -the preferred target recommends setting these features, but this is -not mandatory. +The ioctl returns a struct kvm_vcpu_init instance containing the +preferred CPU target type. The kvm_vcpu_init->features bitmap is +returned empty: userspace selects the vCPU features itself, and their +availability is reported by the capabilities listed under +KVM_ARM_VCPU_INIT. The information returned by this ioctl can be used to prepare an instance of struct kvm_vcpu_init for KVM_ARM_VCPU_INIT ioctl which will result in From 356abf7cedff742621c9b7ae8ecfa7d0febb0b9e Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 17 Aug 2026 10:27:05 +0100 Subject: [PATCH 178/506] KVM: arm64: vgic-its: Reword the comment on a collection-less ITE The comment above the NULL collection check in vgic_its_save_ite() names only MAPC with V=0, which is UNPREDICTABLE. The route that requires the check is clearing GITS_BASER.Valid on a disabled ITS, a legitimate operation that orphans every ITE at once. Name both routes. No functional change intended. Suggested-by: Eric Auger Link: https://lore.kernel.org/all/0ce5e496-a067-48fd-80d2-94fe9981d505@redhat.com/ Signed-off-by: Fuad Tabba Reviewed-by: Eric Auger Link: https://patch.msgid.link/20260817092705.1112634-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-its.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 9e782a4fea7e..9ceecb45995a 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2121,9 +2121,9 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev, u64 val; /* - * MAPC with V=0 keeps the ITEs mapped but drops their collection, - * and with it the ICID. Save a zeroed entry, which the restore path - * reads back as invalid. + * MAPC with V=0 and clearing GITS_BASER.Valid both keep the + * ITEs mapped while dropping their collection, and with it the ICID. + * Save a zeroed entry, which the restore path reads back as invalid. */ if (!ite->collection) return vgic_its_write_entry_lock(its, gpa, 0ULL, ite); From 968c346143b71678a4970d1d0ab3f6505be7dfd9 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:16 +0100 Subject: [PATCH 179/506] tracing: Include linux/types.h in trace_remote_event.h trace_remote_event.h uses bool without including linux/types.h, so a translation unit that includes it ahead of anything else that pulls types.h in fails to build, as with nvhe/trace.h at EL2. Reviewed-by: Vincent Donnefort Reviewed-by: Marc Zyngier Acked-by: Steven Rostedt Signed-off-by: Fuad Tabba Acked-by: Steven Rostedt Link: https://patch.msgid.link/20260901140326.3812068-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- include/linux/trace_remote_event.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/trace_remote_event.h b/include/linux/trace_remote_event.h index c8ae1e1f5e72..e4cc2d4497bc 100644 --- a/include/linux/trace_remote_event.h +++ b/include/linux/trace_remote_event.h @@ -3,6 +3,8 @@ #ifndef _LINUX_TRACE_REMOTE_EVENTS_H #define _LINUX_TRACE_REMOTE_EVENTS_H +#include + struct trace_remote; struct trace_event_fields; struct trace_seq; From 424f7785bcf316645e7f5f4620e3f9d8efdffb0f Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:17 +0100 Subject: [PATCH 180/506] KVM: arm64: nVHE: Share the stacktrace per-CPU declarations with EL2 The declarations for overflow_stack, kvm_stacktrace_info and pkvm_stacktrace are only visible to the host (the first two sit in the host-only section of stacktrace/nvhe.h, the last is private to kvm/stacktrace.c), so the definitions in nvhe/stacktrace.c compile with no declaration in sight and sparse suggests making them static. DECLARE_KVM_NVHE_PER_CPU() resolves to the right symbol name on both sides of the build: move the declarations where both can see them and include the header from the EL2 side unconditionally. No functional change intended. Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-3-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/stacktrace/nvhe.h | 10 ++++++++-- arch/arm64/kvm/hyp/nvhe/stacktrace.c | 3 +-- arch/arm64/kvm/stacktrace.c | 3 --- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/stacktrace/nvhe.h b/arch/arm64/include/asm/stacktrace/nvhe.h index 171f9edef49f..a631a577cbe5 100644 --- a/arch/arm64/include/asm/stacktrace/nvhe.h +++ b/arch/arm64/include/asm/stacktrace/nvhe.h @@ -37,6 +37,14 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state, state->pc = pc; } +DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); +DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info); + +#ifdef CONFIG_PKVM_STACKTRACE +DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], + pkvm_stacktrace); +#endif + #ifndef __KVM_NVHE_HYPERVISOR__ /* * Conventional (non-protected) nVHE HYP stack unwinder @@ -45,8 +53,6 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state, * (by the host in EL1). */ -DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); -DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info); DECLARE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base); void kvm_nvhe_dump_backtrace(unsigned long hyp_offset); diff --git a/arch/arm64/kvm/hyp/nvhe/stacktrace.c b/arch/arm64/kvm/hyp/nvhe/stacktrace.c index 7c832d60d22b..11fadbebbf1d 100644 --- a/arch/arm64/kvm/hyp/nvhe/stacktrace.c +++ b/arch/arm64/kvm/hyp/nvhe/stacktrace.c @@ -8,6 +8,7 @@ #include #include #include +#include DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) __aligned(16); @@ -35,8 +36,6 @@ static void hyp_prepare_backtrace(unsigned long fp, unsigned long pc) } #ifdef CONFIG_PKVM_STACKTRACE -#include - DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], pkvm_stacktrace); static struct stack_info stackinfo_get_overflow(void) diff --git a/arch/arm64/kvm/stacktrace.c b/arch/arm64/kvm/stacktrace.c index 9724c320126b..69377195e18b 100644 --- a/arch/arm64/kvm/stacktrace.c +++ b/arch/arm64/kvm/stacktrace.c @@ -198,9 +198,6 @@ static void hyp_dump_backtrace(unsigned long hyp_offset) } #ifdef CONFIG_PKVM_STACKTRACE -DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], - pkvm_stacktrace); - /* * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace. * From be44596c05ae1b34d7b9029acc1c9d41ee3b8783 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:18 +0100 Subject: [PATCH 181/506] KVM: arm64: nVHE: Declare the hyp event IDs before defining them The defining expansion of HYP_EVENT() in events.c is the first time its translation unit meets hyp_event_id_, so sparse suggests making the symbols static. Include kvm_hypevents.h ahead of define_events.h so the extern declarations come first, as with the tracepoint headers. No functional change intended. Reviewed-by: Vincent Donnefort Tested-by: Vincent Donnefort Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-4-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/events.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/events.c index add9383aadb5..b845be0acd11 100644 --- a/arch/arm64/kvm/hyp/nvhe/events.c +++ b/arch/arm64/kvm/hyp/nvhe/events.c @@ -7,6 +7,8 @@ #include #include +#include + #include int __tracing_enable_event(unsigned short id, bool enable) From 3d7a7a01243408df1b0232288e4f9cf7362980b9 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:19 +0100 Subject: [PATCH 182/506] KVM: arm64: nVHE: Use NULL to reset the trace buffer backing pointer bpages_backing_start is a pointer; resetting it to plain 0 triggers a sparse warning. No functional change intended. Reviewed-by: Vincent Donnefort Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-5-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 65be6c9fb379..5437ebb6dbd6 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -93,7 +93,7 @@ static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace __release_host_mem(start, size); - trace_buffer->bpages_backing_start = 0; + trace_buffer->bpages_backing_start = NULL; trace_buffer->bpages_backing_size = 0; } From c3ac2ba27ed74b58525bd6614c503b30b221bb51 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:20 +0100 Subject: [PATCH 183/506] KVM: arm64: nVHE: Run the source checker under C=2 The custom %.nvhe.o rule reuses rule_cc_o_c, which hooks the source checker only for C=1, and that only when the object is rebuilt. The C=2 hook, cmd_force_checksrc, hangs off the standard %.o rule that nVHE objects do not use, so "make C=2" silently skips every nVHE source file. Call cmd_force_checksrc after the compile rule, as the standard rule does. Fixes: 7621712918ad4 ("KVM: arm64: Add build rules for separate VHE/nVHE object files") Reviewed-by: Vincent Donnefort Tested-by: Vincent Donnefort Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-6-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index f57450ebcb49..ccc1fe839409 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -49,6 +49,7 @@ targets += $(hyp-obj) kvm_nvhe.tmp.o kvm_nvhe.rel.o hyp.lds hyp-reloc.S hyp-relo # avoids file name clashes for files shared with VHE. $(obj)/%.nvhe.o: $(src)/%.c FORCE $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) $(obj)/%.nvhe.o: $(src)/%.S FORCE $(call if_changed_rule,as_o_S) From 44b14ff4aebeae98663d0fd424b4ef88237ee981 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:21 +0100 Subject: [PATCH 184/506] arm64: pi: Run the source checker on the libfdt objects under C=2 The custom lib-%.o rule reuses rule_cc_o_c, which hooks the source checker only for C=1, and that only when the object is rebuilt. The C=2 hook, cmd_force_checksrc, hangs off the standard %.o rule, so "make C=2" silently skips the two libfdt objects. Call cmd_force_checksrc after the compile rule, as the standard rule does. Fixes: aacd149b6238 ("arm64: head: avoid relocating the kernel twice for KASLR") Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-7-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kernel/pi/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kernel/pi/Makefile b/arch/arm64/kernel/pi/Makefile index be92d73c25b2..96243c291e39 100644 --- a/arch/arm64/kernel/pi/Makefile +++ b/arch/arm64/kernel/pi/Makefile @@ -34,6 +34,7 @@ $(obj)/lib-%.pi.o: OBJCOPYFLAGS += --prefix-alloc-sections=.init $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) obj-y := idreg-override.pi.o \ map_kernel.pi.o map_range.pi.o \ From 79a2e4681fd5936f4d7ab26251fe96ee76d60940 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:22 +0100 Subject: [PATCH 185/506] KVM: arm64: nVHE: Pass host VA arguments as pointers Four hypercalls take host VAs as unsigned long: the donated vm, pgd and vcpu regions, and the tracing descriptor. Retype the arguments and their EL2 consumers as void *, so that the typed hypercall declarations introduced later in the series can attach a sparse address space to them; an address space attaches only to pointers. No functional change intended. Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-8-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 5 ++--- arch/arm64/kvm/hyp/include/nvhe/trace.h | 4 ++-- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 8 ++++---- arch/arm64/kvm/hyp/nvhe/pkvm.c | 11 +++++------ arch/arm64/kvm/hyp/nvhe/trace.c | 4 ++-- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index c904647d2f76..2643a1a81966 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -69,10 +69,9 @@ void pkvm_hyp_vm_table_init(void *tbl); int __pkvm_reserve_vm(void); void __pkvm_unreserve_vm(pkvm_handle_t handle); -int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, - unsigned long pgd_hva); +int __pkvm_init_vm(struct kvm *host_kvm, void *vm_hva, void *pgd_hva); int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - unsigned long vcpu_hva); + void *vcpu_hva); int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn); int __pkvm_start_teardown_vm(pkvm_handle_t handle); diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h index 8813ff250f8e..4aa36fd76b9e 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/trace.h +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h @@ -46,7 +46,7 @@ static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt) void *tracing_reserve_entry(unsigned long length); void tracing_commit_entry(void); -int __tracing_load(unsigned long desc_va, size_t desc_size); +int __tracing_load(void *desc_va, size_t desc_size); void __tracing_unload(void); int __tracing_enable(bool enable); int __tracing_swap_reader(unsigned int cpu); @@ -59,7 +59,7 @@ static inline void tracing_commit_entry(void) { } #define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \ static inline void trace_##__name(__proto) {} -static inline int __tracing_load(unsigned long desc_va, size_t desc_size) { return -ENODEV; } +static inline int __tracing_load(void *desc_va, size_t desc_size) { return -ENODEV; } static inline void __tracing_unload(void) { } static inline int __tracing_enable(bool enable) { return -ENODEV; } static inline int __tracing_swap_reader(unsigned int cpu) { return -ENODEV; } diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9a3b92e626ad..61fcb382197d 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -771,8 +771,8 @@ static void handle___pkvm_unreserve_vm(struct kvm_cpu_context *host_ctxt) static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1); - DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2); - DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3); + DECLARE_REG(void *, vm_hva, host_ctxt, 2); + DECLARE_REG(void *, pgd_hva, host_ctxt, 3); host_kvm = kern_hyp_va(host_kvm); cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva); @@ -782,7 +782,7 @@ static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2); - DECLARE_REG(unsigned long, vcpu_hva, host_ctxt, 3); + DECLARE_REG(void *, vcpu_hva, host_ctxt, 3); host_vcpu = kern_hyp_va(host_vcpu); cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva); @@ -828,7 +828,7 @@ static void handle___pkvm_finalize_teardown_vm(struct kvm_cpu_context *host_ctxt static void handle___tracing_load(struct kvm_cpu_context *host_ctxt) { - DECLARE_REG(unsigned long, desc_hva, host_ctxt, 1); + DECLARE_REG(void *, desc_hva, host_ctxt, 1); DECLARE_REG(size_t, desc_size, host_ctxt, 2); cpu_reg(host_ctxt, 1) = __tracing_load(desc_hva, desc_size); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 459bd9eb7e4b..6c8e33ebf8bb 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -657,9 +657,9 @@ static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus) size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus)); } -static void *map_donated_memory_noclear(unsigned long host_va, size_t size) +static void *map_donated_memory_noclear(void *host_va, size_t size) { - void *va = (void *)kern_hyp_va(host_va); + void *va = kern_hyp_va(host_va); if (!PAGE_ALIGNED(va)) return NULL; @@ -671,7 +671,7 @@ static void *map_donated_memory_noclear(unsigned long host_va, size_t size) return va; } -static void *map_donated_memory(unsigned long host_va, size_t size) +static void *map_donated_memory(void *host_va, size_t size) { void *va = map_donated_memory_noclear(host_va, size); @@ -818,8 +818,7 @@ void teardown_selftest_vm(void) * * Return 0 success, negative error code on failure. */ -int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, - unsigned long pgd_hva) +int __pkvm_init_vm(struct kvm *host_kvm, void *vm_hva, void *pgd_hva) { struct pkvm_hyp_vm *hyp_vm = NULL; size_t vm_size, pgd_size; @@ -910,7 +909,7 @@ static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, } int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - unsigned long vcpu_hva) + void *vcpu_hva) { struct pkvm_hyp_vcpu *hyp_vcpu; struct pkvm_hyp_vm *hyp_vm; diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 5437ebb6dbd6..eaa63b06a286 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -206,9 +206,9 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz return true; } -int __tracing_load(unsigned long desc_hva, size_t desc_size) +int __tracing_load(void *desc_hva, size_t desc_size) { - struct hyp_trace_desc *desc = (struct hyp_trace_desc *)kern_hyp_va(desc_hva); + struct hyp_trace_desc *desc = kern_hyp_va(desc_hva); int ret; ret = __admit_host_mem(desc, desc_size); From 3a189b358018d76020c65b5a52c90ea9353ef642 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:23 +0100 Subject: [PATCH 186/506] KVM: arm64: Move the host hypercall interface to its own header Move the kvm_call_hyp() dispatch macros and pkvm_handle_t out of kvm_host.h into a new kvm_hcall.h, giving the host<->hyp hypercall interface a single home that subsequent patches build on to restore type-checking across the boundary. The only adjustment to the moved code is the checkpatch-mandated space in "while (0)". No functional change intended. Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-9-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hcall.h | 68 ++++++++++++++++++++++++++++++ arch/arm64/include/asm/kvm_host.h | 48 +-------------------- 2 files changed, 69 insertions(+), 47 deletions(-) create mode 100644 arch/arm64/include/asm/kvm_hcall.h diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h new file mode 100644 index 000000000000..d925b2c28a3d --- /dev/null +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * The host<->hyp hypercall interface. + * + * Copyright (C) 2026 Google LLC + * Author: Fuad Tabba + */ + +#ifndef __ARM64_KVM_HCALL_H__ +#define __ARM64_KVM_HCALL_H__ + +#include +#include +#include +#include + +#include +#include +#include + +typedef u16 pkvm_handle_t; + +#ifndef __KVM_NVHE_HYPERVISOR__ +#define kvm_call_hyp_nvhe(f, ...) \ + ({ \ + struct arm_smccc_res res; \ + \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \ + ##__VA_ARGS__, &res); \ + if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \ + res.a1 = -EOPNOTSUPP; \ + \ + res.a1; \ + }) + +/* + * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, + * where the eret to EL1 acts as a context synchronization event. + */ +#define kvm_call_hyp(f, ...) \ + do { \ + if (has_vhe()) { \ + f(__VA_ARGS__); \ + isb(); \ + } else { \ + kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ + } \ + } while (0) + +#define kvm_call_hyp_ret(f, ...) \ + ({ \ + typeof(f(__VA_ARGS__)) ret; \ + \ + if (has_vhe()) { \ + ret = f(__VA_ARGS__); \ + } else { \ + ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ + } \ + \ + ret; \ + }) +#else /* __KVM_NVHE_HYPERVISOR__ */ +#define kvm_call_hyp(f, ...) f(__VA_ARGS__) +#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) +#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) +#endif /* __KVM_NVHE_HYPERVISOR__ */ + +#endif /* __ARM64_KVM_HCALL_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..fc1a82777ed4 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #define __KVM_HAVE_ARCH_INTC_INITIALIZED @@ -251,8 +252,6 @@ struct kvm_smccc_features { unsigned long vendor_hyp_bmap_2; /* Function numbers 64-127 */ }; -typedef u16 pkvm_handle_t; - struct kvm_protected_vm { pkvm_handle_t handle; struct kvm_hyp_memcache teardown_mc; @@ -1258,51 +1257,6 @@ void kvm_arm_resume_guest(struct kvm *kvm); #define vcpu_has_run_once(vcpu) (!!READ_ONCE((vcpu)->pid)) -#ifndef __KVM_NVHE_HYPERVISOR__ -#define kvm_call_hyp_nvhe(f, ...) \ - ({ \ - struct arm_smccc_res res; \ - \ - arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \ - ##__VA_ARGS__, &res); \ - if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \ - res.a1 = -EOPNOTSUPP; \ - \ - res.a1; \ - }) - -/* - * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, - * where the eret to EL1 acts as a context synchronization event. - */ -#define kvm_call_hyp(f, ...) \ - do { \ - if (has_vhe()) { \ - f(__VA_ARGS__); \ - isb(); \ - } else { \ - kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ - } \ - } while(0) - -#define kvm_call_hyp_ret(f, ...) \ - ({ \ - typeof(f(__VA_ARGS__)) ret; \ - \ - if (has_vhe()) { \ - ret = f(__VA_ARGS__); \ - } else { \ - ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ - } \ - \ - ret; \ - }) -#else /* __KVM_NVHE_HYPERVISOR__ */ -#define kvm_call_hyp(f, ...) f(__VA_ARGS__) -#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) -#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) -#endif /* __KVM_NVHE_HYPERVISOR__ */ - int handle_exit(struct kvm_vcpu *vcpu, int exception_index); void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index); From db89e04ddbb7838b40f7d7b59b69220212678702 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:24 +0100 Subject: [PATCH 187/506] KVM: arm64: Type-check hypercall arguments at the caller kvm_call_hyp_nvhe() reduces its target to an SMCCC function number, so the compiler never sees a callable: arguments that are wrong in number, type or order are silently marshalled into registers. The kvm_call_hyp() wrappers only catch this on the VHE branch, and the pKVM-only hypercalls have no such branch. Declare each hypercall's signature once in kvm_hcall.h and generate a typed stub from it, in the mold of the syscall wrappers. Make kvm_call_hyp_nvhe() resolve to the stub so every caller is checked against the declared signature; a stale or mistyped call now fails to compile. The stubs inline to the same SMCCC call the untyped macro used to make: the compiled callers are unchanged, apart from hypercall returns now being tested at their declared width. The stage-2 protection arguments are declared u64 rather than enum kvm_pgtable_prot, as kvm_pgtable.h includes linux/kvm_host.h and the enum cannot be completed here. Assisted-by: Antigravity:gemini-3.1-pro Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-10-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hcall.h | 153 ++++++++++++++++++++++++++++- arch/arm64/kvm/hyp_trace.c | 2 +- 2 files changed, 153 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index d925b2c28a3d..c2efcb89e6e7 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -9,6 +9,7 @@ #ifndef __ARM64_KVM_HCALL_H__ #define __ARM64_KVM_HCALL_H__ +#include #include #include #include @@ -16,12 +17,37 @@ #include #include +#include #include typedef u16 pkvm_handle_t; +struct kvm; +struct kvm_s2_mmu; +struct kvm_vcpu; +struct vgic_v3_cpu_if; +struct vgic_v5_cpu_if; + +/* + * Hypercall signatures are declared as (type, name) argument pairs. + * __KVM_HCALL_MAP() applies a macro to each pair, in the mold of __MAP() + * in . The ladder is indexed by list entries, two per + * argument; __KVM_HCALL_MAP_N() takes that count explicitly. + */ +#define __KVM_HCALL_MAP2(m, t, a, ...) m(t, a) +#define __KVM_HCALL_MAP4(m, t, a, ...) m(t, a), __KVM_HCALL_MAP2(m, __VA_ARGS__) +#define __KVM_HCALL_MAP6(m, t, a, ...) m(t, a), __KVM_HCALL_MAP4(m, __VA_ARGS__) +#define __KVM_HCALL_MAP8(m, t, a, ...) m(t, a), __KVM_HCALL_MAP6(m, __VA_ARGS__) +#define __KVM_HCALL_MAP10(m, t, a, ...) m(t, a), __KVM_HCALL_MAP8(m, __VA_ARGS__) +#define __KVM_HCALL_MAP12(m, t, a, ...) m(t, a), __KVM_HCALL_MAP10(m, __VA_ARGS__) +#define __KVM_HCALL_MAP_N(n, m, ...) CONCATENATE(__KVM_HCALL_MAP, n)(m, __VA_ARGS__) +#define __KVM_HCALL_MAP(m, ...) __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), m, __VA_ARGS__) + +#define __KVM_HCALL_DECL(t, a) t a +#define __KVM_HCALL_ARGS(t, a) a + #ifndef __KVM_NVHE_HYPERVISOR__ -#define kvm_call_hyp_nvhe(f, ...) \ +#define __kvm_call_hyp_nvhe(f, ...) \ ({ \ struct arm_smccc_res res; \ \ @@ -33,6 +59,29 @@ typedef u16 pkvm_handle_t; res.a1; \ }) +/* + * Generate a typed stub for each declared hypercall. kvm_call_hyp_nvhe() + * resolves to the stub, so a call with the wrong argument count or types + * fails to compile instead of being silently truncated to an SMCCC function + * number and a pile of registers. The stub inlines to the same SMCCC call + * the untyped macro used to make. + */ +#define DECLARE_KVM_HOST_HCALL(ret, name, ...) \ + static __always_inline \ + ret nvhe_hvc_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) \ + { \ + return (ret)__kvm_call_hyp_nvhe(name, \ + __KVM_HCALL_MAP(__KVM_HCALL_ARGS, __VA_ARGS__));\ + } + +#define DECLARE_KVM_HOST_HCALL0(ret, name) \ + static __always_inline ret nvhe_hvc_##name(void) \ + { \ + return (ret)__kvm_call_hyp_nvhe(name); \ + } + +#define kvm_call_hyp_nvhe(f, ...) nvhe_hvc_##f(__VA_ARGS__) + /* * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, * where the eret to EL1 acts as a context synchronization event. @@ -63,6 +112,108 @@ typedef u16 pkvm_handle_t; #define kvm_call_hyp(f, ...) f(__VA_ARGS__) #define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) #define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) + +#define DECLARE_KVM_HOST_HCALL(ret, name, ...) +#define DECLARE_KVM_HOST_HCALL0(ret, name) #endif /* __KVM_NVHE_HYPERVISOR__ */ +/* Hypercalls that are unavailable once pKVM has finalised. */ +DECLARE_KVM_HOST_HCALL(int, __pkvm_init, + phys_addr_t, phys, unsigned long, size, + unsigned long *, per_cpu_base, u32, hyp_va_bits) +DECLARE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping, + phys_addr_t, phys, size_t, size, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, + enum arm64_hyp_spectre_vector, slot) +DECLARE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) +DECLARE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) +DECLARE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) + +DECLARE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) + +/* Hypercalls that are always available and common to [nh]VHE/pKVM. */ +DECLARE_KVM_HOST_HCALL(void, __kvm_adjust_pc, + struct kvm_vcpu *, vcpu) +DECLARE_KVM_HOST_HCALL(int, __kvm_vcpu_run, + struct kvm_vcpu *, vcpu) +DECLARE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, + struct kvm_s2_mmu *, mmu, phys_addr_t, ipa, int, level) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, + struct kvm_s2_mmu *, mmu, phys_addr_t, ipa, int, level) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, + struct kvm_s2_mmu *, mmu) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, + struct kvm_s2_mmu *, mmu, phys_addr_t, start, unsigned long, pages) +DECLARE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, + struct kvm_s2_mmu *, mmu) +DECLARE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, + u64, cntvoff) +DECLARE_KVM_HOST_HCALL(int, __tracing_load, + void *, desc_hva, size_t, desc_size) +DECLARE_KVM_HOST_HCALL0(void, __tracing_unload) +DECLARE_KVM_HOST_HCALL(int, __tracing_enable, + bool, enable) +DECLARE_KVM_HOST_HCALL(int, __tracing_swap_reader, + unsigned int, cpu) +DECLARE_KVM_HOST_HCALL(void, __tracing_update_clock, + u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc) +DECLARE_KVM_HOST_HCALL(int, __tracing_reset, + unsigned int, cpu) +DECLARE_KVM_HOST_HCALL(int, __tracing_enable_event, + unsigned short, id, bool, enable) +DECLARE_KVM_HOST_HCALL(void, __tracing_write_event, + u64, id) +DECLARE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, + struct vgic_v3_cpu_if *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, + struct vgic_v3_cpu_if *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, + struct vgic_v5_cpu_if *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, + struct vgic_v5_cpu_if *, cpu_if) + +/* Hypercalls that are available only when pKVM has finalised. */ +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, + u64, pfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp, + u64, pfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, + u64, pfn, u64, gfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_guest, + u64, pfn, u64, gfn, u64, nr_pages, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest, + u64, gfn, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, + u64, gfn) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) +DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm, + struct kvm *, host_kvm, void *, vm_hva, void *, pgd_hva) +DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, + pkvm_handle_t, handle, struct kvm_vcpu *, host_vcpu, + void *, vcpu_hva) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) +DECLARE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, + phys_addr_t, phys) +DECLARE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page, + pkvm_handle_t, handle, u64, gfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, + pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2) +DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put) +DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) +DECLARE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, + pkvm_handle_t, handle) + #endif /* __ARM64_KVM_HCALL_H__ */ diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 9644c424819b..44937e257375 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -269,7 +269,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) if (ret) goto err_free_buffer; - ret = kvm_call_hyp_nvhe(__tracing_load, (unsigned long)desc, desc_size); + ret = kvm_call_hyp_nvhe(__tracing_load, desc, desc_size); if (ret) goto err_unload_pages; From ab31289483f54892dc6204a6ce2015420d0d5213 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:25 +0100 Subject: [PATCH 188/506] KVM: arm64: nVHE: Check hypercall handlers against the declared ABI Each hypercall handler unmarshals its arguments from the host context with hand-written DECLARE_REG() casts that nothing ties to what the caller passed: a handler can disagree with its caller in argument type, count or register index without a diagnostic. Generate the unmarshalling instead. DEFINE_KVM_HOST_HCALL() expands to the handle_() glue, modelled on the syscall wrappers, and checks the handler's parameter list against the signature declared in kvm_hcall.h, so both ends of every hypercall are now compiled against the same declaration. Handler bodies keep their logic and lose the DECLARE_REG() and return-register boilerplate. The two get_host_hyp_vcpus() macros existed only to wrap DECLARE_REG(), so one is dropped and the other becomes a function. The compiled handlers are instruction-for-instruction identical, apart from flush_hyp_vcpu() and sync_hyp_vcpu() now being inlined into their only caller. The return-register store is picked by the declared return type, so a void handler needs no macro of its own. The type is pasted into the store's name, which has to be a single word, so __pkvm_create_private_mapping is declared ulong. Assisted-by: Antigravity:gemini-3.1-pro Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-11-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hcall.h | 13 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 438 +++++++++++++---------------- 2 files changed, 209 insertions(+), 242 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index c2efcb89e6e7..c709991d10cd 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -44,6 +44,8 @@ struct vgic_v5_cpu_if; #define __KVM_HCALL_MAP(m, ...) __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), m, __VA_ARGS__) #define __KVM_HCALL_DECL(t, a) t a +#define __KVM_HCALL_LONG(t, a) unsigned long a +#define __KVM_HCALL_CAST(t, a) (__force t) a #define __KVM_HCALL_ARGS(t, a) a #ifndef __KVM_NVHE_HYPERVISOR__ @@ -113,8 +115,15 @@ struct vgic_v5_cpu_if; #define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) #define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) -#define DECLARE_KVM_HOST_HCALL(ret, name, ...) -#define DECLARE_KVM_HOST_HCALL0(ret, name) +/* + * At EL2 each declaration emits the canonical signature of the hypercall, + * which DEFINE_KVM_HOST_HCALL() in hyp-main.c checks the handler + * definition against. + */ +#define DECLARE_KVM_HOST_HCALL(ret, name, ...) \ + typedef ret kvm_host_hcall_sig_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)); +#define DECLARE_KVM_HOST_HCALL0(ret, name) \ + typedef ret kvm_host_hcall_sig_##name(void); #endif /* __KVM_NVHE_HYPERVISOR__ */ /* Hypercalls that are unavailable once pKVM has finalised. */ diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 61fcb382197d..9085f46c2880 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -26,6 +26,48 @@ DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); +/* + * Define a hypercall handler: handle_ unmarshals the arguments from + * the host context and hands them, correctly typed, to the body that + * follows the macro. The parameter list is type-checked against the + * signature declared in , so the handler cannot drift + * from what the typed caller stubs marshal in. Modelled on the syscall + * wrappers. + */ +/* Truncate the fixed list of argument registers to the declared signature. */ +#define KVM_HOST_HCALL_REGS(...) \ + __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), __KVM_HCALL_ARGS \ + ,, cpu_reg(host_ctxt, 1),, cpu_reg(host_ctxt, 2) \ + ,, cpu_reg(host_ctxt, 3),, cpu_reg(host_ctxt, 4) \ + ,, cpu_reg(host_ctxt, 5),, cpu_reg(host_ctxt, 6)) + +#define set_cpu_reg_ulong(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_u64(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_int(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_void(ctxt, r, v) { v; } +#define set_cpu_reg(ctxt, r, t, v) set_cpu_reg_##t(ctxt, r, v) + +#define DEFINE_KVM_HOST_HCALL(ret, name, ...) \ + static kvm_host_hcall_sig_##name __do_##name; \ + static __always_inline \ + ret __se_##name(__KVM_HCALL_MAP(__KVM_HCALL_LONG, __VA_ARGS__)) \ + { \ + return __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_CAST, __VA_ARGS__)); \ + } \ + static void handle_##name(struct kvm_cpu_context *host_ctxt) \ + { \ + set_cpu_reg(host_ctxt, 1, ret, __se_##name(KVM_HOST_HCALL_REGS(__VA_ARGS__))); \ + } \ + static ret __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) + +#define DEFINE_KVM_HOST_HCALL0(ret, name) \ + static kvm_host_hcall_sig_##name __do_##name; \ + static void handle_##name(struct kvm_cpu_context *host_ctxt) \ + { \ + set_cpu_reg(host_ctxt, 1, ret, __do_##name()); \ + } \ + static ret __do_##name(void) + /* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */ unsigned int hyp_gicv3_nr_lr; @@ -279,11 +321,9 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) sync_hyp_vgic_state(hyp_vcpu); } -static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, + pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(unsigned int, vcpu_idx, host_ctxt, 2); - DECLARE_REG(u64, hcr_el2, host_ctxt, 3); struct pkvm_hyp_vcpu *hyp_vcpu; hyp_vcpu = pkvm_load_hyp_vcpu(handle, vcpu_idx); @@ -300,7 +340,7 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) } } -static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put) { struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); @@ -316,7 +356,7 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) } } -static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) { struct pkvm_hyp_vcpu *hyp_vcpu; @@ -346,34 +386,26 @@ static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, return host_vcpu; } -#define get_host_hyp_vcpus(ctxt, regnr, hyp_vcpup) \ - ({ \ - DECLARE_REG(struct kvm_vcpu *, __vcpu, ctxt, regnr); \ - __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ - }) +static struct kvm_vcpu * +__get_host_hyp_vcpus_from_vgic_v3_cpu_if(struct vgic_v3_cpu_if *cpu_if, + struct pkvm_hyp_vcpu **hyp_vcpup) +{ + struct kvm_vcpu *vcpu = container_of(cpu_if, struct kvm_vcpu, + arch.vgic_cpu.vgic_v3); -#define get_host_hyp_vcpus_from_vgic_v3_cpu_if(ctxt, regnr, hyp_vcpup) \ - ({ \ - DECLARE_REG(struct vgic_v3_cpu_if *, cif, ctxt, regnr);\ - struct kvm_vcpu *__vcpu = container_of(cif, \ - struct kvm_vcpu, \ - arch.vgic_cpu.vgic_v3); \ - \ - __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ - }) + return __get_host_hyp_vcpus(vcpu, hyp_vcpup); +} -static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __kvm_vcpu_run, + struct kvm_vcpu *, vcpu) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; int ret; - host_vcpu = get_host_hyp_vcpus(host_ctxt, 1, &hyp_vcpu); - - if (!host_vcpu) { - ret = -EINVAL; - goto out; - } + host_vcpu = __get_host_hyp_vcpus(vcpu, &hyp_vcpu); + if (!host_vcpu) + return -EINVAL; if (unlikely(hyp_vcpu)) { /* @@ -382,10 +414,8 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) * loading a vcpu. Therefore, if SME features enabled the host * is misbehaving. */ - if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) { - ret = -EINVAL; - goto out; - } + if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) + return -EINVAL; flush_hyp_vcpu(hyp_vcpu); @@ -398,8 +428,8 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) ret = __kvm_vcpu_run(host_vcpu); fpsimd_lazy_switch_to_host(host_vcpu); } -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu) @@ -411,184 +441,150 @@ static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu) &host_vcpu->arch.pkvm_memcache); } -static void handle___pkvm_host_donate_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, + u64, pfn, u64, gfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; + int ret; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || !pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; ret = pkvm_refill_memcache(hyp_vcpu); if (ret) - goto out; + return ret; - ret = __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu); } -static void handle___pkvm_host_share_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_guest, + u64, pfn, u64, gfn, u64, nr_pages, u64, prot) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 4); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; + int ret; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; ret = pkvm_refill_memcache(hyp_vcpu); if (ret) - goto out; + return ret; - ret = __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot); } -static void handle___pkvm_host_unshare_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_unshare_guest(gfn, nr_pages, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_relax_perms_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest, + u64, gfn, u64, prot) { - DECLARE_REG(u64, gfn, host_ctxt, 1); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 2); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; - ret = __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot); } -static void handle___pkvm_host_wrprotect_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_wrprotect_guest(gfn, nr_pages, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_test_clear_young_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); - DECLARE_REG(bool, mkold, host_ctxt, 4); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_test_clear_young_guest(gfn, nr_pages, mkold, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_mkyoung_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, + u64, gfn) { - DECLARE_REG(u64, gfn, host_ctxt, 1); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; - ret = __pkvm_host_mkyoung_guest(gfn, hyp_vcpu); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_mkyoung_guest(gfn, hyp_vcpu); } -static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_adjust_pc, + struct kvm_vcpu *, vcpu) { - DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); - __kvm_adjust_pc(kern_hyp_va(vcpu)); } -static void handle___kvm_flush_vm_context(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) { __kvm_flush_vm_context(); } -static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, + struct kvm_s2_mmu *, mmu, phys_addr_t, ipa, int, level) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); - DECLARE_REG(int, level, host_ctxt, 3); - __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); } -static void handle___kvm_tlb_flush_vmid_ipa_nsh(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, + struct kvm_s2_mmu *, mmu, phys_addr_t, ipa, int, level) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); - DECLARE_REG(int, level, host_ctxt, 3); - __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va(mmu), ipa, level); } -static void -handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, + struct kvm_s2_mmu *, mmu, phys_addr_t, start, unsigned long, pages) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, start, host_ctxt, 2); - DECLARE_REG(unsigned long, pages, host_ctxt, 3); - __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, pages); } -static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, + struct kvm_s2_mmu *, mmu) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - __kvm_tlb_flush_vmid(kern_hyp_va(mmu)); } -static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); struct pkvm_hyp_vm *hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) @@ -598,19 +594,19 @@ static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) put_pkvm_hyp_vm(hyp_vm); } -static void handle___kvm_flush_cpu_context(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, + struct kvm_s2_mmu *, mmu) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - __kvm_flush_cpu_context(kern_hyp_va(mmu)); } -static void handle___kvm_timer_set_cntvoff(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, + u64, cntvoff) { - __kvm_timer_set_cntvoff(cpu_reg(host_ctxt, 1)); + __kvm_timer_set_cntvoff(cntvoff); } -static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) { u64 tmp; @@ -619,23 +615,23 @@ static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt) write_sysreg_el2(tmp, SYS_SCTLR); } -static void handle___vgic_v3_get_gic_config(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) { - cpu_reg(host_ctxt, 1) = __vgic_v3_get_gic_config(); + return __vgic_v3_get_gic_config(); } -static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) { __vgic_v3_init_lrs(); } -static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, + struct vgic_v3_cpu_if *, cpu_if) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, - &hyp_vcpu); + host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu); if (!host_vcpu) return; @@ -657,13 +653,13 @@ static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) } } -static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, + struct vgic_v3_cpu_if *, cpu_if) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, - &hyp_vcpu); + host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu); if (!host_vcpu) return; @@ -690,48 +686,39 @@ static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt } } -static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init, + phys_addr_t, phys, unsigned long, size, + unsigned long *, per_cpu_base, u32, hyp_va_bits) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - DECLARE_REG(unsigned long, size, host_ctxt, 2); - DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 3); - DECLARE_REG(u32, hyp_va_bits, host_ctxt, 4); - /* * __pkvm_init() will return only if an error occurred, otherwise it * will tail-call in __pkvm_init_finalise() which will have to deal * with the host context directly. */ - cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, per_cpu_base, hyp_va_bits); + return __pkvm_init(phys, size, per_cpu_base, hyp_va_bits); } -static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, + enum arm64_hyp_spectre_vector, slot) { - DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot); + return pkvm_cpu_set_vector(slot); } -static void handle___pkvm_host_share_hyp(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, + u64, pfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn); + return __pkvm_host_share_hyp(pfn); } -static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp, + u64, pfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn); + return __pkvm_host_unshare_hyp(pfn); } -static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping, + phys_addr_t, phys, size_t, size, u64, prot) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - DECLARE_REG(size_t, size, host_ctxt, 2); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3); - /* * __pkvm_create_private_mapping() populates a pointer with the * hypervisor start address of the allocation. @@ -742,160 +729,131 @@ static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ct * Instead pass the allocation address as the return value (or return * ERR_PTR() on failure). */ - unsigned long haddr; + ulong haddr; int err = __pkvm_create_private_mapping(phys, size, prot, &haddr); if (err) - haddr = (unsigned long)ERR_PTR(err); + haddr = (ulong)ERR_PTR(err); - cpu_reg(host_ctxt, 1) = haddr; + return haddr; } -static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) { - cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize(); + return __pkvm_prot_finalize(); } -static void handle___pkvm_reserve_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) { - cpu_reg(host_ctxt, 1) = __pkvm_reserve_vm(); + return __pkvm_reserve_vm(); } -static void handle___pkvm_unreserve_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - __pkvm_unreserve_vm(handle); } -static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vm, + struct kvm *, host_kvm, void *, vm_hva, void *, pgd_hva) { - DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1); - DECLARE_REG(void *, vm_hva, host_ctxt, 2); - DECLARE_REG(void *, pgd_hva, host_ctxt, 3); - - host_kvm = kern_hyp_va(host_kvm); - cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva); + return __pkvm_init_vm(kern_hyp_va(host_kvm), vm_hva, pgd_hva); } -static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, + pkvm_handle_t, handle, struct kvm_vcpu *, host_vcpu, + void *, vcpu_hva) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2); - DECLARE_REG(void *, vcpu_hva, host_ctxt, 3); - - host_vcpu = kern_hyp_va(host_vcpu); - cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva); + return __pkvm_init_vcpu(handle, kern_hyp_va(host_vcpu), vcpu_hva); } -static void handle___pkvm_vcpu_in_poison_fault(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) { - int ret; struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - ret = hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL; - cpu_reg(host_ctxt, 1) = ret; + return hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL; } -static void handle___pkvm_force_reclaim_guest_page(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, + phys_addr_t, phys) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_force_reclaim_page_guest(phys); + return __pkvm_host_force_reclaim_page_guest(phys); } -static void handle___pkvm_reclaim_dying_guest_page(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page, + pkvm_handle_t, handle, u64, gfn) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - - cpu_reg(host_ctxt, 1) = __pkvm_reclaim_dying_guest_page(handle, gfn); + return __pkvm_reclaim_dying_guest_page(handle, gfn); } -static void handle___pkvm_start_teardown_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_start_teardown_vm(handle); + return __pkvm_start_teardown_vm(handle); } -static void handle___pkvm_finalize_teardown_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_finalize_teardown_vm(handle); + return __pkvm_finalize_teardown_vm(handle); } -static void handle___tracing_load(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_load, + void *, desc_hva, size_t, desc_size) { - DECLARE_REG(void *, desc_hva, host_ctxt, 1); - DECLARE_REG(size_t, desc_size, host_ctxt, 2); - - cpu_reg(host_ctxt, 1) = __tracing_load(desc_hva, desc_size); + return __tracing_load(desc_hva, desc_size); } -static void handle___tracing_unload(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __tracing_unload) { __tracing_unload(); } -static void handle___tracing_enable(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_enable, + bool, enable) { - DECLARE_REG(bool, enable, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_enable(enable); + return __tracing_enable(enable); } -static void handle___tracing_swap_reader(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_swap_reader, + unsigned int, cpu) { - DECLARE_REG(unsigned int, cpu, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_swap_reader(cpu); + return __tracing_swap_reader(cpu); } -static void handle___tracing_update_clock(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __tracing_update_clock, + u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc) { - DECLARE_REG(u32, mult, host_ctxt, 1); - DECLARE_REG(u32, shift, host_ctxt, 2); - DECLARE_REG(u64, epoch_ns, host_ctxt, 3); - DECLARE_REG(u64, epoch_cyc, host_ctxt, 4); - __tracing_update_clock(mult, shift, epoch_ns, epoch_cyc); } -static void handle___tracing_reset(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_reset, + unsigned int, cpu) { - DECLARE_REG(unsigned int, cpu, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_reset(cpu); + return __tracing_reset(cpu); } -static void handle___tracing_enable_event(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_enable_event, + unsigned short, id, bool, enable) { - DECLARE_REG(unsigned short, id, host_ctxt, 1); - DECLARE_REG(bool, enable, host_ctxt, 2); - - cpu_reg(host_ctxt, 1) = __tracing_enable_event(id, enable); + return __tracing_enable_event(id, enable); } -static void handle___tracing_write_event(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __tracing_write_event, + u64, id) { - DECLARE_REG(u64, id, host_ctxt, 1); - trace_selftest(id); } -static void handle___vgic_v5_save_apr(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, + struct vgic_v5_cpu_if *, cpu_if) { - DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); - __vgic_v5_save_apr(kern_hyp_va(cpu_if)); } -static void handle___vgic_v5_restore_vmcr_apr(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, + struct vgic_v5_cpu_if *, cpu_if) { - DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); - __vgic_v5_restore_vmcr_apr(kern_hyp_va(cpu_if)); } From 8bd0e6812a1be71d50620f00445940ded0df897c Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 1 Sep 2026 15:03:26 +0100 Subject: [PATCH 189/506] KVM: arm64: Tag host-VA hypercall parameters __kern The nVHE hypervisor takes host virtual addresses as hypercall arguments and translates each with kern_hyp_va() before use. Nothing marks them as host-owned, so dereferencing one untranslated at EL2 - a recurring bug class - is invisible to the compiler. Add a __kern sparse address space, active only for EL2 code, and tag the host-VA parameters in the hypercall declarations. kern_hyp_va_host() is the only sanctioned unwrap: it translates the address, preserves the pointee type (stripped of qualifiers, as with the percpu accessors) and drops the tag with a __force cast, so an untranslated host VA fails sparse. The tag flows from the shared declaration into the generated handler and on into the donated-memory and tracing-descriptor helpers, so a handler cannot extract a host VA without it. Host code sees plain pointers, and the tag is checker-only: no code is generated. container_of() casts through void * and drops the address space, so __get_host_hyp_vcpus() now takes an already translated vCPU and its callers unwrap. Translating a vgic_v3_cpu_if before taking its container is equivalent, since va_mask spans every bit in which two linear-map addresses differ. Reviewed-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260901140326.3812068-12-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hcall.h | 43 ++++++++++------ arch/arm64/include/asm/kvm_mmu.h | 10 ++++ arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 6 ++- arch/arm64/kvm/hyp/include/nvhe/trace.h | 5 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 65 +++++++++++++------------ arch/arm64/kvm/hyp/nvhe/pkvm.c | 11 +++-- arch/arm64/kvm/hyp/nvhe/trace.c | 4 +- 7 files changed, 86 insertions(+), 58 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index c709991d10cd..9f06af58a4b0 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -28,6 +28,18 @@ struct kvm_vcpu; struct vgic_v3_cpu_if; struct vgic_v5_cpu_if; +/* + * A host VA carried by a hypercall argument. At EL2 such a pointer must not + * be dereferenced until it is translated with kern_hyp_va_host(); sparse + * flags any use that skips the translation. The tag describes the EL2 view + * only: the host dereferences its own VAs freely. + */ +#if defined(__KVM_NVHE_HYPERVISOR__) && defined(__CHECKER__) +#define __kern __attribute__((noderef, address_space(__kern))) +#else +#define __kern +#endif + /* * Hypercall signatures are declared as (type, name) argument pairs. * __KVM_HCALL_MAP() applies a macro to each pair, in the mold of __MAP() @@ -142,24 +154,24 @@ DECLARE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) /* Hypercalls that are always available and common to [nh]VHE/pKVM. */ DECLARE_KVM_HOST_HCALL(void, __kvm_adjust_pc, - struct kvm_vcpu *, vcpu) + struct kvm_vcpu __kern *, vcpu) DECLARE_KVM_HOST_HCALL(int, __kvm_vcpu_run, - struct kvm_vcpu *, vcpu) + struct kvm_vcpu __kern *, vcpu) DECLARE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, - struct kvm_s2_mmu *, mmu, phys_addr_t, ipa, int, level) + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, - struct kvm_s2_mmu *, mmu, phys_addr_t, ipa, int, level) + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, - struct kvm_s2_mmu *, mmu) + struct kvm_s2_mmu __kern *, mmu) DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, - struct kvm_s2_mmu *, mmu, phys_addr_t, start, unsigned long, pages) + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages) DECLARE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, - struct kvm_s2_mmu *, mmu) + struct kvm_s2_mmu __kern *, mmu) DECLARE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, u64, cntvoff) DECLARE_KVM_HOST_HCALL(int, __tracing_load, - void *, desc_hva, size_t, desc_size) + void __kern *, desc_hva, size_t, desc_size) DECLARE_KVM_HOST_HCALL0(void, __tracing_unload) DECLARE_KVM_HOST_HCALL(int, __tracing_enable, bool, enable) @@ -174,13 +186,13 @@ DECLARE_KVM_HOST_HCALL(int, __tracing_enable_event, DECLARE_KVM_HOST_HCALL(void, __tracing_write_event, u64, id) DECLARE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, - struct vgic_v3_cpu_if *, cpu_if) + struct vgic_v3_cpu_if __kern *, cpu_if) DECLARE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, - struct vgic_v3_cpu_if *, cpu_if) + struct vgic_v3_cpu_if __kern *, cpu_if) DECLARE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, - struct vgic_v5_cpu_if *, cpu_if) + struct vgic_v5_cpu_if __kern *, cpu_if) DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, - struct vgic_v5_cpu_if *, cpu_if) + struct vgic_v5_cpu_if __kern *, cpu_if) /* Hypercalls that are available only when pKVM has finalised. */ DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, @@ -205,10 +217,11 @@ DECLARE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, pkvm_handle_t, handle) DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm, - struct kvm *, host_kvm, void *, vm_hva, void *, pgd_hva) + struct kvm __kern *, host_kvm, void __kern *, vm_hva, + void __kern *, pgd_hva) DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, - pkvm_handle_t, handle, struct kvm_vcpu *, host_vcpu, - void *, vcpu_hva) + pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu, + void __kern *, vcpu_hva) DECLARE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) DECLARE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, phys_addr_t, phys) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 6eae7e7e2a68..57f65257bb56 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -7,6 +7,8 @@ #ifndef __ARM64_KVM_MMU_H__ #define __ARM64_KVM_MMU_H__ +#include + #include #include #include @@ -140,6 +142,14 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) #define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v)))) +/* + * Translate a __kern-tagged host VA, dropping the tag: the only sanctioned + * unwrap. Translation only, no ownership or bounds validation; the result + * carries the pointee type stripped of the tag and of any cv-qualifiers. + */ +#define kern_hyp_va_host(v) \ + ((TYPEOF_UNQUAL(*(v)) *)__kern_hyp_va((unsigned long)(__force void *)(v))) + extern u32 __hyp_va_bits; /* diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 2643a1a81966..c1171451e1be 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -7,6 +7,7 @@ #ifndef __ARM64_KVM_NVHE_PKVM_H__ #define __ARM64_KVM_NVHE_PKVM_H__ +#include #include #include @@ -69,9 +70,10 @@ void pkvm_hyp_vm_table_init(void *tbl); int __pkvm_reserve_vm(void); void __pkvm_unreserve_vm(pkvm_handle_t handle); -int __pkvm_init_vm(struct kvm *host_kvm, void *vm_hva, void *pgd_hva); +int __pkvm_init_vm(struct kvm *host_kvm, void __kern *vm_hva, + void __kern *pgd_hva); int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - void *vcpu_hva); + void __kern *vcpu_hva); int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn); int __pkvm_start_teardown_vm(pkvm_handle_t handle); diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h index 4aa36fd76b9e..c2db9f70ea26 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/trace.h +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h @@ -4,6 +4,7 @@ #include +#include #include static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt) @@ -46,7 +47,7 @@ static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt) void *tracing_reserve_entry(unsigned long length); void tracing_commit_entry(void); -int __tracing_load(void *desc_va, size_t desc_size); +int __tracing_load(void __kern *desc_va, size_t desc_size); void __tracing_unload(void); int __tracing_enable(bool enable); int __tracing_swap_reader(unsigned int cpu); @@ -59,7 +60,7 @@ static inline void tracing_commit_entry(void) { } #define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \ static inline void trace_##__name(__proto) {} -static inline int __tracing_load(void *desc_va, size_t desc_size) { return -ENODEV; } +static inline int __tracing_load(void __kern *desc_va, size_t desc_size) { return -ENODEV; } static inline void __tracing_unload(void) { } static inline int __tracing_enable(bool enable) { return -ENODEV; } static inline int __tracing_swap_reader(unsigned int cpu) { return -ENODEV; } diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9085f46c2880..91e74a1a334f 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -367,10 +367,9 @@ DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) sync_hyp_vcpu_state(hyp_vcpu); } -static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, +static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *host_vcpu, struct pkvm_hyp_vcpu **hyp_vcpup) { - struct kvm_vcpu *host_vcpu = kern_hyp_va(arg); struct pkvm_hyp_vcpu *hyp_vcpu = NULL; if (unlikely(is_protected_kvm_enabled())) { @@ -387,23 +386,24 @@ static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, } static struct kvm_vcpu * -__get_host_hyp_vcpus_from_vgic_v3_cpu_if(struct vgic_v3_cpu_if *cpu_if, +__get_host_hyp_vcpus_from_vgic_v3_cpu_if(struct vgic_v3_cpu_if __kern *cpu_if, struct pkvm_hyp_vcpu **hyp_vcpup) { - struct kvm_vcpu *vcpu = container_of(cpu_if, struct kvm_vcpu, - arch.vgic_cpu.vgic_v3); + struct vgic_v3_cpu_if *host_cpu_if = kern_hyp_va_host(cpu_if); + struct kvm_vcpu *host_vcpu = container_of(host_cpu_if, struct kvm_vcpu, + arch.vgic_cpu.vgic_v3); - return __get_host_hyp_vcpus(vcpu, hyp_vcpup); + return __get_host_hyp_vcpus(host_vcpu, hyp_vcpup); } DEFINE_KVM_HOST_HCALL(int, __kvm_vcpu_run, - struct kvm_vcpu *, vcpu) + struct kvm_vcpu __kern *, vcpu) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; int ret; - host_vcpu = __get_host_hyp_vcpus(vcpu, &hyp_vcpu); + host_vcpu = __get_host_hyp_vcpus(kern_hyp_va_host(vcpu), &hyp_vcpu); if (!host_vcpu) return -EINVAL; @@ -548,9 +548,9 @@ DEFINE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, } DEFINE_KVM_HOST_HCALL(void, __kvm_adjust_pc, - struct kvm_vcpu *, vcpu) + struct kvm_vcpu __kern *, vcpu) { - __kvm_adjust_pc(kern_hyp_va(vcpu)); + __kvm_adjust_pc(kern_hyp_va_host(vcpu)); } DEFINE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) @@ -559,27 +559,27 @@ DEFINE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) } DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, - struct kvm_s2_mmu *, mmu, phys_addr_t, ipa, int, level) + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) { - __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); + __kvm_tlb_flush_vmid_ipa(kern_hyp_va_host(mmu), ipa, level); } DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, - struct kvm_s2_mmu *, mmu, phys_addr_t, ipa, int, level) + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) { - __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va(mmu), ipa, level); + __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va_host(mmu), ipa, level); } DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, - struct kvm_s2_mmu *, mmu, phys_addr_t, start, unsigned long, pages) + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages) { - __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, pages); + __kvm_tlb_flush_vmid_range(kern_hyp_va_host(mmu), start, pages); } DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, - struct kvm_s2_mmu *, mmu) + struct kvm_s2_mmu __kern *, mmu) { - __kvm_tlb_flush_vmid(kern_hyp_va(mmu)); + __kvm_tlb_flush_vmid(kern_hyp_va_host(mmu)); } DEFINE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, @@ -595,9 +595,9 @@ DEFINE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, } DEFINE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, - struct kvm_s2_mmu *, mmu) + struct kvm_s2_mmu __kern *, mmu) { - __kvm_flush_cpu_context(kern_hyp_va(mmu)); + __kvm_flush_cpu_context(kern_hyp_va_host(mmu)); } DEFINE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, @@ -626,7 +626,7 @@ DEFINE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) } DEFINE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, - struct vgic_v3_cpu_if *, cpu_if) + struct vgic_v3_cpu_if __kern *, cpu_if) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; @@ -654,7 +654,7 @@ DEFINE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, } DEFINE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, - struct vgic_v3_cpu_if *, cpu_if) + struct vgic_v3_cpu_if __kern *, cpu_if) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; @@ -755,16 +755,17 @@ DEFINE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, } DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vm, - struct kvm *, host_kvm, void *, vm_hva, void *, pgd_hva) + struct kvm __kern *, host_kvm, void __kern *, vm_hva, + void __kern *, pgd_hva) { - return __pkvm_init_vm(kern_hyp_va(host_kvm), vm_hva, pgd_hva); + return __pkvm_init_vm(kern_hyp_va_host(host_kvm), vm_hva, pgd_hva); } DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, - pkvm_handle_t, handle, struct kvm_vcpu *, host_vcpu, - void *, vcpu_hva) + pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu, + void __kern *, vcpu_hva) { - return __pkvm_init_vcpu(handle, kern_hyp_va(host_vcpu), vcpu_hva); + return __pkvm_init_vcpu(handle, kern_hyp_va_host(host_vcpu), vcpu_hva); } DEFINE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) @@ -799,7 +800,7 @@ DEFINE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, } DEFINE_KVM_HOST_HCALL(int, __tracing_load, - void *, desc_hva, size_t, desc_size) + void __kern *, desc_hva, size_t, desc_size) { return __tracing_load(desc_hva, desc_size); } @@ -846,15 +847,15 @@ DEFINE_KVM_HOST_HCALL(void, __tracing_write_event, } DEFINE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, - struct vgic_v5_cpu_if *, cpu_if) + struct vgic_v5_cpu_if __kern *, cpu_if) { - __vgic_v5_save_apr(kern_hyp_va(cpu_if)); + __vgic_v5_save_apr(kern_hyp_va_host(cpu_if)); } DEFINE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, - struct vgic_v5_cpu_if *, cpu_if) + struct vgic_v5_cpu_if __kern *, cpu_if) { - __vgic_v5_restore_vmcr_apr(kern_hyp_va(cpu_if)); + __vgic_v5_restore_vmcr_apr(kern_hyp_va_host(cpu_if)); } typedef void (*hcall_t)(struct kvm_cpu_context *); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 6c8e33ebf8bb..5cdb0318fa7b 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -657,9 +657,9 @@ static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus) size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus)); } -static void *map_donated_memory_noclear(void *host_va, size_t size) +static void *map_donated_memory_noclear(void __kern *host_va, size_t size) { - void *va = kern_hyp_va(host_va); + void *va = kern_hyp_va_host(host_va); if (!PAGE_ALIGNED(va)) return NULL; @@ -671,7 +671,7 @@ static void *map_donated_memory_noclear(void *host_va, size_t size) return va; } -static void *map_donated_memory(void *host_va, size_t size) +static void *map_donated_memory(void __kern *host_va, size_t size) { void *va = map_donated_memory_noclear(host_va, size); @@ -818,7 +818,8 @@ void teardown_selftest_vm(void) * * Return 0 success, negative error code on failure. */ -int __pkvm_init_vm(struct kvm *host_kvm, void *vm_hva, void *pgd_hva) +int __pkvm_init_vm(struct kvm *host_kvm, void __kern *vm_hva, + void __kern *pgd_hva) { struct pkvm_hyp_vm *hyp_vm = NULL; size_t vm_size, pgd_size; @@ -909,7 +910,7 @@ static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, } int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - void *vcpu_hva) + void __kern *vcpu_hva) { struct pkvm_hyp_vcpu *hyp_vcpu; struct pkvm_hyp_vm *hyp_vm; diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index eaa63b06a286..47680a949f61 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -206,9 +206,9 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz return true; } -int __tracing_load(void *desc_hva, size_t desc_size) +int __tracing_load(void __kern *desc_hva, size_t desc_size) { - struct hyp_trace_desc *desc = kern_hyp_va(desc_hva); + struct hyp_trace_desc *desc = kern_hyp_va_host(desc_hva); int ret; ret = __admit_host_mem(desc, desc_size); From bb6c7d2bf564cdaa24b14764b7126f5cab45965a Mon Sep 17 00:00:00 2001 From: Tharit Tangkijwanichakul Date: Sun, 6 Sep 2026 03:41:43 +0000 Subject: [PATCH 190/506] KVM: arm64: selftests: Fix VM leak in early return test_mmio_ease() returns early and skips the call to kvm_vm_free() when ID_AA64PFR1_EL1 is not available. Use a goto so that the early return joins the common exit path. Reported-by: Gokul K Link: https://lore.kernel.org/all/20260808081050.408657-1-gokul02k@gmail.com/ Fixes: a90aac553249 ("KVM: arm64: selftests: Test SEAs are taken to SError vector when EASE=1") Signed-off-by: Tharit Tangkijwanichakul Link: https://patch.msgid.link/20260906034144.14059-1-tharitt97@gmail.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/external_aborts.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/arm64/external_aborts.c b/tools/testing/selftests/kvm/arm64/external_aborts.c index d8fe17a6cc59..7836756a38a6 100644 --- a/tools/testing/selftests/kvm/arm64/external_aborts.c +++ b/tools/testing/selftests/kvm/arm64/external_aborts.c @@ -339,7 +339,7 @@ static void test_mmio_ease(void) pfr1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR1_EL1)); if (!SYS_FIELD_GET(ID_AA64PFR1_EL1, DF2, pfr1)) { pr_debug("Skipping %s\n", __func__); - return; + goto done; } /* @@ -356,6 +356,7 @@ static void test_mmio_ease(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); +done: kvm_vm_free(vm); } From 05fd5844cc83c1a51b0a38113a7ee196a69bda1d Mon Sep 17 00:00:00 2001 From: Like Xu Date: Tue, 18 Aug 2026 19:29:18 +0800 Subject: [PATCH 191/506] KVM: arm64: selftests: Fix EINJ handling in sea_to_user sea_to_user drives EINJ injection through popen("echo ... > file"). A failed write is hidden behind the shell pipeline, so a botched injection is silently ignored and the test proceeds as if a poison were placed. Drive the debugfs interface directly so a write failure is reported rather than swallowed. The EINJ support probe is also unreliable. It checks the ACPI EINJ table with access(R_OK), but reading that table needs CAP_SYS_ADMIN, so the check gives a false negative for unprivileged runs and skips a test that could otherwise report a real problem. Probe by opening the debugfs injection interface instead and classify errno: absence means EINJ is not built or firmware lacks support, EACCES/EPERM means the test is not running as root -- both are skips -- while anything else is a genuine failure. The injection path is only reachable through debugfs; the kernel EINJ driver exposes no other interface, so the test must depend on it. Link: https://lore.kernel.org/kvm/86y0le9cvz.wl-maz@kernel.org/ Signed-off-by: Like Xu Link: https://patch.msgid.link/20260818112920.26252-2-likexu@tencent.com Signed-off-by: Marc Zyngier --- .../testing/selftests/kvm/arm64/sea_to_user.c | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/sea_to_user.c b/tools/testing/selftests/kvm/arm64/sea_to_user.c index e96d8982c28b..1c2a743ca8e2 100644 --- a/tools/testing/selftests/kvm/arm64/sea_to_user.c +++ b/tools/testing/selftests/kvm/arm64/sea_to_user.c @@ -81,25 +81,46 @@ static u64 translate_hva_to_hpa(unsigned long hva) static void write_einj_entry(const char *einj_path, u64 val) { - char cmd[256] = {0}; - FILE *cmdfile = NULL; + char buf[32]; + int fd, len, ret; - sprintf(cmd, "echo %#lx > %s", val, einj_path); - cmdfile = popen(cmd, "r"); + fd = open(einj_path, O_WRONLY); + if (fd < 0) + ksft_exit_fail_perror(einj_path); - if (pclose(cmdfile) == 0) - ksft_print_msg("echo %#lx > %s - done\n", val, einj_path); - else - ksft_exit_fail_perror("Failed to write EINJ entry"); + len = snprintf(buf, sizeof(buf), "%#lx\n", val); + ret = write(fd, buf, len); + if (ret != len) + ksft_exit_fail_perror(einj_path); + + close(fd); + ksft_print_msg("%#lx > %s - done\n", val, einj_path); } static void inject_uer(u64 hpa) { - if (access("/sys/firmware/acpi/tables/EINJ", R_OK) == -1) - ksft_test_result_skip("EINJ table no available in firmware"); + int fd; - if (access(EINJ_ETYPE, R_OK | W_OK) == -1) - ksft_test_result_skip("EINJ module probably not loaded?"); + /* + * EINJ is exposed only through debugfs, and opening it tells us why it + * is unusable far more reliably than access()-ing the ACPI EINJ table: + * reading that table needs CAP_SYS_ADMIN, so access() gives a false + * negative for unprivileged runs. Open the injection interface and let + * errno say whether the test cannot run (skip) or genuinely failed. + */ + fd = open(EINJ_ETYPE, O_WRONLY); + if (fd < 0) { + switch (errno) { + case ENOENT: + ksft_exit_skip("need CONFIG_ACPI_APEI_EINJ and firmware EINJ support\n"); + case EACCES: + case EPERM: + ksft_exit_skip("EINJ requires running as root\n"); + default: + ksft_exit_fail_perror(EINJ_ETYPE); + } + } + close(fd); write_einj_entry(EINJ_ETYPE, ERROR_TYPE_MEMORY_UER); write_einj_entry(EINJ_FLAGS, MASK_MEMORY_UER); From 2665ac1063e71f23888091a0bdb1b96297bd0370 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Tue, 18 Aug 2026 19:29:19 +0800 Subject: [PATCH 192/506] KVM: arm64: selftests: Skip sea_to_user without 1GB hugepages sea_to_user backs guest memory with 1GB hugepages but never checks that any are reserved. On a host with an empty pool it aborts instead of skipping: kvm_syscalls.h:68: mem != MAP_FAILED, errno=12 (ENOMEM) not ok 1 selftests: kvm: sea_to_user # exit=254 The mmap() of the hugetlb region fails with -ENOMEM because no 1GB pages are available, and the test treats that as a hard failure even though it simply cannot run without the backing pages. Check the free 1GB hugepage count up front and skip cleanly when the host cannot satisfy the region. Signed-off-by: Like Xu Reviewed-by: Jiaqi Yan Link: https://patch.msgid.link/20260818112920.26252-3-likexu@tencent.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/sea_to_user.c | 6 ++++++ tools/testing/selftests/kvm/include/test_util.h | 1 + tools/testing/selftests/kvm/lib/test_util.c | 15 +++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/sea_to_user.c b/tools/testing/selftests/kvm/arm64/sea_to_user.c index 1c2a743ca8e2..7cf95da8e594 100644 --- a/tools/testing/selftests/kvm/arm64/sea_to_user.c +++ b/tools/testing/selftests/kvm/arm64/sea_to_user.c @@ -281,6 +281,12 @@ static struct kvm_vm *vm_create_with_sea_handler(struct kvm_vcpu **vcpu) alignment = max(backing_page_size, guest_page_size); num_guest_pages = VM_MEM_SIZE / guest_page_size; + /* + * The region is backed by 1GB hugepages; skip gracefully rather than + * failing with mmap() -ENOMEM if the host has none reserved. + */ + TEST_REQUIRE(get_free_hugepages(backing_page_size) >= VM_MEM_SIZE); + vm = __vm_create_with_one_vcpu(vcpu, num_guest_pages, guest_code); vm_init_descriptor_tables(vm); vcpu_init_descriptor_tables(*vcpu); diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a6a3e1657895..76bb11bb79f3 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -172,6 +172,7 @@ struct vm_mem_backing_src_alias { bool thp_configured(void); size_t get_trans_hugepagesz(void); size_t get_def_hugetlb_pagesz(void); +size_t get_free_hugepages(size_t page_size); const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(u32 i); size_t get_backing_src_pagesz(u32 i); bool is_backing_src_hugetlb(u32 i); diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index 4dff72f6bd34..f9263a715704 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -255,6 +255,21 @@ size_t get_def_hugetlb_pagesz(void) TEST_FAIL("Error in reading /proc/meminfo"); } +size_t get_free_hugepages(size_t page_size) +{ + char path[128]; + size_t free; + + snprintf(path, sizeof(path), + "/sys/kernel/mm/hugepages/hugepages-%zukB/free_hugepages", + page_size >> 10); + if (!test_sysfs_path(path)) + return 0; + + free = get_sysfs_val(path); + return free * page_size; +} + #define ANON_FLAGS (MAP_PRIVATE | MAP_ANONYMOUS) #define ANON_HUGE_FLAGS (ANON_FLAGS | MAP_HUGETLB) From 754c7b7d45074337972c5c3ad76ed9e7b103e5d0 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Tue, 18 Aug 2026 19:29:20 +0800 Subject: [PATCH 193/506] KVM: arm64: selftests: Skip sea_to_user when EINJ places no poison sea_to_user injects a memory UER with EINJ notrigger=1 and expects the guest to consume it and trap to KVM as an SEA. On some firmware the test aborts instead: arm64/sea_to_user.c:223: exit_reason == (41) Wanted KVM exit reason: 41 (ARM_SEA), got: 6 (MMIO) notrigger=1 asks firmware to arm the poison without consuming it, but on these platforms the poison is only armed as part of the trigger step that notrigger skips, so nothing consumable is left in memory. The guest reads back the sentinel, no SEA occurs, and GUEST_FAIL fires. On arm64 a ucall is delivered as an MMIO write, which surfaces as the KVM_EXIT_MMIO above rather than a KVM bug. The file already documents that the test must be skipped when firmware cannot deliver a consumable error. Detect the guest abort that follows a missing SEA and skip, instead of failing on a firmware limitation the test cannot control. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Like Xu Reviewed-by: Jiaqi Yan Link: https://patch.msgid.link/20260818112920.26252-4-likexu@tencent.com Signed-off-by: Marc Zyngier --- .../testing/selftests/kvm/arm64/sea_to_user.c | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/sea_to_user.c b/tools/testing/selftests/kvm/arm64/sea_to_user.c index 7cf95da8e594..d4af6e0eed28 100644 --- a/tools/testing/selftests/kvm/arm64/sea_to_user.c +++ b/tools/testing/selftests/kvm/arm64/sea_to_user.c @@ -215,13 +215,33 @@ static void run_vm(struct kvm_vm *vm, struct kvm_vcpu *vcpu) ksft_print_msg("Dump kvm_run info about KVM_EXIT_%s\n", exit_reason_str(run->exit_reason)); + + /* + * The guest's read of the injected location is expected to trap to KVM + * as an SEA. If it does not, the injected error was never placed as + * consumable poison: some firmware honours EINJ's notrigger request by + * arming the poison only as part of the (now skipped) trigger step, so + * nothing is left in memory for the guest to consume. The guest then + * reads back the sentinel value and reports it via GUEST_FAIL, which + * arm64 delivers as a ucall over MMIO (hence a KVM_EXIT_MMIO here). + * Treat that as "this platform cannot host the test" and skip, matching + * the requirement documented at the top of this file, rather than + * failing on a hardware/firmware limitation the test cannot control. + */ + if (run->exit_reason != KVM_EXIT_ARM_SEA && + get_ucall(vcpu, &uc) == UCALL_ABORT) { + ksft_print_msg("Guest consumed no SEA: %s", uc.buffer); + ksft_exit_skip("EINJ notrigger placed no consumable poison on this platform\n"); + } + + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_SEA); + + /* arm_sea holds valid data only for a KVM_EXIT_ARM_SEA exit. */ ksft_print_msg("kvm_run.arm_sea: esr=%#llx, flags=%#llx\n", run->arm_sea.esr, run->arm_sea.flags); ksft_print_msg("kvm_run.arm_sea: gva=%#llx, gpa=%#llx\n", run->arm_sea.gva, run->arm_sea.gpa); - TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_SEA); - esr = run->arm_sea.esr; TEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_LOW); TEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT); From 65ecca6a1446f635e4a00b893bc6dd5ed9cb6f75 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 19 Sep 2026 12:40:25 +0100 Subject: [PATCH 194/506] KVM: arm64: vgic-v5: Drop __iomem attribute from {vmd,vpet}_base Both vmd_base and vpet_base are pointers to kmalloc'd memory, and therefore cannot carry the __iomem attribute, which designates pointers to MMIO. Drop the abusive qualifier. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202609190650.liMAzwjF-lkp@intel.com/ Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5-tables.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h index b2ffbe68c05b..89096926563f 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.h +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -23,8 +23,8 @@ struct vmtl2_entry { typedef __le64 vpe_entry; struct vgic_v5_vm_info { - void __iomem *vmd_base; - vpe_entry __iomem *vpet_base; + void *vmd_base; + vpe_entry *vpet_base; void *vped_base; u8 vpe_id_bits; From dd5b7887542690fe1fc553eebac9ea235a040265 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 19 Sep 2026 12:43:14 +0100 Subject: [PATCH 195/506] KVM: arm64: vgic-v5: Correctly reset h_lpi_ist to NULL Resetting a pointer to 0 is not completely wrong, but sparse legitimately shouts about it integer/pointer mix. Use NULL as expected. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202609190952.glUgOBvp-lkp@intel.com/ Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5-tables.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index ba8a8d5fcc2a..d7196ed5d1bf 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -1066,7 +1066,7 @@ static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, unsigned int id_bits /* Free the L1 IST again */ vmi = xa_load(&vm_info, vm_id); kfree(vmi->h_lpi_ist); - vmi->h_lpi_ist = 0; + vmi->h_lpi_ist = NULL; return ret; } From 066b1a17fd2dacea1eb92813520917ea3aac799e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 19 Sep 2026 13:04:44 +0100 Subject: [PATCH 196/506] KVM: arm64: vgic-v5: Tidy-up programming of vpe descriptor address Assigning an address calculation to a vpe_entry is poor practice, and results in sparse moaning loudly. Tidy things up by computing the physical address as such, and performing the cpu_to_le64() on that. Whilst at it, simplify the computing of that address, avoiding the shifting back and forth, and kill the pointless GICV5_VPED_ADDR_SHIFT definition. Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5-tables.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index d7196ed5d1bf..2c8d0f360506 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -63,7 +63,6 @@ static DEFINE_XARRAY(vm_info); /* Virtual PE Table Entry */ #define GICV5_VPE_VALID BIT_ULL(0) /* Note that there is no shift for the address by design. */ -#define GICV5_VPED_ADDR_SHIFT 3ULL #define GICV5_VPED_ADDR GENMASK_ULL(55, 3) /* L2 Interrupt State Table Entry */ @@ -725,7 +724,7 @@ int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu) u32 vm_id = vgic_v5_vm_id(vcpu->kvm); u16 vpe_id = vgic_v5_vpe_id(vcpu); struct vgic_v5_vm_info *vmi; - vpe_entry tmp, *vpet_base; + vpe_entry *vpet_base; void *vped; /* Make sure we're not over what the hardware supports */ @@ -751,10 +750,9 @@ int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu) vped = (u8 *)vmi->vped_base + (size_t)vcpu->vcpu_idx * vmt_info->vped_size; - tmp = FIELD_PREP(GICV5_VPED_ADDR, virt_to_phys(vped) >> GICV5_VPED_ADDR_SHIFT); - scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { - WRITE_ONCE(vpet_base[vpe_id], cpu_to_le64(tmp)); + phys_addr_t paddr = virt_to_phys(vped) & GICV5_VPED_ADDR; + WRITE_ONCE(vpet_base[vpe_id], cpu_to_le64(paddr)); vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); } From 3b8ce2b5132ae4eebd81811ffd3bc2395c41ecfd Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Tue, 15 Sep 2026 16:42:59 +0100 Subject: [PATCH 197/506] KVM: arm64: Use a variable for the canonical IPA in kvm_s2_fault_map() Create a variable to store the canonical IPA, instead of calculating it when needed. This will be useful when we need to use the canonical IPA for guest stage-2 tracking later. Signed-off-by: Wei-Lin Chang Tested-by: Itaru Kitayama Tested-by: Wang Han Reviewed-by: Oliver Upton Tested-by: Shuai Xue Link: https://patch.msgid.link/20260915154305.3852871-2-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 2d44cd6a5aed..9372f57e2f85 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2030,6 +2030,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; bool writable = prot & KVM_PGTABLE_PROT_W; struct kvm *kvm = s2fd->vcpu->kvm; + phys_addr_t canonical_ipa; struct kvm_pgtable *pgt; long perm_fault_granule; long mapping_size; @@ -2048,6 +2049,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, mapping_size = s2vi->vma_pagesize; pfn = s2vi->pfn; gfn = s2vi->gfn; + canonical_ipa = gfn_to_gpa(get_canonical_gfn(s2fd, s2vi)); /* * If we are not forced to use page mapping, check if we are @@ -2066,6 +2068,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, goto out_unlock; } } + canonical_ipa = ALIGN_DOWN(canonical_ipa, mapping_size); } if (!perm_fault_granule && !s2vi->map_non_cacheable && kvm_has_mte(kvm)) @@ -2099,11 +2102,9 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, * making sure we adjust the canonical IPA if the mapping size has * been updated (via a THP upgrade, for example). */ - if (writable && !ret) { - phys_addr_t ipa = gfn_to_gpa(get_canonical_gfn(s2fd, s2vi)); - ipa &= ~(mapping_size - 1); - mark_page_dirty_in_slot(kvm, s2fd->memslot, gpa_to_gfn(ipa)); - } + if (writable && !ret) + mark_page_dirty_in_slot(kvm, s2fd->memslot, + gpa_to_gfn(canonical_ipa)); if (ret != -EAGAIN) return ret; From a6ca6cba59598de94eda90e4b3a4e405621aca90 Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Tue, 15 Sep 2026 16:43:00 +0100 Subject: [PATCH 198/506] KVM: arm64: nv: Introduce guest stage-2 tracking structures In order to avoid unmapping all shadow stage-2 mappings when KVM receives a MMU notifier unmap call, we have to keep track of the canonical IPA -> nested IPA relationship of the shadow mappings created. This essentially means tracking the guest's stage-2. To do this, represent each mapping by struct kvm_guest_s2_mapping. It stores the mapping's canonical IPA range and the nested IPA range using two interval tree nodes. Both nodes will be inserted into their respective interval trees called guest_s2_mappings. The canonical IPA ranges will be stored in the tree within the canonical MMU, and the nested IPA ranges will be stored in the corresponding nested MMU's tree. For example: struct kvm_guest_s2_mapping mapping1, mapping2; ---------------------> mapping2.canonical | mapping1.canonical | ^ (both stored in canonical mmu's tree) | | --*****-----------------------*****----------- CIPA \\\\\ ||||| mapping1.nested_mmu \\\\\ \\\\\ | \\\\\ \\\\\ v ------\\\\\---------------------*****--------- NIPA #1 (nested mmu #1) \\\\\ | \\\\\ -> mapping1.nested \\\\\ (stored in nested mmu #1's tree) \\\\\ -----------*****------------------------------ NIPA #2 (nested mmu #2) | ^ -> mapping2.nested | (stored in nested mmu #2's tree) mapping2.nested_mmu Using the trees we can look up nodes in either of the IPA spaces, and for each node, find the corresponding range in the other IPA space from the other node in the enclosing kvm_guest_s2_mapping. Define kvm_guest_s2_mapping and the interval tree here. Guest stage-2 mapping tracking will come in subsequent patches. Signed-off-by: Wei-Lin Chang Reviewed-by: Oliver Upton Tested-by: Shuai Xue Tested-by: Itaru Kitayama Link: https://patch.msgid.link/20260915154305.3852871-3-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 17 +++++++++++++++++ arch/arm64/kvm/mmu.c | 24 ++++++++++++++++++++++++ arch/arm64/kvm/nested.c | 1 + 3 files changed, 42 insertions(+) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index cd9b9d2462f9..365ec57d6d7a 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -150,6 +151,16 @@ struct kvm_vmid { atomic64_t id; }; +/* + * Record of a guest stage-2 mapping, storing canonical and nested IPA + * ranges. Both ranges have the same size. + */ +struct kvm_guest_s2_mapping { + struct interval_tree_node canonical; + struct interval_tree_node nested; + struct kvm_s2_mmu *nested_mmu; +}; + struct kvm_s2_mmu { struct kvm_vmid vmid; @@ -227,6 +238,9 @@ struct kvm_s2_mmu { */ bool pending_unmap; + /* Guest s2 mapping records indexed in this MMU's IPA space. */ + struct rb_root_cached guest_s2_mappings; + /* * 0: Nobody is currently using this, check vttbr for validity * >0: Somebody is actively using this. @@ -326,6 +340,9 @@ struct kvm_arch { size_t nested_mmus_size; int nested_mmus_next; + /* Guest s2 tracking trees access serialization. */ + spinlock_t guest_s2_tracking_lock; + /* Interrupt controller */ struct vgic_dist vgic; diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9372f57e2f85..4467f51547a4 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1042,6 +1043,8 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t mmu->pgd_phys = __pa(pgt->pgd); + mmu->guest_s2_mappings = RB_ROOT_CACHED; + if (kvm_is_nested_s2_mmu(kvm, mmu)) kvm_init_nested_s2_mmu(mmu); @@ -1131,10 +1134,25 @@ void stage2_unmap_vm(struct kvm *kvm) srcu_read_unlock(&kvm->srcu, idx); } +static void guest_s2_tracking_destroy(struct rb_root_cached *tree) +{ + struct kvm_guest_s2_mapping *mapping; + struct interval_tree_node *node; + + while ((node = interval_tree_iter_first(tree, 0, ULONG_MAX))) { + interval_tree_remove(node, tree); + mapping = container_of(node, struct kvm_guest_s2_mapping, + canonical); + kfree(mapping); + cond_resched(); + } +} + void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); struct kvm_pgtable *pgt = NULL; + struct rb_root_cached mappings_tree; write_lock(&kvm->mmu_lock); pgt = mmu->pgt; @@ -1147,12 +1165,18 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) if (kvm_is_nested_s2_mmu(kvm, mmu)) kvm_init_nested_s2_mmu(mmu); + mappings_tree = mmu->guest_s2_mappings; + mmu->guest_s2_mappings = RB_ROOT_CACHED; + write_unlock(&kvm->mmu_lock); if (pgt) { kvm_stage2_destroy(pgt); kfree(pgt); } + + if (!kvm_is_nested_s2_mmu(kvm, mmu)) + guest_s2_tracking_destroy(&mappings_tree); } static void hyp_mc_free_fn(void *addr, void *mc) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index b191365d97cc..81ba9c8532f4 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -52,6 +52,7 @@ int kvm_init_nested(struct kvm *kvm) GFP_KERNEL_ACCOUNT); kvm->arch.nested_mmus_size = 0; atomic_set(&kvm->arch.vncr_tlb_count, 0); + spin_lock_init(&kvm->arch.guest_s2_tracking_lock); return kvm->arch.nested_mmus ? 0 : -ENOMEM; } From 5491980c1a416139ea67faa6c4d52fa1ed0cd4e0 Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Tue, 15 Sep 2026 16:43:02 +0100 Subject: [PATCH 199/506] KVM: arm64: nv: Track guest stage-2 mapping removal kvm_stage2_unmap_range() is the helper to remove mappings from the stage-2 page tables. It is called during guest TLBI handling, memslot removal, nested mmu reuse, etc. Teach it about the guest stage-2 tracking trees and remove mappings from there when shadow mappings are removed. This keeps the tracking trees from having stale mappings pile up. Don't remove tracked mappings from the interval trees if they only partially overlap the removal range. For instance we can have a 4K shadow mapping tracked as a 2M range in the interval trees. It would be wrong to remove the tracked 2M range when a guest TLBI doesn't touch the 4K mapped. Signed-off-by: Wei-Lin Chang Reviewed-by: Oliver Upton Tested-by: Shuai Xue Tested-by: Itaru Kitayama Link: https://patch.msgid.link/20260915154305.3852871-5-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 5 ++++- arch/arm64/include/asm/kvm_nested.h | 2 ++ arch/arm64/kvm/mmu.c | 23 +++++++++++++++++-- arch/arm64/kvm/nested.c | 34 +++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 365ec57d6d7a..f12883a42081 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -340,7 +340,10 @@ struct kvm_arch { size_t nested_mmus_size; int nested_mmus_next; - /* Guest s2 tracking trees access serialization. */ + /* + * Serializes guest s2 tracking trees access when the mmu_lock + * is only held for read. + */ spinlock_t guest_s2_tracking_lock; /* Interrupt controller */ diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 2d71f686064a..954f532bc11b 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -81,6 +81,8 @@ extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, extern void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, gpa_t nested_ipa, size_t map_size, struct kvm_guest_s2_mapping *mapping); +extern void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, + gpa_t nipa, size_t size); extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index f8d3b7fe631f..d19ce58f7736 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -323,6 +323,19 @@ static void invalidate_icache_guest_page(void *va, size_t size) * we then fully enforce cacheability of RAM, no matter what the guest * does. */ + +static int kvm_pgtable_stage2_unmap_tracked(struct kvm_pgtable *pgt, u64 addr, u64 size) +{ + int ret; + + ret = kvm_pgtable_stage2_unmap(pgt, addr, size); + if (ret) + return ret; + + kvm_remove_guest_s2_mappings(pgt->mmu, addr, size); + return 0; +} + /** * __unmap_stage2_range -- Clear stage2 page table entries to unmap a range * @mmu: The KVM stage-2 MMU pointer @@ -340,11 +353,17 @@ static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); phys_addr_t end = start + size; + int (*fn)(struct kvm_pgtable *, u64, u64); lockdep_assert_held_write(&kvm->mmu_lock); WARN_ON(size & ~PAGE_MASK); - WARN_ON(stage2_apply_range(mmu, start, end, KVM_PGT_FN(kvm_pgtable_stage2_unmap), - may_block)); + + if (kvm_is_nested_s2_mmu(kvm, mmu)) + fn = kvm_pgtable_stage2_unmap_tracked; + else + fn = KVM_PGT_FN(kvm_pgtable_stage2_unmap); + + WARN_ON(stage2_apply_range(mmu, start, end, fn, may_block)); } void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start, diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index f5f0e364e56f..288254a798e7 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -897,6 +897,40 @@ void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, interval_tree_insert(&mapping->canonical, &kvm->arch.mmu.guest_s2_mappings); } +void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa, + size_t size) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + struct interval_tree_node *node, *next; + struct kvm_guest_s2_mapping *mapping; + gpa_t nipa_end = nipa + size - 1; + + /* + * Guest s2 tracking interval trees are only accessed while holding the + * mmu_lock, hence we don't have to take guest_s2_tracking_lock if the + * mmu_lock is held for write. + */ + lockdep_assert_held_write(&kvm->mmu_lock); + + node = interval_tree_iter_first(&mmu->guest_s2_mappings, nipa, nipa_end); + while (node) { + next = interval_tree_iter_next(node, nipa, nipa_end); + mapping = container_of(node, struct kvm_guest_s2_mapping, + nested); + /* + * Tracking must be conservative on removal, only remove + * mappings that are within the unmap range. + */ + if (nipa <= mapping->nested.start && nipa_end >= mapping->nested.last) { + interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); + interval_tree_remove(&mapping->canonical, + &kvm->arch.mmu.guest_s2_mappings); + kfree(mapping); + } + node = next; + } +} + void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) { /* CnP being set denotes an invalid entry */ From c2778d14cd2ff757224715221a2ddff89fc6f127 Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Tue, 15 Sep 2026 16:43:01 +0100 Subject: [PATCH 200/506] KVM: arm64: nv: Track guest stage-2 mapping creation During shadow stage-2 faults, in addition to creating mappings in the shadow page tables, also allocate kvm_guest_s2_mapping objects, record the mapping ranges, and insert them into the canonical and nested mmu's guest_s2_mappings tree. Note that because we allow parallel faulting, the interval trees could store mappings that are not live in the shadow page tables. Storing a superset of the live mappings is fine because we will only over-unmap when we use this information later to do the targeted MMU notifier unmap. The mapping is also added to the interval trees if kvm_pgtable_stage2_map() returns -EAGAIN. This is required for example, when a 2M block map (A) races with a 4K page map (B): 1. (A) maps the 2M block in kvm_pgtable_visitor_cb() 2. (B) breaks that block into a table and maps 4K 3. (A) reloads and finds the table after kvm_pgtable_visitor_cb(), then descends into it. 4. (A) maps some 4K, but before it finishes it reads the entry mapped by (B). 5. (A) returns -EAGAIN although it had mapped a few pages. In this case, we don't know what subrange is mapped, just track the whole requested mapping range. Signed-off-by: Wei-Lin Chang Reviewed-by: Oliver Upton Tested-by: Shuai Xue Tested-by: Itaru Kitayama Link: https://patch.msgid.link/20260915154305.3852871-4-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_nested.h | 3 +++ arch/arm64/kvm/mmu.c | 39 +++++++++++++++++++++++++++++ arch/arm64/kvm/nested.c | 25 ++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 586026e85903..2d71f686064a 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -78,6 +78,9 @@ extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, const union tlbi_info *info, void (*)(struct kvm_s2_mmu *, const union tlbi_info *)); +extern void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, + gpa_t nested_ipa, size_t map_size, + struct kvm_guest_s2_mapping *mapping); extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 4467f51547a4..f8d3b7fe631f 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1645,6 +1645,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt; + struct kvm_guest_s2_mapping *mapping = NULL; unsigned long mmu_seq; struct page *page; struct kvm *kvm = s2fd->vcpu->kvm; @@ -1658,6 +1659,11 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) ret = topup_mmu_memcache(s2fd->vcpu, memcache); if (ret) return ret; + if (kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + mapping = kmalloc_obj(struct kvm_guest_s2_mapping, GFP_KERNEL_ACCOUNT); + if (!mapping) + return -ENOMEM; + } } if (s2fd->nested) @@ -1678,6 +1684,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) if (ret) { kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, write_fault, exec_fault, false); + kfree(mapping); return ret; } @@ -1711,11 +1718,22 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE, __pfn_to_phys(pfn), prot, memcache, flags); + /* + * -EAGAIN from kvm_pgtable_stage2_map() can install mappings. + * We don't know which subrange is installed, track the whole + * thing. + */ + if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + kvm_record_guest_s2_mapping(pgt->mmu, gfn << PAGE_SHIFT, + s2fd->fault_ipa, PAGE_SIZE, mapping); + mapping = NULL; + } } out_unlock: kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W); kvm_fault_unlock(kvm); + kfree(mapping); if ((prot & KVM_PGTABLE_PROT_W) && !ret) mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn); @@ -2052,6 +2070,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, void *memcache) { enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; + struct kvm_guest_s2_mapping *mapping = NULL; bool writable = prot & KVM_PGTABLE_PROT_W; struct kvm *kvm = s2fd->vcpu->kvm; phys_addr_t canonical_ipa; @@ -2062,6 +2081,15 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, gfn_t gfn; int ret; + if (kvm_is_nested_s2_mmu(kvm, s2fd->vcpu->arch.hw_mmu)) { + mapping = kmalloc_obj(struct kvm_guest_s2_mapping, + GFP_KERNEL_ACCOUNT); + if (!mapping) { + kvm_release_page_unused(s2vi->page); + return -ENOMEM; + } + } + kvm_fault_lock(kvm); pgt = s2fd->vcpu->arch.hw_mmu->pgt; ret = -EAGAIN; @@ -2115,11 +2143,22 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(gfn), mapping_size, __pfn_to_phys(pfn), prot, memcache, flags); + /* + * -EAGAIN from kvm_pgtable_stage2_map() can install mappings. + * We don't know which subrange is installed, track the whole + * thing. + */ + if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + kvm_record_guest_s2_mapping(pgt->mmu, canonical_ipa, + gfn_to_gpa(gfn), mapping_size, mapping); + mapping = NULL; + } } out_unlock: kvm_release_faultin_page(kvm, s2vi->page, !!ret, writable); kvm_fault_unlock(kvm); + kfree(mapping); /* * Mark the page dirty only if the fault is handled successfully, diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 81ba9c8532f4..f5f0e364e56f 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -5,6 +5,7 @@ */ #include +#include #include #include @@ -872,6 +873,30 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) return s2_mmu; } +void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, + gpa_t nested_ipa, size_t map_size, + struct kvm_guest_s2_mapping *mapping) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + lockdep_assert_held_read(&kvm->mmu_lock); + + canonical_ipa = ALIGN_DOWN(canonical_ipa, map_size); + nested_ipa = ALIGN_DOWN(nested_ipa, map_size); + + mapping->canonical.start = canonical_ipa; + mapping->canonical.last = canonical_ipa + map_size - 1; + + mapping->nested.start = nested_ipa; + mapping->nested.last = nested_ipa + map_size - 1; + + mapping->nested_mmu = mmu; + + guard(spinlock)(&kvm->arch.guest_s2_tracking_lock); + interval_tree_insert(&mapping->nested, &mmu->guest_s2_mappings); + interval_tree_insert(&mapping->canonical, &kvm->arch.mmu.guest_s2_mappings); +} + void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) { /* CnP being set denotes an invalid entry */ From e6084aecaeefe9177280ea74c90127294260d805 Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Tue, 15 Sep 2026 16:43:03 +0100 Subject: [PATCH 201/506] KVM: arm64: nv: Avoid full shadow stage-2 unmap With guest stage-2 tracking in place, we can improve MMU notifier unmaps from unmapping all existing shadow stage-2 mappings to only unmapping the ones affected by the given canonical IPA range. Signed-off-by: Wei-Lin Chang Reviewed-by: Oliver Upton Tested-by: Shuai Xue Tested-by: Itaru Kitayama Link: https://patch.msgid.link/20260915154305.3852871-6-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_nested.h | 2 ++ arch/arm64/kvm/mmu.c | 7 ++-- arch/arm64/kvm/nested.c | 50 +++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 954f532bc11b..0f899addafa8 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -171,6 +171,8 @@ extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans); extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2); extern void kvm_nested_s2_wp(struct kvm *kvm); +extern void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, + size_t unmap_size, bool may_block); extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block); extern void kvm_nested_s2_flush(struct kvm *kvm); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index d19ce58f7736..0c426d4aded2 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2534,8 +2534,9 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) __unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT, (range->end - range->start) << PAGE_SHIFT, range->may_block); - - kvm_nested_s2_unmap(kvm, range->may_block); + kvm_nested_unmap_cipa_range(kvm, range->start << PAGE_SHIFT, + (range->end - range->start) << PAGE_SHIFT, + range->may_block); return false; } @@ -2817,7 +2818,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, write_lock(&kvm->mmu_lock); kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, size, true); - kvm_nested_s2_unmap(kvm, true); + kvm_nested_unmap_cipa_range(kvm, gpa, size, true); write_unlock(&kvm->mmu_lock); } diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 288254a798e7..c321f2f4328f 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -906,9 +906,8 @@ void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa, gpa_t nipa_end = nipa + size - 1; /* - * Guest s2 tracking interval trees are only accessed while holding the - * mmu_lock, hence we don't have to take guest_s2_tracking_lock if the - * mmu_lock is held for write. + * See kvm_nested_unmap_cipa_range() for why guest_s2_tracking_lock + * isn't taken here. */ lockdep_assert_held_write(&kvm->mmu_lock); @@ -1353,6 +1352,51 @@ void kvm_nested_s2_wp(struct kvm *kvm) kvm_invalidate_vncr_ipa_all(kvm); } +void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, size_t unmap_size, + bool may_block) +{ + gpa_t cipa_end = cipa + unmap_size - 1; + struct kvm_guest_s2_mapping *mapping; + struct interval_tree_node *node; + size_t mapping_size; + + /* + * Guest s2 tracking interval trees are only accessed while holding the + * mmu_lock, hence we don't have to take guest_s2_tracking_lock if the + * mmu_lock is held for write. This saves us from having to manually + * lock/unlock guest_s2_tracking_lock below around + * cond_resched_rwlock_write(). + */ + lockdep_assert_held_write(&kvm->mmu_lock); + + if (!kvm->arch.nested_mmus_size) + return; + + while ((node = interval_tree_iter_first(&kvm->arch.mmu.guest_s2_mappings, + cipa, cipa_end))) { + mapping = container_of(node, struct kvm_guest_s2_mapping, canonical); + mapping_size = mapping->nested.last - mapping->nested.start + 1; + + /* We could race against MMU teardown, which frees mmu->pgt. */ + if (mapping->nested_mmu->pgt) { + if (WARN_ON_ONCE(kvm_pgtable_stage2_unmap(mapping->nested_mmu->pgt, + mapping->nested.start, + mapping_size))) + return; + + interval_tree_remove(&mapping->nested, + &mapping->nested_mmu->guest_s2_mappings); + } + interval_tree_remove(node, &kvm->arch.mmu.guest_s2_mappings); + kfree(mapping); + + if (may_block) + cond_resched_rwlock_write(&kvm->mmu_lock); + } + + kvm_invalidate_vncr_ipa(kvm, cipa, cipa + unmap_size); +} + void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) { int i; From 2064f15b08485384d4fca892c63a1c1b4f86e9c8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 15 Sep 2026 16:43:04 +0100 Subject: [PATCH 202/506] KVM: arm64: nv: Drop kvm_s2_mmu pointer from kvm_guest_s2_mapping As it appears that the kvm_guest_s2_mapping structure is quite large, and results in a 128 byte slab allocation, there is some incentive to shrink a bit. For this, replace the S2 MMU back-pointer with an index tucked into the low bits of the nested.start field. This allows us to shrink the structure by 8 bytes, and therefore to fit in a 96 byte slab. The index is also stored in the S2 MMU structure itself, which comes for free as it fits in an existing hole. The structure is also repacked to avoid other disgracious holes. Signed-off-by: Wei-Lin Chang Reviewed-by: Oliver Upton Tested-by: Shuai Xue Tested-by: Itaru Kitayama Link: https://patch.msgid.link/20260915154305.3852871-7-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 19 ++++++----- arch/arm64/kvm/nested.c | 54 ++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index f12883a42081..8dfaa33c6cd5 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -153,12 +153,12 @@ struct kvm_vmid { /* * Record of a guest stage-2 mapping, storing canonical and nested IPA - * ranges. Both ranges have the same size. + * ranges. Both ranges have the same size. The lower bits of nested.start + * store the index of the nested mmu this mapping belongs to. */ struct kvm_guest_s2_mapping { struct interval_tree_node canonical; struct interval_tree_node nested; - struct kvm_s2_mmu *nested_mmu; }; struct kvm_s2_mmu { @@ -222,30 +222,33 @@ struct kvm_s2_mmu { u64 tlb_vttbr; u64 tlb_vtcr; + /* Guest s2 mapping records indexed in this MMU's IPA space. */ + struct rb_root_cached guest_s2_mappings; + /* * true when this represents a nested context where virtual * HCR_EL2.VM == 1 */ bool nested_stage2_enabled; -#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS - struct dentry *shadow_pt_debugfs_dentry; -#endif - /* * true when this MMU needs to be unmapped before being used for a new * purpose. */ bool pending_unmap; - /* Guest s2 mapping records indexed in this MMU's IPA space. */ - struct rb_root_cached guest_s2_mappings; + /* Index in the S2 MMU array, only valid for a shadow S2 */ + u16 s2_mmu_idx; /* * 0: Nobody is currently using this, check vttbr for validity * >0: Somebody is actively using this. */ atomic_t refcnt; + +#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS + struct dentry *shadow_pt_debugfs_dentry; +#endif }; struct kvm_arch_memory_slot { diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index c321f2f4328f..ec754a865a00 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -45,11 +45,12 @@ struct vncr_tlb { * will invalidate them more often). */ #define S2_MMU_PER_VCPU 2 +#define S2_MMU_PER_VM (KVM_MAX_VCPUS * S2_MMU_PER_VCPU) int kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *, - KVM_MAX_VCPUS * S2_MMU_PER_VCPU, + S2_MMU_PER_VM, GFP_KERNEL_ACCOUNT); kvm->arch.nested_mmus_size = 0; atomic_set(&kvm->arch.vncr_tlb_count, 0); @@ -128,8 +129,10 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) guard(write_lock)(&kvm->mmu_lock); - for (i = 0; i < S2_MMU_PER_VCPU; i++) + for (i = 0; i < S2_MMU_PER_VCPU; i++) { + tmp[i].s2_mmu_idx = i + kvm->arch.nested_mmus_size; kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i]; + } kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU; } @@ -873,6 +876,27 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) return s2_mmu; } +#define S2_MMU_IDX_MASK GENMASK_ULL(11, 0) + +static void tag_s2_mapping_mmu(struct kvm_guest_s2_mapping *mapping, + struct kvm_s2_mmu *mmu) +{ + BUILD_BUG_ON(S2_MMU_PER_VM > SZ_4K); + mapping->nested.start &= ~S2_MMU_IDX_MASK; + mapping->nested.start |= mmu->s2_mmu_idx; +} + +static struct kvm_s2_mmu *s2_mapping_to_mmu(struct kvm *kvm, + struct kvm_guest_s2_mapping *mapping) +{ + return kvm->arch.nested_mmus[mapping->nested.start & S2_MMU_IDX_MASK]; +} + +static unsigned long s2_mapping_to_nested_start(struct kvm_guest_s2_mapping *mapping) +{ + return mapping->nested.start & ~S2_MMU_IDX_MASK; +} + void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, gpa_t nested_ipa, size_t map_size, struct kvm_guest_s2_mapping *mapping) @@ -890,7 +914,7 @@ void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, mapping->nested.start = nested_ipa; mapping->nested.last = nested_ipa + map_size - 1; - mapping->nested_mmu = mmu; + tag_s2_mapping_mmu(mapping, mmu); guard(spinlock)(&kvm->arch.guest_s2_tracking_lock); interval_tree_insert(&mapping->nested, &mmu->guest_s2_mappings); @@ -913,6 +937,8 @@ void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa, node = interval_tree_iter_first(&mmu->guest_s2_mappings, nipa, nipa_end); while (node) { + unsigned long nested_start; + next = interval_tree_iter_next(node, nipa, nipa_end); mapping = container_of(node, struct kvm_guest_s2_mapping, nested); @@ -920,7 +946,8 @@ void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa, * Tracking must be conservative on removal, only remove * mappings that are within the unmap range. */ - if (nipa <= mapping->nested.start && nipa_end >= mapping->nested.last) { + nested_start = s2_mapping_to_nested_start(mapping); + if (nipa <= nested_start && nipa_end >= mapping->nested.last) { interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); interval_tree_remove(&mapping->canonical, &kvm->arch.mmu.guest_s2_mappings); @@ -1374,18 +1401,23 @@ void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, size_t unmap_size, while ((node = interval_tree_iter_first(&kvm->arch.mmu.guest_s2_mappings, cipa, cipa_end))) { - mapping = container_of(node, struct kvm_guest_s2_mapping, canonical); - mapping_size = mapping->nested.last - mapping->nested.start + 1; + unsigned long nested_start; + struct kvm_s2_mmu *mmu; + + mapping = container_of(node, struct kvm_guest_s2_mapping, + canonical); + nested_start = s2_mapping_to_nested_start(mapping); + mmu = s2_mapping_to_mmu(kvm, mapping); /* We could race against MMU teardown, which frees mmu->pgt. */ - if (mapping->nested_mmu->pgt) { - if (WARN_ON_ONCE(kvm_pgtable_stage2_unmap(mapping->nested_mmu->pgt, - mapping->nested.start, + if (mmu->pgt) { + mapping_size = mapping->nested.last - nested_start + 1; + + if (WARN_ON_ONCE(kvm_pgtable_stage2_unmap(mmu->pgt, nested_start, mapping_size))) return; - interval_tree_remove(&mapping->nested, - &mapping->nested_mmu->guest_s2_mappings); + interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); } interval_tree_remove(node, &kvm->arch.mmu.guest_s2_mappings); kfree(mapping); From 862fa568078514d0a41bd0946a65c23a0c0edec9 Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Tue, 15 Sep 2026 16:43:05 +0100 Subject: [PATCH 203/506] KVM: arm64: Refactor kvm_unmap_gfn_range() with common variables __unmap_stage2_range() and kvm_nested_unmap_cipa_range() are using the same arguments. Clean this up by using local variables. Signed-off-by: Wei-Lin Chang Tested-by: Itaru Kitayama Tested-by: Wang Han Reviewed-by: Oliver Upton Tested-by: Shuai Xue Link: https://patch.msgid.link/20260915154305.3852871-8-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 0c426d4aded2..d6187295c373 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2528,15 +2528,16 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { + gpa_t gpa = range->start << PAGE_SHIFT; + size_t size = (range->end - range->start) << PAGE_SHIFT; + bool may_block = range->may_block; + if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) return false; - __unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT, - (range->end - range->start) << PAGE_SHIFT, - range->may_block); - kvm_nested_unmap_cipa_range(kvm, range->start << PAGE_SHIFT, - (range->end - range->start) << PAGE_SHIFT, - range->may_block); + __unmap_stage2_range(&kvm->arch.mmu, gpa, size, may_block); + kvm_nested_unmap_cipa_range(kvm, gpa, size, may_block); + return false; } From 0ee2c883b62deb74d8c43007e8049dc006d3a2fb Mon Sep 17 00:00:00 2001 From: Shivank Garg Date: Tue, 1 Sep 2026 07:20:38 +0000 Subject: [PATCH 204/506] KVM: guest_memfd: take the invalidate lock when unbinding a dying file kvm_gmem_unbind() takes mapping->invalidate_lock only if the get_file_active() succeeds. When the guest_memfd file is dying, its reference count is already zero so get_file_active fails and the binding is removed without holding the invalidate lock. kvm_gmem_invalidate_{start,end}() checks f->bindings independently to decide whether to begin or end KVM MMU invalidation. So, the bindings must therefore remain stable between the two calls. Otherwise, unbind can remove a binding after start increments mmu_invalidate_in_progress, but before end finds the binding and decrements it. Example, unbind race with memory failure: CPU 0: memory failure CPU 1: memslot delete ---------------------------------- --------------------------- (guest_memfd file is dying) kvm_gmem_error_folio() kvm_gmem_invalidate_start() finds binding mmu_invalidate_in_progress++ kvm_gmem_unbind() get_file_active() fails removes binding kvm_gmem_invalidate_end() no binding found counter stays elevated mmu_invalidate_retry() then returns 1 forever, so guest page faults retry without ever installing a mapping and the guest hangs. Remove the distinction between live and dying files in the unbind path. Always use slot->gmem.file and take the invalidate lock when it's non-NULL. Normal unbind callers hold slots_lock, which prevents release from clearing the pointer or freeing the file state until unbind completes. Final VM teardown can only see a NULL pointer because guest_memfd pins the KVM until release has cleared all bindings. Fixes: ae431059e75d ("KVM: guest_memfd: Remove bindings on memslot deletion when gmem is dying") Cc: stable@vger.kernel.org Reported-by: Sashiko Closes: https://lore.kernel.org/all/20260728092027.225CF1F000E9@smtp.kernel.org Suggested-by: Sean Christopherson Signed-off-by: Shivank Garg Reviewed-by: David Hildenbrand (Arm) Link: https://patch.msgid.link/20260901-gmem-unbind-fix-v2-1-12febe1d2216@amd.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 54 ++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 625e62e1a031..21bb3710edfe 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -668,11 +668,33 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, return r; } -static void __kvm_gmem_unbind(struct kvm_memory_slot *slot, struct gmem_file *f) +void kvm_gmem_unbind(struct kvm_memory_slot *slot) { + struct file *file = slot->gmem.file; unsigned long start = slot->gmem.pgoff; unsigned long end = start + slot->npages; + struct gmem_file *f; + /* + * Nothing to do if the underlying file was _already_ closed, as + * kvm_gmem_release() invalidates and nullifies all bindings. + */ + if (!file) + return; + + /* + * However, if the file is _being_ closed, then the bindings need to be + * removed as kvm_gmem_release() might not run until after the memslot + * is freed. Modifying the bindings is safe even if the file is dying + * as kvm_gmem_release() nullifies slot->gmem.file under slots_lock, + * and only puts its reference to KVM after destroying all bindings. + * I.e. reaching this point means kvm_gmem_release() hasn't destroyed + * the bindings or freed the gmem_file and can't do so until the caller + * drops slots_lock, so there's no need to verify the file is live. + */ + f = file->private_data; + + filemap_invalidate_lock(file->f_mapping); xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); /* @@ -680,36 +702,6 @@ static void __kvm_gmem_unbind(struct kvm_memory_slot *slot, struct gmem_file *f) * cannot see this memslot. */ WRITE_ONCE(slot->gmem.file, NULL); -} - -void kvm_gmem_unbind(struct kvm_memory_slot *slot) -{ - /* - * Nothing to do if the underlying file was _already_ closed, as - * kvm_gmem_release() invalidates and nullifies all bindings. - */ - if (!slot->gmem.file) - return; - - CLASS(gmem_get_file, file)(slot); - - /* - * However, if the file is _being_ closed, then the bindings need to be - * removed as kvm_gmem_release() might not run until after the memslot - * is freed. Note, modifying the bindings is safe even though the file - * is dying as kvm_gmem_release() nullifies slot->gmem.file under - * slots_lock, and only puts its reference to KVM after destroying all - * bindings. I.e. reaching this point means kvm_gmem_release() hasn't - * yet destroyed the bindings or freed the gmem_file, and can't do so - * until the caller drops slots_lock. - */ - if (!file) { - __kvm_gmem_unbind(slot, slot->gmem.file->private_data); - return; - } - - filemap_invalidate_lock(file->f_mapping); - __kvm_gmem_unbind(slot, file->private_data); filemap_invalidate_unlock(file->f_mapping); } From d85f7dc8cf3c91a131946dae4c3aa8d9c97322d7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 16:56:55 -0700 Subject: [PATCH 205/506] KVM: VMX: Formally define KVM's "0xdead" exit reason used for failed VM-Enter Add EXIT_REASON_UNDEFINED to track KVM's magic "0xdead" value that's used to stuff the exit reason on a failed VM-Enter so that the magic can be reused by TDX, and to make it more obvious that it's an arbitrary, KVM- defined value (though the whole "dead" thing makes it pretty darn obvious). Opportunistically add a compile-time assert to ensure the KVM-defined exit reason never collides with a "real" exit reason. No functional change intended. Cc: Xiaoyao Li Cc: Yan Zhao Cc: Binbin Wu Cc: Rick Edgecombe Cc: stable@vger.kernel.org Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260921235656.1107929-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/vmx.h | 8 ++++++++ arch/x86/kvm/vmx/vmx.c | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 3f1b3096ff04..e49b836c7313 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -20,6 +20,14 @@ #include #include +/* + * UNDEFINED is a KVM-defined value used to indicate that the basic exit reason + * is undefined/invalid, e.g. on VM-Fail or if TDX never attempted VM-Enter. + * The value itself is arbitrary, it just needs to not collide with a real exit + * reason. + */ +#define EXIT_REASON_UNDEFINED 0xdead + struct vmcs_hdr { u32 revision_id:31; u32 shadow_vmcs:1; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..c897dbf0e954 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6388,6 +6388,8 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { static const int kvm_vmx_max_exit_handlers = ARRAY_SIZE(kvm_vmx_exit_handlers); +static_assert(EXIT_REASON_UNDEFINED >= ARRAY_SIZE(kvm_vmx_exit_handlers)); + void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { @@ -7474,7 +7476,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, vmx_enable_fb_clear(vmx); if (unlikely(vmx->fail)) { - vmx->vt.exit_reason.full = 0xdead; + vmx->vt.exit_reason.full = EXIT_REASON_UNDEFINED; goto out; } From f305fd31f52a8d3ce4d2bf0c88c45bb7a373809c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:46 -0700 Subject: [PATCH 206/506] KVM: VMX: Disallow using to_vmx() in common VT code Poison to_vmx() in main.c and posted_intr.c so that attempting to interpret the vCPU as a VMX vCPU will fail at compile time, as opposed to failing at runtime, or worse corrupting state without outright failing. Note, to_tdx() is buried in tdx.c, i.e. isn't broadly reachable, and so doesn't need the same treatment as to_vmx(). Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260904010353.3175819-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/main.c | 2 ++ arch/x86/kvm/vmx/posted_intr.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4c52ab8d0786..ea0d825e556d 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -10,6 +10,8 @@ #include "tdx.h" #include "tdx_arch.h" +#pragma GCC poison to_vmx + #ifdef CONFIG_KVM_INTEL_TDX static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 4a6d9a17da23..24221ba553be 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -14,6 +14,8 @@ #include "vmx.h" #include "tdx.h" +#pragma GCC poison to_vmx + /* * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler() * when a WAKEUP_VECTOR interrupted is posted. vCPUs are added to the list when From d93395058c1951bb657c8895dd1fc0f22263f1cc Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:55 +0800 Subject: [PATCH 207/506] KVM: TDX: Enable Notify VM exit Enable Notify VM exit functionality for TDX guests. Notify VM exit is an existing feature supported by KVM. Userspace can enable Notify VM exit through KVM_CAP_X86_NOTIFY_VMEXIT when it's reported as supported. However, KVM reports the support of this CAP just based on the hardware capability but doesn't differentiate between VMX and TDX. This leads to the issue that userspace can enable this cap for TDX guests without getting an error, but the feature is not actually enabled because KVM doesn't call the TDX module API to program the relevant TD VMCS fields. Enable Notify VM exit for TDX guests by: - Invoking TDX module API calls to set NOTIFY_VM_EXITING and Notify Window in TD VMCS. It's done in tdx_vcpu_init() where other TD VMCS bits are set. Since TDX vCPU cannot be reset, it only needs to be configured once when initializing the TDX vCPU. - Adding corresponding exit handler for TDX Notify VM Exit. Notify VM exit can happen when executing the IRET instruction. If the IRET unblocks the NMI blocking state, bit 12 of the exit qualification is set. In this case, the VMM needs to restore the "blocked by NMI" state when it decides to re-enter the guest. For TDX, KVM cannot manage the GUEST_INTERRUPTIBILITY_INFO and it's TDX module's responsibility to handle it. Extract the common part without NMI blocking handling into a helper in common.h so that it can be shared between VMX and TDX. Note, KVM uses "pre-production" terminology for the feature formally called Notify VM-Exit. All public versions of the SDM refer to the feature as Instruction Timeout. This will be remedied in the near future, for now, use KVM's terminology for consistency. Note, #2, there is no enumeration bit for Notify VM exit by TDX module because all TDX modules support it, and allow to set the corresponding TD VMCS fields as long as the hardware supports the feature. Fixes: 161d34609f9b ("KVM: TDX: Make TDX VM type supported") Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Reviewed-by: Rick Edgecombe Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260819094903.3060020-2-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 19 +++++++++++++++++++ arch/x86/kvm/vmx/tdx.c | 10 ++++++++++ arch/x86/kvm/vmx/vmx.c | 13 +------------ 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 08005676702c..7fce1bbabc78 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -4,6 +4,7 @@ #include #include +#include #include "mmu.h" @@ -183,6 +184,24 @@ static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); } +static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, + unsigned long exit_qual) +{ + bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; + + ++vcpu->stat.notify_window_exits; + + if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || + context_invalid) { + vcpu->run->exit_reason = KVM_EXIT_NOTIFY; + vcpu->run->notify.flags = context_invalid ? + KVM_NOTIFY_CONTEXT_INVALID : 0; + return 0; + } + + return 1; +} + noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..d557840687d2 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2126,6 +2126,9 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) * - If it's not an MSMI, no need to do anything here. */ return 1; + case EXIT_REASON_NOTIFY: + /* NMI blocking state is handled by TDX module */ + return __vt_handle_notify(vcpu, vmx_get_exit_qual(vcpu)); default: break; } @@ -3154,6 +3157,13 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd) td_vmcs_write64(tdx, POSTED_INTR_DESC_ADDR, __pa(&tdx->vt.pi_desc)); td_vmcs_setbit32(tdx, PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_POSTED_INTR); + if (kvm_notify_vmexit_enabled(vcpu->kvm)) { + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_NOTIFY_VM_EXITING); + td_vmcs_write32(tdx, NOTIFY_WINDOW, + vcpu->kvm->arch.notify_window); + } + tdx->state = VCPU_TD_STATE_INITIALIZED; return 0; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c897dbf0e954..965141e0d4c8 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6281,9 +6281,6 @@ static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) static int handle_notify(struct kvm_vcpu *vcpu) { unsigned long exit_qual = vmx_get_exit_qual(vcpu); - bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; - - ++vcpu->stat.notify_window_exits; /* * Notify VM exit happened while executing iret from NMI, @@ -6293,15 +6290,7 @@ static int handle_notify(struct kvm_vcpu *vcpu) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); - if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || - context_invalid) { - vcpu->run->exit_reason = KVM_EXIT_NOTIFY; - vcpu->run->notify.flags = context_invalid ? - KVM_NOTIFY_CONTEXT_INVALID : 0; - return 0; - } - - return 1; + return __vt_handle_notify(vcpu, exit_qual); } static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) From 0966c881ea13be5ec89820c185de4d55568e80ed Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:56 +0800 Subject: [PATCH 208/506] KVM: TDX: Check if there is valid exit infos based on vp_enter_ret Check if there is valid exit info based on vp_enter_ret instead of relying on the clobbered Exit Reason, in tdx_get_exit_info(). Current KVM uses "Exit Reason is not equal to the synthesized invalid Exit Reason, -1u," as the condition to identify there is a real TD Exit and valid exit infos. However, there is one issue with this approach: KVM updates the Exit Reason to the synthesized invalid Exit Reason for real EPT MISCONFIG as well. This is a false positive for real EPT MISCONFIG, which has valid exit infos. Though the issue can be addressed by changing the handling for real EPT MISCONFIG to not update the Exit Reason to the synthesized one, relying on the clobbered Exit Reason itself is brittle. Instead, check vp_enter_ret directly to identify if it is a valid Exit Reason. Fixes: da407fe45908 ("KVM: TDX: Handle EPT violation/misconfig exit") Cc: stable@vger.kernel.org Suggested-by: Sean Christopherson Signed-off-by: Xiaoyao Li Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260819094903.3060020-3-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index d557840687d2..1dead84e6077 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -921,21 +921,27 @@ static __always_inline u32 tdcall_to_vmx_exit_reason(struct kvm_vcpu *vcpu) return EXIT_REASON_TDCALL; } -static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret) { - struct vcpu_tdx *tdx = to_tdx(vcpu); - u32 exit_reason; - - switch (tdx->vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { + switch (vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { case TDX_SUCCESS: case TDX_NON_RECOVERABLE_VCPU: case TDX_NON_RECOVERABLE_TD: case TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE: case TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE: - break; + return true; default: - return -1u; + return false; } +} + +static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +{ + struct vcpu_tdx *tdx = to_tdx(vcpu); + u32 exit_reason; + + if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) + return -1u; exit_reason = tdx->vp_enter_ret; @@ -2144,7 +2150,7 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, struct vcpu_tdx *tdx = to_tdx(vcpu); *reason = tdx->vt.exit_reason.full; - if (*reason != -1u) { + if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) { *info1 = vmx_get_exit_qual(vcpu); *info2 = tdx->ext_exit_qualification; *intr_info = vmx_get_intr_info(vcpu); From b43c520b198ab70868307a489ef2117ed6d6c4cc Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:57 +0800 Subject: [PATCH 209/506] KVM: TDX: Set bits 31:16 to 0 and use UNDEFINED when synthesizing exit reason Set bits 31:16 to 0 instead of all-1s when synthesizing an exit reason, e.g. if the TDX Module never attempted VM-Enter, or when stuffing the exit reason due to an unexpected "real" EPT Misconfig. Opportunistically use VMX's KVM-defined EXIT_REASON_UNDEFINED (0xdead) instead of -1u. KVM is going to support Bus Lock VM exit for TDX, after which bit 26 of the Exit Reason becomes meaningful and indicates that a bus lock happened. The existing synthesized Exit Reason, -1u, will cause a false positive in that case. Change the synthesized Exit Reason from -1u to U16_MAX, so that bits 31:16 are set to 0. This also avoids the potential issues when other bits in 31:16 become valid in the future. As a bonus, the check for synthesized Exit Reason in tdx_failed_vmentry() becomes unnecessary. Just drop it. Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Reviewed-by: Rick Edgecombe Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260819094903.3060020-4-xiaoyao.li@intel.com [sean: use EXIT_REASON_UNDEFINED instead of another magic value] Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 1dead84e6077..2664123caf26 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -940,8 +940,13 @@ static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) struct vcpu_tdx *tdx = to_tdx(vcpu); u32 exit_reason; + /* + * Return the synthesized invalid Exit Reason, as the TDX module + * never attempted to run the vCPU, i.e. the Exit Reason is undefined, + * but this is NOT a failed VM-Enter. + */ if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) - return -1u; + return EXIT_REASON_UNDEFINED; exit_reason = tdx->vp_enter_ret; @@ -956,7 +961,7 @@ static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in * non-instrumentable code with interrupts disabled. */ - return -1u; + return EXIT_REASON_UNDEFINED; default: break; } @@ -987,8 +992,7 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu) { - return vmx_get_exit_reason(vcpu).failed_vmentry && - vmx_get_exit_reason(vcpu).full != -1u; + return vmx_get_exit_reason(vcpu).failed_vmentry; } static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) From bc335ce04408750585b4b32f248c543c9493d545 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:58 +0800 Subject: [PATCH 210/506] KVM: TDX: Don't assume exit_reason[31:16] is all-0 in tdx_to_vmx_exit_reason() When handling the real Exit Reason, don't assume the upper 16 bits as all-0 in tdx_to_vmx_exit_reason(), in preparation for enabling Bus Lock VM exit. When Bus Lock VM exit is enabled, the bit 26 of Exit Reason becomes valid and it can be 1 with various exit reasons. Change the logic in tdx_to_vmx_exit_reason() to check the 'basic' Exit Reason for correctness. Also preserve bit[31:16] when changing the (basic) Exit Reason, to not lose the information in bit[31:16]. Change the return type of tdx_to_vmx_exit_reason() to "union vmx_exit_reason" for the convenience of manipulating the basic field. Fixes: c42856af8f70 ("KVM: TDX: Add a place holder for handler of TDX hypercalls (TDG.VP.VMCALL)") Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Link: https://patch.msgid.link/20260819094903.3060020-5-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 2664123caf26..f4d74c02fa1e 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -935,10 +935,10 @@ static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret) } } -static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline union vmx_exit_reason tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) { struct vcpu_tdx *tdx = to_tdx(vcpu); - u32 exit_reason; + union vmx_exit_reason exit_reason; /* * Return the synthesized invalid Exit Reason, as the TDX module @@ -946,22 +946,26 @@ static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) * but this is NOT a failed VM-Enter. */ if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) - return EXIT_REASON_UNDEFINED; + return (union vmx_exit_reason) { + .basic = EXIT_REASON_UNDEFINED, + }; - exit_reason = tdx->vp_enter_ret; + exit_reason.full = (u32)tdx->vp_enter_ret; - switch (exit_reason) { + switch (exit_reason.basic) { case EXIT_REASON_TDCALL: if (tdvmcall_exit_type(vcpu)) - return EXIT_REASON_VMCALL; - - return tdcall_to_vmx_exit_reason(vcpu); + exit_reason.basic = EXIT_REASON_VMCALL; + else + exit_reason.basic = tdcall_to_vmx_exit_reason(vcpu); + break; case EXIT_REASON_EPT_MISCONFIG: /* * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in * non-instrumentable code with interrupts disabled. */ - return EXIT_REASON_UNDEFINED; + exit_reason.basic = EXIT_REASON_UNDEFINED; + break; default: break; } @@ -978,7 +982,7 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args); - vt->exit_reason.full = tdx_to_vmx_exit_reason(vcpu); + vt->exit_reason = tdx_to_vmx_exit_reason(vcpu); vt->exit_qualification = tdx->vp_enter_args.rcx; tdx->ext_exit_qualification = tdx->vp_enter_args.rdx; From e5f1d62c59aea4f270773de1bf6e8867f5b1124c Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:48:59 +0800 Subject: [PATCH 211/506] KVM: TDX: Stuff exit_reason with UNDEFINED on wait_for_sept_zap return Forcefully update the exit_reason to KVM's "standard" EXIT_REASON_UNDEFINED if tdx_vcpu_run() bails without trying to run the vCPU because KVM needs to wait for an in-progress S-EPT zap to complete. This avoids handling the previous exit multiple times due to a stale exit_reason. Fixes: 4b2abc49712b ("KVM: TDX: Kick off vCPUs when SEAMCALL is busy during TD page removal") Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Link: https://patch.msgid.link/20260819094903.3060020-6-xiaoyao.li@intel.com [sean: use EXIT_REASON_UNDEFINED, drop comment about bits 31:16] Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index f4d74c02fa1e..6762724a80f5 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1085,8 +1085,16 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * allowing vCPU entry to avoid contention with tdh_vp_enter() and * TDCALLs. */ - if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) + if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) { + /* + * The vCPU never entered the guest, but this looks like a + * handled exit to the caller. Synthesize an invalid exit + * reason so the previous exit's stale value isn't consumed + * a second time. + */ + vt->exit_reason.full = EXIT_REASON_UNDEFINED; return EXIT_FASTPATH_EXIT_HANDLED; + } trace_kvm_entry(vcpu, run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT); From cef48693da1e964e9f192bfc16cd0ed287c48337 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:49:00 +0800 Subject: [PATCH 212/506] KVM: VMX: Preserve negative return value in vmx_handle_exit() with bus lock detected Preserve the negative return value from __vmx_handle_exit() when a bus lock is detected, instead of always overwriting it with 0. The purpose of bus_lock_detected handling is to force a userspace exit to inform userspace that a bus lock happened. The negative return value can achieve this purpose, and changing the negative value to 0 fails to return an error to userspace. So, preserve the negative return value. Fixes: fe6b6bc802b4 ("KVM: VMX: Enable bus lock VM exit") Cc: stable@vger.kernel.org Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260805034602.5B2BB1F000E9@smtp.kernel.org/ Signed-off-by: Xiaoyao Li Link: https://patch.msgid.link/20260819094903.3060020-7-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 965141e0d4c8..6df390ae8708 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6864,11 +6864,12 @@ int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) * a bus lock in guest. */ if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) + if (ret > 0) { vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - return 0; } return ret; } From 326fa2c9b4d3e6cab58c139eb55aaabd7179192e Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:49:01 +0800 Subject: [PATCH 213/506] KVM: VMX: Make handle_bus_lock_vmexit() a shared helper Move handle_bus_lock_vmexit() from vmx.c to main.c to make it a shared helper so that TDX can use it for Bus Lock VM exit handling as well. Rename it to add vt_ prefix to reflect that it is a helper for both VMX and TDX. Cc: stable@vger.kernel.org Signed-off-by: Xiaoyao Li Reviewed-by: Nikolay Borisov Link: https://patch.msgid.link/20260819094903.3060020-8-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 1 + arch/x86/kvm/vmx/main.c | 11 +++++++++++ arch/x86/kvm/vmx/vmx.c | 13 +------------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 7fce1bbabc78..bbeec197a507 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -203,5 +203,6 @@ static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, } noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); +int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu); #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index ea0d825e556d..835de375e8c8 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -878,6 +878,17 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, #define vt_op_tdx_only(name) NULL #endif /* CONFIG_KVM_INTEL_TDX */ +int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) +{ + /* + * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK + * VM-Exits. Unconditionally set the flag here and leave the handling + * to .handle_exit() callback. + */ + to_vt(vcpu)->exit_reason.bus_lock_detected = true; + return 1; +} + #define VMX_REQUIRED_APICV_INHIBITS \ (BIT(APICV_INHIBIT_REASON_DISABLED) | \ BIT(APICV_INHIBIT_REASON_ABSENT) | \ diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 6df390ae8708..3ba1e762e7e7 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6267,17 +6267,6 @@ static int handle_encls(struct kvm_vcpu *vcpu) } #endif /* CONFIG_X86_SGX_KVM */ -static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) -{ - /* - * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK - * VM-Exits. Unconditionally set the flag here and leave the handling to - * vmx_handle_exit(). - */ - to_vt(vcpu)->exit_reason.bus_lock_detected = true; - return 1; -} - static int handle_notify(struct kvm_vcpu *vcpu) { unsigned long exit_qual = vmx_get_exit_qual(vcpu); @@ -6366,7 +6355,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { [EXIT_REASON_VMFUNC] = handle_vmx_instruction, [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, [EXIT_REASON_ENCLS] = handle_encls, - [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, + [EXIT_REASON_BUS_LOCK] = vt_handle_bus_lock_vmexit, [EXIT_REASON_NOTIFY] = handle_notify, [EXIT_REASON_SEAMCALL] = handle_tdx_instruction, [EXIT_REASON_TDCALL] = handle_tdx_instruction, From 63d196e2e20429de5ba501f1b6059d9b93140007 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:49:02 +0800 Subject: [PATCH 214/506] KVM: TDX: Enable Bus Lock VM exit Enable Bus Lock VM exit functionality for TDX guests. Bus Lock VM exit is an existing feature supported by KVM. Userspace can enable Bus Lock VM exit through KVM_BUS_LOCK_DETECTION_EXIT when it's reported as supported. However, KVM reports the support of this CAP just based on the hardware capability but doesn't differentiate between VMX and TDX. This leads to the issue that userspace can enable this cap for TDX guests without getting an error, but the feature is not actually enabled because KVM doesn't call the TDX module API to program the relevant TD VMCS fields. Enable Bus Lock VM exit for TDX guests by programming the BUS_LOCK_DETECTION control in the TD VMCS and by adding the exit handler. Note, there is no enumeration bit for this feature by TDX module because all TDX modules support it and allow to set the TD VMCS as long as the hardware supports the feature. Fixes: 161d34609f9b ("KVM: TDX: Make TDX VM type supported") Cc: stable@vger.kernel.org Originally-by: Chenyi Qiang Signed-off-by: Xiaoyao Li Reviewed-by: Nikolay Borisov Link: https://patch.msgid.link/20260819094903.3060020-9-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 6762724a80f5..3d093efe8820 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2050,7 +2050,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) } -int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) +static int __tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; @@ -2151,6 +2151,8 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) case EXIT_REASON_NOTIFY: /* NMI blocking state is handled by TDX module */ return __vt_handle_notify(vcpu, vmx_get_exit_qual(vcpu)); + case EXIT_REASON_BUS_LOCK: + return vt_handle_bus_lock_vmexit(vcpu); default: break; } @@ -2160,6 +2162,22 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) return 0; } +int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) +{ + int ret = __tdx_handle_exit(vcpu, fastpath); + + /* Exit to user space when bus lock was detected */ + if (vmx_get_exit_reason(vcpu).bus_lock_detected) { + if (ret > 0) { + vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } + + vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; + } + return ret; +} + void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { @@ -3186,6 +3204,10 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd) vcpu->kvm->arch.notify_window); } + if (vcpu->kvm->arch.bus_lock_detection_enabled) + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_BUS_LOCK_DETECTION); + tdx->state = VCPU_TD_STATE_INITIALIZED; return 0; From 213bbf18f07aed935e21df322fa61a84a19f9169 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 19 Aug 2026 17:49:03 +0800 Subject: [PATCH 215/506] KVM: VMX: Consolidate the exit handler for VMX and TDX The exit handlers for VMX and TDX have the similar pattern. Consolidate them into a single vt_handle_exit() helper. Changes in v4: - new patch. Signed-off-by: Xiaoyao Li Reviewed-by: Nikolay Borisov Link: https://patch.msgid.link/20260819094903.3060020-10-xiaoyao.li@intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/main.c | 38 ++++++++++++++++++++++++++++---------- arch/x86/kvm/vmx/tdx.c | 18 +----------------- arch/x86/kvm/vmx/vmx.c | 21 +-------------------- 3 files changed, 30 insertions(+), 47 deletions(-) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 835de375e8c8..fc7787b2bbb5 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -156,15 +156,6 @@ static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) return vmx_vcpu_run(vcpu, run_flags); } -static int vt_handle_exit(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion fastpath) -{ - if (is_td_vcpu(vcpu)) - return tdx_handle_exit(vcpu, fastpath); - - return vmx_handle_exit(vcpu, fastpath); -} - static bool vt_unhandleable_emulation_required(struct kvm_vcpu *vcpu) { if (is_td_vcpu(vcpu)) { @@ -889,6 +880,33 @@ int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) return 1; } +static int vt_handle_exit(struct kvm_vcpu *vcpu, + enum exit_fastpath_completion fastpath) +{ + int ret; + +#ifdef CONFIG_KVM_INTEL_TDX + if (is_td_vcpu(vcpu)) + ret = tdx_handle_exit(vcpu, fastpath); + else +#endif + ret = vmx_handle_exit(vcpu, fastpath); + + /* + * Exit to user space when bus lock detected to inform that there is + * a bus lock in guest. + */ + if (vmx_get_exit_reason(vcpu).bus_lock_detected) { + if (ret > 0) { + vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } + + vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; + } + return ret; +} + #define VMX_REQUIRED_APICV_INHIBITS \ (BIT(APICV_INHIBIT_REASON_DISABLED) | \ BIT(APICV_INHIBIT_REASON_ABSENT) | \ @@ -962,7 +980,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .vcpu_needs_initialization = vt_op_tdx_only(vcpu_needs_initialization), .vcpu_run = vt_op(vcpu_run), - .handle_exit = vt_op(handle_exit), + .handle_exit = vt_handle_exit, .skip_emulated_instruction = vmx_skip_emulated_instruction, .update_emulated_instruction = vmx_update_emulated_instruction, .unhandleable_emulation_required = vt_op(unhandleable_emulation_required), diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 3d093efe8820..8c0fdb032bb4 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2050,7 +2050,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) } -static int __tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) +int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; @@ -2162,22 +2162,6 @@ static int __tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) return 0; } -int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) -{ - int ret = __tdx_handle_exit(vcpu, fastpath); - - /* Exit to user space when bus lock was detected */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) { - vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; - ret = 0; - } - - vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - } - return ret; -} - void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3ba1e762e7e7..a9afffae951a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6685,7 +6685,7 @@ void dump_vmcs(struct kvm_vcpu *vcpu) * The guest has exited. See if we can fix it or if we need userspace * assistance. */ -static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) +int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_vmx *vmx = to_vmx(vcpu); union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); @@ -6844,25 +6844,6 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) return 0; } -int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) -{ - int ret = __vmx_handle_exit(vcpu, exit_fastpath); - - /* - * Exit to user space when bus lock detected to inform that there is - * a bus lock in guest. - */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) { - vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; - ret = 0; - } - - vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - } - return ret; -} - void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { int tpr_threshold; From f502a81663dd8520f83af75849803b168479be05 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:47 -0700 Subject: [PATCH 216/506] KVM: VMX: Move the shared "IRQs off" exit handler(s) to common code Move vmx_handle_exit_irqoff() and its helpers to common.h / main.c to capture that it's a common handler and to allow guarding against incorrectly using to_vmx(). Opportunistically use a "vt" prefix instead of "vmx" to communicate that it's a shared handler. No functional change intended. Cc: Rick Edgecombe Cc: Xiaoyao Li Cc: Binbin Wu Cc: Kai Huang Cc: Yan Zhao Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 6 ++++ arch/x86/kvm/vmx/main.c | 71 +++++++++++++++++++++++++++++++++++- arch/x86/kvm/vmx/vmx.c | 74 -------------------------------------- arch/x86/kvm/vmx/x86_ops.h | 1 - 4 files changed, 76 insertions(+), 76 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index bbeec197a507..88c525231e3a 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -75,6 +75,12 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; } #endif +static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_fpu.fpstate->xfd && + !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); +} + static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa) { /* For TDX the direct mask is the shared mask. */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index fc7787b2bbb5..87e9893d5ada 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include "x86_ops.h" @@ -880,6 +881,74 @@ int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) return 1; } +static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) +{ + /* + * Save xfd_err to guest_fpu before interrupt is enabled, so the + * MSR value is not clobbered by the host activity before the guest + * has chance to consume it. + * + * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM + * interception may have been caused by L1 interception. Per the SDM, + * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. + * + * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. + * unlike CR2 and DR6, the value is not a payload that is attached to + * the #NM exception. + */ + if (is_xfd_nm_fault(vcpu)) + rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); +} + +static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) +{ + /* if exit due to PF check for async PF */ + if (is_page_fault(intr_info)) + vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); + /* if exit due to NM, handle before interrupts are enabled */ + else if (is_nm_fault(intr_info)) + handle_nm_fault_irqoff(vcpu); + /* Handle machine checks before interrupts are enabled */ + else if (is_machine_check(intr_info)) + kvm_machine_check(); +} + +static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, + u32 intr_info) +{ + unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; + + if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, + "unexpected VM-Exit interrupt info: 0x%x", intr_info)) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); + x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); + kvm_after_interrupt(vcpu); + + vcpu->arch.at_instruction_boundary = true; +} + +static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu) +{ + if (to_vt(vcpu)->emulation_required) + return; + + switch (vmx_get_exit_reason(vcpu).basic) { + case EXIT_REASON_EXTERNAL_INTERRUPT: + handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); + break; + case EXIT_REASON_EXCEPTION_NMI: + handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); + break; + case EXIT_REASON_MCE_DURING_VMENTRY: + kvm_machine_check(); + break; + default: + break; + } +} + static int vt_handle_exit(struct kvm_vcpu *vcpu, enum exit_fastpath_completion fastpath) { @@ -1031,7 +1100,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .load_mmu_pgd = vt_op(load_mmu_pgd), .check_intercept = vmx_check_intercept, - .handle_exit_irqoff = vmx_handle_exit_irqoff, + .handle_exit_irqoff = vt_handle_exit_irqoff, .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a9afffae951a..4c444a3cae33 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5381,12 +5381,6 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); } -static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.guest_fpu.fpstate->xfd && - !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); -} - static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) { unsigned long cr2 = vmx_get_exit_qual(vcpu); @@ -7107,74 +7101,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); } -static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) -{ - /* - * Save xfd_err to guest_fpu before interrupt is enabled, so the - * MSR value is not clobbered by the host activity before the guest - * has chance to consume it. - * - * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM - * interception may have been caused by L1 interception. Per the SDM, - * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. - * - * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. - * unlike CR2 and DR6, the value is not a payload that is attached to - * the #NM exception. - */ - if (is_xfd_nm_fault(vcpu)) - rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); -} - -static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) -{ - /* if exit due to PF check for async PF */ - if (is_page_fault(intr_info)) - vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); - /* if exit due to NM, handle before interrupts are enabled */ - else if (is_nm_fault(intr_info)) - handle_nm_fault_irqoff(vcpu); - /* Handle machine checks before interrupts are enabled */ - else if (is_machine_check(intr_info)) - kvm_machine_check(); -} - -static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, - u32 intr_info) -{ - unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; - - if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, - "unexpected VM-Exit interrupt info: 0x%x", intr_info)) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); - x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); - kvm_after_interrupt(vcpu); - - vcpu->arch.at_instruction_boundary = true; -} - -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) -{ - if (to_vt(vcpu)->emulation_required) - return; - - switch (vmx_get_exit_reason(vcpu).basic) { - case EXIT_REASON_EXTERNAL_INTERRUPT: - handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_EXCEPTION_NMI: - handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_MCE_DURING_VMENTRY: - kvm_machine_check(); - break; - default: - break; - } -} - /* * The kvm parameter can be NULL (module initialization, or invocation before * VM creation). Be sure to check the kvm parameter before using it. diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 054fd14bb2e1..7ac02c68457f 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -27,7 +27,6 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void vmx_vcpu_put(struct kvm_vcpu *vcpu); int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath); -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu); int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu); void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu); bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu); From a06210034254b6a5800ff0a8c1b69739adf16014 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 09:59:15 -0700 Subject: [PATCH 217/506] KVM: VMX: Move the shared NMI handler/trampoline to common code Move vmx_handle_nmi() to main.c to capture that it's a common handler and to allow guarding against incorrectly using to_vmx(). Opportunistically use a "vt" prefix instead of "vmx" to communicate that it's a shared handler, and move the declaration higher up in common.h so that it doesn't get buried in a pile of inline helpers (more are on the way). No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 3 ++- arch/x86/kvm/vmx/main.c | 11 +++++++++++ arch/x86/kvm/vmx/tdx.c | 2 +- arch/x86/kvm/vmx/vmx.c | 13 +------------ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 88c525231e3a..dbc1bca47ed6 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -56,6 +56,8 @@ struct vcpu_vt { #endif }; +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu); + #ifdef CONFIG_KVM_INTEL_TDX static __always_inline bool is_td(struct kvm *kvm) @@ -208,7 +210,6 @@ static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, return 1; } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu); #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 87e9893d5ada..0f2f9532e048 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -881,6 +881,17 @@ int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) return 1; } +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) +{ + if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || + !is_nmi(vmx_get_intr_info(vcpu))) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); + x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); + kvm_after_interrupt(vcpu); +} + static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) { /* diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 8c0fdb032bb4..ed1bd4e37026 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -989,7 +989,7 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) tdx->exit_gpa = tdx->vp_enter_args.r8; vt->exit_intr_info = tdx->vp_enter_args.r9; - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); guest_state_exit_irqoff(); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 4c444a3cae33..7b867e273e59 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7327,17 +7327,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, } } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu) -{ - if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || - !is_nmi(vmx_get_intr_info(vcpu))) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); - x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); - kvm_after_interrupt(vcpu); -} - static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, unsigned int flags) { @@ -7370,7 +7359,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry)) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); out: guest_state_exit_irqoff(); From 8f0140d8c777f0fbe89cc59ac7d3ca1f47da9f49 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:49 -0700 Subject: [PATCH 218/506] KVM: VMX: Rename posted interrupt prefixes from "vmx" to "vt" Rename all common posted interrupt APIs that use a "vmx" prefix to instead use a "vt" prefix to capture that they are used for both VMX and TDX vCPUs. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 4 ++-- arch/x86/kvm/vmx/main.c | 4 ++-- arch/x86/kvm/vmx/posted_intr.c | 22 +++++++++++----------- arch/x86/kvm/vmx/posted_intr.h | 8 ++++---- arch/x86/kvm/vmx/tdx.c | 6 +++--- arch/x86/kvm/vmx/vmx.c | 6 +++--- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index dbc1bca47ed6..f19fe019ee8a 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -173,8 +173,8 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, * Post an interrupt to a vCPU's PIR and trigger the vCPU to process the * interrupt if necessary. */ -static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, - struct pi_desc *pi_desc, int vector) +static inline void __vt_deliver_posted_interrupt(struct kvm_vcpu *vcpu, + struct pi_desc *pi_desc, int vector) { if (pi_test_and_set_pir(vector, pi_desc)) return; diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 0f2f9532e048..4e342fbd4d67 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -1115,8 +1115,8 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), - .pi_update_irte = vmx_pi_update_irte, - .pi_start_bypass = vmx_pi_start_bypass, + .pi_update_irte = vt_pi_update_irte, + .pi_start_bypass = vt_pi_start_bypass, #ifdef CONFIG_X86_64 .set_hv_timer = vt_op(set_hv_timer), diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 24221ba553be..2644add54b76 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -56,7 +56,7 @@ static int pi_try_set_control(struct pi_desc *pi_desc, u64 *pold, u64 new) return 0; } -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); struct vcpu_vt *vt = to_vt(vcpu); @@ -146,11 +146,11 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) pi_set_on(pi_desc); } -static bool vmx_can_use_vtd_pi(struct kvm *kvm) +static bool vt_can_use_vtd_pi(struct kvm *kvm) { /* * Note, reading the number of possible bypass IRQs can race with a - * bypass IRQ being attached to the VM. vmx_pi_start_bypass() ensures + * bypass IRQ being attached to the VM. vt_pi_start_bypass() ensures * blockng vCPUs will see an elevated count or get KVM_REQ_UNBLOCK. */ return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() && @@ -208,7 +208,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) __apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); } -static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) +static bool vt_needs_pi_wakeup(struct kvm_vcpu *vcpu) { /* * The default posted interrupt vector does nothing when @@ -219,14 +219,14 @@ static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) * back to the pi_wakeup_handler() function. */ return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) || - vmx_can_use_vtd_pi(vcpu->kvm); + vt_can_use_vtd_pi(vcpu->kvm); } -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); - if (!vmx_needs_pi_wakeup(vcpu)) + if (!vt_needs_pi_wakeup(vcpu)) return; /* @@ -294,17 +294,17 @@ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) /* * Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as * blocking vCPUs may scheduled out without reconfiguring PID.NV to the wakeup - * vector, i.e. if the bypass IRQ came along after vmx_vcpu_pi_put(). + * vector, i.e. if the bypass IRQ came along after vt_vcpu_pi_put(). */ -void vmx_pi_start_bypass(struct kvm *kvm) +void vt_pi_start_bypass(struct kvm *kvm) { - if (WARN_ON_ONCE(!vmx_can_use_vtd_pi(kvm))) + if (WARN_ON_ONCE(!vt_can_use_vtd_pi(kvm))) return; kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK); } -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector) { diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h index a4af39948cf0..c5860ea828ae 100644 --- a/arch/x86/kvm/vmx/posted_intr.h +++ b/arch/x86/kvm/vmx/posted_intr.h @@ -8,16 +8,16 @@ #include -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu); +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu); void pi_wakeup_handler(void); void __init pi_init_cpu(int cpu); void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu); bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu); -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector); -void vmx_pi_start_bypass(struct kvm *kvm); +void vt_pi_start_bypass(struct kvm *kvm); static inline int pi_find_highest_vector(struct pi_desc *pi_desc) { diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index ed1bd4e37026..3a685c4ce332 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -716,7 +716,7 @@ void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_tdx *tdx = to_tdx(vcpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); if (vcpu->cpu == cpu || !is_hkid_assigned(to_kvm_tdx(vcpu->kvm))) return; @@ -826,7 +826,7 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) void tdx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); tdx_prepare_switch_to_host(vcpu); } @@ -1917,7 +1917,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, struct vcpu_tdx *tdx = to_tdx(vcpu); /* TDX supports only posted interrupt. No lapic emulation. */ - __vmx_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 7b867e273e59..e99b5184518a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1577,12 +1577,12 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load_vmcs(vcpu, cpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); } void vmx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); vmx_prepare_switch_to_host(to_vmx(vcpu)); } @@ -4426,7 +4426,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) if (!vcpu->arch.apic->apicv_active) return -1; - __vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); return 0; } From 3d2ad6cfb6e1ef5fc6f3e86aa8b0f2e4bcab1b02 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:50 -0700 Subject: [PATCH 219/506] KVM: VMX: Rename EPT violation handler prefix from "vmx" to "vt" Rename __vmx_handle_ept_violation() to __vt_handle_ept_violation to capture that it is shared between VMX and TDX. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 4 ++-- arch/x86/kvm/vmx/tdx.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index f19fe019ee8a..4af85ec70989 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -89,8 +89,8 @@ static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa) return !kvm_is_addr_direct(kvm, gpa); } -static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, - unsigned long exit_qualification) +static inline int __vt_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, + unsigned long exit_qualification) { u64 error_code; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 3a685c4ce332..76bcf2fabb83 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2009,7 +2009,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) while (1) { struct kvm_memory_slot *slot; - ret = __vmx_handle_ept_violation(vcpu, gpa, exit_qual); + ret = __vt_handle_ept_violation(vcpu, gpa, exit_qual); if (ret != RET_PF_RETRY || !local_retry) break; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index e99b5184518a..d0a617797661 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5993,7 +5993,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) return kvm_emulate_instruction(vcpu, 0); - return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification); + return __vt_handle_ept_violation(vcpu, gpa, exit_qualification); } static int handle_ept_misconfig(struct kvm_vcpu *vcpu) From 978540f908addea83672e66b3af3461a8ddfc744 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:51 -0700 Subject: [PATCH 220/506] KVM: VMX: Use dummy pseudo-overlay struct for to_vt() and vt_to_vcpu() Define a combined vcpu_vmx_tdx structure to represent the common layout of vcpu_vmx and vcpu_tdx, instead of having vcpu_tdx piggyback vcpu_vmx. I.e. require VMX and TDX to match the common definition, not for TDX to match VMX. This will allow moving to_vt() and vt_to_vcpu() to common.h where they belong. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/main.c | 4 +++- arch/x86/kvm/vmx/vmx.h | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4e342fbd4d67..9b304f420d4c 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -13,8 +13,10 @@ #pragma GCC poison to_vmx +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_vmx, vt)); + #ifdef CONFIG_KVM_INTEL_TDX -static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_tdx, vt)); static void vt_disable_virtualization_cpu(void) { diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..0415d328e1a5 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -304,14 +304,19 @@ struct kvm_vmx { u64 *pid_table; }; +struct vcpu_vmx_tdx { + struct kvm_vcpu vcpu; + struct vcpu_vt vt; +}; + static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) { - return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt); + return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); } static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) { - return &(container_of(vt, struct vcpu_vmx, vt)->vcpu); + return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); } static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) From 82e11cefb7464d402ee3323c0dbff0b44abf7509 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:52 -0700 Subject: [PATCH 221/506] KVM: VMX: Move common VT getters/converters to common.h Move the "to" vCPU converters and common exit information getters from vmx.h to common.h to capture that they are indeed common VT code. Defer renaming their prefixes to isolate the code movement from the massive churn that will result from said renaming. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 44 +++++++++++++++++++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.h | 42 ------------------------------------- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 4af85ec70989..4e00832f132a 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -2,11 +2,13 @@ #ifndef __KVM_X86_VMX_COMMON_H #define __KVM_X86_VMX_COMMON_H +#include #include #include #include #include "mmu.h" +#include "vmx_ops.h" union vmx_exit_reason { struct { @@ -77,6 +79,48 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; } #endif +struct vcpu_vmx_tdx { + struct kvm_vcpu vcpu; + struct vcpu_vt vt; +}; + +static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) +{ + return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); +} + +static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) +{ + return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); +} + +static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) +{ + return to_vt(vcpu)->exit_reason; +} + +static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + + return vt->exit_qualification; +} + +static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + + return vt->exit_intr_info; +} + static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) { return vcpu->arch.guest_fpu.fpstate->xfd && diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 0415d328e1a5..3247b55b3484 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -304,48 +304,6 @@ struct kvm_vmx { u64 *pid_table; }; -struct vcpu_vmx_tdx { - struct kvm_vcpu vcpu; - struct vcpu_vt vt; -}; - -static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) -{ - return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); -} - -static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) -{ - return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); -} - -static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) -{ - return to_vt(vcpu)->exit_reason; -} - -static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - - return vt->exit_qualification; -} - -static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - - return vt->exit_intr_info; -} - void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu); int allocate_vpid(void); void free_vpid(int vpid); From 72dea4f43c78207a6828fc1b2deb7e3727342807 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Sep 2026 18:03:53 -0700 Subject: [PATCH 222/506] KVM: VMX: Rename common exit info getters prefixes from "vmx" to "vt" Rename all common prefixes for the APIs that get exit information common to VMX and TDX from "vmx" to "vt" to capture that they are used for both VMX and TDX vCPUs. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Kai Huang Tested-by: Kai Huang Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260904010353.3175819-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 6 ++-- arch/x86/kvm/vmx/main.c | 12 ++++---- arch/x86/kvm/vmx/nested.c | 36 ++++++++++++------------ arch/x86/kvm/vmx/tdx.c | 22 +++++++-------- arch/x86/kvm/vmx/vmx.c | 58 +++++++++++++++++++-------------------- 5 files changed, 67 insertions(+), 67 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 4e00832f132a..e22c159c3b84 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -94,12 +94,12 @@ static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); } -static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline union vmx_exit_reason vt_get_exit_reason(struct kvm_vcpu *vcpu) { return to_vt(vcpu)->exit_reason; } -static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) +static __always_inline unsigned long vt_get_exit_qual(struct kvm_vcpu *vcpu) { struct vcpu_vt *vt = to_vt(vcpu); @@ -110,7 +110,7 @@ static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) return vt->exit_qualification; } -static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) +static __always_inline u32 vt_get_intr_info(struct kvm_vcpu *vcpu) { struct vcpu_vt *vt = to_vt(vcpu); diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 9b304f420d4c..c11732cfadf0 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -885,8 +885,8 @@ int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) { - if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || - !is_nmi(vmx_get_intr_info(vcpu))) + if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || + !is_nmi(vt_get_intr_info(vcpu))) return; kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); @@ -947,12 +947,12 @@ static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu) if (to_vt(vcpu)->emulation_required) return; - switch (vmx_get_exit_reason(vcpu).basic) { + switch (vt_get_exit_reason(vcpu).basic) { case EXIT_REASON_EXTERNAL_INTERRUPT: - handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); + handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu)); break; case EXIT_REASON_EXCEPTION_NMI: - handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); + handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu)); break; case EXIT_REASON_MCE_DURING_VMENTRY: kvm_machine_check(); @@ -978,7 +978,7 @@ static int vt_handle_exit(struct kvm_vcpu *vcpu, * Exit to user space when bus lock detected to inform that there is * a bus lock in guest. */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { + if (vt_get_exit_reason(vcpu).bus_lock_detected) { if (ret > 0) { vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; ret = 0; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 151873407abd..47599e7312cf 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -446,7 +446,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * "NMI unblocking due to IRET", i.e. the bit can be propagated * as-is from the original EXIT_QUALIFICATION. */ - exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; + exit_qualification = vt_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; } else { if (fault->error_code & PFERR_RSVD_MASK) { vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; @@ -472,7 +472,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * and walks the faulting GPA. */ if (from_hardware) - exit_qualification |= vmx_get_exit_qual(vcpu) & mask; + exit_qualification |= vt_get_exit_qual(vcpu) & mask; else exit_qualification |= fault->exit_qualification & mask; @@ -4747,7 +4747,7 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, { /* update exit information fields: */ vmcs12->vm_exit_reason = vm_exit_reason; - if (vmx_get_exit_reason(vcpu).enclave_mode) + if (vt_get_exit_reason(vcpu).enclave_mode) vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; vmcs12->exit_qualification = exit_qualification; @@ -5373,7 +5373,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, struct x86_exception e; int r; - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmcs_read32(VMX_INSTRUCTION_INFO), false, sizeof(*vmpointer), &gva)) { *ret = 1; @@ -5665,7 +5665,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5771,7 +5771,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5960,7 +5960,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) /* Emulate the VMPTRST instruction */ static int handle_vmptrst(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); + unsigned long exit_qual = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; struct x86_exception e; @@ -6021,7 +6021,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) /* According to the Intel VMX instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6104,7 +6104,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) /* according to the intel vmx instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6236,8 +6236,8 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) * EXIT_REASON_VMFUNC as the exit reason. */ nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full, - vmx_get_intr_info(vcpu), - vmx_get_exit_qual(vcpu)); + vt_get_intr_info(vcpu), + vt_get_exit_qual(vcpu)); return 1; } @@ -6288,7 +6288,7 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; @@ -6314,7 +6314,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM || exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) - msr_index = vmx_get_exit_qual(vcpu); + msr_index = vt_get_exit_qual(vcpu); else msr_index = kvm_ecx_read(vcpu); @@ -6350,7 +6350,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int cr = exit_qualification & 15; int reg; unsigned long val; @@ -6484,7 +6484,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6567,7 +6567,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6737,14 +6737,14 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) * need to be synthesized by querying the in-kernel LAPIC, but external * interrupts are never reflected to L1 so it's a non-issue. */ - exit_intr_info = vmx_get_intr_info(vcpu); + exit_intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(exit_intr_info)) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); vmcs12->vm_exit_intr_error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); } - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); reflect_vmexit: nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 76bcf2fabb83..53fd4e63ce40 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -741,7 +741,7 @@ bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu) * interrupt is always allowed unless TDX guest calls TDVMCALL with HLT, * which passes the interrupt blocked flag. */ - return vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + return vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || !to_tdx(vcpu)->vp_enter_args.r12; } @@ -759,7 +759,7 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu) * otherwise the interrupt would have been serviced at the instruction * boundary. */ - if (vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + if (vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || to_tdx(vcpu)->vp_enter_args.r12) return false; @@ -996,7 +996,7 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu) { - return vmx_get_exit_reason(vcpu).failed_vmentry; + return vt_get_exit_reason(vcpu).failed_vmentry; } static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) @@ -1153,7 +1153,7 @@ void tdx_inject_nmi(struct kvm_vcpu *vcpu) static int tdx_handle_exception_nmi(struct kvm_vcpu *vcpu) { - u32 intr_info = vmx_get_intr_info(vcpu); + u32 intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -1925,7 +1925,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcpu) { u64 eeq_type = to_tdx(vcpu)->ext_exit_qualification & TDX_EXT_EXIT_QUAL_TYPE_MASK; - u64 eq = vmx_get_exit_qual(vcpu); + u64 eq = vt_get_exit_qual(vcpu); if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION) return false; @@ -1961,7 +1961,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) /* Only private GPA triggers zero-step mitigation */ local_retry = true; } else { - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); /* * EPT violation due to instruction fetch should never be * triggered from shared memory in TDX guest. If such EPT @@ -2043,7 +2043,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) return 1; } - if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) + if (vt_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) tdvmcall_set_return_val(vcpu, kvm_read_edx_eax(vcpu)); return 1; @@ -2054,7 +2054,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); if (fastpath != EXIT_FASTPATH_NONE) return 1; @@ -2150,7 +2150,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) return 1; case EXIT_REASON_NOTIFY: /* NMI blocking state is handled by TDX module */ - return __vt_handle_notify(vcpu, vmx_get_exit_qual(vcpu)); + return __vt_handle_notify(vcpu, vt_get_exit_qual(vcpu)); case EXIT_REASON_BUS_LOCK: return vt_handle_bus_lock_vmexit(vcpu); default: @@ -2169,9 +2169,9 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, *reason = tdx->vt.exit_reason.full; if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) { - *info1 = vmx_get_exit_qual(vcpu); + *info1 = vt_get_exit_qual(vcpu); *info2 = tdx->ext_exit_qualification; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); } else { *info1 = 0; *info2 = 0; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d0a617797661..21d2d96fd969 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1791,7 +1791,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, * so that guest userspace can't DoS the guest simply by triggering * emulation (enclaves are CPL3 only). */ - if (vmx_get_exit_reason(vcpu).enclave_mode) { + if (vt_get_exit_reason(vcpu).enclave_mode) { kvm_queue_exception(vcpu, UD_VECTOR); return X86EMUL_PROPAGATE_FAULT; } @@ -1806,7 +1806,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, static int skip_emulated_instruction(struct kvm_vcpu *vcpu) { - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); unsigned long rip, orig_rip; u32 instr_len; @@ -5383,7 +5383,7 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) { - unsigned long cr2 = vmx_get_exit_qual(vcpu); + unsigned long cr2 = vt_get_exit_qual(vcpu); if (vcpu->arch.apf.host_apf_flags) goto handle_pf; @@ -5431,7 +5431,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) u32 vect_info; vect_info = vmx->idt_vectoring_info; - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -5512,7 +5512,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) switch (ex_no) { case DB_VECTOR: - dr6 = vmx_get_exit_qual(vcpu); + dr6 = vt_get_exit_qual(vcpu); if (!(vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { /* @@ -5588,7 +5588,7 @@ static int handle_io(struct kvm_vcpu *vcpu) int size, in, string; unsigned port; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); string = (exit_qualification & 16) != 0; ++vcpu->stat.io_exits; @@ -5677,7 +5677,7 @@ static int handle_cr(struct kvm_vcpu *vcpu) int err; int ret; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); cr = exit_qualification & 15; reg = (exit_qualification >> 8) & 15; switch ((exit_qualification >> 4) & 3) { @@ -5755,7 +5755,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) int dr, dr7, reg; int err = 1; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); dr = exit_qualification & DEBUG_REG_ACCESS_NUM; /* First, if DR does not exist, trigger UD */ @@ -5851,7 +5851,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu) static int handle_invlpg(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); kvm_mmu_invlpg(vcpu, exit_qualification); return kvm_skip_emulated_instruction(vcpu); @@ -5860,7 +5860,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu) static int handle_apic_access(struct kvm_vcpu *vcpu) { if (likely(fasteoi)) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int access_type, offset; access_type = exit_qualification & APIC_ACCESS_TYPE; @@ -5881,7 +5881,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int vector = exit_qualification & 0xff; /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ @@ -5891,7 +5891,7 @@ static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) static int handle_apic_write(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); /* * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and @@ -5919,7 +5919,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); reason = (u32)exit_qualification >> 30; if (reason == TASK_SWITCH_GATE && idt_v) { @@ -5965,7 +5965,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) static int handle_ept_violation(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); gpa_t gpa; /* @@ -6156,7 +6156,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) /* According to the Intel instruction reference, the memory operand * is read even if it isn't needed (e.g., for type==all) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; @@ -6170,7 +6170,7 @@ static int handle_pml_full(struct kvm_vcpu *vcpu) trace_kvm_pml_full(vcpu->vcpu_id); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); /* * PML buffer FULL happened while executing iret from NMI, @@ -6263,7 +6263,7 @@ static int handle_encls(struct kvm_vcpu *vcpu) static int handle_notify(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); + unsigned long exit_qual = vt_get_exit_qual(vcpu); /* * Notify VM exit happened while executing iret from NMI, @@ -6283,13 +6283,13 @@ static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) static int handle_rdmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_rdmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_rdmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } static int handle_wrmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } @@ -6368,10 +6368,10 @@ void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, struct vcpu_vmx *vmx = to_vmx(vcpu); *reason = vmx->vt.exit_reason.full; - *info1 = vmx_get_exit_qual(vcpu); + *info1 = vt_get_exit_qual(vcpu); if (!(vmx->vt.exit_reason.failed_vmentry)) { *info2 = vmx->idt_vectoring_info; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(*intr_info)) *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); else @@ -6682,7 +6682,7 @@ void dump_vmcs(struct kvm_vcpu *vcpu) int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_vmx *vmx = to_vmx(vcpu); - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); u32 vectoring_info = vmx->idt_vectoring_info; u16 exit_handler_index; @@ -7140,7 +7140,7 @@ static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) if (vmx->loaded_vmcs->nmi_known_unmasked) return; - exit_intr_info = vmx_get_intr_info(&vmx->vcpu); + exit_intr_info = vt_get_intr_info(&vmx->vcpu); unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; vector = exit_intr_info & INTR_INFO_VECTOR_MASK; /* @@ -7307,14 +7307,14 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, * the fastpath even, all other exits must use the slow path. */ if (is_guest_mode(vcpu) && - vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) + vt_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) return EXIT_FASTPATH_NONE; - switch (vmx_get_exit_reason(vcpu).basic) { + switch (vt_get_exit_reason(vcpu).basic) { case EXIT_REASON_MSR_WRITE: return handle_fastpath_wrmsr(vcpu); case EXIT_REASON_MSR_WRITE_IMM: - return handle_fastpath_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return handle_fastpath_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); case EXIT_REASON_PREEMPTION_TIMER: return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); @@ -7356,7 +7356,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, } vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON); - if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry)) + if (likely(!vt_get_exit_reason(vcpu).failed_vmentry)) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); vt_handle_nmi(vcpu); @@ -7496,7 +7496,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * checking. */ if (vcpu->arch.nested_run_pending && - !vmx_get_exit_reason(vcpu).failed_vmentry) + !vt_get_exit_reason(vcpu).failed_vmentry) ++vcpu->stat.nested_run; vcpu->arch.nested_run_pending = 0; @@ -7507,7 +7507,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) trace_kvm_exit(vcpu, KVM_ISA_VMX); - if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry)) + if (unlikely(vt_get_exit_reason(vcpu).failed_vmentry)) return EXIT_FASTPATH_NONE; vmx->loaded_vmcs->launched = 1; From 6088c9a220127c1902902f2791d127dbbd9bca34 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 17:01:52 -0700 Subject: [PATCH 223/506] KVM: VMX: Move bus lock handler from main.c => common.h Move the bus lock VM-Exit handler, which is basically a one-liner, from main.c to vmx.h now that to_vt() is usable in common.h. No functional change intended. Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260922000152.1113202-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/common.h | 11 ++++++++++- arch/x86/kvm/vmx/main.c | 11 ----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index e22c159c3b84..b2905335a2d2 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -254,6 +254,15 @@ static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, return 1; } -int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu); +static inline int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) +{ + /* + * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK + * VM-Exits. Unconditionally set the flag here and leave the handling + * to .handle_exit() callback. + */ + to_vt(vcpu)->exit_reason.bus_lock_detected = true; + return 1; +} #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index c11732cfadf0..2985742e20f1 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -872,17 +872,6 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, #define vt_op_tdx_only(name) NULL #endif /* CONFIG_KVM_INTEL_TDX */ -int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) -{ - /* - * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK - * VM-Exits. Unconditionally set the flag here and leave the handling - * to .handle_exit() callback. - */ - to_vt(vcpu)->exit_reason.bus_lock_detected = true; - return 1; -} - noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) { if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || From 1a66112cbc0214743c0d6d52427609379ac858f7 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Wed, 26 Aug 2026 13:03:21 +0300 Subject: [PATCH 224/506] KVM: TDX: Remove always true ifdef CONFIG_KVM_INTEL_TDX depends on KVM_TDX_HOST, which in turn depends on CONFIG_X86_64. So it's guaranteed that any code under ifdef CONFIG_KVM_INTEL_TDX is also running under CONFIG_X86_64 being true, remove pointless ifdef guard. Signed-off-by: Nikolay Borisov Link: https://patch.msgid.link/20260826100321.1309685-1-nik.borisov@suse.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/main.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 2985742e20f1..250666cee0c2 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -803,7 +803,6 @@ static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) vmx_write_tsc_multiplier(vcpu); } -#ifdef CONFIG_X86_64 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired) { @@ -822,7 +821,6 @@ static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) vmx_cancel_hv_timer(vcpu); } -#endif static void vt_setup_mce(struct kvm_vcpu *vcpu) { From aec196ba2affa5b396ed1bc16340067d66972f24 Mon Sep 17 00:00:00 2001 From: Haotian Jiang Date: Thu, 6 Aug 2026 10:06:01 -0700 Subject: [PATCH 225/506] KVM: TDX: Reject INIT_MEM_REGION if number of bytes would overflow a u64 The nr_pages field in struct kvm_tdx_init_mem_region is a u64 that comes directly from userspace via copy_from_user(). The current validation uses a manual overflow check: region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa When nr_pages >= 2^52, the shift (nr_pages << PAGE_SHIFT) wraps around to a small value, bypassing the wrap check. While downstream protections (gfn_to_memslot() returning NULL for GFNs outside any memslot, and kvm_slot_has_gmem() checking for NULL) prevent any actual out-of-bounds access, the overflow itself is a real bug that should be caught at the validation layer. Replace the manual overflow check with check_shl_overflow() to correctly detect the wrap-around. Note, the manual wrap-around check on the gpa+size technically has a benign off-by-one bug, and can also use check_add_overflow(). Those flaws will be addressed shortly. Opportunistically separate the initial sanity checks from the more involved checks to try and make the code easier to read. Fixes: c846b451d3c5 ("KVM: TDX: Add an ioctl to create initial guest memory") Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260630214952.98C851F000E9@smtp.kernel.org Cc: Yan Zhao Cc: Binbin Wu Cc: Ackerley Tng Signed-off-by: Haotian Jiang [sean: use gpa_t, isolate check_shl_overflow() change, tweak changelog] Reviewed-by: Binbin Wu Reviewed-by: Yan Zhao Tested-by: Yan Zhao Link: https://patch.msgid.link/20260806170602.4112602-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 53fd4e63ce40..d991996259a8 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3257,6 +3257,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); struct kvm_tdx_init_mem_region region; struct tdx_gmem_post_populate_arg arg; + gpa_t nr_bytes; long gmem_ret; int ret; @@ -3274,10 +3275,13 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c return -EFAULT; if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr || - !PAGE_ALIGNED(region.gpa) || !region.nr_pages || - region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || + !PAGE_ALIGNED(region.gpa) || !region.nr_pages) + return -EINVAL; + + if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) || + region.gpa + nr_bytes <= region.gpa || !vt_is_tdx_private_gpa(kvm, region.gpa) || - !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1)) + !vt_is_tdx_private_gpa(kvm, region.gpa + nr_bytes - 1)) return -EINVAL; ret = 0; From ea93efec4dda2ca8c33dc88cf4d389a980a358df Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Aug 2026 10:06:02 -0700 Subject: [PATCH 226/506] KVM: TDX: Fix a benign off-by-one bug on the end GPA for INIT_MEM_REGION When verifying that the incoming GPA rage for INIT_MEM_REGION doesn't wrap, check the inclusive last GPA, not the exclusive last GPA. Super duper technically, it's ok if the very last GPA is -1ull. In practice, the flaw is benign as KVM x86 disallows memslots with GPAs that exceed MAXPHYADDR, i.e. INIT_MEM_REGION would fail with -EINVAL anyways due to the memslot check in kvm_gmem_populate(). Opportunistically use check_add_overflow() instead of manually checking for wrap, mostly so that the inclusive math doesn't need to be copy+pasted in the "is private" check. Fixes: c846b451d3c5 ("KVM: TDX: Add an ioctl to create initial guest memory") Reviewed-by: Yan Zhao Tested-by: Yan Zhao Reviewed-by: Binbin Wu Link: https://patch.msgid.link/20260806170602.4112602-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index d991996259a8..2a3acf493cb5 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3257,7 +3257,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); struct kvm_tdx_init_mem_region region; struct tdx_gmem_post_populate_arg arg; - gpa_t nr_bytes; + gpa_t nr_bytes, end_gpa; long gmem_ret; int ret; @@ -3279,9 +3279,9 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c return -EINVAL; if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) || - region.gpa + nr_bytes <= region.gpa || + check_add_overflow(region.gpa, nr_bytes - 1, &end_gpa) || !vt_is_tdx_private_gpa(kvm, region.gpa) || - !vt_is_tdx_private_gpa(kvm, region.gpa + nr_bytes - 1)) + !vt_is_tdx_private_gpa(kvm, end_gpa)) return -EINVAL; ret = 0; From 53c34396c19e45cd45e6b20c47ef9f47f289698b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:56:47 -0700 Subject: [PATCH 227/506] KVM: guest_memfd: Elaborate on how release() vs. get_pfn() is safe against UAF Add more context and information to the comment in kvm_gmem_release() that explains why there's no synchronization on RCU _or_ kvm->srcu. Point (b) from commit 67b43038ce14 ("KVM: guest_memfd: Remove RCU-protected attribute from slot->gmem.file") b) kvm->srcu ensures that kvm_gmem_unbind() and freeing of a memslot occur after the memslot is no longer visible to kvm_gmem_get_pfn(). is especially difficult to fully grok, particularly in light of commit ae431059e75d ("KVM: guest_memfd: Remove bindings on memslot deletion when gmem is dying"), which addressed a race between unbind() and release(). See the extended on-list discussion[*] for more details about exactly what KVM guards against, and how. Opportunistically update the comment in kvm_gmem_unbind() to clarify the rules for unbinding, i.e. to be more explicit about why it's ok to nullify the file pointer even though kvm_gmem_get_pfn() doesn't take the invalidate lock. No functional change intended. Link: https://lore.kernel.org/all/CAEvNRgGmyd1yqQXsnz5hWRZpBZUs%3DpiEWbEaqP9%2Bcz9ZqEMQ6g@mail.gmail.com [*] Cc: Yan Zhao Cc: Vishal Annapurve Reviewed-by: Yan Zhao Link: https://patch.msgid.link/20260826165647.769231-1-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 56 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 21bb3710edfe..b8797382cc5b 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -300,17 +300,56 @@ static int kvm_gmem_release(struct inode *inode, struct file *file) * dereferencing the slot for existing bindings needs to be protected * against memslot updates, specifically so that unbind doesn't race * and free the memslot (kvm_gmem_get_file() will return NULL). - * - * Since .release is called only when the reference count is zero, - * after which file_ref_get() and get_file_active() fail, - * kvm_gmem_get_pfn() cannot be using the file concurrently. - * file_ref_put() provides a full barrier, and get_file_active() the - * matching acquire barrier. */ mutex_lock(&kvm->slots_lock); filemap_invalidate_lock(inode->i_mapping); + /* + * Note! synchronize_srcu() is _not_ needed after nullifying memslot + * bindings as slot->gmem.file cannot be set back to a non-null value + * without the memslot first being deleted. I.e. this relies on the + * synchronize_srcu_expedited() in kvm_swap_active_memslots() to ensure + * kvm_gmem_get_pfn() (which runs with kvm->srcu held for read) can't + * grab a reference to slot->gmem.file even if the struct file object + * is reallocated (for use as a different file). + * + * file_ref_put() provides a full barrier, and __get_file_rcu() the + * matching acquire barrier, to ensure that kvm_gmem_get_file() (via + * __get_file_rcu()) sees refcount==0 or fails the "file reloaded" + * check (file != NULL due to nullifying the file pointer here). + * + * Unlike most other users of get_file_rcu(), where callers don't care + * if they race with a write, only that they have a reference to _a_ + * live file, kvm_gmem_get_pfn() needs to get the exact file that is + * associated with the memslot. Without the aforementioned SRCU + * synchronization, the following could happen: + * + * CPU0 CPU1 + * kvm_gmem_get_pfn() + * slot->gmem.file == A + * kvm_gmem_release()) + * slot->gmem.file = NULL + * + * kvm_set_memory_region() + * slot deleted + * + * kvm_set_memory_region() + * slot created + * slot->gmem.file = B (alloc the same object) + * + * get_file_active() + * file = B + * file_reloaded = B + * + * + * + * Obviously KVM would be broken in many places if the synchronization + * were omitted, but it's important to note that get_file_active() does + * NOT guarantee a reference to the "original" file was obtained, only + * that it grabbed a reference for the returned file, i.e. didn't grab + * a reference for file A, but then returned a pointer to file B. + */ xa_for_each(&f->bindings, index, slot) WRITE_ONCE(slot->gmem.file, NULL); @@ -698,8 +737,9 @@ void kvm_gmem_unbind(struct kvm_memory_slot *slot) xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); /* - * synchronize_srcu(&kvm->srcu) ensured that kvm_gmem_get_pfn() - * cannot see this memslot. + * Note, the caller is responsible for ensuring the slot is unreachable + * before unbinding, e.g. by synchronizing SRCU after deleting the slot, + * to guarantee kvm_gmem_get_pfn() can't see the slot+file. */ WRITE_ONCE(slot->gmem.file, NULL); filemap_invalidate_unlock(file->f_mapping); From e6ecb88711d93b9e014532f17d3d29b57e80932c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 17:13:27 -0700 Subject: [PATCH 228/506] KVM: guest_memfd: Gracefully handle xarray errors when binding a memslot If inserting a memslot into a guest_memfd's bindings xarray fails, propagate the error back to the caller, i.e. fail memslot creation as well. Signalling success and continuing on with memslot creation results in use-after-free, as the guest_memfd instance will remain reachable via the memslot after the file is freed (kvm_gmem_release() won't nullify the file pointer due to lack of a valid binding). Opportunistically WARN and reject binding if KVM_MEMSLOT_GMEM_ONLY is already set, partly to guard against goofs elsewhere, but mostly so that KVM doesn't need to worry about clobbering flags when unwinding on failure. Regarding the unwind, the slot must be fully prepared before inserting it into the bindings, at which point the slot becomes reachable. I.e. waiting to update the slot in order to avoid the ugly unwind isn't an option. And as part of the unwind, explicitly nullify the relevant bindings, as xarray can store a subset of entries when populating a range. Fixes: a7800aa80ea4 ("KVM: Add KVM_CREATE_GUEST_MEMFD ioctl() for guest-specific backing memory") Cc: stable@vger.kernel.org Reported-by: Stefan Teodorescu Reported-by: Dennis Tighe Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260823135031.4F6DC1F000E9%40smtp.kernel.org Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260922001332.1121266-2-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index b8797382cc5b..751d0ad94f74 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -655,6 +655,9 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset)); BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff)); + if (WARN_ON_ONCE(slot->flags & KVM_MEMSLOT_GMEM_ONLY)) + return -EINVAL; + file = fget(fd); if (!file) return -EBADF; @@ -693,7 +696,13 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, if (kvm_gmem_supports_mmap(inode)) slot->flags |= KVM_MEMSLOT_GMEM_ONLY; - xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL); + r = xa_err(xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL)); + if (r) { + xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + slot->gmem.file = NULL; + slot->gmem.pgoff = 0; + slot->flags &= ~KVM_MEMSLOT_GMEM_ONLY; + } filemap_invalidate_unlock(inode->i_mapping); /* @@ -701,7 +710,6 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, * not the other way 'round. Active bindings are invalidated if the * file is closed before memslots are destroyed. */ - r = 0; err: fput(file); return r; From 23a205797cec86f391d0be886b9bc741e1262063 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 17:13:28 -0700 Subject: [PATCH 229/506] KVM: Use goto to handle errors during memslot preparation Use a goto to unwind early memslot changes if preparing for a memslot operation fails. This will allow moving the creation of guest_memfd bindings into kvm_set_memslot() without needing to copy+paste the unwind logic. No functional change intended. Cc: stable@vger.kernel.org Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260922001332.1121266-3-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..3c0dbe60a5b4 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1931,21 +1931,8 @@ static int kvm_set_memslot(struct kvm *kvm, } r = kvm_prepare_memory_region(kvm, old, new, change); - if (r) { - /* - * For DELETE/MOVE, revert the above INVALID change. No - * modifications required since the original slot was preserved - * in the inactive slots. Changing the active memslots also - * release slots_arch_lock. - */ - if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { - kvm_activate_memslot(kvm, invalid_slot, old); - kfree(invalid_slot); - } else { - mutex_unlock(&kvm->slots_arch_lock); - } - return r; - } + if (r) + goto err; /* * For DELETE and MOVE, the working slot is now active as the INVALID @@ -1977,6 +1964,20 @@ static int kvm_set_memslot(struct kvm *kvm, kvm_commit_memory_region(kvm, old, new, change); return 0; + +err: + /* + * For DELETE/MOVE, revert the above INVALID change. No modifications + * required since the original slot was preserved in the inactive slots. + * Changing the active memslots also release slots_arch_lock. + */ + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { + kvm_activate_memslot(kvm, invalid_slot, old); + kfree(invalid_slot); + } else { + mutex_unlock(&kvm->slots_arch_lock); + } + return r; } static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id, From 513333a5546017da2a4a6173bfb70372f83733b3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 17:13:29 -0700 Subject: [PATCH 230/506] KVM: Only bind memslot to guest_memfd instance for CREATE operations For additional defense-in-depth, and to avoid having to handle impossible unwind scenarios when binding to a memslot fails, bind a memslot to a gmem instance only for CREATE operations, i.e. don't attempt to establish a binding for MOVE and FLAGS_ONLY operations. And when FLAGS_ONLY operations are eventually supported (this is currently all dead code), creating a new binding would be incorrect; KVM instead needs to do a 1:1 replacement of the existing binding, i.e. FLAGS_ONLY will need its own dedicated handling. Update the relevant TODO to make a better guess as to what needs to be done to support toggling dirty logging for guest_memfd memslots. Because it's dead code, no functional change intended. Cc: stable@vger.kernel.org Reviewed-by: David Hildenbrand (Arm) Link: https://patch.msgid.link/20260922001332.1121266-4-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 3c0dbe60a5b4..c53a88cea1f2 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1742,10 +1742,10 @@ static void kvm_commit_memory_region(struct kvm *kvm, kvm_destroy_dirty_bitmap(old); /* - * Unbind the guest_memfd instance as needed; the @new slot has - * already created its own binding. TODO: Drop the WARN when - * dirty logging guest_memfd memslots is supported. Until then, - * flags-only changes on guest_memfd slots should be impossible. + * TODO: Drop the WARN and do the unbind() call only for MOVE + * when dirty logging guest_memfd memslots is supported. Until + * then, flags-only changes on guest_memfd slots should also be + * impossible; unbind the old memslot for defense-in-depth. */ if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD)) kvm_gmem_unbind(old); @@ -2106,7 +2106,7 @@ static int kvm_set_memory_region(struct kvm *kvm, new->npages = npages; new->flags = mem->flags; new->userspace_addr = mem->userspace_addr; - if (mem->flags & KVM_MEM_GUEST_MEMFD) { + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) { r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset); if (r) goto out; @@ -2119,7 +2119,7 @@ static int kvm_set_memory_region(struct kvm *kvm, return 0; out_unbind: - if (mem->flags & KVM_MEM_GUEST_MEMFD) + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) kvm_gmem_unbind(new); out: kfree(new); From 83411f32b8ad96c6b05b1511d1a588daab44bd25 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 17:13:30 -0700 Subject: [PATCH 231/506] KVM: guest_memfd: Split bind() into prepare()+commit() phases Split binding a memslot to a guest_memfd instance into prepare() and commit() phases so that KVM can separate preparing the memslot from binding the memslot to the gmem instance, i.e. from committing the memslot. This will allow waiting to commit the memslot+gmem binding until the memslot is fully prepared, which is necessary as the memslot becomes reachable when the binding is created. As a bonus, drop the unwind-on-failure from the commit phase (other than nullifying the bindings), as the only reason bind() did the full unwind is because it technically didn't own the memslot, i.e. "needed" to leave memslot in the same state it started in. No functional change intended (the unwinding down on bind() failure was effectively dead code since KVM simply deletes the memslot on failure, i.e. there was nothing that could actually observe the unwind). Cc: stable@vger.kernel.org Reviewed-by: David Hildenbrand (Arm) Link: https://patch.msgid.link/20260922001332.1121266-5-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 68 ++++++++++++++++++++++++------------------ virt/kvm/guest_memfd.h | 19 ++++++++---- virt/kvm/kvm_main.c | 18 ++++++++++- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 751d0ad94f74..c813c20de7f3 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -642,15 +642,14 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) return __kvm_gmem_create(kvm, size, flags); } -int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset) +int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset) { uoff_t size = slot->npages << PAGE_SHIFT; - unsigned long start, end; struct gmem_file *f; struct inode *inode; struct file *file; - int r = -EINVAL; + BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset)); BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff)); @@ -674,44 +673,55 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, if (!PAGE_ALIGNED(offset) || offset + size > i_size_read(inode)) goto err; - filemap_invalidate_lock(inode->i_mapping); - - start = offset >> PAGE_SHIFT; - end = start + slot->npages; - - if (!xa_empty(&f->bindings) && - xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) { - r = -EEXIST; - filemap_invalidate_unlock(inode->i_mapping); - goto err; - } - /* * memslots of flag KVM_MEM_GUEST_MEMFD are immutable to change, so * kvm_gmem_bind() must occur on a new memslot. Because the memslot * is not visible yet, kvm_gmem_get_pfn() is guaranteed to see the file. */ WRITE_ONCE(slot->gmem.file, file); - slot->gmem.pgoff = start; + slot->gmem.pgoff = offset >> PAGE_SHIFT; if (kvm_gmem_supports_mmap(inode)) slot->flags |= KVM_MEMSLOT_GMEM_ONLY; - r = xa_err(xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL)); - if (r) { - xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); - slot->gmem.file = NULL; - slot->gmem.pgoff = 0; - slot->flags &= ~KVM_MEMSLOT_GMEM_ONLY; - } - filemap_invalidate_unlock(inode->i_mapping); - /* - * Drop the reference to the file, even on success. The file pins KVM, - * not the other way 'round. Active bindings are invalidated if the - * file is closed before memslots are destroyed. + * Gift the caller a reference to the file. The reference will be + * dropped after bindings are established, or if installing the new + * memslot ultimately fails. */ + return 0; + err: fput(file); + return -EINVAL; +} + +int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot) +{ + struct gmem_file *f = slot->gmem.file->private_data; + struct inode *inode = file_inode(slot->gmem.file); + unsigned long start, end; + int r; + + if (WARN_ON_ONCE(slot->gmem.file->f_op != &kvm_gmem_fops)) + return -EIO; + + filemap_invalidate_lock(inode->i_mapping); + + start = slot->gmem.pgoff; + end = start + slot->npages; + + if (!xa_empty(&f->bindings) && + xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) { + filemap_invalidate_unlock(inode->i_mapping); + return -EEXIST; + } + + r = xa_err(xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL)); + if (r) + xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + + filemap_invalidate_unlock(inode->i_mapping); + return r; } diff --git a/virt/kvm/guest_memfd.h b/virt/kvm/guest_memfd.h index 0f9c6f840838..01bd359d27e3 100644 --- a/virt/kvm/guest_memfd.h +++ b/virt/kvm/guest_memfd.h @@ -8,8 +8,9 @@ int kvm_gmem_init(struct module *module); void kvm_gmem_exit(void); int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args); -int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset); +int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset); +int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot); void kvm_gmem_unbind(struct kvm_memory_slot *slot); #else static inline int kvm_gmem_init(struct module *module) @@ -17,9 +18,17 @@ static inline int kvm_gmem_init(struct module *module) return 0; } static inline void kvm_gmem_exit(void) {}; -static inline int kvm_gmem_bind(struct kvm *kvm, - struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset) + +static inline int kvm_gmem_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset) +{ + WARN_ON_ONCE(1); + return -EIO; +} + +static inline int kvm_gmem_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *slot) { WARN_ON_ONCE(1); return -EIO; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index c53a88cea1f2..c7526c1617f7 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2107,7 +2107,23 @@ static int kvm_set_memory_region(struct kvm *kvm, new->flags = mem->flags; new->userspace_addr = mem->userspace_addr; if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) { - r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset); + r = kvm_gmem_prepare_memory_region(kvm, new, mem->guest_memfd, + mem->guest_memfd_offset); + if (r) + goto out; + + r = kvm_gmem_commit_memory_region(kvm, new); + + /* + * Drop the reference to the file, even on success. The file + * pins KVM, not the other way 'round. Active bindings are + * invalidated if the file is closed before memslots are + * destroyed. + */ +#ifdef CONFIG_KVM_GUEST_MEMFD + fput(new->gmem.file); +#endif + if (r) goto out; } From 3fe5d2944e5ac3fc5073344b0982c73703332e18 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 17:13:31 -0700 Subject: [PATCH 232/506] KVM: guest_memfd: Establish memslot<=>guest_memfd bindings *after* memslot is ready Wait to bind a memslot to a guest_memfd instance until *after* the memslot is fully prepared, as creating the binding in guest_memfd will effectively expose the memslot to readers. As pointed out by Sashiko, binding the memslot before it's ready to be exposed to the rest of the world can break various memslot assumption and rules. E.g. x86 could observe a NULL rmap pointer if a PUNCH_HOLE hit the guest_memfd after the binding was created, but before KVM made it through kvm_prepare_memory_region(). Fixes: a7800aa80ea4 ("KVM: Add KVM_CREATE_GUEST_MEMFD ioctl() for guest-specific backing memory") Cc: stable@vger.kernel.org Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260826170551.BEF801F000E9@smtp.kernel.org Link: https://patch.msgid.link/20260922001332.1121266-6-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index c7526c1617f7..f26986136bc6 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1934,6 +1934,15 @@ static int kvm_set_memslot(struct kvm *kvm, if (r) goto err; + if (change == KVM_MR_CREATE && (new->flags & KVM_MEM_GUEST_MEMFD)) { + r = kvm_gmem_commit_memory_region(kvm, new); + if (r) { + kvm_arch_free_memslot(kvm, new); + kvm_destroy_dirty_bitmap(new); + goto err; + } + } + /* * For DELETE and MOVE, the working slot is now active as the INVALID * version of the old slot. MOVE is particularly special as it reuses @@ -2111,32 +2120,25 @@ static int kvm_set_memory_region(struct kvm *kvm, mem->guest_memfd_offset); if (r) goto out; - - r = kvm_gmem_commit_memory_region(kvm, new); - - /* - * Drop the reference to the file, even on success. The file - * pins KVM, not the other way 'round. Active bindings are - * invalidated if the file is closed before memslots are - * destroyed. - */ -#ifdef CONFIG_KVM_GUEST_MEMFD - fput(new->gmem.file); -#endif - - if (r) - goto out; } r = kvm_set_memslot(kvm, old, new, change); + + /* + * Drop the reference to the gmem file, even on success. The file pins + * KVM, not the other way 'round. Active bindings are invalidated if + * the file is closed before memslots are destroyed. + */ +#ifdef CONFIG_KVM_GUEST_MEMFD + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) + fput(new->gmem.file); +#endif + if (r) - goto out_unbind; + goto out; return 0; -out_unbind: - if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) - kvm_gmem_unbind(new); out: kfree(new); return r; From 48252615eb78e151f93183370a650f5eb7e572fb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 17:13:32 -0700 Subject: [PATCH 233/506] KVM: guest_memfd: Drop superfluous WRITE_ONCE() when binding a memslot Drop the superfluous WRITE_ONCE() when setting a memslot's guest_memfd file during initial binding, as the memslot *must* be inactive and unreachable. The superfluous WRITE_ONCE() was added by commit 67b43038ce14 ("KVM: guest_memfd: Remove RCU-protected attribute from slot->gmem.file") to maintain rough "parity" with the existing rcu_assign_pointer(), not realizing that the only reason rcu_assign_pointer() was used was to make sparse and other checkers happy. Cc: Yan Zhao Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Ackerley Tng Reviewed-by: Yan Zhao Link: https://patch.msgid.link/20260922001332.1121266-7-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index c813c20de7f3..90ee0b9a9a6a 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -678,7 +678,7 @@ int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot * kvm_gmem_bind() must occur on a new memslot. Because the memslot * is not visible yet, kvm_gmem_get_pfn() is guaranteed to see the file. */ - WRITE_ONCE(slot->gmem.file, file); + slot->gmem.file = file; slot->gmem.pgoff = offset >> PAGE_SHIFT; if (kvm_gmem_supports_mmap(inode)) slot->flags |= KVM_MEMSLOT_GMEM_ONLY; From 211606532680fdb8113734b0aa8cbd490647523f Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Wed, 26 Aug 2026 09:01:46 +0000 Subject: [PATCH 234/506] KVM: SEV: Treat unassigned RMP entry as benign race on PSMASH failure When handling an RMP fault, KVM attempts to split a 2MB page via PSMASH. If PSMASH fails, the only expected return value is FAIL_BADADDR, which does not distinguish the reason for the bad address. Hence, another RMP entry lookup is required to determine whether the failure was benign. Specifically, KVM re-checks the RMP entry to determine if another CPU raced and already smashed the entry into 4KB pages. A concurrent operation (such as guest_memfd truncation or hole punching) can also race and transition the page to shared, removing the page from the RMP table and causing PSMASH to fail. This can happen even if the page is still referenced by KVM, because guest_memfd reclaim transitions the RMP entry to shared when the folio is removed from the page cache. Treat an unassigned RMP entry as an expected race when re-checking after a failed PSMASH, and skip logging an error warning. Fixes: c63cf135cc99 ("KVM: SEV: Add support to handle RMP nested page faults") Reviewed-by: Michael Roth Signed-off-by: Ackerley Tng Link: https://patch.msgid.link/20260826-gmem-no-return-page-v4-1-3bb9c1ddb4e3@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5705723f1f41..e198469eb074 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5090,10 +5090,11 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) /* * Look it up again. If it's 4K now then the PSMASH may have * raced with another process and the issue has already resolved - * itself. + * itself. If it's not assigned, then this must have raced with + * another process that made this page shared. */ if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && - assigned && rmp_level == PG_LEVEL_4K) + ((assigned && rmp_level == PG_LEVEL_4K) || !assigned)) goto out; pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", From 375b32931b5da6a86ccb017e7e12a67a2def03f0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:01:47 +0000 Subject: [PATCH 235/506] KVM: SEV: Drop page refcount early during RMP fault handling Rework KVM's handling of RMP faults to rely on MMU invalidation logic for safety, instead of the current approach of holding onto a folio reference until the RMP operations are complete. I.e. drop the reference gifted by guest_memfd immediately after getting the PFN, and instead do RMP updates under mmu_lock, after checking for relevant MMU invalidations. This will allow dropping guest_memfd's reference gifting entirely, which is ideally how KVM would operate for all "follow PFN" operations (GUP has many more complications, which is why KVM holds a reference across page faults *on top* of the standard MMU invalidation logic). Reviewed-by: Michael Roth Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Link: https://patch.msgid.link/20260826-gmem-no-return-page-v4-2-3bb9c1ddb4e3@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index e198469eb074..0d027cff734c 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5019,6 +5019,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) struct kvm_memory_slot *slot; struct kvm *kvm = vcpu->kvm; int order, rmp_level, ret; + unsigned long mmu_seq; struct page *page; bool assigned; kvm_pfn_t pfn; @@ -5046,18 +5047,22 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) return; } + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &page, &order); if (ret) { pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n", gpa); return; } + kvm_release_page_unused(page); ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); if (ret || !assigned) { pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", gpa, pfn, ret); - goto out_no_trace; + return; } /* @@ -5085,27 +5090,31 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) if (rmp_level == PG_LEVEL_4K) goto out; - ret = snp_rmptable_psmash(pfn); - if (ret) { - /* - * Look it up again. If it's 4K now then the PSMASH may have - * raced with another process and the issue has already resolved - * itself. If it's not assigned, then this must have raced with - * another process that made this page shared. - */ - if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && - ((assigned && rmp_level == PG_LEVEL_4K) || !assigned)) + scoped_guard(read_lock, &kvm->mmu_lock) { + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) goto out; - pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", - gpa, pfn, ret); + ret = snp_rmptable_psmash(pfn); + if (ret) { + /* + * Look it up again. If it's 4K now then the PSMASH may + * have raced with another process and the issue has + * already resolved itself. If it's not assigned, then + * this must have raced with another process that made + * this page shared. + */ + if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && + ((assigned && rmp_level == PG_LEVEL_4K) || !assigned)) + goto out; + + pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", + gpa, pfn, ret); + } } kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD); out: trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret); -out_no_trace: - kvm_release_page_unused(page); } static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end) From acafcf32c83ec16631463eeed9e87e44177c2e49 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Wed, 26 Aug 2026 09:01:48 +0000 Subject: [PATCH 236/506] KVM: SEV: Check for invalidation before warning on unassigned RMP entry When handling an RMP fault, KVM looks up the RMP entry for the backing PFN to determine if a 2MB page needs to be split via PSMASH. If the RMP entry is not assigned (or lookup fails), KVM logs a rate-limited warning under the assumption that private memory should always have an assigned RMP entry. However, a concurrent invalidation (such as guest_memfd hole punching) can race with RMP fault handling and transition the page to shared, unassigning the RMP entry after KVM fetched the PFN. In such cases, not finding an assigned RMP entry is benign. Check mmu_invalidate_retry_gfn() under mmu_lock before warning about a missing or unassigned RMP entry, and suppress the spurious warning if an invalidation occurred for the faulting GFN. Fixes: c63cf135cc99 ("KVM: SEV: Add support to handle RMP nested page faults") Signed-off-by: Ackerley Tng Link: https://patch.msgid.link/20260826-gmem-no-return-page-v4-3-3bb9c1ddb4e3@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0d027cff734c..f9dce8acac8e 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5060,8 +5060,12 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); if (ret || !assigned) { - pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", - gpa, pfn, ret); + guard(read_lock)(&kvm->mmu_lock); + + if (!mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) + pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", + gpa, pfn, ret); + return; } From 3c427e5c03476f6267a75116e11c8646683378e6 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Wed, 26 Aug 2026 09:01:49 +0000 Subject: [PATCH 237/506] KVM: SEV: Drop page refcount early in VMSA reload When reloading the guest VMSA for an SEV-SNP vCPU, KVM retrieves the PFN from guest_memfd. Drop the page reference immediately after retrieving the PFN instead of holding it across MMU lock acquisition in preparation for a follow-up patch to stop returning page pointers from guest_memfd PFN lookups. This is safe because the page's validity and presence are governed by KVM's MMU invalidation protocol rather than the page reference. No functional change intended. Reviewed-by: Michael Roth Signed-off-by: Ackerley Tng Link: https://patch.msgid.link/20260826-gmem-no-return-page-v4-4-3bb9c1ddb4e3@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index f9dce8acac8e..f094b19226b9 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4078,6 +4078,7 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) */ if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL)) return; + kvm_release_page_clean(page); read_lock(&kvm->mmu_lock); /* @@ -4092,8 +4093,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) else svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); read_unlock(&kvm->mmu_lock); - - kvm_release_page_clean(page); } /* From 89fbe3de01ccfc1b4862af6dfc79ab65bdc69e19 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 09:01:50 +0000 Subject: [PATCH 238/506] KVM: guest_memfd: Stop returning struct page from PFN lookup KVM currently expects guest_memfd PFN lookups to return a refcounted struct page, which callers hold across fault handling. Drop the page's refcount before returning from kvm_gmem_get_pfn() to prepare for the in-place conversions series. CoCo shared-to-private conversion handling must inspect folio refcounts to ensure exclusive ownership by guest_memfd. A concurrent guest page fault taking a temporary reference on the folio causes conversions to fail due to an elevated refcount. While this refcount is also taken on host userspace page faults, that refcount is taken on behalf of the host userspace page tables. This refcount will be dropped when conversions unmaps the page. Either way, once there's an mmap() or userspace mapping, the pages are open to way more refcounts, transient or not. This patch focuses on just dropping refcounts before handing KVM a page. guest_memfd already notifies KVM of page invalidations, so callers within KVM only need to respect the MMU invalidation protocol to safely rely on guest_memfd for page presence. Since the page refcounts are dropped, don't return the struct page pointer. Not returning the struct page from the guest_memfd PFN lookup moves KVM closer toward supporting memory backends that are not backed by struct page. Here are some notes on the cleanup in the callers of kvm_gmem_get_pfn(): kvm_release_faultin_page() in ARM's gmem_abort() originally also serves to set the page dirty and accessed under some conditions. The dirty and accessed flags don't matter for guest_memfd anyway, so it is safe to just drop the call to kvm_release_faultin_page(). For ARM's kvm_translate_vncr(), the local page pointer must be initialized to NULL so that the shared cleanup path that releases faulted-in pages safely no-ops for guest_memfd. For x86, no additional changes are required in the MMU fault path because the page fault tracking structure is zero-initialized at the start of page fault handling, ensuring the refcounted page pointer is already NULL. Reported-by: Yan Zhao Closes: https://lore.kernel.org/all/anZ4W9o5pTWIEgMY@yzhao56-desk.sh.intel.com/ Co-developed-by: Yan Zhao Signed-off-by: Yan Zhao Reviewed-by: Suzuki K Poulose Reviewed-by: Michael Roth Tested-by: Michael Roth Tested-by: Yan Zhao Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Reviewed-by: Gavin Shan Link: https://patch.msgid.link/20260826-gmem-no-return-page-v4-5-3bb9c1ddb4e3@google.com Signed-off-by: Sean Christopherson --- arch/arm64/kvm/mmu.c | 4 +--- arch/arm64/kvm/nested.c | 4 ++-- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/svm/sev.c | 8 ++------ include/linux/kvm_host.h | 6 ++---- virt/kvm/guest_memfd.c | 9 ++------- 6 files changed, 10 insertions(+), 23 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9ba86450fe4a..d57f8181f1b7 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1613,7 +1613,6 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt; unsigned long mmu_seq; - struct page *page; struct kvm *kvm = s2fd->vcpu->kvm; void *memcache = NULL; kvm_pfn_t pfn; @@ -1641,7 +1640,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ smp_rmb(); - ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, NULL); if (ret) { kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, write_fault, exec_fault, false); @@ -1681,7 +1680,6 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) } out_unlock: - kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W); kvm_fault_unlock(kvm); if ((prot & KVM_PGTABLE_PROT_W) && !ret) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 3c4fc566eafc..68acedc526c4 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1415,7 +1415,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) bool write_fault, writable; unsigned long mmu_seq; struct vncr_tlb *vt; - struct page *page; + struct page *page = NULL; u64 va, pfn, gfn; int ret; @@ -1471,7 +1471,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) return -EFAULT; } } else { - ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, NULL); if (ret) { kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE, write_fault, false, false); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..947c9cd84345 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4628,7 +4628,7 @@ static int kvm_mmu_faultin_pfn_gmem(struct kvm_vcpu *vcpu, } r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn, - &fault->refcounted_page, &max_order); + &max_order); if (r) { kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return r; diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index f094b19226b9..0a887f8e05d3 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4032,7 +4032,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) struct kvm *kvm = vcpu->kvm; gfn_t gfn = gpa_to_gfn(gpa); unsigned long mmu_seq; - struct page *page; kvm_pfn_t pfn; lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); @@ -4076,9 +4075,8 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) * The new VMSA will be private memory guest memory, so retrieve the * PFN from the gmem backend. */ - if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL)) + if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, NULL)) return; - kvm_release_page_clean(page); read_lock(&kvm->mmu_lock); /* @@ -5019,7 +5017,6 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) struct kvm *kvm = vcpu->kvm; int order, rmp_level, ret; unsigned long mmu_seq; - struct page *page; bool assigned; kvm_pfn_t pfn; gfn_t gfn; @@ -5049,13 +5046,12 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) mmu_seq = kvm->mmu_invalidate_seq; smp_rmb(); - ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &page, &order); + ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &order); if (ret) { pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n", gpa); return; } - kvm_release_page_unused(page); ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); if (ret || !assigned) { diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..502465119ca0 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2586,13 +2586,11 @@ static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) #ifdef CONFIG_KVM_GUEST_MEMFD int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order); + gfn_t gfn, kvm_pfn_t *pfn, int *max_order); #else static inline int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, - kvm_pfn_t *pfn, struct page **page, - int *max_order) + kvm_pfn_t *pfn, int *max_order) { KVM_BUG_ON(1, kvm); return -EIO; diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 90ee0b9a9a6a..77876d20f5a8 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -801,8 +801,7 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, } int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order) + gfn_t gfn, kvm_pfn_t *pfn, int *max_order) { pgoff_t index = kvm_gmem_get_index(slot, gfn); struct folio *folio; @@ -830,11 +829,7 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, #endif folio_unlock(folio); - - if (!r) - *page = folio_file_page(folio, index); - else - folio_put(folio); + folio_put(folio); return r; } From 54623a16bbdca9f273ce5658854e0b0d1637bc4d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:27 -0700 Subject: [PATCH 239/506] KVM: guest_memfd: Optimize away conversion overheads via dead-code elimination Add and use kvm_arch_has_gmem_convert() to guard guest_memfd's invocation of arch hooks related to converting memory between private and shared, as only one half of the x86 CoCo duo needs the runtime hooks (any pre-work is pure overhead for TDX). At this exact moment, the overhead is negligible, but that will change when in-place conversion comes along, at which point to-shared conversions will "need" to find all affected folios prior to calling into arch code. In quotes because very technically that work could be pushed to arch code, but that would bleed guest_memfd details into arch code and would be far worse than adding yet another kvm_arch_has... hook. Opportunistically provide the kvm_arch_gmem_make_private() declaration, and rely on dead-code elimination to eliminate the call to non-existent code when CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT=n. Reported-by: Binbin Wu Closes: https://lore.kernel.org/all/1ec08cd8-3072-4753-ad5e-cd34956647f8@linux.intel.com Suggested-by: Ackerley Tng Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Xiaoyao Li Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-1-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 3 +++ include/linux/kvm_host.h | 3 ++- virt/kvm/guest_memfd.c | 5 ++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 683bb8bf43a9..6beddd8a0c74 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1854,6 +1854,9 @@ enum kvm_intr_type { #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) #endif +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() (!!kvm_x86_ops.gmem_make_private) +#endif #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 502465119ca0..b7516e19ef41 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2597,9 +2597,10 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#ifndef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() false #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 77876d20f5a8..551b19ad2296 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -822,11 +822,10 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, folio_mark_uptodate(folio); } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - if (kvm_gmem_is_private_mem(file_inode(file), index)) + if (kvm_arch_has_gmem_convert() && + kvm_gmem_is_private_mem(file_inode(file), index)) r = kvm_arch_gmem_make_private(kvm, gfn, *pfn, (kvm_pfn_t)1 << *max_order); -#endif folio_unlock(folio); folio_put(folio); From fa7b8b1bb5d068ed737e64bcf6836922a70983d5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:28 -0700 Subject: [PATCH 240/506] KVM: guest_memfd: Use kvm_mem_is_private() when populating guest_memfd memory Use kvm_mem_is_private() when populating guest_memfd instead of using an open coded equivalent. In addition to simplifying the populate code *now*, this avoids the need to provide a range-based gmem lookup API in the future as well. No functional change intended. Suggested-by: Xiaoyao Li Signed-off-by: Ackerley Tng Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Reviewed-by: Fuad Tabba Reviewed-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-2-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 551b19ad2296..73974fece401 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -855,9 +855,7 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, folio_unlock(folio); - if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1, - KVM_MEMORY_ATTRIBUTE_PRIVATE, - KVM_MEMORY_ATTRIBUTE_PRIVATE)) { + if (!kvm_mem_is_private(kvm, gfn)) { ret = -EINVAL; goto out_put_folio; } From 1ae5be958ff702749b5ac8488dc18e2c6c175200 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:29 -0700 Subject: [PATCH 241/506] KVM: guest_memfd: Introduce per-gmem attributes, use to guard user mappings Start plumbing in guest_memfd support for in-place private<=>shared conversions by tracking attributes via a maple tree. KVM currently tracks private vs. shared attributes on a per-VM basis, which made sense when a guest_memfd _only_ supported private memory, but tracking per-VM simply can't work for in-place conversions as the shared/private status of a given page needs to be per-gmem_inode, not per-VM. Use the filemap invalidation lock to protect the maple tree, as taking the lock for read when faulting in memory (for userspace or the guest) isn't expected to result in meaningful contention, and using a separate lock would add significant complexity (avoiding deadlock is quite difficult). In kvm_gmem_get_pfn(), drop the folio refcount before releasing filemap_invalidate_lock(). This ensures that a competing conversion request from userspace (to be added in a later patch), which also takes the filemap_invalidate_lock(), will never see an elevated refcount due to kvm_gmem_get_pfn(). Co-developed-by: Vishal Annapurve Signed-off-by: Vishal Annapurve Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-3-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 136 +++++++++++++++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 17 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 73974fece401..fc92c45bb249 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,13 @@ struct gmem_inode { struct list_head gmem_file_list; u64 flags; + /* + * Every index in this inode, whether memory is populated or + * not, is tracked in attributes. The entire range of indices, + * corresponding to the size of this inode, is represented in + * this maple tree. + */ + struct maple_tree attributes; }; static __always_inline struct gmem_inode *GMEM_I(struct inode *inode) @@ -61,9 +69,28 @@ static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn) return gfn - slot->base_gfn + slot->gmem.pgoff; } +static u64 kvm_gmem_get_default_attributes(struct inode *inode) +{ + bool init_shared = GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED; + + return init_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE; +} + +static u64 kvm_gmem_get_attributes(struct inode *inode, void *entry) +{ + if (WARN_ON_ONCE(!entry)) + return kvm_gmem_get_default_attributes(inode); + + return xa_to_value(entry); +} + static bool kvm_gmem_is_private_mem(struct inode *inode, pgoff_t index) { - return !(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED); + struct maple_tree *mt = &GMEM_I(inode)->attributes; + void *entry = mtree_load(mt, index); + + return kvm_gmem_get_attributes(inode, entry) & + KVM_MEMORY_ATTRIBUTE_PRIVATE; } static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) @@ -403,10 +430,13 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) return VM_FAULT_SIGBUS; - if (!kvm_gmem_is_shared_mem(inode, vmf->pgoff)) - return VM_FAULT_SIGBUS; + filemap_invalidate_lock_shared(inode->i_mapping); + if (kvm_gmem_is_shared_mem(inode, vmf->pgoff)) + folio = kvm_gmem_get_folio(inode, vmf->pgoff); + else + folio = ERR_PTR(-EACCES); + filemap_invalidate_unlock_shared(inode->i_mapping); - folio = kvm_gmem_get_folio(inode, vmf->pgoff); if (IS_ERR(folio)) { if (PTR_ERR(folio) == -EAGAIN) return VM_FAULT_RETRY; @@ -559,6 +589,51 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm) return true; } +static int kvm_gmem_init_inode(struct inode *inode, loff_t size, u64 flags) +{ + struct gmem_inode *gi = GMEM_I(inode); + MA_STATE(mas, &gi->attributes, 0, (size >> PAGE_SHIFT) - 1); + u64 attrs; + int r; + + inode->i_op = &kvm_gmem_iops; + inode->i_mapping->a_ops = &kvm_gmem_aops; + inode->i_mode |= S_IFREG; + inode->i_size = size; + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + + /* + * guest_memfd memory is neither migratable nor swappable: set + * inaccessible to gate off both. + */ + mapping_set_inaccessible(inode->i_mapping); + WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); + + gi->flags = flags; + + mt_set_external_lock(&gi->attributes, + &inode->i_mapping->invalidate_lock); + + /* + * Store default attributes for the entire gmem instance. Ensuring every + * index is represented in the maple tree at all times simplifies the + * conversion and merging logic. + */ + attrs = kvm_gmem_get_default_attributes(inode); + + /* + * Acquire the invalidation lock purely to make lockdep happy. The + * maple tree library expects all stores to be protected via the lock, + * and the library can't know when the tree is reachable only by the + * caller, as is the case here. + */ + filemap_invalidate_lock(inode->i_mapping); + r = mas_store_gfp(&mas, xa_mk_value(attrs), GFP_KERNEL); + filemap_invalidate_unlock(inode->i_mapping); + + return r; +} + static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { static const char *name = "[kvm-gmem]"; @@ -589,16 +664,9 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) goto err_fops; } - inode->i_op = &kvm_gmem_iops; - inode->i_mapping->a_ops = &kvm_gmem_aops; - inode->i_mode |= S_IFREG; - inode->i_size = size; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); - mapping_set_inaccessible(inode->i_mapping); - /* Unmovable mappings are supposed to be marked unevictable as well. */ - WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); - - GMEM_I(inode)->flags = flags; + err = kvm_gmem_init_inode(inode, size, flags); + if (err) + goto err_inode; file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, &kvm_gmem_fops); if (IS_ERR(file)) { @@ -813,9 +881,13 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, if (!file) return -EFAULT; + filemap_invalidate_lock_shared(file_inode(file)->i_mapping); + folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order); - if (IS_ERR(folio)) - return PTR_ERR(folio); + if (IS_ERR(folio)) { + r = PTR_ERR(folio); + goto out; + } if (!folio_test_uptodate(folio)) { clear_highpage(folio_page(folio, 0)); @@ -830,6 +902,8 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, folio_unlock(folio); folio_put(folio); +out: + filemap_invalidate_unlock_shared(file_inode(file)->i_mapping); return r; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn); @@ -959,6 +1033,15 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) mpol_shared_policy_init(&gi->policy, NULL); + /* + * Memory attributes are protected by the filemap invalidation lock, but + * the lock structure isn't available at this time. Immediately mark + * maple tree as using external locking so that accessing the tree + * before it's fully initialized results in NULL pointer dereferences + * and not more subtle bugs. + */ + mt_init_flags(&gi->attributes, MT_FLAGS_LOCK_EXTERN | MT_FLAGS_USE_RCU); + gi->flags = 0; INIT_LIST_HEAD(&gi->gmem_file_list); return &gi->vfs_inode; @@ -966,7 +1049,26 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) static void kvm_gmem_destroy_inode(struct inode *inode) { - mpol_free_shared_policy(&GMEM_I(inode)->policy); + struct gmem_inode *gi = GMEM_I(inode); + + mpol_free_shared_policy(&gi->policy); + + /* + * Note! Checking for an empty tree is functionally necessary + * to avoid explosions if the tree hasn't been fully + * initialized, i.e. if the inode is being destroyed before + * guest_memfd can set the external lock, lockdep would find + * that the tree's internal ma_lock was not held. + */ + if (!mtree_empty(&gi->attributes)) { + /* + * Acquire the invalidation lock purely to make lockdep happy, + * the inode is unreachable at this point. + */ + filemap_invalidate_lock(inode->i_mapping); + __mt_destroy(&gi->attributes); + filemap_invalidate_unlock(inode->i_mapping); + } } static void kvm_gmem_free_inode(struct inode *inode) From 44e17d0a3663eeba1629d4ec5f14a16dc445d659 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:30 -0700 Subject: [PATCH 242/506] KVM: Rename KVM_GENERIC_MEMORY_ATTRIBUTES to KVM_VM_MEMORY_ATTRIBUTES Rename the per-VM memory attributes Kconfig to make it explicitly about per-VM attributes in anticipation of adding memory attributes support to guest_memfd, at which point it will be possible (and desirable) to have memory attributes without the per-VM support, even in x86. No functional change intended. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-4-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/Kconfig | 6 +++--- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/x86.c | 2 +- include/linux/kvm_host.h | 8 ++++---- include/trace/events/kvm.h | 4 ++-- virt/kvm/Kconfig | 2 +- virt/kvm/kvm_main.c | 14 +++++++------- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6beddd8a0c74..744c1f6ff03e 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1851,7 +1851,7 @@ enum kvm_intr_type { ((vcpu) && (vcpu)->arch.handling_intr_from_guest && \ (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI))) -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 538ed1e80332..2f8e1b5a8a72 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -84,7 +84,7 @@ config KVM_SW_PROTECTED_VM bool "Enable support for KVM software-protected VMs" depends on EXPERT depends on KVM_X86 && X86_64 - select KVM_GENERIC_MEMORY_ATTRIBUTES + select KVM_VM_MEMORY_ATTRIBUTES help Enable support for KVM software-protected VMs. Currently, software- protected VMs are purely a development and testing vehicle for @@ -135,7 +135,7 @@ config KVM_INTEL_TDX bool "Intel Trust Domain Extensions (TDX) support" default y depends on INTEL_TDX_HOST - select KVM_GENERIC_MEMORY_ATTRIBUTES + select KVM_VM_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_POPULATE help Provides support for launching Intel Trust Domain Extensions (TDX) @@ -159,7 +159,7 @@ config KVM_AMD_SEV depends on KVM_AMD && X86_64 depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) select ARCH_HAS_CC_PLATFORM - select KVM_GENERIC_MEMORY_ATTRIBUTES + select KVM_VM_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_CONVERT select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 947c9cd84345..b88c00e11854 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -8097,7 +8097,7 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread); } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, int level) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..229224957031 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10140,7 +10140,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm, } } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES kvm_mmu_init_memslot_memory_attributes(kvm, slot); #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index b7516e19ef41..cffa16647f2a 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -722,7 +722,7 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) } #endif -#ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifndef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static inline bool kvm_arch_has_private_mem(struct kvm *kvm) { return false; @@ -872,7 +872,7 @@ struct kvm { #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER struct notifier_block pm_notifier; #endif -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* Protected by slots_lock (for writes) and RCU (for reads) */ struct xarray mem_attr_array; #endif @@ -2560,7 +2560,7 @@ static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot) return slot->flags & KVM_MEMSLOT_GMEM_ONLY; } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) { return xa_to_value(xa_load(&kvm->mem_attr_array, gfn)); @@ -2582,7 +2582,7 @@ static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) { return false; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ #ifdef CONFIG_KVM_GUEST_MEMFD int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h index b282e3a86769..1ba72bd73ea2 100644 --- a/include/trace/events/kvm.h +++ b/include/trace/events/kvm.h @@ -358,7 +358,7 @@ TRACE_EVENT(kvm_dirty_ring_exit, TP_printk("vcpu %d", __entry->vcpu_id) ); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* * @start: Starting address of guest memory range * @end: End address of guest memory range @@ -383,7 +383,7 @@ TRACE_EVENT(kvm_vm_set_mem_attributes, TP_printk("%#016llx -- %#016llx [0x%lx]", __entry->start, __entry->end, __entry->attr) ); -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ TRACE_EVENT(kvm_unmap_hva_range, TP_PROTO(unsigned long start, unsigned long end), diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index c3c0ee253fc7..df575449d97b 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -100,7 +100,7 @@ config KVM_ELIDE_TLB_FLUSH_IF_YOUNG config KVM_MMU_LOCKLESS_AGING bool -config KVM_GENERIC_MEMORY_ATTRIBUTES +config KVM_VM_MEMORY_ATTRIBUTES bool config KVM_GUEST_MEMFD diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index f26986136bc6..2b425b4607b9 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1116,7 +1116,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) spin_lock_init(&kvm->mn_invalidate_lock); rcuwait_init(&kvm->mn_memslots_update_rcuwait); xa_init(&kvm->vcpu_array); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES xa_init(&kvm->mem_attr_array); #endif @@ -1301,7 +1301,7 @@ static void kvm_destroy_vm(struct kvm *kvm) cleanup_srcu_struct(&kvm->irq_srcu); srcu_barrier(&kvm->srcu); cleanup_srcu_struct(&kvm->srcu); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES xa_destroy(&kvm->mem_attr_array); #endif kvm_arch_free_vm(kvm); @@ -2438,7 +2438,7 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, } #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static u64 kvm_supported_mem_attributes(struct kvm *kvm) { if (!kvm || kvm_arch_has_private_mem(kvm)) @@ -2643,7 +2643,7 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes); } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { @@ -4960,7 +4960,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_SYSTEM_EVENT_DATA: case KVM_CAP_DEVICE_CTRL: return 1; -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_CAP_MEMORY_ATTRIBUTES: return kvm_supported_mem_attributes(kvm); #endif @@ -5364,7 +5364,7 @@ static long kvm_vm_ioctl(struct file *filp, break; } #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_SET_MEMORY_ATTRIBUTES: { struct kvm_memory_attributes attrs; @@ -5375,7 +5375,7 @@ static long kvm_vm_ioctl(struct file *filp, r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs); break; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ case KVM_CREATE_DEVICE: { struct kvm_create_device cd; From c2ae2bc1a311975b1c9c808cf9ad2758faf94495 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:31 -0700 Subject: [PATCH 243/506] KVM: Enumerate support for PRIVATE memory iff kvm_arch_has_private_mem is defined Explicitly guard reporting support for KVM_MEMORY_ATTRIBUTE_PRIVATE based on kvm_arch_has_private_mem being #defined in anticipation of tracking PRIVATE vs. SHARED state per-guest_memfd, not per-VM (to allow in-place conversion). guest_memfd support for memory attributes is expected to be unconditional to avoid yet more macros (all architectures that support guest_memfd are expected to use per-gmem attributes at some point), at which point enumerating support KVM_MEMORY_ATTRIBUTE_PRIVATE based solely on memory attributes being supported by KVM at-large would result in a system-scope check (NULL @kvm) over-reporting support on arm64. Give architectures full control over overriding the default definition of kvm_arch_has_private_mem() by removing the coupling with CONFIG_KVM_VM_MEMORY_ATTRIBUTES. In a later patch, kvm_arch_has_private_mem() will be defined based on whether architectural features are compiled in, and made orthogonal to CONFIG_KVM_VM_MEMORY_ATTRIBUTES. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-5-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 2 +- virt/kvm/kvm_main.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index cffa16647f2a..b45fb80b0aca 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -722,7 +722,7 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) } #endif -#ifndef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +#ifndef kvm_arch_has_private_mem static inline bool kvm_arch_has_private_mem(struct kvm *kvm) { return false; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 2b425b4607b9..798afcf14194 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2441,8 +2441,10 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static u64 kvm_supported_mem_attributes(struct kvm *kvm) { +#ifdef kvm_arch_has_private_mem if (!kvm || kvm_arch_has_private_mem(kvm)) return KVM_MEMORY_ATTRIBUTE_PRIVATE; +#endif return 0; } From a84ef89cbf91e88fff493b01b142cc949bb9bd7b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:32 -0700 Subject: [PATCH 244/506] KVM: Rename memory attribute APIs to prepare for in-place gmem conversion Rename memory attribute APIs to add a "vm_" in the name in anticipation of moving PRIVATE tracking into guest_memfd, to allow in-place conversion between SHARED and PRIVATE. At that point, there will effectively be two (potential) sources of memory attributes: the VM and guest_memfd. kvm_vm_set_mem_attributes() already has "vm" in the name to indicate that it is a VM ioctl. Rename it to kvm_set_vm_mem_attributes() to show that it is setting the VM's memory attributes. (Drop the VM-ioctl scoping since it is a helper local to the file.) Update the accompanying trace function to match. No functional change intended. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-6-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 14 +++++++------- include/linux/kvm_host.h | 16 ++++++++-------- include/trace/events/kvm.h | 2 +- virt/kvm/kvm_main.c | 32 ++++++++++++++++---------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b88c00e11854..10e59507d252 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -8116,8 +8116,8 @@ static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; } -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { struct kvm_memory_slot *slot = range->slot; int level; @@ -8186,18 +8186,18 @@ static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot, const unsigned long end = start + KVM_PAGES_PER_HPAGE(level); if (level == PG_LEVEL_2M) - return kvm_range_has_memory_attributes(kvm, start, end, ~0, attrs); + return kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attrs); for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) { if (hugepage_test_mixed(slot, gfn, level - 1) || - attrs != kvm_get_memory_attributes(kvm, gfn)) + attrs != kvm_get_vm_memory_attributes(kvm, gfn)) return false; } return true; } -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { unsigned long attrs = range->arg.attributes; struct kvm_memory_slot *slot = range->slot; @@ -8290,7 +8290,7 @@ void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm, * be manually checked as the attributes may already be mixed. */ for (gfn = start; gfn < end; gfn += nr_pages) { - unsigned long attrs = kvm_get_memory_attributes(kvm, gfn); + unsigned long attrs = kvm_get_vm_memory_attributes(kvm, gfn); if (hugepage_has_attrs(kvm, slot, gfn, level, attrs)) hugepage_clear_mixed(slot, gfn, level); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index b45fb80b0aca..6ca81f5349db 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2561,21 +2561,21 @@ static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot) } #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES -static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) +static inline unsigned long kvm_get_vm_memory_attributes(struct kvm *kvm, gfn_t gfn) { return xa_to_value(xa_load(&kvm->mem_attr_array, gfn)); } -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs); -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs); +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) { - return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; + return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; } #else static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h index 1ba72bd73ea2..c4d514d2350c 100644 --- a/include/trace/events/kvm.h +++ b/include/trace/events/kvm.h @@ -364,7 +364,7 @@ TRACE_EVENT(kvm_dirty_ring_exit, * @end: End address of guest memory range * @attr: The value of the attribute being set. */ -TRACE_EVENT(kvm_vm_set_mem_attributes, +TRACE_EVENT(kvm_set_vm_mem_attributes, TP_PROTO(gfn_t start, gfn_t end, unsigned long attr), TP_ARGS(start, end, attr), diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 798afcf14194..960060a442bc 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2439,7 +2439,7 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES -static u64 kvm_supported_mem_attributes(struct kvm *kvm) +static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm) { #ifdef kvm_arch_has_private_mem if (!kvm || kvm_arch_has_private_mem(kvm)) @@ -2453,19 +2453,19 @@ static u64 kvm_supported_mem_attributes(struct kvm *kvm) * Returns true if _all_ gfns in the range [@start, @end) have attributes * such that the bits in @mask match @attrs. */ -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs) +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs) { XA_STATE(xas, &kvm->mem_attr_array, start); unsigned long index; void *entry; - mask &= kvm_supported_mem_attributes(kvm); + mask &= kvm_supported_vm_mem_attributes(kvm); if (attrs & ~mask) return false; if (end == start + 1) - return (kvm_get_memory_attributes(kvm, start) & mask) == attrs; + return (kvm_get_vm_memory_attributes(kvm, start) & mask) == attrs; guard(rcu)(); if (!attrs) @@ -2536,8 +2536,8 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm, KVM_MMU_UNLOCK(kvm); } -static bool kvm_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +static bool kvm_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { /* * Unconditionally add the range to the invalidation set, regardless of @@ -2552,18 +2552,18 @@ static bool kvm_pre_set_memory_attributes(struct kvm *kvm, */ kvm_mmu_invalidate_range_add(kvm, range->start, range->end); - return kvm_arch_pre_set_memory_attributes(kvm, range); + return kvm_arch_pre_set_vm_memory_attributes(kvm, range); } /* Set @attributes for the gfn range [@start, @end). */ -static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, +static int kvm_set_vm_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, unsigned long attributes) { struct kvm_mmu_notifier_range pre_set_range = { .start = start, .end = end, .arg.attributes = attributes, - .handler = kvm_pre_set_memory_attributes, + .handler = kvm_pre_set_vm_memory_attributes, .on_lock = kvm_mmu_invalidate_start, .flush_on_ret = true, .may_block = true, @@ -2572,7 +2572,7 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, .start = start, .end = end, .arg.attributes = attributes, - .handler = kvm_arch_post_set_memory_attributes, + .handler = kvm_arch_post_set_vm_memory_attributes, .on_lock = kvm_mmu_invalidate_end, .may_block = true, }; @@ -2582,12 +2582,12 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, entry = attributes ? xa_mk_value(attributes) : NULL; - trace_kvm_vm_set_mem_attributes(start, end, attributes); + trace_kvm_set_vm_mem_attributes(start, end, attributes); mutex_lock(&kvm->slots_lock); /* Nothing to do if the entire range has the desired attributes. */ - if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes)) + if (kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attributes)) goto out_unlock; /* @@ -2626,7 +2626,7 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, /* flags is currently not used. */ if (attrs->flags) return -EINVAL; - if (attrs->attributes & ~kvm_supported_mem_attributes(kvm)) + if (attrs->attributes & ~kvm_supported_vm_mem_attributes(kvm)) return -EINVAL; if (attrs->size == 0 || attrs->address + attrs->size < attrs->address) return -EINVAL; @@ -2643,7 +2643,7 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, */ BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long)); - return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes); + return kvm_set_vm_mem_attributes(kvm, start, end, attrs->attributes); } #endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ @@ -4964,7 +4964,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) return 1; #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_CAP_MEMORY_ATTRIBUTES: - return kvm_supported_mem_attributes(kvm); + return kvm_supported_vm_mem_attributes(kvm); #endif #ifdef CONFIG_KVM_GUEST_MEMFD case KVM_CAP_GUEST_MEMFD: From 98fd6e30bd148d6c76aa39407a98281ea4256e3c Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:33 -0700 Subject: [PATCH 245/506] KVM: Rename kvm_mem_is_private() to kvm_is_private_gfn() Rename kvm_mem_is_private() to kvm_is_private_gfn() to prepare for in-place conversion, where there will be two lookup functions, kvm_vm_is_private_gfn() and kvm_gmem_is_private_gfn(). This renaming allows consistent prefixing of "vm" vs "gmem" for kvm_*_is_private_gfn(), as opposed to kvm_gmem_mem_is_private(), which looks like a typo. Suggested-by: Sean Christopherson Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-7-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 12 ++++++------ arch/x86/kvm/svm/sev.c | 2 +- include/linux/kvm_host.h | 4 ++-- virt/kvm/guest_memfd.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 10e59507d252..62f751952ad8 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3421,7 +3421,7 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault, is_private = fault->is_private; } else { max_level = PG_LEVEL_NUM; - is_private = kvm_mem_is_private(kvm, gfn); + is_private = kvm_is_private_gfn(kvm, gfn); } max_level = min(max_level, max_huge_page_level); @@ -3634,13 +3634,13 @@ static bool page_fault_can_be_fast(struct kvm *kvm, struct kvm_page_fault *fault * guest spinning on a #PF indefinitely, so don't attempt the fast path * in this case. * - * Note that the kvm_mem_is_private() check might race with an + * Note that the kvm_is_private_gfn() check might race with an * attribute update, but this will either result in the guest spinning * on RET_PF_SPURIOUS until the update completes, or an actual spurious * case might go down the slow path. Either case will resolve itself. */ if (kvm->arch.has_private_mem && - fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) + fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) return false; /* @@ -4708,7 +4708,7 @@ static int kvm_mmu_faultin_pfn(struct kvm_vcpu *vcpu, * Now that we have a snapshot of mmu_invalidate_seq we can check for a * private vs. shared mismatch. */ - if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) { + if (fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) { kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return -EFAULT; } @@ -5111,7 +5111,7 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, direct_bits = 0; if (kvm_arch_has_private_mem(vcpu->kvm) && - kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa))) + kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(range->gpa))) error_code |= PFERR_PRIVATE_ACCESS; else direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm)); @@ -6584,7 +6584,7 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && !(error_code & PFERR_RSVD_MASK) && vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM && - kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa))) + kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(cr2_or_gpa))) error_code |= PFERR_PRIVATE_ACCESS; r = RET_PF_INVALID; diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 0a887f8e05d3..dff684b88f15 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5030,7 +5030,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) * userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults * for shared pages should not end up here. */ - if (!kvm_mem_is_private(kvm, gfn)) { + if (!kvm_is_private_gfn(kvm, gfn)) { pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n", gpa); return; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 6ca81f5349db..9b97266f61ec 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2573,12 +2573,12 @@ bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, struct kvm_gfn_range *range); -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; } #else -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { return false; } diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index fc92c45bb249..3b05c6443f0f 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -929,7 +929,7 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, folio_unlock(folio); - if (!kvm_mem_is_private(kvm, gfn)) { + if (!kvm_is_private_gfn(kvm, gfn)) { ret = -EINVAL; goto out_put_folio; } From 8ab5719fbcdc96f5e636b67f14204ef7670abde1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:34 -0700 Subject: [PATCH 246/506] KVM: Provide generic interface for checking memory private/shared status Introduce a generic kvm_is_private_gfn() interface using a static call to determine if a GFN is private. This allows the implementation for checking a GFN's private/shared status to be set at runtime. In preparation for choosing implementations between a guest_memfd lookup and the existing VM attribute lookup, rename the existing VM-attribute-based check to kvm_vm_is_private_gfn() to emphasize that it looks up VM attributes. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Xiaoyao Li Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-8-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 14 ++++++++++++-- virt/kvm/kvm_main.c | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 9b97266f61ec..3579feb63177 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2573,16 +2573,26 @@ bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, struct kvm_gfn_range *range); -static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +static inline bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; } +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ + +#ifdef kvm_arch_has_private_mem +typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn); +DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t); + +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return static_call(__kvm_is_private_gfn)(kvm, gfn); +} #else static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { return false; } -#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ +#endif /* kvm_arch_has_private_mem */ #ifdef CONFIG_KVM_GUEST_MEMFD int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 960060a442bc..19098a51c35d 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2647,6 +2647,20 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, } #endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ +#ifdef kvm_arch_has_private_mem +DEFINE_STATIC_CALL_RET0(__kvm_is_private_gfn, kvm_is_private_gfn_t); +EXPORT_STATIC_CALL_GPL(__kvm_is_private_gfn); + +static void kvm_init_memory_attributes(void) +{ +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES + static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn); +#endif +} +#else +static void kvm_init_memory_attributes(void) { } +#endif + struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { return __gfn_to_memslot(kvm_memslots(kvm), gfn); @@ -6560,6 +6574,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out; + kvm_init_memory_attributes(); kvm_init_debug(); r = kvm_vfio_ops_init(); From ee1168fb73136dc1bd6b6f26142fdaa2399a2d60 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:35 -0700 Subject: [PATCH 247/506] KVM: guest_memfd: Stub in ability to enable in-place shared<=>private conversion Stub in global variable to enable in-place guest_memfd private<=>shared memory conversion, which will eventually be exposed to userspace via a module param, and wire up the __kvm_is_private_gfn() static call to the guest_memfd version when in-place conversion is enabled, i.e. when gmem is the sole authority on private vs. shared memory. Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: David Hildenbrand (Arm) Cc: Fuad Tabba Cc: Xiaoyao Li Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-9-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 17 ++++++++++++----- include/linux/kvm_host.h | 6 ++++++ virt/kvm/guest_memfd.c | 26 ++++++++++++++++++++++++++ virt/kvm/kvm_main.c | 12 +++++++++++- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc750c9..09967e030803 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6381,11 +6381,16 @@ mapping for userspace_addr is not required to be valid/populated at the time of KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated on-demand. -When mapping a gfn into the guest, KVM selects shared vs. private, i.e consumes -userspace_addr vs. guest_memfd, based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE -state. At VM creation time, all memory is shared, i.e. the PRIVATE attribute -is '0' for all gfns. Userspace can control whether memory is shared/private by -toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via KVM_SET_MEMORY_ATTRIBUTES as needed. +When mapping a gfn into the guest, KVM selects shared vs. private, i.e. consumes +userspace_addr vs. guest_memfd, based on the state in guest_memfd, which is the +sole authority on private vs. shared memory. See :ref:`KVM_CREATE_GUEST_MEMFD` +to find out more about the creation-time shared/private status. + +If in-place conversion is disabled, KVM selects shared vs. private based on the +gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE state. At VM creation time, all memory is +shared, i.e. the PRIVATE attribute is '0' for all gfns. Userspace can control +whether memory is shared/private by toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via +KVM_SET_MEMORY_ATTRIBUTES as needed. S390: ^^^^^ @@ -6429,6 +6434,8 @@ the state of a gfn/page as needed. The "flags" field is reserved for future extensions and must be '0'. +.. _KVM_CREATE_GUEST_MEMFD: + 4.142 KVM_CREATE_GUEST_MEMFD ---------------------------- diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 3579feb63177..b0110143d2ff 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2580,6 +2580,8 @@ static inline bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn) #endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ #ifdef kvm_arch_has_private_mem +extern bool gmem_in_place_conversion; + typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn); DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t); @@ -2588,6 +2590,8 @@ static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) return static_call(__kvm_is_private_gfn)(kvm, gfn); } #else +#define gmem_in_place_conversion false + static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { return false; @@ -2595,6 +2599,8 @@ static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) #endif /* kvm_arch_has_private_mem */ #ifdef CONFIG_KVM_GUEST_MEMFD +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn); + int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, kvm_pfn_t *pfn, int *max_order); #else diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 3b05c6443f0f..134fed0e24fa 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -517,6 +517,32 @@ static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) return 0; } +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); + struct inode *inode; + + if (!slot) + return 0; + + CLASS(gmem_get_file, file)(slot); + if (!file) + return 0; + + inode = file_inode(file); + + /* + * Rely on the maple tree's internal RCU lock to ensure a stable result. + * This result can become stale as soon as the lock is dropped, so the + * caller _must_ protect consumption of private vs. shared either by + * holding guest_memfd's invalidate lock for the entire duration, or by + * checking mmu_invalidate_retry_gfn() under mmu_lock to serialize + * against concurrent attribute updates. + */ + return kvm_gmem_is_private_mem(inode, kvm_gmem_get_index(slot, gfn)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_is_private_gfn); + static struct file_operations kvm_gmem_fops = { .mmap = kvm_gmem_mmap, .open = generic_file_open, diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 19098a51c35d..2532d1252a10 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -102,6 +102,10 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink); static bool __ro_after_init allow_unsafe_mappings; module_param(allow_unsafe_mappings, bool, 0444); +#ifdef kvm_arch_has_private_mem +bool __ro_after_init gmem_in_place_conversion = false; +#endif + /* * Ordering of locks: * @@ -2442,6 +2446,9 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm) { #ifdef kvm_arch_has_private_mem + if (gmem_in_place_conversion) + return 0; + if (!kvm || kvm_arch_has_private_mem(kvm)) return KVM_MEMORY_ATTRIBUTE_PRIVATE; #endif @@ -2653,8 +2660,11 @@ EXPORT_STATIC_CALL_GPL(__kvm_is_private_gfn); static void kvm_init_memory_attributes(void) { + if (gmem_in_place_conversion) + static_call_update(__kvm_is_private_gfn, kvm_gmem_is_private_gfn); #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES - static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn); + else + static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn); #endif } #else From bcc640eabbb289b96acd06c35adc623739d20548 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:36 -0700 Subject: [PATCH 248/506] KVM: Consolidate private memory and guest_memfd ifdeffery in kvm_host.h Move the kvm_arch_has_private_mem() stub and a few guest_memfd function definitions/declarations "down" in kvm_host.h to utilize existing #ifdefs, and so that related code is clustered together. No functional change intended. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Reviewed-by: Fuad Tabba Reviewed-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-10-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index b0110143d2ff..ab87effdd221 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -722,27 +722,6 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) } #endif -#ifndef kvm_arch_has_private_mem -static inline bool kvm_arch_has_private_mem(struct kvm *kvm) -{ - return false; -} -#endif - -#ifdef CONFIG_KVM_GUEST_MEMFD -bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); - -static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) -{ - u64 flags = GUEST_MEMFD_FLAG_MMAP; - - if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) - flags |= GUEST_MEMFD_FLAG_INIT_SHARED; - - return flags; -} -#endif - #ifndef kvm_arch_has_readonly_mem static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm) { @@ -2592,6 +2571,11 @@ static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) #else #define gmem_in_place_conversion false +static inline bool kvm_arch_has_private_mem(struct kvm *kvm) +{ + return false; +} + static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { return false; @@ -2600,6 +2584,17 @@ static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) #ifdef CONFIG_KVM_GUEST_MEMFD bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn); +bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); + +static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) +{ + u64 flags = GUEST_MEMFD_FLAG_MMAP; + + if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) + flags |= GUEST_MEMFD_FLAG_INIT_SHARED; + + return flags; +} int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, kvm_pfn_t *pfn, int *max_order); From e088c7f86befedcfb391ab54c924d65d36f0c328 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:37 -0700 Subject: [PATCH 249/506] KVM: guest_memfd: Invalidate both SHARED and PRIVATE mappings for in-place conversions When removing one or more folios from a guest_memfd instance, invalidate both SHARED and PRIVATE mappings if in-place conversion is enabled, because stating the obvious, KVM needs to ensure that all mappings to the folio(s) are dropped. Opportunistically rename the helper to capture that it returns a filter for all gfns in anticipation of zapping only the previous mapping types on conversion. I.e. when doing in-place conversion to PRIVATE, only SHARED mappings need to be zapped (ignoring that KVM would ideally not invalidate ranges whose attributes aren't changing in the first place). Note, precisely zapping only the possible mapping types when in-place conversion is disabled is important for functional correctness, not just for performance. Specifically, if KVM zaps both when SHARED vs. PRIVATE is tracked per-VM, then a PUNCH_HOLE operation on a PRIVATE guest_memfd will incorrectly zap SHARED mappings that have nothing to do with that gmem instance (because they're mapped via a VMA, not a gmem fd). The incorrect over-zapping of SHARED memory that doesn't belong to the gmem fd requesting the zapping will be resolved in a later patch, where, if in-place conversion is enabled, KVM will use both shared and private memory from the guest_memfd. If both shared and private memory are from the guest_memfd, invalidation will only zap memory belonging to the given gmem instance. Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-11-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 134fed0e24fa..2af2b6f0ab61 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -139,8 +139,11 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) return folio; } -static enum kvm_gfn_range_filter kvm_gmem_get_invalidate_filter(struct inode *inode) +static enum kvm_gfn_range_filter kvm_gmem_get_all_gfns_filter(struct inode *inode) { + if (gmem_in_place_conversion) + return KVM_FILTER_SHARED | KVM_FILTER_PRIVATE; + if (GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED) return KVM_FILTER_SHARED; @@ -191,11 +194,9 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, static void kvm_gmem_invalidate_start(struct inode *inode, pgoff_t start, pgoff_t end) { - enum kvm_gfn_range_filter attr_filter; + enum kvm_gfn_range_filter attr_filter = kvm_gmem_get_all_gfns_filter(inode); struct gmem_file *f; - attr_filter = kvm_gmem_get_invalidate_filter(inode); - kvm_gmem_for_each_file(f, inode) __kvm_gmem_invalidate_start(f, start, end, attr_filter); } @@ -386,7 +387,7 @@ static int kvm_gmem_release(struct inode *inode, struct file *file) * memory, as its lifetime is associated with the inode, not the file. */ __kvm_gmem_invalidate_start(f, 0, -1ul, - kvm_gmem_get_invalidate_filter(inode)); + kvm_gmem_get_all_gfns_filter(inode)); __kvm_gmem_invalidate_end(f, 0, -1ul); list_del(&f->entry); From 2e646bcfcf900cf35d5982e5d8d9bb8dab828380 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:38 -0700 Subject: [PATCH 250/506] KVM: guest_memfd: Always fault from guest_memfd if in-place conversion is enabled If a guest_memfd memslot is created but the guest_memfd does not have the GUEST_MEMFD_FLAG_MMAP set, KVM still fulfils guest faults by looking up the memslot's userspace_addr. Set KVM_MEMSLOT_GMEM_ONLY if in-place conversion is enabled so that the guest_memfd's memory will be used for both shared and private memory. With in-place conversion, guest_memfd will be the only backing memory for the memslot. No validation is performed to require userspace_addr to be a mapping from the associated guest_memfd because even after validation, userspace is free to remap something else at the provided userspace_addr. userspace_addr will still be used by functions like kvm_read_guest(), and if userspace_addr does not match up with the corresponding memory in the memslot's guest_memfd (whether userspace_addr points to the wrong offset or some non-guest_memfd memory, etc), that is a user error. Requiring both shared and private memory to come from the only associated guest_memfd simplifies invalidation in stage 2 page tables. On a PUNCH_HOLE operation on a guest_memfd, the invalidation is now guaranteed to be invalidating only memory mapped from the given guest_memfd. Suggested-by: Sean Christopherson Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-12-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 22 ++++++++++++++-------- virt/kvm/guest_memfd.c | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 09967e030803..35d94c51cc56 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6381,10 +6381,16 @@ mapping for userspace_addr is not required to be valid/populated at the time of KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated on-demand. -When mapping a gfn into the guest, KVM selects shared vs. private, i.e. consumes -userspace_addr vs. guest_memfd, based on the state in guest_memfd, which is the -sole authority on private vs. shared memory. See :ref:`KVM_CREATE_GUEST_MEMFD` -to find out more about the creation-time shared/private status. +When mapping a gfn into the guest, guest faults are always serviced from +guest_memfd regardless of whether memory is shared or private. KVM determines +shared vs. private based on the state in guest_memfd, which is the sole +authority on private vs. shared memory. See :ref:`KVM_CREATE_GUEST_MEMFD` to +find out more about the creation-time shared/private status. + +userspace_addr is expected to be the mmap()-ed address corresponding to the +right offset within the guest_memfd. Any mismatch between userspace_addr and +guest_memfd is not validated and is a user error. userspace_addr is only used +for host-side guest accesses such as kvm_read_guest(). If in-place conversion is disabled, KVM selects shared vs. private based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE state. At VM creation time, all memory is @@ -6490,10 +6496,10 @@ specified via KVM_CREATE_GUEST_MEMFD. Currently defined flags: page tables. Private memory cannot. ============================ ================================================ -When the KVM MMU performs a PFN lookup to service a guest fault and the backing -guest_memfd has the GUEST_MEMFD_FLAG_MMAP set, then the fault will always be -consumed from guest_memfd, regardless of whether it is a shared or a private -fault. +When the KVM MMU performs a PFN lookup to service a guest fault, the fault will +always be consumed from guest_memfd, regardless of whether it is a shared or a +private fault (unless in-place conversion is disabled and the backing +guest_memfd does not have the GUEST_MEMFD_FLAG_MMAP flag set). See KVM_SET_USER_MEMORY_REGION2 for additional details. diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 2af2b6f0ab61..10831f9d7ab0 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -775,7 +775,7 @@ int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot */ slot->gmem.file = file; slot->gmem.pgoff = offset >> PAGE_SHIFT; - if (kvm_gmem_supports_mmap(inode)) + if (gmem_in_place_conversion || kvm_gmem_supports_mmap(inode)) slot->flags |= KVM_MEMSLOT_GMEM_ONLY; /* From 2f7f4e102880706a2cadb46ba77859b5e968768c Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:39 -0700 Subject: [PATCH 251/506] KVM: guest_memfd: Pass mapping type filter to invalidation helper Accept the mapping type filter as a parameter in the invalidation start helper instead of querying it internally. This allows callers to specify which mappings (shared, private, or both) should be invalidated. In the next patch, the conversion process will use this new parameter to invalidate mappings only when they're different from the target state of the conversion, i.e. invalidate only shared mappings on a shared to private conversion and not both. No functional change intended. Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-13-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 10831f9d7ab0..a7615a8bf50c 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -192,13 +192,13 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, } static void kvm_gmem_invalidate_start(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, + enum kvm_gfn_range_filter filter) { - enum kvm_gfn_range_filter attr_filter = kvm_gmem_get_all_gfns_filter(inode); struct gmem_file *f; kvm_gmem_for_each_file(f, inode) - __kvm_gmem_invalidate_start(f, start, end, attr_filter); + __kvm_gmem_invalidate_start(f, start, end, filter); } static void __kvm_gmem_invalidate_end(struct gmem_file *f, pgoff_t start, @@ -224,6 +224,7 @@ static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t start, static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { + enum kvm_gfn_range_filter filter = kvm_gmem_get_all_gfns_filter(inode); pgoff_t start = offset >> PAGE_SHIFT; pgoff_t end = (offset + len) >> PAGE_SHIFT; @@ -233,7 +234,7 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) */ filemap_invalidate_lock(inode->i_mapping); - kvm_gmem_invalidate_start(inode, start, end); + kvm_gmem_invalidate_start(inode, start, end, filter); truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1); @@ -561,6 +562,8 @@ static int kvm_gmem_migrate_folio(struct address_space *mapping, static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *folio) { + struct inode *inode = mapping->host; + enum kvm_gfn_range_filter filter; pgoff_t start, end; filemap_invalidate_lock_shared(mapping); @@ -568,7 +571,8 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol start = folio->index; end = start + folio_nr_pages(folio); - kvm_gmem_invalidate_start(mapping->host, start, end); + filter = kvm_gmem_get_all_gfns_filter(inode); + kvm_gmem_invalidate_start(inode, start, end, filter); /* * Do not truncate the range, what action is taken in response to the From 799c9fadc639949beeab9328930f99fbf873d01a Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:40 -0700 Subject: [PATCH 252/506] KVM: guest_memfd: Add base support for KVM_SET_MEMORY_ATTRIBUTES2 Add a new ioctl (and matching struct), KVM_SET_MEMORY_ATTRIBUTES2, using the same base ioctl number (0xd2), but with R/W semantics for the kernel instead of just read semantics. "Officially" documenting that KVM writes to the payload will allow KVM to support partial/incremental conversions, instead of all-or-nothing updates (which requires complex unwinding), by recording the failing offset if an error occurs. Opportunistically add a new struct as well, even though KVM could squeeze the error offset into "struct kvm_memory_attributes", as there's no cost to doing so in practice. Pad the struct with a pile of extra space to try and avoid ending up with "struct kvm_memory_attributes3" in the future. Use the same layout for the fields common to version 1 of the struct, e.g. to ease upgrading userspace, and to provide flexibility if KVM ever adds support for KVM_SET_MEMORY_ATTRIBUTES2 at VM scope. Introduce KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES to advertise the availability of the KVM_SET_MEMORY_ATTRIBUTES2 ioctl. Update the KVM API documentation to define the new ioctl and its behavior, and add the necessary UAPI definitions and capability checks. The process of setting memory attributes has a clear point of no return because, for CoCo VMs, zapping stage 2 page tables is a destructive operation. Unlike regular VMs, where re-faulting pages into the stage 2 page tables merely incurs a performance penalty, CoCo guests must (re-):accept pages after every fault. To preserve CoCo security guarantees, guests will not accept pages they did not explicitly request faults for. Consequently, during memory conversions, any operation that could cause the process to abort must be completed before the stage 2 page tables are zapped. Zap only the ranges that are not already in the requested state to avoid inadvertently destroying (CoCo) data. ARM CCA guests will try to mark the entire DRAM as private at boot. If there are no shared pages at all, the to-private conversion can be skipped, but the existence of a single shared page would require the conversion process to proceed, and if it proceeds, zapping both shared and private pages would destroy data and break the guest. Suggested-by: Michael Roth Suggested-by: Suzuki K Poulose Co-developed-by: Vishal Annapurve Signed-off-by: Vishal Annapurve Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: Suzuki K Poulose Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-14-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 71 +++++++++++++++++++- include/uapi/linux/kvm.h | 15 +++++ virt/kvm/guest_memfd.c | 119 +++++++++++++++++++++++++++++++++ virt/kvm/kvm_main.c | 23 +++++-- 4 files changed, 219 insertions(+), 9 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 35d94c51cc56..027156508d04 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -117,7 +117,7 @@ description: x86 includes both i386 and x86_64. Type: - system, vm, or vcpu. + system, vm, vcpu or guest_memfd. Parameters: what parameters are accepted by the ioctl. @@ -6385,7 +6385,9 @@ When mapping a gfn into the guest, guest faults are always serviced from guest_memfd regardless of whether memory is shared or private. KVM determines shared vs. private based on the state in guest_memfd, which is the sole authority on private vs. shared memory. See :ref:`KVM_CREATE_GUEST_MEMFD` to -find out more about the creation-time shared/private status. +find out more about the creation-time shared/private status. Userspace can +control whether memory is shared/private by toggling +KVM_MEMORY_ATTRIBUTE_PRIVATE via :ref:`KVM_SET_MEMORY_ATTRIBUTES2` as needed. userspace_addr is expected to be the mmap()-ed address corresponding to the right offset within the guest_memfd. Any mismatch between userspace_addr and @@ -6404,6 +6406,8 @@ S390: Returns -EINVAL if the VM has the KVM_VM_S390_UCONTROL flag set. Returns -EINVAL if called on a protected VM. +.. _KVM_SET_MEMORY_ATTRIBUTES: + 4.141 KVM_SET_MEMORY_ATTRIBUTES ------------------------------- @@ -6440,6 +6444,8 @@ the state of a gfn/page as needed. The "flags" field is reserved for future extensions and must be '0'. +See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES2`. + .. _KVM_CREATE_GUEST_MEMFD: 4.142 KVM_CREATE_GUEST_MEMFD @@ -6676,6 +6682,67 @@ significant bit): Userspace should use the defined constants from ```` rather than hardcoding bit positions. +.. _KVM_SET_MEMORY_ATTRIBUTES2: + +4.146 KVM_SET_MEMORY_ATTRIBUTES2 +--------------------------------- + +:Capability: KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES +:Architectures: all +:Type: guest_memfd ioctl +:Parameters: struct kvm_memory_attributes2 (in) +:Returns: 0 on success, <0 on error + +Errors: + + ========== =============================================================== + EINVAL The specified `offset` or `size` was invalid (e.g. not + page aligned, causes an overflow, or size is zero). + EFAULT The parameter address was invalid. + ENOMEM Ran out of memory trying to track private/shared state + ========== =============================================================== + +KVM_SET_MEMORY_ATTRIBUTES2 is an extension to +KVM_SET_MEMORY_ATTRIBUTES that supports returning (writing) values to +userspace. The original (pre-extension) fields are shared with +KVM_SET_MEMORY_ATTRIBUTES identically. + +Attribute values are shared with KVM_SET_MEMORY_ATTRIBUTES. + +:: + + struct kvm_memory_attributes2 { + union { + __u64 address; + __u64 offset; + }; + __u64 size; + __u64 attributes; + __u64 flags; + __u64 reserved[12]; + }; + + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) + +Set attributes for a range of offsets within a guest_memfd to +KVM_MEMORY_ATTRIBUTE_PRIVATE to limit the specified guest_memfd backed +memory range for guest use. Even if KVM_CAP_GUEST_MEMFD_MMAP is +supported, after a successful call to set +KVM_MEMORY_ATTRIBUTE_PRIVATE, the requested range will not be mappable +into host userspace and will only be mappable by the guest. + +To allow the range to be mappable into host userspace again, call +KVM_SET_MEMORY_ATTRIBUTES2 on the guest_memfd again with +KVM_MEMORY_ATTRIBUTE_PRIVATE unset. + +KVM does not directly manipulate the memory contents of pages during +attribute updates. However, the process of setting these attributes, +which includes operations such as unmapping pages from the host or +stage-2 page tables, may result in side effects on memory contents +that vary across different trusted firmware implementations. + +See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES`. + .. _kvm_run: 5. The kvm_run structure diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index ac2d77d14963..ac371a50041c 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -999,6 +999,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_HPAGE_2G 249 #define KVM_CAP_PPC_COMPAT_CAPS 250 #define KVM_CAP_ARM_PMU_V3_STRICT 251 +#define KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES 252 struct kvm_irq_routing_irqchip { __u32 irqchip; @@ -1653,6 +1654,20 @@ struct kvm_memory_attributes { __u64 flags; }; +/* Available with KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES */ +#define KVM_SET_MEMORY_ATTRIBUTES2 _IOWR(KVMIO, 0xd2, struct kvm_memory_attributes2) + +struct kvm_memory_attributes2 { + union { + __u64 address; + __u64 offset; + }; + __u64 size; + __u64 attributes; + __u64 flags; + __u64 reserved[12]; +}; + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) #define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index a7615a8bf50c..0baf5c342d83 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -545,11 +545,130 @@ bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_is_private_gfn); +/* + * Preallocate memory for attributes to be stored on a maple tree, pointed to + * by mas. Adjacent ranges with attributes identical to the new attributes + * will be merged. Also sets mas's bounds up for storing attributes. + * + * This maintains the invariant that ranges with the same attributes will + * always be merged. + */ +static int kvm_gmem_mas_preallocate(struct ma_state *mas, u64 attributes, + pgoff_t start, size_t nr_pages) +{ + pgoff_t end = start + nr_pages; + pgoff_t last = end - 1; + void *entry; + + /* Try extending range. entry is NULL on overflow/wrap-around. */ + mas_set(mas, end); + entry = mas_find(mas, end); + if (entry && xa_to_value(entry) == attributes) + last = mas->last; + + if (start > 0) { + mas_set(mas, start - 1); + entry = mas_find(mas, start - 1); + if (entry && xa_to_value(entry) == attributes) + start = mas->index; + } + + mas_set_range(mas, start, last); + return mas_preallocate(mas, xa_mk_value(attributes), GFP_KERNEL); +} + +static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, + size_t nr_pages, uint64_t attrs) +{ + bool to_private = attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE; + struct address_space *mapping = inode->i_mapping; + struct gmem_inode *gi = GMEM_I(inode); + enum kvm_gfn_range_filter filter; + pgoff_t end = start + nr_pages; + struct maple_tree *mt; + struct ma_state mas; + int r; + + mt = &gi->attributes; + + filemap_invalidate_lock(mapping); + + mas_init(&mas, mt, start); + r = kvm_gmem_mas_preallocate(&mas, attrs, start, nr_pages); + if (r) + goto out; + + /* + * From this point on guest_memfd has performed necessary + * checks and can proceed to do guest-breaking changes. + */ + + filter = to_private ? KVM_FILTER_SHARED : KVM_FILTER_PRIVATE; + kvm_gmem_invalidate_start(inode, start, end, filter); + mas_store_prealloc(&mas, xa_mk_value(attrs)); + kvm_gmem_invalidate_end(inode, start, end); +out: + filemap_invalidate_unlock(mapping); + return r; +} + +static long kvm_gmem_set_attributes(struct file *file, void __user *argp) +{ + struct gmem_file *f = file->private_data; + struct inode *inode = file_inode(file); + struct kvm_memory_attributes2 attrs; + size_t nr_pages; + pgoff_t index; + int i; + + if (copy_from_user(&attrs, argp, sizeof(attrs))) + return -EFAULT; + + if (attrs.flags) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(attrs.reserved); i++) { + if (attrs.reserved[i]) + return -EINVAL; + } + if (!kvm_arch_has_private_mem(f->kvm)) + return -EINVAL; + if (attrs.attributes & ~KVM_MEMORY_ATTRIBUTE_PRIVATE) + return -EINVAL; + if (attrs.size == 0 || attrs.offset + attrs.size < attrs.offset) + return -EINVAL; + if (!PAGE_ALIGNED(attrs.offset) || !PAGE_ALIGNED(attrs.size)) + return -EINVAL; + + if (attrs.offset >= i_size_read(inode) || + attrs.offset + attrs.size > i_size_read(inode)) + return -EINVAL; + + nr_pages = attrs.size >> PAGE_SHIFT; + index = attrs.offset >> PAGE_SHIFT; + return __kvm_gmem_set_attributes(inode, index, nr_pages, + attrs.attributes); +} + +static long kvm_gmem_ioctl(struct file *file, unsigned int ioctl, + unsigned long arg) +{ + switch (ioctl) { + case KVM_SET_MEMORY_ATTRIBUTES2: + if (!gmem_in_place_conversion) + return -ENOTTY; + + return kvm_gmem_set_attributes(file, (void __user *)arg); + default: + return -ENOTTY; + } +} + static struct file_operations kvm_gmem_fops = { .mmap = kvm_gmem_mmap, .open = generic_file_open, .release = kvm_gmem_release, .fallocate = kvm_gmem_fallocate, + .unlocked_ioctl = kvm_gmem_ioctl, }; static int kvm_gmem_migrate_folio(struct address_space *mapping, diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 2532d1252a10..82e85a72c323 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2442,18 +2442,22 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, } #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ +#ifdef kvm_arch_has_private_mem +static u64 kvm_supports_private_mem(struct kvm *kvm) +{ + return !kvm || kvm_arch_has_private_mem(kvm); +} +#else +#define kvm_supports_private_mem(kvm) false +#endif + #ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm) { -#ifdef kvm_arch_has_private_mem - if (gmem_in_place_conversion) + if (gmem_in_place_conversion || !kvm_supports_private_mem(kvm)) return 0; - if (!kvm || kvm_arch_has_private_mem(kvm)) - return KVM_MEMORY_ATTRIBUTE_PRIVATE; -#endif - - return 0; + return KVM_MEMORY_ATTRIBUTE_PRIVATE; } /* @@ -4995,6 +4999,11 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) return 1; case KVM_CAP_GUEST_MEMFD_FLAGS: return kvm_gmem_get_supported_flags(kvm); + case KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES: + if (!gmem_in_place_conversion || !kvm_supports_private_mem(kvm)) + return 0; + + return KVM_MEMORY_ATTRIBUTE_PRIVATE; #endif default: break; From dcde2f853ff46836aa2b2d6530a47ba164cb49ab Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:41 -0700 Subject: [PATCH 253/506] KVM: guest_memfd: Ensure pages are not in use before conversion When converting memory to private in guest_memfd, it is necessary to ensure that the pages are not currently being accessed by any other part of the kernel or userspace to avoid any current user writing to guest private memory. guest_memfd checks for any outstanding references to determine whether a page is still in use. The only expected references after unmapping the range requested for conversion are those that are held by guest_memfd itself. A folio will have outstanding references if it is present in a per-CPU lru_add fbatch. guest_memfd does not actually participate in LRU, but freshly-allocated folios are still added to the lru_add fbatch for batch LRU statistics processing. A folio may also have extra refcounts if it is on the mlock fbatch. These two known usages of the folio are handled by draining both the lru_add and mlock fbatches for the folio. After draining, if the refcount is still elevated, then there are truly outstanding references. If the page may be DMA-pinned, DMA is using it and hence there are outstanding references. Checking if the page may be DMA-pinned can have false positives, but that is only with a significant number of refcounts, at which point draining LRU is not going to move the needle - it can still be concluded that the folio has outstanding references. If the page is still mapped after guest_memfd tried to unmap it earlier in the conversion process, it also has outstanding references. Exit early to avoid unnecessary draining in these two cases. If an outstanding reference is detected, stop scanning, record the failing offset, and return immediately. Track the drain status to avoid repeatedly draining LRU caches across multiple folios while scanning the requested range. Update the kvm_memory_attributes2 structure to include an error_offset field. This allows KVM to report the exact offset where a conversion failed. If the safety check fails, return -EAGAIN and copy the error_offset back to userspace so that it can potentially retry the operation or handle the failure gracefully. Report error_offset if preallocating memory to track attributes fails with -ENOMEM as well, in which case the start offset of the range is returned. Update documentation to document the error_offset field and the possible -EAGAIN error. Suggested-by: David Hildenbrand Co-developed-by: Vishal Annapurve Signed-off-by: Vishal Annapurve Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: David Hildenbrand (Arm) Acked-by: Vlastimil Babka (SUSE) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-15-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 10 ++++ include/uapi/linux/kvm.h | 3 +- virt/kvm/guest_memfd.c | 90 ++++++++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 6 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 027156508d04..67f0f290797a 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6741,6 +6741,16 @@ which includes operations such as unmapping pages from the host or stage-2 page tables, may result in side effects on memory contents that vary across different trusted firmware implementations. +If this ioctl returns -EAGAIN, the offset of the page with unexpected +refcounts will be returned in ``error_offset``. This can occur if +there are transient refcounts on the pages, taken by other parts of +the kernel. + +Userspace is expected to figure out how to remove all known refcounts +on the shared pages, such as refcounts taken by get_user_pages(), and +try the ioctl again. A possible source of these long term refcounts is +if the guest_memfd memory was pinned in IOMMU page tables. + See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES`. .. _kvm_run: diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index ac371a50041c..8dff2fc1972e 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1665,7 +1665,8 @@ struct kvm_memory_attributes2 { __u64 size; __u64 attributes; __u64 flags; - __u64 reserved[12]; + __u64 error_offset; + __u64 reserved[11]; }; #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 0baf5c342d83..7a244cb2be54 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "kvm_mm.h" #include "guest_memfd.h" @@ -577,8 +578,58 @@ static int kvm_gmem_mas_preallocate(struct ma_state *mas, u64 attributes, return mas_preallocate(mas, xa_mk_value(attributes), GFP_KERNEL); } +static bool __folio_has_outstanding_references(struct folio *folio, + enum lru_cache_drained *drained) +{ + if (folio_maybe_dma_pinned(folio) || folio_mapped(folio)) + return true; + + /* 1 reference held by filemap_get_folios() in the folio batch. */ + lru_cache_drain_for_folio(folio, 1, drained); + + /* + * Outstanding references are anything other than those from the page + * cache, plus 1 temporary reference held by filemap_get_folios() in the + * folio batch. + */ + return folio_ref_count(folio) != folio_nr_pages(folio) + 1; +} + +static bool kvm_gmem_has_outstanding_references(struct inode *inode, + pgoff_t start, size_t nr_pages, + pgoff_t *err_index) +{ + enum lru_cache_drained drained = LRU_CACHE_NOT_DRAINED; + struct address_space *mapping = inode->i_mapping; + pgoff_t last = start + nr_pages - 1; + struct folio_batch fbatch; + pgoff_t next; + int i; + + folio_batch_init(&fbatch); + + next = start; + while (filemap_get_folios(mapping, &next, last, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + + if (__folio_has_outstanding_references(folio, &drained)) { + *err_index = max(start, folio->index); + folio_batch_release(&fbatch); + return true; + } + } + + folio_batch_release(&fbatch); + cond_resched(); + } + + return false; +} + static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, - size_t nr_pages, uint64_t attrs) + size_t nr_pages, uint64_t attrs, + pgoff_t *err_index) { bool to_private = attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE; struct address_space *mapping = inode->i_mapping; @@ -595,8 +646,28 @@ static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, mas_init(&mas, mt, start); r = kvm_gmem_mas_preallocate(&mas, attrs, start, nr_pages); - if (r) + if (r) { + *err_index = start; goto out; + } + + if (to_private) { + /* + * Forcefully unmap the pages from all userspace page tables, + * and then verify there are no outstanding references, e.g. + * acquired via GUP or similar. Tell userspace to try again if + * there are outstanding references and hope that whatever has + * pinned the page will put its reference "soon". + */ + unmap_mapping_pages(mapping, start, nr_pages, false); + + if (kvm_gmem_has_outstanding_references(inode, start, nr_pages, + err_index)) { + mas_destroy(&mas); + r = -EAGAIN; + goto out; + } + } /* * From this point on guest_memfd has performed necessary @@ -617,9 +688,10 @@ static long kvm_gmem_set_attributes(struct file *file, void __user *argp) struct gmem_file *f = file->private_data; struct inode *inode = file_inode(file); struct kvm_memory_attributes2 attrs; + pgoff_t err_index; size_t nr_pages; pgoff_t index; - int i; + int i, r; if (copy_from_user(&attrs, argp, sizeof(attrs))) return -EFAULT; @@ -645,8 +717,16 @@ static long kvm_gmem_set_attributes(struct file *file, void __user *argp) nr_pages = attrs.size >> PAGE_SHIFT; index = attrs.offset >> PAGE_SHIFT; - return __kvm_gmem_set_attributes(inode, index, nr_pages, - attrs.attributes); + r = __kvm_gmem_set_attributes(inode, index, nr_pages, attrs.attributes, + &err_index); + if (r) { + attrs.error_offset = ((uint64_t)err_index) << PAGE_SHIFT; + + if (copy_to_user(argp, &attrs, sizeof(attrs))) + return -EFAULT; + } + + return r; } static long kvm_gmem_ioctl(struct file *file, unsigned int ioctl, From 6fa1dab756fd49d4b3162a68775de3aeec78646e Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:42 -0700 Subject: [PATCH 254/506] KVM: guest_memfd: Call arch make_shared callback for to-shared conversion When doing in-place conversion from PRIVATE to SHARED, immediately inform arch code of the conversion for all allocated pages/folios, e.g. so that arch code can put hardware metadata tables in the correct state. Eagerly updating the table for to SHARED conversions avoids having to implement on-demand updates, e.g. when faulting in host userspace mappings. Skip the entire flow if the arch doesn't implement conversion callbacks, as getting folios from the filemap is noticeably expensive, especially when converting large chunks of memory. Deliberately don't eagerly update the metadata table on conversions from SHARED to PRIVATE, because assigning a page to a VM (versus "returning" it to the host) requires the exact GFN associated with the page, i.e would require walking the memslot bindings. And because KVM *must* do on-demand metadata updates when getting a PFN for KVM-internal usage, as that's the only time a relevant memslot binding is guaranteed to exist. Note! Inform arch code of the conversion within the protection of the invalidation sequence, to ensure that any existing mappings are dropped before hardware is updated, and to ensure that new mappings can't be established until after the conversion is complete. Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-16-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 2 +- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/x86.c | 5 ++++ include/linux/kvm_host.h | 1 + virt/kvm/guest_memfd.c | 42 ++++++++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index e213c9ae3e30..67b43c167045 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -150,7 +150,7 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT KVM_X86_OP_OPTIONAL_RET0(gmem_make_private) #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM) KVM_X86_OP_OPTIONAL(gmem_make_shared) #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 744c1f6ff03e..83e26ce45fb7 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1732,7 +1732,7 @@ struct kvm_x86_ops { int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM) void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 229224957031..75e03a2f79db 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10653,6 +10653,11 @@ int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, { return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); } + +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +{ + kvm_x86_call(gmem_make_shared)(pfn, nr_pages); +} #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ab87effdd221..485f18454eb4 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2610,6 +2610,7 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); #ifndef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT #define kvm_arch_has_gmem_convert() false #endif diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 7a244cb2be54..5b6b58ea2acb 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -627,6 +627,43 @@ static bool kvm_gmem_has_outstanding_references(struct inode *inode, return false; } +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) +{ + struct folio_batch fbatch; + pgoff_t next = start; + int i; + + folio_batch_init(&fbatch); + while (filemap_get_folios(inode->i_mapping, &next, end - 1, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + pgoff_t start_index, end_index; + kvm_pfn_t start_pfn; + kvm_pfn_t nr_pages; + + start_index = max(start, folio->index); + end_index = min(end, folio_next_index(folio)); + /* + * end_index is either in folio or points to + * the first page of the next folio. Hence, + * all pages in range [start_index, end_index) + * are contiguous. + */ + start_pfn = folio_file_pfn(folio, start_index); + nr_pages = end_index - start_index; + + kvm_arch_gmem_make_shared(start_pfn, nr_pages); + } + + folio_batch_release(&fbatch); + cond_resched(); + } +} +#else +static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) {} +#endif + static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, size_t nr_pages, uint64_t attrs, pgoff_t *err_index) @@ -676,7 +713,12 @@ static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, filter = to_private ? KVM_FILTER_SHARED : KVM_FILTER_PRIVATE; kvm_gmem_invalidate_start(inode, start, end, filter); + + if (!to_private && kvm_arch_has_gmem_convert()) + kvm_gmem_make_shared(inode, start, end); + mas_store_prealloc(&mas, xa_mk_value(attrs)); + kvm_gmem_invalidate_end(inode, start, end); out: filemap_invalidate_unlock(mapping); From 856852ef2108fd4e70f29d450487fe15324c647c Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:43 -0700 Subject: [PATCH 255/506] KVM: guest_memfd: Return early if range already has requested attributes Provide a function to check that a range has given attributes. Optimize setting memory attributes by returning early if all pages in the requested range already have the requested attributes. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Reviewed-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-17-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 5b6b58ea2acb..8eedff42a96f 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -99,6 +99,24 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) return !kvm_gmem_is_private_mem(inode, index); } +static bool kvm_gmem_range_has_attributes(struct inode *inode, + pgoff_t index, size_t nr_pages, + u64 attributes) +{ + struct maple_tree *mt = &GMEM_I(inode)->attributes; + pgoff_t end = index + nr_pages - 1; + void *entry; + + lockdep_assert(mt_lock_is_held(mt)); + + mt_for_each(mt, entry, index, end) { + if (kvm_gmem_get_attributes(inode, entry) != attributes) + return false; + } + + return true; +} + /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -675,12 +693,15 @@ static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, pgoff_t end = start + nr_pages; struct maple_tree *mt; struct ma_state mas; - int r; + int r = 0; mt = &gi->attributes; filemap_invalidate_lock(mapping); + if (kvm_gmem_range_has_attributes(inode, start, nr_pages, attrs)) + goto out; + mas_init(&mas, mt, start); r = kvm_gmem_mas_preallocate(&mas, attrs, start, nr_pages); if (r) { From cb0544296a33f8dae0d60b24cfb4de839b332b98 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:44 -0700 Subject: [PATCH 256/506] KVM: guest_memfd: Zero page while getting pfn Move the folio initialization logic from kvm_gmem_get_pfn() into __kvm_gmem_get_pfn() to also zero pages if the page is to be used in kvm_gmem_populate(). With in-place conversion, the existing data in a guest_memfd page can be populated into guest memory through platform-specific ioctls. Without first zeroing the page obtained using __kvm_gmem_get_pfn(), it might contain uninitialized host memory, which would leak to the guest if the populate completes. guest_memfd pages are zeroed at most once in the page's entire lifetime with guest_memfd, and that is tracked using the uptodate flag. Zeroing the page in __kvm_gmem_get_pfn() is chosen over zeroing in kvm_gmem_get_folio() since other flows, such as a future write() syscall, can get a page, write to the page and then set page uptodate without zeroing. There may be some performance penalty due to redundant zeroing, but this would pale in comparison to the cost of actually assigning the page to the VM. This aligns with the concept of zeroing before first use - the other place where zeroing happens is in kvm_gmem_fault_user_mapping(). On populate failure, the page is not re-zeroed, since on SNP, if firmware rejects a CPUID page, the expected CPUID values provided by firmware are returned to userspace via page contents. More generally, page contents may be modified on populate failure. Don't mark the page uptodate again after populating, since the page would already be marked uptodate before the post_populate() call. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Reviewed-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-18-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 8eedff42a96f..a13445c26d9d 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -1154,6 +1154,11 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, return ERR_PTR(-EHWPOISON); } + if (!folio_test_uptodate(folio)) { + clear_highpage(folio_page(folio, 0)); + folio_mark_uptodate(folio); + } + *pfn = folio_file_pfn(folio, index); if (max_order) *max_order = 0; @@ -1182,11 +1187,6 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, goto out; } - if (!folio_test_uptodate(folio)) { - clear_highpage(folio_page(folio, 0)); - folio_mark_uptodate(folio); - } - if (kvm_arch_has_gmem_convert() && kvm_gmem_is_private_mem(file_inode(file), index)) r = kvm_arch_gmem_make_private(kvm, gfn, *pfn, @@ -1228,8 +1228,6 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, } ret = post_populate(kvm, gfn, pfn, src_page, opaque); - if (!ret) - folio_mark_uptodate(folio); out_put_folio: folio_put(folio); From edcc3f42981e9018d73c65ad8cfaf9e2ec70c07f Mon Sep 17 00:00:00 2001 From: Michael Roth Date: Thu, 10 Sep 2026 16:55:45 -0700 Subject: [PATCH 257/506] KVM: SEV: Make 'uaddr' parameter optional for KVM_SEV_SNP_LAUNCH_UPDATE Make the source page for populating an SNP guest_memfd instance optional if in-place conversion/population is enabled. If KVM can convert the page in-place, then it's possible for guest memory to be initialized directly from userspace by mmap()'ing the guest_memfd and writing to it while the corresponding GPA ranges are in a 'shared' state, before converting them to the 'private' state expected by KVM_SEV_SNP_LAUNCH_UPDATE. Update the handling/documentation for KVM_SEV_SNP_LAUNCH_UPDATE to allow for 'uaddr' to be set to NULL when in-place conversion is enabled, which SNP_LAUNCH_UPDATE will then use to determine when it should/shouldn't copy in data from a separate memory location. Continue to enforce non-NULL when PRIVATE is tracked per-VM, not per-guest_memfd. [ Sean: Moved condition to snp_launch_update ] Signed-off-by: Michael Roth Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-19-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../virt/kvm/x86/amd-memory-encryption.rst | 14 ++++++++++---- arch/x86/kvm/svm/sev.c | 11 ++++++----- virt/kvm/kvm_main.c | 1 + 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index bd04a908a8db..5977fbe33b98 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -503,7 +503,8 @@ secrets. It is required that the GPA ranges initialized by this command have had the KVM_MEMORY_ATTRIBUTE_PRIVATE attribute set in advance. See the documentation -for KVM_SET_MEMORY_ATTRIBUTES for more details on this aspect. +for KVM_SET_MEMORY_ATTRIBUTES/KVM_SET_MEMORY_ATTRIBUTES2 for more details on +this aspect. Upon success, this command is not guaranteed to have processed the entire range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of @@ -511,9 +512,14 @@ range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of remaining range that has yet to be processed. The caller should continue calling this command until those fields indicate the entire range has been processed, e.g. ``len`` is 0, ``gfn_start`` is equal to the last GFN in the -range plus 1, and ``uaddr`` is the last byte of the userspace-provided source -buffer address plus 1. In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, -``uaddr`` will be ignored completely. +range plus 1, and ``uaddr`` (if specified) is the last byte of the +userspace-provided source buffer address plus 1. + +In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, ``uaddr`` will be +ignored completely. For all other page types, ``uaddr`` is optional if in-place +conversion is enabled (i.e. when the data had been written directly to +guest_memfd while the page was in the shared state) and is required if in-place +conversion is disabled. Parameters (in): struct kvm_sev_snp_launch_update diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index dff684b88f15..63e8cfa9bf55 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2400,7 +2400,7 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, */ if (ret && !snp_page_reclaim(kvm, pfn) && sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID && - sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) { + sev_populate_args->fw_error == SEV_RET_INVALID_PARAM && src_page) { void *src_vaddr = kmap_local_page(src_page); void *dst_vaddr = kmap_local_pfn(pfn); @@ -2433,8 +2433,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) return -EFAULT; - pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d\n", __func__, - params.gfn_start, params.len, params.type, params.flags); + pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d src %llx\n", __func__, + params.gfn_start, params.len, params.type, params.flags, params.uaddr); if (!params.len || !PAGE_ALIGNED(params.len) || params.flags || (params.type != KVM_SEV_SNP_PAGE_TYPE_NORMAL && @@ -2446,7 +2446,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO) src = NULL; - else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr)) + else if ((!gmem_in_place_conversion && !params.uaddr) || + !PAGE_ALIGNED(params.uaddr)) return -EINVAL; else src = u64_to_user_ptr(params.uaddr); @@ -2493,7 +2494,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) params.gfn_start += count; params.len -= count * PAGE_SIZE; - if (params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) + if (src && params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) params.uaddr += count * PAGE_SIZE; if (copy_to_user(u64_to_user_ptr(argp->data), ¶ms, sizeof(params))) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 82e85a72c323..94b50fbe6614 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -104,6 +104,7 @@ module_param(allow_unsafe_mappings, bool, 0444); #ifdef kvm_arch_has_private_mem bool __ro_after_init gmem_in_place_conversion = false; +EXPORT_SYMBOL_FOR_KVM_INTERNAL(gmem_in_place_conversion); #endif /* From af602a96f401124a788836239c5fd775a687d8e5 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:46 -0700 Subject: [PATCH 258/506] KVM: TDX: Make source page optional for KVM_TDX_INIT_MEM_REGION Update tdx_gmem_post_populate() to handle cases where userspace requests "no source page". To handle "no source page", populate (perform TDH.MEM.PAGE.ADD) using memory in-place at the target PFN. Allow "no source page" only when gmem_in_place_conversion is enabled, because retroactively adding support for out-of-place conversion would mean requiring a userspace update for a feature that's being deprecated. Also, KVM supporting "no source page" without in-place conversion would effectively be an obscure zero-page optimization that relies on the page being zeroed when it is allocated by guest_memfd. Rejecting "no source page" without in-place conversion scenario is valuable for KVM developers since it helps newcomers understand what exactly is and isn't possible. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Tested-by: Yan Zhao Reviewed-by: Binbin Wu Reviewed-by: Yan Zhao Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-20-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/x86/intel-tdx.rst | 4 ++++ arch/x86/kvm/vmx/tdx.c | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Documentation/virt/kvm/x86/intel-tdx.rst b/Documentation/virt/kvm/x86/intel-tdx.rst index 6a222e9d0954..d8d9409120e6 100644 --- a/Documentation/virt/kvm/x86/intel-tdx.rst +++ b/Documentation/virt/kvm/x86/intel-tdx.rst @@ -158,6 +158,10 @@ KVM_TDX_INIT_MEM_REGION Initialize @nr_pages TDX guest private memory starting from @gpa with userspace provided data from @source_addr. @source_addr must be PAGE_SIZE-aligned. +If guest_memfd in-place conversion is enabled, pass 0 for @source_addr +to represent "no source page". A source page is required if in-place +conversion is not enabled or not supported. + Note, before calling this sub command, memory attribute of the range [gpa, gpa + nr_pages] needs to be private. Userspace can use KVM_SET_MEMORY_ATTRIBUTES to set the attribute. diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..7654a9f5fa12 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3189,7 +3189,7 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm)) return -EIO; - kvm_tdx->page_add_src = src_page; + kvm_tdx->page_add_src = src_page ?: pfn_to_page(pfn); ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn); kvm_tdx->page_add_src = NULL; @@ -3235,7 +3235,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c if (copy_from_user(®ion, u64_to_user_ptr(cmd->data), sizeof(region))) return -EFAULT; - if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr || + if (!PAGE_ALIGNED(region.source_addr) || + (!gmem_in_place_conversion && !region.source_addr) || !PAGE_ALIGNED(region.gpa) || !region.nr_pages || region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || !vt_is_tdx_private_gpa(kvm, region.gpa) || @@ -3266,7 +3267,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c break; } - region.source_addr += PAGE_SIZE; + if (region.source_addr) + region.source_addr += PAGE_SIZE; region.gpa += PAGE_SIZE; region.nr_pages--; From 31670c382efc5c7b3e40bb74ebc1b17bc8780fcc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:47 -0700 Subject: [PATCH 259/506] KVM: Move KVM_VM_MEMORY_ATTRIBUTES config definition to x86 Bury KVM_VM_MEMORY_ATTRIBUTES in x86 to discourage other architectures from adding support for per-VM memory attributes, because tracking private vs. shared memory on a per-VM basis is now deprecated in favor of tracking on a per-guest_memfd basis, and while RWX memory attributes are on the horizon, they too are expected to be x86-only. This will also allow modifying KVM_VM_MEMORY_ATTRIBUTES to be user-selectable (in x86) without creating weirdness in KVM's Kconfigs. Now that guest_memfd supports in-place conversions, it's entirely possible to run x86 CoCo VMs without support for KVM_VM_MEMORY_ATTRIBUTES. Leave the code itself in common KVM so that it's trivial to undo this change if new per-VM attributes do come along. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Reviewed-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-21-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/Kconfig | 3 +++ virt/kvm/Kconfig | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 2f8e1b5a8a72..abb108886733 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -80,6 +80,9 @@ config KVM_WERROR If in doubt, say "N". +config KVM_VM_MEMORY_ATTRIBUTES + bool + config KVM_SW_PROTECTED_VM bool "Enable support for KVM software-protected VMs" depends on EXPERT diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index df575449d97b..a0678ef8ee3f 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -100,9 +100,6 @@ config KVM_ELIDE_TLB_FLUSH_IF_YOUNG config KVM_MMU_LOCKLESS_AGING bool -config KVM_VM_MEMORY_ATTRIBUTES - bool - config KVM_GUEST_MEMFD select XARRAY_MULTI bool From 537ec2b15fdd7396c6c50860c17ebbf21f6f47d1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:48 -0700 Subject: [PATCH 260/506] KVM: Let userspace disable per-VM mem attributes, enable per-gmem attributes Allow the user to disable KVM_VM_MEMORY_ATTRIBUTES even when KVM supports PRIVATE and SHARED attributes, and expose gmem_in_place_conversion as a module parameter when per-VM attributes are supported. I.e. let userspace enable in-place PRIVATE<=>SHARED conversion of guest_memfd pages. Provide both a Kconfig option and a (conditional) module param so that deployments that use a custom kernel can fully disable per-VM tracking, while not forcing distros to ship two separate kernels in order to provide backwards compatibility for downstream users. Don't allow running VMs with mixed tracking for a given instance of KVM, i.e. disallow toggling the module param after KVM is loaded, as the extra complexity needed to handle per-VM behavior far outweighs any potential benefit. E.g. neither TDX nor SNP supports live migration, so in effect the requirement is that existing deployments that want to support both the old and the new models would need to tell their VMM which flavor of tracking to use. [ Xiaoyao: Define module_param only if CONFIG_KVM_VM_MEMORY_ATTRIBUTES is enabled ] Suggested-by: Xiaoyao Li Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Xiaoyao Li Acked-by: David Hildenbrand (Arm) Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-22-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../admin-guide/kernel-parameters.txt | 25 +++++++++++++++++++ arch/x86/include/asm/kvm_host.h | 4 ++- arch/x86/kvm/Kconfig | 14 ++++++++--- virt/kvm/kvm_main.c | 5 +++- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 33cd30996e47..49560b1b54c4 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3156,6 +3156,31 @@ Kernel parameters kvm.enable_vmware_backdoor=[KVM] Support VMware backdoor PV interface. Default is false (don't support). + kvm.gmem_in_place_conversion= + [KVM] Controls whether KVM enables in-place conversion + support for guest_memfd and tracks the private/shared + state of memory per guest_memfd instead of per VM. + + If enabled, KVM enables the KVM_SET_MEMORY_ATTRIBUTES2 + ioctl on guest_memfd file descriptors and disables the + legacy VM-scoped KVM_SET_MEMORY_ATTRIBUTES ioctl for + private memory state tracking. Only the + KVM_MEMORY_ATTRIBUTE_PRIVATE attribute moves to + per-guest_memfd tracking; other attributes remain + per-VM. + + This parameter toggles KVM's in-place conversion + capability support. Whether a VMM uses separate backends + or out-of-place memory management is determined by + userspace VMM design. + + Note, this parameter is only available when + CONFIG_KVM_VM_MEMORY_ATTRIBUTES=y. When + CONFIG_KVM_VM_MEMORY_ATTRIBUTES is not set, in-place + conversion is unconditionally enabled. + + Default is N (off). + kvm.nx_huge_pages= [KVM] Controls the software workaround for the X86_BUG_ITLB_MULTIHIT bug. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 83e26ce45fb7..e840418427a1 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1851,7 +1851,9 @@ enum kvm_intr_type { ((vcpu) && (vcpu)->arch.handling_intr_from_guest && \ (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI))) -#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +#if defined(CONFIG_KVM_SW_PROTECTED_VM) || \ + defined(CONFIG_KVM_INTEL_TDX) || \ + defined(CONFIG_KVM_AMD_SEV) #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index abb108886733..2c3c22aeafa5 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -81,13 +81,21 @@ config KVM_WERROR If in doubt, say "N". config KVM_VM_MEMORY_ATTRIBUTES - bool + bool "Enable per-VM PRIVATE vs. SHARED attributes (for CoCo VMs)" + depends on KVM_SW_PROTECTED_VM || KVM_INTEL_TDX || KVM_AMD_SEV + help + Enable support for tracking PRIVATE vs. SHARED memory using per-VM + memory attributes. Using per-VM attributes is deprecated in favor of + tracking PRIVATE state in guest_memfd. Select this if you need to run + CoCo VMs using a VMM that doesn't support guest_memfd memory + attributes. + + If unsure, say N. config KVM_SW_PROTECTED_VM bool "Enable support for KVM software-protected VMs" depends on EXPERT depends on KVM_X86 && X86_64 - select KVM_VM_MEMORY_ATTRIBUTES help Enable support for KVM software-protected VMs. Currently, software- protected VMs are purely a development and testing vehicle for @@ -138,7 +146,6 @@ config KVM_INTEL_TDX bool "Intel Trust Domain Extensions (TDX) support" default y depends on INTEL_TDX_HOST - select KVM_VM_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_POPULATE help Provides support for launching Intel Trust Domain Extensions (TDX) @@ -162,7 +169,6 @@ config KVM_AMD_SEV depends on KVM_AMD && X86_64 depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) select ARCH_HAS_CC_PLATFORM - select KVM_VM_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_CONVERT select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 94b50fbe6614..3ed481c4a15b 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -103,7 +103,10 @@ static bool __ro_after_init allow_unsafe_mappings; module_param(allow_unsafe_mappings, bool, 0444); #ifdef kvm_arch_has_private_mem -bool __ro_after_init gmem_in_place_conversion = false; +bool __ro_after_init gmem_in_place_conversion = !IS_ENABLED(CONFIG_KVM_VM_MEMORY_ATTRIBUTES); +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +module_param(gmem_in_place_conversion, bool, 0444); +#endif EXPORT_SYMBOL_FOR_KVM_INTERNAL(gmem_in_place_conversion); #endif From b5ed1ee2a7a0a9b2e70308e6a7306b5ce04d5ab7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:49 -0700 Subject: [PATCH 261/506] KVM: guest_memfd: Enable INIT_SHARED on guest_memfd for x86 Coco VMs Now that guest_memfd supports tracking private vs. shared within gmem itself, allow userspace to specify INIT_SHARED on a guest_memfd instance for x86 Confidential Computing (CoCo) VMs, so long as in-place conversion is enabled, i.e. when it's actually possible for a guest_memfd instance to contain shared memory. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-23-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 75e03a2f79db..c081e4b9b25d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10637,14 +10637,15 @@ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) } #ifdef CONFIG_KVM_GUEST_MEMFD -/* - * KVM doesn't yet support initializing guest_memfd memory as shared for VMs - * with private memory (the private vs. shared tracking needs to be moved into - * guest_memfd). - */ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) { - return !kvm_arch_has_private_mem(kvm); + /* + * INIT_SHARED is supported if in-place conversion is enabled, or if + * the VM doesn't support private memory. If the VM has private memory + * and in-place conversion is disabled, then guest_memfd can _only_ be + * used for private memory. + */ + return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm); } #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT From b312cab4e48c654d58a09abc4100a488e344d4fb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:50 -0700 Subject: [PATCH 262/506] KVM: selftests: Create gmem fd before "regular" fd when adding memslot When adding a memslot associated with a guest_memfd instance, create/dup the guest_memfd before creating the "normal" backing file. This will allow dup'ing the gmem fd as the normal fd when guest_memfd supports mmap(), i.e. to make guest_memfd the _only_ backing source for the memslot. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-24-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/kvm_util.c | 45 +++++++++++----------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9ddc047d5c27..3413d45e34fa 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1077,6 +1077,29 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, if (alignment > 1) region->mmap_size += alignment; + if (flags & KVM_MEM_GUEST_MEMFD) { + if (guest_memfd < 0) { + u32 guest_memfd_flags = 0; + + TEST_ASSERT(!guest_memfd_offset, + "Offset must be zero when creating new guest_memfd"); + guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags); + } else { + /* + * Install a unique fd for each memslot so that the fd + * can be closed when the region is deleted without + * needing to track if the fd is owned by the framework + * or by the caller. + */ + guest_memfd = kvm_dup(guest_memfd); + } + + region->region.guest_memfd = guest_memfd; + region->region.guest_memfd_offset = guest_memfd_offset; + } else { + region->region.guest_memfd = -1; + } + region->fd = -1; if (backing_src_is_shared(src_type)) region->fd = kvm_memfd_alloc(region->mmap_size, @@ -1106,28 +1129,6 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, region->backing_src_type = src_type; - if (flags & KVM_MEM_GUEST_MEMFD) { - if (guest_memfd < 0) { - u32 guest_memfd_flags = 0; - TEST_ASSERT(!guest_memfd_offset, - "Offset must be zero when creating new guest_memfd"); - guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags); - } else { - /* - * Install a unique fd for each memslot so that the fd - * can be closed when the region is deleted without - * needing to track if the fd is owned by the framework - * or by the caller. - */ - guest_memfd = kvm_dup(guest_memfd); - } - - region->region.guest_memfd = guest_memfd; - region->region.guest_memfd_offset = guest_memfd_offset; - } else { - region->region.guest_memfd = -1; - } - region->unused_phy_pages = sparsebit_alloc(); if (vm_arch_has_protected_memory(vm)) region->protected_phy_pages = sparsebit_alloc(); From 8d5f49000f8bb92a88f712c967ed8582d3f08904 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:51 -0700 Subject: [PATCH 263/506] KVM: selftests: Rename guest_memfd{,_offset} to gmem_{fd,offset} Rename local variables and function parameters for the guest memory file descriptor and its offset to use a "gmem_" prefix instead of "guest_memfd_". No functional change intended. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-25-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 6 ++--- tools/testing/selftests/kvm/lib/kvm_util.c | 26 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 5a7a455b5387..cf49a84de02e 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -690,17 +690,17 @@ int __vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva); void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags); void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd_fd, u64 guest_memfd_offset); + int gmem_fd, u64 gmem_offset); #ifndef vm_arch_has_protected_memory static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 3413d45e34fa..af501a27211a 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -970,7 +970,7 @@ void vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset) + u32 gmem_fd, u64 gmem_offset) { struct kvm_userspace_memory_region2 region = { .slot = slot, @@ -978,8 +978,8 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, .guest_phys_addr = gpa, .memory_size = size, .userspace_addr = (uintptr_t)hva, - .guest_memfd = guest_memfd, - .guest_memfd_offset = guest_memfd_offset, + .guest_memfd = gmem_fd, + .guest_memfd_offset = gmem_offset, }; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -989,10 +989,10 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset) + u32 gmem_fd, u64 gmem_offset) { int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva, - guest_memfd, guest_memfd_offset); + gmem_fd, gmem_offset); TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)", errno, strerror(errno)); @@ -1002,7 +1002,7 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, /* FIXME: This thing needs to be ripped apart and rewritten. */ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd, u64 guest_memfd_offset) + int gmem_fd, u64 gmem_offset) { int ret; struct userspace_mem_region *region; @@ -1078,12 +1078,12 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, region->mmap_size += alignment; if (flags & KVM_MEM_GUEST_MEMFD) { - if (guest_memfd < 0) { - u32 guest_memfd_flags = 0; + if (gmem_fd < 0) { + u32 gmem_flags = 0; - TEST_ASSERT(!guest_memfd_offset, + TEST_ASSERT(!gmem_offset, "Offset must be zero when creating new guest_memfd"); - guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags); + gmem_fd = vm_create_guest_memfd(vm, mem_size, gmem_flags); } else { /* * Install a unique fd for each memslot so that the fd @@ -1091,11 +1091,11 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, * needing to track if the fd is owned by the framework * or by the caller. */ - guest_memfd = kvm_dup(guest_memfd); + gmem_fd = kvm_dup(gmem_fd); } - region->region.guest_memfd = guest_memfd; - region->region.guest_memfd_offset = guest_memfd_offset; + region->region.guest_memfd = gmem_fd; + region->region.guest_memfd_offset = gmem_offset; } else { region->region.guest_memfd = -1; } From ee455183c92be7661ab08eedd68cd2bed0874975 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:52 -0700 Subject: [PATCH 264/506] KVM: selftests: Add support for mmap() on guest_memfd in core library Accept gmem_flags in vm_mem_add() to be able to create a guest_memfd within vm_mem_add(). When vm_mem_add() is used to set up a guest_memfd for a memslot, set up the provided (or created) gmem_fd as the fd for the user memory region. This makes it available to be mmap()-ed from just like fds from other memory sources. For guest_memfds, mmap() using gmem_offset instead of 0 all the time. Always use MAP_SHARED if mmap-ing from guest_memfd instead of reading flag from the configured src_type, which doesn't include guest_memfd. Add a kvm_slot_to_fd() helper to provide convenient access to the file descriptor of a memslot. Update existing callers of vm_mem_add() to pass 0 for gmem_flags to preserve existing behavior. Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-26-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 7 ++++- tools/testing/selftests/kvm/lib/kvm_util.c | 28 +++++++++++-------- .../kvm/x86/private_mem_conversions_test.c | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index cf49a84de02e..a091c495442a 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -700,7 +700,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, gpa_t gpa, u32 slot, u64 npages, u32 flags); void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int gmem_fd, u64 gmem_offset); + int gmem_fd, u64 gmem_offset, u64 gmem_flags); #ifndef vm_arch_has_protected_memory static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) @@ -732,6 +732,11 @@ void *addr_gva2hva(struct kvm_vm *vm, gva_t gva); gpa_t addr_hva2gpa(struct kvm_vm *vm, void *hva); void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa); +static inline int kvm_slot_to_fd(struct kvm_vm *vm, u32 slot) +{ + return memslot2region(vm, slot)->fd; +} + #ifndef vcpu_arch_put_guest #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) #endif diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index af501a27211a..08d0bbcd421e 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1002,12 +1002,14 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, /* FIXME: This thing needs to be ripped apart and rewritten. */ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int gmem_fd, u64 gmem_offset) + int gmem_fd, u64 gmem_offset, u64 gmem_flags) { int ret; struct userspace_mem_region *region; size_t backing_src_pagesz = get_backing_src_pagesz(src_type); + int mmap_flags = vm_mem_backing_src_alias(src_type)->flag; size_t mem_size = npages * vm->page_size; + off_t mmap_offset = 0; size_t alignment = 1; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -1079,8 +1081,6 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, if (flags & KVM_MEM_GUEST_MEMFD) { if (gmem_fd < 0) { - u32 gmem_flags = 0; - TEST_ASSERT(!gmem_offset, "Offset must be zero when creating new guest_memfd"); gmem_fd = vm_create_guest_memfd(vm, mem_size, gmem_flags); @@ -1101,13 +1101,17 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, } region->fd = -1; - if (backing_src_is_shared(src_type)) + if (flags & KVM_MEM_GUEST_MEMFD && gmem_flags & GUEST_MEMFD_FLAG_MMAP) { + region->fd = kvm_dup(gmem_fd); + mmap_flags = MAP_SHARED; + mmap_offset = gmem_offset; + } else if (backing_src_is_shared(src_type)) { region->fd = kvm_memfd_alloc(region->mmap_size, src_type == VM_MEM_SRC_SHARED_HUGETLB); + } - region->mmap_start = kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd); + region->mmap_start = __kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, + mmap_flags, region->fd, mmap_offset); TEST_ASSERT(!is_backing_src_hugetlb(src_type) || region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz), @@ -1153,10 +1157,10 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, /* If shared memory, create an alias. */ if (region->fd >= 0) { - region->mmap_alias = kvm_mmap(region->mmap_size, - PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd); + region->mmap_alias = __kvm_mmap(region->mmap_size, + PROT_READ | PROT_WRITE, + mmap_flags, region->fd, + mmap_offset); /* Align host alias address */ region->host_alias = align_ptr_up(region->mmap_alias, alignment); @@ -1167,7 +1171,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags) { - vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0); + vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0, 0); } /* diff --git a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c index 3aa0673f79f3..f3de2a4ab055 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c @@ -399,7 +399,7 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_v for (i = 0; i < nr_memslots; i++) vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i, BASE_DATA_SLOT + i, slot_size / vm->page_size, - KVM_MEM_GUEST_MEMFD, memfd, slot_size * i); + KVM_MEM_GUEST_MEMFD, memfd, slot_size * i, 0); for (i = 0; i < nr_vcpus; i++) { gpa_t gpa = BASE_DATA_GPA + i * per_cpu_size; From c3e48c166e35a5031df811459c96c303a2774cc1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:53 -0700 Subject: [PATCH 265/506] KVM: selftests: Add selftests global for guest memory attributes capability Add a global variable, kvm_has_gmem_attributes, to make the result of checking for KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES available to all tests. kvm_has_gmem_attributes is true if guest_memfd tracks memory attributes, as opposed to VM-level tracking. This global variable is synced to the guest for testing convenience, to avoid introducing subtle bugs when host/guest state is desynced. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-27-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/test_util.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a6a3e1657895..d5825211f272 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -116,6 +116,8 @@ struct kvm_random_state { extern u32 kvm_random_seed; extern struct kvm_random_state kvm_rng; +extern bool kvm_has_gmem_attributes; + struct kvm_random_state new_kvm_random_state(u32 seed); u32 kvm_random_u32(struct kvm_random_state *state); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 08d0bbcd421e..cd468b2bb834 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -32,6 +32,8 @@ static void kvm_seed_rng(u32 seed) kvm_rng = new_kvm_random_state(kvm_random_seed); } +bool kvm_has_gmem_attributes; + static size_t vcpu_mmap_sz(void); int __open_path_or_exit(const char *path, int flags, const char *enoent_help) @@ -527,6 +529,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, kvm_seed_rng(kvm_random_seed); sync_global_to_guest(vm, kvm_rng); + sync_global_to_guest(vm, kvm_has_gmem_attributes); kvm_arch_vm_post_create(vm, nr_runnable_vcpus); @@ -2311,6 +2314,8 @@ void __attribute((constructor)) kvm_selftest_init(void) srandom(time(0)); kvm_seed_rng(random()); + kvm_has_gmem_attributes = kvm_has_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES); + kvm_selftest_arch_init(); } From 0bbbd2b02bf0f66a65e4c6cc427fefbad512a861 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:55:54 -0700 Subject: [PATCH 266/506] KVM: selftests: Add helpers for calling ioctls on guest_memfd Add helper functions to kvm_util.h to support calling ioctls, specifically KVM_SET_MEMORY_ATTRIBUTES2, on a guest_memfd file descriptor. Introduce gmem_ioctl() and __gmem_ioctl() macros, modeled after the existing vm_ioctl() helpers, to provide a standard way to call ioctls on a guest_memfd. Add gmem_set_memory_attributes() and its derivatives (gmem_set_private(), gmem_set_shared()) to set memory attributes on a guest_memfd region. Also provide "__" variants that return the ioctl error code instead of aborting the test. These helpers will be used by upcoming guest_memfd tests. To avoid code duplication, factor out the check for supported memory attributes into a new macro, TEST_ASSERT_SUPPORTED_ATTRIBUTES, and use it in both the existing vm_set_memory_attributes() and the new gmem_set_memory_attributes() helpers. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-28-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 94 +++++++++++++++++-- 1 file changed, 86 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index a091c495442a..575032e9791d 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -392,6 +392,16 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ }) +#define __gmem_ioctl(gmem_fd, cmd, arg) \ + kvm_do_ioctl(gmem_fd, cmd, arg) + +#define gmem_ioctl(gmem_fd, cmd, arg) \ +({ \ + int ret = __gmem_ioctl(gmem_fd, cmd, arg); \ + \ + TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ +}) + /* * Looks up and returns the value corresponding to the capability * (KVM_CAP_*) given by cap. @@ -418,8 +428,16 @@ static inline void vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); } +/* + * KVM_SET_MEMORY_ATTRIBUTES{,2} overwrites _all_ attributes. These + * flows need significant enhancements to support multiple attributes. + */ +#define TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes) \ + TEST_ASSERT(!(attributes) || (attributes) == KVM_MEMORY_ATTRIBUTE_PRIVATE, \ + "Update me to support multiple attributes!") + static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, - u64 size, u64 attributes) + size_t size, u64 attributes) { struct kvm_memory_attributes attr = { .attributes = attributes, @@ -428,17 +446,11 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, .flags = 0, }; - /* - * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows - * need significant enhancements to support multiple attributes. - */ - TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, - "Update me to support multiple attributes!"); + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); } - static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa, u64 size) { @@ -451,6 +463,72 @@ static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa, vm_set_memory_attributes(vm, gpa, size, 0); } +static inline int __gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes, + u64 *error_offset) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + .error_offset = 0, + }; + int r; + + r = __gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); + + /* Copy error_offset regardless of r so caller can check. */ + if (error_offset) + *error_offset = attr.error_offset; + + return r; +} + +static inline int __gmem_set_private(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE, + error_offset); +} + +static inline int __gmem_set_shared(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, 0, + error_offset); +} + +static inline void gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + }; + + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); + + __TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) > 0, + "No valid attributes for guest_memfd ioctl!"); + + gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); +} + +static inline void gmem_set_private(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); +} + +static inline void gmem_set_shared(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, 0); +} + void vm_guest_mem_fallocate(struct kvm_vm *vm, gpa_t gpa, u64 size, bool punch_hole); From eaef4ed5fabcb62890bfc4596230c43703841384 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:55 -0700 Subject: [PATCH 267/506] KVM: selftests: Test basic single-page conversion flow Add a selftest for the guest_memfd memory attribute conversion ioctls. The test starts the guest_memfd as all-private (the default state), and verifies the basic flow of converting a single page to shared and then back to private. Add infrastructure that supports extensions to other conversion flow tests. This infrastructure will be used in upcoming patches for other conversion tests. Add test as an x86-specific test since guest_memfd's testing vehicle (KVM_X86_SW_PROTECTED_VM) is x86-specific. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-29-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../kvm/x86/guest_memfd_conversions_test.c | 200 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 96bab7002d39..239bc61ea384 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -155,6 +155,7 @@ TEST_GEN_PROGS_x86 += x86/max_vcpuid_cap_test TEST_GEN_PROGS_x86 += x86/triple_fault_event_test TEST_GEN_PROGS_x86 += x86/recalc_apic_map_test TEST_GEN_PROGS_x86 += x86/aperfmperf_test +TEST_GEN_PROGS_x86 += x86/guest_memfd_conversions_test TEST_GEN_PROGS_x86 += access_tracking_perf_test TEST_GEN_PROGS_x86 += coalesced_io_test TEST_GEN_PROGS_x86 += dirty_log_perf_test diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c new file mode 100644 index 000000000000..77f95d49349c --- /dev/null +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, Google LLC. + */ +#include +#include + +#include +#include +#include + +#include "kvm_util.h" +#include "kselftest_harness.h" +#include "test_util.h" +#include "ucall_common.h" + +FIXTURE(gmem_conversions) { + struct kvm_vcpu *vcpu; + int gmem_fd; + /* HVA of the first byte of the memory mmap()-ed from gmem_fd. */ + char *mem; +}; + +typedef FIXTURE_DATA(gmem_conversions) test_data_t; + +FIXTURE_SETUP(gmem_conversions) { } + +static size_t page_size; + +static void guest_do_rmw(void); +#define GUEST_MEMFD_SHARING_TEST_GVA 0x90000000ULL + +/* + * Defer setup until the individual test is invoked so that tests can specify + * the number of pages and flags for the guest_memfd instance. + */ +static void gmem_conversions_do_setup(test_data_t *t, int nr_pages, + int gmem_flags) +{ + const struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, + }; + /* + * Use high GPA above APIC_DEFAULT_PHYS_BASE to avoid clashing with + * APIC_DEFAULT_PHYS_BASE. + */ + const gpa_t gpa = SZ_4G; + const u32 slot = 1; + struct kvm_vm *vm; + + vm = __vm_create_shape_with_one_vcpu(shape, &t->vcpu, nr_pages, guest_do_rmw); + + vm_mem_add(vm, VM_MEM_SRC_SHMEM, gpa, slot, nr_pages, + KVM_MEM_GUEST_MEMFD, -1, 0, gmem_flags); + + t->gmem_fd = kvm_slot_to_fd(vm, slot); + t->mem = addr_gpa2hva(vm, gpa); + virt_map(vm, GUEST_MEMFD_SHARING_TEST_GVA, gpa, nr_pages); +} + +static void gmem_conversions_do_teardown(test_data_t *t) +{ + /* No need to close gmem_fd, it's owned by the VM structure. */ + kvm_vm_free(t->vcpu->vm); +} + +FIXTURE_TEARDOWN(gmem_conversions) +{ + gmem_conversions_do_teardown(self); +} + +/* + * In these test definition macros, __nr_pages and nr_pages is used to set up + * the total number of pages in the guest_memfd under test. This will be + * available in the test definitions as nr_pages. + */ + +#define __GMEM_CONVERSION_TEST(test, __nr_pages, flags) \ +static void __gmem_conversions_##test(test_data_t *t, int nr_pages); \ + \ +TEST_F(gmem_conversions, test) \ +{ \ + const int nr = (__nr_pages); \ + \ + gmem_conversions_do_setup(self, nr, flags); \ + __gmem_conversions_##test(self, nr); \ +} \ +static void __gmem_conversions_##test(test_data_t *t, int nr_pages) \ + +#define GMEM_CONVERSION_TEST(test, __nr_pages, flags) \ + __GMEM_CONVERSION_TEST(test, __nr_pages, (flags) | GUEST_MEMFD_FLAG_MMAP) + +#define __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, __nr_pages) \ + GMEM_CONVERSION_TEST(test, __nr_pages, 0) + +#define GMEM_CONVERSION_TEST_INIT_PRIVATE(test) \ + __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, 1) + +struct guest_check_data { + void *mem; + char expected_val; + char write_val; +}; +static struct guest_check_data guest_data; + +static void guest_do_rmw(void) +{ + for (;;) { + char *mem = READ_ONCE(guest_data.mem); + + GUEST_ASSERT_EQ(READ_ONCE(*mem), READ_ONCE(guest_data.expected_val)); + WRITE_ONCE(*mem, READ_ONCE(guest_data.write_val)); + + GUEST_SYNC(0); + } +} + +static void run_guest_do_rmw(struct kvm_vcpu *vcpu, u64 pgoff, + char expected_val, char write_val) +{ + struct ucall uc; + int r; + + guest_data.mem = (void *)GUEST_MEMFD_SHARING_TEST_GVA + pgoff * page_size; + guest_data.expected_val = expected_val; + guest_data.write_val = write_val; + sync_global_to_guest(vcpu->vm, guest_data); + + do { + r = __vcpu_run(vcpu); + } while (r == -1 && errno == EINTR); + + TEST_ASSERT_EQ(r, 0); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_SYNC: + break; + default: + TEST_FAIL("Unexpected ucall %lu", uc.cmd); + } +} + +static void host_do_rmw(char *mem, u64 pgoff, char expected_val, + char write_val) +{ + TEST_ASSERT_EQ(READ_ONCE(mem[pgoff * page_size]), expected_val); + WRITE_ONCE(mem[pgoff * page_size], write_val); +} + +static void test_private(test_data_t *t, u64 pgoff, char starting_val, + char write_val) +{ + TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[pgoff * page_size], write_val)); + run_guest_do_rmw(t->vcpu, pgoff, starting_val, write_val); + TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[pgoff * page_size])); +} + +static void test_convert_to_private(test_data_t *t, u64 pgoff, + char starting_val, char write_val) +{ + gmem_set_private(t->gmem_fd, pgoff * page_size, page_size); + test_private(t, pgoff, starting_val, write_val); +} + +static void test_shared(test_data_t *t, u64 pgoff, char starting_val, + char host_write_val, char write_val) +{ + host_do_rmw(t->mem, pgoff, starting_val, host_write_val); + run_guest_do_rmw(t->vcpu, pgoff, host_write_val, write_val); + TEST_ASSERT_EQ(READ_ONCE(t->mem[pgoff * page_size]), write_val); +} + +static void test_convert_to_shared(test_data_t *t, u64 pgoff, + char starting_val, char host_write_val, + char write_val) +{ + gmem_set_shared(t->gmem_fd, pgoff * page_size, page_size); + test_shared(t, pgoff, starting_val, host_write_val, write_val); +} + +GMEM_CONVERSION_TEST_INIT_PRIVATE(init_private) +{ + test_private(t, 0, 0, 'A'); + test_convert_to_shared(t, 0, 'A', 'B', 'C'); + test_convert_to_private(t, 0, 'C', 'E'); +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) & + KVM_MEMORY_ATTRIBUTE_PRIVATE); + + page_size = getpagesize(); + + return test_harness_run(argc, argv); +} From e5fa04dc920e4542404da89b31a8eb5a2d3360eb Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:56 -0700 Subject: [PATCH 268/506] KVM: selftests: Test conversion flow when INIT_SHARED Add a test case to verify that conversions between private and shared memory work correctly when the memory is initially created as shared. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-30-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/guest_memfd_conversions_test.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c index 77f95d49349c..4c20d3d8e934 100644 --- a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -97,6 +97,12 @@ static void __gmem_conversions_##test(test_data_t *t, int nr_pages) \ #define GMEM_CONVERSION_TEST_INIT_PRIVATE(test) \ __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, 1) +#define __GMEM_CONVERSION_TEST_INIT_SHARED(test, __nr_pages) \ + GMEM_CONVERSION_TEST(test, __nr_pages, GUEST_MEMFD_FLAG_INIT_SHARED) + +#define GMEM_CONVERSION_TEST_INIT_SHARED(test) \ + __GMEM_CONVERSION_TEST_INIT_SHARED(test, 1) + struct guest_check_data { void *mem; char expected_val; @@ -188,6 +194,13 @@ GMEM_CONVERSION_TEST_INIT_PRIVATE(init_private) test_convert_to_private(t, 0, 'C', 'E'); } +GMEM_CONVERSION_TEST_INIT_SHARED(init_shared) +{ + test_shared(t, 0, 0, 'A', 'B'); + test_convert_to_private(t, 0, 'B', 'C'); + test_convert_to_shared(t, 0, 'C', 'D', 'E'); +} + int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); From 492080a3d4e5812c6c232ee69758a856c303f723 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:57 -0700 Subject: [PATCH 269/506] KVM: selftests: Test conversion precision in guest_memfd The existing guest_memfd conversion tests only use single-page memory regions. This provides no coverage for multi-page guest_memfd objects, specifically whether KVM correctly handles the page index for conversion operations. An incorrect implementation could, for example, always operate on the first page regardless of the index provided. Add a new test case to verify that conversions between private and shared memory correctly target the specified page within a multi-page guest_memfd. This test also verifies the precision of memory conversions by converting a single page and then iterating through all other pages to ensure they remain in their original state. To support this test, add a new GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED macro that handles setting up and tearing down the VM for each page iteration. The teardown logic is adjusted to prevent a double-free in this new scenario. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-31-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/guest_memfd_conversions_test.c | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c index 4c20d3d8e934..a8d89df53e46 100644 --- a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -61,8 +61,13 @@ static void gmem_conversions_do_setup(test_data_t *t, int nr_pages, static void gmem_conversions_do_teardown(test_data_t *t) { + /* Use NULL to avoid second free in FIXTURE_TEARDOWN (multipage tests). */ + if (!t->vcpu) + return; + /* No need to close gmem_fd, it's owned by the VM structure. */ kvm_vm_free(t->vcpu->vm); + t->vcpu = NULL; } FIXTURE_TEARDOWN(gmem_conversions) @@ -103,6 +108,30 @@ static void __gmem_conversions_##test(test_data_t *t, int nr_pages) \ #define GMEM_CONVERSION_TEST_INIT_SHARED(test) \ __GMEM_CONVERSION_TEST_INIT_SHARED(test, 1) +/* + * Repeats test over nr_pages in a guest_memfd of size nr_pages, providing each + * test iteration with test_page, the index of the page under test in + * guest_memfd. test_page takes values 0..(nr_pages - 1) inclusive. + */ +#define GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(test, __nr_pages) \ +static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \ + const int test_page); \ + \ +TEST_F(gmem_conversions, test) \ +{ \ + const u64 flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; \ + const int nr = (__nr_pages); \ + int i; \ + \ + for (i = 0; i < nr; ++i) { \ + gmem_conversions_do_setup(self, nr, flags); \ + __gmem_conversions_multipage_##test(self, nr, i); \ + gmem_conversions_do_teardown(self); \ + } \ +} \ +static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \ + const int test_page) + struct guest_check_data { void *mem; char expected_val; @@ -201,6 +230,44 @@ GMEM_CONVERSION_TEST_INIT_SHARED(init_shared) test_convert_to_shared(t, 0, 'C', 'D', 'E'); } +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(indexing, 4) +{ + int i; + + /* Get a char that varies with both i and n. */ +#define combine(x, n) (((x) << 4) + (n)) +#define i_(n) (combine(i, n)) +#define t_(n) (combine(test_page, n)) + + /* + * Start with the highest index, to catch any errors when, perhaps, the + * first page is returned even for the last index. + */ + for (i = nr_pages - 1; i >= 0; --i) + test_shared(t, i, 0, i_(0), i_(2)); + + test_convert_to_private(t, test_page, t_(2), t_(3)); + + for (i = 0; i < nr_pages; ++i) { + if (i == test_page) + test_private(t, test_page, t_(3), t_(4)); + else + test_shared(t, i, i_(2), i_(3), i_(4)); + } + + test_convert_to_shared(t, test_page, t_(4), t_(5), t_(6)); + + for (i = 0; i < nr_pages; ++i) { + char expected = i == test_page ? t_(6) : i_(4); + + test_shared(t, i, expected, i_(7), i_(8)); + } + +#undef t_ +#undef i_ +#undef combine +} + int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); From 97137c1403607ed8316865867186382a71b4ea66 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:58 -0700 Subject: [PATCH 270/506] KVM: selftests: Test conversion before allocation Add two test cases to the guest_memfd conversions selftest to cover the scenario where a conversion is requested before any memory has been allocated in the guest_memfd region. The KVM_SET_MEMORY_ATTRIBUTES2 ioctl can be called on a memory region at any time. If the guest had not yet faulted in any pages for that region, the kernel must record the conversion request and apply the requested state when the pages are eventually allocated. The new tests cover both conversion directions. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-32-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/guest_memfd_conversions_test.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c index a8d89df53e46..f46f1530a94b 100644 --- a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -268,6 +268,20 @@ GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(indexing, 4) #undef combine } +/* + * Test that even if there are no folios yet, conversion requests are recorded + * in guest_memfd. + */ +GMEM_CONVERSION_TEST_INIT_SHARED(before_allocation_shared) +{ + test_convert_to_private(t, 0, 0, 'A'); +} + +GMEM_CONVERSION_TEST_INIT_PRIVATE(before_allocation_private) +{ + test_convert_to_shared(t, 0, 0, 'A', 'B'); +} + int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); From 5362c5fb88f6df1a00eda33ab8487ce09ebdb71a Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:55:59 -0700 Subject: [PATCH 271/506] KVM: selftests: Convert with allocated folios in different layouts Add a guest_memfd selftest to verify that memory conversions work correctly with allocated folios in different layouts. By iterating through which pages are initially faulted, the test covers various layouts of contiguous allocated and unallocated regions, exercising conversion with different range layouts. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-33-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/guest_memfd_conversions_test.c | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c index f46f1530a94b..acdff2d15404 100644 --- a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -282,6 +282,36 @@ GMEM_CONVERSION_TEST_INIT_PRIVATE(before_allocation_private) test_convert_to_shared(t, 0, 0, 'A', 'B'); } +/* + * Test that when some of the folios in the conversion range are allocated, + * conversion requests are handled correctly in guest_memfd. Vary the ranges + * allocated before conversion, using test_page, to cover various layouts of + * contiguous allocated and unallocated regions. + */ +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(unallocated_folios, 8) +{ + const int second_page_to_fault = 4; + int i; + + /* + * Fault 2 of the pages to test filemap range operations except when + * test_page == second_page_to_fault. + */ + host_do_rmw(t->mem, test_page, 0, 'A'); + if (test_page != second_page_to_fault) + host_do_rmw(t->mem, second_page_to_fault, 0, 'A'); + + gmem_set_private(t->gmem_fd, 0, nr_pages * page_size); + for (i = 0; i < nr_pages; ++i) { + char expected = (i == test_page || i == second_page_to_fault) ? 'A' : 0; + + test_private(t, i, expected, 'B'); + } + + for (i = 0; i < nr_pages; ++i) + test_convert_to_shared(t, i, 'B', 'C', 'D'); +} + int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); From bfb9b5a286ea6a190bfed191de3a6b6f0bf801c4 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:56:00 -0700 Subject: [PATCH 272/506] KVM: selftests: Test that truncation does not change shared/private status Add a test to verify that deallocating a page in a guest memfd region via fallocate() with FALLOC_FL_PUNCH_HOLE does not alter the shared or private status of the corresponding memory range. When a page backing a guest memfd mapping is deallocated, e.g., by punching a hole or truncating the file, and then subsequently faulted back in, the new page must inherit the correct shared/private status tracked by guest_memfd. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-34-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/guest_memfd_conversions_test.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c index acdff2d15404..b7a54d2a9092 100644 --- a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -10,6 +10,7 @@ #include #include "kvm_util.h" +#include "kvm_syscalls.h" #include "kselftest_harness.h" #include "test_util.h" #include "ucall_common.h" @@ -312,6 +313,19 @@ GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(unallocated_folios, 8) test_convert_to_shared(t, i, 'B', 'C', 'D'); } +/* Truncation should not affect shared/private status. */ +GMEM_CONVERSION_TEST_INIT_SHARED(truncate) +{ + host_do_rmw(t->mem, 0, 0, 'A'); + kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size); + host_do_rmw(t->mem, 0, 0, 'A'); + + test_convert_to_private(t, 0, 'A', 'B'); + + kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size); + test_private(t, 0, 0, 'A'); +} + int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); From 8b72eaa766b6b11a38faafa0b936d1952f1b7ba6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:56:01 -0700 Subject: [PATCH 273/506] KVM: selftests: Test that shared/private status is consistent across processes Add a test to verify that a guest_memfd's shared/private status is consistent across processes, and that any shared pages previously mapped in any process are unmapped from all processes. The test forks a child process after creating the shared guest_memfd region so that the second process exists alongside the main process for the entire test. The processes then take turns to access memory to check that the shared/private status is consistent across processes. Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-35-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/guest_memfd_conversions_test.c | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c index b7a54d2a9092..a972a09c5c6e 100644 --- a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -2,7 +2,9 @@ /* * Copyright (c) 2024, Google LLC. */ +#include #include +#include #include #include @@ -326,6 +328,122 @@ GMEM_CONVERSION_TEST_INIT_SHARED(truncate) test_private(t, 0, 0, 'A'); } +/* Test that shared/private memory protections work and are seen from any process. */ +GMEM_CONVERSION_TEST_INIT_SHARED(forked_accesses) +{ + enum test_state { + STATE_INIT, + STATE_CHECK_SHARED, + STATE_DONE_CHECKING_SHARED, + STATE_CHECK_PRIVATE, + STATE_DONE_CHECKING_PRIVATE, + }; + + struct sync_state { + pthread_mutex_t mutex; + pthread_cond_t cond; + enum test_state step; + } *sync; + + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; + pid_t child_pid, parent_pid; + int status; + + sync = kvm_mmap(sizeof(*sync), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); + + pthread_mutexattr_init(&mattr); + pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); + pthread_mutex_init(&sync->mutex, &mattr); + pthread_mutexattr_destroy(&mattr); + + pthread_condattr_init(&cattr); + pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); + pthread_cond_init(&sync->cond, &cattr); + pthread_condattr_destroy(&cattr); + + sync->step = STATE_INIT; + +#define TEST_STATE_AWAIT(__state) \ + do { \ + pthread_mutex_lock(&sync->mutex); \ + while (sync->step != (__state)) { \ + struct timespec ts, stop; \ + int ret; \ + \ + clock_gettime(CLOCK_REALTIME, &ts); \ + stop = timespec_add_ns(ts, 100 * 1000000UL); \ + \ + ret = pthread_cond_timedwait(&sync->cond, &sync->mutex, &stop); \ + if (ret == ETIMEDOUT) { \ + bool alive = (child_pid == 0) ? \ + (getppid() == parent_pid) : \ + (waitpid(child_pid, NULL, WNOHANG) == 0); \ + TEST_ASSERT(alive, "Other process exited prematurely"); \ + } else { \ + TEST_ASSERT(!ret, "pthread_cond_timedwait failed"); \ + } \ + } \ + pthread_mutex_unlock(&sync->mutex); \ + } while (0) + +#define TEST_STATE_SET(__state) \ + do { \ + pthread_mutex_lock(&sync->mutex); \ + sync->step = (__state); \ + pthread_cond_broadcast(&sync->cond); \ + pthread_mutex_unlock(&sync->mutex); \ + } while (0) + + parent_pid = getpid(); + child_pid = fork(); + TEST_ASSERT(child_pid != -1, "fork failed"); + + if (child_pid == 0) { + const char inconsequential = 0xdd; + + TEST_STATE_AWAIT(STATE_CHECK_SHARED); + + /* + * This maps the pages into the child process as well, and tests + * that the conversion process will unmap the guest_memfd memory + * from all processes. + */ + host_do_rmw(t->mem, 0, 0xB, 0xC); + + TEST_STATE_SET(STATE_DONE_CHECKING_SHARED); + TEST_STATE_AWAIT(STATE_CHECK_PRIVATE); + + TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[0])); + TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[0], inconsequential)); + + TEST_STATE_SET(STATE_DONE_CHECKING_PRIVATE); + exit(0); + } + + test_shared(t, 0, 0, 0xA, 0xB); + + TEST_STATE_SET(STATE_CHECK_SHARED); + TEST_STATE_AWAIT(STATE_DONE_CHECKING_SHARED); + + test_convert_to_private(t, 0, 0xC, 0xD); + + TEST_STATE_SET(STATE_CHECK_PRIVATE); + TEST_STATE_AWAIT(STATE_DONE_CHECKING_PRIVATE); + + TEST_ASSERT_EQ(waitpid(child_pid, &status, 0), child_pid); + TEST_ASSERT(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "Child exited with unexpected status"); + + pthread_mutex_destroy(&sync->mutex); + pthread_cond_destroy(&sync->cond); + kvm_munmap(sync, sizeof(*sync)); + +#undef TEST_STATE_SET +#undef TEST_STATE_AWAIT +} + int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); From e8a40b9a604e560b99f4dc4fbb4971143c75807e Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:56:02 -0700 Subject: [PATCH 274/506] KVM: selftests: Add helpers to pin pages with CONFIG_GUP_TEST Add helper functions to allow KVM selftests to pin memory using CONFIG_GUP_TEST. This is useful for creating test scenarios where some page has an increased refcount, such as when testing guest_memfd in-place conversion. The helpers open /sys/kernel/debug/gup_test and invoke the PIN_LONGTERM_TEST_START and PIN_LONGTERM_TEST_STOP ioctls. Since this functionality depends on the kernel being built with CONFIG_GUP_TEST, provide stub implementations that trigger a test failure if the configuration is missing. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-36-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 3 +++ tools/testing/selftests/kvm/lib/kvm_util.c | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 575032e9791d..3cfe243d7c8b 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -1206,6 +1206,9 @@ static inline int pin_self_to_any_cpu(void) return pin_task_to_any_cpu(pthread_self()); } +void pin_pages(void *vaddr, uint64_t size); +void unpin_pages(void); + void kvm_print_vcpu_pinning_help(void); void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[], int nr_vcpus); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index cd468b2bb834..b1456ed76e27 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -19,6 +19,8 @@ #include #include +#include "../../../../../mm/gup_test.h" + #define KVM_UTIL_MIN_PFN 2 u32 kvm_random_seed; @@ -645,6 +647,29 @@ int __pin_task_to_cpu(pthread_t task, int cpu) return pthread_setaffinity_np(task, sizeof(cpuset), &cpuset); } +static int gup_test_fd = -1; + +void pin_pages(void *vaddr, uint64_t size) +{ + const struct pin_longterm_test args = { + .addr = (uint64_t)vaddr, + .size = size, + .flags = PIN_LONGTERM_TEST_FLAG_USE_WRITE, + }; + + gup_test_fd = __open_path_or_exit("/sys/kernel/debug/gup_test", O_RDWR, + "Is CONFIG_GUP_TEST enabled?"); + + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_START, &args), 0); +} + +void unpin_pages(void) +{ + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_STOP), 0); + + kvm_free_fd(gup_test_fd); +} + static u32 parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask) { u32 pcpu = atoi_non_negative("CPU number", cpu_str); From 0db3f14f40c42dd0ac6a85b5e8fd605d37c8128a Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:56:03 -0700 Subject: [PATCH 275/506] KVM: selftests: Test conversion with elevated page refcount Add a selftest to verify that converting a shared guest_memfd page to a private page fails if the page has an elevated reference count. When KVM converts a shared page to a private one, it expects the page to have a reference count equal to the reference counts taken by the filemap. If another kernel subsystem holds a reference to the page, the conversion must be aborted. The test asserts that both bulk and single-page conversion attempts correctly fail with EAGAIN for the pinned page. After the page is unpinned, the test verifies that subsequent conversions succeed. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-37-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/guest_memfd_conversions_test.c | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c index a972a09c5c6e..e1c84df0f059 100644 --- a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -444,6 +444,62 @@ GMEM_CONVERSION_TEST_INIT_SHARED(forked_accesses) #undef TEST_STATE_AWAIT } +static void test_convert_to_private_fails(test_data_t *t, u64 pgoff, + size_t nr_pages, + u64 expected_error_offset) +{ + /* +1 to make it anything but expected_error_offset. */ + u64 error_offset = expected_error_offset + 1; + u64 offset = pgoff * page_size; + int ret; + + do { + ret = __gmem_set_private(t->gmem_fd, offset, + nr_pages * page_size, &error_offset); + } while (ret == -1 && errno == EINTR); + TEST_ASSERT(ret == -1 && errno == EAGAIN, + "Wanted EAGAIN on page %lu, got %d (ret = %d)", pgoff, + errno, ret); + TEST_ASSERT_EQ(error_offset, expected_error_offset); +} + +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(elevated_refcount, 4) +{ + int i; + + pin_pages(t->mem + test_page * page_size, page_size); + + for (i = 0; i < nr_pages; i++) + test_shared(t, i, 0, 'A', 'B'); + + /* + * Converting in bulk should fail as long any page in the range has + * unexpected refcounts. + */ + test_convert_to_private_fails(t, 0, nr_pages, test_page * page_size); + + for (i = 0; i < nr_pages; i++) { + /* + * Converting page-wise should also fail as long any page in the + * range has unexpected refcounts. + */ + if (i == test_page) + test_convert_to_private_fails(t, i, 1, test_page * page_size); + else + test_convert_to_private(t, i, 'B', 'C'); + } + + unpin_pages(); + + gmem_set_private(t->gmem_fd, 0, nr_pages * page_size); + + for (i = 0; i < nr_pages; i++) { + char expected = i == test_page ? 'B' : 'C'; + + test_private(t, i, expected, 'D'); + } +} + int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); From c235250e8bc1826ddab92565b659350e06e9d40c Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:56:04 -0700 Subject: [PATCH 276/506] KVM: selftests: Reset shared memory after hole-punching private_mem_conversions_test used to reset the shared memory that was used for the test to an initial pattern at the end of each test iteration. Then, it would punch out the pages, which would zero memory. Without in-place conversion, the resetting would write shared memory, and hole-punching will zero private memory, hence resetting the test to the state at the beginning of the for loop. With in-place conversion, resetting writes memory as shared, and hole-punching zeroes the same physical memory, hence undoing the reset done before the hole punch. Move the resetting after the hole-punching, and reset the entire PER_CPU_DATA_SIZE instead of just the tested range. With in-place conversion, this zeroes and then resets the same physical memory. Without in-place conversion, the private memory is zeroed, and the shared memory is reset to init_p. This is sufficient since at each test stage, the memory is assumed to start as shared, and private memory is always assumed to start zeroed. Conversion zeroes memory, so the future test stages will work as expected. Fixes: 43f623f350ce1 ("KVM: selftests: Add x86-only selftest for private memory conversions") Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-38-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/x86/private_mem_conversions_test.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c index f3de2a4ab055..cf50e9a332c5 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c @@ -202,15 +202,18 @@ static void guest_test_explicit_conversion(u64 base_gpa, bool do_fallocate) guest_sync_shared(gpa, size, p3, p4); memcmp_g(gpa, p4, size); - /* Reset the shared memory back to the initial pattern. */ - memset((void *)gpa, init_p, size); - /* * Free (via PUNCH_HOLE) *all* private memory so that the next * iteration starts from a clean slate, e.g. with respect to * whether or not there are pages/folios in guest_mem. */ guest_map_shared(base_gpa, PER_CPU_DATA_SIZE, true); + + /* + * Hole-punching above zeroed private memory. Reset shared + * memory in preparation for the next GUEST_STAGE. + */ + memset((void *)base_gpa, init_p, PER_CPU_DATA_SIZE); } } From ee6f4c381903e015b52a6797a7ef066966dc65ec Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:56:05 -0700 Subject: [PATCH 277/506] KVM: selftests: Provide function to look up guest_memfd details from gpa Introduce a new helper, kvm_gpa_to_guest_memfd(), to find the guest_memfd-related details of a memory region that contains a given guest physical address (GPA). The function returns the file descriptor for the memfd, the offset into the file that corresponds to the GPA, and the number of bytes remaining in the region from that GPA. kvm_gpa_to_guest_memfd() was factored out from vm_guest_mem_fallocate(); refactor vm_guest_mem_fallocate() to use the new helper. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-39-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 3 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 37 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 3cfe243d7c8b..795c9eb31107 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -428,6 +428,9 @@ static inline void vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes); + /* * KVM_SET_MEMORY_ATTRIBUTES{,2} overwrites _all_ attributes. These * flows need significant enhancements to support multiple attributes. diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index b1456ed76e27..1188d0a6c0a5 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1332,27 +1332,20 @@ void vm_guest_mem_fallocate(struct kvm_vm *vm, u64 base, u64 size, bool punch_hole) { const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0); - struct userspace_mem_region *region; u64 end = base + size; - gpa_t gpa, len; off_t fd_offset; - int ret; + int fd, ret; + size_t len; + gpa_t gpa; for (gpa = base; gpa < end; gpa += len) { - u64 offset; + fd = kvm_gpa_to_guest_memfd(vm, gpa, &fd_offset, &len); + len = min(end - gpa, len); - region = userspace_mem_region_find(vm, gpa, gpa); - TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, - "Private memory region not found for GPA 0x%lx", gpa); - - offset = gpa - region->region.guest_phys_addr; - fd_offset = region->region.guest_memfd_offset + offset; - len = min_t(u64, end - gpa, region->region.memory_size - offset); - - ret = fallocate(region->region.guest_memfd, mode, fd_offset, len); + ret = fallocate(fd, mode, fd_offset, len); TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx", punch_hole ? "punch hole" : "allocate", gpa, len, - region->region.guest_memfd, mode, fd_offset); + fd, mode, fd_offset); } } @@ -1689,6 +1682,22 @@ void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa) return (void *) ((uintptr_t) region->host_alias + offset); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes) +{ + struct userspace_mem_region *region; + gpa_t gpa_offset; + + region = userspace_mem_region_find(vm, gpa, gpa); + TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, + "guest_memfd memory region not found for GPA 0x%lx", gpa); + + gpa_offset = gpa - region->region.guest_phys_addr; + *fd_offset = region->region.guest_memfd_offset + gpa_offset; + *nr_bytes = region->region.memory_size - gpa_offset; + return region->region.guest_memfd; +} + /* Create an interrupt controller chip for the specified VM. */ void vm_create_irqchip(struct kvm_vm *vm) { From f49dd8ba330e89457c23448d4137b9f984cb0904 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:56:06 -0700 Subject: [PATCH 278/506] KVM: selftests: Provide common function to set memory attributes Introduce vm_mem_set_memory_attributes(), which handles setting of memory attributes for a range of guest physical addresses, regardless of whether the attributes should be set via guest_memfd or via the memory attributes at the VM level. Refactor existing vm_mem_set_{shared,private} functions to use the new function. Opportunistically update the size parameter to use size_t instead of u64. Co-developed-by: Ackerley Tng Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-40-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 795c9eb31107..777fa3dbf88d 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -454,18 +454,6 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); } -static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa, - u64 size) -{ - vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); -} - -static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa, - u64 size) -{ - vm_set_memory_attributes(vm, gpa, size, 0); -} - static inline int __gmem_set_memory_attributes(int fd, u64 offset, size_t size, u64 attributes, u64 *error_offset) @@ -532,6 +520,40 @@ static inline void gmem_set_shared(int fd, u64 offset, size_t size) gmem_set_memory_attributes(fd, offset, size, 0); } +static inline void vm_mem_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, + size_t size, u64 attrs) +{ + if (kvm_has_gmem_attributes) { + gpa_t end = gpa + size; + off_t fd_offset; + gpa_t addr; + size_t len; + int fd; + + for (addr = gpa; addr < end; addr += len) { + fd = kvm_gpa_to_guest_memfd(vm, addr, &fd_offset, &len); + len = min(end - addr, len); + + gmem_set_memory_attributes(fd, fd_offset, len, attrs); + } + } else { + vm_set_memory_attributes(vm, gpa, size, attrs); + } +} + +static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa, + size_t size) +{ + vm_mem_set_memory_attributes(vm, gpa, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); +} + +static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa, + size_t size) +{ + vm_mem_set_memory_attributes(vm, gpa, size, 0); +} + void vm_guest_mem_fallocate(struct kvm_vm *vm, gpa_t gpa, u64 size, bool punch_hole); From ef72e71da099c0cfb4fe581ac94168531ad2c1ea Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:56:07 -0700 Subject: [PATCH 279/506] KVM: selftests: Make TEST_EXPECT_SIGBUS thread-safe The TEST_EXPECT_SIGBUS macro is not thread-safe as it uses a global sigjmp_buf and installs a global SIGBUS signal handler. If multiple threads execute the macro concurrently, they will race on installing the signal handler and stomp on other threads' jump buffers, leading to incorrect test behavior. Make TEST_EXPECT_SIGBUS thread-safe with the following changes: Share the KVM tests' global signal handler. sigaction() applies to all threads; without sharing a global signal handler, one thread may have removed the signal handler that another thread added, hence leading to unexpected signals. The alternative of layering signal handlers was considered, but calling sigaction() within TEST_EXPECT_SIGBUS() necessarily creates a race. To avoid adding new setup and teardown routines to do sigaction() and keep usage of TEST_EXPECT_SIGBUS() simple, share the KVM tests' global signal handler. Opportunistically rename report_unexpected_signal to catchall_signal_handler. To continue to only expect SIGBUS within specific regions of code, use a thread-specific variable, expecting_sigbus, to replace installing and removing signal handlers. Make the execution environment for the thread, sigjmp_buf, a thread-specific variable. As part of TEST_EXPECT_SIGBUS(), assert the prerequisite for this setup, that the current signal handler is the catchall_signal_handler. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-41-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/test_util.h | 30 ++++++++++--------- tools/testing/selftests/kvm/lib/kvm_util.c | 18 +++++++---- tools/testing/selftests/kvm/lib/test_util.c | 7 ----- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index d5825211f272..75e98ec14107 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -83,21 +83,23 @@ do { \ __builtin_unreachable(); \ } while (0) -extern sigjmp_buf expect_sigbus_jmpbuf; -void expect_sigbus_handler(int signum); +extern __thread sigjmp_buf expect_sigbus_jmpbuf; +extern __thread volatile sig_atomic_t expecting_sigbus; +void catchall_signal_handler(int signum); -#define TEST_EXPECT_SIGBUS(action) \ -do { \ - struct sigaction sa_old, sa_new = { \ - .sa_handler = expect_sigbus_handler, \ - }; \ - \ - sigaction(SIGBUS, &sa_new, &sa_old); \ - if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ - action; \ - TEST_FAIL("'%s' should have triggered SIGBUS", #action); \ - } \ - sigaction(SIGBUS, &sa_old, NULL); \ +#define TEST_EXPECT_SIGBUS(action) \ +do { \ + struct sigaction __sa = {}; \ + \ + TEST_ASSERT_EQ(sigaction(SIGBUS, NULL, &__sa), 0); \ + TEST_ASSERT_EQ(__sa.sa_handler, &catchall_signal_handler); \ + \ + expecting_sigbus = true; \ + if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ + action; \ + TEST_FAIL("'%s' should have triggered SIGBUS", #action);\ + } \ + expecting_sigbus = false; \ } while (0) size_t parse_size(const char *size); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 1188d0a6c0a5..cac5624e9977 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -2316,13 +2316,20 @@ __weak void kvm_selftest_arch_init(void) { } -static void report_unexpected_signal(int signum) +__thread sigjmp_buf expect_sigbus_jmpbuf; +__thread volatile sig_atomic_t expecting_sigbus; + +void catchall_signal_handler(int signum) { + switch (signum) { + case SIGBUS: { + if (expecting_sigbus) + siglongjmp(expect_sigbus_jmpbuf, 1); + + TEST_FAIL("Unexpected SIGBUS (%d)\n", signum); + } #define KVM_CASE_SIGNUM(sig) \ case sig: TEST_FAIL("Unexpected " #sig " (%d)\n", signum) - - switch (signum) { - KVM_CASE_SIGNUM(SIGBUS); KVM_CASE_SIGNUM(SIGSEGV); KVM_CASE_SIGNUM(SIGILL); KVM_CASE_SIGNUM(SIGFPE); @@ -2334,12 +2341,13 @@ static void report_unexpected_signal(int signum) void __attribute((constructor)) kvm_selftest_init(void) { struct sigaction sig_sa = { - .sa_handler = report_unexpected_signal, + .sa_handler = catchall_signal_handler, }; /* Tell stdout not to buffer its content. */ setbuf(stdout, NULL); + expecting_sigbus = false; sigaction(SIGBUS, &sig_sa, NULL); sigaction(SIGSEGV, &sig_sa, NULL); sigaction(SIGILL, &sig_sa, NULL); diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index 4dff72f6bd34..89d6d1b57295 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -19,13 +19,6 @@ #include "test_util.h" #include "kvm_syscalls.h" -sigjmp_buf expect_sigbus_jmpbuf; - -void __attribute__((used)) expect_sigbus_handler(int signum) -{ - siglongjmp(expect_sigbus_jmpbuf, 1); -} - /* * Random number generator that is usable from guest code. This is the * Park-Miller LCG using standard constants. From 710169986d9b7df56e435a0c5890b355ac8603d8 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:56:08 -0700 Subject: [PATCH 280/506] KVM: selftests: Set up page size and alignment independently for guest_memfd Currently, vm_mem_add derives the backing source page size, alignment padding, and mmap size from the backing source type upfront before checking if guest_memfd is being mmapped. With shared memory also mmap()-ed from guest_memfd, the alignment of the mmap-ed address needs to respect guest_memfd's backing page size. Refactor the backing store setup to configure the backing source page size, alignment, mmap flags, and mmap offset directly for guest_memfd when it is mmapped, ignoring the backing source type. Skip hugepage validation and anonymous memory madvise calls when mmapping from guest_memfd, since those are not applicable when mmapping guest_memfd. Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-42-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/kvm_util.c | 74 +++++++++++++--------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index cac5624e9977..3f559f207408 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1034,11 +1034,12 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, { int ret; struct userspace_mem_region *region; - size_t backing_src_pagesz = get_backing_src_pagesz(src_type); - int mmap_flags = vm_mem_backing_src_alias(src_type)->flag; size_t mem_size = npages * vm->page_size; - off_t mmap_offset = 0; - size_t alignment = 1; + size_t backing_src_pagesz; + off_t mmap_offset; + bool is_gmem_mmap; + size_t alignment; + int mmap_flags; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -1090,19 +1091,31 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, /* Allocate and initialize new mem region structure. */ region = calloc(1, sizeof(*region)); TEST_ASSERT(region != NULL, "Insufficient Memory"); - region->mmap_size = mem_size; - /* - * When using THP mmap is not guaranteed to returned a hugepage aligned - * address so we have to pad the mmap. Padding is not needed for HugeTLB - * because mmap will always return an address aligned to the HugeTLB - * page size. - */ - if (src_type == VM_MEM_SRC_ANONYMOUS_THP) - alignment = max(backing_src_pagesz, alignment); + is_gmem_mmap = (flags & KVM_MEM_GUEST_MEMFD) && + (gmem_flags & GUEST_MEMFD_FLAG_MMAP); + + if (is_gmem_mmap) { + backing_src_pagesz = getpagesize(); + alignment = 1; + mmap_flags = MAP_SHARED; + mmap_offset = gmem_offset; + } else { + backing_src_pagesz = get_backing_src_pagesz(src_type); + /* + * When using THP mmap is not guaranteed to returned a hugepage aligned + * address so we have to pad the mmap. Padding is not needed for HugeTLB + * because mmap will always return an address aligned to the HugeTLB + * page size. + */ + alignment = src_type == VM_MEM_SRC_ANONYMOUS_THP ? backing_src_pagesz : 1; + mmap_flags = vm_mem_backing_src_alias(src_type)->flag; + mmap_offset = 0; + } TEST_ASSERT_EQ(gpa, align_up(gpa, backing_src_pagesz)); + region->mmap_size = mem_size; /* Add enough memory to align up if necessary */ if (alignment > 1) region->mmap_size += alignment; @@ -1129,10 +1142,8 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, } region->fd = -1; - if (flags & KVM_MEM_GUEST_MEMFD && gmem_flags & GUEST_MEMFD_FLAG_MMAP) { + if (is_gmem_mmap) { region->fd = kvm_dup(gmem_fd); - mmap_flags = MAP_SHARED; - mmap_offset = gmem_offset; } else if (backing_src_is_shared(src_type)) { region->fd = kvm_memfd_alloc(region->mmap_size, src_type == VM_MEM_SRC_SHARED_HUGETLB); @@ -1141,22 +1152,27 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, region->mmap_start = __kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, mmap_flags, region->fd, mmap_offset); - TEST_ASSERT(!is_backing_src_hugetlb(src_type) || - region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz), - "mmap_start %p is not aligned to HugeTLB page size 0x%lx", - region->mmap_start, backing_src_pagesz); - /* Align host address */ region->host_mem = align_ptr_up(region->mmap_start, alignment); - /* As needed perform madvise */ - if ((src_type == VM_MEM_SRC_ANONYMOUS || - src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { - ret = madvise(region->host_mem, mem_size, - src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); - TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", - region->host_mem, mem_size, - vm_mem_backing_src_alias(src_type)->name); + if (!is_gmem_mmap) { + TEST_ASSERT(!is_backing_src_hugetlb(src_type) || + region->mmap_start == + align_ptr_up(region->mmap_start, backing_src_pagesz), + "mmap_start %p is not aligned to HugeTLB page size 0x%lx", + region->mmap_start, backing_src_pagesz); + + /* As needed perform madvise */ + if ((src_type == VM_MEM_SRC_ANONYMOUS || + src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { + int advice = src_type == VM_MEM_SRC_ANONYMOUS ? + MADV_NOHUGEPAGE : MADV_HUGEPAGE; + + ret = madvise(region->host_mem, mem_size, advice); + TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", + region->host_mem, mem_size, + vm_mem_backing_src_alias(src_type)->name); + } } region->backing_src_type = src_type; From e24e534ff850b51520a45699cc9c7c1e2396c67d Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 10 Sep 2026 16:56:09 -0700 Subject: [PATCH 281/506] KVM: selftests: Update private_mem_conversions_test for in-place conversions Update private_mem_conversions_test for in-place conversions. In-place conversions support is detected in selftests with kvm_has_gmem_attributes. With in-place conversions, specifying userspace_addr from some other memory provider that is not the guest_memfd associated with the memslot is a user error, since KVM will only use both shared and private memory from the guest_memfd. Hence, when kvm_has_gmem_attributes, only test in-place conversions with single backing, where guest_memfd provides both shared and private memory. For single backing, guest_memfd must be created with GUEST_MEMFD_FLAG_MMAP. Initialize the guest_memfd as shared to align with how memory would default to shared when shared/private state was tracked at the VM level (the test expects this, it was written for non-in-place-conversions). When handling a hypercall to set attributes, use vm_mem_set_memory_attributes() to send the ioctl to the guest_memfd instead of the VM. When testing in-place conversions (single-backing), don't allow the user to configure src_type, since src_type will be ignored. Don't use src_type to determine alignment for rounding up per-cpu test memory size, since guest_memfd's backing page size is always the system PAGE_SIZE. Co-developed-by: Sean Christopherson Signed-off-by: Ackerley Tng Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-43-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/private_mem_conversions_test.c | 57 ++++++++++++++----- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c index cf50e9a332c5..33b610383693 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c @@ -306,9 +306,12 @@ static void handle_exit_hypercall(struct kvm_vcpu *vcpu) if (do_fallocate) vm_guest_mem_fallocate(vm, gpa, size, map_shared); - if (set_attributes) - vm_set_memory_attributes(vm, gpa, size, - map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE); + if (set_attributes) { + u64 attrs = map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE; + + vm_mem_set_memory_attributes(vm, gpa, size, attrs); + } + run->hypercall.ret = 0; } @@ -352,8 +355,21 @@ static void *__test_mem_conversions(void *__vcpu) size_t nr_bytes = min_t(size_t, vm->page_size, size - i); u8 *hva = addr_gpa2hva(vm, gpa + i); - /* In all cases, the host should observe the shared data. */ - memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + if (kvm_has_gmem_attributes && + uc.args[0] == SYNC_PRIVATE) { + TEST_EXPECT_SIGBUS(READ_ONCE(*hva)); + } else { + /* + * If not testing in-place conversion + * (dual backing), the host should + * always observe shared data, since the + * shared memory is a separate page. + * + * For SYNC_SHARED, test that the host + * can see shared memory. + */ + memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + } /* For shared, write the new pattern to guest memory. */ if (uc.args[0] == SYNC_SHARED) @@ -369,20 +385,29 @@ static void *__test_mem_conversions(void *__vcpu) } } +/* Align each vCPU's chunk of memory naturally to the size of the backing store. */ +static size_t compute_per_cpu_size(enum vm_mem_backing_src_type src_type) +{ + size_t alignment; + + if (kvm_has_gmem_attributes) + alignment = getpagesize(); + else + alignment = get_backing_src_pagesz(src_type); + + return align_up(PER_CPU_DATA_SIZE, max_t(size_t, SZ_2M, alignment)); +} + static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_vcpus, u32 nr_memslots) { - /* - * Allocate enough memory so that each vCPU's chunk of memory can be - * naturally aligned with respect to the size of the backing store. - */ - const size_t alignment = max_t(size_t, SZ_2M, get_backing_src_pagesz(src_type)); - const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment); + const size_t per_cpu_size = compute_per_cpu_size(src_type); const size_t memfd_size = per_cpu_size * nr_vcpus; const size_t slot_size = memfd_size / nr_memslots; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; pthread_t threads[KVM_MAX_VCPUS]; struct kvm_vm *vm; + u64 gmem_flags; int memfd, i; const struct vm_shape shape = { @@ -397,12 +422,16 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_v vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE)); - memfd = vm_create_guest_memfd(vm, memfd_size, 0); + gmem_flags = 0; + if (kvm_has_gmem_attributes) + gmem_flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; + + memfd = vm_create_guest_memfd(vm, memfd_size, gmem_flags); for (i = 0; i < nr_memslots; i++) vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i, BASE_DATA_SLOT + i, slot_size / vm->page_size, - KVM_MEM_GUEST_MEMFD, memfd, slot_size * i, 0); + KVM_MEM_GUEST_MEMFD, memfd, slot_size * i, gmem_flags); for (i = 0; i < nr_vcpus; i++) { gpa_t gpa = BASE_DATA_GPA + i * per_cpu_size; @@ -462,6 +491,8 @@ int main(int argc, char *argv[]) while ((opt = getopt(argc, argv, "hm:s:n:")) != -1) { switch (opt) { case 's': + TEST_ASSERT(!kvm_has_gmem_attributes, + "src_type is only configurable when testing without in-place conversion"); src_type = parse_backing_src_type(optarg); break; case 'n': From 37e0791600e5f1e2837a270c885296a172f5825e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 10 Sep 2026 16:56:10 -0700 Subject: [PATCH 282/506] KVM: selftests: Update private memory exits test to work with per-gmem attributes Skip setting memory to private in the private memory exits test when using per-gmem memory attributes, as memory is initialized to private by default for guest_memfd, and using vm_mem_set_private() on a guest_memfd instance requires creating guest_memfd with GUEST_MEMFD_FLAG_MMAP (which is totally doable, but would need to be conditional and is ultimately unnecessary). Expect an emulated MMIO instead of a memory fault exit when attributes are per-gmem, as deleting the memslot effectively drops the private status, i.e. the GPA becomes shared and thus supports emulated MMIO. Skip the "memslot not private" test entirely, as private vs. shared state for x86 software-protected VMs comes from the memory attributes themselves, and so when doing in-place conversions there can never be a disconnect between the expected and actual states. Signed-off-by: Ackerley Tng Tested-by: Shivank Garg Reviewed-by: Fuad Tabba Tested-by: Yan Zhao Link: https://patch.msgid.link/20260910-gmem-inplace-conversion-v13-44-dd6fbf94f4e1@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/private_mem_kvm_exits_test.c | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c index e6d16e9d6120..f7228e0f1ac6 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c @@ -62,8 +62,9 @@ static void test_private_access_memslot_deleted(void) virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES); - /* Request to access page privately */ - vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); + /* Request to access page privately. */ + if (!kvm_has_gmem_attributes) + vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); kvm_pthread_create(&vm_thread, NULL, (pthread_fn_t)run_vcpu_get_exit_reason, (void *)vcpu); @@ -73,10 +74,26 @@ static void test_private_access_memslot_deleted(void) kvm_pthread_join(vm_thread, &thread_return); exit_reason = (u32)(u64)thread_return; - TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); - TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); - TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); - TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + /* + * If attributes are tracked per-gmem, deleting the memslot that points + * at the gmem instance effectively makes the memory shared, and so the + * read should trigger emulated MMIO. + * + * If attributes are tracked per-VM, deleting the memslot shouldn't + * affect the private attribute, and so KVM should generate a memory + * fault exit (emulated MMIO on private GPAs is disallowed). + */ + if (kvm_has_gmem_attributes) { + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MMIO); + TEST_ASSERT_EQ(vcpu->run->mmio.phys_addr, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->mmio.len, sizeof(u64)); + TEST_ASSERT_EQ(vcpu->run->mmio.is_write, false); + } else { + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); + TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); + TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + } kvm_vm_free(vm); } @@ -87,6 +104,13 @@ static void test_private_access_memslot_not_private(void) struct kvm_vcpu *vcpu; u32 exit_reason; + /* + * Accessing non-private memory as private with a software-protected VM + * isn't possible when doing in-place conversions. + */ + if (kvm_has_gmem_attributes) + return; + vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu, guest_repeatedly_read); From 7d7199bdf77a3a6ed74d0037133f435a7febe97c Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Thu, 24 Sep 2026 10:49:50 +0100 Subject: [PATCH 283/506] KVM: arm64: ptdump: Check the page tables aren't freed when accessing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An open debugfs file keeps the KVM structure alive, but does not prevent mmu notifier release from freeing the stage-2 page tables when the VMM’s address space is torn down. Therefore page tables belonging to the mmus could have been freed when a thread opens or reads the ptdump files. Take the mmu_lock and check mmu->pgt is still alive before accessing the page tables. Also for the ipa_range file and stage2_levels file, switch from keeping the pointer to the page tables to keeping the pointer to the mmu. Fixes: 7c4f73548ed1 ("KVM: arm64: Register ptdump with debugfs on guest creation") Reported-by: Sashiko AI Closes: https://lore.kernel.org/kvmarm/20260623144054.B91D21F000E9@smtp.kernel.org/ Reviewed-by: Vincent Donnefort Tested-by: Vincent Donnefort Tested-by: Itaru Kitayama Signed-off-by: Wei-Lin Chang Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260924094951.727352-2-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/ptdump.c | 43 +++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/ptdump.c b/arch/arm64/kvm/ptdump.c index 69899797dbad..b5a0584260e8 100644 --- a/arch/arm64/kvm/ptdump.c +++ b/arch/arm64/kvm/ptdump.c @@ -115,15 +115,28 @@ static int kvm_ptdump_build_levels(struct ptdump_pg_level *level, u32 start_lvl) static struct kvm_ptdump_guest_state *kvm_ptdump_parser_create(struct kvm_s2_mmu *mmu) { + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); struct kvm_ptdump_guest_state *st; - struct kvm_pgtable *pgtable = mmu->pgt; + s8 start_level; int ret; + /* + * We only need the pgt start level to initialize the ptdump, get it + * while holding the mmu_lock. It's fine if the pgt gets freed + * afterwards, we'll check again when doing the actual dump. + */ + scoped_guard(read_lock, &kvm->mmu_lock) { + if (mmu->pgt) + start_level = mmu->pgt->start_level; + else + return ERR_PTR(-ENOENT); + } + st = kzalloc_obj(struct kvm_ptdump_guest_state, GFP_KERNEL_ACCOUNT); if (!st) return ERR_PTR(-ENOMEM); - ret = kvm_ptdump_build_levels(&st->level[0], pgtable->start_level); + ret = kvm_ptdump_build_levels(&st->level[0], start_level); if (ret) { kfree(st); return ERR_PTR(ret); @@ -149,6 +162,9 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused) }; guard(write_lock)(&kvm->mmu_lock); + if (!mmu->pgt) + return 0; + st->parser_state = (struct ptdump_pg_state) { .marker = &st->ipa_marker[0], .end_address = BIT(mmu->pgt->ia_bits), @@ -211,17 +227,27 @@ static const struct file_operations kvm_ptdump_guest_fops = { static int kvm_pgtable_range_show(struct seq_file *m, void *unused) { - struct kvm_pgtable *pgtable = m->private; + struct kvm_s2_mmu *mmu = m->private; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + guard(read_lock)(&kvm->mmu_lock); + + if (mmu->pgt) + seq_printf(m, "%2u\n", mmu->pgt->ia_bits); - seq_printf(m, "%2u\n", pgtable->ia_bits); return 0; } static int kvm_pgtable_levels_show(struct seq_file *m, void *unused) { - struct kvm_pgtable *pgtable = m->private; + struct kvm_s2_mmu *mmu = m->private; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + guard(read_lock)(&kvm->mmu_lock); + + if (mmu->pgt) + seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - mmu->pgt->start_level); - seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - pgtable->start_level); return 0; } @@ -230,15 +256,12 @@ static int kvm_pgtable_debugfs_open(struct inode *m, struct file *file, { struct kvm_s2_mmu *mmu = m->i_private; struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - struct kvm_pgtable *pgtable; int ret; if (!kvm_get_kvm_safe(kvm)) return -ENOENT; - pgtable = mmu->pgt; - - ret = single_open(file, show, pgtable); + ret = single_open(file, show, mmu); if (ret < 0) kvm_put_kvm(kvm); return ret; From b1cc219eb21ab5f31fe4caa32be8057159e42db5 Mon Sep 17 00:00:00 2001 From: Wei-Lin Chang Date: Thu, 24 Sep 2026 10:49:51 +0100 Subject: [PATCH 284/506] KVM: arm64: ptdump: Fix shadow ptdump sleep-in-atomic-context problem Previously we exposed shadow page tables by creating a debugfs ptdump file whenever a nested mmu instance gets bound to a new context, and deleting the debugfs file whose context was getting unbound. This turned out to be buggy, as the instance<->context binding process is done with the mmu_lock held, and debugfs creation/deletion can sleep. Change the approach and create a shadow ptdump file for each nested mmu instead during kvm_vcpu_init_nested(), in a sleepable context. The files will be named nested_mmu, and reading it will return the nested context's VTTBR, VTCR, and s2 enabled or not before dumping the shadow page tables, given the nested mmu is valid. Fixes: 19e15dc73f0f ("KVM: arm64: nv: Expose shadow page tables in debugfs") Reported-by: Itaru Kitayama Closes: https://lore.kernel.org/kvmarm/aiuF0KSvvv-ZozI1@sm-arm-grace07/ Tested-by: Itaru Kitayama Signed-off-by: Wei-Lin Chang Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260924094951.727352-3-weilin.chang@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 4 ---- arch/arm64/include/asm/kvm_mmu.h | 6 ++---- arch/arm64/kvm/nested.c | 9 ++++----- arch/arm64/kvm/ptdump.c | 32 +++++++++++++++---------------- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 8dfaa33c6cd5..255b30658d6f 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -245,10 +245,6 @@ struct kvm_s2_mmu { * >0: Somebody is actively using this. */ atomic_t refcnt; - -#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS - struct dentry *shadow_pt_debugfs_dentry; -#endif }; struct kvm_arch_memory_slot { diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 6eae7e7e2a68..ac2b0637692a 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -392,12 +392,10 @@ static inline bool kvm_supports_cacheable_pfnmap(void) #ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS void kvm_s2_ptdump_create_debugfs(struct kvm *kvm); -void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu); -void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu); +void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx); #else static inline void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) {} -static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) {} -static inline void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) {} +static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx) {} #endif /* CONFIG_PTDUMP_STAGE2_DEBUGFS */ #endif /* __ASSEMBLER__ */ diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index ec754a865a00..b9aa1472a037 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -127,6 +127,9 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) return ret; } + for (i = 0; i < S2_MMU_PER_VCPU; i++) + kvm_nested_s2_ptdump_create_debugfs(&tmp[i], i + kvm->arch.nested_mmus_size); + guard(write_lock)(&kvm->mmu_lock); for (i = 0; i < S2_MMU_PER_VCPU; i++) { @@ -842,10 +845,8 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; /* Make sure we don't forget to do the laundry */ - if (kvm_s2_mmu_valid(s2_mmu)) { - kvm_nested_s2_ptdump_remove_debugfs(s2_mmu); + if (kvm_s2_mmu_valid(s2_mmu)) s2_mmu->pending_unmap = true; - } /* * The virtual VMID (modulo CnP) will be used as a key when matching @@ -859,8 +860,6 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; - kvm_nested_s2_ptdump_create_debugfs(s2_mmu); - out: atomic_inc(&s2_mmu->refcnt); diff --git a/arch/arm64/kvm/ptdump.c b/arch/arm64/kvm/ptdump.c index b5a0584260e8..7829fe376385 100644 --- a/arch/arm64/kvm/ptdump.c +++ b/arch/arm64/kvm/ptdump.c @@ -17,7 +17,7 @@ #define MARKERS_LEN 2 #define KVM_PGTABLE_MAX_LEVELS (KVM_PGTABLE_LAST_LEVEL + 1) -#define S2FNAMESZ sizeof("0x0123456789abcdef-0x0123456789abcdef-s2-disabled") +#define S2FNAMESZ sizeof("nested_mmu0000") struct kvm_ptdump_guest_state { struct kvm_s2_mmu *mmu; @@ -173,6 +173,15 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused) .seq = m, }; + if (kvm_is_nested_s2_mmu(kvm, mmu)) { + if (kvm_s2_mmu_valid(mmu)) + seq_printf(m, "VTCR: 0x%016llx VTTBR: 0x%016llx s2: %s\n", + mmu->tlb_vtcr, mmu->tlb_vttbr, + mmu->nested_stage2_enabled ? "enabled" : "disabled"); + else + return 0; + } + ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker); if (ret) return ret; @@ -299,26 +308,15 @@ static const struct file_operations kvm_pgtable_levels_fops = { .release = kvm_pgtable_debugfs_close, }; -void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) +void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx) { - struct dentry *dent; char file_name[S2FNAMESZ]; - snprintf(file_name, sizeof(file_name), "0x%016llx-0x%016llx-s2-%sabled", - mmu->tlb_vttbr, - mmu->tlb_vtcr, - mmu->nested_stage2_enabled ? "en" : "dis"); + snprintf(file_name, sizeof(file_name), "nested_mmu%d", idx); - dent = debugfs_create_file(file_name, 0400, - mmu->arch->debugfs_nv_dentry, mmu, - &kvm_ptdump_guest_fops); - - mmu->shadow_pt_debugfs_dentry = dent; -} - -void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) -{ - debugfs_remove(mmu->shadow_pt_debugfs_dentry); + debugfs_create_file(file_name, 0400, + mmu->arch->debugfs_nv_dentry, mmu, + &kvm_ptdump_guest_fops); } void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) From caa63620f9e37a5eec4d0af13534ebd6c0d33f97 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:44 +0100 Subject: [PATCH 285/506] KVM: Allow architectures to disallow pre-fault All existing architectures which implement KVM pre-fault (x86, s390) have mechanisms for disallowing pre-faulting. Currently these are open coded as part of kvm_arch_vcpu_pre_fault_memory(). Formalise this by moving them into a new kvm_arch_pre_fault_allowed() hook, which every architecture selecting CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY must implement, returning an error code if the operation is disallowed or 0 otherwise. The hook is called early in the generic code, allowing architectures to disallow the operation prior to vCPU load. This is important, as kvm_vcpu_pre_fault_memory() is the only place where generic code can call vcpu_load() on a vCPU that has not yet been initialised. This lays the foundation for a future change which implements pre-faulting for arm64 which needs to disallow the mechanism for uninitialised vCPUs. Suggested-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Acked-by: Sean Christopherson Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-1-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/s390/kvm/s390/s390.c | 11 ++++++++--- arch/x86/kvm/mmu/mmu.c | 11 ++++++++--- arch/x86/kvm/vmx/tdx.c | 2 +- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 10 +++++++++- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index 5c73f43782a7..47fe032444f4 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -5784,6 +5784,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, s390_kvm_mmu_commit_memory_region(kvm, old, new, change); } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + + return 0; +} + /** * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables * @vcpu: the vcpu that shall appear to have generated the fault-in. @@ -5810,9 +5818,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_ gpa_t end; int rc; - if (kvm_is_ucontrol(vcpu->kvm)) - return -EINVAL; - rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); if (rc == PGM_ADDRESSING) return -ENOENT; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..c35fd2868c20 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5086,6 +5086,14 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, } } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (!vcpu->kvm->arch.pre_fault_allowed) + return -EOPNOTSUPP; + + return 0; +} + long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) { @@ -5095,9 +5103,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, u64 end; int r; - if (!vcpu->kvm->arch.pre_fault_allowed) - return -EOPNOTSUPP; - if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa))) return -EINVAL; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..a4d4479e2b28 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1708,7 +1708,7 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK); /* - * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory() + * Ensure pre_fault_allowed is read by kvm_arch_pre_fault_allowed() * before kvm_tdx->state. Userspace must not be allowed to pre-fault * arbitrary memory until the initial memory image is finalized. Pairs * with the smp_wmb() in tdx_td_finalize(). diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..39a4d345aeca 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2643,6 +2643,7 @@ void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu); long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range); #endif diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..41356577bee7 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4353,7 +4353,7 @@ static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) { - int idx; + int idx, err; long r; u64 full_size; @@ -4365,6 +4365,14 @@ static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, range->gpa + range->size <= range->gpa) return -EINVAL; + /* + * Certain architectures (e.g. arm64) need to reject the ioctl 'early' + * before vcpu_load(). + */ + err = kvm_arch_pre_fault_allowed(vcpu); + if (err) + return err; + vcpu_load(vcpu); idx = srcu_read_lock(&vcpu->kvm->srcu); From eb2eb48c8d72b19a7163cfc9e4b416e60f5b9e2d Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:45 +0100 Subject: [PATCH 286/506] arm64: Add ESR fault helpers Add helper functions which operate directly on an ESR value rather than trying to access the register itself. These mirror equivalent KVM vCPU wrappers in kvm_emulate.h and allow those wrappers and KVM's stage-2 fault handling to operate on a plain ESR value. This is needed to later generate a synthetic fault for the stage-2 page table pre-faulting mechanism. No functional change intended. Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-2-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/esr.h | 29 ++++++++++++++++++++++++++++ arch/arm64/include/asm/kvm_emulate.h | 10 ++++++++++ 2 files changed, 39 insertions(+) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index f816f5d77f1a..86756fc9bb5a 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -437,6 +437,35 @@ #ifndef __ASSEMBLER__ #include +static __always_inline bool esr_trap_is_iabt(unsigned long esr) +{ + return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; +} + +/* Always check for S1PTW *before* using this. */ +static __always_inline bool esr_dabt_is_write(unsigned long esr) +{ + return esr & ESR_ELx_WNR; +} + +static __always_inline bool esr_dabt_is_cm(unsigned long esr) +{ + return esr & ESR_ELx_CM; +} + +static __always_inline bool esr_abt_is_sea(unsigned long esr) +{ + switch (esr & ESR_ELx_FSC) { + case ESR_ELx_FSC_EXTABT: + case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3): + case ESR_ELx_FSC_SECC: + case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3): + return true; + default: + return false; + } +} + static inline unsigned long esr_brk_comment(unsigned long esr) { return esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index a3c1928bdf74..654e8c1b4461 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -336,6 +336,16 @@ static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu) return vcpu->arch.fault.esr_el2; } +static __always_inline bool esr_abt_is_s1ptw(unsigned long esr) +{ + return esr & ESR_ELx_S1PTW; +} + +static __always_inline bool esr_abt_is_exec_fault(unsigned long esr) +{ + return esr_trap_is_iabt(esr) && !esr_abt_is_s1ptw(esr); +} + static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu) { u64 esr = kvm_vcpu_get_esr(vcpu); From 18b94675b1fb22b5ce9b607f9de818d916b3da16 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:46 +0100 Subject: [PATCH 287/506] KVM: arm64: Use ESR helpers in guest abort handling Convert kvm_vcpu_* ESR wrappers in kvm_emulate.h and the stage-2 abort handling logic in mmu.c to use the newly introduced ESR helpers. kvm_is_write_fault() is split in two, with esr_abt_is_write_fault() handling the esr parts of the operation and kvm_is_write_fault() wrapping it. Introduce/modify kvm_s2_fault_is_{perm,exec,write}(), kvm_s2_perm_fault_granule() so the abort path reads the ESR from a single place. This is to allow a later change to permit stage 2 pre-faulting via a synthetic ESR value. Remove now-unused kvm_vcpu_dabt_is_cm(), kvm_vcpu_trap_is_exec_fault() and kvm_vcpu_trap_get_perm_fault_granule(). No functional change intended. Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-3-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_emulate.h | 50 +++++------------- arch/arm64/kvm/mmu.c | 78 +++++++++++++++++----------- 2 files changed, 61 insertions(+), 67 deletions(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 654e8c1b4461..708a4b6d8823 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -421,18 +421,13 @@ static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu) static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW); + return esr_abt_is_s1ptw(kvm_vcpu_get_esr(vcpu)); } /* Always check for S1PTW *before* using this. */ static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR; -} - -static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu) -{ - return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM); + return esr_dabt_is_write(kvm_vcpu_get_esr(vcpu)); } static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu) @@ -453,12 +448,7 @@ static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW; -} - -static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu); + return esr_trap_is_iabt(kvm_vcpu_get_esr(vcpu)); } static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) @@ -478,26 +468,9 @@ bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu) return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu)); } -static inline -u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu) -{ - unsigned long esr = kvm_vcpu_get_esr(vcpu); - - BUG_ON(!esr_fsc_is_permission_fault(esr)); - return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL)); -} - static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu) { - switch (kvm_vcpu_trap_get_fault(vcpu)) { - case ESR_ELx_FSC_EXTABT: - case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3): - case ESR_ELx_FSC_SECC: - case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3): - return true; - default: - return false; - } + return esr_abt_is_sea(kvm_vcpu_get_esr(vcpu)); } static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) @@ -506,9 +479,9 @@ static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) return ESR_ELx_SYS64_ISS_RT(esr); } -static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) +static inline bool esr_abt_is_write_fault(unsigned long esr) { - if (kvm_vcpu_abt_iss1tw(vcpu)) { + if (esr_abt_is_s1ptw(esr)) { /* * Only a permission fault on a S1PTW should be * considered as a write. Otherwise, page tables baked @@ -521,13 +494,18 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) * first), then a permission fault to allow the flags * to be set. */ - return kvm_vcpu_trap_is_permission_fault(vcpu); + return esr_fsc_is_permission_fault(esr); } - if (kvm_vcpu_trap_is_iabt(vcpu)) + if (esr_trap_is_iabt(esr)) return false; - return kvm_vcpu_dabt_iswrite(vcpu); + return esr_dabt_is_write(esr); +} + +static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) +{ + return esr_abt_is_write_fault(kvm_vcpu_get_esr(vcpu)); } static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index f8d3b7fe631f..cfa6dec2d99f 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1638,10 +1638,35 @@ struct kvm_s2_fault_desc { unsigned long hva; }; +static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_fsc_is_permission_fault(kvm_vcpu_get_esr(s2fd->vcpu)); +} + +static bool kvm_s2_fault_is_exec(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_abt_is_exec_fault(kvm_vcpu_get_esr(s2fd->vcpu)); +} + +static bool kvm_s2_fault_is_write(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_abt_is_write_fault(kvm_vcpu_get_esr(s2fd->vcpu)); +} + +static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd) +{ + u64 level; + + if (!kvm_s2_fault_is_perm(s2fd)) + return 0; + level = kvm_vcpu_get_esr(s2fd->vcpu) & ESR_ELx_FSC_LEVEL; + return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); +} + static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) { bool write_fault, exec_fault; - bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); + bool perm_fault = kvm_s2_fault_is_perm(s2fd); enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt; @@ -1671,8 +1696,8 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) else gfn = s2fd->fault_ipa >> PAGE_SHIFT; - write_fault = kvm_is_write_fault(s2fd->vcpu); - exec_fault = kvm_vcpu_trap_is_exec_fault(s2fd->vcpu); + write_fault = kvm_s2_fault_is_write(s2fd); + exec_fault = kvm_s2_fault_is_exec(s2fd); VM_WARN_ON_ONCE(write_fault && exec_fault); @@ -1891,11 +1916,6 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, return vma_shift; } -static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) -{ - return kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); -} - static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd, struct kvm_s2_fault_vma_info *s2vi) { @@ -1961,7 +1981,7 @@ static int kvm_s2_fault_pin_pfn(const struct kvm_s2_fault_desc *s2fd, return ret; s2vi->pfn = __kvm_faultin_pfn(s2fd->memslot, get_canonical_gfn(s2fd, s2vi), - kvm_is_write_fault(s2fd->vcpu) ? FOLL_WRITE : 0, + kvm_s2_fault_is_write(s2fd) ? FOLL_WRITE : 0, &s2vi->map_writable, &s2vi->page); if (unlikely(is_error_noslot_pfn(s2vi->pfn))) { if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) { @@ -2019,7 +2039,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, { struct kvm *kvm = s2fd->vcpu->kvm; - if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu) && s2vi->map_non_cacheable) + if (kvm_s2_fault_is_exec(s2fd) && s2vi->map_non_cacheable) return -ENOEXEC; /* @@ -2037,13 +2057,13 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, if (s2vi->map_writable && (s2vi->device || !memslot_is_logging(s2fd->memslot) || - kvm_is_write_fault(s2fd->vcpu))) + kvm_s2_fault_is_write(s2fd))) *prot |= KVM_PGTABLE_PROT_W; if (s2fd->nested) *prot = adjust_nested_fault_perms(s2fd->nested, *prot); - if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu)) + if (kvm_s2_fault_is_exec(s2fd)) *prot |= KVM_PGTABLE_PROT_X; if (s2vi->map_non_cacheable) @@ -2096,8 +2116,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, if (mmu_invalidate_retry(kvm, s2vi->mmu_seq)) goto out_unlock; - perm_fault_granule = (kvm_s2_fault_is_perm(s2fd) ? - kvm_vcpu_trap_get_perm_fault_granule(s2fd->vcpu) : 0); + perm_fault_granule = kvm_s2_perm_fault_granule(s2fd); mapping_size = s2vi->vma_pagesize; pfn = s2vi->pfn; gfn = s2vi->gfn; @@ -2176,7 +2195,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) { - bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); + bool perm_fault = kvm_s2_fault_is_perm(s2fd); struct kvm_s2_fault_vma_info s2vi = {}; enum kvm_pgtable_prot prot; void *memcache; @@ -2323,7 +2342,7 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu) int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) { struct kvm_s2_trans nested_trans, *nested = NULL; - unsigned long esr; + unsigned long esr = kvm_vcpu_get_esr(vcpu); phys_addr_t fault_ipa; /* The address we faulted on */ phys_addr_t ipa; /* Always the IPA in the L1 guest phys space */ struct kvm_memory_slot *memslot; @@ -2332,11 +2351,9 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) gfn_t gfn; int ret, idx; - if (kvm_vcpu_abt_issea(vcpu)) + if (esr_abt_is_sea(esr)) return kvm_handle_guest_sea(vcpu); - esr = kvm_vcpu_get_esr(vcpu); - /* * The fault IPA should be reliable at this point as we're not dealing * with an SEA. @@ -2345,7 +2362,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm)) return -EFAULT; - is_iabt = kvm_vcpu_trap_is_iabt(vcpu); + is_iabt = esr_trap_is_iabt(esr); if (esr_fsc_is_translation_fault(esr)) { /* Beyond sanitised PARange (which is the IPA limit) */ @@ -2362,7 +2379,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } } - trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu), + trace_kvm_guest_fault(*vcpu_pc(vcpu), esr, kvm_vcpu_get_hfar(vcpu), fault_ipa); /* Check the stage-2 fault is trans. fault or write fault */ @@ -2370,10 +2387,10 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) !esr_fsc_is_permission_fault(esr) && !esr_fsc_is_access_flag_fault(esr) && !esr_fsc_is_excl_atomic_fault(esr)) { - kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", - kvm_vcpu_trap_get_class(vcpu), - (unsigned long)kvm_vcpu_trap_get_fault(vcpu), - (unsigned long)kvm_vcpu_get_esr(vcpu)); + kvm_err("Unsupported FSC: EC=%#lx xFSC=%#lx ESR_EL2=%#lx\n", + ESR_ELx_EC(esr), + (unsigned long)(esr & ESR_ELx_FSC), + (unsigned long)esr); return -EFAULT; } @@ -2422,7 +2439,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) gfn = ipa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); - write_fault = kvm_is_write_fault(vcpu); + write_fault = esr_abt_is_write_fault(esr); if (kvm_is_error_hva(hva) || (write_fault && !writable)) { /* * The guest has put either its instructions or its page-tables @@ -2435,7 +2452,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) goto out; } - if (kvm_vcpu_abt_iss1tw(vcpu)) { + if (esr_abt_is_s1ptw(esr)) { ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); goto out_unlock; } @@ -2450,7 +2467,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * So let's assume that the guest is just being * cautious, and skip the instruction. */ - if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) { + if (kvm_is_error_hva(hva) && esr_dabt_is_cm(esr)) { kvm_incr_pc(vcpu); ret = 1; goto out_unlock; @@ -2487,9 +2504,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) if (kvm_vm_is_protected(vcpu->kvm)) { ret = pkvm_mem_abort(&s2fd); } else { - VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && - !write_fault && - !kvm_vcpu_trap_is_exec_fault(vcpu)); + VM_WARN_ON_ONCE(kvm_s2_fault_is_perm(&s2fd) && !write_fault && + !kvm_s2_fault_is_exec(&s2fd)); if (kvm_slot_has_gmem(memslot)) ret = gmem_abort(&s2fd); From 2b8033a99631b744a67a6fa5df46d0d7b3b9ab7c Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:47 +0100 Subject: [PATCH 288/506] KVM: arm64: Propagate and use esr in s2fd when handling guest aborts kvm_handle_guest_abort() establishes a kvm_s2_fault_desc data structure, s2fd, to store and propagate state to either pkvm_mem_abort(), gmem_abort() or user_mem_abort() handlers. Each of these, however, examines the Exception Syndrome Register (ESR) via s2fd->vcpu. Introduce an s2fd->esr field to abstract this and propagate it to callers. The value of this (beyond refactoring) is to be able to later generate faults with a synthetic esr, specifically to implement stage 2 page table pre-faulting. Abort handlers which use kvm_s2_fault_desc - gmem_abort() and user_mem_abort() - now only reference s2fd->esr and do not look it up in any other way, which makes it safe to pass a synthetic s2fd->esr value to these functions. No functional change intended. Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-4-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index cfa6dec2d99f..01678cc7a285 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1636,21 +1636,22 @@ struct kvm_s2_fault_desc { struct kvm_s2_trans *nested; struct kvm_memory_slot *memslot; unsigned long hva; + unsigned long esr; }; static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) { - return esr_fsc_is_permission_fault(kvm_vcpu_get_esr(s2fd->vcpu)); + return esr_fsc_is_permission_fault(s2fd->esr); } static bool kvm_s2_fault_is_exec(const struct kvm_s2_fault_desc *s2fd) { - return esr_abt_is_exec_fault(kvm_vcpu_get_esr(s2fd->vcpu)); + return esr_abt_is_exec_fault(s2fd->esr); } static bool kvm_s2_fault_is_write(const struct kvm_s2_fault_desc *s2fd) { - return esr_abt_is_write_fault(kvm_vcpu_get_esr(s2fd->vcpu)); + return esr_abt_is_write_fault(s2fd->esr); } static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd) @@ -1659,7 +1660,7 @@ static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd) if (!kvm_s2_fault_is_perm(s2fd)) return 0; - level = kvm_vcpu_get_esr(s2fd->vcpu) & ESR_ELx_FSC_LEVEL; + level = s2fd->esr & ESR_ELx_FSC_LEVEL; return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); } @@ -2048,7 +2049,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, * and trigger the exception here. Since the memslot is valid, inject * the fault back to the guest. */ - if (esr_fsc_is_excl_atomic_fault(kvm_vcpu_get_esr(s2fd->vcpu))) { + if (esr_fsc_is_excl_atomic_fault(s2fd->esr)) { kvm_inject_dabt_excl_atomic(s2fd->vcpu, kvm_vcpu_get_hfar(s2fd->vcpu)); return 1; } @@ -2499,6 +2500,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) .nested = nested, .memslot = memslot, .hva = hva, + .esr = esr, }; if (kvm_vm_is_protected(vcpu->kvm)) { From b477bcdfa0342145084948a079b606e058066d80 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:48 +0100 Subject: [PATCH 289/506] KVM: arm64: Propagate and use mmu in s2fd when handling guest aborts kvm_handle_guest_abort() establishes a kvm_s2_fault_desc data structure, s2fd, to store and propagate state to either pkvm_mem_abort(), gmem_abort() or user_mem_abort() handlers. Each of these, however, examines the state of the stage 2 MMU via vcpu->arch.hw_mmu. Introduce an s2fd->mmu field to abstract this and propagate it to callers. Similar to adding the esr field, this allows injection of synthetic faults with the ultimate intention of implementing stage 2 page table pre-faulting. No functional change intended. Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-5-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 01678cc7a285..f398e0857684 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1637,6 +1637,7 @@ struct kvm_s2_fault_desc { struct kvm_memory_slot *memslot; unsigned long hva; unsigned long esr; + struct kvm_s2_mmu *mmu; }; static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) @@ -1670,7 +1671,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) bool perm_fault = kvm_s2_fault_is_perm(s2fd); enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; - struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt; + struct kvm_pgtable *pgt = s2fd->mmu->pgt; struct kvm_guest_s2_mapping *mapping = NULL; unsigned long mmu_seq; struct page *page; @@ -1786,7 +1787,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) { unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; struct kvm_vcpu *vcpu = s2fd->vcpu; - struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt; + struct kvm_pgtable *pgt = s2fd->mmu->pgt; struct mm_struct *mm = current->mm; struct kvm *kvm = vcpu->kvm; void *hyp_memcache; @@ -2102,7 +2103,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, gfn_t gfn; int ret; - if (kvm_is_nested_s2_mmu(kvm, s2fd->vcpu->arch.hw_mmu)) { + if (kvm_is_nested_s2_mmu(kvm, s2fd->mmu)) { mapping = kmalloc_obj(struct kvm_guest_s2_mapping, GFP_KERNEL_ACCOUNT); if (!mapping) { @@ -2112,7 +2113,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, } kvm_fault_lock(kvm); - pgt = s2fd->vcpu->arch.hw_mmu->pgt; + pgt = s2fd->mmu->pgt; ret = -EAGAIN; if (mmu_invalidate_retry(kvm, s2vi->mmu_seq)) goto out_unlock; @@ -2344,6 +2345,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) { struct kvm_s2_trans nested_trans, *nested = NULL; unsigned long esr = kvm_vcpu_get_esr(vcpu); + struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; phys_addr_t fault_ipa; /* The address we faulted on */ phys_addr_t ipa; /* Always the IPA in the L1 guest phys space */ struct kvm_memory_slot *memslot; @@ -2373,7 +2375,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } /* Falls between the IPA range and the PARange? */ - if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(vcpu->arch.hw_mmu->vtcr))) { + if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(mmu->vtcr))) { fault_ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(vcpu)); return kvm_inject_sea(vcpu, is_iabt, fault_ipa); @@ -2410,8 +2412,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * nothing to walk and we treat it as a 1:1 before going through the * canonical translation. */ - if (kvm_is_nested_s2_mmu(vcpu->kvm,vcpu->arch.hw_mmu) && - vcpu->arch.hw_mmu->nested_stage2_enabled) { + if (kvm_is_nested_s2_mmu(vcpu->kvm, mmu) && + mmu->nested_stage2_enabled) { u32 esr; ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans); @@ -2486,7 +2488,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } /* Userspace should not be able to register out-of-bounds IPAs */ - VM_BUG_ON(ipa >= kvm_phys_size(vcpu->arch.hw_mmu)); + VM_BUG_ON(ipa >= kvm_phys_size(mmu)); if (esr_fsc_is_access_flag_fault(esr)) { handle_access_fault(vcpu, fault_ipa); @@ -2501,6 +2503,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) .memslot = memslot, .hva = hva, .esr = esr, + .mmu = mmu, }; if (kvm_vm_is_protected(vcpu->kvm)) { From f1fbc7ef12caf0785fdb3ea24193dbd9c425d7b7 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:49 +0100 Subject: [PATCH 290/506] KVM: arm64: Propagate and use kvm_s2_fault_result on S2 fault When stage 2 page tables fault the net result may either be that a page is mapped, an error occurred or the fault should be retried (-EAGAIN). When a fault succeeds it may be upgraded to a PMD size via transparent_hugepage_adjust(). In order to support KVM pre-faulting the outcome of the fault and the mapping size must be recorded. Track the mapping size via new kvm_s2_fault_result struct, which is threaded through gmem_abort(), user_mem_abort() and kvm_s2_fault_map(). PKVM and SEA aren't relevant to synthetic pre-faulting so neither kvm_inject_sea() nor pkvm_mem_abort() are altered. Actual hardware faulting doesn't require this information, so kvm_handle_guest_abort() simply passes NULL kvm_s2_fault_result to gmem_abort() and user_mem_abort(). A non-NULL result also tells the abort handlers that the caller is pre-faulting rather than a vcpu, in which case -EAGAIN is propagated to the caller to allow the pre-fault to be retried. Likewise, only prepare a KVM_EXIT_MEMORY_FAULT in gmem_abort() when a real fault is being handled, as this updates vcpu state. No functional change intended. Suggested-by: Vincent Donnefort Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-6-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index f398e0857684..71ce5b361e52 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1640,6 +1640,10 @@ struct kvm_s2_fault_desc { struct kvm_s2_mmu *mmu; }; +struct kvm_s2_fault_result { + unsigned long mapping_size; +}; + static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) { return esr_fsc_is_permission_fault(s2fd->esr); @@ -1665,7 +1669,8 @@ static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd) return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); } -static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) +static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_result *result) { bool write_fault, exec_fault; bool perm_fault = kvm_s2_fault_is_perm(s2fd); @@ -1709,8 +1714,10 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL); if (ret) { - kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, - write_fault, exec_fault, false); + /* If result is non-NULL this is a synthetic fault. */ + if (!result) + kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, + write_fault, exec_fault, false); kfree(mapping); return ret; } @@ -1765,7 +1772,13 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) if ((prot & KVM_PGTABLE_PROT_W) && !ret) mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn); - return ret != -EAGAIN ? ret : 0; + if (ret == -EAGAIN) + return result ? ret : 0; + + if (result && !ret) + result->mapping_size = PAGE_SIZE; + + return ret; } struct kvm_s2_fault_vma_info { @@ -2089,7 +2102,8 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, const struct kvm_s2_fault_vma_info *s2vi, enum kvm_pgtable_prot prot, - void *memcache) + void *memcache, + struct kvm_s2_fault_result *result) { enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; struct kvm_guest_s2_mapping *mapping = NULL; @@ -2190,12 +2204,17 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, mark_page_dirty_in_slot(kvm, s2fd->memslot, gpa_to_gfn(canonical_ipa)); - if (ret != -EAGAIN) - return ret; - return 0; + if (ret == -EAGAIN) + return result ? ret : 0; + + if (result && !ret) + result->mapping_size = mapping_size; + + return ret; } -static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) +static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_result *result) { bool perm_fault = kvm_s2_fault_is_perm(s2fd); struct kvm_s2_fault_vma_info s2vi = {}; @@ -2234,7 +2253,7 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) return ret; } - return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache); + return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache, result); } /* Resolve the access fault by making the page young again. */ @@ -2513,9 +2532,9 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) !kvm_s2_fault_is_exec(&s2fd)); if (kvm_slot_has_gmem(memslot)) - ret = gmem_abort(&s2fd); + ret = gmem_abort(&s2fd, NULL); else - ret = user_mem_abort(&s2fd); + ret = user_mem_abort(&s2fd, NULL); } if (ret == 0) From d27acc152c62f368d4d4d9d3f0faa891c154e4ce Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:50 +0100 Subject: [PATCH 291/506] KVM: arm64: Size the stage-2 memcache from the fault MMU topup_mmu_memcache() sizes the page-table cache from vcpu->arch.hw_mmu, which is the MMU that faulted. Pass the target s2fd->mmu instead, so the mmu used can be controlled by the caller. For a guest abort this is vcpu->arch.hw_mmu, so this commit introduces no functional change in practice. However, it's necessary for a future change which introduces stage 2 pre-faulting which must always target the canonical MMU regardless of the vCPU's last-run context. No functional change intended. Reviewed-by: Oliver Upton Reviewed-by: Fuad Tabba Signed-off-by: Lorenzo Stoakes (ARM) Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-7-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 71ce5b361e52..cfe0b87631ff 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1587,9 +1587,9 @@ static void *get_mmu_memcache(struct kvm_vcpu *vcpu) return &vcpu->arch.pkvm_memcache; } -static int topup_mmu_memcache(struct kvm_vcpu *vcpu, void *memcache) +static int topup_mmu_memcache(struct kvm_s2_mmu *mmu, void *memcache) { - int min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu); + int min_pages = kvm_mmu_cache_min_pages(mmu); if (!is_protected_kvm_enabled()) return kvm_mmu_topup_memory_cache(memcache, min_pages); @@ -1688,7 +1688,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, if (!perm_fault) { memcache = get_mmu_memcache(s2fd->vcpu); - ret = topup_mmu_memcache(s2fd->vcpu, memcache); + ret = topup_mmu_memcache(s2fd->mmu, memcache); if (ret) return ret; if (kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { @@ -1808,7 +1808,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) int ret; hyp_memcache = get_mmu_memcache(vcpu); - ret = topup_mmu_memcache(vcpu, hyp_memcache); + ret = topup_mmu_memcache(s2fd->mmu, hyp_memcache); if (ret) return -ENOMEM; @@ -2234,7 +2234,7 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd, memcache = get_mmu_memcache(s2fd->vcpu); if (!perm_fault || memslot_is_logging(s2fd->memslot) || is_protected_kvm_enabled()) { - ret = topup_mmu_memcache(s2fd->vcpu, memcache); + ret = topup_mmu_memcache(s2fd->mmu, memcache); if (ret) return ret; } From 33f849d808e2329f569bcd8fdbfe5ce22449466f Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:51 +0100 Subject: [PATCH 292/506] KVM: arm64: Propagate EHWPOISON in kvm_s2_fault_pin_pfn() Currently kvm_s2_fault_pin_pfn() handles a poisoned page directly by sending a SIGBUS signal itself. This is an odd place to do it, the caller should decide what to do with errors, so move the handling to the sole caller, user_mem_abort(). This lays the foundation for stage 2 pre-faulting which, arising from a synthetic fault, should not send a signal. In order to do so, check to see if user_mem_abort()'s caller has set result - i.e. whether it wants to be informed about the outcome of the fault handling. If it does, then it is implied that it should handle the -EHWPOISON error itself. This is the case for pre-faulting. Otherwise this is real hardware, so send the signal. No functional change intended. Reviewed-by: Oliver Upton Reviewed-by: Fuad Tabba Signed-off-by: Lorenzo Stoakes (ARM) Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-8-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index cfe0b87631ff..2496fc390ab3 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1999,10 +1999,8 @@ static int kvm_s2_fault_pin_pfn(const struct kvm_s2_fault_desc *s2fd, kvm_s2_fault_is_write(s2fd) ? FOLL_WRITE : 0, &s2vi->map_writable, &s2vi->page); if (unlikely(is_error_noslot_pfn(s2vi->pfn))) { - if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) { - kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi->vma_pagesize)); - return 0; - } + if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) + return -EHWPOISON; return -EFAULT; } @@ -2244,6 +2242,13 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd, * get block mapping for device MMIO region. */ ret = kvm_s2_fault_pin_pfn(s2fd, &s2vi); + if (ret == -EHWPOISON) { + /* If result is specified, let the caller handle this. */ + if (result) + return -EHWPOISON; + kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi.vma_pagesize)); + return 0; + } if (ret != 1) return ret; From ee26443f3dd70a961ad5e9384d5f41a8675213d1 Mon Sep 17 00:00:00 2001 From: Jack Thomson Date: Wed, 23 Sep 2026 16:15:52 +0100 Subject: [PATCH 293/506] KVM: arm64: Pass walk flags to kvm_pgtable_get_leaf() Allow callers of kvm_pgtable_get_leaf() to specify the page-table walk flags, in preparation for performing walks under the MMU read lock. Reading a stage-2 leaf while only holding the read lock requires KVM_PGTABLE_WALK_SHARED: parallel faults (which also only hold the read lock) can unlink table pages and free them via RCU, so the walker must be inside an RCU read-side critical section, which the shared walk flag provides via kvm_pgtable_walk_begin(). All existing callers either hold the write lock, walk with interrupts disabled, or run at hyp where shared walks are rejected; they keep the current behaviour by passing no flags. No functional change intended. Signed-off-by: Jack Thomson [ljs: fixup new kvm_pgtable_get_leaf() invocation] Reviewed-by: Oliver Upton Reviewed-by: Fuad Tabba Signed-off-by: Lorenzo Stoakes (ARM) Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-9-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pgtable.h | 5 ++++- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 10 +++++----- arch/arm64/kvm/hyp/nvhe/mm.c | 2 +- arch/arm64/kvm/hyp/pgtable.c | 5 +++-- arch/arm64/kvm/mmu.c | 2 +- arch/arm64/kvm/nested.c | 2 +- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 41a8687938eb..d0167f7dfbee 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -859,6 +859,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, * @addr: Input address for the start of the walk. * @ptep: Pointer to storage for the retrieved PTE. * @level: Pointer to storage for the level of the retrieved PTE. + * @flags: Flags to control the page-table walk + * (see struct kvm_pgtable_visit_ctx). * * The offset of @addr within a page is ignored. * @@ -869,7 +871,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, * Return: 0 on success, negative error code on failure. */ int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, - kvm_pte_t *ptep, s8 *level); + kvm_pte_t *ptep, s8 *level, + enum kvm_pgtable_walk_flags flags); /** * kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index a6a47c1e058b..443ee433119e 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -540,7 +540,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) int ret; hyp_assert_lock_held(&host_mmu.lock); - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level, 0); if (ret) return ret; @@ -930,7 +930,7 @@ static int get_valid_guest_pte(struct pkvm_hyp_vm *vm, u64 ipa, kvm_pte_t *ptep, s8 level; int ret; - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) return ret; if (guest_pte_is_poisoned(pte)) @@ -979,7 +979,7 @@ int __pkvm_vcpu_in_poison_fault(struct pkvm_hyp_vcpu *hyp_vcpu) ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(&hyp_vcpu->vcpu)); guest_lock_component(vm); - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) goto unlock; @@ -1335,7 +1335,7 @@ static int host_stage2_get_guest_info(phys_addr_t phys, struct pkvm_hyp_vm **vm, return -EPERM; } - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level, 0); if (ret) return ret; @@ -1564,7 +1564,7 @@ static int __check_host_shared_guest(struct pkvm_hyp_vm *vm, u64 *__phys, u64 ip s8 level; int ret; - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) return ret; if (!kvm_pte_valid(pte)) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 29ab5ee9d57f..bcf2fd1ddd75 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -486,7 +486,7 @@ static int check_page_ownership(phys_addr_t phys) return -EPERM; } - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL, 0); if (ret) return ret; diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index b74dd5ce1efd..347eec3957d6 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -298,12 +298,13 @@ static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx, } int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, - kvm_pte_t *ptep, s8 *level) + kvm_pte_t *ptep, s8 *level, + enum kvm_pgtable_walk_flags flags) { struct leaf_walk_data data; struct kvm_pgtable_walker walker = { .cb = leaf_walker, - .flags = KVM_PGTABLE_WALK_LEAF, + .flags = flags | KVM_PGTABLE_WALK_LEAF, .arg = &data, }; int ret; diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 2496fc390ab3..e21dd223c12a 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -876,7 +876,7 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr) * IPI-ing threads). */ local_irq_save(flags); - ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level); + ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level, 0); local_irq_restore(flags); if (ret) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index f5f0e364e56f..1b71530c89e3 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -659,7 +659,7 @@ static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) return 0; tmp &= ~(sz - 1); - if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) + if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL, 0)) goto again; if (!(pte & PTE_VALID)) goto again; From 34cfa9b3af17d0554ffd6c655fc1097854b49487 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:53 +0100 Subject: [PATCH 294/506] KVM: arm64: Implement KVM_PRE_FAULT_MEMORY Implement KVM stage 2 page table pre-faulting for the arm64 architecture, with the core of the implementation in kvm_arch_vcpu_pre_fault_memory(). Pre-fault by first trying a page table walk under the MMU read lock then, if it fails, injecting a synthetic data abort at the page table level at which the page table walk failed. This is necessarily racey as reclaim might happen at any time. Successfully pre-faulting can therefore only guarantee that each GPA was observed to be mapped at least once. Protected KVM (pKVM) is not supported at all because pKVM creates VMs and vCPUs when first run, meaning any attempt to pre-fault prior to this cannot succeed. Additionally, in pKVM mode, the stage 2 page tables are owned by the hypervisor rather than the host - the host can neither walk them nor populate them directly, so it's not clear that the pre-fault mechanism correctly maps onto pKVM. Reject both pKVM hosts and uninitialised vCPUs, which are not in a safe state to be loaded, from the kvm_arch_pre_fault_allowed() hook so they are refused before the vCPU is loaded. A retry mechanic is implemented when user_mem_abort() or gmem_abort() fail to map memory due to a benign failure where a hardware abort would not result in an error. An invalid memslot (i.e. a memslot with the KVM_MEMSLOT_INVALID flag set) results in the operation returning -EAGAIN without a retry mechanic, because an invalid memslot means the pre-fault operation raced with memslot reclaim, and since the SRCU lock is held, progress cannot be made, and the user must retry. This work is based with gratitude on Jack Thomson's original series, its previous revisions and the feedback they received. Link: https://patch.msgid.link/20260612162354.73378-1-jackabt.amazon@gmail.com/ Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-10-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pkvm.h | 2 +- arch/arm64/kvm/Kconfig | 1 + arch/arm64/kvm/arm.c | 1 + arch/arm64/kvm/mmu.c | 144 ++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index cad60569f061..c1831c8e421d 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -44,9 +44,9 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) case KVM_CAP_ARM_PTRAUTH_GENERIC: return true; case KVM_CAP_ARM_MTE: - return false; case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: + case KVM_CAP_PRE_FAULT_MEMORY: return false; default: return !kvm || !kvm_vm_is_protected(kvm); diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 449154f9a485..71233068b7cb 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -37,6 +37,7 @@ menuconfig KVM select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS select KVM_GUEST_MEMFD + select KVM_GENERIC_PRE_FAULT_MEMORY help Support hosting virtualized guest machines. diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index eaf583b77193..5de2ea5d6e7c 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -410,6 +410,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_COUNTER_OFFSET: case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS: case KVM_CAP_ARM_SEA_TO_USER: + case KVM_CAP_PRE_FAULT_MEMORY: r = 1; break; case KVM_CAP_SET_GUEST_DEBUG2: diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index e21dd223c12a..dc597d33f77b 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -2914,3 +2915,146 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); } + +/* + * Try to walk to the specified GPA in canonical mmu - if unmapped returns 0, if + * mapped returns the granule size, otherwise returns an error. + */ +static long kvm_walk_s2(struct kvm_pgtable *pgt, + gpa_t gpa, s8 *level) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu); + kvm_pte_t pte; + long ret; + + guard(read_lock)(&kvm->mmu_lock); + + ret = kvm_pgtable_get_leaf(pgt, gpa, &pte, level, + KVM_PGTABLE_WALK_SHARED); + if (ret) + return ret; + /* Unpopulated, must fault. */ + if (!kvm_pte_valid(pte)) + return 0; + return kvm_granule_size(*level); +} + +/* Synthesised data abort at specified page table level. */ +#define PRE_FAULT_ESR(level) \ + ((ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | \ + ESR_ELx_IL | ESR_ELx_FSC_FAULT_L(level)) + +/* Retrieve either a read-only or a read/write hva. */ +static hva_t gfn_to_hva_memslot_read(struct kvm_memory_slot *slot, gfn_t gfn) +{ + return gfn_to_hva_memslot_prot(slot, gfn, /*writable=*/NULL); +} + +static long __pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu, + gpa_t gpa, struct kvm_memory_slot *memslot, s8 level) +{ + const bool is_gmem = kvm_slot_has_gmem(memslot); + const gfn_t gfn = gpa_to_gfn(gpa); + const hva_t hva = is_gmem ? 0 : gfn_to_hva_memslot_read(memslot, gfn); + const struct kvm_s2_fault_desc s2fd = { + .vcpu = vcpu, + .fault_ipa = gpa, + .nested = NULL, + .memslot = memslot, + .hva = hva, + .esr = PRE_FAULT_ESR(level), + .mmu = mmu, + }; + struct kvm_s2_fault_result result = {}; + long ret; + + if (kvm_is_error_hva(hva)) + return -EFAULT; + + if (is_gmem) + ret = gmem_abort(&s2fd, &result); + else + ret = user_mem_abort(&s2fd, &result); + if (IS_ERR_VALUE(ret)) + return ret; + return result.mapping_size; +} + +static long pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu, + gpa_t gpa, struct kvm_memory_slot *memslot) +{ + s8 level; + long ret; + + /* Try a walk first. */ + ret = kvm_walk_s2(mmu->pgt, gpa, &level); + if (ret) + return ret; + /* OK, have to fault page in. */ + return __pre_fault_s2(mmu, vcpu, gpa, memslot, level); +} + +static unsigned long +pre_fault_bytes_consumed(gpa_t gpa, unsigned long granule_size, + unsigned long bytes_remaining) +{ + /* Granules are always a power-of-2. */ + const unsigned long granule_bytes_remaining = + granule_size - (gpa % granule_size); + + return min(granule_bytes_remaining, bytes_remaining); +} + +/* If you lose the race this many times, time to give up. */ +#define MAX_PRE_FAULT_RETRIES 3 + +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (is_protected_kvm_enabled()) + return -EOPNOTSUPP; + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + return 0; +} + +/** + * kvm_arch_vcpu_pre_fault_memory - pre-fault stage-2 page tables for the + * specified GPA. + * @vcpu: The VCPU pointer + * @range: {gpa, size, flags} tuple + * + * The mapping performed is always best-effort - faulting in is necessarily + * racey. The ranges faulted in are canonical, nested page tables are ignored. + * + * @range->gpa specifies the GPA to pre-fault, @range->size specifies how many + * bytes remain to be pre-faulted and @range->flags is reserved and must be 0. + * + * Returns: the number of bytes the pre-fault consumed, or an error. + */ +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, + struct kvm_pre_fault_memory *range) +{ + struct kvm *kvm = vcpu->kvm; + const u64 bytes_remaining = range->size; + struct kvm_s2_mmu *mmu = &kvm->arch.mmu; /* Canonical. */ + struct kvm_memory_slot *memslot; + const gpa_t gpa = range->gpa; + int num_retries = 0; + long ret; + + memslot = gfn_to_memslot(kvm, gpa_to_gfn(gpa)); + if (!memslot) + return -ENOENT; + /* SRCU must be released for progress and only userland can do that. */ + if (memslot->flags & KVM_MEMSLOT_INVALID) + return -EAGAIN; + + do { + ret = pre_fault_s2(mmu, vcpu, gpa, memslot); + } while (ret == -EAGAIN && num_retries++ < MAX_PRE_FAULT_RETRIES); + + if (IS_ERR_VALUE(ret)) + return ret; + return pre_fault_bytes_consumed(gpa, ret, bytes_remaining); +} From 28187f51e4c470c473350905141a8054ec0f1f2d Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:54 +0100 Subject: [PATCH 295/506] Documentation: KVM: document arm64 KVM_PRE_FAULT_MEMORY Update the KVM API documentation to reflect the fact that arm64 now supports this functionality. Also document error values and arm64-specific behaviour. Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-11-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- Documentation/virt/kvm/api.rst | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc750c9..712345ac4b2f 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6494,7 +6494,7 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details. --------------------------- :Capability: KVM_CAP_PRE_FAULT_MEMORY -:Architectures: none +:Architectures: x86, s390, arm64 :Type: vcpu ioctl :Parameters: struct kvm_pre_fault_memory (in/out) :Returns: 0 if at least one page is processed, < 0 on error @@ -6502,12 +6502,15 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details. Errors: ========== =============================================================== + EAGAIN A race occurred before progress was made, but a retry may succeed. EINVAL The specified `gpa` and `size` were invalid (e.g. not page aligned, causes an overflow, or size is zero), or the VM is UCONTROL (s390). ENOENT The specified `gpa` is outside defined memslots. + ENOEXEC The vCPU has not been initialised (arm64). EINTR An unmasked signal is pending and no page was processed. EFAULT The parameter address was invalid. + EHWPOISON A poisoned host page was encountered. EOPNOTSUPP Mapping memory for a GPA is unsupported by the hypervisor, and/or for the current vCPU state/mode. EIO unexpected error conditions (also causes a WARN) @@ -6527,7 +6530,17 @@ Errors: KVM_PRE_FAULT_MEMORY populates KVM's stage-2 page tables used to map memory for the current vCPU state. KVM maps memory as if the vCPU generated a stage-2 read page fault, e.g. faults in memory as needed, but doesn't break -CoW. On x86, KVM does not mark any newly created stage-2 PTE as Accessed. +CoW. On arm64, KVM marks newly created stage-2 PTEs as Accessed, as it +does for any stage-2 fault, but leaves the Accessed state of existing PTEs +unchanged. On x86, KVM does not mark any newly created stage-2 PTE as +Accessed, and for s390 it is not applicable. + +On arm64, a GPA is interpreted as an IPA, and never interpreted as the IPA +of a nested guest. Pre-faulting only populates canonical stage-2 page +tables. + +The feature is not supported on arm64 if the protected KVM (pKVM) feature +is enabled. In the case of confidential VM types where there is an initial set up of private guest memory before the guest is 'finalized'/measured, this ioctl @@ -6542,7 +6555,7 @@ When the ioctl returns, the input values are updated to point to the remaining range. If `size` > 0 on return, the caller can just issue the ioctl again with the same `struct kvm_map_memory` argument. -Shadow page tables cannot support this ioctl because they +On x86, shadow page tables cannot support this ioctl because they are indexed by virtual address or nested guest physical address. Calling this ioctl when the guest is using shadow page tables (for example because it is running a nested guest with nested page tables) From 8ab194f836391fdcb1a8328d5ff0f0c7f0afe557 Mon Sep 17 00:00:00 2001 From: Jack Thomson Date: Wed, 23 Sep 2026 16:15:55 +0100 Subject: [PATCH 296/506] KVM: selftests: Enable pre_fault_memory_test for arm64 Enable the pre_fault_memory_test to run on arm64 by making it work with different guest page sizes and testing multiple guest configurations. Add an assert on the exit reason, comparing against UCALL_EXIT_REASON so the check is portable, as arm64 exits with KVM_EXIT_MMIO while x86 uses KVM_EXIT_IO. [ljs: merge conflict resolution with commit c847704619da ("KVM: selftests: Fix pre_fault_memory_test to run on s390"), commit message accuracy fixup] Signed-off-by: Jack Thomson Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-12-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/pre_fault_memory_test.c | 117 ++++++++++++++---- 2 files changed, 96 insertions(+), 22 deletions(-) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 6a1482e3a286..874ff9097384 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -205,6 +205,7 @@ TEST_GEN_PROGS_arm64 += guest_memfd_test TEST_GEN_PROGS_arm64 += mmu_stress_test TEST_GEN_PROGS_arm64 += rseq_test TEST_GEN_PROGS_arm64 += steal_time +TEST_GEN_PROGS_arm64 += pre_fault_memory_test TEST_GEN_PROGS_s390 = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_s390 += s390/memop diff --git a/tools/testing/selftests/kvm/pre_fault_memory_test.c b/tools/testing/selftests/kvm/pre_fault_memory_test.c index c57631aab3d3..a5993f8e1feb 100644 --- a/tools/testing/selftests/kvm/pre_fault_memory_test.c +++ b/tools/testing/selftests/kvm/pre_fault_memory_test.c @@ -12,19 +12,29 @@ #include #include #include +#include /* Arbitrarily chosen values */ -#define TEST_SIZE (SZ_2M + PAGE_SIZE) -#define TEST_NPAGES (TEST_SIZE / PAGE_SIZE) +#define TEST_BASE_SIZE SZ_2M #define TEST_SLOT 10 +/* Storage of test info to share with guest code */ +struct test_config { + u64 page_size; + u64 test_size; + u64 test_num_pages; +}; + +static struct test_config test_config; + static void guest_code(u64 base_gva) { volatile u64 val __used; + struct test_config *config = &test_config; int i; - for (i = 0; i < TEST_NPAGES; i++) { - u64 *src = (u64 *)(base_gva + i * PAGE_SIZE); + for (i = 0; i < config->test_num_pages; i++) { + u64 *src = (u64 *)(base_gva + i * config->page_size); val = *src; } @@ -57,7 +67,7 @@ static void *delete_slot_worker(void *__data) cpu_relax(); vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, data->gpa, - TEST_SLOT, TEST_NPAGES, data->flags); + TEST_SLOT, test_config.test_num_pages, data->flags); return NULL; } @@ -150,8 +160,8 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, /* * Assert success if prefaulting the entire range should succeed, i.e. * complete with no bytes remaining. Otherwise prefaulting should have - * failed due to ENOENT (due to RET_PF_EMULATE for emulated MMIO when - * no memslot exists). + * failed due to ENOENT (no memslot exists for the GPA; on x86 this + * surfaces via RET_PF_EMULATE). */ if (!expected_left) TEST_ASSERT_VM_VCPU_IOCTL(!ret, KVM_PRE_FAULT_MEMORY, ret, vcpu->vm); @@ -160,39 +170,72 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, KVM_PRE_FAULT_MEMORY, ret, vcpu->vm); } -static void __test_pre_fault_memory(unsigned long vm_type, bool private) +struct test_params { + unsigned long vm_type; + bool private; +}; + +static void __test_pre_fault_memory(enum vm_guest_mode guest_mode, void *arg) { - gpa_t gpa, gva, alignment, guest_page_size; + gpa_t gpa, gva, alignment, guest_page_size, host_page_size; + struct test_params *p = arg; const struct vm_shape shape = { - .mode = VM_MODE_DEFAULT, - .type = vm_type, + .mode = guest_mode, + .type = p->vm_type, }; struct kvm_vcpu *vcpu; + struct kvm_run *run; struct kvm_vm *vm; struct ucall uc; + pr_info("Testing guest mode: %s\n", vm_guest_mode_string(guest_mode)); + vm = vm_create_shape_with_one_vcpu(shape, &vcpu, guest_code); - alignment = guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size; - gpa = (vm->max_gfn - TEST_NPAGES) * guest_page_size; + guest_page_size = vm_guest_mode_params[guest_mode].page_size; + host_page_size = getpagesize(); + + test_config.page_size = guest_page_size; + test_config.test_size = align_up(TEST_BASE_SIZE + test_config.page_size, + host_page_size); + test_config.test_num_pages = vm_calc_num_guest_pages(vm->mode, test_config.test_size); + + gpa = (vm->max_gfn - test_config.test_num_pages) * test_config.page_size; alignment = SZ_2M; + alignment = max(alignment, host_page_size); gpa = align_down(gpa, alignment); gva = gpa & ((1ULL << (vm->va_bits - 1)) - 1); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa, TEST_SLOT, - TEST_NPAGES, private ? KVM_MEM_GUEST_MEMFD : 0); - virt_map(vm, gva, gpa, TEST_NPAGES); + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, + gpa, TEST_SLOT, test_config.test_num_pages, + p->private ? KVM_MEM_GUEST_MEMFD : 0); + virt_map(vm, gva, gpa, test_config.test_num_pages); - if (private) - vm_mem_set_private(vm, gpa, TEST_SIZE); + if (p->private) + vm_mem_set_private(vm, gpa, test_config.test_size); - pre_fault_memory(vcpu, gpa, 0, SZ_2M, 0, private); - pre_fault_memory(vcpu, gpa, SZ_2M, PAGE_SIZE * 2, PAGE_SIZE, private); - pre_fault_memory(vcpu, gpa, TEST_SIZE, PAGE_SIZE, PAGE_SIZE, private); + pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, p->private); + /* Retry the same range after the first prefault attempt. */ + pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, p->private); + pre_fault_memory(vcpu, gpa, + test_config.test_size - host_page_size, + host_page_size * 2, host_page_size, p->private); + pre_fault_memory(vcpu, gpa, test_config.test_size, + host_page_size, host_page_size, p->private); vcpu_args_set(vcpu, 1, gva); + + /* Export the shared variables to the guest. */ + sync_global_to_guest(vm, test_config); + vcpu_run(vcpu); + run = vcpu->run; + TEST_ASSERT(run->exit_reason == UCALL_EXIT_REASON, + "Wanted %s, got exit reason: %u (%s)", + exit_reason_str(UCALL_EXIT_REASON), + run->exit_reason, exit_reason_str(run->exit_reason)); + switch (get_ucall(vcpu, &uc)) { case UCALL_ABORT: REPORT_GUEST_ASSERT(uc); @@ -209,16 +252,46 @@ static void __test_pre_fault_memory(unsigned long vm_type, bool private) static void test_pre_fault_memory(unsigned long vm_type, bool private) { + struct test_params p = { + .vm_type = vm_type, + .private = private, + }; + if (vm_type && !(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(vm_type))) { pr_info("Skipping tests for vm_type 0x%lx\n", vm_type); return; } - __test_pre_fault_memory(vm_type, private); + for_each_guest_mode(__test_pre_fault_memory, &p); +} + +static void help(char *name) +{ + puts(""); + printf("usage: %s [-h] [-m mode]\n", name); + puts(""); + guest_modes_help(); + puts(""); } int main(int argc, char *argv[]) { + int opt; + + guest_modes_append_default(); + + while ((opt = getopt(argc, argv, "hm:")) != -1) { + switch (opt) { + case 'm': + guest_modes_cmdline(optarg); + break; + case 'h': + default: + help(argv[0]); + exit(0); + } + } + TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY)); test_pre_fault_memory(0, false); From c4560edbaa81291ee79a97b5e793934cea88500b Mon Sep 17 00:00:00 2001 From: Jack Thomson Date: Wed, 23 Sep 2026 16:15:56 +0100 Subject: [PATCH 297/506] KVM: selftests: Add option for different backing in pre-fault tests Add a -s option to specify different memory backing types for the pre-fault tests (e.g. anonymous, hugetlb), allowing testing of the pre-fault functionality across different memory configurations. Signed-off-by: Jack Thomson Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-13-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- .../selftests/kvm/pre_fault_memory_test.c | 51 +++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/tools/testing/selftests/kvm/pre_fault_memory_test.c b/tools/testing/selftests/kvm/pre_fault_memory_test.c index a5993f8e1feb..3082fe09b95f 100644 --- a/tools/testing/selftests/kvm/pre_fault_memory_test.c +++ b/tools/testing/selftests/kvm/pre_fault_memory_test.c @@ -46,6 +46,7 @@ struct slot_worker_data { struct kvm_vm *vm; gpa_t gpa; u32 flags; + enum vm_mem_backing_src_type mem_backing_src; bool worker_ready; bool prefault_ready; bool recreate_slot; @@ -66,14 +67,16 @@ static void *delete_slot_worker(void *__data) while (!READ_ONCE(data->recreate_slot)) cpu_relax(); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, data->gpa, + vm_userspace_mem_region_add(vm, data->mem_backing_src, data->gpa, TEST_SLOT, test_config.test_num_pages, data->flags); return NULL; } static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, - u64 size, u64 expected_left, bool private) + u64 size, u64 expected_left, + enum vm_mem_backing_src_type mem_backing_src, + bool private) { struct kvm_pre_fault_memory range = { .gpa = base_gpa + offset, @@ -84,6 +87,7 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, .vm = vcpu->vm, .gpa = base_gpa, .flags = private ? KVM_MEM_GUEST_MEMFD : 0, + .mem_backing_src = mem_backing_src, }; bool slot_recreated = false; pthread_t slot_worker; @@ -173,11 +177,13 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, struct test_params { unsigned long vm_type; bool private; + enum vm_mem_backing_src_type mem_backing_src; }; static void __test_pre_fault_memory(enum vm_guest_mode guest_mode, void *arg) { gpa_t gpa, gva, alignment, guest_page_size, host_page_size; + gpa_t backing_src_pagesz, mem_page_size; struct test_params *p = arg; const struct vm_shape shape = { .mode = guest_mode, @@ -189,24 +195,28 @@ static void __test_pre_fault_memory(enum vm_guest_mode guest_mode, void *arg) struct ucall uc; pr_info("Testing guest mode: %s\n", vm_guest_mode_string(guest_mode)); + pr_info("Testing memory backing src type: %s\n", + vm_mem_backing_src_alias(p->mem_backing_src)->name); vm = vm_create_shape_with_one_vcpu(shape, &vcpu, guest_code); guest_page_size = vm_guest_mode_params[guest_mode].page_size; host_page_size = getpagesize(); + backing_src_pagesz = get_backing_src_pagesz(p->mem_backing_src); + mem_page_size = max(host_page_size, backing_src_pagesz); test_config.page_size = guest_page_size; test_config.test_size = align_up(TEST_BASE_SIZE + test_config.page_size, - host_page_size); + mem_page_size); test_config.test_num_pages = vm_calc_num_guest_pages(vm->mode, test_config.test_size); gpa = (vm->max_gfn - test_config.test_num_pages) * test_config.page_size; alignment = SZ_2M; - alignment = max(alignment, host_page_size); + alignment = max(alignment, mem_page_size); gpa = align_down(gpa, alignment); gva = gpa & ((1ULL << (vm->va_bits - 1)) - 1); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, + vm_userspace_mem_region_add(vm, p->mem_backing_src, gpa, TEST_SLOT, test_config.test_num_pages, p->private ? KVM_MEM_GUEST_MEMFD : 0); virt_map(vm, gva, gpa, test_config.test_num_pages); @@ -214,14 +224,18 @@ static void __test_pre_fault_memory(enum vm_guest_mode guest_mode, void *arg) if (p->private) vm_mem_set_private(vm, gpa, test_config.test_size); - pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, p->private); + pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, + p->mem_backing_src, p->private); /* Retry the same range after the first prefault attempt. */ - pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, p->private); + pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, + p->mem_backing_src, p->private); pre_fault_memory(vcpu, gpa, test_config.test_size - host_page_size, - host_page_size * 2, host_page_size, p->private); + host_page_size * 2, host_page_size, + p->mem_backing_src, p->private); pre_fault_memory(vcpu, gpa, test_config.test_size, - host_page_size, host_page_size, p->private); + host_page_size, host_page_size, + p->mem_backing_src, p->private); vcpu_args_set(vcpu, 1, gva); @@ -250,11 +264,13 @@ static void __test_pre_fault_memory(enum vm_guest_mode guest_mode, void *arg) kvm_vm_free(vm); } -static void test_pre_fault_memory(unsigned long vm_type, bool private) +static void test_pre_fault_memory(unsigned long vm_type, enum vm_mem_backing_src_type backing_src, + bool private) { struct test_params p = { .vm_type = vm_type, .private = private, + .mem_backing_src = backing_src, }; if (vm_type && !(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(vm_type))) { @@ -268,23 +284,28 @@ static void test_pre_fault_memory(unsigned long vm_type, bool private) static void help(char *name) { puts(""); - printf("usage: %s [-h] [-m mode]\n", name); + printf("usage: %s [-h] [-m mode] [-s mem-type]\n", name); puts(""); guest_modes_help(); + backing_src_help("-s"); puts(""); } int main(int argc, char *argv[]) { + enum vm_mem_backing_src_type backing = DEFAULT_VM_MEM_SRC; int opt; guest_modes_append_default(); - while ((opt = getopt(argc, argv, "hm:")) != -1) { + while ((opt = getopt(argc, argv, "hm:s:")) != -1) { switch (opt) { case 'm': guest_modes_cmdline(optarg); break; + case 's': + backing = parse_backing_src_type(optarg); + break; case 'h': default: help(argv[0]); @@ -294,10 +315,10 @@ int main(int argc, char *argv[]) TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY)); - test_pre_fault_memory(0, false); + test_pre_fault_memory(0, backing, false); #ifdef __x86_64__ - test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, false); - test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, true); + test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, backing, false); + test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, backing, true); #endif return 0; } From 6fddb4f6f76910fe0cc2d3d78872b86357899018 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 23 Sep 2026 16:15:57 +0100 Subject: [PATCH 298/506] KVM: selftests: Add nested pre-fault test for arm64 Add an arm64 nested-virtualisation selftest for KVM_PRE_FAULT_MEMORY. The guest enters vEL2, sets up its EL2 translation configuration into the real EL1 registers then ERETs to vEL1 and exits to userspace so its vCPU last-run context is nested backed by a shadow stage 2 MMU. The test then asserts that pre-faulting ignores that and instead targets the canonical stage 2 page tables only. Userspace cannot observe a mistargeted pre-fault, so simply assert that the pre-faulting works correctly when the last-run context was nested. Based on a patch by Jack Thomson with gratitude. The test here is simplified and focused on what can be asserted. Link: https://lore.kernel.org/kvmarm/20260612162354.73378-6-jackabt.amazon@gmail.com/ Reviewed-by: Oliver Upton Signed-off-by: Lorenzo Stoakes (ARM) Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260923-kvm-arm-prefault-v4-14-d4b0b4dfa8c3@kernel.org Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../kvm/arm64/nv_pre_fault_memory_test.c | 158 ++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 874ff9097384..908bdc7cf4f5 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -178,6 +178,7 @@ TEST_GEN_PROGS_arm64 += arm64/debug-exceptions TEST_GEN_PROGS_arm64 += arm64/hello_el2 TEST_GEN_PROGS_arm64 += arm64/host_sve TEST_GEN_PROGS_arm64 += arm64/hypercalls +TEST_GEN_PROGS_arm64 += arm64/nv_pre_fault_memory_test TEST_GEN_PROGS_arm64 += arm64/external_aborts TEST_GEN_PROGS_arm64 += arm64/mmio_sign_ext TEST_GEN_PROGS_arm64 += arm64/page_fault_test diff --git a/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c b/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c new file mode 100644 index 000000000000..09c1db303856 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * nv_pre_fault_memory_test - Test KVM_PRE_FAULT_MEMORY on a vCPU whose + * last-run context is nested. + * + * The guest enters vEL2, sets up its EL2 translation configuration into the + * real EL1 registers then ERETs to vEL1 and exits to userspace so its vCPU + * last-run context is nested backed by a shadow stage 2 MMU. + * + * Assert that pre-faulting ignores that and targets the canonical stage-2 + * page tables only. + */ +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" +#include "ucall.h" + +#include +#include + +#define TEST_MEM_SLOT 10 +#define TEST_MEM_SIZE SZ_2M +#define TEST_MEM_GPA SZ_1G + +static void guest_el1_code(void) +{ + u64 offset; + + GUEST_ASSERT_EQ(get_current_el(), 1); + + /* Exit to userspace with the vEL1 (nested) context live. */ + GUEST_SYNC(1); + + /* + * Touch the prefaulted range. vstage-2 is disabled, so the shadow + * stage-2 is a 1:1 view of the canonical IPA space. + */ + for (offset = 0; offset < TEST_MEM_SIZE; offset += SZ_4K) + READ_ONCE(*(u64 *)(TEST_MEM_GPA + offset)); + + GUEST_DONE(); +} + +static void guest_code(void) +{ + u64 sp; + + GUEST_ASSERT_EQ(get_current_el(), 2); + + /* + * Mirror the EL2 translation regime into the real EL1 registers so + * that vEL1 runs on the test's stage-1 page tables. With E2H=1, the + * _EL1 accessors read the EL2 registers, and the _EL12 accessors + * write the real EL1 registers. + */ + write_sysreg_s(read_sysreg(sctlr_el1), SYS_SCTLR_EL12); + write_sysreg_s(read_sysreg(tcr_el1), SYS_TCR_EL12); + write_sysreg_s(read_sysreg(ttbr0_el1), SYS_TTBR0_EL12); + write_sysreg_s(read_sysreg(mair_el1), SYS_MAIR_EL12); + write_sysreg_s(read_sysreg(cpacr_el1), SYS_CPACR_EL12); + + /* Run vEL1 on the same stack. */ + asm volatile("mov %0, sp" : "=r"(sp)); + write_sysreg(sp, sp_el1); + + /* + * Drop TGE so that vEL1 is a nested context rather than host EL0. + * KVM backs it with a shadow stage-2 MMU even though vstage-2 is + * disabled (HCR_EL2.VM=0). + */ + write_sysreg(read_sysreg(hcr_el2) & ~HCR_EL2_TGE, hcr_el2); + isb(); + + write_sysreg(PSR_MODE_EL1h | PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | + PSR_D_BIT, spsr_el2); + write_sysreg((u64)guest_el1_code, elr_el2); + asm volatile("eret"); + + GUEST_ASSERT(false); +} + +static void pre_fault(struct kvm_vcpu *vcpu, u64 gpa, u64 size) +{ + struct kvm_pre_fault_memory range = { + .gpa = gpa, + .size = size, + }; + int ret; + + do { + ret = __vcpu_ioctl(vcpu, KVM_PRE_FAULT_MEMORY, &range); + } while ((!ret && range.size) || + (ret < 0 && (errno == EINTR || errno == EAGAIN))); + + TEST_ASSERT(!ret, "KVM_PRE_FAULT_MEMORY failed, ret: %d errno: %d", + ret, errno); + TEST_ASSERT_EQ(range.size, 0); +} + +int main(void) +{ + struct kvm_vcpu_init init; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + u64 npages; + + TEST_REQUIRE(test_supports_el2()); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY)); + + vm = vm_create(1); + + kvm_get_default_vcpu_target(vm, &init); + init.features[0] |= BIT(KVM_ARM_VCPU_HAS_EL2); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + kvm_arch_vm_finalize_vcpus(vm); + + npages = TEST_MEM_SIZE / vm->page_size; + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, TEST_MEM_GPA, + TEST_MEM_SLOT, npages, 0); + virt_map(vm, TEST_MEM_GPA, TEST_MEM_GPA, npages); + + /* Run the guest until it has ERET'd from vEL2 to vEL1. */ + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_SYNC: + TEST_ASSERT_EQ(uc.args[1], 1); + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Unhandled ucall: %ld", uc.cmd); + } + + /* + * The vCPU's last-run context is vEL1, so its hw_mmu is a shadow + * stage-2 MMU. + * + * Pre-faulting must ignore that and populate the canonical stage-2. + */ + pre_fault(vcpu, TEST_MEM_GPA, TEST_MEM_SIZE); + + /* Resume at vEL1 and touch the prefaulted range. */ + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Unhandled ucall: %ld", uc.cmd); + } + + kvm_vm_free(vm); + return 0; +} From 7ef1b9e11dd7b3832124e26a6fee703b342d6f7f Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 14 Sep 2026 13:58:26 +0100 Subject: [PATCH 299/506] KVM: arm64: Report SMCCC_VERSION and SMCCC_ARCH_FEATURES as implemented KVM reports SMCCC v1.1 to guests, which makes SMCCC_VERSION and SMCCC_ARCH_FEATURES mandatory, and DEN0028H 7.3.6 requires SMCCC_ARCH_FEATURES to return SUCCESS for both function ids: "This function must return SUCCESS when arch_func_id is the SMCCC_VERSION or SMCCC_ARCH_FEATURES function id." With arch_func_id set to either, KVM's guest handler returns NOT_SUPPORTED. Return SUCCESS. A Linux guest is unaffected: it discovers SMCCC_VERSION through PSCI_FEATURES, and issues SMCCC_ARCH_FEATURES only with arch_func_id set to ARCH_SOC_ID, the ARCH_WORKAROUND_* ids or HV_PV_TIME_FEATURES. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260914125827.1645980-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hypercalls.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index b11b8821c9fb..66f16bf9a8ea 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -293,6 +293,10 @@ int kvm_smccc_call_handler(struct kvm_vcpu *vcpu) case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: feature = smccc_get_arg1(vcpu); switch (feature) { + case ARM_SMCCC_VERSION_FUNC_ID: + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + val[0] = SMCCC_RET_SUCCESS; + break; case ARM_SMCCC_ARCH_WORKAROUND_1: switch (arm64_get_spectre_v2_state()) { case SPECTRE_VULNERABLE: From 67d00fcc00761e9fe4debeac781eb4f992dd8506 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 14 Sep 2026 13:58:27 +0100 Subject: [PATCH 300/506] KVM: arm64: selftests: Check the mandatory SMCCC_ARCH_FEATURES queries tools/testing/selftests/kvm/arm64/hypercalls.c checks SMCCC_ARCH_FEATURES with two expectations: hvc_info[] expects NOT_SUPPORTED with the pseudo-firmware bitmaps cleared and something else with them set, and false_hvc_info[] expects NOT_SUPPORTED for mismatched pairs. DEN0028H 7.3.6 mandates SUCCESS for arch_func_id SMCCC_VERSION and SMCCC_ARCH_FEATURES itself, whatever the bitmaps hold, which neither expectation allows. Check those two in the feature-disabled and feature-enabled stages, so the answer is asserted with the bitmaps cleared and again at their reset values. They pass only with the preceding patch applied. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260914125827.1645980-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- .../testing/selftests/kvm/arm64/hypercalls.c | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/arm64/hypercalls.c b/tools/testing/selftests/kvm/arm64/hypercalls.c index 5d96cdf382c4..e74410cef0fe 100644 --- a/tools/testing/selftests/kvm/arm64/hypercalls.c +++ b/tools/testing/selftests/kvm/arm64/hypercalls.c @@ -6,7 +6,8 @@ * via the psuedo-firmware bitmap register. This includes the registers' * read/write behavior before and after the VM has started, and if the * hypercalls are properly masked or unmasked to the guest when disabled or - * enabled from the KVM userspace, respectively. + * enabled from the KVM userspace, respectively. It also checks the two + * SMCCC_ARCH_FEATURES queries whose answer SMCCC pins to SUCCESS. */ #include #include @@ -97,6 +98,12 @@ static const struct test_hvc_info false_hvc_info[] = { TEST_HVC_INFO(ARM_SMCCC_HV_PV_TIME_FEATURES, ARM_SMCCC_TRNG_RND64), }; +/* SMCCC pins the answer to SUCCESS, whatever the bitmaps hold. */ +static const struct test_hvc_info always_hvc_info[] = { + TEST_HVC_INFO(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, ARM_SMCCC_VERSION_FUNC_ID), + TEST_HVC_INFO(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, ARM_SMCCC_ARCH_FEATURES_FUNC_ID), +}; + static void guest_test_hvc(const struct test_hvc_info *hc_info) { unsigned int i; @@ -128,6 +135,23 @@ static void guest_test_hvc(const struct test_hvc_info *hc_info) } } +static void guest_test_always_hvc(void) +{ + struct arm_smccc_res res; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(always_hvc_info); i++) { + memset(&res, 0, sizeof(res)); + do_smccc(always_hvc_info[i].func_id, always_hvc_info[i].arg1, + 0, 0, 0, 0, 0, 0, &res); + + __GUEST_ASSERT(res.a0 == SMCCC_RET_SUCCESS, + "a0 = 0x%lx, func_id = 0x%x, arg1 = 0x%lx, stage = %u", + res.a0, always_hvc_info[i].func_id, + always_hvc_info[i].arg1, stage); + } +} + static void guest_code(void) { while (stage != TEST_STAGE_END) { @@ -137,6 +161,7 @@ static void guest_code(void) case TEST_STAGE_HVC_IFACE_FEAT_DISABLED: case TEST_STAGE_HVC_IFACE_FEAT_ENABLED: guest_test_hvc(hvc_info); + guest_test_always_hvc(); break; case TEST_STAGE_HVC_IFACE_FALSE_INFO: guest_test_hvc(false_hvc_info); From 0dea17ffe6757a6249d143e607e786144aceee9c Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 24 Sep 2026 11:04:59 +0100 Subject: [PATCH 301/506] KVM: arm64: Disable stage-2 ptdump of pKVM With pKVM enabled, guest stage-2 page tables are owned and managed by the hypervisor. The host's pgt is uninitialised. Skip registering stage-2 ptdump debugfs files when protected KVM is enabled. Fixes: fce886a60207 ("KVM: arm64: Plumb the pKVM MMU in KVM") Reviewed-by: Fuad Tabba Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260924100459.831013-1-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/ptdump.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/ptdump.c b/arch/arm64/kvm/ptdump.c index 69899797dbad..87e5e7a4e612 100644 --- a/arch/arm64/kvm/ptdump.c +++ b/arch/arm64/kvm/ptdump.c @@ -300,6 +300,10 @@ void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) { + /* pKVM handles the page-tables in the hypervisor */ + if (is_protected_kvm_enabled()) + return; + debugfs_create_file("stage2_page_tables", 0400, kvm->debugfs_dentry, &kvm->arch.mmu, &kvm_ptdump_guest_fops); debugfs_create_file("ipa_range", 0400, kvm->debugfs_dentry, From 8642afb74be6de014e49937c2c4de7576f45531c Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 24 Sep 2026 10:51:54 +0100 Subject: [PATCH 302/506] KVM: arm64: Remove PAGE_SIZE alignment for hyp event ELF sections .hyp.event_ids is mapped as part of .hyp.rodata. As a consequence this section does not need any page-alignment. Remove it to save a page worth of memory. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260924095154.807726-1-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kernel/vmlinux.lds.S | 2 +- arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index af1d72020976..c0b090db8be4 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -15,7 +15,7 @@ #ifdef CONFIG_NVHE_EL2_TRACING #define HYPERVISOR_EVENT_IDS \ - . = ALIGN(PAGE_SIZE); \ + . = ALIGN(SZ_8); \ __hyp_event_ids_start = .; \ *(HYP_SECTION_NAME(.event_ids)) \ __hyp_event_ids_end = .; diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S index 7a02837203d1..89222425359a 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S @@ -17,7 +17,6 @@ SECTIONS { HYP_SECTION(.data..ro_after_init) HYP_SECTION(.rodata) #ifdef CONFIG_NVHE_EL2_TRACING - . = ALIGN(PAGE_SIZE); BEGIN_HYP_SECTION(.event_ids) *(SORT(.hyp.event_ids.*)) END_HYP_SECTION From 883719f57f5d3898637b130d6db4742471cad618 Mon Sep 17 00:00:00 2001 From: Keqiang Duan Date: Thu, 20 Aug 2026 20:35:12 +0800 Subject: [PATCH 303/506] KVM: x86: Clear hardware HLT state when userspace makes a vCPU not-halted Force a vCPU out of its hardware-tracked halted state when userspace puts the vCPU into any state other than HALTED via KVM_SET_MP_STATE, i.e. clear VMCS.GUEST_ACTIVITY_STATE if it says the vCPU is halted. Add an optional kvm_x86_ops hook to do the clearing, as SVM has no equivalent VMCB field. When HLT-exiting is disabled for a VM (KVM_CAP_X86_DISABLE_EXITS with KVM_X86_DISABLE_EXITS_HLT, e.g. QEMU's "-overcommit cpu-pm=on"), a guest HLT halts the physical CPU instead of exiting to KVM, and hardware saves GUEST_ACTIVITY_STATE=HLT into the VMCS on the next VM-Exit. That field is sticky: it survives VM-Exit/VM-Enter and is only cleared by vmx_clear_hlt() on event injection, or by vmx_vcpu_reset() on INIT / vCPU creation. Nothing clears it on a userspace-driven state change. KVM_SET_REGS only writes the software register cache and KVM_SET_MP_STATE only writes vcpu->arch.mp_state; kvm_vcpu_running() likewise consults software state only. A VMM that emulates a machine reset therefore ends up with a vCPU that KVM happily VM-Enters while hardware refuses to fetch instructions. Reproduce with a Linux guest by triggering a panic/kdump on a non-boot vCPU: nmi_shootdown_cpus() parks the other vCPUs -- including vCPU0 -- in crash_nmi_callback(), which does local_irq_disable() followed by a bare HLT. The capture kernel then resets the machine via port 0xCF9. QEMU rewrites RIP to 0xfff0 and sets mp_state to RUNNABLE, but vCPU0's GUEST_ACTIVITY_STATE is still HLT, so the BSP never executes the reset vector, never sends SIPIs, and the entire VM hangs at "reboot: machine restart" forever. Only destroying and recreating the VM recovers it. Key off "not HALTED" rather than "is RUNNABLE" as HALTED is the only target state for which retaining the hardware halted state is meaningful; every other transition either can't be reached on VMX, e.g. AP_RESET_HOLD, or goes through vmx_vcpu_reset(), which stuffs GUEST_ACTIVITY_ACTIVE anyways. This also captures the intent more directly: KVM only ever drops the hardware halted state, it never installs it. Hook KVM_SET_MP_STATE rather than the VM-Enter path so that the clearing is driven by an explicit userspace declaration, and so that no work is added to vmx_vcpu_run(). Gate the call on kvm_hlt_in_guest() even though vmx_clear_hlt() checks it as well, as the check is cheap and makes it obvious in common code when the hardware state can exist at all. Do nothing for TDX vCPUs as TDX doesn't support disabling HLT-exiting and KVM cannot access a TD's VMCS, i.e. the VMCS used to run the vCPU can't in HALTED state, and even if it were, KVM can't change it. Fixes: caa057a2cad6 ("KVM: X86: Provide a capability to disable HLT intercepts") Cc: stable@vger.kernel.org Tested-by: Qinguang Chen Signed-off-by: Zhiping Du Signed-off-by: Keqiang Duan Reviewed-by: Chao Gao Link: https://patch.msgid.link/20260820123512.87236-1-duankeqiangcym@gmail.com [sean: drop claim that spurious wakeups are ok, massage changelog] Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 1 + arch/x86/include/asm/kvm_host.h | 7 +++++++ arch/x86/kvm/vmx/main.c | 13 +++++++++++++ arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/vmx/x86_ops.h | 1 + arch/x86/kvm/x86.c | 13 +++++++++++++ 6 files changed, 36 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index e213c9ae3e30..dd9026c1071c 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -82,6 +82,7 @@ KVM_X86_OP(interrupt_allowed) KVM_X86_OP(nmi_allowed) KVM_X86_OP(get_nmi_mask) KVM_X86_OP(set_nmi_mask) +KVM_X86_OP_OPTIONAL(clear_hlt) KVM_X86_OP(enable_nmi_window) KVM_X86_OP(enable_irq_window) KVM_X86_OP_OPTIONAL(update_cr8_intercept) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3af7395c2430..4cc39c3592ee 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1608,6 +1608,13 @@ struct kvm_x86_ops { int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); + /* + * Force the vCPU out of any hardware-tracked halted/inactive state so + * that it will fetch and execute instructions on the next VM-Enter. + * Only needed by VMX, where VMCS.GUEST_ACTIVITY_STATE persists across + * VM-Exit/VM-Enter; SVM has no equivalent VMCB field. + */ + void (*clear_hlt)(struct kvm_vcpu *vcpu); /* Whether or not a virtual NMI is pending in hardware. */ bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); /* diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 250666cee0c2..8bf1702f5165 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -597,6 +597,18 @@ static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) vmx_set_nmi_mask(vcpu, masked); } +static void vt_clear_hlt(struct kvm_vcpu *vcpu) +{ + /* + * TDX doesn't support disabling HLT-exiting, and KVM can't access a + * TD's VMCS, so there is never any hardware halted state to clear. + */ + if (is_td_vcpu(vcpu)) + return; + + vmx_clear_hlt(vcpu); +} + static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) { /* Refer to the comments in tdx_inject_nmi(). */ @@ -1064,6 +1076,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .nmi_allowed = vt_op(nmi_allowed), .get_nmi_mask = vt_op(get_nmi_mask), .set_nmi_mask = vt_op(set_nmi_mask), + .clear_hlt = vt_op(clear_hlt), .enable_nmi_window = vt_op(enable_nmi_window), .enable_irq_window = vt_op(enable_irq_window), .update_cr8_intercept = vt_op(update_cr8_intercept), diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 21d2d96fd969..cf32aee0aab7 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1909,7 +1909,7 @@ int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) return skip_emulated_instruction(vcpu); } -static void vmx_clear_hlt(struct kvm_vcpu *vcpu) +void vmx_clear_hlt(struct kvm_vcpu *vcpu) { /* * Ensure that we clear the HLT state in the VMCS. We don't need to diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 7ac02c68457f..9ead79c90ba8 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -94,6 +94,7 @@ int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection); int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection); bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu); void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked); +void vmx_clear_hlt(struct kvm_vcpu *vcpu); void vmx_enable_nmi_window(struct kvm_vcpu *vcpu); void vmx_enable_irq_window(struct kvm_vcpu *vcpu); void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..6f590f9be849 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9058,6 +9058,19 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, } kvm_set_mp_state(vcpu, mp_state->mp_state); + + /* + * Force the vCPU out of any hardware-tracked halted state, e.g. VMX's + * GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state + * other than HALTED. The hardware state is sticky across VM-Exit and + * VM-Enter and is not touched by any other ioctl, so a vCPU that halted + * with HLT-exiting disabled stays wedged even after userspace rewrites + * its registers, e.g. when a VMM emulates a machine reset. + */ + if (kvm_hlt_in_guest(vcpu->kvm) && + mp_state->mp_state != KVM_MP_STATE_HALTED) + kvm_x86_call(clear_hlt)(vcpu); + kvm_make_request(KVM_REQ_EVENT, vcpu); ret = 0; From fe02f37155c80ccfe8cbe10fd2bec1b0689028be Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 11:09:12 +0200 Subject: [PATCH 304/506] Documentation: KVM: SEV: Use the uapi structure names The document names struct kvm_sev_launch_update, struct kvm_sev_attestation, struct kvm_sev_launch_send_update_data and struct kvm_sev_launch_receive_update_data. None of them exists in arch/x86/include/uapi/asm/kvm.h; the structures are kvm_sev_launch_update_data, kvm_sev_attestation_report, kvm_sev_send_update_data and kvm_sev_receive_update_data. Use the uapi names. Fixes: dc48bae01e5a ("KVM: Define SEV key management command id") Fixes: 2c07ded06427 ("KVM/SVM: add support for SEV attestation command") Fixes: d3d1af85e2c7 ("KVM: SVM: Add KVM_SEND_UPDATE_DATA command") Fixes: 15fb7de1a7f5 ("KVM: SVM: Add KVM_SEV_RECEIVE_UPDATE_DATA command") Assisted-by: LLM Signed-off-by: Karl Mehltretter Link: https://patch.msgid.link/20260905090912.40740-1-kmehltretter@gmail.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/x86/amd-memory-encryption.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index bd04a908a8db..9ebd99aa1d0b 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -170,7 +170,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_update { + struct kvm_sev_launch_update_data { __u64 uaddr; /* userspace address to be encrypted (must be 16-byte aligned) */ __u32 len; /* length of the data to be encrypted (must be 16-byte aligned) */ }; @@ -316,7 +316,7 @@ used by the guest owner with the KVM_SEV_LAUNCH_MEASURE. If len is zero on entry, the measurement blob length is written to len and uaddr is unused. -Parameters (in): struct kvm_sev_attestation +Parameters (in): struct kvm_sev_attestation_report Returns: 0 on success, -negative on error @@ -377,7 +377,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_send_update_data { + struct kvm_sev_send_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; @@ -447,7 +447,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_receive_update_data { + struct kvm_sev_receive_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; From 07de87723b6f7a7d4a699eb94e75ebe85b6b8c98 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 11:10:08 +0200 Subject: [PATCH 305/506] Documentation: KVM: x86: Use struct kvm_clock_pairing in hypercalls.rst struct kvm_clock_offset does not exist; the name is struct kvm_clock_pairing. Use struct kvm_clock_pairing. Fixes: 55dd00a73a51 ("KVM: x86: add KVM_HC_CLOCK_PAIRING hypercall") Assisted-by: LLM Signed-off-by: Karl Mehltretter Link: https://patch.msgid.link/20260905091008.40802-1-kmehltretter@gmail.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/x86/hypercalls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/x86/hypercalls.rst b/Documentation/virt/kvm/x86/hypercalls.rst index 521ecf9a8a36..e0b6c11f19ce 100644 --- a/Documentation/virt/kvm/x86/hypercalls.rst +++ b/Documentation/virt/kvm/x86/hypercalls.rst @@ -109,7 +109,7 @@ The template for each hypercall is: Usage: a0: guest physical address where host copies -"struct kvm_clock_offset" structure. +"struct kvm_clock_pairing" structure. a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0) is supported (corresponding to the host's CLOCK_REALTIME clock). From ba70065fec49766cd9e5a53ca6540703c9f53b85 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 17 Sep 2026 06:03:20 -0400 Subject: [PATCH 306/506] KVM: x86: remove "full" out-argument from flush_tlb_gva The "full" out-argument in flush_tlb_gva is only ever set on AMD processors, where the TLB flush is little more than a memory write and thus extremely cheap anyway. Simplify the code by letting the implementation of Hyper-V TLB flush hypercalls always go through all the addresses queued in tlb_flush_fifo; there are no performance concerns since that is already the case on VMX, and also was on SVM prior to commit 26505e1b5b54 ("KVM: SVM: make svm_flush_tlb_gva do a full asid flush if NPT enabled", 2026-08-06). Suggested-by: Sean Christopherson Signed-off-by: Paolo Bonzini Link: https://patch.msgid.link/20260917100320.21899-1-pbonzini@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/hyperv.c | 7 +++---- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/svm/svm.c | 4 +--- arch/x86/kvm/vmx/main.c | 4 ++-- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/vmx/x86_ops.h | 2 +- 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index e369faf3c285..1bfe56fbb8a6 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1580,7 +1580,7 @@ struct kvm_x86_ops { * Can potentially get non-canonical addresses through INVLPGs, which * the implementation may choose to ignore if appropriate. */ - void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full); + void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr); /* * Flush any TLB entries created by the guest. Like tlb_flush_gva(), diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 8d2669d8ef34..5d2e3617c952 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1997,7 +1997,6 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; int i, j, count; gva_t gva; - bool full = false; if (!tdp_enabled) return -EINVAL; @@ -2008,7 +2007,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); - for (i = 0; i < count && !full; i++) { + for (i = 0; i < count; i++) { if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) goto out_flush_all; @@ -2017,11 +2016,11 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) * pages to flush. */ gva = entries[i] & PAGE_MASK; - for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) { + for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) { if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) continue; - kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full); + kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); } ++vcpu->stat.tlb_flush; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 0601964934a1..f037fa665414 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6710,7 +6710,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w, if (is_noncanonical_invlpg_address(addr, vcpu)) return; - kvm_x86_call(flush_tlb_gva)(vcpu, addr, NULL); + kvm_x86_call(flush_tlb_gva)(vcpu, addr); if (tdp_enabled) return; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7d59d301e1e5..351b9c7233f0 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4245,7 +4245,7 @@ static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) svm_flush_tlb_asid(vcpu); } -static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) +static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) { struct vcpu_svm *svm = to_svm(vcpu); @@ -4261,8 +4261,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) } svm_flush_tlb_guest(vcpu); - if (full) - *full = true; } static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4c52ab8d0786..0ff3230fd95e 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -543,12 +543,12 @@ static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) vmx_flush_tlb_current(vcpu); } -static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { if (is_td_vcpu(vcpu)) return; - vmx_flush_tlb_gva(vcpu, addr, full); + vmx_flush_tlb_gva(vcpu, addr); } static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 58078e50c4de..4395d0809833 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3379,7 +3379,7 @@ void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) vpid_sync_context(vmx_get_current_vpid(vcpu)); } -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { /* * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 054fd14bb2e1..cdb38d940cfb 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -82,7 +82,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); bool vmx_get_if_flag(struct kvm_vcpu *vcpu); void vmx_flush_tlb_all(struct kvm_vcpu *vcpu); void vmx_flush_tlb_current(struct kvm_vcpu *vcpu); -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full); +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr); void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu); void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask); u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu); From 1065274e5cbbf43683c125c654b66a7867ead140 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 7 Sep 2026 06:38:59 +0000 Subject: [PATCH 307/506] KVM: VMX: Pass @vcpu, not @vmx to init_vmcs() Pass @vcpu instead of @vmx to init_vmcs(), and switch all of the vmx->vcpu usage to a simple vcpu. No functional change intended. Signed-off-by: Nikunj A Dadhania Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260907063906.1964557-2-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 4395d0809833..928627a2ac8c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4883,10 +4883,11 @@ int vmx_vcpu_precreate(struct kvm *kvm) #define VMX_XSS_EXIT_BITMAP 0 -static void init_vmcs(struct vcpu_vmx *vmx) +static void init_vmcs(struct kvm_vcpu *vcpu) { - struct kvm *kvm = vmx->vcpu.kvm; + struct kvm *kvm = vcpu->kvm; struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); + struct vcpu_vmx *vmx = to_vmx(vcpu); if (nested) nested_vmx_set_vmcs_shadowing_bitmap(); @@ -4911,7 +4912,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (cpu_has_tertiary_exec_ctrls()) tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); - if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { + if (enable_apicv && lapic_in_kernel(vcpu)) { vmcs_write64(EOI_EXIT_BITMAP0, 0); vmcs_write64(EOI_EXIT_BITMAP1, 0); vmcs_write64(EOI_EXIT_BITMAP2, 0); @@ -4923,7 +4924,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc))); } - if (vmx_can_use_ipiv(&vmx->vcpu)) { + if (vmx_can_use_ipiv(vcpu)) { vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); } @@ -4958,15 +4959,15 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) - vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); + vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat); vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl()); /* 22.2.1, 20.8.1 */ vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl()); - vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); - vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); + vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); + vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); set_cr4_guest_host_mask(vmx); @@ -4981,7 +4982,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); } - vmx_write_encls_bitmap(&vmx->vcpu, NULL); + vmx_write_encls_bitmap(vcpu, NULL); if (vmx_pt_mode_is_host_guest()) { memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); @@ -4994,13 +4995,13 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_writel(GUEST_SYSENTER_ESP, 0); vmcs_writel(GUEST_SYSENTER_EIP, 0); - vmx_guest_debugctl_write(&vmx->vcpu, 0); + vmx_guest_debugctl_write(vcpu, 0); if (cpu_has_vmx_tpr_shadow()) { vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); - if (cpu_need_tpr_shadow(&vmx->vcpu)) + if (cpu_need_tpr_shadow(vcpu)) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - __pa(vmx->vcpu.arch.apic->regs)); + __pa(vcpu->arch.apic->regs)); vmcs_write32(TPR_THRESHOLD, 0); } @@ -5011,7 +5012,7 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - init_vmcs(vmx); + init_vmcs(vcpu); if (nested && kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) From d75bb23606ff5d142d115f12adac339e3266eb5c Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:00 +0000 Subject: [PATCH 308/506] KVM: x86: Move PML page to common vcpu arch structure Move the PML page pointer from VMX-specific vcpu_vmx structure to the common kvm_vcpu_arch structure to enable sharing between VMX and SVM implementations. Only the page pointer is moved to x86 common code while keeping allocation logic vendor-specific, since AMD requires snp_safe_alloc_page() for PML buffer allocation. Update all VMX references accordingly. No functional change, restructuring to prepare for SVM PML support. Suggested-by: Kai Huang Reviewed-by: Kai Huang Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-3-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/vmx/vmx.c | 21 ++++++++++----------- arch/x86/kvm/vmx/vmx.h | 7 ------- arch/x86/kvm/x86.h | 5 +++++ 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1bfe56fbb8a6..6fce5a0061c2 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -789,6 +789,8 @@ struct kvm_vcpu_arch { */ struct kvm_mmu_memory_cache mmu_external_spt_cache; + struct page *pml_page; + /* * QEMU userspace and the guest each have their own FPU state. * In vcpu_run, we switch between the user and guest FPU contexts. diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 928627a2ac8c..c214633453a8 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4978,7 +4978,7 @@ static void init_vmcs(struct kvm_vcpu *vcpu) vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); if (enable_pml) { - vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); + vmcs_write64(PML_ADDRESS, page_to_phys(vcpu->arch.pml_page)); vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); } @@ -6418,17 +6418,16 @@ void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) *error_code = 0; } -static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) +static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu) { - if (vmx->pml_pg) { - __free_page(vmx->pml_pg); - vmx->pml_pg = NULL; + if (vcpu->arch.pml_page) { + __free_page(vcpu->arch.pml_page); + vcpu->arch.pml_page = NULL; } } static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) { - struct vcpu_vmx *vmx = to_vmx(vcpu); u16 pml_idx, pml_tail_index; u64 *pml_buf; int i; @@ -6451,7 +6450,7 @@ static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) * Read the entries in the same order they were written, to ensure that * the dirty ring is filled in the same order the CPU wrote them. */ - pml_buf = page_address(vmx->pml_pg); + pml_buf = page_address(vcpu->arch.pml_page); for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { u64 gpa; @@ -7648,7 +7647,7 @@ void vmx_vcpu_free(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); if (enable_pml) - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid(vmx->vpid); nested_vmx_free_vcpu(vcpu); free_loaded_vmcs(vmx->loaded_vmcs); @@ -7680,8 +7679,8 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) * for the guest), etc. */ if (enable_pml) { - vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!vmx->pml_pg) + vcpu->arch.pml_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!vcpu->arch.pml_page) goto free_vpid; } @@ -7752,7 +7751,7 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) free_vmcs: free_loaded_vmcs(vmx->loaded_vmcs); free_pml: - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid: free_vpid(vmx->vpid); return err; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..6bcca3487c59 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -265,13 +265,6 @@ struct vcpu_vmx { unsigned int ple_window; bool ple_window_dirty; - /* Support for PML */ -#define PML_LOG_NR_ENTRIES 512 - /* PML is written backwards: this is the first entry written by the CPU */ -#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) - - struct page *pml_pg; - /* apic deadline value in host tsc */ u64 hv_deadline_tsc; diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 0f5919b092e4..05c31ee984c9 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -900,4 +900,9 @@ int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); +/* Support for PML */ +#define PML_LOG_NR_ENTRIES 512 +/* PML is written backwards: this is the first entry written by the CPU */ +#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) + #endif From e1e1934fea48bf4193dcd3c2778421ce5610f5ee Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:01 +0000 Subject: [PATCH 309/506] KVM: x86: Carve out PML flush routine Move the PML (Page Modification Logging) buffer flushing logic from VMX-specific code to common x86 KVM code to enable reuse by SVM and avoid code duplication. The AMD SVM PML implementations share the same behavior as VMX PML: 1) The PML buffer is a 4K page with 512 entries 2) Hardware records dirty GPAs in reverse order (from index 511 to 0) 3) Hardware clears bits 11:0 when recording GPAs No functional change intended for VMX, except tone down the WARN_ON() to WARN_ON_ONCE() for the page alignment check. If hardware exhibits this behavior once, it's likely to occur repeatedly, so use WARN_ON_ONCE() to avoid log flooding while still capturing the unexpected condition. The refactoring prepares for SVM to leverage the same PML flushing implementation. Reviewed-by: Kai Huang Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-4-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 26 ++------------------------ arch/x86/kvm/x86.c | 31 +++++++++++++++++++++++++++++++ arch/x86/kvm/x86.h | 2 ++ 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c214633453a8..2650b6e299af 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6428,37 +6428,15 @@ static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu) static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) { - u16 pml_idx, pml_tail_index; - u64 *pml_buf; - int i; + u16 pml_idx; pml_idx = vmcs_read16(GUEST_PML_INDEX); /* Do nothing if PML buffer is empty */ if (pml_idx == PML_HEAD_INDEX) return; - /* - * PML index always points to the next available PML buffer entity - * unless PML log has just overflowed. - */ - pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; - /* - * PML log is written backwards: the CPU first writes the entry 511 - * then the entry 510, and so on. - * - * Read the entries in the same order they were written, to ensure that - * the dirty ring is filled in the same order the CPU wrote them. - */ - pml_buf = page_address(vcpu->arch.pml_page); - - for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { - u64 gpa; - - gpa = pml_buf[i]; - WARN_ON(gpa & (PAGE_SIZE - 1)); - kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); - } + kvm_flush_pml_buffer(vcpu, pml_idx); /* reset PML index */ vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 28172424c502..85ce54fa268b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3941,6 +3941,37 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) kvm_vcpu_kick(vcpu); } +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx) +{ + u16 pml_tail_index; + u64 *pml_buf; + int i; + + /* + * PML index always points to the next available PML buffer entity + * unless PML log has just overflowed. + */ + pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; + + /* + * PML log is written backwards: the CPU first writes the entry 511 + * then the entry 510, and so on. + * + * Read the entries in the same order they were written, to ensure that + * the dirty ring is filled in the same order the CPU wrote them. + */ + pml_buf = page_address(vcpu->arch.pml_page); + + for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { + u64 gpa; + + gpa = pml_buf[i]; + WARN_ON_ONCE(gpa & (PAGE_SIZE - 1)); + kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_pml_buffer); + int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) { diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 05c31ee984c9..2d747c20965c 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -905,4 +905,6 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); /* PML is written backwards: this is the first entry written by the CPU */ #define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx); + #endif From 8ac111b6dc4a523c406f5d827afb5379fbd4e025 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:02 +0000 Subject: [PATCH 310/506] KVM: VMX: Use cpu_dirty_log_size instead of enable_pml for PML checks Replace the enable_pml check with cpu_dirty_log_size in VMX PML code to determine whether PML is enabled on a per-VM basis. The enable_pml module parameter is a global setting that doesn't reflect per-VM capabilities, whereas cpu_dirty_log_size accurately indicates whether a specific VM has PML enabled. For example, TDX VMs don't yet support PML. Using cpu_dirty_log_size ensures the check correctly reflects this, while enable_pml would incorrectly indicate PML is available. This also improves consistency with kvm_mmu_update_cpu_dirty_logging(), which already uses cpu_dirty_log_size to determine PML enablement. Suggested-by: Kai Huang Reviewed-by: Kai Huang Reviewed-by: Pankaj Gupta Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-5-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 2650b6e299af..d11642e4da95 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8421,7 +8421,7 @@ void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (WARN_ON_ONCE(!enable_pml)) + if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) return; guard(vmx_vmcs01)(vcpu); From 8301c447b24031b73d51ba04de04636c59317828 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:03 +0000 Subject: [PATCH 311/506] KVM: x86: Carve out common dirty logging update Move the PML enable/disable decision logic from the VMX into a new common wrapper kvm_update_cpu_dirty_logging() in x86.c to avoid code duplication. Update the update_cpu_dirty_logging() signature to accept a bool enable parameter accordingly. Note, this subtly changes the ordering for VMX: currently guard(vmx_vmcs01) is taken before the atomic_read of nr_memslots_dirty_logging; after this refactoring the atomic_read happens in common code before guard(vmx_vmcs01) is taken. The ordering flip is harmless as a concurrent change to nr_memslots_dirty_logging will queue another update request. No functional change intended for VMX, restructuring to prepare for SVM PML support. Suggested-by: Sean Christopherson Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-6-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/vmx/main.c | 4 ++-- arch/x86/kvm/vmx/vmx.c | 12 ++---------- arch/x86/kvm/vmx/vmx.h | 2 +- arch/x86/kvm/vmx/x86_ops.h | 2 +- arch/x86/kvm/x86.c | 17 ++++++++++++++++- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6fce5a0061c2..3335c4b1afad 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1673,7 +1673,7 @@ struct kvm_x86_ops { struct x86_exception *exception); void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); - void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); + void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable); void (*vcpu_blocking)(struct kvm_vcpu *vcpu); void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 0ff3230fd95e..797206912ae9 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -108,7 +108,7 @@ static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load(vcpu, cpu); } -static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { /* * Basic TDX does not support feature PML. KVM does not enable PML in @@ -117,7 +117,7 @@ static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) if (WARN_ON_ONCE(is_td_vcpu(vcpu))) return; - vmx_update_cpu_dirty_logging(vcpu); + vmx_update_cpu_dirty_logging(vcpu, enable); } static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d11642e4da95..c9f20144a70a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8417,21 +8417,13 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit } #endif -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) - return; - guard(vmx_vmcs01)(vcpu); - /* - * Note, nr_memslots_dirty_logging can be changed concurrent with this - * code, but in that case another update request will be made and so - * the guest will never run with a stale PML value. - */ - if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) + if (enable) secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); else secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 6bcca3487c59..5f505ad14dde 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -390,7 +390,7 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated); bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated); diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index cdb38d940cfb..3351b91e924e 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -118,7 +118,7 @@ u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu); u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); void vmx_write_tsc_offset(struct kvm_vcpu *vcpu); void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); #ifdef CONFIG_X86_64 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 85ce54fa268b..be117e09cdd2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8073,6 +8073,21 @@ static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) kvm_x86_call(set_apic_access_page_addr)(vcpu); } +static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +{ + /* + * Note, nr_memslots_dirty_logging can be changed concurrent with this + * code, but in that case another update request will be made and so + * the guest will never run with a stale PML value. + */ + bool enable = atomic_read(&vcpu->kvm->nr_memslots_dirty_logging); + + if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) + return; + + kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable); +} + /* * Called within kvm->srcu read side. * Returns 1 to let vcpu_run() continue the guest execution loop without @@ -8238,7 +8253,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_call(recalc_intercepts)(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) - kvm_x86_call(update_cpu_dirty_logging)(vcpu); + kvm_update_cpu_dirty_logging(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) { kvm_vcpu_reset(vcpu, true); From 732cf7ec3224f05f29258c77648d11edc489130c Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:04 +0000 Subject: [PATCH 312/506] x86/cpufeatures: Add Page modification logging Page modification logging(PML) is a hardware feature designed to track guest modified memory pages. PML enables the hypervisor to identify which pages in a guest's memory have been changed since the last checkpoint or during live migration. The PML feature is advertised via CPUID leaf 0x8000000A ECX[4] bit. Acked-by: Borislav Petkov (AMD) Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-7-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/kernel/cpu/scattered.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index f70ee74b5f92..cbfcc66a18e9 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -238,6 +238,7 @@ // free: was #define X86_FEATURE_PVUNLOCK ( 8*32+20) /* PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* PV vcpu_is_preempted function */ #define X86_FEATURE_TDX_GUEST ( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */ +#define X86_FEATURE_PML ( 8*32+23) /* AMD Page Modification logging */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* "fsgsbase" RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 8665a6474806..b18cc9c88dd5 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -55,6 +55,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_AMD_FAST_CPPC, CPUID_EDX, 15, 0x80000007, 0 }, { X86_FEATURE_CPPC_PERF_PRIO, CPUID_EDX, 16, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, + { X86_FEATURE_PML, CPUID_ECX, 4, 0x8000000a, 0 }, { X86_FEATURE_X2AVIC_EXT, CPUID_ECX, 6, 0x8000000a, 0 }, { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, From 5ebaa85a746e886ca23f03b8d2990a3c16d7f01f Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:05 +0000 Subject: [PATCH 313/506] KVM: SVM: Use BIT_ULL for 64-bit misc_ctl bit definitions Replace BIT() with BIT_ULL() for SVM misc control bit definitions since misc_ctl is a 64-bit field in the VMCB control area structure. No functional change intended. Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-8-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/svm.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index aa63431ba92c..f199c52709df 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -240,10 +240,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) -#define SVM_MISC_ENABLE_NP BIT(0) -#define SVM_MISC_ENABLE_SEV BIT(1) -#define SVM_MISC_ENABLE_SEV_ES BIT(2) -#define SVM_MISC_ENABLE_GMET BIT(3) +#define SVM_MISC_ENABLE_NP BIT_ULL(0) +#define SVM_MISC_ENABLE_SEV BIT_ULL(1) +#define SVM_MISC_ENABLE_SEV_ES BIT_ULL(2) +#define SVM_MISC_ENABLE_GMET BIT_ULL(3) #define SVM_MISC2_ENABLE_V_LBR BIT_ULL(0) #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE BIT_ULL(1) From 0d7fbc059f33527caadf1f0387d3eea61962c402 Mon Sep 17 00:00:00 2001 From: Nikunj A Dadhania Date: Mon, 7 Sep 2026 06:39:06 +0000 Subject: [PATCH 314/506] KVM: SVM: Add Page modification logging support Currently, dirty logging relies on write protecting guest memory and marking dirty GFNs during subsequent write faults. This method works but incurs overhead due to additional write faults for each dirty GFN. Implement support for the Page Modification Logging (PML) feature, a hardware-assisted method for efficient dirty logging. PML automatically logs dirty GPA[51:12] to a 4K buffer when the CPU sets NPT D-bits. Two new VMCB fields are utilized: PML_ADDR and PML_INDEX. The PML_INDEX is initialized to 511 (8 bytes per GPA entry), and the CPU decreases the PML_INDEX after logging each GPA. When the PML buffer is full, a VMEXIT(PML_FULL) with exit code 0x407 is generated. Since PML_INDEX in the VMCB control area remains valid after an intercepted SHUTDOWN, only initialize it on reset and leave it unchanged on INIT to avoid discarding already-logged entries that haven't been flushed. PML operates on guest physical addresses at the NPT level, tracking D-bit updates in page tables rather than memory content. This allows it to work identically for normal and confidential computing guests (SEV/SEV-ES/SEV-SNP), enabling cpu_dirty_log_size to be set uniformly for all AMD VMs without special-casing encrypted guests. Use vmcb01 directly when updating PML controls to ensure L1's state remains correct, as svm->vmcb points to vmcb02 when L2 is active. PML is not enabled in hardware for nested guests; treat PML_FULL as unexpected exits. Add a new module parameter to enable/disable PML, and enable it by default when supported. Acked-by: Kai Huang Signed-off-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260907063906.1964557-9-nikunj@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/svm.h | 6 +- arch/x86/include/uapi/asm/svm.h | 2 + arch/x86/kvm/svm/nested.c | 14 +++++ arch/x86/kvm/svm/sev.c | 2 +- arch/x86/kvm/svm/svm.c | 106 +++++++++++++++++++++++++++++++- arch/x86/kvm/svm/svm.h | 1 + 6 files changed, 127 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index f199c52709df..e0a7549a7c72 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -165,7 +165,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u8 reserved_9[22]; u64 allowed_sev_features; /* Offset 0x138 */ u64 guest_sev_features; /* Offset 0x140 */ - u8 reserved_10[664]; + u8 reserved_10[128]; + u64 pml_addr; /* Offset 0x1c8 */ + u16 pml_index; /* Offset 0x1d0 */ + u8 reserved_11[526]; /* * Offset 0x3e0, 32 bytes reserved * for use by hypervisor/software. @@ -244,6 +247,7 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_MISC_ENABLE_SEV BIT_ULL(1) #define SVM_MISC_ENABLE_SEV_ES BIT_ULL(2) #define SVM_MISC_ENABLE_GMET BIT_ULL(3) +#define SVM_MISC_ENABLE_PML BIT_ULL(11) #define SVM_MISC2_ENABLE_V_LBR BIT_ULL(0) #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE BIT_ULL(1) diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 010a45c9f614..e80676185092 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -101,6 +101,7 @@ #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 #define SVM_EXIT_VMGEXIT 0x403 +#define SVM_EXIT_PML_FULL 0x407 /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001ull @@ -236,6 +237,7 @@ { SVM_EXIT_AVIC_INCOMPLETE_IPI, "avic_incomplete_ipi" }, \ { SVM_EXIT_AVIC_UNACCELERATED_ACCESS, "avic_unaccelerated_access" }, \ { SVM_EXIT_VMGEXIT, "vmgexit" }, \ + { SVM_EXIT_PML_FULL, "pml_full" }, \ { SVM_VMGEXIT_MMIO_READ, "vmgexit_mmio_read" }, \ { SVM_VMGEXIT_MMIO_WRITE, "vmgexit_mmio_write" }, \ { SVM_VMGEXIT_NMI_COMPLETE, "vmgexit_nmi_complete" }, \ diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 23d29597d6bf..ddbded0dee87 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -905,6 +905,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); + /* + * PML is never enabled in hardware for L2. Make sure that an + * unexpected PML write would trigger a PML_FULL VM-Exit. + */ + if (pml) + vmcb02->control.pml_index = -1; + /* * Stash vmcb02's counter if the guest hasn't moved past the guilty * instruction; otherwise, reset the counter to '0'. @@ -1824,6 +1831,13 @@ int nested_svm_exit_special(struct vcpu_svm *svm) if (nested_svm_is_l2_tlb_flush_hcall(vcpu)) return NESTED_EXIT_HOST; break; + case SVM_EXIT_PML_FULL: + /* + * All PML full exits are handled by KVM. KVM emulates PML in + * software for L1, but never enables PML in hardware on behalf + * of L1. + */ + return NESTED_EXIT_HOST; default: break; } diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5705723f1f41..cfe990f80504 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4994,7 +4994,7 @@ struct page *snp_safe_alloc_page_node(int node, gfp_t gfp) * Allocate an SNP-safe page to workaround the SNP erratum where * the CPU will incorrectly signal an RMP violation #PF if a * hugepage (2MB or 1GB) collides with the RMP entry of a - * 2MB-aligned VMCB, VMSA, or AVIC backing page. + * 2MB-aligned VMCB, VMSA, PML or AVIC backing page. * * Allocate one extra page, choose a page which is not * 2MB-aligned, and free the other. diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 351b9c7233f0..842c16483f6c 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -179,6 +179,9 @@ module_param(vnmi, bool, 0444); module_param(enable_mediated_pmu, bool, 0444); +bool __ro_after_init pml = true; +module_param(pml, bool, 0444); + static bool __ro_after_init svm_gp_erratum_intercept = true; static u8 rsm_ins_bytes[] = "\x0f\xaa"; @@ -1267,6 +1270,23 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) if (vcpu->kvm->arch.bus_lock_detection_enabled) svm_set_intercept(svm, INTERCEPT_BUSLOCK); + if (pml) { + /* + * Populate the page address and index here, PML is enabled + * when dirty logging is enabled on the memslot through + * svm_update_cpu_dirty_logging() + */ + control->pml_addr = (u64)__sme_set(page_to_phys(vcpu->arch.pml_page)); + + /* + * The VMCB control area remains valid after an intercepted SHUTDOWN + * (AMD APM Rev 3.45+), so only initialize PML index on reset to avoid + * discarding already-logged entries that haven't been flushed. + */ + if (!init_event) + control->pml_index = PML_HEAD_INDEX; + } + if (is_sev_guest(vcpu)) sev_init_vmcb(svm, init_event); @@ -1326,9 +1346,15 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) if (!vmcb01_page) goto out; + if (pml) { + vcpu->arch.pml_page = snp_safe_alloc_page(); + if (!vcpu->arch.pml_page) + goto error_free_vmcb_page; + } + err = sev_vcpu_create(vcpu); if (err) - goto error_free_vmcb_page; + goto error_free_pml_page; err = avic_init_vcpu(svm); if (err) @@ -1353,6 +1379,9 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) error_free_sev: sev_free_vcpu(vcpu); +error_free_pml_page: + if (vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); error_free_vmcb_page: __free_page(vmcb01_page); out: @@ -1370,6 +1399,9 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu) sev_free_vcpu(vcpu); + if (pml && vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); + __free_page(__sme_pa_to_page(svm->vmcb01.pa)); svm_vcpu_free_msrpm(svm->msrpm); } @@ -3344,6 +3376,45 @@ static int vmmcall_interception(struct kvm_vcpu *vcpu) return kvm_emulate_hypercall(vcpu); } +static void svm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb01 = svm->vmcb01.ptr; + + if (enable) + vmcb01->control.misc_ctl |= SVM_MISC_ENABLE_PML; + else + vmcb01->control.misc_ctl &= ~SVM_MISC_ENABLE_PML; + + vmcb_mark_dirty(vmcb01, VMCB_NPT); +} + +static void svm_flush_pml_buffer(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_control_area *control = &svm->vmcb->control; + + /* Do nothing if PML buffer is empty */ + if (control->pml_index == PML_HEAD_INDEX) + return; + + kvm_flush_pml_buffer(vcpu, control->pml_index); + + /* Reset the PML index */ + control->pml_index = PML_HEAD_INDEX; +} + +static int pml_full_interception(struct kvm_vcpu *vcpu) +{ + trace_kvm_pml_full(vcpu->vcpu_id); + + /* + * PML buffer is already flushed at the beginning of svm_handle_exit(). + * Nothing to do here. + */ + return 1; +} + static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { [SVM_EXIT_READ_CR0] = cr_interception, [SVM_EXIT_READ_CR3] = cr_interception, @@ -3421,6 +3492,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { #ifdef CONFIG_KVM_AMD_SEV [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, #endif + [SVM_EXIT_PML_FULL] = pml_full_interception, }; static void dump_vmcb(struct kvm_vcpu *vcpu) @@ -3470,8 +3542,14 @@ static void dump_vmcb(struct kvm_vcpu *vcpu) pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); - pr_err("%-20s%lld\n", "misc_ctl:", control->misc_ctl); + pr_err("%-20s%llx\n", "misc_ctl:", control->misc_ctl); pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); + + if (pml) { + pr_err("%-20s%016llx\n", "pml_addr:", control->pml_addr); + pr_err("%-20s%04x\n", "pml_index:", control->pml_index); + } + pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); pr_err("%-20s%08x\n", "event_inj:", control->event_inj); @@ -3649,6 +3727,13 @@ int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 __exit_code) (u64)exit_code != __exit_code) goto unexpected_vmexit; + /* + * PML is never enabled when running L2, bail immediately if a PML full + * exit occurs as something is horribly wrong. + */ + if (unlikely(is_guest_mode(vcpu) && exit_code == SVM_EXIT_PML_FULL)) + goto unexpected_vmexit; + #ifdef CONFIG_MITIGATION_RETPOLINE if (exit_code == SVM_EXIT_MSR) return msr_interception(vcpu); @@ -3717,6 +3802,14 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; + /* + * Opportunistically flush the PML buffer on VM exit. This keeps the + * dirty bitmap current by processing logged GPAs rather than waiting for + * PML_FULL exit. + */ + if (vcpu->kvm->arch.cpu_dirty_log_size && !is_guest_mode(vcpu)) + svm_flush_pml_buffer(vcpu); + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) return 0; @@ -5309,6 +5402,9 @@ static int svm_vm_init(struct kvm *kvm) if (!pause_filter_count || !pause_filter_thresh) kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); + if (pml) + kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES; + svm_srso_vm_init(); return 0; } @@ -5427,6 +5523,8 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .check_intercept = svm_check_intercept, .handle_exit_irqoff = svm_handle_exit_irqoff, + .update_cpu_dirty_logging = svm_update_cpu_dirty_logging, + .deliver_interrupt = svm_deliver_interrupt, .pi_update_irte = avic_pi_update_irte, .setup_mce = svm_setup_mce, @@ -5687,6 +5785,10 @@ static __init int svm_hardware_setup(void) nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS); + pml = pml && npt_enabled && cpu_feature_enabled(X86_FEATURE_PML); + if (pml) + pr_info("Page modification logging supported\n"); + if (lbrv) { if (!boot_cpu_has(X86_FEATURE_LBRV)) lbrv = false; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index e958943b8162..f566dae44bb0 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -53,6 +53,7 @@ extern int vgif; extern bool intercept_smi; extern bool vnmi; extern int lbrv; +extern bool pml; extern int tsc_aux_uret_slot __ro_after_init; From 1c24893fdb4df1e8713d921a849ef14626e50954 Mon Sep 17 00:00:00 2001 From: Luka Absandze Date: Sun, 20 Sep 2026 15:54:53 +0000 Subject: [PATCH 315/506] KVM: x86/pmu: Don't retry a counter whose config was rejected kvm_pmu_handle_event() re-arms the reprogram bit for every failed reprogram, on the assumption that the failure is transient and a later refresh will succeed. That is true for contention, e.g. the -EBUSY from x86_reserve_hardware(), but not for a configuration the host PMU driver rejects outright. A rejected config can never succeed on retry, so the counter is reprogrammed on every PMU refresh for as long as the guest leaves it enabled, and every attempt fails the same way. Re-arm only for the errnos that indicate a transient condition, i.e. -EBUSY and -ENOMEM. Suggested-by: Sean Christopherson Link: https://lore.kernel.org/all/20260916123315.89042-1-absandze@amazon.de/ Signed-off-by: Luka Absandze Reviewed-by: Sandipan Das Link: https://patch.msgid.link/20260920155453.10836-1-absandze@amazon.de Signed-off-by: Sean Christopherson --- arch/x86/kvm/pmu.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index a7d60c8785cd..03a470c49a74 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -662,7 +662,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; - int bit; + int bit, r; bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); @@ -676,12 +676,14 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) kvm_for_each_pmc(pmu, pmc, bit, bitmap) { /* - * If reprogramming fails, e.g. due to contention, re-set the - * reprogram bit, i.e. opportunistically try again on the next - * PMU refresh. Don't make a new request as doing so can stall - * the guest if reprogramming repeatedly fails. + * If reprogramming fails on a transient condition, e.g. due to + * contention, re-set the reprogram bit, i.e. opportunistically + * try again on the next PMU refresh. Don't make a new request + * as doing so can stall the guest if reprogramming repeatedly + * fails. */ - if (reprogram_counter(pmc)) + r = reprogram_counter(pmc); + if (r == -EBUSY || r == -ENOMEM) set_bit(pmc->idx, pmu->reprogram_pmi); } From bf260dc9e7b8bfcd06c1a8e739797b1cf0917c72 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Thu, 3 Sep 2026 22:32:58 +0000 Subject: [PATCH 316/506] KVM: SVM: Trigger new ASID allocation in both VMCBs on pCPU switch When the pCPU where the VMCB was most recently used is switched, reset the ASID generation in both VMCBs, triggering new ASID allocation for the immediate VMRUN as well as the next VMRUN on the other VMCB. The ASID is shared between vmcb01 and vmcb02, and gets flushed on every nested transition. However, since pCPU tracking is done per VMCB, it is possible for one VMCB to allocate a new ASID when migrated to a new pCPU, and then the other VMCB reuses that ASID on the old pCPU. This can result in the same ASID being used by multiple vCPUs on the old pCPU. Example scenario: - vCPU runs on pCPU A, vmcb01 is active, asid=1. - vCPU migrates to pCPU B, vmcb01 pCPU changes, new asid=2. - Another vCPU runs on pCPU A and allocates asid=2 as well. - vCPU migrates back to pCPU A, and then switches to vmcb02 before it runs again with vmcb01. - No pCPU switch is detected for vmcb02, so VMRUN is done with asid=2. - Two vCPUs end up using asid=2 on the same pCPU. Keep the VMCB dirtying to the active VMCB only. Clean bits are tracked by a pCPU for each VMCB, so do not unnecessarily dirty a VMCB if its pCPU does not change. Additionally, initialize the tracked pCPU for vmcb02 to -1 on nested enablement as hardening, so that a new ASID allocation is always triggered when nested is disabled and re-enabled. No performance regression was noticed when overcommitting L1 vCPUs in L0 (to force rescheduling), pinning L1 <-> L2 vCPUs, and running CPUID in a tight loop bouncing between 2 vCPUs in L2. An alternative (and perhaps more proper) fix would be tracking the ASID per-VMCB instead (e.g. [1]). However, that's a more involved change, and it would result in having different ASIDs for L1 and L2 without actually properly maintaining them. It would probably work because all TLB flushes target the current VMCB, and the other VMCB is always flushed on nested transitions, but the code ends up in an arguably more fragile state. Punt a proper clean fix to an incoming (and overdue) overhaul of SVM's ASID usage [2]. [1] https://lore.kernel.org/lkml/20250205182402.2147495-2-yosry.ahmed@linux.dev [2] https://lore.kernel.org/kvm/20260728003557.1136583-1-yosry@kernel.org Cc: stable@vger.kernel.org Reported-by: Stefan Teodorescu Signed-off-by: Yosry Ahmed Reviewed-by: Nikunj A Dadhania Link: https://patch.msgid.link/20260903223258.486034-1-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 1 + arch/x86/kvm/svm/svm.c | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 73f37b050d0a..0c55c71fc601 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -1494,6 +1494,7 @@ int svm_allocate_nested(struct vcpu_svm *svm) if (!svm->nested.msrpm) goto err_free_vmcb02; + svm->nested.vmcb02.cpu = -1; svm->nested.initialized = true; return 0; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7d59d301e1e5..2332814cd203 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3765,14 +3765,25 @@ static int pre_svm_run(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); /* - * If the previous vmrun of the vmcb occurred on a different physical - * cpu, then mark the vmcb dirty and assign a new asid. Hardware's - * vmcb clean bits are per logical CPU, as are KVM's asid assignments. + * If the previous VMRUN of the VMCB occurred on a different physical + * cpu, then mark the VMCB dirty as hardware's clean bits are per pCPU. + * + * Reset the ASID generation in both VMCBs. This will lead to assigning + * a new ASID now, and then again when switching to the other VMCB. + * However, this is needed as the ASID is shared between the VMCBs, and + * otherwise it would be possible to use an ASID allocated on one pCPU + * on another, for example: + * - vCPU migrates from pCPU A to pCPU B, allocates a new ASID. + * - vCPU migrates back to pCPU A, and then switches the VMCB. + * - The new VMCB does not detect a pCPU change and runs on pCPU A with + * the new ASID allocated on pCPU B, which is potentially used by + * another vCPU/VM. */ if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { - svm->current_vmcb->asid_generation = 0; vmcb_mark_all_dirty(svm->vmcb); svm->current_vmcb->cpu = vcpu->cpu; + svm->vmcb01.asid_generation = 0; + svm->nested.vmcb02.asid_generation = 0; } if (is_sev_guest(vcpu)) From 681f3b68f4356f223ad32e1fa2253b2929750685 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 28 Sep 2026 19:02:50 +0100 Subject: [PATCH 317/506] KVM: guest_memfd: Fix the in-place conversion documentation Commit dcde2f853ff46 ("KVM: guest_memfd: Ensure pages are not in use before conversion") added error_offset to struct kvm_memory_attributes2, written back on failure, and a -EAGAIN return. The KVM_SET_MEMORY_ATTRIBUTES2 section of api.rst still lists the old struct, marks it input-only and leaves EAGAIN out of the Errors table. It also refers to KVM_CAP_GUEST_MEMFD_MMAP, which had become KVM_CAP_GUEST_MEMFD_FLAGS before this ioctl was added. kvm.gmem_in_place_conversion's entry says in-place conversion is unconditionally enabled without CONFIG_KVM_VM_MEMORY_ATTRIBUTES, but a build without kvm_arch_has_private_mem has it disabled: arm64 always, and x86 without CONFIG_KVM_SW_PROTECTED_VM, CONFIG_KVM_INTEL_TDX or CONFIG_KVM_AMD_SEV. Bring both in line with the code: replace the stale capability with GUEST_MEMFD_FLAG_MMAP, and describe KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES the way the KVM_CREATE_GUEST_MEMFD section describes KVM_CAP_GUEST_MEMFD_FLAGS. Fixes: 799c9fadc6399 ("KVM: guest_memfd: Add base support for KVM_SET_MEMORY_ATTRIBUTES2") Fixes: dcde2f853ff46 ("KVM: guest_memfd: Ensure pages are not in use before conversion") Fixes: 537ec2b15fdd7 ("KVM: Let userspace disable per-VM mem attributes, enable per-gmem attributes") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260928180250.3075584-1-fuad.tabba@linux.dev Signed-off-by: Sean Christopherson --- Documentation/admin-guide/kernel-parameters.txt | 3 ++- Documentation/virt/kvm/api.rst | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 49560b1b54c4..27e1f05c44c0 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3177,7 +3177,8 @@ Kernel parameters Note, this parameter is only available when CONFIG_KVM_VM_MEMORY_ATTRIBUTES=y. When CONFIG_KVM_VM_MEMORY_ATTRIBUTES is not set, in-place - conversion is unconditionally enabled. + conversion is enabled (and cannot be disabled) if KVM + is built with private memory support. Default is N (off). diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 67f0f290797a..3d6a4bf01c70 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6690,7 +6690,7 @@ significant bit): :Capability: KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES :Architectures: all :Type: guest_memfd ioctl -:Parameters: struct kvm_memory_attributes2 (in) +:Parameters: struct kvm_memory_attributes2 (in/out) :Returns: 0 on success, <0 on error Errors: @@ -6700,6 +6700,8 @@ Errors: page aligned, causes an overflow, or size is zero). EFAULT The parameter address was invalid. ENOMEM Ran out of memory trying to track private/shared state + EAGAIN Pages in the range have outstanding references, see + ``error_offset`` below. ========== =============================================================== KVM_SET_MEMORY_ATTRIBUTES2 is an extension to @@ -6719,15 +6721,19 @@ Attribute values are shared with KVM_SET_MEMORY_ATTRIBUTES. __u64 size; __u64 attributes; __u64 flags; - __u64 reserved[12]; + __u64 error_offset; + __u64 reserved[11]; }; #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) +The capability KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES enumerates the attributes +that can be set via KVM_SET_MEMORY_ATTRIBUTES2. + Set attributes for a range of offsets within a guest_memfd to KVM_MEMORY_ATTRIBUTE_PRIVATE to limit the specified guest_memfd backed -memory range for guest use. Even if KVM_CAP_GUEST_MEMFD_MMAP is -supported, after a successful call to set +memory range for guest use. Even if the guest_memfd was created with +GUEST_MEMFD_FLAG_MMAP, after a successful call to set KVM_MEMORY_ATTRIBUTE_PRIVATE, the requested range will not be mappable into host userspace and will only be mappable by the guest. From ad06cf5338bc5501bd989584db6dc71972bb436a Mon Sep 17 00:00:00 2001 From: Hao Zhang Date: Mon, 27 Jul 2026 10:37:41 +0800 Subject: [PATCH 318/506] KVM: guest_memfd: Avoid deadlock in error_remove_folio() memory_failure() invokes ->error_remove_folio() while holding the poisoned folio lock. KVM's guest_memfd callback then takes mapping->invalidate_lock for read before invalidating guest mappings. This can deadlock with paths that hold mapping->invalidate_lock for write and then try to lock the same folio, e.g. truncation from MADV_REMOVE or FALLOC_FL_PUNCH_HOLE. Because rwsems are fair, the problem is not limited to guest_memfd punch-hole: a pending writer can also block future readers, so ->error_remove_folio() must not block on the invalidate lock while holding the poisoned folio lock. Use filemap_invalidate_trylock_shared() in kvm_gmem_error_folio(). If the invalidate lock is contended, fail the recovery instead of potentially deadlocking the memory-failure path. Fixes: a7800aa80ea4 ("KVM: Add KVM_CREATE_GUEST_MEMFD ioctl() for guest-specific backing memory") Signed-off-by: Hao Zhang Link: https://lore.kernel.org/all/tencent_7DA064CAC9B463EB9B651801011E2CE4230A@qq.com/ Link: https://patch.msgid.link/ambEdSPjerZIVN0b@192.168.1.215 Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index a13445c26d9d..93f3d29d5751 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -828,7 +828,17 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol enum kvm_gfn_range_filter filter; pgoff_t start, end; - filemap_invalidate_lock_shared(mapping); + /* + * memory_failure() invokes ->error_remove_folio() while holding the + * poisoned folio lock. Do not block on mapping->invalidate_lock, as + * a writer that already holds the invalidate lock can be waiting for + * the same folio, e.g. truncation from MADV_REMOVE/FALLOC_FL_PUNCH_HOLE. + * + * If the invalidate lock is contended, fail the recovery instead of + * risking a deadlock in the memory-failure path. + */ + if (!filemap_invalidate_trylock_shared(mapping)) + return -EBUSY; start = folio->index; end = start + folio_nr_pages(folio); From 003892d6e802a1c08ad03e8ea793453465cae7f4 Mon Sep 17 00:00:00 2001 From: Jacky Li Date: Wed, 23 Sep 2026 20:18:12 +0000 Subject: [PATCH 319/506] KVM: SEV: Return INVALID_INPUT on SNP req/resp buffer access failure Currently, snp_handle_(ext_)guest_req() returns -EIO when kvm_{read/clear}_guest() fails while accessing guest-provided buffers. Returning -EIO causes KVM_RUN to exit to userspace, likely killing the VM. Fix this by returning GHCB_HV_RESP_MALFORMED_INPUT with sub-error code GHCB_ERR_INVALID_INPUT to the guest and resuming the vCPU. Per the GHCB specification, guest-provided GPA buffers that cannot be accessed by the hypervisor (e.g. private pages) should be treated as guest input errors. Because kvm_{read/write/clear}_guest() only returns -EFAULT on failure, treating this failure as an invalid input aligns with the definition of -EFAULT ("Bad address"). Returning GHCB_ERR_INVALID_INPUT also matches existing SNP handling in KVM, which already returns this error code for unaligned or overlapping buffers. It also aligns with other hypercall implementations in KVM (e.g. Hyper-V returning INVALID_HYPERCALL_INPUT on kvm_read_guest() failures in kvm_hv_flush_tlb()). Additionally, tickle the response buffer before issuing the PSP command so that an invalid resp_gpa fails upfront with GHCB_ERR_INVALID_INPUT, i.e. to avoid terminating the guest if it provides a bad buffer. Do not return an error to the guest if the post-firmware write fails, as the firmware has already incremented the VMPCK sequence number. Exiting to userspace avoids creating an ambiguous failure where the guest has no way to know whether to advance its VMPCK sequence number. I.e. the "tickle" is a best effort compromise between terminating the guest on bad input, and horribly confusing the guest on bad *host userspace* behavior. Fixes: 88caf544c930 ("KVM: SEV: Provide support for SNP_GUEST_REQUEST NAE event") Fixes: 74458e4859d8 ("KVM: SEV: Provide support for SNP_EXTENDED_GUEST_REQUEST NAE event") Signed-off-by: Jacky Li Link: https://patch.msgid.link/20260923-snp-invalid-input-v2-1-c00e0dd077ca@google.com [sean: massage comments and changelog regarding the tickle] Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5705723f1f41..7a1563cb471a 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4221,6 +4221,7 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ struct kvm *kvm = svm->vcpu.kvm; struct kvm_sev_info *sev = to_kvm_sev_info(kvm); sev_ret_code fw_err = 0; + u8 tickle = 0; int ret; if (!is_sev_snp_guest(&svm->vcpu)) @@ -4228,8 +4229,21 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ guard(mutex)(&sev->guest_req_mutex); - if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE)) - return -EIO; + if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } + + /* + * "Tickle" the response buffer to verify it's writable before sending + * the request to firmware, which will modify the VMPCK sequence number. + * Note, this is a best effort check and doesn't guard against TOCTOU + * issues. See below for more information. + */ + if (kvm_write_guest(kvm, resp_gpa, &tickle, sizeof(tickle))) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } data.gctx_paddr = __psp_pa(sev->snp_context); data.req_paddr = __psp_pa(sev->guest_req_buf); @@ -4244,6 +4258,13 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ if (ret && !fw_err) return ret; + /* + * Exit to userspace if writing the response fails, e.g. if the mapping + * changed between the initial tickle and the actual write, to avoid + * creating an ambiguous failure ABI with the guest. Don't return an + * error to the guest because firmware already incremented the VMPCK + * sequence number, but there's no way to communicate that to the guest. + */ if (kvm_write_guest(kvm, resp_gpa, sev->guest_resp_buf, PAGE_SIZE)) return -EIO; @@ -4295,8 +4316,10 @@ static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t r return -EINVAL; if (kvm_read_guest(kvm, req_gpa + offsetof(struct snp_guest_msg_hdr, msg_type), - &msg_type, 1)) - return -EIO; + &msg_type, 1)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } /* * As per GHCB spec, requests of type MSG_REPORT_REQ also allow for @@ -4335,8 +4358,10 @@ static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t r * As per GHCB spec (see "SNP Extended Guest Request"), the * certificate table is terminated by 24-bytes of zeroes. */ - if (data_npages && kvm_clear_guest(kvm, data_gpa, 24)) - return -EIO; + if (data_npages && kvm_clear_guest(kvm, data_gpa, 24)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } } return snp_handle_guest_req(svm, req_gpa, resp_gpa); From 98a843669196ace8789a859796e45caffb970e40 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Sun, 20 Sep 2026 07:24:59 +0000 Subject: [PATCH 320/506] KVM: x86/mmu: Convert can't-happen fault path EFAULTs to KVM_BUG_ON() + -EIO Four -EFAULT returns in x86's page fault path are guarded by WARN_ON_ONCE() because they describe KVM bugs, not conditions the guest or userspace can reach: - direct_map() and FNAME(fetch)() completing the walk at a level other than the goal level, i.e. KVM's shadow page table walk disagreeing with the mapping level KVM itself just computed. - a CR2 with bits 63:32 set on 32-bit KVM, which the hardware cannot produce. - PFERR_PRIVATE_ACCESS set on a reserved-bit fault. KVM sets that synthetic bit itself, and only when PFERR_RSVD_MASK is clear. Returning -EFAULT for a KVM bug conflicts with KVM_CAP_MEMORY_FAULT_INFO, which promises that an -EFAULT out of KVM_RUN on a guest page fault VM-Exit is accompanied by kvm_run.memory_fault. Describing a broken KVM invariant as a memory fault would be actively misleading, as there is no guest access for userspace to resolve and retry. Convert the four sites to KVM_BUG_ON() + -EIO, the established way to report that KVM is hosed, so that -EFAULT out of the fault path always means "guest memory access KVM could not resolve". Suggested-by: Sean Christopherson Link: https://lore.kernel.org/all/Zr-8M9rYplgN6IS3@google.com/ Signed-off-by: Mikhail Malyshev Link: https://patch.msgid.link/20260920072459.3485710-3-mike.malyshev@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 12 ++++++------ arch/x86/kvm/mmu/paging_tmpl.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 2bf6ed878589..f14440a2b547 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3577,8 +3577,8 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) fault->req_level >= it.level); } - if (WARN_ON_ONCE(it.level != fault->goal_level)) - return -EFAULT; + if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm)) + return -EIO; ret = mmu_set_spte(vcpu, fault->slot, it.sptep, access, base_gfn, fault->pfn, fault); @@ -4941,8 +4941,8 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, #ifndef CONFIG_X86_64 /* A 64-bit CR2 should be impossible on 32-bit KVM. */ - if (WARN_ON_ONCE(fault_address >> 32)) - return -EFAULT; + if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm)) + return -EIO; #endif /* * Legacy #PF exception only have a 32-bit error code. Simply drop the @@ -6651,8 +6651,8 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err r = RET_PF_INVALID; if (unlikely(error_code & PFERR_RSVD_MASK)) { - if (WARN_ON_ONCE(error_code & PFERR_PRIVATE_ACCESS)) - return -EFAULT; + if (KVM_BUG_ON(error_code & PFERR_PRIVATE_ACCESS, vcpu->kvm)) + return -EIO; r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct); if (r == RET_PF_EMULATE) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 27427e7f22fa..4f5369cf3753 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -775,8 +775,8 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, fault->req_level >= it.level); } - if (WARN_ON_ONCE(it.level != fault->goal_level)) - return -EFAULT; + if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm)) + return -EIO; ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access, base_gfn, fault->pfn, fault); From 64f1cf2884403895feed3c442c65e1cb785945b9 Mon Sep 17 00:00:00 2001 From: Anish Moorthy Date: Sun, 20 Sep 2026 07:24:58 +0000 Subject: [PATCH 321/506] KVM: x86/mmu: Report a memory fault exit when the fault handler EFAULTs KVM_CAP_MEMORY_FAULT_INFO documents that KVM_RUN will fill kvm_run.memory_fault when KVM cannot resolve a guest page fault VM-Exit, "e.g. if there is a valid memslot but no backing VMA for the corresponding host virtual address". kvm_handle_error_pfn() does not honor that guarantee. It returns a bare -EFAULT, leaving userspace with no indication of which guest physical address faulted, or even that the exit was a memory fault at all. Userspace cannot distinguish a transient, resolvable condition from a fatal one, so in practice the VMM terminates the guest. Fill kvm_run.memory_fault before returning -EFAULT. A concrete user is an Intel integrated GPU assigned to a guest via vfio-pci. The guest driver clears PCI_COMMAND.MEM on one vCPU while another vCPU is mid-MMIO to a BAR of the same device. Clearing PCI_COMMAND.MEM makes vfio-pci zap the BAR's mmap, so the second vCPU's fault finds a valid memslot whose VMA can no longer supply a PFN, and KVM_RUN fails with a bare -EFAULT. The VM dies, even though the guest did nothing architecturally invalid and the condition clears as soon as the driver re-enables memory decoding. Reproduce by pairing a vCPU that spins on accesses to the assigned device's BAR0 with a vCPU that toggles PCI_COMMAND.MEM; the race is hit within minutes. The same crash has been observed in the field on production edge hardware. Reporting the fault does not by itself define the access semantics. Userspace still has to decide what a read or write to a BAR with memory decoding disabled returns. But it is the information userspace needs in order to make that decision instead of killing the guest. Suggested-by: Sean Christopherson Fixes: 16f95f3b95ca ("KVM: Add KVM_EXIT_MEMORY_FAULT exit to report faults to userspace") Link: https://lore.kernel.org/all/20240809205158.1340255-1-amoorthy@google.com/ Link: https://lore.kernel.org/all/Zr-8M9rYplgN6IS3@google.com/ Signed-off-by: Anish Moorthy Co-developed-by: Mikhail Malyshev Signed-off-by: Mikhail Malyshev Link: https://patch.msgid.link/20260920072459.3485710-2-mike.malyshev@gmail.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f14440a2b547..4d22e0d3f8a1 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3616,6 +3616,7 @@ static int kvm_handle_error_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa return RET_PF_RETRY; } + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return -EFAULT; } From e70f812ccdd6f45b002ff054b0108bbf8db1c0ed Mon Sep 17 00:00:00 2001 From: Kyle Zeng Date: Thu, 17 Sep 2026 20:36:58 -0700 Subject: [PATCH 322/506] KVM: x86: Restrict saved GPA writes to hardware write faults A GPA supplied by a hardware page fault describes the access that faulted, not an arbitrary instruction decoded afterwards. A guest can change an MMIO read into a store before KVM fetches the instruction. The saved-GPA shortcut then skips the write-aware guest page-table walk and can issue a write through a guest-read-only mapping. Carry hardware write-fault information into the emulator and retain it alongside the saved GPA. For guests without protected page tables, only reuse that GPA for a write when hardware reported a write access. Reads and faults with unknown access direction do not authorize an emulated write; fall back to the existing permission-aware translation in those cases. Keep the authorization across MMIO completion without decoding a new instruction, and reset it when initializing a fresh emulator context. This also covers writes reached through the cmpxchg fallback. Preserve the shortcut for hardware-reported writes and for the read side of read-modify-write emulation after such a fault. For guests with protected page tables, preserve the existing behavior and always allow writes through the saved GPA. An RMW instruction can fault on its initial read, and KVM cannot walk the protected guest page tables to check the subsequent write. Track this property in common KVM state for SEV and TDX. Also account for legacy SEV initialization and encryption-context copies and moves. Fixes: 0f89b207b04a ("kvm: svm: Use the hardware provided GPA instead of page walk") Reported-by: Oleg Boiko Assisted-by: Codex:gpt-6-astra Signed-off-by: Kyle Zeng Link: https://patch.msgid.link/20260918033658.21842-1-kylebot@openai.com [sean: clear has_protected_page_tables on the source after migration] Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/kvm_emulate.h | 4 ++-- arch/x86/kvm/mmu/mmu.c | 3 +++ arch/x86/kvm/svm/sev.c | 5 +++++ arch/x86/kvm/vmx/tdx.c | 1 + arch/x86/kvm/x86.c | 17 ++++++++++++++--- arch/x86/kvm/x86.h | 3 +++ 7 files changed, 29 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3335c4b1afad..2159f1ba4f33 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1168,6 +1168,7 @@ struct kvm_arch { u8 mmu_valid_gen; u8 vm_type; bool has_private_mem; + bool has_protected_page_tables; bool has_protected_state; bool has_protected_eoi; bool has_protected_pmu; diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 3e375af15c03..10886bea82c9 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -354,8 +354,8 @@ struct x86_emulate_ctxt { bool have_exception; struct x86_exception exception; - /* GPA available */ - bool gpa_available; + /* Saved GPA and permitted emulated accesses. */ + u64 gpa_access; gpa_t gpa_val; /* diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f037fa665414..21a9a11b9ce6 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6638,6 +6638,9 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err return r; emulate: + if (error_code & PFERR_WRITE_MASK) + emulation_type |= EMULTYPE_PF_WRITE; + return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn, insn_len); } diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index cfe990f80504..dd0943cd0133 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -560,6 +560,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, INIT_LIST_HEAD(&sev->regions_list); INIT_LIST_HEAD(&sev->mirror_vms); sev->need_init = false; + kvm->arch.has_protected_page_tables = true; kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV); @@ -2041,12 +2042,14 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) dst->pages_locked = src->pages_locked; dst->es_active = src->es_active; dst->vmsa_features = src->vmsa_features; + dst_kvm->arch.has_protected_page_tables = true; src->asid = 0; src->active = false; src->handle = 0; src->pages_locked = 0; src->es_active = false; + src_kvm->arch.has_protected_page_tables = false; list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); @@ -2907,6 +2910,7 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) mutex_unlock(&sev_mirror_lock); mirror_sev->active = true; + kvm->arch.has_protected_page_tables = true; mirror_sev->asid = source_sev->asid; mirror_sev->fd = source_sev->fd; mirror_sev->es_active = source_sev->es_active; @@ -2965,6 +2969,7 @@ void sev_vm_init(struct kvm *kvm) kvm->arch.has_protected_state = true; fallthrough; case KVM_X86_SEV_VM: + kvm->arch.has_protected_page_tables = true; kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem; to_kvm_sev_info(kvm)->need_init = true; break; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index a0bc9f818f43..2d2febd4a5ef 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -616,6 +616,7 @@ int tdx_vm_init(struct kvm *kvm) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); + kvm->arch.has_protected_page_tables = true; kvm->arch.has_protected_state = true; /* * TDX Module doesn't allow the hypervisor to modify the EOI-bitmap, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index be117e09cdd2..de77b222870e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -5120,7 +5120,8 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, * operation using rep will only have the initial GPA from the NPF * occurred. */ - if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) && + if ((ctxt->gpa_access & (write ? ACC_WRITE_MASK : ACC_READ_MASK)) && + emulator_can_use_gpa(ctxt) && (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { gpa = ctxt->gpa_val; ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); @@ -5969,7 +5970,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - ctxt->gpa_available = false; + ctxt->gpa_access = 0; ctxt->eflags = kvm_get_rflags(vcpu); ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; @@ -6485,7 +6486,17 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, /* With shadow page tables, cr2 contains a GVA or nGPA. */ if (vcpu->arch.mmu->root_role.direct) { - ctxt->gpa_available = true; + ctxt->gpa_access = ACC_READ_MASK; + /* + * Always allow writes for guests with protected page + * tables, as KVM can't walk the guest's page tables, + * i.e. KVM can't get the RMW protections for a given + * GVA to see if the write side of a RMW operation + * should be allowed. + */ + if ((emulation_type & EMULTYPE_PF_WRITE) || + vcpu->kvm->arch.has_protected_page_tables) + ctxt->gpa_access |= ACC_WRITE_MASK; ctxt->gpa_val = cr2_or_gpa; } } else { diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 2d747c20965c..cbcf8e3ff31a 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -407,6 +407,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case * the CR2/GPA value pass on the stack is valid. * + * EMULTYPE_PF_WRITE - Set with EMULTYPE_PF when hardware reports a write access. + * * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility * state and inject single-step #DBs after skipping * an instruction (after completing userspace I/O). @@ -445,6 +447,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7) #define EMULTYPE_WRITE_PF_TO_SP (1 << 8) #define EMULTYPE_SKIP_SOFT_INT (1 << 9) +#define EMULTYPE_PF_WRITE (1 << 10) #define EMULTYPE_SET_SOFT_INT_VECTOR(v) ((u32)((v) & 0xff) << 16) #define EMULTYPE_GET_SOFT_INT_VECTOR(e) (((e) >> 16) & 0xff) From dd572ec94640b07e495237d2a6afd3d29a4c2a7c Mon Sep 17 00:00:00 2001 From: Gokul K Date: Fri, 18 Sep 2026 17:06:30 +0530 Subject: [PATCH 323/506] KVM: selftests: Delete the declaration of vcpu_set_cpuid_maxphyaddr() vcpu_set_cpuid_maxphyaddr() is declared in include/x86/processor.h and defined nowhere. The declaration is the only mention of the name in tools/testing/selftests, so nothing has ever been able to call it. No Fixes: tag: a prototype for a function that does not exist cannot have regressed anything, and there is nothing to backport. No functional change intended. Signed-off-by: Gokul K Link: https://patch.msgid.link/20260918113631.634233-3-gokul02k@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/processor.h | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 7579e692bc2b..ffe95ce1ad49 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -1180,7 +1180,6 @@ static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu) void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu, struct kvm_x86_cpu_property property, u32 value); -void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, u8 maxphyaddr); void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function); From 3069f4036dff6a36936c9f43f38c408db58bc846 Mon Sep 17 00:00:00 2001 From: Gokul K Date: Fri, 18 Sep 2026 11:14:20 +0530 Subject: [PATCH 324/506] KVM: selftests: Move test_read() into elf.c and delete lib/io.c lib/io.c has outlived its purpose. test_read() has exactly one caller, lib/elf.c; test_write() has none at all; and test_seq_read() is a declaration in test_util.h with no definition anywhere in the tree. Both surviving helpers open with TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count); which can never fire, because @count is a size_t. Building with -Wextra points this out: lib/io.c:51:27: warning: comparison of unsigned expression in '>= 0' is always true [-Wtype-limits] lib/io.c:128:27: warning: comparison of unsigned expression in '>= 0' is always true [-Wtype-limits] The comment above each assertion is worse than the assertion itself: it cites the read(2) and write(2) manpages to claim that a count of zero is allowed, but a zero count lands in "case 0" and fails the test outright. The helper has never accepted what the comment promises. Move test_read() into elf.c as a static elf_read(), named for what it does rather than reading as "test the read() syscall", and assert that @count is non-zero, which is what the code has always required and what every caller already passes. Drop the commentary describing retry behaviour that the code makes obvious. Nothing is then left in lib/io.c, so remove the file along with test_write() and the stale test_seq_read() declaration. No functional change intended. Suggested-by: Sean Christopherson Link: https://lore.kernel.org/all/aqxNbR9u5F1sqB3p@google.com Signed-off-by: Gokul K Link: https://patch.msgid.link/20260918054420.338842-1-gokul02k@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 1 - .../testing/selftests/kvm/include/test_util.h | 4 - tools/testing/selftests/kvm/lib/elf.c | 48 +++++- tools/testing/selftests/kvm/lib/io.c | 157 ------------------ 4 files changed, 43 insertions(+), 167 deletions(-) delete mode 100644 tools/testing/selftests/kvm/lib/io.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 96bab7002d39..d12c1c738956 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -6,7 +6,6 @@ all: LIBKVM += lib/assert.c LIBKVM += lib/elf.c LIBKVM += lib/guest_modes.c -LIBKVM += lib/io.c LIBKVM += lib/kvm_util.c LIBKVM += lib/lru_gen_util.c LIBKVM += lib/memstress.c diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a6a3e1657895..e558346c3b69 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -49,10 +49,6 @@ do { \ #define TEST_REQUIRE(f) __TEST_REQUIRE(f, "Requirement not met: %s", #f) -ssize_t test_write(int fd, const void *buf, size_t count); -ssize_t test_read(int fd, void *buf, size_t count); -int test_seq_read(const char *path, char **bufp, size_t *sizep); - void __printf(5, 6) test_assert(bool exp, const char *exp_str, const char *file, unsigned int line, const char *fmt, ...); diff --git a/tools/testing/selftests/kvm/lib/elf.c b/tools/testing/selftests/kvm/lib/elf.c index 1924a9895834..d5adb68fd7c0 100644 --- a/tools/testing/selftests/kvm/lib/elf.c +++ b/tools/testing/selftests/kvm/lib/elf.c @@ -12,6 +12,44 @@ #include "kvm_util.h" +static ssize_t elf_read(int fd, void *buf, size_t count) +{ + ssize_t rc; + ssize_t num_read = 0; + size_t num_left = count; + char *ptr = buf; + + TEST_ASSERT(count, "Count must be non-zero"); + + do { + rc = read(fd, ptr, num_left); + + switch (rc) { + case -1: + TEST_ASSERT(errno == EAGAIN || errno == EINTR, + "Unexpected read failure,\n" + " rc: %zi errno: %i", rc, errno); + break; + + case 0: + TEST_FAIL("Unexpected EOF,\n" + " rc: %zi num_read: %zi num_left: %zu", + rc, num_read, num_left); + break; + + default: + TEST_ASSERT(rc > 0, "Unexpected ret from read,\n" + " rc: %zi errno: %i", rc, errno); + num_read += rc; + num_left -= rc; + ptr += rc; + break; + } + } while (num_read < count); + + return num_read; +} + static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) { off_t offset_rv; @@ -31,7 +69,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) * the real size of the ELF header. */ unsigned char ident[EI_NIDENT]; - test_read(fd, ident, sizeof(ident)); + elf_read(fd, ident, sizeof(ident)); TEST_ASSERT((ident[EI_MAG0] == ELFMAG0) && (ident[EI_MAG1] == ELFMAG1) && (ident[EI_MAG2] == ELFMAG2) && (ident[EI_MAG3] == ELFMAG3), "ELF MAGIC Mismatch,\n" @@ -79,7 +117,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) offset_rv = lseek(fd, 0, SEEK_SET); TEST_ASSERT(offset_rv == 0, "Seek to ELF header failed,\n" " rv: %zi expected: %i", offset_rv, 0); - test_read(fd, hdrp, sizeof(*hdrp)); + elf_read(fd, hdrp, sizeof(*hdrp)); TEST_ASSERT(hdrp->e_phentsize == sizeof(Elf64_Phdr), "Unexpected physical header size,\n" " hdrp->e_phentsize: %x\n" @@ -146,7 +184,7 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) /* Read in the program header. */ Elf64_Phdr phdr; - test_read(fd, &phdr, sizeof(phdr)); + elf_read(fd, &phdr, sizeof(phdr)); /* Skip if this header doesn't describe a loadable segment. */ if (phdr.p_type != PT_LOAD) @@ -186,8 +224,8 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) " expected: 0x%jx", n1, errno, (intmax_t) offset_rv, (intmax_t) phdr.p_offset); - test_read(fd, addr_gva2hva(vm, phdr.p_vaddr), - phdr.p_filesz); + elf_read(fd, addr_gva2hva(vm, phdr.p_vaddr), + phdr.p_filesz); } } close(fd); diff --git a/tools/testing/selftests/kvm/lib/io.c b/tools/testing/selftests/kvm/lib/io.c deleted file mode 100644 index fedb2a741f0b..000000000000 --- a/tools/testing/selftests/kvm/lib/io.c +++ /dev/null @@ -1,157 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * tools/testing/selftests/kvm/lib/io.c - * - * Copyright (C) 2018, Google LLC. - */ - -#include "test_util.h" - -/* Test Write - * - * A wrapper for write(2), that automatically handles the following - * special conditions: - * - * + Interrupted system call (EINTR) - * + Write of less than requested amount - * + Non-block return (EAGAIN) - * - * For each of the above, an additional write is performed to automatically - * continue writing the requested data. - * There are also many cases where write(2) can return an unexpected - * error (e.g. EIO). Such errors cause a TEST_ASSERT failure. - * - * Note, for function signature compatibility with write(2), this function - * returns the number of bytes written, but that value will always be equal - * to the number of requested bytes. All other conditions in this and - * future enhancements to this function either automatically issue another - * write(2) or cause a TEST_ASSERT failure. - * - * Args: - * fd - Opened file descriptor to file to be written. - * count - Number of bytes to write. - * - * Output: - * buf - Starting address of data to be written. - * - * Return: - * On success, number of bytes written. - * On failure, a TEST_ASSERT failure is caused. - */ -ssize_t test_write(int fd, const void *buf, size_t count) -{ - ssize_t rc; - ssize_t num_written = 0; - size_t num_left = count; - const char *ptr = buf; - - /* Note: Count of zero is allowed (see "RETURN VALUE" portion of - * write(2) manpage for details. - */ - TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count); - - do { - rc = write(fd, ptr, num_left); - - switch (rc) { - case -1: - TEST_ASSERT(errno == EAGAIN || errno == EINTR, - "Unexpected write failure,\n" - " rc: %zi errno: %i", rc, errno); - continue; - - case 0: - TEST_FAIL("Unexpected EOF,\n" - " rc: %zi num_written: %zi num_left: %zu", - rc, num_written, num_left); - break; - - default: - TEST_ASSERT(rc >= 0, "Unexpected ret from write,\n" - " rc: %zi errno: %i", rc, errno); - num_written += rc; - num_left -= rc; - ptr += rc; - break; - } - } while (num_written < count); - - return num_written; -} - -/* Test Read - * - * A wrapper for read(2), that automatically handles the following - * special conditions: - * - * + Interrupted system call (EINTR) - * + Read of less than requested amount - * + Non-block return (EAGAIN) - * - * For each of the above, an additional read is performed to automatically - * continue reading the requested data. - * There are also many cases where read(2) can return an unexpected - * error (e.g. EIO). Such errors cause a TEST_ASSERT failure. Note, - * it is expected that the file opened by fd at the current file position - * contains at least the number of requested bytes to be read. A TEST_ASSERT - * failure is produced if an End-Of-File condition occurs, before all the - * data is read. It is the callers responsibility to assure that sufficient - * data exists. - * - * Note, for function signature compatibility with read(2), this function - * returns the number of bytes read, but that value will always be equal - * to the number of requested bytes. All other conditions in this and - * future enhancements to this function either automatically issue another - * read(2) or cause a TEST_ASSERT failure. - * - * Args: - * fd - Opened file descriptor to file to be read. - * count - Number of bytes to read. - * - * Output: - * buf - Starting address of where to write the bytes read. - * - * Return: - * On success, number of bytes read. - * On failure, a TEST_ASSERT failure is caused. - */ -ssize_t test_read(int fd, void *buf, size_t count) -{ - ssize_t rc; - ssize_t num_read = 0; - size_t num_left = count; - char *ptr = buf; - - /* Note: Count of zero is allowed (see "If count is zero" portion of - * read(2) manpage for details. - */ - TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count); - - do { - rc = read(fd, ptr, num_left); - - switch (rc) { - case -1: - TEST_ASSERT(errno == EAGAIN || errno == EINTR, - "Unexpected read failure,\n" - " rc: %zi errno: %i", rc, errno); - break; - - case 0: - TEST_FAIL("Unexpected EOF,\n" - " rc: %zi num_read: %zi num_left: %zu", - rc, num_read, num_left); - break; - - default: - TEST_ASSERT(rc > 0, "Unexpected ret from read,\n" - " rc: %zi errno: %i", rc, errno); - num_read += rc; - num_left -= rc; - ptr += rc; - break; - } - } while (num_read < count); - - return num_read; -} From 31adfe155db60a58b61e5b3d472cf85336ff3daf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20L=C3=B3pez?= Date: Wed, 16 Sep 2026 16:33:11 +0200 Subject: [PATCH 325/506] KVM: x86/mmu: Fix NX page unaccounting for TDP MMU pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit unaccount_nx_huge_page() unconditionally passes KVM_SHADOW_MMU to untrack_possible_nx_huge_page(), which is incorrect, particularly on the path from kvm_recover_nx_huge_pages(), which handles pages from both MMUs. Fix this by passing down the MMU type from the callers. Note that __kvm_mmu_prepare_zap_page() is only used for shadow MMU pages. Fixes: 6777885605e1 ("KVM: x86/mmu: Track possible NX huge pages separately for TDP vs. Shadow MMU") Signed-off-by: Carlos López Link: https://patch.msgid.link/20260916143310.1909993-2-clopez@suse.de Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 4d22e0d3f8a1..021fbc33a2b8 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -864,11 +864,12 @@ void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp, list_del_init(&sp->possible_nx_huge_page_link); } -static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp) +static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp, + enum kvm_mmu_type mmu_type) { sp->nx_huge_page_disallowed = false; - untrack_possible_nx_huge_page(kvm, sp, KVM_SHADOW_MMU); + untrack_possible_nx_huge_page(kvm, sp, mmu_type); } static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, @@ -2829,7 +2830,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, } if (sp->nx_huge_page_disallowed) - unaccount_nx_huge_page(kvm, sp); + unaccount_nx_huge_page(kvm, sp, KVM_SHADOW_MMU); sp->role.invalid = 1; @@ -8050,7 +8051,7 @@ static void kvm_recover_nx_huge_pages(struct kvm *kvm, WARN_ON_ONCE(!sp->nx_huge_page_disallowed); WARN_ON_ONCE(!sp->role.direct); - unaccount_nx_huge_page(kvm, sp); + unaccount_nx_huge_page(kvm, sp, mmu_type); if (is_tdp_mmu) spin_unlock(tdp_mmu_pages_lock); From 19868b67e3e73c34493efdd5b0791ccfdc7bf192 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 29 Sep 2026 12:57:58 +0100 Subject: [PATCH 326/506] KVM: arm64: Finalize guest-wide sysregs prior to per-vCPU sysregs In commit d82d09d5ba4b ("KVM: arm64: Don't skip per-vcpu NV initialisation") the NV register sanitisation was moved earlier in kvm_finalize_sys_regs() so that it runs for each vCPU rather than only once per guest. This means that for the first vCPU it runs prior to vGIC finalization, but the vGIC finalization updates the ID registers which the NV initialization uses so we may end up with a mismatch. For example, HFGRTR_EL2.ICC_IGRPENn_EL1 depends on GICv3 being enabled in ID_AA64PFR0_EL1.GIC so may be mistakenly marked or not marked as RES0. Split the initialization which runs once per guest into a separate function and run that before the per-vCPU initialisation for NV, renaming the per-vCPU function to make it clear that it does per-vCPU setup. Fixes: d82d09d5ba4b ("KVM: arm64: Don't skip per-vcpu NV initialisation") Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Reviewed-by: Lorenzo Stoakes (ARM) Tested-by: Lorenzo Stoakes (ARM) Signed-off-by: Mark Brown Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260929-kvm-arm64-idreg-final-v4-1-75296dde994d@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 2 +- arch/arm64/kvm/sys_regs.c | 40 ++++++++++++++++++++++++++------------- arch/arm64/kvm/sys_regs.h | 2 +- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..4f044280dec0 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -949,7 +949,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; } - ret = kvm_finalize_sys_regs(vcpu); + ret = kvm_vcpu_finalize_sys_regs(vcpu); if (ret) return ret; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 44aae52c473d..57abbce52540 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -5861,24 +5861,15 @@ void kvm_calculate_traps(struct kvm_vcpu *vcpu) } /* - * Perform last adjustments to the ID registers that are implied by the + * Do system register finalization that is shared by the whole guest. This + * includes last adjustments to the ID registers that are implied by the * configuration outside of the ID regs themselves, as well as any * initialisation that directly depend on these ID registers (such as * RES0/RES1 behaviours). This is not the place to configure traps though. - * - * Because this can be called once per CPU, changes must be idempotent. */ -int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) +static int kvm_vm_finalize_sys_regs(struct kvm *kvm) { - struct kvm *kvm = vcpu->kvm; - - guard(mutex)(&kvm->arch.config_lock); - - if (vcpu_has_nv(vcpu)) { - int ret = kvm_init_nv_sysregs(vcpu); - if (ret) - return ret; - } + lockdep_assert_held(&kvm->arch.config_lock); if (kvm_vm_has_ran_once(kvm)) return 0; @@ -5931,6 +5922,29 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) return 0; } +/* + * Because this can be called once per CPU, changes must be idempotent. + */ +int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + int ret; + + guard(mutex)(&kvm->arch.config_lock); + + ret = kvm_vm_finalize_sys_regs(kvm); + if (ret) + return ret; + + if (vcpu_has_nv(vcpu)) { + ret = kvm_init_nv_sysregs(vcpu); + if (ret) + return ret; + } + + return 0; +} + int __init kvm_sys_reg_table_init(void) { const struct sys_reg_desc *gicv3_regs; diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index bd56a45abbf9..a3cccad2766f 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -254,7 +254,7 @@ int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg, bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index); -int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu); +int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu); #define AA32(_x) .aarch32_map = AA32_##_x #define Op0(_x) .Op0 = _x From 2ce80f62403200f0a59f898dd9cd2ccf756a5a1a Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 29 Sep 2026 12:57:59 +0100 Subject: [PATCH 327/506] KVM: arm64: Block ID register changes after we rely on the values In commit c5bac1ef7df6b ("KVM: arm64: Move existing feature disabling over to FGU infrastructure") a check was added to suppress duplicate recalculation of FGUs based on a flag KVM_ARCH_FLAG_FGU_INITIALIZED. This flag is set when we complete kvm_calculate_traps(), which is called from kvm_arch_vcpu_run_pid_change(). There are several points where that function could fail after we have calculated FGUs (eg, due to an invalid timer configuration). If this happens then userspace will still be able to write to the ID registers, writes to which are gated on KVM_ARCH_FLAG_HAS_RAN_ONCE being set. This in turn means that the FGU configuration for a running guest may not match the ID register configuration. This will result in issues based on the hypervisor assuming a consistent configuration, for example it allows the creation of guests which have untrapped access to system registers which are not context switched for the guest. A similar issue exists in kvm_init_nv_sysregs() where once sysreg_masks is allocated the RES0/RES1 masks for registers are fixed based on the ID register values at the time the function ran, and also for copying the implementation ID registers to the hypervisor for pKVM. There is a further issue with vGIC setup, creating a vGIC includes updating the ID registers to reflect the GIC configuration. We refuse to create a vGIC after the first vCPU has run but if a vCPU fails its first run we may already have finalized the ID register values. Avoid these issues by adding a new flag that we set when we finalize the system registers, blocking ID register changes after that has been set even if something fails later on. Do this in kvm_vm_finalize_sys_regs(), this is where we finalize the GIC fields in the ID registers and happens before we do the FGU and RES0/1 setup. A VMM which tries to create an irqchip after failing to run a vCPU will now get -EBUSY rather than a likely misconfigured guest. Userspace is not expected to try to run a guest that fails to start, never mind try to repair the guest configuration after doing so, so this is not expected to have any impact on practical users. There is a preexisting flag KVM_ARCH_FLAG_ID_REGS_INITIALIZED, this was added as part of the series that originally enabled writable ID registers[1]. That is set when the vCPU feature flags are finalized in KVM_ARM_VCPU_INIT when we initiailise the ID registers, we need to be able to write to the ID registers after that point since the features can influence ID registers (eg, ID_AA64ZFR0_EL1). Given this and the fact that the flag was introduced as part of making the ID registers writable it appears to be a deliberate and desired ABI design decision to not use this flag to block writes to the ID registers. Introducing the new flag preserves the existing behaviour. [1] https://lore.kernel.org/r/20230609190054.1542113-7-oliver.upton@linux.dev Fixes: c5bac1ef7df6b ("KVM: arm64: Move existing feature disabling over to FGU infrastructure") Fixes: 888f088070229 ("KVM: arm64: nv: Add sanitising to VNCR-backed sysregs") Fixes: 03e1b89d051f ("KVM: arm64: Copy MIDR_EL1 into hyp VM when it is writable") Fixes: 8a9866ff8600 ("KVM: arm64: Set ID_{AA64PFR0,PFR1}_EL1.GIC when GICv3 is configured") Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Reviewed-by: Lorenzo Stoakes (ARM) Tested-by: Lorenzo Stoakes (ARM) Signed-off-by: Mark Brown Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260929-kvm-arm64-idreg-final-v4-2-75296dde994d@kernel.org [maz: dropped vcpu_id_regs_final() helper] Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 5 +++++ arch/arm64/kvm/sys_regs.c | 17 ++++++++++------- arch/arm64/kvm/vgic/vgic-init.c | 6 ++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..ca68b614d9a8 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -367,6 +367,8 @@ struct kvm_arch { #define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS 10 /* Unhandled SEAs are taken to userspace */ #define KVM_ARCH_FLAG_EXIT_SEA 11 + /* No further ID register changes possible */ +#define KVM_ARCH_FLAG_ID_REGS_FINAL 12 unsigned long flags; /* VM-wide vCPU feature set */ @@ -1149,6 +1151,9 @@ struct kvm_vcpu_arch { #define vcpu_has_ptrauth(vcpu) false #endif +#define kvm_id_regs_final(kvm) \ + test_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &(kvm)->arch.flags) + #define vcpu_on_unsupported_cpu(vcpu) \ vcpu_get_flag(vcpu, ON_UNSUPPORTED_CPU) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 57abbce52540..e1d2897bfb61 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2511,9 +2511,10 @@ static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, /* * Once the VM has started the ID registers are immutable. Reject any - * write that does not match the final register value. + * write that does not match the final register value once we have + * got far enough into first running the VM to use the values. */ - if (kvm_vm_has_ran_once(vcpu->kvm)) { + if (kvm_id_regs_final(vcpu->kvm)) { if (val != read_id_reg(vcpu, rd)) ret = -EBUSY; else @@ -2547,7 +2548,7 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val) lockdep_assert_held(&kvm->arch.config_lock); - if (KVM_BUG_ON(kvm_vm_has_ran_once(kvm) || !p, kvm)) + if (KVM_BUG_ON(kvm_id_regs_final(kvm) || !p, kvm)) return; *p = val; @@ -3243,10 +3244,10 @@ static int set_imp_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, return -EINVAL; /* - * Once the VM has started the ID registers are immutable. Reject the - * write if userspace tries to change it. + * Once we have been far enough into starting the VM the ID registers + * are immutable. Reject the write if userspace tries to change it. */ - if (kvm_vm_has_ran_once(kvm)) + if (kvm_id_regs_final(kvm)) return -EBUSY; /* @@ -5871,7 +5872,7 @@ static int kvm_vm_finalize_sys_regs(struct kvm *kvm) { lockdep_assert_held(&kvm->arch.config_lock); - if (kvm_vm_has_ran_once(kvm)) + if (kvm_id_regs_final(kvm)) return 0; /* @@ -5919,6 +5920,8 @@ static int kvm_vm_finalize_sys_regs(struct kvm *kvm) kvm_vgic_finalize_idregs(kvm); } + set_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &kvm->arch.flags); + return 0; } diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 4012df6002ea..247c211bd68b 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -123,10 +123,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) goto out_unlock; } - kvm_for_each_vcpu(i, vcpu, kvm) { - if (vcpu_has_run_once(vcpu)) - goto out_unlock; - } + if (kvm_id_regs_final(kvm)) + goto out_unlock; ret = 0; if (type == KVM_DEV_TYPE_ARM_VGIC_V2) From 00e309933879cbf181a95a0bd02384715a381cff Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 29 Sep 2026 12:58:00 +0100 Subject: [PATCH 328/506] KVM: arm64: selftests: Check ID regs are immutable after a failed run Add a set_id_regs case covering ID register immutability when a vCPU's first KVM_RUN fails after finalization but before KVM_ARCH_FLAG_HAS_RAN_ONCE is set. The test provokes such a failure with a PMUv3-enabled vCPU whose PMU is left uninitialized, then checks that KVM_SET_ONE_REG on the feature and implementation ID registers, and KVM_CREATE_DEVICE for a vGIC, are all rejected with -EBUSY. Assisted-by: LLM [Fixed for latest tag standard -- broonie] Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260805064740.3013538-1-fuad.tabba@linux.dev Acked-by: Lorenzo Stoakes (ARM) Tested-by: Lorenzo Stoakes (ARM) Signed-off-by: Mark Brown Acked-by: Mark Brown Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20260929-kvm-arm64-idreg-final-v4-3-75296dde994d@kernel.org Signed-off-by: Marc Zyngier --- .../testing/selftests/kvm/arm64/set_id_regs.c | 106 +++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/arm64/set_id_regs.c b/tools/testing/selftests/kvm/arm64/set_id_regs.c index 7429a1055df5..10849d21c0dd 100644 --- a/tools/testing/selftests/kvm/arm64/set_id_regs.c +++ b/tools/testing/selftests/kvm/arm64/set_id_regs.c @@ -13,6 +13,7 @@ #include "kvm_util.h" #include "processor.h" #include "test_util.h" +#include "vgic.h" #include enum ftr_type { @@ -803,6 +804,107 @@ static void test_reset_preserves_id_regs(struct kvm_vcpu *vcpu) ksft_test_result_pass("%s\n", __func__); } +/* + * ID registers must stay immutable even when a vCPU's first KVM_RUN fails + * after finalization but before KVM_ARCH_FLAG_HAS_RAN_ONCE is set. + */ +static void test_idreg_frozen_after_failed_run(void) +{ + static const u32 imp_id_regs[] = { + SYS_MIDR_EL1, + SYS_REVIDR_EL1, + SYS_AIDR_EL1, + }; + struct kvm_vcpu_init init; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int r; + + if (!kvm_has_cap(KVM_CAP_ARM_PMU_V3)) { + ksft_print_msg("PMUv3 unsupported, cannot fail the first run\n"); + ksft_test_result_skip("%s\n", __func__); + return; + } + + /* Skip the default vGIC so the KVM_CREATE_DEVICE gate is reachable. */ + test_disable_default_vgic(); + + vm = vm_create(1); + vm_enable_cap(vm, KVM_CAP_ARM_WRITABLE_IMP_ID_REGS, 0); + kvm_get_default_vcpu_target(vm, &init); + init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + kvm_arch_vm_finalize_vcpus(vm); + + /* + * A PMUv3 vCPU left without PMU init is rejected by + * kvm_arm_pmu_v3_enable(), which runs after sysreg finalization. + */ + r = _vcpu_run(vcpu); + TEST_ASSERT(r < 0 && errno == EINVAL, + "first KVM_RUN should fail post-finalization: r=%d errno=%d", + r, errno); + + /* + * Feature ID registers: use values that would have been accepted before + * finalization, so that a rejection means the registers are final + * rather than the value being invalid. + */ + for (int i = 0; i < ARRAY_SIZE(test_regs); i++) { + const struct reg_ftr_bits *ftr_bits = test_regs[i].ftr_bits; + u64 reg = KVM_ARM64_SYS_REG(test_regs[i].reg); + u64 val = vcpu_get_reg(vcpu, reg); + + for (int j = 0; ftr_bits[j].type != FTR_END; j++) { + u64 ftr = (val & ftr_bits[j].mask) >> ftr_bits[j].shift; + u64 safe = get_safe_value(&ftr_bits[j], ftr); + u64 new_val; + + if (safe == ftr) + continue; + + new_val = (val & ~ftr_bits[j].mask) | + (safe << ftr_bits[j].shift); + + r = __vcpu_set_reg(vcpu, reg, new_val); + TEST_ASSERT(r < 0 && errno == EBUSY, + "%s write after failed first run: r=%d errno=%d", + ftr_bits[j].name, r, errno); + TEST_ASSERT_EQ(vcpu_get_reg(vcpu, reg), val); + } + + /* A write matching the finalized value is still accepted. */ + vcpu_set_reg(vcpu, reg, val); + } + + /* + * The VM-wide implementation ID registers are gated separately. Bit 0 + * is within the writable mask of all three, so flipping it is a change + * KVM would otherwise accept. + */ + for (int i = 0; i < ARRAY_SIZE(imp_id_regs); i++) { + u64 reg = KVM_ARM64_SYS_REG(imp_id_regs[i]); + u64 val = vcpu_get_reg(vcpu, reg); + + r = __vcpu_set_reg(vcpu, reg, val ^ 1); + TEST_ASSERT(r < 0 && errno == EBUSY, + "implementation ID reg write after failed first run: r=%d errno=%d", + r, errno); + TEST_ASSERT_EQ(vcpu_get_reg(vcpu, reg), val); + } + + /* Creating an in-kernel irqchip would change the ID registers too. */ + if (kvm_supports_vgic_v3()) { + r = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3); + TEST_ASSERT(r < 0 && errno == EBUSY, + "vGIC creation after failed first run: r=%d errno=%d", + r, errno); + } + + kvm_vm_free(vm); + ksft_test_result_pass("%s\n", __func__); +} + int main(void) { struct kvm_vcpu *vcpu; @@ -828,7 +930,7 @@ int main(void) ksft_print_header(); - test_cnt = 3 + MPAM_IDREG_TEST + MTE_IDREG_TEST; + test_cnt = 4 + MPAM_IDREG_TEST + MTE_IDREG_TEST; for (i = 0; i < ARRAY_SIZE(test_regs); i++) for (j = 0; test_regs[i].ftr_bits[j].type != FTR_END; j++) test_cnt++; @@ -847,5 +949,7 @@ int main(void) kvm_vm_free(vm); + test_idreg_frozen_after_failed_run(); + ksft_finished(); } From 3bac3b77c5e6b4403701cab9df43030daf984a2e Mon Sep 17 00:00:00 2001 From: Gokul K Date: Fri, 18 Sep 2026 17:06:31 +0530 Subject: [PATCH 329/506] KVM: selftests: Delete the unused guest_snprintf() Delete guest_snprintf() as it isn't used, has never been used, and in all likelihood will never be used. Because variadic functions can't pass along va_args verbatim, guest_snprintf() can't be used as-is by things like ucall_fmt(). guest_snprintf() could be used to dedup the ucall code by turning it into a macro, but that would still require implicitly consuming the caller's variadic arguments and would risk creating variable shadows, i.e. introduces fragility for very little benefit. No functional change intended. Link: https://lore.kernel.org/all/20260930002238.649421F000FF@smtp.kernel.org Signed-off-by: Gokul K Closes: https://lore.kernel.org/all/20260918113631.634233-4-gokul02k@gmail.com Link: https://patch.msgid.link/20260918113631.634233-4-gokul02k@gmail.com [sean: massage changelog] Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/test_util.h | 1 - tools/testing/selftests/kvm/lib/guest_sprintf.c | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index e558346c3b69..610d910e64af 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -234,7 +234,6 @@ static inline u32 atoi_non_negative(const char *name, const char *num_str) } int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args); -__printf(3, 4) int guest_snprintf(char *buf, int n, const char *fmt, ...); char *strdup_printf(const char *fmt, ...) __attribute__((format(printf, 1, 2), nonnull(1))); diff --git a/tools/testing/selftests/kvm/lib/guest_sprintf.c b/tools/testing/selftests/kvm/lib/guest_sprintf.c index 7a33965349a7..6caebb456f3d 100644 --- a/tools/testing/selftests/kvm/lib/guest_sprintf.c +++ b/tools/testing/selftests/kvm/lib/guest_sprintf.c @@ -300,15 +300,3 @@ int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args) *str = '\0'; return str - buf; } - -int guest_snprintf(char *buf, int n, const char *fmt, ...) -{ - va_list va; - int len; - - va_start(va, fmt); - len = guest_vsnprintf(buf, n, fmt, va); - va_end(va); - - return len; -} From 719a1d58bf43edd03b271df90b67c58c1e9a7da4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:23 -0700 Subject: [PATCH 330/506] KVM: x86: Preserve DR6.BLD (Bus Lock Detect) when delivering #DB payload Preserve Bus Lock Detect if it isn't present in the payload. Unlike RTM, which is explicitly modified on every #DB, DR6.BLD is modified if and only if the #DB is due to a (coincident) Bus Lock Detect. Per Intel's SDM: Other debug exceptions do not modify this bit. And AMD's APM All other #DB exceptions leave DR6[BLD] unmodified, Restore the explicit RTM handling that was clobbered by commit 9a3ecd5e2aa1 ("KVM: X86: Rename DR6_INIT to DR6_ACTIVE_LOW"), as the semantics of any given bit are feature specific, i.e. are obviously not consistent for active-high versus active-low. In other words, while the bug was technically introduced when support for Bus Lock Detect was added, it's really that attempt to genericize handling of active-low bits that set KVM up to fail. Fixes: e8ea85fb280e ("KVM: X86: Add support for the emulation of DR6_BUS_LOCK bit") Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260930193634.3415089-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 52 ++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index de77b222870e..997a6b79c03c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -399,30 +399,37 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, switch (ex->vector) { case DB_VECTOR: /* - * "Certain debug exceptions may clear bit 0-3. The - * remaining contents of the DR6 register are never - * cleared by the processor". + * DR6 is a mess. Reserved/unused bits are fixed-to-1, and so + * to maintain backwards compatibility with existing software, + * features that use previously-reserved bits have active-low + * semantics, i.e. clear the bit when the feature is present in + * the payload. + * + * Further complicating matters, some DR6 bits are preserved by + * hardware, while others are explicitly modified on every #DB. + * The trap bits are always set based on the payload, as is the + * RTM flag (but it's active low). All other bits are modified + * if and only if a relevant debug exception occurs, e.g. BD, + * BS, and BT are never cleared by hardware, and BLD is never + * set by hardware (when supported, excepting RESET). + * + * Lastly, the payload does NOT have active-low semantics, e.g. + * so that it's compatible VMX's pending debug exceptions and + * qualification fields, and to avoid bleeding the DR6 madness + * into other KVM code. + * + * To compute DR6: + * + * 1. "Reset" the bits that are modified on all #DBs + * 2. Clear active-low bits that are present in the payload. + * 3. Set active-high bits that are present in the payload. + * 4. Clear fixed-0 bits. + * 5. Set fixed-1 bits. */ vcpu->arch.dr6 &= ~DR_TRAP_BITS; - /* - * In order to reflect the #DB exception payload in guest - * dr6, three components need to be considered: active low - * bit, FIXED_1 bits and active high bits (e.g. DR6_BD, - * DR6_BS and DR6_BT) - * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits. - * In the target guest dr6: - * FIXED_1 bits should always be set. - * Active low bits should be cleared if 1-setting in payload. - * Active high bits should be set if 1-setting in payload. - * - * Note, the payload is compatible with the pending debug - * exceptions/exit qualification under VMX, that active_low bits - * are active high in payload. - * So they need to be flipped for DR6. - */ - vcpu->arch.dr6 |= DR6_ACTIVE_LOW; - vcpu->arch.dr6 |= ex->payload; - vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW; + vcpu->arch.dr6 |= DR6_RTM; + vcpu->arch.dr6 &= ~(ex->payload & DR6_ACTIVE_LOW); + vcpu->arch.dr6 |= (ex->payload & ~DR6_ACTIVE_LOW); /* * The #DB payload is defined as compatible with the 'pending @@ -431,6 +438,7 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, * breakpoint), it is reserved and must be zero in DR6. */ vcpu->arch.dr6 &= ~BIT(12); + vcpu->arch.dr6 |= DR6_FIXED_1; break; case PF_VECTOR: vcpu->arch.cr2 = ex->payload; From cc551e8d4f77c278fd0bb0ed45d1103cb5a90889 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:24 -0700 Subject: [PATCH 331/506] KVM: x86: Rename kvm_dr6_fixed() => kvm_get_dr6_fixed_1() Rename kvm_dr6_fixed() to kvm_get_dr6_fixed_1() to make it more clear that the helper retrieves the fixed-1 DR6 bits for a given vCPU. No functional change intended. Link: https://patch.msgid.link/20260930193634.3415089-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index fad31b59c622..62075443f6b9 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -772,7 +772,7 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) +static u64 kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { u64 fixed = DR6_FIXED_1; @@ -798,7 +798,7 @@ int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) case 6: if (!kvm_dr6_valid(val)) return 1; /* #GP */ - vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); + vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_get_dr6_fixed_1(vcpu); break; case 5: default: /* 7 */ From 28bb033664eef3c18c6acb2c075fc01d7ada9296 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:25 -0700 Subject: [PATCH 332/506] KVM: Return an "unsigned long", not "u64" for the fixed-1 DR6 bits Return an "unsigned long" instead of a "u64" from the helper to get the fixed-1 DR6 bits, as debug registers are natural width registers, not 64-bit registers. No functional change intended (the fixed-1 bits only set bits in 31:0, and the sole caller truncates the value to an "unsigned long"). Link: https://patch.msgid.link/20260930193634.3415089-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 62075443f6b9..f74a29621661 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -772,15 +772,16 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static u64 kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) +static unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { - u64 fixed = DR6_FIXED_1; + unsigned long fixed = DR6_FIXED_1; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) fixed |= DR6_RTM; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) fixed |= DR6_BUS_LOCK; + return fixed; } From 7b88577a0983af221a78fc66c56532eb97d8fd2f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:26 -0700 Subject: [PATCH 333/506] KVM: x86: Force fixed-1 bits in DR6 after synchronizing with hardware Set all fixed-1 bits in KVM's version of DR6 after synchronizing with hardware to paper over as much of the virtualization hole as possible. Because KVM dynamically disables DR interception, a guest can write any DR6 bit that isn't fixed in bare metal, even if the bit is supposed to be fixed from the guest's perspective. In addition to providing some amount of consistency in KVM, this will allow adding sanity checks that KVM never ends up with fixed-1 bits clear in its version of DR6. Link: https://patch.msgid.link/20260930193634.3415089-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 2 +- arch/x86/kvm/regs.h | 1 + arch/x86/kvm/x86.c | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index f74a29621661..db43ade8ceb1 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -772,7 +772,7 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { unsigned long fixed = DR6_FIXED_1; diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 447f0ec3e63e..3ca5cd8a8d95 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -62,6 +62,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr); +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 997a6b79c03c..c2e24f4d2a1e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8461,6 +8461,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) WARN_ON(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH); kvm_x86_call(sync_dirty_debug_regs)(vcpu); kvm_update_dr0123(vcpu); + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); kvm_update_dr7(vcpu); } From c62ffee78db8837a6251af83798907af37b53ed4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:27 -0700 Subject: [PATCH 334/506] KVM: x86: Set guest's fixed-1 DR6 bits when delivering #DB payload When delivering a #DB payload, set all DR6 bits that are fixed-1 from the guest's perspective, not just the host's perspective. Don't WARN on a "bad" payload (or existing DR6) even though KVM itself shouldn't set DR6 or synthesize a payload without the fixed-1 bits set, as userspace can provide a completely arbitrary payload via KVM_SET_VCPU_EVENTS (and "hardening" that flow is not worth the effort as there's no benefit in practice). Link: https://patch.msgid.link/20260930193634.3415089-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c2e24f4d2a1e..1e24ae7b8879 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -438,7 +438,7 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, * breakpoint), it is reserved and must be zero in DR6. */ vcpu->arch.dr6 &= ~BIT(12); - vcpu->arch.dr6 |= DR6_FIXED_1; + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); break; case PF_VECTOR: vcpu->arch.cr2 = ex->payload; From 0e2342b023801746ece4320d1ad0714f48ea6b1e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:28 -0700 Subject: [PATCH 335/506] KVM: x86: Kill off DR6_FIXED_1 to prevent future misuse Now that the only user of DR6_FIXED_1 is kvm_get_dr6_fixed_1(), drop the hardcoded macro to force all references to fixed-1 bits to use the vCPU- aware helper. Opportunistically reword the comment about DR6_ACTIVE_LOW to explain why treating fixed-1 bits as active-low is correct (barring a *very* egregious architectural goof). No functional change intended. Link: https://patch.msgid.link/20260930193634.3415089-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 2 +- arch/x86/kvm/regs.h | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index db43ade8ceb1..0c76cf17e884 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -774,7 +774,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { - unsigned long fixed = DR6_FIXED_1; + unsigned long fixed = DR6_ACTIVE_LOW & ~DR6_VOLATILE; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) fixed |= DR6_RTM; diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 3ca5cd8a8d95..97c17b120a57 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -38,16 +38,14 @@ static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS)); #define DR6_BT (1 << 15) #define DR6_RTM (1 << 16) /* - * DR6_ACTIVE_LOW combines fixed-1 and active-low bits. - * We can regard all the bits in DR6_FIXED_1 as active_low bits; - * they will never be 0 for now, but when they are defined - * in the future it will require no code change. + * DR6_ACTIVE_LOW combines fixed-1 and active-low bits (bits that are currently + * fixed-1 are guaranteed to have active-low semantics if they are ever used to + * support a new feature). * - * DR6_ACTIVE_LOW is also used as the init/reset value for DR6. + * Note, DR6_ACTIVE_LOW is also the INIT/RESET value for DR6. */ #define DR6_ACTIVE_LOW 0xffff0ff0 #define DR6_VOLATILE 0x0001e80f -#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE) #define DR7_BP_EN_MASK 0x000000ff #define DR7_GE (1 << 9) From 7faa73133e40c59d15abd7201145b75834b067c6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:29 -0700 Subject: [PATCH 336/506] KVM: SVM: Add helper to query if LBR virtualization needs to be enabled Extract the logic for determining whether or not LBR virtualization needs to be enabled into a helper in anticipation of growing the list of features that depend on LBR virtualization. No functional change intended. Link: https://patch.msgid.link/20260930193634.3415089-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 842c16483f6c..2bb1c6576ded 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -884,13 +884,22 @@ static void __svm_disable_lbrv(struct kvm_vcpu *vcpu) to_svm(vcpu)->vmcb->control.misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; } +static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) + return true; + + return is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && + (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); +} + void svm_update_lbrv(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool current_enable_lbrv = svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; - bool enable_lbrv = (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) || - (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)); + bool enable_lbrv = svm_need_lbr_virtualization(vcpu); if (enable_lbrv && !current_enable_lbrv) __svm_enable_lbrv(vcpu); From 5cb0577b74971d578ab105f3b6a8f43fce1f3968 Mon Sep 17 00:00:00 2001 From: Shivansh Dhiman Date: Wed, 30 Sep 2026 12:36:30 -0700 Subject: [PATCH 337/506] KVM: nSVM: Disable LBRV in nested control cache when unsupported Clear SVM_MISC2_ENABLE_V_LBR in __nested_copy_vmcb_control_to_cache() when the vCPU does not support LBR Virtualization. This lets the cached value be consumed directly instead of re-checking X86_FEATURE_LBRV on every access. Suggested-by: Yosry Ahmed Signed-off-by: Shivansh Dhiman Link: https://patch.msgid.link/20260930193634.3415089-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 8 +++++--- arch/x86/kvm/svm/svm.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index ddbded0dee87..64464bdbe644 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -527,12 +527,16 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, /* Always clear misc_ctl bits that the guest cannot use */ to->misc_ctl = from->misc_ctl; + to->misc_ctl2 = from->misc_ctl2; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT)) to->misc_ctl &= ~SVM_MISC_ENABLE_NP; if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET)) to->misc_ctl &= ~SVM_MISC_ENABLE_GMET; + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV)) + to->misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; + to->iopm_base_pa = from->iopm_base_pa & PAGE_MASK; to->msrpm_base_pa = from->msrpm_base_pa & PAGE_MASK; to->tsc_offset = from->tsc_offset; @@ -550,7 +554,6 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, to->event_inj_err = from->event_inj_err; to->next_rip = from->next_rip; to->nested_cr3 = from->nested_cr3; - to->misc_ctl2 = from->misc_ctl2; to->pause_filter_count = from->pause_filter_count; to->pause_filter_thresh = from->pause_filter_thresh; @@ -737,8 +740,7 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) { - return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); + return to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; } static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 2bb1c6576ded..cbd4b0806d23 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -891,7 +891,7 @@ static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) if (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) return true; - return is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && + return is_guest_mode(vcpu) && (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); } From cab4449bddbdcd8bf121ebe67ddb44fc61ca1b8e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 12:36:31 -0700 Subject: [PATCH 338/506] KVM: nSVM: Open code check on LBR virtualization being enabled in vmcb12 Now that KVM clears ENABLE_V_LBR if LBR virtualization isn't supported when caching vmcb12 control fields, i.e. now that the logic is a simple one-liner, open the check and drop the now-superfluous wrapper. No functional change intended. Link: https://patch.msgid.link/20260930193634.3415089-10-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 64464bdbe644..3c1c3e9698bb 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -738,11 +738,6 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, return 0; } -static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) -{ - return to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; -} - static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) { struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; @@ -817,7 +812,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) vmcb_mark_dirty(vmcb02, VMCB_DR); } - if (nested_vmcb12_has_lbrv(vcpu)) { + if (control->misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) { /* * Reserved bits of DEBUGCTL are ignored. Be consistent with * svm_set_msr's definition of reserved bits. @@ -1310,7 +1305,7 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) vmcb12->control.next_rip = vmcb02->control.next_rip; - if (nested_vmcb12_has_lbrv(vcpu)) + if (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) svm_copy_lbrs(&vmcb12->save, &vmcb02->save); vmcb12->control.event_inj = 0; @@ -1387,7 +1382,7 @@ void nested_svm_vmexit(struct vcpu_svm *svm) if (!nested_exit_on_intr(svm)) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - if (!nested_vmcb12_has_lbrv(vcpu)) { + if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)) { svm_copy_lbrs(&vmcb01->save, &vmcb02->save); vmcb_mark_dirty(vmcb01, VMCB_LBR); } From 39080c3ca8b1edefdca7a4a189696427bbd8f6f7 Mon Sep 17 00:00:00 2001 From: Shivansh Dhiman Date: Wed, 30 Sep 2026 12:36:32 -0700 Subject: [PATCH 339/506] KVM: nSVM: Don't assume all active-low bits DR6 are fixed-1 When preparing vmcb02 for nested VMRUN, force only the actual fixed-1 bits instead of setting all active-low bits. The flaw is currently benign, as the only active-low bits supported by KVM are RTM (Restricted Transactional Memory) and BLD (Bus Lock Detect), neither of which is currently supported on SVM, but that's about to change. Signed-off-by: Shivansh Dhiman [sean: massage changelog] Link: https://patch.msgid.link/20260930193634.3415089-11-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 1 + arch/x86/kvm/svm/nested.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 0c76cf17e884..3621f1a47d37 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -784,6 +784,7 @@ unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) return fixed; } +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr6_fixed_1); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) { diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3c1c3e9698bb..eb1ea172f2bb 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -808,7 +808,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) { vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; - svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; + svm->vcpu.arch.dr6 = svm->nested.save.dr6 | kvm_get_dr6_fixed_1(vcpu); vmcb_mark_dirty(vmcb02, VMCB_DR); } From cd0e6f4d7cec11be97fe6476309719b23ab5accb Mon Sep 17 00:00:00 2001 From: Shivansh Dhiman Date: Wed, 30 Sep 2026 12:36:33 -0700 Subject: [PATCH 340/506] KVM: SVM: Add a vCPU-aware helper to get supported DEBUGCTL bits Replace DEBUGCTL_RESERVED_BITS with a vCPU-aware helper that provides the set of supported bits so that KVM can handle bits that are conditionally supported based on the vCPU model, e.g. for features like Bus Lock Detect. No functional change intended. Signed-off-by: Shivansh Dhiman Reviewed-by: Nikunj A Dadhania [sean: invert from reserved => supported, massage changelog] Link: https://patch.msgid.link/20260930193634.3415089-12-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 8 ++++---- arch/x86/kvm/svm/svm.c | 6 +++--- arch/x86/kvm/svm/svm.h | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index eb1ea172f2bb..3f547c01bd20 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -818,7 +818,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) * svm_set_msr's definition of reserved bits. */ svm_copy_lbrs(&vmcb02->save, save); - vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; + vmcb02->save.dbgctl &= svm_get_supported_debugctl(vcpu); } else { svm_copy_lbrs(&vmcb02->save, &vmcb01->save); } @@ -1207,7 +1207,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) } /* Copy state save area fields which are handled by VMRUN */ -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save) { to_save->es = from_save->es; @@ -1234,7 +1234,7 @@ void svm_copy_vmrun_state(struct vmcb_save_area *to_save, if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) { svm_copy_lbrs(to_save, from_save); - to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS; + to_save->dbgctl &= svm_get_supported_debugctl(vcpu); } } @@ -2080,7 +2080,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, save); nested_copy_vmcb_control_to_cache(svm, ctl); svm_switch_vmcb(svm, &svm->nested.vmcb02); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index cbd4b0806d23..88349a9862c2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3216,7 +3216,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) data &= ~DEBUGCTLMSR_BTF; } - if (data & DEBUGCTL_RESERVED_BITS) + if (data & ~svm_get_supported_debugctl(vcpu)) return 1; if (svm->vmcb->save.dbgctl == data) @@ -5152,7 +5152,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); - svm_copy_vmrun_state(m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); + svm_copy_vmrun_state(vcpu, m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); return 0; } @@ -5192,7 +5192,7 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) * used during SMM (see svm_enter_smm()) */ - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, m_save.map.hva + 0x400); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, m_save.map.hva + 0x400); /* * Enter the nested guest now diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index f566dae44bb0..61f9c5e4cee2 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -786,7 +786,10 @@ BUILD_SVM_MSR_BITMAP_HELPERS(bool, test, test) BUILD_SVM_MSR_BITMAP_HELPERS(void, clear, __clear) BUILD_SVM_MSR_BITMAP_HELPERS(void, set, __set) -#define DEBUGCTL_RESERVED_BITS (~DEBUGCTLMSR_LBR) +static inline u64 svm_get_supported_debugctl(struct kvm_vcpu *vcpu) +{ + return DEBUGCTLMSR_LBR; +} /* svm.c */ extern bool dump_invalid_vmcb; @@ -876,7 +879,7 @@ void svm_leave_nested(struct kvm_vcpu *vcpu); void svm_free_nested(struct vcpu_svm *svm); int svm_allocate_nested(struct vcpu_svm *svm); int nested_svm_vmrun(struct kvm_vcpu *vcpu); -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save); void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb); void nested_svm_vmexit(struct vcpu_svm *svm); From 62ad461079e72c7bc16fcc535aa72a49ad89a54f Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Wed, 30 Sep 2026 12:36:34 -0700 Subject: [PATCH 341/506] KVM: SVM: Add support for virtualizating Bus Lock Detect Advertise support for virtualizing Bus Lock Detect and allow the guest to enable BUS_LOCK_DETECT if LBR Virtualization is enabled (Bus Lock Detect is enabled through MSR_IA32_DEBUGCTLMSR, which context-switched if and only if LBR Virtualization is enabled). Signed-off-by: Ravi Bangoria Reviewed-by: Tom Lendacky Co-developed-by: Shivansh Dhiman Signed-off-by: Shivansh Dhiman [sean: massage changelog] Link: https://patch.msgid.link/20260930193634.3415089-13-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 12 ++++++++++-- arch/x86/kvm/svm/svm.h | 7 ++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 88349a9862c2..3331fb4d9cd4 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -888,7 +888,7 @@ static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - if (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) + if (svm->vmcb->save.dbgctl & (DEBUGCTLMSR_LBR | DEBUGCTLMSR_BUS_LOCK_DETECT)) return true; return is_guest_mode(vcpu) && @@ -5700,9 +5700,17 @@ static __init void svm_set_cpu_caps(void) * Clear capabilities that are automatically configured by common code, * but that require explicit SVM support (that isn't yet implemented). */ - kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM); + /* + * LBR Virtualization must be enabled to support BusLockTrap inside the + * guest, since BusLockTrap is enabled through MSR_IA32_DEBUGCTLMSR and + * MSR_IA32_DEBUGCTLMSR is virtualized only if LBR Virtualization is + * enabled. + */ + if (!lbrv) + kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 61f9c5e4cee2..14094a94f6aa 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -788,7 +788,12 @@ BUILD_SVM_MSR_BITMAP_HELPERS(void, set, __set) static inline u64 svm_get_supported_debugctl(struct kvm_vcpu *vcpu) { - return DEBUGCTLMSR_LBR; + u64 debugctl = DEBUGCTLMSR_LBR; + + if (guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; + + return debugctl; } /* svm.c */ From 192dbad1bc0fb23460beef0c72b3590bfe613f35 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 30 Sep 2026 14:21:10 -0700 Subject: [PATCH 342/506] KVM: x86: Advertise EFER_LMSLE_MBZ when KVM disallows EFER.LMSLE Set EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], in KVM's supported CPUID whenever KVM refuses to set EFER.LMSLE, not just when hardware enumerates the defeature. KVM allows EFER.LMSLE if and only if nested SVM is supported, but passes through hardware's EFER_LMSLE_MBZ as-is, i.e. KVM tells userspace that long mode segment limits are available on an Intel CPU, and on an AMD CPU with kvm_amd.nested=0, and then rejects WRMSR(EFER) with EFER.LMSLE=1. The bit means precisely "EFER.LMSLE must be zero." Note, this is purely an enumeration change. EFER.LMSLE is already gated on nested SVM: kvm_setup_efer_caps() adds EFER_LMSLE to supported_efer_bits only if X86_FEATURE_SVM is supported, and svm_set_cpu_caps() sets X86_FEATURE_SVM if and only if "nested" is true. KVM's handling of EFER is unchanged; only what KVM tells userspace changes. Note #2, KVM now sets the defeature on Intel CPUs as well. That is no different than KVM setting NullSelectorClearsBase, another AMD-defined bit, on CPUs that don't have the errata. And LMSLE's dependency on nested SVM is a historical artifact of commit eec4b140c924 ("KVM: SVM: Allow EFER.LMSLE to be set with nested svm"), not an architectural requirement. Be aware that this changes KVM_GET_SUPPORTED_CPUID on all Intel hosts and on AMD hosts with kvm_amd.nested=0, i.e. userspace that reflects KVM's supported CPUID into the guest will start enumerating EFER_LMSLE_MBZ=1 to newly created guests. That's the correct enumeration, and it takes away only an EFER bit that KVM has never allowed to be set on such hosts, but it *is* guest visible. Deliberately omit a Fixes: tag: the misenumeration is benign in practice, as KVM's WRMSR(EFER) behavior is and always has been correct, and backporting a guest-visible CPUID change isn't worth the risk. Opportunistically key the EFER_LMSLE decision off kvm_cpu_cap_has() instead of boot_cpu_has(), as PASSTHROUGH_F() sets EFER_LMSLE_MBZ based on *raw* CPUID, i.e. "clearcpuid=13:20" would otherwise get KVM to enumerate the defeature and allow EFER.LMSLE=1, which is guaranteed to fail on VMRUN. Document KVM's enumeration of the defeature in api.rst. Assisted-by: LLM Signed-off-by: Jim Mattson Link: https://patch.msgid.link/20260930212114.3500045-2-seanjc@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 11 +++++++++++ arch/x86/kvm/cpuid.c | 4 ++++ arch/x86/kvm/svm/svm.c | 10 ++++++++++ arch/x86/kvm/vmx/vmx.c | 7 +++++++ arch/x86/kvm/x86.c | 9 ++++++++- 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc750c9..3c5bcdb0923f 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -9635,6 +9635,17 @@ On older versions of Linux, CPU[EAX=1]:ECX[24] (TSC_DEADLINE) is not reported by is present and the kernel has enabled in-kernel emulation of the local APIC. On newer versions, ``KVM_GET_SUPPORTED_CPUID`` does report the bit as available. +Long mode segment limits +~~~~~~~~~~~~~~~~~~~~~~~~ + +CPU[EAX=0x80000008]:EBX[20] (EFER_LMSLE_MBZ) is a "defeature" bit: it is set +when the CPU does *not* support long mode segment limits, and so requires +EFER.LMSLE to be zero. KVM reports the bit via ``KVM_GET_SUPPORTED_CPUID`` if +and only if KVM refuses to set EFER.LMSLE, i.e. if the CPU doesn't support long +mode segment limits, or if nested SVM is unsupported. KVM therefore reports +the bit on all Intel hosts, as KVM allows EFER.LMSLE only when nested SVM is +enabled. + CPU topology ~~~~~~~~~~~~ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index cbf1fe4ae01f..65029e16d5d8 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1157,6 +1157,10 @@ void kvm_initialize_cpu_caps(void) F(AMD_STIBP), F(AMD_STIBP_ALWAYS_ON), F(AMD_IBRS_SAME_MODE), + /* + * Vendor code also sets EFER_LMSLE_MBZ if KVM itself + * can't support EFER.LMSLE, e.g. if nested SVM is disabled. + */ PASSTHROUGH_F(EFER_LMSLE_MBZ), F(AMD_PSFD), F(AMD_IBPB_RET), diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 3331fb4d9cd4..b09640629a14 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5676,6 +5676,16 @@ static __init void svm_set_cpu_caps(void) boot_cpu_has(X86_FEATURE_AMD_SSBD)) kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); + /* + * Tell userspace that EFER.LMSLE must be zero if nested SVM is + * disabled, as KVM allows EFER.LMSLE if and only if nested SVM is + * supported (a historical artifact of commit eec4b140c924 ("KVM: SVM: + * Allow EFER.LMSLE to be set with nested svm"), not an architectural + * requirement). + */ + if (!nested) + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + if (enable_pmu) { /* * Enumerate support for PERFCTR_CORE if and only if KVM has diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c9f20144a70a..50f98343fc80 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8114,6 +8114,13 @@ static __init void vmx_set_cpu_caps(void) kvm_cpu_cap_clear(X86_FEATURE_IBT); } + /* + * CPUID 0x80000008. Tell userspace that EFER.LMSLE must be zero; KVM + * never allows EFER.LMSLE to be set on Intel CPUs, as KVM supports long + * mode segment limits only in conjunction with nested SVM. + */ + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1e24ae7b8879..5efd17b360de 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6980,7 +6980,14 @@ static void kvm_setup_efer_caps(void) if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { kvm_caps.supported_efer_bits |= EFER_SVME; - if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) + + /* + * Enumerating EFER_LMSLE_MBZ and allowing EFER.LMSLE=1 + * would be nonsensical. Note, vendor code sets the defeature + * if KVM can't support EFER.LMSLE for any reason, i.e. this + * needs to consult KVM's capabilities, not just raw CPUID. + */ + if (!kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) kvm_caps.supported_efer_bits |= EFER_LMSLE; } } From 7358d13e1f1975bc8ddd2b02df70d2cdf638273a Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 30 Sep 2026 14:21:11 -0700 Subject: [PATCH 343/506] KVM: x86: Honor the guest's EFER_LMSLE_MBZ Reject guest attempts to set EFER.LMSLE if userspace enumerates EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], in guest CPUID, so that userspace can hide long mode segment limits from the guest even on CPUs that support LMSLE, e.g. for migration compatibility between Rome and Milan. Enumerate the defeature as *partially* emulated, i.e. as supported by KVM but never advertised, so that kvm_vcpu_after_set_cpuid() propagates userspace's bit into vcpu->arch.cpu_caps. Checking guest_cpu_cap_has() in __kvm_valid_efer() doesn't suffice on its own, because KVM clears the defeature in kvm_cpu_caps precisely on the CPUs where it needs to be emulated, i.e. cpu_caps would never hold the bit no matter what userspace puts in guest CPUID. Don't advertise the defeature in KVM_GET_SUPPORTED_CPUID, as that would take EFER.LMSLE away from userspace that blindly stuffs KVM's supported CPUID into KVM_SET_CPUID2. MONITOR/MWAIT gets the same treatment. Enforce the defeature only for guest-initiated writes, as it's a guest CPUID consistency check, not a host capability. See commit 11988499e62b ("KVM: x86: Skip EFER vs. guest CPUID checks for host-initiated writes"). Note, KVM_SET_SREGS does reject EFER.LMSLE, as it runs the full set of guest CPUID checks, as does nested VMRUN, i.e. L1 can't sneak EFER.LMSLE into L2 via vmcb12. Mask EFER_LMSLE out of the value consumed by efer_trap(), which handles SVM_EXIT_EFER_WRITE_TRAP. EFER writes are *trapped*, not intercepted, i.e. hardware has already committed the write by the time KVM gains control, and the trap is enabled if and only if the guest is SEV-ES, whose EFER lives in the encrypted VMSA and so can't be fixed up by KVM. Rejecting the write would inject a #GP *and* leave EFER.LMSLE set in the guest, which is strictly worse than honoring a write that hardware itself allowed. EFER_SVME is already masked out for a similar "KVM can't enforce this here" reason. Opportunistically fix the whitespace damage around __kvm_valid_efer(). Document the resulting two-level contract in api.rst, i.e. that userspace may set the defeature even when KVM doesn't enumerate it. Suggested-by: Sean Christopherson Assisted-by: LLM Signed-off-by: Jim Mattson Link: https://patch.msgid.link/20260930212114.3500045-3-seanjc@google.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/api.rst | 14 ++++++++++++++ arch/x86/kvm/cpuid.c | 16 ++++++++++++++++ arch/x86/kvm/msrs.c | 12 +++++++++++- arch/x86/kvm/svm/svm.c | 11 ++++++++++- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 3c5bcdb0923f..2852bf94828b 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -9646,6 +9646,20 @@ mode segment limits, or if nested SVM is unsupported. KVM therefore reports the bit on all Intel hosts, as KVM allows EFER.LMSLE only when nested SVM is enabled. +KVM never reports the bit via ``KVM_GET_EMULATED_CPUID``, but userspace may set +it via ``KVM_SET_CPUID2`` even on a host where KVM doesn't report it. KVM +honors the guest's enumeration and rejects EFER.LMSLE=1 accordingly. That lets +userspace defeature a vCPU on a host that *does* support long mode segment +limits, so that the vCPU can later be migrated to a host that doesn't, e.g. so +that a vCPU created on AMD Rome can be migrated to Milan and later, which +dropped support for long mode segment limits. The opposite direction needs no +emulation, as a host that lacks long mode segment limits already enumerates the +defeature. + +Note, ``KVM_SET_MSRS`` is exempt from the check, as host-initiated MSR writes +skip guest CPUID checks so that userspace can set MSRs before it sets guest +CPUID. ``KVM_SET_SREGS`` and nested VMRUN are not exempt. + CPU topology ~~~~~~~~~~~~ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 65029e16d5d8..697d3009816e 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1405,6 +1405,22 @@ static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 ind if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) entry->ecx = feature_bit(RDPID); return 1; + case 0x80000008: + /* + * Honor the guest's EFER_LMSLE_MBZ even if the underlying CPU + * allows setting EFER.LMSLE, e.g. to allow migrating a vCPU + * between hosts with and without EFER.LMSLE support. To avoid + * breaking existing setups that reflect KVM's supported CPUID + * into the guest, KVM doesn't advertise EFER_LMSLE_MBZ unless + * KVM *can't* support EFER.LMSLE=1. + */ + if (include_partially_emulated && + !kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) { + entry->ebx |= feature_bit(EFER_LMSLE_MBZ); + return 1; + } + /* Nothing in 0x80000008 is fully emulated, don't emit an entry. */ + return 0; default: return 0; } diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index dd3bb04878ca..b519fb90776e 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -598,9 +598,19 @@ static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX)) return false; - return true; + /* + * EFER_LMSLE_MBZ is a "defeature" bit, i.e. is set when the CPU does + * *not* support long mode segment limits, and so is the only EFER + * check whose polarity is inverted: EFER.LMSLE is legal if and only if + * the guest does *not* have the defeature. + */ + if (efer & EFER_LMSLE && + guest_cpu_cap_has(vcpu, X86_FEATURE_EFER_LMSLE_MBZ)) + return false; + return true; } + bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) { if (efer & ~kvm_caps.supported_efer_bits) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index b09640629a14..aacc493e1b90 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2800,10 +2800,19 @@ static int efer_trap(struct kvm_vcpu *vcpu) * bit in svm_set_efer(), but __kvm_valid_efer() checks it against * whether the guest has X86_FEATURE_SVM - this avoids a failure if * the guest doesn't have X86_FEATURE_SVM. + * + * Clear EFER_LMSLE for a related reason: EFER writes are *trapped*, + * not intercepted, i.e. hardware has already committed the write by + * the time KVM gains control, and the trap is enabled if and only if + * the guest is SEV-ES, whose EFER lives in the encrypted VMSA and so + * can't be fixed up by KVM. Rejecting EFER.LMSLE=1 would inject a #GP + * *and* leave EFER.LMSLE set in the guest, which is strictly worse + * than honoring a write that hardware itself allowed. */ msr_info.host_initiated = false; msr_info.index = MSR_EFER; - msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; + msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & + ~(EFER_SVME | EFER_LMSLE); ret = kvm_set_msr_common(vcpu, &msr_info); return kvm_complete_insn_gp(vcpu, ret); From cc2b9cc15ab702a3d4ada479f129fd287e40bd7e Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 30 Sep 2026 14:21:12 -0700 Subject: [PATCH 344/506] KVM: selftests: Rename svm_nested_clear_efer_svme to svm_nested_efer_test Rename svm_nested_clear_efer_svme.c to svm_nested_efer_test.c so that the test can host coverage for other EFER bits whose behavior is tied to nested SVM, and so that the name matches the svm_nested__test convention used by svm_nested_pat_test, svm_nested_shutdown_test, etc. No functional change intended. Assisted-by: LLM Signed-off-by: Jim Mattson Link: https://patch.msgid.link/20260930212114.3500045-4-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 2 +- .../{svm_nested_clear_efer_svme.c => svm_nested_efer_test.c} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tools/testing/selftests/kvm/x86/{svm_nested_clear_efer_svme.c => svm_nested_efer_test.c} (100%) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 96bab7002d39..2554464d8b96 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -117,7 +117,7 @@ TEST_GEN_PROGS_x86 += x86/state_test TEST_GEN_PROGS_x86 += x86/vmx_preemption_timer_test TEST_GEN_PROGS_x86 += x86/svm_vmcall_test TEST_GEN_PROGS_x86 += x86/svm_int_ctl_test -TEST_GEN_PROGS_x86 += x86/svm_nested_clear_efer_svme +TEST_GEN_PROGS_x86 += x86/svm_nested_efer_test TEST_GEN_PROGS_x86 += x86/svm_nested_shutdown_test TEST_GEN_PROGS_x86 += x86/svm_nested_soft_inject_test TEST_GEN_PROGS_x86 += x86/svm_nested_vmcb12_gpa diff --git a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c similarity index 100% rename from tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c rename to tools/testing/selftests/kvm/x86/svm_nested_efer_test.c From 58f4dc83da6e5baf8b9a9752e433360c6b17df38 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 14:21:13 -0700 Subject: [PATCH 345/506] KVM: selftests: Add module param API to check if nested virtualization is enabled Add an API to check if the nested virtualization module param is enabled, e.g. so that tests can verify KVM's own enumeration of the corresponding CPU feature. Provide a big warning above the API to make it as obvious as possible clear that tests should use "kvm_cpu_has(X86_FEATURE_VMX) || kvm_cpu_has(X86_FEATURE_SVM)" when checking for nested virtualization. It doesn't really matter which method tests use, as KVM is supposed to keep "nested" synchronized with VMX/SVM enumeration, but make the comment super scary to try and prevent KVM selftests from having two different ways of doing the same thing. Note, the bool vs. int mess is intentional, because KVM's params are terrible. kvm_amd.ko provides an "int", while kvm_intel.ko provides a "bool". Link: https://patch.msgid.link/20260930212114.3500045-5-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/x86/processor.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 6e6f70035508..3d3a7d9f2a95 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -1472,6 +1472,19 @@ static inline bool kvm_is_lbrv_enabled(void) return !!get_kvm_amd_param_integer("lbrv"); } +/* + * Do NOT use this to check for nVMX or nSVM support. Querying kvm_cpu_has() + * for either of X86_FEATURE_{VMX,SVM} is the idiomatic way to check for nested + * virtualization support. Use this *only* to validate KVM's own enumeration. + */ +static inline bool kvm_is_nested_virtualization_enabled(void) +{ + if (host_cpu_is_intel) + return get_kvm_intel_param_bool("nested"); + + return get_kvm_amd_param_integer("nested"); +} + u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva); u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3); From 34f749161dabac48a6aeae36ffd5f55f84023d61 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Wed, 30 Sep 2026 14:21:14 -0700 Subject: [PATCH 346/506] KVM: selftests: Add coverage for the EFER_LMSLE_MBZ defeature Add coverage for KVM's virtualization of EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], which is set when the CPU does *not* support long mode segment limits. Verify that KVM's enumeration of the defeature matches the expectation derived from hardware plus kvm_amd's "nested" module param, so that the assertion doesn't compare KVM's enumeration against itself, and that WRMSR(EFER), nested VMRUN, and KVM_SET_SREGS reject EFER.LMSLE=1 if and only if guest CPUID enumerates the defeature. Also verify that host-initiated KVM_SET_MSRS is exempt from the check, as host-initiated writes skip guest CPUID checks so that userspace can set MSRs before it sets guest CPUID. Extend svm_nested_efer_test rather than add yet another test binary for a single CPUID bit; the test already has the L1/L2 harness needed to exercise EFER.LMSLE. Note, only a CPU that supports LMSLE, i.e. Rome and earlier, exercises the emulation path; elsewhere KVM enumerates the defeature straight from hardware. Opportunistically drop the unnecessary #include of vmx.h. Assisted-by: LLM Signed-off-by: Jim Mattson [sean: drop specific APM volume references, use common "nested" getter] Link: https://patch.msgid.link/20260930212114.3500045-6-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/include/x86/processor.h | 1 + .../selftests/kvm/x86/svm_nested_efer_test.c | 244 +++++++++++++++++- 2 files changed, 231 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 3d3a7d9f2a95..6f110987c51a 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -216,6 +216,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) +#define X86_FEATURE_EFER_LMSLE_MBZ KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 20) #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) diff --git a/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c index 6bc301207cbc..fa0d76a9c390 100644 --- a/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c +++ b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c @@ -1,16 +1,20 @@ // SPDX-License-Identifier: GPL-2.0-only /* + * Tests for KVM's handling of EFER bits whose behavior is tied to nested SVM. + * * Copyright (C) 2026, Google LLC. */ +#include "test_util.h" #include "kvm_util.h" -#include "vmx.h" +#include "processor.h" #include "svm_util.h" #include "kselftest.h" +static bool l2_ran; -static void l2_guest_code(void) +static void l2_clear_efer_svme(void) { - unsigned long efer = rdmsr(MSR_EFER); + u64 efer = rdmsr(MSR_EFER); /* generic_svm_setup() initializes EFER_SVME set for L2 */ GUEST_ASSERT(efer & EFER_SVME); @@ -20,31 +24,243 @@ static void l2_guest_code(void) GUEST_ASSERT(0); } -static void l1_guest_code(struct svm_test_data *svm) +static void l1_clear_efer_svme(struct svm_test_data *svm) { - generic_svm_setup(svm, l2_guest_code); + generic_svm_setup(svm, l2_clear_efer_svme); run_guest(svm->vmcb, svm->vmcb_gpa); /* Unreachable, L1 should be shutdown */ GUEST_ASSERT(0); } -int main(int argc, char *argv[]) +static void l2_lmsle(void) +{ + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + l2_ran = true; + vmmcall(); +} + +static void l1_lmsle(struct svm_test_data *svm) +{ + bool lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + struct vmcb *vmcb = svm->vmcb; + u64 efer = rdmsr(MSR_EFER); + + /* + * Selftests' vCPUs are created with EFER.LMSLE clear; the sub-tests + * below need to start from a clean slate. + */ + GUEST_ASSERT(!(efer & EFER_LMSLE)); + GUEST_ASSERT(!l2_ran); + + /* + * Per the APM, if EFER_LMSLE_MBZ is enumerated in CPUID, "64-bit mode + * segment limit checking is not supported and attempting to set + * EFER.LMSLE = 1 causes a #GP exception". + */ + if (lmsle_mbz) { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), GP_VECTOR); + GUEST_ASSERT(!(rdmsr(MSR_EFER) & EFER_LMSLE)); + } else { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), 0); + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + + /* + * Restore EFER so that generic_svm_setup() doesn't propagate + * EFER.LMSLE into vmcb12 on its own, i.e. so that the VMRUN + * sub-test actually tests what it thinks it's testing. + */ + wrmsr(MSR_EFER, efer); + } + + /* + * VMRUN's consistency checks reject "any MBZ bit of EFER", i.e. a + * vmcb12 with EFER.LMSLE set must generate VMEXIT_INVALID when the + * defeature is enumerated. + */ + generic_svm_setup(svm, l2_lmsle); + vmcb->save.efer |= EFER_LMSLE; + run_guest(vmcb, svm->vmcb_gpa); + + if (lmsle_mbz) { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_ERR); + GUEST_ASSERT(!l2_ran); + } else { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_VMMCALL); + GUEST_ASSERT(l2_ran); + GUEST_ASSERT(vmcb->save.efer & EFER_LMSLE); + } + + GUEST_DONE(); +} + +static struct kvm_vcpu *create_l1_vcpu(struct kvm_vm **vm, void *l1_guest_code) +{ + struct kvm_vcpu *vcpu; + gva_t svm_gva; + + *vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + + vcpu_alloc_svm(*vm, &svm_gva); + vcpu_args_set(vcpu, 1, svm_gva); + + return vcpu; +} + +static void test_enumeration(void) +{ + bool lmsle_mbz; + + /* + * EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], is a "defeature" bit, + * i.e. is set when the CPU does *not* support long mode segment + * limits. KVM enumerates the defeature if and only if KVM refuses to + * set EFER.LMSLE, i.e. if the CPU doesn't support LMSLE, or if KVM + * doesn't support nested SVM. Derive the expectation from raw CPUID + * and kvm_amd's "nested" module param rather than from + * kvm_cpu_has(X86_FEATURE_SVM), so that the assertion doesn't simply + * compare KVM's enumeration to itself. + */ + lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ) || + !this_cpu_has(X86_FEATURE_SVM) || + !kvm_is_nested_virtualization_enabled(); + + TEST_ASSERT_EQ(kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ), lmsle_mbz); + + ksft_test_result_pass("KVM enumerates EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_clear_efer_svme(void) { struct kvm_vcpu *vcpu; struct kvm_vm *vm; - gva_t nested_gva = 0; - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM)); - - vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); - - vcpu_alloc_svm(vm, &nested_gva); - vcpu_args_set(vcpu, 1, nested_gva); + vcpu = create_l1_vcpu(&vm, l1_clear_efer_svme); vcpu_run(vcpu); TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_SHUTDOWN); kvm_vm_free(vm); - return 0; + ksft_test_result_pass("L2 clearing EFER.SVME shuts down L1\n"); +} + +static void test_lmsle(bool lmsle_mbz) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + + vcpu = create_l1_vcpu(&vm, l1_lmsle); + + vcpu_set_or_clear_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ, + lmsle_mbz); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_DONE: + break; + default: + TEST_FAIL("Unexpected ucall: %lu", uc.cmd); + } + + kvm_vm_free(vm); + ksft_test_result_pass("Guest EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_host_initiated_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + u64 efer; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * EFER_LMSLE_MBZ is a guest CPUID consistency check, not a host + * capability, i.e. must not be enforced against host-initiated writes, + * so that userspace can set MSRs before it sets guest CPUID. + */ + efer = vcpu_get_msr(vcpu, MSR_EFER); + TEST_ASSERT(!(efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + vcpu_set_msr(vcpu, MSR_EFER, efer | EFER_LMSLE); + TEST_ASSERT_EQ(vcpu_get_msr(vcpu, MSR_EFER), efer | EFER_LMSLE); + + kvm_vm_free(vm); + ksft_test_result_pass("Host-initiated EFER.LMSLE=1 is allowed\n"); +} + +static void test_sregs_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_sregs sregs; + struct kvm_vm *vm; + int rc; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * Unlike KVM_SET_MSRS, KVM_SET_SREGS runs the full set of guest CPUID + * checks, i.e. rejects EFER.LMSLE even though it's host-initiated. + */ + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT(!(sregs.efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + sregs.efer |= EFER_LMSLE; + rc = _vcpu_sregs_set(vcpu, &sregs); + TEST_ASSERT(rc, "KVM allowed EFER.LMSLE with EFER_LMSLE_MBZ set"); + + kvm_vm_free(vm); + ksft_test_result_pass("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); +} + +int main(int argc, char *argv[]) +{ + bool has_nested_svm, has_lmsle; + + ksft_print_header(); + ksft_set_plan(6); + + test_enumeration(); + + /* + * The sub-tests below need to actually run a nested guest, and the + * EFER.LMSLE sub-tests additionally need KVM to allow EFER.LMSLE. + * It's KVM's view of the world, not raw CPUID, that dictates whether + * EFER.LMSLE is allowed, i.e. whether the defeature is emulated. + */ + has_nested_svm = kvm_cpu_has(X86_FEATURE_SVM); + has_lmsle = has_nested_svm && + !kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + + if (!has_nested_svm) + ksft_print_msg("Nested SVM unsupported\n"); + else if (!has_lmsle) + ksft_print_msg("KVM doesn't support EFER.LMSLE\n"); + + if (has_nested_svm) { + test_clear_efer_svme(); + test_lmsle(true); + } else { + ksft_test_result_skip("L2 clearing EFER.SVME shuts down L1\n"); + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=1\n"); + } + + if (has_lmsle) { + test_lmsle(false); + test_host_initiated_lmsle(); + test_sregs_lmsle(); + } else { + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=0\n"); + ksft_test_result_skip("Host-initiated EFER.LMSLE=1 is allowed\n"); + ksft_test_result_skip("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); + } + + ksft_finished(); } From b701a8139fd6d51462a5dcac79c92c3ff6524f6c Mon Sep 17 00:00:00 2001 From: Amirmohammad Eftekhar Date: Wed, 30 Sep 2026 10:36:15 -0700 Subject: [PATCH 347/506] KVM: x86: Saturate L2's TSC frequency if it exceeds hardware supports When computing the TSC multiplier for L2, use the minimum/maximum frequency supported by hardware if the resulting multiplier can't be programmed into hardware, i.e. saturate L2's frequency on both sides. This can happen if userspace is running L1 at a low/high frequency relative to L0, and L1 is doing the same for L2 (relative to L1), because although each of the L1 and L2 inputs are constrained and validated, the end result can exceed SVM's and VMX's architectural limits. Don't bother trying to log an error or do something more sophisticated, because in practice only a misbehaving L1 (and/or host userspace) will run afoul of the flaw. On SVM, which has the smallest "range" by far (8 bits for the integer multiplier, versus 16 bits on VMX), KVM can still run L2 with a frequency 255x that of L0. Even assuming a pessimistic L0 TSC frequency of 1GHz (modern CPUs run TSC at 2GHz+), L2 would need to be running at a whopping ~255GHz for the limitation to be a problem. Not to mention that if L2 is running at a legitimate frequency, then running that VM on existing hardware is already doomed irrespective of the nested angle. Open code the mul_u64_u64_shr() if the compiler natively supports 128-bit integers, mostly to avoid having to do the multiply twice for what should be a very rare scenario, but also so that there's a slightly more scrutable sequence that can be read to understand what all is going on. Signed-off-by: Amirmohammad Eftekhar [sean: optimize for INT128, rewrite comment+changelog to be less AI] Link: https://patch.msgid.link/20260930173635.3362655-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 55 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..b7eb410c423c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1164,11 +1164,58 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_offset); u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) { - if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio) - return mul_u64_u64_shr(l1_multiplier, l2_multiplier, - kvm_caps.tsc_scaling_ratio_frac_bits); + u8 frac_bits = kvm_caps.tsc_scaling_ratio_frac_bits; + u64 nested_multiplier; - return l1_multiplier; + if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio) + return l1_multiplier; + + /* + * The shift is fixed on both AMD and Intel, and operates on a 64-bit + * value. I.e. a shift greater than 63 is completely nonsensical. + */ + if (WARN_ON_ONCE(frac_bits > 63)) + return l1_multiplier; + + /* + * If the resulting multiplier can't be programmed into hardware, run + * L2 at the minimum/maximum frequency supported by hardware, i.e. + * saturate L2's frequency on both sides. Because L2's frequency needs + * to be distilled down to a single multiplier to get from: + * + * L2 = (((L0 * L1_mult) >> frac) * L2_mult) >> frac) + * + * to: + * + * L2 = (L0 * mult) >> frac + * + * very small/large L1 and L2 multipliers can underflow/overflow the + * minimum/maximum multiplier supported by hardware when combined into + * a single value. + * + * Manually check for the case where the result would overflow a u64, + * i.e. if the multiplier would be silently truncated before the "too + * large" check. Avoid doing the multiply twice in the common case + * where the compiler natively supports 128-bit values. + */ +#ifdef CONFIG_ARCH_SUPPORTS_INT128 + unsigned __int128 m = (unsigned __int128)l1_multiplier * l2_multiplier; + + if (m >> (64 + frac_bits)) + return kvm_caps.max_tsc_scaling_ratio; + + nested_multiplier = m >> frac_bits; +#else + if (mul_u64_u64_shr(l1_multiplier, l2_multiplier, 64 + frac_bits)) + return kvm_caps.max_tsc_scaling_ratio; + + nested_multiplier = mul_u64_u64_shr(l1_multiplier, l2_multiplier, frac_bits); +#endif + if (nested_multiplier > kvm_caps.max_tsc_scaling_ratio) + return kvm_caps.max_tsc_scaling_ratio; + + /* The minimum multiplier is '1' on both AMD and Intel. */ + return nested_multiplier ?: 1; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_multiplier); From 967e40fb1b08110e5528d19fc96b4b5410e53109 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:16 -0700 Subject: [PATCH 348/506] KVM: SVM: Fallback to the default TSC ratio if KVM tries to use a bad multiplier Explicitly WARN and fallback to the default TSC ratio, i.e. run the guest at L0's TSC frequency, if KVM tries to program a bad multiplier value. As pointed out by Sashiko, leaving the MSR as-is bleeds state from the vCPU that last ran on the pCPU into the likely-misbehaving current vCPU. Leaking a vCPU's frequency isn't very interesting, and corrupting KVM's cache isn't a big deal either since KVM would only refuse to try to write the same bad value in the future, but falling back to the default value is trivial, and explicitly WARNing ensures a KVM bug won't slip by silently. E.g. because ex_handler_msr() only WARNs once for *all* WRMSRs, it's possible for the potentially-fatal-to-the-guest issue to not exhibit any visible symptoms in the host. Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260722091414.E842B1F000E9@smtp.kernel.org Link: https://patch.msgid.link/20260930173635.3362655-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7d59d301e1e5..26d458e9af54 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -530,6 +530,15 @@ static int svm_check_processor_compat(void) static void __svm_write_tsc_multiplier(u64 multiplier) { + /* + * Fallback to the default ratio if KVM is buggy and tries to program + * an unsupported scaling ratio, e.g. so that the guest has a chance of + * surviving, so that the cache isn't stale/corrupted, and so that KVM + * doesn't leak state across VMs. + */ + if (WARN_ON_ONCE(multiplier & SVM_TSC_RATIO_RSVD)) + multiplier = SVM_TSC_RATIO_DEFAULT; + if (multiplier == __this_cpu_read(current_tsc_ratio)) return; From 95eaeb7f202baddbe81ab832d8523acbb158edfe Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:17 -0700 Subject: [PATCH 349/506] KVM: x86: Allow userspace to set KVM's max supported guest TSC frequency Reject KVM_SET_TSC_KHZ if the incoming frequency is strictly greater than KVM's max supported frequency, not if the frequency is greater than *or* equal to the max frequency. The effective off-by-one bug came about via (dubious) review feedback, which also subtly collided with a functional change in later versions of the original TSC scaling series. In v2 of the original TSC scaling series[1], KVM set its absolute min/max to [1, UINT_MAX], i.e. allowed any value that would fit in the u32 passed to KVM_SET_TSC_KHZ[2]. min = max(1ULL, __scale_tsc(tsc_khz, TSC_RATIO_MIN)); max = min(0xffffffffULL, __scale_tsc(tsc_khz, TSC_RATIO_MAX)); That prompted Avi to suggest rejecting the "equals" case, presumably because the check would always succeed given an absolute max of UINT_MAX? But even that doesn't hold up to scrutiny, as allowing the actual min/max isn't inherently unsafe. Regardless, that feedback was taken and applied to future versions, but only for the maximum, not the minimum. > + > + r = -EINVAL; > + if (user_tsc_khz< kvm_min_guest_tsc_khz || > + user_tsc_khz> kvm_max_guest_tsc_khz) <= and >= are probably safer. v3 of the series[3] then also realized that allowing UINT_MAX would lead to undesirable interactions with KVM_GET_TSC_KHZ, which returns a *signed* 32-bit integer that is implicitly converted into a signed 64-bit value on 64-bit kernels. I.e. allowing a value greater than INT_MAX would result in userspace observing a negative value when doing KVM_GET_TSC_KHZ after KVM_SET_TSC_KHZ. /* * Make sure the user can only configure tsc_khz values that * fit into a signed integer. * A min value is not calculated needed because it will always * be 1 on all machines and a value of 0 is used to disable * tsc-scaling for the vcpu. */ max = min(0x7fffffffULL, __scale_tsc(tsc_khz, TSC_RATIO_MAX)); kvm_max_guest_tsc_khz = max; v3 also dropped the explicit minimum tracking, as both AMD and Intel support a minimum *fractional* ratio of 1, i.e. AMD and Intel support a minimum frequency of "host / 2^32" and "host / 2^48" respectively. And because KVM_GET_TSC_KHZ (and KVM itself) only supports frequencies that fit in a signed 32-bit integer, even AMD's more coarse-grained ratio can scale down any host frequency to '1', i.e. KVM can always support a minimum frequency of 1KHz. Rather than explicitly reject a frequency of 1KHz, v3 simply dropped the minimum check, i.e. ignored the "<=" suggestion, but kept the ">=" side of things. As a result, KVM now has a bizarre uABI where KVM_SET_TSC_KHZ tops out at INT_MAX-1 for no discernible reason. Fix the off-by-one flaw to provide a less weird uABI, and so that KVM_SET_TSC_KHZ accepts the maximum possible value that can be returned by KVM_GET_TSC_KHZ (without running afoul of casting issues; KVM doesn't actually sanity check that tsc_khz fits in a signed 32-bit value, which is a non-issue in practice because the units are KHz, not Hz, i.e. KVM_GET_TSC_KHZ is fine until CPUs with a TSC frequency greater than ~2.147PHz come along). Note, in practice, no real world VMM is likely to care. As above, running afoul of the off-by-one issue would mean trying to configure a virtual TSC frequency that is three orders of magnitude greater than what current CPUs support. Opportunistically explain *why* KVM restricts KVM_SET_TSC_KHZ to values that fit in a signed integer, as it requires far too much spelunking to piece together the connection to KVM_GET_TSC_KHZ. Link: https://lore.kernel.org/all/1300952424-32014-1-git-send-email-joerg.roedel@amd.com [1] Link: https://lore.kernel.org/all/1300952424-32014-7-git-send-email-joerg.roedel@amd.com [2] Link: https://lore.kernel.org/all/1301042691-22929-7-git-send-email-joerg.roedel@amd.com [3] Link: https://patch.msgid.link/20260930173635.3362655-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index b7eb410c423c..baf171804748 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3764,7 +3764,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, user_tsc_khz = (u32)arg; if (kvm_caps.has_tsc_control && - user_tsc_khz >= kvm_caps.max_guest_tsc_khz) + user_tsc_khz > kvm_caps.max_guest_tsc_khz) goto out; if (user_tsc_khz == 0) @@ -4700,7 +4700,7 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) user_tsc_khz = (u32)arg; if (kvm_caps.has_tsc_control && - user_tsc_khz >= kvm_caps.max_guest_tsc_khz) + user_tsc_khz > kvm_caps.max_guest_tsc_khz) goto out; if (user_tsc_khz == 0) @@ -7178,7 +7178,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) if (kvm_caps.has_tsc_control) { /* * Make sure the user can only configure tsc_khz values that - * fit into a signed integer. + * fit into a signed integer, otherwise KVM_GET_TSC_KHZ would + * return a negative value and confuse userspace. * A min value is not calculated because it will always * be 1 on all machines. */ From ed95a2725719f41a7580480543eda9892164fe82 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:18 -0700 Subject: [PATCH 350/506] KVM: selftests: Use KVM's pRNG to randomize L1's TSC ratio in TSC scaling test Use KVM's pRNG to randomize L1's TSC ratio in the nested TSC scaling test, so that reproducing a failure with a specific seed is slightly easier. Link: https://patch.msgid.link/20260930173635.3362655-5-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index c763f4cf0f62..65de46f80a3e 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -166,8 +166,7 @@ int main(int argc, char *argv[]) * referenced by both main() and l1_guest_code() and using a global * variable does not work. */ - srand(time(NULL)); - l1_scale_factor = (rand() % 9) + 2; + l1_scale_factor = (kvm_random_u32(&kvm_rng) % 9) + 2; printf("L1's scale down factor is: %"PRIu64"\n", l1_scale_factor); printf("L2's scale up factor is: %llu\n", L2_SCALE_FACTOR); From b955d8fdabfe13d6ee6a52b1c0048cb9e5931912 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:19 -0700 Subject: [PATCH 351/506] KVM: selftests: Drop redundant VMWRITE of TSC_MULTIPLIER_HIGH Drop a redundant VMWRITE of TSC_MULTIPLIER_HIGH in the nested TSC scaling test. KVM selftests are 64-bit only, i.e. the full 64-bit multiplier is always written by setting TSC_MULTIPLIER. Link: https://patch.msgid.link/20260930173635.3362655-6-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 65de46f80a3e..052ddd61482d 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -122,7 +122,6 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) vmwrite(TSC_OFFSET, TSC_OFFSET_L2); vmwrite(TSC_MULTIPLIER, TSC_MULTIPLIER_L2); - vmwrite(TSC_MULTIPLIER_HIGH, TSC_MULTIPLIER_L2 >> 32); /* launch L2 */ vmlaunch(); From 17919fe61684235f9e66da14933ec09c3081df2a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:20 -0700 Subject: [PATCH 352/506] KVM: selftests: Drop unnecessary use of PRIu64 in nested TSC scaling test Use a plain "lu" instead of "PRIu64" to print u64s in the nested TSC scaling test, as KVM selftests are 64-bit only (and supporting 32-bit x86 selftest would be an absurdly large lift, for something absolutely no one cares about). Link: https://patch.msgid.link/20260930173635.3362655-7-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 052ddd61482d..b8b2217b9efa 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -41,14 +41,9 @@ static void compare_tsc_freq(u64 actual, u64 expected) thresh_low = expected - tolerance; thresh_high = expected + tolerance; - TEST_ASSERT(thresh_low < actual, - "TSC freq is expected to be between %"PRIu64" and %"PRIu64 - " but it actually is %"PRIu64, - thresh_low, thresh_high, actual); - TEST_ASSERT(thresh_high > actual, - "TSC freq is expected to be between %"PRIu64" and %"PRIu64 - " but it actually is %"PRIu64, - thresh_low, thresh_high, actual); + TEST_ASSERT(thresh_low < actual && thresh_high > actual, + "TSC freq is '%lu', expected to be between %lu and %lu", + actual, thresh_low, thresh_high); } static void check_tsc_freq(int level) @@ -166,7 +161,7 @@ int main(int argc, char *argv[]) * variable does not work. */ l1_scale_factor = (kvm_random_u32(&kvm_rng) % 9) + 2; - printf("L1's scale down factor is: %"PRIu64"\n", l1_scale_factor); + printf("L1's scale down factor is: %lu\n", l1_scale_factor); printf("L2's scale up factor is: %llu\n", L2_SCALE_FACTOR); tsc_start = rdtsc(); @@ -174,7 +169,7 @@ int main(int argc, char *argv[]) tsc_end = rdtsc(); l0_tsc_freq = tsc_end - tsc_start; - printf("real TSC frequency is around: %"PRIu64"\n", l0_tsc_freq); + printf("real TSC frequency is around: %lu\n", l0_tsc_freq); vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); @@ -207,16 +202,14 @@ int main(int argc, char *argv[]) break; case UCHECK_L1: l1_tsc_freq = uc.args[1]; - printf("L1's TSC frequency is around: %"PRIu64 - "\n", l1_tsc_freq); + printf("L1's TSC frequency is around: %lu\n", l1_tsc_freq); compare_tsc_freq(l1_tsc_freq, l0_tsc_freq / l1_scale_factor); break; case UCHECK_L2: l2_tsc_freq = uc.args[1]; - printf("L2's TSC frequency is around: %"PRIu64 - "\n", l2_tsc_freq); + printf("L2's TSC frequency is around: %lu\n", l2_tsc_freq); compare_tsc_freq(l2_tsc_freq, l1_tsc_freq * L2_SCALE_FACTOR); From 19d963ed9a4072fba776673619c44fbee45fc9d2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:21 -0700 Subject: [PATCH 353/506] KVM: selftests: Randomize L2's scale factor in nested TSC scaling test Randomize L2's scale factor in the nested TSC scaling test; the test's comment that "using a global variable does not work" just means the author didn't know how to use sync_global_to_guest(). Link: https://patch.msgid.link/20260930173635.3362655-8-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index b8b2217b9efa..84312146e93f 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -17,10 +17,9 @@ #include "kselftest.h" /* L2 is scaled up (from L1's perspective) by this factor */ -#define L2_SCALE_FACTOR 4ULL +static u64 l2_scale_factor; #define TSC_OFFSET_L2 ((u64)-33125236320908) -#define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48) enum { USLEEP, UCHECK_L1, UCHECK_L2 }; #define GUEST_SLEEP(sec) ucall(UCALL_SYNC, 2, USLEEP, sec) @@ -81,7 +80,7 @@ static void l1_svm_code(struct svm_test_data *svm) generic_svm_setup(svm, l2_guest_code); /* enable TSC scaling for L2 */ - wrmsr(MSR_AMD64_TSC_RATIO, L2_SCALE_FACTOR << 32); + wrmsr(MSR_AMD64_TSC_RATIO, l2_scale_factor << 32); /* launch L2 */ run_guest(svm->vmcb, svm->vmcb_gpa); @@ -116,7 +115,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) vmwrite(SECONDARY_VM_EXEC_CONTROL, control); vmwrite(TSC_OFFSET, TSC_OFFSET_L2); - vmwrite(TSC_MULTIPLIER, TSC_MULTIPLIER_L2); + vmwrite(TSC_MULTIPLIER, l2_scale_factor << 48); /* launch L2 */ vmlaunch(); @@ -154,15 +153,12 @@ int main(int argc, char *argv[]) TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL)); TEST_REQUIRE(sys_clocksource_is_based_on_tsc()); - /* - * We set L1's scale factor to be a random number from 2 to 10. - * Ideally we would do the same for L2's factor but that one is - * referenced by both main() and l1_guest_code() and using a global - * variable does not work. - */ + /* Scale L1 "down" and L2 "up" at a random factor from 2 to 10. */ l1_scale_factor = (kvm_random_u32(&kvm_rng) % 9) + 2; printf("L1's scale down factor is: %lu\n", l1_scale_factor); - printf("L2's scale up factor is: %llu\n", L2_SCALE_FACTOR); + + l2_scale_factor = (kvm_random_u32(&kvm_rng) % 9) + 2; + printf("L2's scale up factor is: %lu\n", l2_scale_factor); tsc_start = rdtsc(); sleep(1); @@ -172,6 +168,7 @@ int main(int argc, char *argv[]) printf("real TSC frequency is around: %lu\n", l0_tsc_freq); vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + sync_global_to_guest(vm, l2_scale_factor); if (kvm_cpu_has(X86_FEATURE_VMX)) vcpu_alloc_vmx(vm, &guest_gva); @@ -212,7 +209,7 @@ int main(int argc, char *argv[]) printf("L2's TSC frequency is around: %lu\n", l2_tsc_freq); compare_tsc_freq(l2_tsc_freq, - l1_tsc_freq * L2_SCALE_FACTOR); + l1_tsc_freq * l2_scale_factor); break; } break; From 771902aecbb46e0908dec93fa75fba544fbba39d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:22 -0700 Subject: [PATCH 354/506] KVM: selftests: Rename TSC freq checkers in nested TSC scaling test Rename the nested TSC scaling test's helpers to clarify which one is used by guest code and which one is used by host code, and to make it more obvious that the host version *checks* the result and asserts on errors. Contrary to what "compare" suggests, the helper doesn't return whether or not the actual vs. expected values match. Link: https://patch.msgid.link/20260930173635.3362655-9-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 84312146e93f..e5fc4c9bb9f9 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -32,7 +32,7 @@ enum { USLEEP, UCHECK_L1, UCHECK_L2 }; * measurements, a difference of 1% between the actual and the expected value * is tolerated. */ -static void compare_tsc_freq(u64 actual, u64 expected) +static void host_check_tsc_freq(u64 actual, u64 expected) { u64 tolerance, thresh_low, thresh_high; @@ -45,7 +45,7 @@ static void compare_tsc_freq(u64 actual, u64 expected) actual, thresh_low, thresh_high); } -static void check_tsc_freq(int level) +static void guest_check_tsc_freq(int level) { u64 tsc_start, tsc_end, tsc_freq; @@ -66,7 +66,7 @@ static void check_tsc_freq(int level) static void l2_guest_code(void) { - check_tsc_freq(UCHECK_L2); + guest_check_tsc_freq(UCHECK_L2); /* exit to L1 */ __asm__ __volatile__("vmcall"); @@ -75,7 +75,7 @@ static void l2_guest_code(void) static void l1_svm_code(struct svm_test_data *svm) { /* check that L1's frequency looks alright before launching L2 */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCHECK_L1); generic_svm_setup(svm, l2_guest_code); @@ -87,7 +87,7 @@ static void l1_svm_code(struct svm_test_data *svm) GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL); /* check that L1's frequency still looks good */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCHECK_L1); GUEST_DONE(); } @@ -97,7 +97,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) u32 control; /* check that L1's frequency looks alright before launching L2 */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCHECK_L1); prepare_for_vmx_operation(vmx_pages); load_vmcs(vmx_pages); @@ -122,7 +122,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* check that L1's frequency still looks good */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCHECK_L1); GUEST_DONE(); } @@ -201,14 +201,14 @@ int main(int argc, char *argv[]) l1_tsc_freq = uc.args[1]; printf("L1's TSC frequency is around: %lu\n", l1_tsc_freq); - compare_tsc_freq(l1_tsc_freq, + host_check_tsc_freq(l1_tsc_freq, l0_tsc_freq / l1_scale_factor); break; case UCHECK_L2: l2_tsc_freq = uc.args[1]; printf("L2's TSC frequency is around: %lu\n", l2_tsc_freq); - compare_tsc_freq(l2_tsc_freq, + host_check_tsc_freq(l2_tsc_freq, l1_tsc_freq * l2_scale_factor); break; } From 6ab38c295d062121bd246403afb667662e5186f9 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:23 -0700 Subject: [PATCH 355/506] KVM: selftests: Extract guts of nested TSC scaling test to helper function Extract the guts of the nested TSC scaling test to a separate helper in anticipation of extending the test beyond its single "scale L1 up, L2 down" testcase. No functional change intended. Link: https://patch.msgid.link/20260930173635.3362655-10-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index e5fc4c9bb9f9..d3208b57eafc 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -135,38 +135,18 @@ static void l1_guest_code(void *data) l1_svm_code(data); } -int main(int argc, char *argv[]) +static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale) { + u64 tsc_khz, l1_tsc_freq, l2_tsc_freq; struct kvm_vcpu *vcpu; struct kvm_vm *vm; - gva_t guest_gva = 0; + gva_t guest_gva; - u64 tsc_start, tsc_end; - u64 tsc_khz; - u64 l1_scale_factor; - u64 l0_tsc_freq = 0; - u64 l1_tsc_freq = 0; - u64 l2_tsc_freq = 0; + l2_scale_factor = l2_scale; - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || - kvm_cpu_has(X86_FEATURE_SVM)); - TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL)); - TEST_REQUIRE(sys_clocksource_is_based_on_tsc()); - - /* Scale L1 "down" and L2 "up" at a random factor from 2 to 10. */ - l1_scale_factor = (kvm_random_u32(&kvm_rng) % 9) + 2; printf("L1's scale down factor is: %lu\n", l1_scale_factor); - - l2_scale_factor = (kvm_random_u32(&kvm_rng) % 9) + 2; printf("L2's scale up factor is: %lu\n", l2_scale_factor); - tsc_start = rdtsc(); - sleep(1); - tsc_end = rdtsc(); - - l0_tsc_freq = tsc_end - tsc_start; - printf("real TSC frequency is around: %lu\n", l0_tsc_freq); - vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); sync_global_to_guest(vm, l2_scale_factor); @@ -183,6 +163,9 @@ int main(int argc, char *argv[]) /* scale down L1's TSC frequency */ vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *) (tsc_khz / l1_scale_factor)); + /* L1 will communicate its frequency before the L2 check.*/ + l1_tsc_freq = 0; + for (;;) { struct ucall uc; @@ -222,5 +205,28 @@ int main(int argc, char *argv[]) done: kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + u64 l0_tsc_freq, tsc_start, tsc_end, l1_scale, l2_scale; + + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || + kvm_cpu_has(X86_FEATURE_SVM)); + TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL)); + TEST_REQUIRE(sys_clocksource_is_based_on_tsc()); + + tsc_start = rdtsc(); + sleep(1); + tsc_end = rdtsc(); + + l0_tsc_freq = tsc_end - tsc_start; + printf("real TSC frequency is around: %lu\n", l0_tsc_freq); + + /* Scale L1 "down" and L2 "up" at a random factor from 2 to 10. */ + l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; + l2_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; + test_tsc_scaling(l0_tsc_freq, l1_scale, l2_scale); + return 0; } From ff18d56a08b2b1671d89a5be6d1b2bdd857de1ef Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:24 -0700 Subject: [PATCH 356/506] KVM: selftests: Track L2 multiplier, not scale-up factor, in nested TSC test Track L2's multiplier, not just the non-fractional scale-up factor, so that the nested TSC scaling test can also validate scaling L2's frequency *down*. No functional change intended. Link: https://patch.msgid.link/20260930173635.3362655-11-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index d3208b57eafc..af85fd469ef4 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -16,8 +16,8 @@ #include "svm_util.h" #include "kselftest.h" -/* L2 is scaled up (from L1's perspective) by this factor */ -static u64 l2_scale_factor; +/* L2's TSC multiplier, relative to L1. */ +static u64 l2_multiplier; #define TSC_OFFSET_L2 ((u64)-33125236320908) @@ -80,7 +80,7 @@ static void l1_svm_code(struct svm_test_data *svm) generic_svm_setup(svm, l2_guest_code); /* enable TSC scaling for L2 */ - wrmsr(MSR_AMD64_TSC_RATIO, l2_scale_factor << 32); + wrmsr(MSR_AMD64_TSC_RATIO, l2_multiplier); /* launch L2 */ run_guest(svm->vmcb, svm->vmcb_gpa); @@ -115,7 +115,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) vmwrite(SECONDARY_VM_EXEC_CONTROL, control); vmwrite(TSC_OFFSET, TSC_OFFSET_L2); - vmwrite(TSC_MULTIPLIER, l2_scale_factor << 48); + vmwrite(TSC_MULTIPLIER, l2_multiplier); /* launch L2 */ vmlaunch(); @@ -135,20 +135,22 @@ static void l1_guest_code(void *data) l1_svm_code(data); } -static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale) +static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_factor) { u64 tsc_khz, l1_tsc_freq, l2_tsc_freq; struct kvm_vcpu *vcpu; struct kvm_vm *vm; gva_t guest_gva; - - l2_scale_factor = l2_scale; + u8 frac_bits; printf("L1's scale down factor is: %lu\n", l1_scale_factor); printf("L2's scale up factor is: %lu\n", l2_scale_factor); + frac_bits = kvm_cpu_has(X86_FEATURE_VMX) ? 48 : 32; + l2_multiplier = l2_scale_factor << frac_bits; + vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); - sync_global_to_guest(vm, l2_scale_factor); + sync_global_to_guest(vm, l2_multiplier); if (kvm_cpu_has(X86_FEATURE_VMX)) vcpu_alloc_vmx(vm, &guest_gva); From df1613249f6f65cebd6fff8782852013ca76b936 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:25 -0700 Subject: [PATCH 357/506] KVM: selftests: Print out the failing L{0,1,2} level in nested TSC scaling test Print out which one of L0, L1, or L2 TSC frequency checks failed if the nested TSC scaling test detects an out-of-bounds frequency. Triaging test failures without the level information is unnecessarily painful. Link: https://patch.msgid.link/20260930173635.3362655-12-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/x86/nested_tsc_scaling_test.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index af85fd469ef4..8c96b8e501f9 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -32,7 +32,7 @@ enum { USLEEP, UCHECK_L1, UCHECK_L2 }; * measurements, a difference of 1% between the actual and the expected value * is tolerated. */ -static void host_check_tsc_freq(u64 actual, u64 expected) +static void host_check_tsc_freq(int level, u64 actual, u64 expected) { u64 tolerance, thresh_low, thresh_high; @@ -41,8 +41,8 @@ static void host_check_tsc_freq(u64 actual, u64 expected) thresh_high = expected + tolerance; TEST_ASSERT(thresh_low < actual && thresh_high > actual, - "TSC freq is '%lu', expected to be between %lu and %lu", - actual, thresh_low, thresh_high); + "L%u TSC freq is '%lu', expected to be between %lu and %lu", + level, actual, thresh_low, thresh_high); } static void guest_check_tsc_freq(int level) @@ -186,14 +186,14 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_ l1_tsc_freq = uc.args[1]; printf("L1's TSC frequency is around: %lu\n", l1_tsc_freq); - host_check_tsc_freq(l1_tsc_freq, + host_check_tsc_freq(1, l1_tsc_freq, l0_tsc_freq / l1_scale_factor); break; case UCHECK_L2: l2_tsc_freq = uc.args[1]; printf("L2's TSC frequency is around: %lu\n", l2_tsc_freq); - host_check_tsc_freq(l2_tsc_freq, + host_check_tsc_freq(2, l2_tsc_freq, l1_tsc_freq * l2_scale_factor); break; } From faa4f010187c5d4cc4ddb3adecb9943ff87472f5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:26 -0700 Subject: [PATCH 358/506] KVM: selftests: Allow +/- 1 tolerance if expected TSC frequency is <100 Massage the nested TSC scaling test's checker to allow for validating an expected TSC frequency of 1Hz, which is achievable for L2 since L1 can use a completely arbitrary fractional ratio. To do so, set the minimum tolerance to 1, and make the thresholds inclusive, i.e. for an expected frequency of 1, allow the actual frequency to be in the range of [0,2]. Because scaling L2's frequency down to a single Hz makes the virtual TSC run soooo comically slow, and because all of the math is done with basic integer arithmetic, it's very easy for the test to observe what appears to be a frequency of 0. Link: https://patch.msgid.link/20260930173635.3362655-13-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 8c96b8e501f9..d790ab36a198 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -36,11 +36,11 @@ static void host_check_tsc_freq(int level, u64 actual, u64 expected) { u64 tolerance, thresh_low, thresh_high; - tolerance = expected / 100; + tolerance = max(expected / 100, (u64)1); thresh_low = expected - tolerance; thresh_high = expected + tolerance; - TEST_ASSERT(thresh_low < actual && thresh_high > actual, + TEST_ASSERT(thresh_low <= actual && thresh_high >= actual, "L%u TSC freq is '%lu', expected to be between %lu and %lu", level, actual, thresh_low, thresh_high); } From 43f58b1ee6fe030d0e425cab0382d1e07841f897 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:27 -0700 Subject: [PATCH 359/506] KVM: selftests: Use KVM's reported TSC KHz as L0's frequency (sanity checked) Use KVM's reported default TSC frequency as L0's frequency instead of calibrating L0's frequency using RDTSC. Trying to calibrate the frequency in userspace is all but guaranteed to be wildly inaccurate, e.g. due to interrupts, preemption, etc, and no known CPU supports TSC scaling without also having a constant TSC. I.e. there's no need to calibrate the current frequency, as it should also be the same as KVM's default frequency. Opportunistically sanity check KVM's reported frequency against what is observed via RDTSC, e.g. so that KVM_GET_TSC_KHZ or platform issues show up much earlier in the test, before VMs get involved. Link: https://patch.msgid.link/20260930173635.3362655-14-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index d790ab36a198..a78585af0ffd 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -137,7 +137,7 @@ static void l1_guest_code(void *data) static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_factor) { - u64 tsc_khz, l1_tsc_freq, l2_tsc_freq; + u64 l1_tsc_freq, l2_tsc_freq; struct kvm_vcpu *vcpu; struct kvm_vm *vm; gva_t guest_gva; @@ -159,11 +159,8 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_ vcpu_args_set(vcpu, 1, guest_gva); - tsc_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL); - TEST_ASSERT(tsc_khz != -1, "vcpu ioctl KVM_GET_TSC_KHZ failed"); - /* scale down L1's TSC frequency */ - vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *) (tsc_khz / l1_scale_factor)); + vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *) (l0_tsc_freq / l1_scale_factor)); /* L1 will communicate its frequency before the L2 check.*/ l1_tsc_freq = 0; @@ -212,18 +209,30 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_ int main(int argc, char *argv[]) { u64 l0_tsc_freq, tsc_start, tsc_end, l1_scale, l2_scale; + struct kvm_vm *vm; TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || kvm_cpu_has(X86_FEATURE_SVM)); TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL)); TEST_REQUIRE(sys_clocksource_is_based_on_tsc()); + /* + * Create a dummy VM to get KVM's default TSC frequency. All CPUs that + * support TSC scaling should have a constant TSC, i.e. there's no need + * to calibrate the "real" TSC. But do sanity check that the observed + * TSC is within range of KVM's reported TSC frequency. + */ + vm = vm_create_barebones(); + l0_tsc_freq = (u64)__vm_ioctl(vm, KVM_GET_TSC_KHZ, NULL) * 1000; + TEST_ASSERT(l0_tsc_freq, "vcpu ioctl KVM_GET_TSC_KHZ failed"); + kvm_vm_free(vm); + + printf("L0 TSC frequency is: %lu\n", l0_tsc_freq); + tsc_start = rdtsc(); sleep(1); tsc_end = rdtsc(); - - l0_tsc_freq = tsc_end - tsc_start; - printf("real TSC frequency is around: %lu\n", l0_tsc_freq); + host_check_tsc_freq(0, tsc_end - tsc_start, l0_tsc_freq); /* Scale L1 "down" and L2 "up" at a random factor from 2 to 10. */ l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; From b77a34ac9d8fd19623dbe744bbde5a3f8b051add Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:28 -0700 Subject: [PATCH 360/506] KVM: selftests: Explicitly pass TSC frequencies to guts of TSC scaling test Explicitly pass the L1 and L2 TSC frequencies, along with the L2 multiplier, to the main test routine in the nested TSC scaling test. This will allow testing both "scale up" and "scale down" scenarios, and that KVM saturates L2's frequency if it's scale up/down beyond what hardware can support, in which case the expected frequency will not exactly match the L2 multiplier. Link: https://patch.msgid.link/20260930173635.3362655-15-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 42 ++++++++----------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index a78585af0ffd..a8f9307ed2a4 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -8,7 +8,7 @@ * both L1 and L2 are scaled using different ratios. For this test we scale * L1 down and scale L2 up. */ - +#include #include #include "kvm_util.h" @@ -135,21 +135,19 @@ static void l1_guest_code(void *data) l1_svm_code(data); } -static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_factor) +static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_tsc_freq, u64 l2_tsc_freq, + u64 __l2_multiplier) { - u64 l1_tsc_freq, l2_tsc_freq; struct kvm_vcpu *vcpu; struct kvm_vm *vm; gva_t guest_gva; - u8 frac_bits; - printf("L1's scale down factor is: %lu\n", l1_scale_factor); - printf("L2's scale up factor is: %lu\n", l2_scale_factor); - - frac_bits = kvm_cpu_has(X86_FEATURE_VMX) ? 48 : 32; - l2_multiplier = l2_scale_factor << frac_bits; + printf("Testing L0 freq = %lu, L1 freq = %lu, L2 freq = %lu, L2 mult = 0x%lx\n", + l0_tsc_freq, l1_tsc_freq, l2_tsc_freq, __l2_multiplier); vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + + l2_multiplier = __l2_multiplier; sync_global_to_guest(vm, l2_multiplier); if (kvm_cpu_has(X86_FEATURE_VMX)) @@ -159,11 +157,7 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_ vcpu_args_set(vcpu, 1, guest_gva); - /* scale down L1's TSC frequency */ - vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *) (l0_tsc_freq / l1_scale_factor)); - - /* L1 will communicate its frequency before the L2 check.*/ - l1_tsc_freq = 0; + vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *)(l1_tsc_freq / 1000)); for (;;) { struct ucall uc; @@ -180,18 +174,12 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_ sleep(uc.args[1]); break; case UCHECK_L1: - l1_tsc_freq = uc.args[1]; - printf("L1's TSC frequency is around: %lu\n", l1_tsc_freq); - - host_check_tsc_freq(1, l1_tsc_freq, - l0_tsc_freq / l1_scale_factor); + printf("L1's observed TSC frequency: %lu\n", uc.args[1]); + host_check_tsc_freq(1, uc.args[1], l1_tsc_freq); break; case UCHECK_L2: - l2_tsc_freq = uc.args[1]; - printf("L2's TSC frequency is around: %lu\n", l2_tsc_freq); - - host_check_tsc_freq(2, l2_tsc_freq, - l1_tsc_freq * l2_scale_factor); + printf("L2's observed TSC frequency: %lu\n", uc.args[1]); + host_check_tsc_freq(2, uc.args[1], l2_tsc_freq); break; } break; @@ -209,6 +197,7 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_scale_factor, u64 l2_scale_ int main(int argc, char *argv[]) { u64 l0_tsc_freq, tsc_start, tsc_end, l1_scale, l2_scale; + u8 frac_bits = kvm_cpu_has(X86_FEATURE_VMX) ? 48 : 32; struct kvm_vm *vm; TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || @@ -237,7 +226,10 @@ int main(int argc, char *argv[]) /* Scale L1 "down" and L2 "up" at a random factor from 2 to 10. */ l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; l2_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; - test_tsc_scaling(l0_tsc_freq, l1_scale, l2_scale); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale, + mul_u64_u64_div64(l0_tsc_freq, l2_scale, l1_scale), + (l2_scale << frac_bits)); return 0; } From 6760d12380a4e03aa4b1f2a4f769a3000d2543c1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:29 -0700 Subject: [PATCH 361/506] KVM: selftests: Sanity check KVM's default TSC freq in nested TSC scaling test Explicitly verify that both L1 and L2 can run at KVM's default TSC frequency reported via KVM_GET_TSC_KHZ, e.g. that KVM_SET_TSC_KHZ can take in what KVM_GET_TSC_KHZ spits out. Link: https://patch.msgid.link/20260930173635.3362655-16-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index a8f9307ed2a4..ee8fda310e33 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -223,6 +223,9 @@ int main(int argc, char *argv[]) tsc_end = rdtsc(); host_check_tsc_freq(0, tsc_end - tsc_start, l0_tsc_freq); + /* Sanity check the frequency reported by KVM_GET_TSC_KHZ. */ + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq, l0_tsc_freq, BIT_ULL(frac_bits)); + /* Scale L1 "down" and L2 "up" at a random factor from 2 to 10. */ l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; l2_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; From 245932d3880ca22382c9c727a9cdab54c3441d6a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:30 -0700 Subject: [PATCH 362/506] KVM: selftests: Test L1 "up" and L2 "down" in nested TSC scaling test Now that the guts of the nested TSC scaling test support scaling L2 down, validate all four combinations of scaling up/down L1 and L2. Ideally, the test would also validate integer+fractional scale up scenarios, but prioritize keeping the test and math somewhat understandable. Opportunistically drop the stale file comment about the test's name. Link: https://patch.msgid.link/20260930173635.3362655-17-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index ee8fda310e33..1d578af393bd 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -1,12 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * vmx_nested_tsc_scaling_test - * * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * This test case verifies that nested TSC scaling behaves as expected when - * both L1 and L2 are scaled using different ratios. For this test we scale - * L1 down and scale L2 up. + * both L1 and L2 are scaled using different ratios. */ #include #include @@ -226,7 +223,21 @@ int main(int argc, char *argv[]) /* Sanity check the frequency reported by KVM_GET_TSC_KHZ. */ test_tsc_scaling(l0_tsc_freq, l0_tsc_freq, l0_tsc_freq, BIT_ULL(frac_bits)); - /* Scale L1 "down" and L2 "up" at a random factor from 2 to 10. */ + /* + * Scale L1 and L2 up and down at random factors from 2 to 10. Current + * CPUs have two full orders of magnitude of "breathing room" before an + * ultrafast L0 TSC frequency multiplied by 10x will encounter KVM's + * signed 32-bit limit on L1's frequency, and it's highly unlikely a + * CPU that supports TSC scaling will show up running at 100MHz, i.e. + * underflowing KVM's minimum 1KHz frequency is extremely unlikely. + * + * For L2, the frequency is limited only by what hardware can support, + * not by KVM's limits. SVM provides 8 bits of integer scale up, and + * 32 bits of fractional scale down, i.e. can scale up 255x and down a + * comical amount, so scaling up 100x and down 1/100 is well within + * hardware's capabilities (VMX provides 16 bits of "up" and 48 bits of + * "down"). + */ l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; l2_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; @@ -234,5 +245,17 @@ int main(int argc, char *argv[]) mul_u64_u64_div64(l0_tsc_freq, l2_scale, l1_scale), (l2_scale << frac_bits)); + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale, + mul_u64_u64_div64(l0_tsc_freq, l1_scale, l2_scale), + (1ull << frac_bits) / l2_scale); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale, + l0_tsc_freq / l1_scale / l2_scale, + (1ull << frac_bits) / l2_scale); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale, + l0_tsc_freq * l1_scale * l2_scale, + (l2_scale << frac_bits)); + return 0; } From c2d3a10b90b13434aa5b945f661cf06a1b1ba56c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:31 -0700 Subject: [PATCH 363/506] KVM: selftests: Verify that KVM saturates L2 TSC freq on {under,over}flow Extend the nested TSC scaling test to validate the extreme ends of what hardware supports, i.e. validate that KVM allows L1 to scale L2's virtual TSC frequency up/down to the architectural maximum/minimum, from L1's perspective, and that KVM saturates L2's effective frequency when L2's frequency can't be virtualized by hardware (which is possible when stacking multipliers/ratios across L1 and L2). Link: https://patch.msgid.link/20260930173635.3362655-18-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 1d578af393bd..4cacd87f129b 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -193,6 +193,7 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_tsc_freq, u64 l2_tsc_freq, int main(int argc, char *argv[]) { + u64 min_freq, min_multiplier, max_freq, max_multiplier, l1_max_freq, l1_min_freq; u64 l0_tsc_freq, tsc_start, tsc_end, l1_scale, l2_scale; u8 frac_bits = kvm_cpu_has(X86_FEATURE_VMX) ? 48 : 32; struct kvm_vm *vm; @@ -257,5 +258,39 @@ int main(int argc, char *argv[]) l0_tsc_freq * l1_scale * l2_scale, (l2_scale << frac_bits)); + /* + * Test that KVM saturates L2's frequency on both ends if the resulting + * L2 TSC frequency would be below or above what hardware can support. + * Because L2 = L0 * (L1_mult >> frac) * (L2_mult >> frac) needs to be + * distilled down to a single multiplier, very small/large multipliers + * will underflow/overflow the minimum/maximum multiplier supported by + * hardware when L1 and L2 multipliers are combined. KVM's behavior is + * saturate on {under,over}flow, i.e. to run at the min/max frequency. + * + * Note, userspace can only program L1's frequency in KHz, i.e. can't + * specify an exact multiplier. As a result, the minimum and maximum + * frequencies are different for L1 vs L2, because L1 is constrained by + * hardware *and* KVM, whereas L2 is constrained only by hardware. + */ + min_multiplier = 1; + min_freq = mul_u64_u64_div64(l0_tsc_freq, min_multiplier, BIT_ULL(frac_bits)); + min_freq = max(min_freq, (u64)1); + l1_min_freq = max(min_freq, (u64)1 * 1000); + test_tsc_scaling(l0_tsc_freq, l1_min_freq, min_freq, 1); + + /* + * SVM takes a 40-bit value (right shifted by 32), while VMX takes a + * 64-bit value (right shifted by 48). mul_u64_u64_shr() isn't (yet) + * available in selftests, but mul_u64_u64_div64() does nicely since, + * albeit more slowly (performance is obviously not a concern). Note, + * because KVM_GET_TSC_KHZ returns a signed 32-bit integer, KVM limits + * KVM_SET_TSC_KHZ to INT_MAX, even though hardware (both SVM and VMX) + * supports much higher frequencies. + */ + max_multiplier = kvm_cpu_has(X86_FEATURE_VMX) ? -1ull : GENMASK_U64(39, 0); + max_freq = mul_u64_u64_div64(l0_tsc_freq, max_multiplier, BIT_ULL(frac_bits)); + l1_max_freq = min(max_freq, (u64)INT32_MAX * 1000); + test_tsc_scaling(l0_tsc_freq, l1_max_freq, max_freq, max_multiplier); + return 0; } From baada72b9c013cc6363a01b0e9af6263e07d696c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:32 -0700 Subject: [PATCH 364/506] KVM: selftests: Test non-zero TSC offset on SVM in nested TSC scaling test Set L2's TSC offset for SVM as well as VMX in the nested TSC scaling test. Odds are good skipping SVM was pure oversight. Fixes: e6bcdd212238 ("KVM: selftests: Extend vmx_nested_tsc_scaling_test to cover SVM") Link: https://patch.msgid.link/20260930173635.3362655-19-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 4cacd87f129b..571fc214c925 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -75,6 +75,7 @@ static void l1_svm_code(struct svm_test_data *svm) guest_check_tsc_freq(UCHECK_L1); generic_svm_setup(svm, l2_guest_code); + svm->vmcb->control.tsc_offset = TSC_OFFSET_L2; /* enable TSC scaling for L2 */ wrmsr(MSR_AMD64_TSC_RATIO, l2_multiplier); From 5f13a511c78215817fc603202b98293d4598f33d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:33 -0700 Subject: [PATCH 365/506] KVM: selftests: Randomize L2's TSC offset in the nested TSC scaling test Randomize L2's TSC offset in the nested TSC scaling test, not because it's all that interesting, but because hardcoding a copmletely arbitrary value is all kinds of silly. Link: https://patch.msgid.link/20260930173635.3362655-20-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/x86/nested_tsc_scaling_test.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 571fc214c925..be6246c20d7b 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -13,10 +13,9 @@ #include "svm_util.h" #include "kselftest.h" -/* L2's TSC multiplier, relative to L1. */ +/* L2's TSC multiplier and offset, relative to L1. */ static u64 l2_multiplier; - -#define TSC_OFFSET_L2 ((u64)-33125236320908) +static u64 l2_offset; enum { USLEEP, UCHECK_L1, UCHECK_L2 }; #define GUEST_SLEEP(sec) ucall(UCALL_SYNC, 2, USLEEP, sec) @@ -75,7 +74,7 @@ static void l1_svm_code(struct svm_test_data *svm) guest_check_tsc_freq(UCHECK_L1); generic_svm_setup(svm, l2_guest_code); - svm->vmcb->control.tsc_offset = TSC_OFFSET_L2; + svm->vmcb->control.tsc_offset = l2_offset; /* enable TSC scaling for L2 */ wrmsr(MSR_AMD64_TSC_RATIO, l2_multiplier); @@ -112,7 +111,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) control |= SECONDARY_EXEC_TSC_SCALING; vmwrite(SECONDARY_VM_EXEC_CONTROL, control); - vmwrite(TSC_OFFSET, TSC_OFFSET_L2); + vmwrite(TSC_OFFSET, l2_offset); vmwrite(TSC_MULTIPLIER, l2_multiplier); /* launch L2 */ @@ -145,6 +144,9 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_tsc_freq, u64 l2_tsc_freq, vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + l2_offset = kvm_random_u64(&kvm_rng); + sync_global_to_guest(vm, l2_offset); + l2_multiplier = __l2_multiplier; sync_global_to_guest(vm, l2_multiplier); From a744d1c1655d4c18769acf78a98119cb88bc3ada Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:34 -0700 Subject: [PATCH 366/506] KVM: selftests: Use GUEST_SYNC2() in nested TSC scaling test Use GUEST_SYNC2() instead of open coding equivalent functionality using raw ucall() invocations in the nested TSC scaling test. No functional change intended. Link: https://patch.msgid.link/20260930173635.3362655-21-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index be6246c20d7b..b21f79e9ec71 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -18,9 +18,6 @@ static u64 l2_multiplier; static u64 l2_offset; enum { USLEEP, UCHECK_L1, UCHECK_L2 }; -#define GUEST_SLEEP(sec) ucall(UCALL_SYNC, 2, USLEEP, sec) -#define GUEST_CHECK(level, freq) ucall(UCALL_SYNC, 2, level, freq) - /* * This function checks whether the "actual" TSC frequency of a guest matches @@ -52,12 +49,12 @@ static void guest_check_tsc_freq(int level) * be good enough for the purposes of this test. */ tsc_start = rdmsr(MSR_IA32_TSC); - GUEST_SLEEP(1); + GUEST_SYNC2(USLEEP, 1); tsc_end = rdmsr(MSR_IA32_TSC); tsc_freq = tsc_end - tsc_start; - GUEST_CHECK(level, tsc_freq); + GUEST_SYNC2(level, tsc_freq); } static void l2_guest_code(void) From bef953c069806148ae2f10714b39a08726c9a076 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 30 Sep 2026 10:36:35 -0700 Subject: [PATCH 367/506] KVM: selftests: Spell out UCALL in nested TSC scaling test's enums Explicitly spell out UCALL instead of prefixing only U in the nested TSC scaling test's ucall commands, as it's all too easy to think USLEEP means "do usleep()", when it actually means "do sleep() in userspace". Opportunistically name the enum and take an enum instead of a bare integer for the to-be-checked level, to make it more obvious that the fact that the level values happen to correspond to '1' and '2' for the nesting level is largely coincidental (or perhaps it was intentional, but it's not a hard requirement for the test). Link: https://patch.msgid.link/20260930173635.3362655-22-seanjc@google.com Signed-off-by: Sean Christopherson --- .../kvm/x86/nested_tsc_scaling_test.c | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index b21f79e9ec71..bfb1ff441c78 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -17,7 +17,7 @@ static u64 l2_multiplier; static u64 l2_offset; -enum { USLEEP, UCHECK_L1, UCHECK_L2 }; +enum ucall_cmd { UCALL_SLEEP, UCALL_CHECK_L1, UCALL_CHECK_L2 }; /* * This function checks whether the "actual" TSC frequency of a guest matches @@ -38,7 +38,7 @@ static void host_check_tsc_freq(int level, u64 actual, u64 expected) level, actual, thresh_low, thresh_high); } -static void guest_check_tsc_freq(int level) +static void guest_check_tsc_freq(enum ucall_cmd check_level) { u64 tsc_start, tsc_end, tsc_freq; @@ -49,17 +49,17 @@ static void guest_check_tsc_freq(int level) * be good enough for the purposes of this test. */ tsc_start = rdmsr(MSR_IA32_TSC); - GUEST_SYNC2(USLEEP, 1); + GUEST_SYNC2(UCALL_SLEEP, 1); tsc_end = rdmsr(MSR_IA32_TSC); tsc_freq = tsc_end - tsc_start; - GUEST_SYNC2(level, tsc_freq); + GUEST_SYNC2(check_level, tsc_freq); } static void l2_guest_code(void) { - guest_check_tsc_freq(UCHECK_L2); + guest_check_tsc_freq(UCALL_CHECK_L2); /* exit to L1 */ __asm__ __volatile__("vmcall"); @@ -68,7 +68,7 @@ static void l2_guest_code(void) static void l1_svm_code(struct svm_test_data *svm) { /* check that L1's frequency looks alright before launching L2 */ - guest_check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); generic_svm_setup(svm, l2_guest_code); svm->vmcb->control.tsc_offset = l2_offset; @@ -81,7 +81,7 @@ static void l1_svm_code(struct svm_test_data *svm) GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL); /* check that L1's frequency still looks good */ - guest_check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); GUEST_DONE(); } @@ -91,7 +91,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) u32 control; /* check that L1's frequency looks alright before launching L2 */ - guest_check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); prepare_for_vmx_operation(vmx_pages); load_vmcs(vmx_pages); @@ -116,7 +116,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* check that L1's frequency still looks good */ - guest_check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); GUEST_DONE(); } @@ -167,14 +167,14 @@ static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_tsc_freq, u64 l2_tsc_freq, REPORT_GUEST_ASSERT(uc); case UCALL_SYNC: switch (uc.args[0]) { - case USLEEP: + case UCALL_SLEEP: sleep(uc.args[1]); break; - case UCHECK_L1: + case UCALL_CHECK_L1: printf("L1's observed TSC frequency: %lu\n", uc.args[1]); host_check_tsc_freq(1, uc.args[1], l1_tsc_freq); break; - case UCHECK_L2: + case UCALL_CHECK_L2: printf("L2's observed TSC frequency: %lu\n", uc.args[1]); host_check_tsc_freq(2, uc.args[1], l2_tsc_freq); break; From 295d3df9b1e425099a4eb37954a02c4a08fdb496 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:20 +0200 Subject: [PATCH 368/506] arm64: ptrace: Use constants for compat register numbers Define an enum to determine the register number for the mapping from AArch32 and AArch64 registers. This allows for a more flexible use of the register mappings. Reviewed-by: Marc Zyngier Acked-by: Catalin Marinas Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-6-seiden@linux.ibm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/ptrace.h | 60 ++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index f7dc5fb9427d..49a83a0636b9 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -102,26 +102,48 @@ #define COMPAT_USER_SZ 296 /* Architecturally defined mapping between AArch32 and AArch64 registers */ +enum { + __compat_fp = 11, + __compat_sp = 13, + __compat_lr = 14, + __compat_sp_hyp = 15, + __compat_lr_irq = 16, + __compat_sp_irq = 17, + __compat_lr_svc = 18, + __compat_sp_svc = 19, + __compat_lr_abt = 20, + __compat_sp_abt = 21, + __compat_lr_und = 22, + __compat_sp_und = 23, + __compat_r8_fiq = 24, + __compat_r9_fiq = 25, + __compat_r10_fiq = 26, + __compat_r11_fiq = 27, + __compat_r12_fiq = 28, + __compat_sp_fiq = 29, + __compat_lr_fiq = 30, +}; + #define compat_usr(x) regs[(x)] -#define compat_fp regs[11] -#define compat_sp regs[13] -#define compat_lr regs[14] -#define compat_sp_hyp regs[15] -#define compat_lr_irq regs[16] -#define compat_sp_irq regs[17] -#define compat_lr_svc regs[18] -#define compat_sp_svc regs[19] -#define compat_lr_abt regs[20] -#define compat_sp_abt regs[21] -#define compat_lr_und regs[22] -#define compat_sp_und regs[23] -#define compat_r8_fiq regs[24] -#define compat_r9_fiq regs[25] -#define compat_r10_fiq regs[26] -#define compat_r11_fiq regs[27] -#define compat_r12_fiq regs[28] -#define compat_sp_fiq regs[29] -#define compat_lr_fiq regs[30] +#define compat_fp regs[__compat_fp] +#define compat_sp regs[__compat_sp] +#define compat_lr regs[__compat_lr] +#define compat_sp_hyp regs[__compat_sp_hyp] +#define compat_lr_irq regs[__compat_lr_irq] +#define compat_sp_irq regs[__compat_sp_irq] +#define compat_lr_svc regs[__compat_lr_svc] +#define compat_sp_svc regs[__compat_sp_svc] +#define compat_lr_abt regs[__compat_lr_abt] +#define compat_sp_abt regs[__compat_sp_abt] +#define compat_lr_und regs[__compat_lr_und] +#define compat_sp_und regs[__compat_sp_und] +#define compat_r8_fiq regs[__compat_r8_fiq] +#define compat_r9_fiq regs[__compat_r9_fiq] +#define compat_r10_fiq regs[__compat_r10_fiq] +#define compat_r11_fiq regs[__compat_r11_fiq] +#define compat_r12_fiq regs[__compat_r12_fiq] +#define compat_sp_fiq regs[__compat_sp_fiq] +#define compat_lr_fiq regs[__compat_lr_fiq] static inline unsigned long compat_psr_to_pstate(const unsigned long psr) { From d3d78520068779779e1ed7aa0dd8a3ce63e1ca43 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:21 +0200 Subject: [PATCH 369/506] arm64: sysreg: Convert SPSR_ELx to automatic register generation Convert SPSR_EL{1, 2, 12} to automatic register generation as per DDI0601. No functional change. Acked-by: Catalin Marinas Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-7-seiden@linux.ibm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/sysreg.h | 3 -- arch/arm64/tools/sysreg | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 7aa08d59d494..387950b19cac 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -295,7 +295,6 @@ #define SYS_APGAKEYLO_EL1 sys_reg(3, 0, 2, 3, 0) #define SYS_APGAKEYHI_EL1 sys_reg(3, 0, 2, 3, 1) -#define SYS_SPSR_EL1 sys_reg(3, 0, 4, 0, 0) #define SYS_ELR_EL1 sys_reg(3, 0, 4, 0, 1) #define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0) @@ -518,7 +517,6 @@ #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0) #define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6) -#define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) #define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0) #define SYS_SPSR_irq sys_reg(3, 4, 4, 3, 0) @@ -604,7 +602,6 @@ #define SYS_BRBCR_EL12 sys_reg(2, 5, 9, 0, 0) #define SYS_TTBR0_EL12 sys_reg(3, 5, 2, 0, 0) #define SYS_TTBR1_EL12 sys_reg(3, 5, 2, 0, 1) -#define SYS_SPSR_EL12 sys_reg(3, 5, 4, 0, 0) #define SYS_ELR_EL12 sys_reg(3, 5, 4, 0, 1) #define SYS_AFSR0_EL12 sys_reg(3, 5, 5, 1, 0) #define SYS_AFSR1_EL12 sys_reg(3, 5, 5, 1, 1) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index e2d37ee221b8..893b5f9df575 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -401,6 +401,57 @@ Res0 3:1 Field 0 SO EndSysreg +SysregFields SPSR_ELx +Res0 63:37 +Field 36 UINJ +Field 35 PACM +Field 34 EXLOCK +Res0 33 +Field 32 PM +Field 31 N +Field 30 Z +Field 29 C +Field 28 V +Res0 27:26 +Field 25 TCO +Field 24 DIT +Field 23 UAO +Field 22 PAN +Field 21 SS +Field 20 IL +Res0 19:14 +Field 13 ALLINT +Field 12 SSBS +Field 11:10 BTYPE +Field 9 D +Field 8 A +Field 7 I +Field 6 F +Res0 5 +Field 4 M4 +Enum 3:0 M + 0b0000 EL0 + 0b0100 EL1t + 0b0101 EL1h + 0b1000 EL2t + 0b1001 EL2h + 0b1100 EL3t + 0b1101 EL3h +EndEnum +EndSysregFields + +Sysreg SPSR_EL1 3 0 4 0 0 +Fields SPSR_ELx +EndSysreg + +Sysreg SPSR_EL2 3 4 4 0 0 +Fields SPSR_ELx +EndSysreg + +Sysreg SPSR_EL12 3 5 4 0 0 +Mapping SPSR_EL1 +EndSysreg + Sysreg ID_PFR0_EL1 3 0 0 1 0 Res0 63:32 UnsignedEnum 31:28 RAS From 29d92585ab987bb80b1e78d811aaea2a8db86e62 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:22 +0200 Subject: [PATCH 370/506] KVM: arm64: Access elements of vcpu_gp_regs individually While for arm64 the members of vcpu_gp_regs are allocated continuous this is not necessarily true for other architectures implementing ARM. Let vcpu_gp_regs() no longer return the address of the user_pt_regs in the vcpu context but the address of the gp-register array field in the user_pt_reg struct. Note, this change semantically excludes pc from the GPRs. No functional change. Co-developed-by: Nina Schoetterl-Glausch Signed-off-by: Nina Schoetterl-Glausch Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-8-seiden@linux.ibm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_emulate.h | 14 ++++++++------ arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/kvm/guest.c | 2 +- arch/arm64/kvm/hyp/exception.c | 4 ++-- arch/arm64/kvm/hyp/include/hyp/adjust_pc.h | 4 ++-- arch/arm64/kvm/hyp/include/hyp/switch.h | 4 ++-- 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index a3c1928bdf74..8603766c1b4d 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -133,12 +133,12 @@ static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr) static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) { - return (unsigned long *)&vcpu_gp_regs(vcpu)->pc; + return (unsigned long *)&vcpu->arch.ctxt.regs.pc; } static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) { - return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate; + return (unsigned long *)&vcpu->arch.ctxt.regs.pstate; } static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) @@ -167,14 +167,14 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu, u8 reg_num) { - return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num]; + return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)[reg_num]; } static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, unsigned long val) { if (reg_num != 31) - vcpu_gp_regs(vcpu)->regs[reg_num] = val; + vcpu_gp_regs(vcpu)[reg_num] = val; } static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt) @@ -750,13 +750,15 @@ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) pstate = VCPU_RESET_PSTATE_EL1; /* Reset core registers */ - memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); + memset(vcpu_gp_regs(vcpu), 0, sizeof(vcpu_gp_regs(vcpu))); + *vcpu_pc(vcpu) = 0; + vcpu->arch.ctxt.regs.sp = 0; memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); vcpu->arch.ctxt.spsr_abt = 0; vcpu->arch.ctxt.spsr_und = 0; vcpu->arch.ctxt.spsr_irq = 0; vcpu->arch.ctxt.spsr_fiq = 0; - vcpu_gp_regs(vcpu)->pstate = pstate; + *vcpu_cpsr(vcpu) = pstate; } /* PSCI reset handling for a vcpu. */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..46c8d7928b3f 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1158,7 +1158,7 @@ struct kvm_vcpu_arch { #define vcpu_clear_on_unsupported_cpu(vcpu) \ vcpu_clear_flag(vcpu, ON_UNSUPPORTED_CPU) -#define vcpu_gp_regs(v) (&(v)->arch.ctxt.regs) +#define vcpu_gp_regs(v) ((v)->arch.ctxt.regs.regs) /* * Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index b01d6622b872..81d850d5a328 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -134,7 +134,7 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) KVM_REG_ARM_CORE_REG(regs.regs[30]): off -= KVM_REG_ARM_CORE_REG(regs.regs[0]); off /= 2; - return &vcpu->arch.ctxt.regs.regs[off]; + return &vcpu_gp_regs(vcpu)[off]; case KVM_REG_ARM_CORE_REG(regs.sp): return &vcpu->arch.ctxt.regs.sp; diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 754e2dc1df54..0640c9ccaa3c 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -276,12 +276,12 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) switch(mode) { case PSR_AA32_MODE_ABT: __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr)); - vcpu_gp_regs(vcpu)->compat_lr_abt = return_address; + vcpu_gp_regs(vcpu)[__compat_lr_abt] = return_address; break; case PSR_AA32_MODE_UND: __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr)); - vcpu_gp_regs(vcpu)->compat_lr_und = return_address; + vcpu_gp_regs(vcpu)[__compat_lr_und] = return_address; break; } diff --git a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h index 4fdfeabefeb4..a049983bd5c3 100644 --- a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h +++ b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h @@ -33,11 +33,11 @@ static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) static inline void __kvm_skip_instr(struct kvm_vcpu *vcpu) { *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); - vcpu_gp_regs(vcpu)->pstate = read_sysreg_el2(SYS_SPSR); + *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); kvm_skip_instr(vcpu); - write_sysreg_el2(vcpu_gp_regs(vcpu)->pstate, SYS_SPSR); + write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); } diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 1ce7130e2549..a2a054fec2e2 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -470,7 +470,7 @@ static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) u64 spsr; *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); - arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); + arm64_mops_reset_regs(&vcpu->arch.ctxt.regs, vcpu->arch.fault.esr_el2); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); /* @@ -902,7 +902,7 @@ static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu) /* * Check for the conditions of Cortex-A510's #2077057. When these occur * SPSR_EL2 can't be trusted, but isn't needed either as it is - * unchanged from the value in vcpu_gp_regs(vcpu)->pstate. + * unchanged from the value in vcpu_cpsr(vcpu). * Are we single-stepping the guest, and took a PAC exception from the * active-not-pending state? */ From 89ed62818f10e38e64fea46d43ee6455a1beb647 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:23 +0200 Subject: [PATCH 371/506] KVM: arm64: Use accessor functions for core regs Instead of accessing the value directly use accessor functions that abstract the actual location of the register values. Add a helper function to reset fpsimd registers. Enables other KVM implementations to reuse this code. No functional changes. Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-9-seiden@linux.ibm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_emulate.h | 34 ++++++++++++++++++++++++++-- arch/arm64/kvm/guest.c | 14 ++++++------ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 8603766c1b4d..b31d50bdf59e 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -141,6 +141,31 @@ static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) return (unsigned long *)&vcpu->arch.ctxt.regs.pstate; } +static __always_inline unsigned long *vcpu_sp_el0(struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.ctxt.regs.sp; +} + +static __always_inline u64 *vcpu_sp_el1(struct kvm_vcpu *vcpu) +{ + return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); +} + +static __always_inline __u128 *vcpu_vreg(struct kvm_vcpu *vcpu, int n) +{ + return &vcpu->arch.ctxt.fp_regs.vregs[n]; +} + +static __always_inline __u32 *vcpu_fpsr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.ctxt.fp_regs.fpsr; +} + +static __always_inline __u32 *vcpu_fpcr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.ctxt.fp_regs.fpcr; +} + static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) { return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT); @@ -737,6 +762,11 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) } } +static inline void kvm_reset_fpsimd(struct kvm_vcpu *vcpu) +{ + memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); +} + /* Reset a vcpu's core registers. */ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) { @@ -752,8 +782,8 @@ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) /* Reset core registers */ memset(vcpu_gp_regs(vcpu), 0, sizeof(vcpu_gp_regs(vcpu))); *vcpu_pc(vcpu) = 0; - vcpu->arch.ctxt.regs.sp = 0; - memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); + *vcpu_sp_el0(vcpu) = 0; + kvm_reset_fpsimd(vcpu); vcpu->arch.ctxt.spsr_abt = 0; vcpu->arch.ctxt.spsr_und = 0; vcpu->arch.ctxt.spsr_irq = 0; diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 81d850d5a328..773f6c8e5026 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -137,16 +137,16 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return &vcpu_gp_regs(vcpu)[off]; case KVM_REG_ARM_CORE_REG(regs.sp): - return &vcpu->arch.ctxt.regs.sp; + return vcpu_sp_el0(vcpu); case KVM_REG_ARM_CORE_REG(regs.pc): - return &vcpu->arch.ctxt.regs.pc; + return vcpu_pc(vcpu); case KVM_REG_ARM_CORE_REG(regs.pstate): - return &vcpu->arch.ctxt.regs.pstate; + return vcpu_cpsr(vcpu); case KVM_REG_ARM_CORE_REG(sp_el1): - return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); + return vcpu_sp_el1(vcpu); case KVM_REG_ARM_CORE_REG(elr_el1): return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); @@ -170,13 +170,13 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]); off /= 4; - return &vcpu->arch.ctxt.fp_regs.vregs[off]; + return vcpu_vreg(vcpu, off); case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): - return &vcpu->arch.ctxt.fp_regs.fpsr; + return vcpu_fpsr(vcpu); case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): - return &vcpu->arch.ctxt.fp_regs.fpcr; + return vcpu_fpcr(vcpu); default: return NULL; From df6a70516adec8b7f5fa49e1755d8ee33d980d9c Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:24 +0200 Subject: [PATCH 372/506] arm64: Prepare sharing arm64 headers with s390 Prepare the sharing of arm64 headers with s390 by moving definitions inside headers to reduce the number of shared markers. Clean up includes to further streamline the sharing of headers. No functional change. Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-10-seiden@linux.ibm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/esr.h | 4 +- arch/arm64/include/asm/kvm_arm.h | 5 +- arch/arm64/include/asm/kvm_host.h | 102 ++++++++++++++++-------------- arch/arm64/include/asm/sysreg.h | 2 +- 4 files changed, 59 insertions(+), 54 deletions(-) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index f816f5d77f1a..da085c83b57e 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -7,7 +7,7 @@ #ifndef __ASM_ESR_H #define __ASM_ESR_H -#include +#include #include #define ESR_ELx_EC_UNKNOWN UL(0x00) @@ -435,7 +435,7 @@ #define ESR_ELx_IT_GCSPOPX 7 #ifndef __ASSEMBLER__ -#include +#include static inline unsigned long esr_brk_comment(unsigned long esr) { diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 4bfbd827c5aa..57edd8437200 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -7,10 +7,11 @@ #ifndef __ARM64_KVM_ARM_H__ #define __ARM64_KVM_ARM_H__ +#include +#include +#include #include -#include #include -#include /* * Because I'm terribly lazy and that repainting the whole of the KVM diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 46c8d7928b3f..5261c21a437c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -56,6 +56,46 @@ #define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10) #define KVM_REQ_VGIC_PROCESS_UPDATE KVM_ARCH_REQ(11) +/* Bit indices for kvm_arch::flags */ +enum { + /* + * If we encounter a data abort without valid instruction syndrome + * information, report this to user space. User space can (and + * should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is + * supported. + */ + KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, + /* Memory Tagging Extension enabled for the guest */ + KVM_ARCH_FLAG_MTE_ENABLED, + /* At least one vCPU has ran in the VM */ + KVM_ARCH_FLAG_HAS_RAN_ONCE, + /* The vCPU feature set for the VM is configured */ + KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, + /* PSCI SYSTEM_SUSPEND enabled for the guest */ + KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, + /* VM counter offset */ + KVM_ARCH_FLAG_VM_COUNTER_OFFSET, + /* Timer PPIs made immutable */ + KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, + /* Initial ID reg values loaded */ + KVM_ARCH_FLAG_ID_REGS_INITIALIZED, + /* Fine-Grained UNDEF initialised */ + KVM_ARCH_FLAG_FGU_INITIALIZED, + /* SVE exposed to guest */ + KVM_ARCH_FLAG_GUEST_HAS_SVE, + /* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */ + KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, + /* Unhandled SEAs are taken to userspace */ + KVM_ARCH_FLAG_EXIT_SEA, +}; + +struct vcpu_reset_state { + unsigned long pc; + unsigned long r0; + bool be; + bool reset; +}; + #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -338,35 +378,6 @@ struct kvm_arch { /* Protects VM-scoped configuration data */ struct mutex config_lock; - /* - * If we encounter a data abort without valid instruction syndrome - * information, report this to user space. User space can (and - * should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is - * supported. - */ -#define KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER 0 - /* Memory Tagging Extension enabled for the guest */ -#define KVM_ARCH_FLAG_MTE_ENABLED 1 - /* At least one vCPU has ran in the VM */ -#define KVM_ARCH_FLAG_HAS_RAN_ONCE 2 - /* The vCPU feature set for the VM is configured */ -#define KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED 3 - /* PSCI SYSTEM_SUSPEND enabled for the guest */ -#define KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED 4 - /* VM counter offset */ -#define KVM_ARCH_FLAG_VM_COUNTER_OFFSET 5 - /* Timer PPIs made immutable */ -#define KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE 6 - /* Initial ID reg values loaded */ -#define KVM_ARCH_FLAG_ID_REGS_INITIALIZED 7 - /* Fine-Grained UNDEF initialised */ -#define KVM_ARCH_FLAG_FGU_INITIALIZED 8 - /* SVE exposed to guest */ -#define KVM_ARCH_FLAG_GUEST_HAS_SVE 9 - /* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */ -#define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS 10 - /* Unhandled SEAs are taken to userspace */ -#define KVM_ARCH_FLAG_EXIT_SEA 11 unsigned long flags; /* VM-wide vCPU feature set */ @@ -835,13 +846,6 @@ extern s64 kvm_nvhe_sym(hyp_physvirt_offset); extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS]; #define hyp_cpu_logical_map CHOOSE_NVHE_SYM(hyp_cpu_logical_map) -struct vcpu_reset_state { - unsigned long pc; - unsigned long r0; - bool be; - bool reset; -}; - struct vncr_tlb; struct kvm_vcpu_arch { @@ -1046,6 +1050,11 @@ struct kvm_vcpu_arch { /* pKVM VCPU setup completed */ #define VCPU_PKVM_FINALIZED __vcpu_single_flag(cflags, BIT(2)) +int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); +bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); + +#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED) + /* Exception pending */ #define PENDING_EXCEPTION __vcpu_single_flag(iflags, BIT(0)) /* @@ -1239,14 +1248,6 @@ struct kvm_vcpu_stat { u64 exits; }; -unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); -int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); -int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); - -unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); - int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events); @@ -1325,6 +1326,14 @@ int __init populate_nv_trap_config(void); void kvm_calculate_traps(struct kvm_vcpu *vcpu); +unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); +int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); +int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); +int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); + +unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu); +int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); + /* MMIO helpers */ void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data); unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); @@ -1508,11 +1517,6 @@ struct kvm *kvm_arch_alloc_vm(void); #define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm) -int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); -bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); - -#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED) - #define kvm_has_mte(kvm) \ (system_supports_mte() && \ test_bit(KVM_ARCH_FLAG_MTE_ENABLED, &(kvm)->arch.flags)) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 387950b19cac..30950014971d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -1263,6 +1263,6 @@ FIELD_PREP(reg##_##field##_MASK, \ SYS_FIELD_VALUE(reg, field, val)) -#endif +#endif /* __ASSEMBLER__ */ #endif /* __ASM_SYSREG_H */ From c7e18ba462536e870b0ff4deca581babcac9135b Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:25 +0200 Subject: [PATCH 373/506] arm64: Share arm64 headers with s390 Allow sharing of arm64 headers with s390 by marking the shared regions or by adding a comment that the whole file is shared. The shared regions are marked with: #if ARM64_S390_COMMON == 1 /* insert shared definitions here */ #endif /* ARM64_S390_COMMON == 1 */ The preprocessor symbol ARM64_S390_COMMON is always defined to 1 for any arm64 build. s390 later will generate its own view of arm64 headers based on the content inside the #if/#endif block. No functional change. Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-11-seiden@linux.ibm.com Signed-off-by: Marc Zyngier --- arch/arm64/Makefile | 3 +++ arch/arm64/include/asm/brk-imm.h | 1 + arch/arm64/include/asm/esr.h | 1 + arch/arm64/include/asm/kvm_arm.h | 1 + arch/arm64/include/asm/kvm_emulate.h | 18 ++++++++++++++++++ arch/arm64/include/asm/kvm_host.h | 13 +++++++++++++ arch/arm64/include/asm/ptrace.h | 4 ++++ arch/arm64/include/asm/sysreg.h | 15 +++++++++++++++ 8 files changed, 56 insertions(+) diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 6b005c8fef70..a2ed76cfdc0a 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -45,6 +45,9 @@ KBUILD_CFLAGS += $(CC_FLAGS_NO_FPU) \ KBUILD_CFLAGS += $(call cc-disable-warning, psabi) KBUILD_AFLAGS += $(compat_vdso) +# Enable all code shared to s390 +KBUILD_CPPFLAGS += -DARM64_S390_COMMON=1 + ifeq ($(call rustc-min-version, 108500),y) KBUILD_RUSTFLAGS += --target=aarch64-unknown-none-softfloat else diff --git a/arch/arm64/include/asm/brk-imm.h b/arch/arm64/include/asm/brk-imm.h index beb42c62b6ac..dd2d153dc0d8 100644 --- a/arch/arm64/include/asm/brk-imm.h +++ b/arch/arm64/include/asm/brk-imm.h @@ -2,6 +2,7 @@ /* * Copyright (C) 2012 ARM Ltd. */ +/* Whole file is shared with s390 */ #ifndef __ASM_BRK_IMM_H #define __ASM_BRK_IMM_H diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index da085c83b57e..2cb73bde7346 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -3,6 +3,7 @@ * Copyright (C) 2013 - ARM Ltd * Author: Marc Zyngier */ +/* Whole file is shared with s390 */ #ifndef __ASM_ESR_H #define __ASM_ESR_H diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 57edd8437200..678f0daed445 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -3,6 +3,7 @@ * Copyright (C) 2012,2013 - ARM Ltd * Author: Marc Zyngier */ +/* Whole file is shared with s390 */ #ifndef __ARM64_KVM_ARM_H__ #define __ARM64_KVM_ARM_H__ diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index b31d50bdf59e..e19d61f5ccd8 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -44,6 +44,7 @@ enum exception_type { bool kvm_condition_valid32(const struct kvm_vcpu *vcpu); void kvm_skip_instr32(struct kvm_vcpu *vcpu); +#if ARM64_S390_COMMON == 1 void kvm_inject_undefined(struct kvm_vcpu *vcpu); void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr); int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr); @@ -75,6 +76,8 @@ static inline int kvm_inject_serror(struct kvm_vcpu *vcpu) void kvm_vcpu_wfi(struct kvm_vcpu *vcpu); +#endif /* ARM64_S390_COMMON == 1 */ + void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu); int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2); int kvm_inject_nested_irq(struct kvm_vcpu *vcpu); @@ -184,6 +187,7 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) *vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT; } +#if ARM64_S390_COMMON == 1 /* * vcpu_get_reg and vcpu_set_reg should always be passed a register number * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on @@ -202,6 +206,8 @@ static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, vcpu_gp_regs(vcpu)[reg_num] = val; } +#endif /* ARM64_S390_COMMON == 1 */ + static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt) { switch (ctxt->regs.pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) { @@ -404,6 +410,7 @@ static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu) return vcpu->arch.fault.disr_el1; } +#if ARM64_S390_COMMON == 1 static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu) { return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK; @@ -481,6 +488,8 @@ static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC; } +#endif /* ARM64_S390_COMMON == 1 */ + static inline bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) { @@ -521,6 +530,7 @@ static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) return ESR_ELx_SYS64_ISS_RT(esr); } +#if ARM64_S390_COMMON == 1 static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) { if (kvm_vcpu_abt_iss1tw(vcpu)) { @@ -545,6 +555,8 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) return kvm_vcpu_dabt_iswrite(vcpu); } +#endif /* ARM64_S390_COMMON == 1 */ + static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) { return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; @@ -586,6 +598,7 @@ static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) return vcpu_read_sys_reg(vcpu, r) & bit; } +#if ARM64_S390_COMMON == 1 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu, unsigned long data, unsigned int len) @@ -661,6 +674,8 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu) vcpu_set_flag((v), e); \ } while (0) +#endif /* ARM64_S390_COMMON == 1 */ + /* * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE * format if E2H isn't set. @@ -767,6 +782,7 @@ static inline void kvm_reset_fpsimd(struct kvm_vcpu *vcpu) memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); } +#if ARM64_S390_COMMON == 1 /* Reset a vcpu's core registers. */ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) { @@ -791,6 +807,8 @@ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) *vcpu_cpsr(vcpu) = pstate; } +#endif /* ARM64_S390_COMMON == 1 */ + /* PSCI reset handling for a vcpu. */ static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu, struct vcpu_reset_state *reset_state) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 5261c21a437c..85ad32883f14 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -39,6 +39,7 @@ #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS +#if ARM64_S390_COMMON == 1 #define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) @@ -96,6 +97,8 @@ struct vcpu_reset_state { bool reset; }; +#endif /* ARM64_S390_COMMON == 1 */ + #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -955,6 +958,7 @@ struct kvm_vcpu_arch { pid_t pid; }; +#if ARM64_S390_COMMON == 1 /* * Each 'flag' is composed of a comma-separated triplet: * @@ -1113,6 +1117,8 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); /* KVM is currently emulating an L2 to L1 exception */ #define IN_NESTED_EXCEPTION __vcpu_single_flag(sflags, BIT(9)) +#endif /* ARM64_S390_COMMON == 1 */ + #define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) #define vcpu_sve_zcr_elx(vcpu) \ @@ -1326,6 +1332,7 @@ int __init populate_nv_trap_config(void); void kvm_calculate_traps(struct kvm_vcpu *vcpu); +#if ARM64_S390_COMMON == 1 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); @@ -1341,6 +1348,8 @@ unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); int kvm_handle_mmio_return(struct kvm_vcpu *vcpu); int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa); +#endif /* ARM64_S390_COMMON == 1 */ + /* * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event, * arrived in guest context. For arm64, any event that arrives while a vCPU is @@ -1533,11 +1542,15 @@ static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature) return test_bit(feature, ka->vcpu_features); } +#if ARM64_S390_COMMON == 1 + #define kvm_vcpu_has_feature(k, f) __vcpu_has_feature(&(k)->arch, (f)) #define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f)) #define kvm_vcpu_initialized(v) vcpu_get_flag(v, VCPU_INITIALIZED) +#endif /* ARM64_S390_COMMON == 1 */ + int kvm_trng_call(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM extern phys_addr_t hyp_mem_base; diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 49a83a0636b9..4d88ed5cc425 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -26,6 +26,8 @@ #define GIC_PRIO_PSR_I_SET GICV3_PRIO_PSR_I_SET +#if ARM64_S390_COMMON == 1 + /* Additional SPSR bits not exposed in the UABI */ #define PSR_MODE_THREAD_BIT (1 << 0) #define PSR_IL_BIT (1 << 20) @@ -66,6 +68,8 @@ #define PSR_AA32_IT_MASK 0x0600fc00 /* If-Then execution state mask */ #define PSR_AA32_GE_MASK 0x000f0000 +#endif /* ARM64_S390_COMMON == 1 */ + #ifdef CONFIG_CPU_BIG_ENDIAN #define PSR_AA32_ENDSTATE PSR_AA32_E_BIT #else diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 30950014971d..bef66021b1cf 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -16,6 +16,8 @@ #include +#if ARM64_S390_COMMON == 1 + /* * ARMv8 ARM reserves the following encoding for system registers: * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview", @@ -50,6 +52,8 @@ #define sys_reg_CRm(id) (((id) >> CRm_shift) & CRm_mask) #define sys_reg_Op2(id) (((id) >> Op2_shift) & Op2_mask) +#endif /* ARM64_S390_COMMON == 1 */ + #ifndef CONFIG_BROKEN_GAS_INST #ifdef __ASSEMBLER__ @@ -123,6 +127,8 @@ #define GSB_SYS_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 0, 31) #define GSB_ACK_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 1, 31) +#if ARM64_S390_COMMON == 1 + /* Data cache zero operations */ #define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2) #define SYS_DC_IGSW sys_insn(1, 0, 7, 6, 4) @@ -832,6 +838,8 @@ #define SCTLR_ELx_A (BIT(1)) #define SCTLR_ELx_M (BIT(0)) +#endif /* ARM64_S390_COMMON == 1 */ + #ifdef CONFIG_CPU_BIG_ENDIAN #define ENDIAN_SET_EL2 SCTLR_ELx_EE #else @@ -866,6 +874,7 @@ SCTLR_EL1_LSMAOE | SCTLR_EL1_nTLSMD | SCTLR_EL1_EIS | \ SCTLR_EL1_TSCXT | SCTLR_EL1_EOS) +#if ARM64_S390_COMMON == 1 /* MAIR_ELx memory attributes (used by Linux) */ #define MAIR_ATTR_DEVICE_nGnRnE UL(0x00) #define MAIR_ATTR_DEVICE_nGnRE UL(0x04) @@ -1102,6 +1111,8 @@ #define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29) #define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0) +#endif /* ARM64_S390_COMMON == 1 */ + #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) #define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn) @@ -1251,6 +1262,8 @@ par; \ }) +#if ARM64_S390_COMMON == 1 + #define SYS_FIELD_VALUE(reg, field, val) reg##_##field##_##val #define SYS_FIELD_GET(reg, field, val) \ @@ -1263,6 +1276,8 @@ FIELD_PREP(reg##_##field##_MASK, \ SYS_FIELD_VALUE(reg, field, val)) +#endif /* ARM64_S390_COMMON == 1 */ + #endif /* __ASSEMBLER__ */ #endif /* __ASM_SYSREG_H */ From dcb776bfcccc9662001a75b99dbfbef55f46eb31 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:26 +0200 Subject: [PATCH 374/506] KVM: arm64: Share arm64 code with s390 Mark functions that s390 can use to implement arm on s390 as shared functions. No functional change. Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-12-seiden@linux.ibm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 3 +++ arch/arm64/kvm/guest.c | 6 ++++++ arch/arm64/kvm/mmio.c | 2 ++ 3 files changed, 11 insertions(+) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..e22ecd1a65f4 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1603,6 +1603,7 @@ static unsigned long system_supported_vcpu_features(void) return features; } +#if ARM64_S390_COMMON == 1 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, const struct kvm_vcpu_init *init) { @@ -1656,6 +1657,8 @@ static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu, KVM_VCPU_MAX_FEATURES); } +#endif /* ARM64_S390_COMMON == 1 */ + static int kvm_setup_vcpu(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm; diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 773f6c8e5026..0259c515997b 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -62,6 +62,7 @@ const struct kvm_stats_header kvm_vcpu_stats_header = { sizeof(kvm_vcpu_stats_desc), }; +#if ARM64_S390_COMMON == 1 static bool core_reg_offset_is_vreg(u64 off) { return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && @@ -306,6 +307,8 @@ static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return err; } +#endif /* ARM64_S390_COMMON == 1 */ + #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) @@ -543,6 +546,7 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) return -EINVAL; } +#if ARM64_S390_COMMON == 1 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, u64 __user *uindices) { @@ -591,6 +595,8 @@ static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) return copy_core_reg_indices(vcpu, NULL); } +#endif /* ARM64_S390_COMMON == 1 */ + static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) { const unsigned int slices = vcpu_sve_slices(vcpu); diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index d1c3a352d5a2..3eaf687eae07 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -10,6 +10,7 @@ #include "trace.h" +#if ARM64_S390_COMMON == 1 void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data) { void *datap = NULL; @@ -258,3 +259,4 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) run->exit_reason = KVM_EXIT_MMIO; return 0; } +#endif /* ARM64_S390_COMMON == 1 */ From 6c29285e78b5e4811145ebda382ac0ba732ce56a Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Mon, 28 Sep 2026 10:54:50 +0100 Subject: [PATCH 375/506] KVM: arm64: Fix MMFR0 TGRAN advertisement for pVMs Protected VM ID register emulation leaves unlisted fields at 0. This creates two issues for TGRAN: * TGRAN16: 0 means non-implemented. It prevents 16KB protected VMs from booting even when the hardware supports it. * TGRAN4|TGRAN64: 0 means implemented. This may falsely advertise the feature even when the hardware does not support it. Advertise all the TGRAN fields in pvmid_aa64mmfr0 so that the hardware support is properly propagated. Cc: stable@vger.kernel.org Fixes: 6c30bfb18d0b ("KVM: arm64: Add handlers for protected VM System Registers") Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260928095454.3255918-2-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/sys_regs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8758c6801776..ec469d96516a 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -121,6 +121,9 @@ static const struct pvm_ftr_bits pvmid_aa64mmfr0[] = { MAX_FEAT(ID_AA64MMFR0_EL1, SNSMEM, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, BIGENDEL0, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, EXS, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN4, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN16, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN64, IMP), FEAT_END }; From 10c5cc50f056a6425305557063ad70c3eb0e686b Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Mon, 28 Sep 2026 10:54:51 +0100 Subject: [PATCH 376/506] KVM: arm64: Pass kvm_s2_fault_desc to fault_supports_stage2_huge_mapping() Pass the fault descriptor to fault_supports_stage2_huge_mapping() instead of passing memslot and hva separately so it can access other s2fd members such as vcpu. This implies to pass that same s2fd argument to transparent_hugepage_adjust(). No functional change intended. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260928095454.3255918-3-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9ba86450fe4a..9dfaf4f277b5 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1353,12 +1353,21 @@ static void kvm_send_hwpoison_signal(unsigned long address, short lsb) send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current); } -static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, - unsigned long hva, +struct kvm_s2_fault_desc { + struct kvm_vcpu *vcpu; + phys_addr_t fault_ipa; + struct kvm_s2_trans *nested; + struct kvm_memory_slot *memslot; + unsigned long hva; +}; + +static bool fault_supports_stage2_huge_mapping(const struct kvm_s2_fault_desc *s2fd, unsigned long map_size) { - gpa_t gpa_start; + struct kvm_memory_slot *memslot = s2fd->memslot; + unsigned long hva = s2fd->hva; hva_t uaddr_start, uaddr_end; + gpa_t gpa_start; size_t size; /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ @@ -1427,9 +1436,9 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, * Returns the size of the mapping. */ static long -transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, - unsigned long hva, kvm_pfn_t *pfnp, gfn_t *gfnp) +transparent_hugepage_adjust(const struct kvm_s2_fault_desc *s2fd, kvm_pfn_t *pfnp, gfn_t *gfnp) { + struct kvm *kvm = s2fd->vcpu->kvm; kvm_pfn_t pfn = *pfnp; gfn_t gfn = *gfnp; @@ -1438,8 +1447,8 @@ transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, * sure that the HVA and IPA are sufficiently aligned and that the * block map is contained within the memslot. */ - if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { - int sz = get_user_mapping_size(kvm, hva); + if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) { + int sz = get_user_mapping_size(kvm, s2fd->hva); if (sz < 0) return sz; @@ -1597,14 +1606,6 @@ static enum kvm_pgtable_prot adjust_nested_exec_perms(struct kvm *kvm, return prot; } -struct kvm_s2_fault_desc { - struct kvm_vcpu *vcpu; - phys_addr_t fault_ipa; - struct kvm_s2_trans *nested; - struct kvm_memory_slot *memslot; - unsigned long hva; -}; - static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) { bool write_fault, exec_fault; @@ -1791,7 +1792,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, switch (vma_shift) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SHIFT: - if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PUD_SIZE)) + if (fault_supports_stage2_huge_mapping(s2fd, PUD_SIZE)) break; fallthrough; #endif @@ -1799,7 +1800,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, vma_shift = PMD_SHIFT; fallthrough; case PMD_SHIFT: - if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PMD_SIZE)) + if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) break; fallthrough; case CONT_PTE_SHIFT: @@ -2049,9 +2050,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, if (perm_fault_granule > PAGE_SIZE) { mapping_size = perm_fault_granule; } else { - mapping_size = transparent_hugepage_adjust(kvm, s2fd->memslot, - s2fd->hva, &pfn, - &gfn); + mapping_size = transparent_hugepage_adjust(s2fd, &pfn, &gfn); if (mapping_size < 0) { ret = mapping_size; goto out_unlock; From 80b0b86dc5e4344e62a2fd3ee43d0ef816fe9122 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 28 Sep 2026 10:54:52 +0100 Subject: [PATCH 377/506] KVM: arm64: Use kvm_s2_fault_vma_info in gmem_abort() gmem_abort() maps at s2fd->fault_ipa, which HPFAR_EL2 holds at 4K granularity whatever the page size. The generic page-table code aligns it, but pkvm_pgtable_stage2_map() looks up existing mappings over [addr, addr + size), so on a pKVM host with pages larger than 4K a guest_memfd-backed guest that faults past the first 4K of a page can find its neighbour's mapping, get -EAGAIN and take the same fault forever. The memory fault exit reports s2fd->fault_ipa too, which for a nested guest's fault is the nested IPA rather than the canonical one kvm_gmem_get_pfn() failed on. Take the addresses from kvm_s2_fault_vma_info instead, as user_mem_abort() does, and report the canonical gfn in the memory fault exit. kvm_s2_fault_get_vma_info() itself isn't called: a guest_memfd memslot's userspace_addr doesn't need to be backed by a VMA. Fixes: a7b57e0995927 ("KVM: arm64: Handle guest_memfd-backed guest page faults") Reported-by: Sashiko Closes: https://lore.kernel.org/r/20260913175105.A57AC1F000FF@smtp.kernel.org Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Signed-off-by: Vincent Donnefort Link: https://patch.msgid.link/20260928095454.3255918-4-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 67 ++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9dfaf4f277b5..658ac6a09e7b 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1606,6 +1606,24 @@ static enum kvm_pgtable_prot adjust_nested_exec_perms(struct kvm *kvm, return prot; } +struct kvm_s2_fault_vma_info { + unsigned long mmu_seq; + long vma_pagesize; + vm_flags_t vm_flags; + unsigned long max_map_size; + struct page *page; + kvm_pfn_t pfn; + gfn_t gfn; + bool device; + bool mte_allowed; + bool is_vma_cacheable; + bool map_writable; + bool map_non_cacheable; +}; + +static gfn_t get_canonical_gfn(const struct kvm_s2_fault_desc *s2fd, + const struct kvm_s2_fault_vma_info *s2vi); + static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) { bool write_fault, exec_fault; @@ -1613,12 +1631,10 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt; - unsigned long mmu_seq; - struct page *page; + struct kvm_s2_fault_vma_info s2vi = {}; struct kvm *kvm = s2fd->vcpu->kvm; void *memcache = NULL; - kvm_pfn_t pfn; - gfn_t gfn; + gfn_t canonical_gfn; int ret; if (!perm_fault) { @@ -1628,24 +1644,23 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) return ret; } - if (s2fd->nested) - gfn = kvm_s2_trans_output(s2fd->nested) >> PAGE_SHIFT; - else - gfn = s2fd->fault_ipa >> PAGE_SHIFT; + s2vi.vma_pagesize = PAGE_SIZE; + s2vi.gfn = ALIGN_DOWN(s2fd->fault_ipa, s2vi.vma_pagesize) >> PAGE_SHIFT; + canonical_gfn = get_canonical_gfn(s2fd, &s2vi); write_fault = kvm_is_write_fault(s2fd->vcpu); exec_fault = kvm_vcpu_trap_is_exec_fault(s2fd->vcpu); VM_WARN_ON_ONCE(write_fault && exec_fault); - mmu_seq = kvm->mmu_invalidate_seq; + s2vi.mmu_seq = kvm->mmu_invalidate_seq; /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ smp_rmb(); - ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, &s2vi.page, NULL); if (ret) { - kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, - write_fault, exec_fault, false); + kvm_prepare_memory_fault_exit(s2fd->vcpu, gfn_to_gpa(canonical_gfn), + s2vi.vma_pagesize, write_fault, exec_fault, false); return ret; } @@ -1662,7 +1677,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) prot = adjust_nested_exec_perms(kvm, s2fd->nested, prot); kvm_fault_lock(kvm); - if (mmu_invalidate_retry(kvm, mmu_seq)) { + if (mmu_invalidate_retry(kvm, s2vi.mmu_seq)) { ret = -EAGAIN; goto out_unlock; } @@ -1673,39 +1688,25 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) * PTE, which will be preserved. */ prot &= ~KVM_NV_GUEST_MAP_SZ; - ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, s2fd->fault_ipa, + ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, gfn_to_gpa(s2vi.gfn), prot, flags); } else { - ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE, - __pfn_to_phys(pfn), prot, + ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(s2vi.gfn), + s2vi.vma_pagesize, + __pfn_to_phys(s2vi.pfn), prot, memcache, flags); } out_unlock: - kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W); + kvm_release_faultin_page(kvm, s2vi.page, !!ret, prot & KVM_PGTABLE_PROT_W); kvm_fault_unlock(kvm); if ((prot & KVM_PGTABLE_PROT_W) && !ret) - mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn); + mark_page_dirty_in_slot(kvm, s2fd->memslot, canonical_gfn); return ret != -EAGAIN ? ret : 0; } -struct kvm_s2_fault_vma_info { - unsigned long mmu_seq; - long vma_pagesize; - vm_flags_t vm_flags; - unsigned long max_map_size; - struct page *page; - kvm_pfn_t pfn; - gfn_t gfn; - bool device; - bool mte_allowed; - bool is_vma_cacheable; - bool map_writable; - bool map_non_cacheable; -}; - static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) { unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; From 0cdcc1039200b3c4fbc2ae3f9b5c1be1bc9155e8 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Mon, 28 Sep 2026 10:54:53 +0100 Subject: [PATCH 378/506] KVM: arm64: Use kvm_s2_fault_vma_info in pkvm_mem_abort() To paraphrase Marc in commit 08f97454b7fa ("KVM: arm64: Fix protected mode handling of pages larger than 4kB"), pkvm_pgtable_stage2_map() assumes the address passed as a parameter is aligned to the size of the intended mapping, while HPFAR_EL2 gives the IPA minus the bottom 12 bits, regardless of the system page size configuration. To fix this alignment, bring support for kvm_s2_fault_vma_info in pkvm_mem_abort() and use its members where possible. They do contain the page-alignment we need to fix the fault on system with pages larger than 4K. Also, add a check at the start of pkvm_pgtable_stage2_map(), it does not support !PAGE_ALIGNED arguments. Fixes: ea03466e806f ("KVM: arm64: Handle aborts from protected VMs") Cc: stable@vger.kernel.org Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260928095454.3255918-5-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 27 ++++++++++++++++++--------- arch/arm64/kvm/pkvm.c | 3 +++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 658ac6a09e7b..9ee5a5c122cb 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1374,9 +1374,11 @@ static bool fault_supports_stage2_huge_mapping(const struct kvm_s2_fault_desc *s if (map_size == PAGE_SIZE) return true; - /* pKVM only supports PMD_SIZE huge-mappings */ - if (is_protected_kvm_enabled() && map_size != PMD_SIZE) - return false; + /* pKVM only supports PMD_SIZE huge-mappings for non-protected VMs */ + if (is_protected_kvm_enabled()) { + if (vcpu_is_protected(s2fd->vcpu) || map_size != PMD_SIZE) + return false; + } size = memslot->npages * PAGE_SIZE; @@ -1621,6 +1623,9 @@ struct kvm_s2_fault_vma_info { bool map_non_cacheable; }; +static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_vma_info *s2vi); + static gfn_t get_canonical_gfn(const struct kvm_s2_fault_desc *s2fd, const struct kvm_s2_fault_vma_info *s2vi); @@ -1710,12 +1715,12 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) { unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; + struct kvm_s2_fault_vma_info s2vi = {}; struct kvm_vcpu *vcpu = s2fd->vcpu; struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt; struct mm_struct *mm = current->mm; struct kvm *kvm = vcpu->kvm; void *hyp_memcache; - struct page *page; int ret; hyp_memcache = get_mmu_memcache(vcpu); @@ -1723,12 +1728,16 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) if (ret) return -ENOMEM; + ret = kvm_s2_fault_get_vma_info(s2fd, &s2vi); + if (ret) + return ret; + ret = account_locked_vm(mm, 1, true); if (ret) return ret; mmap_read_lock(mm); - ret = pin_user_pages(s2fd->hva, 1, flags, &page); + ret = pin_user_pages(s2fd->hva, 1, flags, &s2vi.page); mmap_read_unlock(mm); if (ret == -EHWPOISON) { @@ -1738,7 +1747,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) } else if (ret != 1) { ret = -EFAULT; goto dec_account; - } else if (!folio_test_swapbacked(page_folio(page))) { + } else if (!folio_test_swapbacked(page_folio(s2vi.page))) { /* * We really can't deal with page-cache pages returned by GUP * because (a) we may trigger writeback of a page for which we @@ -1758,8 +1767,8 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) } write_lock(&kvm->mmu_lock); - ret = pkvm_pgtable_stage2_map(pgt, s2fd->fault_ipa, PAGE_SIZE, - page_to_phys(page), KVM_PGTABLE_PROT_RWX, + ret = pkvm_pgtable_stage2_map(pgt, gfn_to_gpa(s2vi.gfn), PAGE_SIZE, + page_to_phys(s2vi.page), KVM_PGTABLE_PROT_RWX, hyp_memcache, 0); write_unlock(&kvm->mmu_lock); if (ret) { @@ -1770,7 +1779,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) return 0; unpin: - unpin_user_pages(&page, 1); + unpin_user_page(s2vi.page); dec_account: account_locked_vm(mm, 1, false); return ret; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4c6e4bec12..c298a5b9b12a 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -414,6 +414,9 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 end = addr + size; int ret; + if (WARN_ON_ONCE(!PAGE_ALIGNED(addr | size))) + return -EINVAL; + lockdep_assert_held_write(&kvm->mmu_lock); mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1); From 3cf59906cdc6ed59672ab7e937ff54cdc67b0843 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 19 Sep 2026 13:16:56 +0100 Subject: [PATCH 379/506] KVM: arm64: vgic-v5: Correctly handle host ISTE __le32 conversion The vgic-v5 defines the host ISTE (h_iste) as a __le32, but uses get_user/put_user on a userspace buffer declared as u32 *. This leads to sparse having yet another fit. Instead, define h_iste as a u32, and perform the conversion at the point of doing the access on the architectural state. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202609191215.to1MBYak-lkp@intel.com/ Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-v5-tables.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index 2c8d0f360506..d577e485ebe9 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -1457,7 +1457,6 @@ static int vgic_v5_get_lpi_ist_desc(struct kvm *kvm, static int vgic_v5_save_linear_ist(const struct vgic_v5_ist_desc *ist, u32 __user *uaddr, size_t nr_entries) { - __le32 h_iste; size_t index; int ret; @@ -1466,8 +1465,9 @@ static int vgic_v5_save_linear_ist(const struct vgic_v5_ist_desc *ist, for (index = 0; index < nr_entries; index++) { __le32 *h_iste_addr = ist->base + index * ist->iste_size; + u32 h_iste; - h_iste = READ_ONCE(*h_iste_addr); + h_iste = le32_to_cpu(READ_ONCE(*h_iste_addr)); ret = put_user(h_iste, uaddr); if (ret) return ret; @@ -1490,7 +1490,6 @@ static int vgic_v5_save_two_level_ist(const struct vgic_v5_ist_desc *ist, struct vgic_v5_two_level_ist_shape shape; size_t h_l1_index, h_l2_index; void *h_l2_ist_base; - __le32 h_iste; int ret; shape = vgic_v5_two_level_ist_shape(ist); @@ -1517,8 +1516,8 @@ static int vgic_v5_save_two_level_ist(const struct vgic_v5_ist_desc *ist, shape.l2_entries * ist->iste_size); for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) { - h_iste = *(__le32 *)(h_l2_ist_base + - h_l2_index * ist->iste_size); + void *h_iste_addr = h_l2_ist_base + h_l2_index * ist->iste_size; + u32 h_iste = le32_to_cpu(*(__le32 *)h_iste_addr); ret = put_user(h_iste, uaddr); if (ret) @@ -1710,19 +1709,19 @@ static int vgic_v5_restore_linear_ist(struct kvm *kvm, u32 __user *uaddr, size_t nr_entries, u32 intid_type) { - __le32 h_iste; size_t index; int ret; for (index = 0; index < nr_entries; index++) { void *h_iste_addr = ist->base + index * ist->iste_size; + u32 h_iste; ret = get_user(h_iste, uaddr); if (ret) return ret; ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr, - h_iste, index, intid_type); + cpu_to_le32(h_iste), index, intid_type); if (ret) return ret; @@ -1744,7 +1743,6 @@ static int vgic_v5_restore_two_level_ist(struct kvm *kvm, struct vgic_v5_two_level_ist_shape shape; size_t h_l1_index, h_l2_index; void *h_l2_ist_base; - __le32 h_iste; int ret; shape = vgic_v5_two_level_ist_shape(ist); @@ -1771,13 +1769,14 @@ static int vgic_v5_restore_two_level_ist(struct kvm *kvm, void *h_iste_addr = h_l2_ist_base + h_l2_index * ist->iste_size; u32 intid = h_l1_index * shape.l2_entries + h_l2_index; + u32 h_iste; ret = get_user(h_iste, uaddr); if (ret) return ret; ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr, - h_iste, intid, + cpu_to_le32(h_iste), intid, intid_type); if (ret) return ret; From deaf98ff52ed9226dc84d7abb315d67fc6398c12 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 28 Sep 2026 15:07:02 +0200 Subject: [PATCH 380/506] KVM: Introduce file_to_kvm_() infrastructure Add a macro mechanism that generates an opt-in and arch-namespaced file_to_kvm_() to convert a file reference to a kvm object if the file handle represents a kvm handle. Activate for x86 and for s390 (native KVM only). Suggested-by: Sean Christopherson Signed-off-by: Steffen Eiden Reviewed-by: Claudio Imbrenda Acked-by: Janosch Frank Acked-by: Sean Christopherson Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/kvm_host_s390.h | 2 ++ arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/Makefile | 1 + include/linux/kvm_host.h | 12 ++++++++++++ virt/kvm/kvm_main.c | 11 +++++++++++ 5 files changed, 28 insertions(+) diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h index cd692f8fb764..8a7eed5847e1 100644 --- a/arch/s390/include/asm/kvm_host_s390.h +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -27,6 +27,8 @@ #include #include +#define kvm_file_to_kvm_arch s390 + #define KVM_HAVE_MMU_RWLOCK #define KVM_MAX_VCPUS 255 diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 683bb8bf43a9..e460073ff176 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -44,6 +44,8 @@ #include +#define kvm_file_to_kvm_arch x86 + #define __KVM_HAVE_ARCH_VCPU_DEBUGFS /* diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 0474604ab8a1..e6d395ede9e2 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -61,6 +61,7 @@ exports_grep_trailer := --include='*.[ch]' -nrw $(srctree)/virt/kvm $(srctree)/a -e kvm_page_track_unregister_notifier \ -e kvm_write_track_add_gfn \ -e kvm_write_track_remove_gfn \ + -e kvm_file_to_kvm_fn \ -e kvm_get_kvm \ -e kvm_get_kvm_safe \ -e kvm_put_kvm diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..e840ceda9f16 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1082,6 +1082,18 @@ void kvm_get_kvm(struct kvm *kvm); bool kvm_get_kvm_safe(struct kvm *kvm); void kvm_put_kvm(struct kvm *kvm); bool file_is_kvm(struct file *file); + +/* + * Architectures define kvm_file_to_kvm_arch to + * to get a typed, arch-namespaced helper: + * + * struct kvm *file_to_kvm_(struct file *file) + */ +#ifdef kvm_file_to_kvm_arch +#define kvm_file_to_kvm_fn CONCATENATE(file_to_kvm_, kvm_file_to_kvm_arch) +struct kvm *kvm_file_to_kvm_fn(struct file *file); +#endif /* kvm_file_to_kvm_arch */ + void kvm_put_kvm_no_destroy(struct kvm *kvm); static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..0172ca94f82e 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5496,6 +5496,17 @@ bool file_is_kvm(struct file *file) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(file_is_kvm); +#ifdef kvm_file_to_kvm_arch +struct kvm *kvm_file_to_kvm_fn(struct file *file) +{ + if (!file || file->f_op != &kvm_vm_fops) + return NULL; + + return file->private_data; +} +EXPORT_SYMBOL_GPL(kvm_file_to_kvm_fn); +#endif + static int kvm_dev_ioctl_create_vm(unsigned long type) { char fdname[ITOA_MAX_LEN + 1]; From 8b5cb9769ca70a4f08c1d3c390285632995b248f Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 28 Sep 2026 15:07:03 +0200 Subject: [PATCH 381/506] KVM: Add file back-pointer to struct kvm Add kvm->file as a no-reference back-pointer to the VM file in struct kvm. The pointer is set at VM creation time and cleared under WRITE_ONCE() in kvm_vm_release() before the file is freed. Callers storing the file must take their own reference. Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Steffen Eiden Reviewed-by: Claudio Imbrenda Acked-by: Janosch Frank Acked-by: Sean Christopherson Signed-off-by: Christian Borntraeger --- include/linux/kvm_host.h | 6 ++++++ virt/kvm/kvm_main.c | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index e840ceda9f16..943b108f01cd 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -784,6 +784,12 @@ struct kvm { * kvm_swap_active_memslots(). */ struct mutex slots_arch_lock; + /* + * Back-reference to the VM file for subsystems (e.g., VFIO). Holds no + * reference to avoid pinning the VM. Callers storing the file must + * take their own reference. + */ + struct file *file; struct mm_struct *mm; /* userspace tied to this vm */ unsigned long nr_memslot_pages; /* The two memslot sets - active and inactive (per address space) */ diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 0172ca94f82e..bab71d398236 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1352,6 +1352,8 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) kvm_irqfd_release(kvm); + WRITE_ONCE(kvm->file, NULL); + kvm_put_kvm(kvm); return 0; } @@ -5532,6 +5534,8 @@ static int kvm_dev_ioctl_create_vm(unsigned long type) goto put_kvm; } + kvm->file = file; + /* * Don't call kvm_put_kvm anymore at this point; file->f_op is * already set, with ->release() being kvm_vm_release(). In error From 77037cf77a3c1b575e514a483527d88c12bb682c Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 28 Sep 2026 15:07:04 +0200 Subject: [PATCH 382/506] KVM: x86: Use file_to_kvm_x86() in SEV Use the new arch-namespaced helper instead of open-coding the file check and private_data cast separately. Suggested-by: Sean Christopherson Signed-off-by: Steffen Eiden Reviewed-by: Claudio Imbrenda Acked-by: Janosch Frank Acked-by: Sean Christopherson Signed-off-by: Christian Borntraeger --- arch/x86/kvm/svm/sev.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5705723f1f41..6b0eacde4f06 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2145,10 +2145,10 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (fd_empty(f)) return -EBADF; - if (!file_is_kvm(fd_file(f))) + source_kvm = file_to_kvm_x86(fd_file(f)); + if (!source_kvm) return -EBADF; - source_kvm = fd_file(f)->private_data; ret = sev_lock_two_vms(kvm, source_kvm); if (ret) return ret; @@ -2866,10 +2866,10 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (fd_empty(f)) return -EBADF; - if (!file_is_kvm(fd_file(f))) + source_kvm = file_to_kvm_x86(fd_file(f)); + if (!source_kvm) return -EBADF; - source_kvm = fd_file(f)->private_data; ret = sev_lock_two_vms(kvm, source_kvm); if (ret) return ret; From 5c3ea6d3e836debc1f7f5518a5ec0c7e058fb04a Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 28 Sep 2026 15:07:05 +0200 Subject: [PATCH 383/506] KVM/vfio: Use file-based reference counting for KVM Replace manual module reference counting with file-based reference counting for KVM integration. Previously, VFIO used symbol_get() to obtain function pointers for kvm_get_kvm_safe() and kvm_put_kvm(), then manually tracked module references through these symbols. This approach required storing the put_kvm function pointer in each device and carefully managing symbol references. Pass struct file pointers instead of struct kvm pointers throughout the VFIO-KVM interface, leveraging the kernel's existing file reference counting via get_file()/get_file_active() and fput(). Convert the x86 page-track API and update s390 vfio to use file_to_kvm_(). This simplifies the code. Suggested-by: Jason Gunthorpe Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Steffen Eiden Acked-by: Alex Williamson Reviewed-by: Claudio Imbrenda Acked-by: Janosch Frank Acked-by: Sean Christopherson [borntraeger@linux.ibm.com remove _safe from vfio_device_get_kvm_safe] Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/kvm_host_s390.h | 2 +- arch/s390/kvm/s390/pci.c | 9 ++++- arch/x86/include/asm/kvm_page_track.h | 10 +++-- arch/x86/kvm/mmu/page_track.c | 22 +++++++---- drivers/s390/crypto/vfio_ap_ops.c | 20 +++++++--- drivers/vfio/device_cdev.c | 2 +- drivers/vfio/group.c | 13 +++++- drivers/vfio/vfio.h | 14 +++---- drivers/vfio/vfio_main.c | 57 +++++++++------------------ include/linux/vfio.h | 5 +-- virt/kvm/vfio.c | 13 ++++-- 11 files changed, 90 insertions(+), 77 deletions(-) diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h index 8a7eed5847e1..9519b8028b10 100644 --- a/arch/s390/include/asm/kvm_host_s390.h +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -722,7 +722,7 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} void kvm_arch_free_vm(struct kvm *kvm); struct zpci_kvm_hook { - int (*kvm_register)(void *opaque, struct kvm *kvm); + int (*kvm_register)(void *opaque, struct file *kvm_file); void (*kvm_unregister)(void *opaque); }; diff --git a/arch/s390/kvm/s390/pci.c b/arch/s390/kvm/s390/pci.c index 82892e1e03d9..d79bd3bdc68e 100644 --- a/arch/s390/kvm/s390/pci.c +++ b/arch/s390/kvm/s390/pci.c @@ -498,17 +498,22 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev) * available, enable them and let userspace indicate whether or not they will * be used (specify SHM bit to disable). */ -static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm) +static int kvm_s390_pci_register_kvm(void *opaque, struct file *kvm_file) { struct zpci_dev *zdev = opaque; + struct kvm *kvm; int rc; if (!zdev) return -EINVAL; + kvm = file_to_kvm_s390(kvm_file); + if (!kvm) + return -ENOENT; + mutex_lock(&zdev->kzdev_lock); - if (zdev->kzdev || zdev->gisa != 0 || !kvm) { + if (zdev->kzdev || zdev->gisa != 0) { mutex_unlock(&zdev->kzdev_lock); return -EINVAL; } diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index 3d040741044b..ea885dd5c9af 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h @@ -44,13 +44,15 @@ struct kvm_page_track_notifier_node { struct kvm_page_track_notifier_node *node); }; -int kvm_page_track_register_notifier(struct kvm *kvm, +struct file; + +int kvm_page_track_register_notifier(struct file *file, struct kvm_page_track_notifier_node *n); -void kvm_page_track_unregister_notifier(struct kvm *kvm, +void kvm_page_track_unregister_notifier(struct file *file, struct kvm_page_track_notifier_node *n); -int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn); -int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn); +int kvm_write_track_add_gfn(struct file *file, gfn_t gfn); +int kvm_write_track_remove_gfn(struct file *file, gfn_t gfn); #else /* * Allow defining a node in a structure even if page tracking is disabled, e.g. diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 7e8195a311bb..f12558dfcd81 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -16,6 +16,8 @@ #include #include +#include + #include "mmu.h" #include "mmu_internal.h" #include "page_track.h" @@ -237,10 +239,11 @@ static int kvm_enable_external_write_tracking(struct kvm *kvm) * register the notifier so that event interception for the tracked guest * pages can be received. */ -int kvm_page_track_register_notifier(struct kvm *kvm, +int kvm_page_track_register_notifier(struct file *file, struct kvm_page_track_notifier_node *n) { struct kvm_page_track_notifier_head *head; + struct kvm *kvm = file_to_kvm_x86(file); int r; if (!kvm || kvm->mm != current->mm) @@ -252,7 +255,7 @@ int kvm_page_track_register_notifier(struct kvm *kvm, return r; } - kvm_get_kvm(kvm); + get_file(file); head = &kvm->arch.track_notifier_head; @@ -267,10 +270,11 @@ EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); * stop receiving the event interception. It is the opposed operation of * kvm_page_track_register_notifier(). */ -void kvm_page_track_unregister_notifier(struct kvm *kvm, +void kvm_page_track_unregister_notifier(struct file *file, struct kvm_page_track_notifier_node *n) { struct kvm_page_track_notifier_head *head; + struct kvm *kvm = file_to_kvm_x86(file); head = &kvm->arch.track_notifier_head; @@ -279,7 +283,7 @@ void kvm_page_track_unregister_notifier(struct kvm *kvm, write_unlock(&kvm->mmu_lock); synchronize_srcu(&head->track_srcu); - kvm_put_kvm(kvm); + fput(file); } EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); @@ -336,11 +340,12 @@ void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot) * add guest page to the tracking pool so that corresponding access on that * page will be intercepted. * - * @kvm: the guest instance we are interested in. + * @file: the VM file of the guest instance we are interested in. * @gfn: the guest page. */ -int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn) +int kvm_write_track_add_gfn(struct file *file, gfn_t gfn) { + struct kvm *kvm = file_to_kvm_x86(file); struct kvm_memory_slot *slot; int idx; @@ -366,11 +371,12 @@ EXPORT_SYMBOL_GPL(kvm_write_track_add_gfn); * remove the guest page from the tracking pool which stops the interception * of corresponding access on that page. * - * @kvm: the guest instance we are interested in. + * @file: the VM file of the guest instance we are interested in. * @gfn: the guest page. */ -int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn) +int kvm_write_track_remove_gfn(struct file *file, gfn_t gfn) { + struct kvm *kvm = file_to_kvm_x86(file); struct kvm_memory_slot *slot; int idx; diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 940c0ff668be..556e643244f2 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1822,17 +1822,27 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { /** * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed - * to manage AP resources for the guest whose state is represented by @kvm + * to manage AP resources for the guest whose state is represented by + * @kvm_file * * @matrix_mdev: a mediated matrix device - * @kvm: reference to KVM instance + * @kvm_file: the KVM VM file this vfio device is associated with * - * Return: 0 if no other mediated matrix device has a reference to @kvm; + * Return: 0 if no other mediated matrix device has a reference to the VM; * otherwise, returns an -EPERM. */ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, - struct kvm *kvm) + struct file *kvm_file) { + struct kvm *kvm; + + if (!kvm_file) + return -ENOENT; + + kvm = file_to_kvm_s390(kvm_file); + if (!kvm) + return -ENOENT; + if (kvm->arch.crypto.crycbd) { get_update_locks_for_kvm(kvm); if (kvm->arch.crypto.pqap_hook) { @@ -1841,7 +1851,6 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, } kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; - kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; vfio_ap_mdev_update_guest_apcb(matrix_mdev); release_update_locks_for_kvm(kvm); @@ -1894,7 +1903,6 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) matrix_mdev->kvm = NULL; release_update_locks_for_kvm(kvm); - kvm_put_kvm(kvm); } } diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c index 1d9515c967b0..c1c26d7a9aec 100644 --- a/drivers/vfio/device_cdev.c +++ b/drivers/vfio/device_cdev.c @@ -65,7 +65,7 @@ int vfio_device_fops_cdev_open(struct inode *inode, struct file *filep) static void vfio_df_get_kvm_safe(struct vfio_device_file *df) { spin_lock(&df->kvm_ref_lock); - vfio_device_get_kvm_safe(df->device, df->kvm); + vfio_device_get_kvm(df->device, df->kvm); spin_unlock(&df->kvm_ref_lock); } diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index b2299e5bc6df..185f359f75d3 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c @@ -163,7 +163,7 @@ static int vfio_group_ioctl_set_container(struct vfio_group *group, static void vfio_device_group_get_kvm_safe(struct vfio_device *device) { spin_lock(&device->group->kvm_ref_lock); - vfio_device_get_kvm_safe(device, device->group->kvm); + vfio_device_get_kvm(device, device->group->kvm); spin_unlock(&device->group->kvm_ref_lock); } @@ -860,11 +860,20 @@ bool vfio_group_enforced_coherent(struct vfio_group *group) return ret; } -void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm) +void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm) { + struct file *old; + + if (kvm) + get_file(kvm); + spin_lock(&group->kvm_ref_lock); + old = group->kvm; group->kvm = kvm; spin_unlock(&group->kvm_ref_lock); + + if (old) + fput(old); } /** diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index 7728bc99b63d..2eeb526a8da5 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h @@ -23,7 +23,7 @@ struct vfio_device_file { u8 access_granted; u32 devid; /* only valid when iommufd is valid */ spinlock_t kvm_ref_lock; /* protect kvm field */ - struct kvm *kvm; + struct file *kvm; struct iommufd_ctx *iommufd; /* protected by struct vfio_device_set::lock */ }; @@ -88,7 +88,7 @@ struct vfio_group { #endif enum vfio_group_type type; struct mutex group_lock; - struct kvm *kvm; + struct file *kvm; struct file *opened_file; struct iommufd_ctx *iommufd; spinlock_t kvm_ref_lock; @@ -107,7 +107,7 @@ void vfio_device_group_unuse_iommu(struct vfio_device *device); void vfio_df_group_close(struct vfio_device_file *df); struct vfio_group *vfio_group_from_file(struct file *file); bool vfio_group_enforced_coherent(struct vfio_group *group); -void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm); +void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm); bool vfio_device_has_container(struct vfio_device *device); int __init vfio_group_init(void); void vfio_group_cleanup(void); @@ -165,7 +165,7 @@ static inline bool vfio_group_enforced_coherent(struct vfio_group *group) return true; } -static inline void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm) +static inline void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm) { } @@ -429,11 +429,11 @@ static inline void vfio_virqfd_exit(void) #endif #if IS_ENABLED(CONFIG_KVM) -void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm); +void vfio_device_get_kvm(struct vfio_device *device, struct file *kvm); void vfio_device_put_kvm(struct vfio_device *device); #else -static inline void vfio_device_get_kvm_safe(struct vfio_device *device, - struct kvm *kvm) +static inline void vfio_device_get_kvm(struct vfio_device *device, + struct file *kvm) { } diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 423ead48aafe..ea8e6cc13c45 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -472,36 +472,14 @@ void vfio_unregister_group_dev(struct vfio_device *device) EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); #if IS_ENABLED(CONFIG_KVM) -void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) +void vfio_device_get_kvm(struct vfio_device *device, struct file *kvm) { - void (*pfn)(struct kvm *kvm); - bool (*fn)(struct kvm *kvm); - bool ret; - lockdep_assert_held(&device->dev_set->lock); if (!kvm) return; - pfn = symbol_get(kvm_put_kvm); - if (WARN_ON(!pfn)) - return; - - fn = symbol_get(kvm_get_kvm_safe); - if (WARN_ON(!fn)) { - symbol_put(kvm_put_kvm); - return; - } - - ret = fn(kvm); - symbol_put(kvm_get_kvm_safe); - if (!ret) { - symbol_put(kvm_put_kvm); - return; - } - - device->put_kvm = pfn; - device->kvm = kvm; + device->kvm = get_file(kvm); } void vfio_device_put_kvm(struct vfio_device *device) @@ -511,14 +489,7 @@ void vfio_device_put_kvm(struct vfio_device *device) if (!device->kvm) return; - if (WARN_ON(!device->put_kvm)) - goto clear; - - device->put_kvm(device->kvm); - device->put_kvm = NULL; - symbol_put(kvm_put_kvm); - -clear: + fput(device->kvm); device->kvm = NULL; } #endif @@ -1544,9 +1515,13 @@ bool vfio_file_enforced_coherent(struct file *file) } EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); -static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) +static void vfio_device_file_set_kvm(struct file *file, struct file *kvm) { struct vfio_device_file *df = file->private_data; + struct file *old; + + if (kvm) + get_file(kvm); /* * The kvm is first recorded in the vfio_device_file, and will @@ -1554,28 +1529,32 @@ static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) * iommufd successfully in the vfio device cdev path. */ spin_lock(&df->kvm_ref_lock); + old = df->kvm; df->kvm = kvm; spin_unlock(&df->kvm_ref_lock); + + if (old) + fput(old); } /** * vfio_file_set_kvm - Link a kvm with VFIO drivers - * @file: VFIO group file or VFIO device file - * @kvm: KVM to link + * @vfio_file: VFIO group file or VFIO device file + * @kvm: KVM file to link * * When a VFIO device is first opened the KVM will be available in * device->kvm if one was associated with the file. */ -void vfio_file_set_kvm(struct file *file, struct kvm *kvm) +void vfio_file_set_kvm(struct file *vfio_file, struct file *kvm) { struct vfio_group *group; - group = vfio_group_from_file(file); + group = vfio_group_from_file(vfio_file); if (group) vfio_group_set_kvm(group, kvm); - if (vfio_device_from_file(file)) - vfio_device_file_set_kvm(file, kvm); + if (vfio_device_from_file(vfio_file)) + vfio_device_file_set_kvm(vfio_file, kvm); } EXPORT_SYMBOL_GPL(vfio_file_set_kvm); diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 45f08986359e..0cc91c6f96d2 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -54,7 +54,7 @@ struct vfio_device { struct list_head dev_set_list; unsigned int migration_flags; u8 precopy_info_v2; - struct kvm *kvm; + struct file *kvm; /* Members below here are private, not for driver use */ unsigned int index; @@ -66,7 +66,6 @@ struct vfio_device { unsigned int open_count; struct completion comp; struct iommufd_access *iommufd_access; - void (*put_kvm)(struct kvm *kvm); struct inode *inode; #if IS_ENABLED(CONFIG_IOMMUFD) struct iommufd_device *iommufd_device; @@ -378,7 +377,7 @@ static inline bool vfio_file_has_dev(struct file *file, struct vfio_device *devi #endif bool vfio_file_is_valid(struct file *file); bool vfio_file_enforced_coherent(struct file *file); -void vfio_file_set_kvm(struct file *file, struct kvm *kvm); +void vfio_file_set_kvm(struct file *vfio_file, struct file *kvm); #define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long)) diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index 6cdc4e9a333a..19548a430942 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -35,15 +35,15 @@ struct kvm_vfio { bool noncoherent; }; -static void kvm_vfio_file_set_kvm(struct file *file, struct kvm *kvm) +static void kvm_vfio_file_set_kvm(struct file *vfio_file, struct file *kvm) { - void (*fn)(struct file *file, struct kvm *kvm); + void (*fn)(struct file *vfio_file, struct file *kvm); fn = symbol_get(vfio_file_set_kvm); if (!fn) return; - fn(file, kvm); + fn(vfio_file, kvm); symbol_put(vfio_file_set_kvm); } @@ -144,6 +144,7 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) { struct kvm_vfio *kv = dev->private; struct kvm_vfio_file *kvf; + struct file *kvm_file __free(fput) = NULL; struct file *filp __free(fput) = NULL; filp = fget(fd); @@ -154,6 +155,10 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) if (!kvm_vfio_file_is_valid(filp)) return -EINVAL; + kvm_file = get_file_active(&dev->kvm->file); + if (!kvm_file) + return -ENOENT; + guard(mutex)(&kv->lock); list_for_each_entry(kvf, &kv->file_list, node) { @@ -168,7 +173,7 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) kvf->file = get_file(filp); list_add_tail(&kvf->node, &kv->file_list); - kvm_vfio_file_set_kvm(kvf->file, dev->kvm); + kvm_vfio_file_set_kvm(kvf->file, kvm_file); kvm_vfio_update_coherency(dev); return 0; From 00aa2c8a02b8f5214f41f50b67bf43d9380471a0 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 28 Sep 2026 15:07:06 +0200 Subject: [PATCH 384/506] KVM: Restrict kvm_get_kvm/kvm_put_kvm export to internal KVM modules Switch kvm_get_kvm, kvm_get_kvm_safe, and kvm_put_kvm from EXPORT_SYMBOL_GPL to EXPORT_SYMBOL_FOR_KVM_INTERNAL. These symbols are now only used within KVM itself. No external module needs them now that VFIO uses file-based reference counting. Remove the corresponding exemptions from the x86 KVM Makefile exports check. Suggested-by: Sean Christopherson Signed-off-by: Steffen Eiden Reviewed-by: Claudio Imbrenda Acked-by: Janosch Frank Acked-by: Sean Christopherson Signed-off-by: Christian Borntraeger --- arch/x86/kvm/Makefile | 5 +---- virt/kvm/kvm_main.c | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index e6d395ede9e2..c6bf463f5032 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -61,10 +61,7 @@ exports_grep_trailer := --include='*.[ch]' -nrw $(srctree)/virt/kvm $(srctree)/a -e kvm_page_track_unregister_notifier \ -e kvm_write_track_add_gfn \ -e kvm_write_track_remove_gfn \ - -e kvm_file_to_kvm_fn \ - -e kvm_get_kvm \ - -e kvm_get_kvm_safe \ - -e kvm_put_kvm + -e kvm_file_to_kvm_fn # Force grep to emit a goofy group separator that can in turn be replaced with # the above newline macro (newlines in Make are a nightmare). Note, grep only diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index bab71d398236..891a67cb365a 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1314,7 +1314,7 @@ void kvm_get_kvm(struct kvm *kvm) { refcount_inc(&kvm->users_count); } -EXPORT_SYMBOL_GPL(kvm_get_kvm); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_kvm); /* * Make sure the vm is not during destruction, which is a safe version of @@ -1324,14 +1324,14 @@ bool kvm_get_kvm_safe(struct kvm *kvm) { return refcount_inc_not_zero(&kvm->users_count); } -EXPORT_SYMBOL_GPL(kvm_get_kvm_safe); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_kvm_safe); void kvm_put_kvm(struct kvm *kvm) { if (refcount_dec_and_test(&kvm->users_count)) kvm_destroy_vm(kvm); } -EXPORT_SYMBOL_GPL(kvm_put_kvm); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_put_kvm); /* * Used to put a reference that was taken on behalf of an object associated From c22295d0d75f5590fdde5493cb90e2d50d5f508e Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 28 Sep 2026 15:07:07 +0200 Subject: [PATCH 385/506] KVM: Remove unused file_is_kvm Remove the now unused file_is_kvm function. Signed-off-by: Steffen Eiden Reviewed-by: Claudio Imbrenda Acked-by: Janosch Frank Acked-by: Sean Christopherson Signed-off-by: Christian Borntraeger --- include/linux/kvm_host.h | 1 - virt/kvm/kvm_main.c | 6 ------ 2 files changed, 7 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 943b108f01cd..b6a1f4312a73 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1087,7 +1087,6 @@ void kvm_exit(void); void kvm_get_kvm(struct kvm *kvm); bool kvm_get_kvm_safe(struct kvm *kvm); void kvm_put_kvm(struct kvm *kvm); -bool file_is_kvm(struct file *file); /* * Architectures define kvm_file_to_kvm_arch to diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 891a67cb365a..563e7b0becc3 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5492,12 +5492,6 @@ static struct file_operations kvm_vm_fops = { KVM_COMPAT(kvm_vm_compat_ioctl), }; -bool file_is_kvm(struct file *file) -{ - return file && file->f_op == &kvm_vm_fops; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(file_is_kvm); - #ifdef kvm_file_to_kvm_arch struct kvm *kvm_file_to_kvm_fn(struct file *file) { From 3708f6342f010d66a767b587dc881f18d95ceb3e Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:56:54 +0100 Subject: [PATCH 386/506] KVM: arm64: Sync HCR_EL2.VSE back to the host vCPU under pKVM flush_hyp_vcpu() copies HCR_EL2.VSE from the host vCPU into the hyp vCPU on every entry, and nothing copies it back. Once the guest takes the vSError the hyp vCPU's copy clears with the hardware bit while the host's stays set, so the next entry pends the same SError again, KVM_GET_VCPU_EVENTS keeps reporting it, and kvm_arch_vcpu_runnable() never lets the vCPU block in WFI. Reflect VSE back on every exit, which the flush-side comment already states. Fixes: 734dc8c01c838 ("KVM: arm64: Implement lazy vCPU state sync for non-protected guests") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9a3b92e626ad..b6bfe502bcd0 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -276,6 +276,10 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; + /* Cleared by hardware once the guest takes the vSError. */ + host_vcpu->arch.hcr_el2 &= ~HCR_VSE; + host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE; + sync_hyp_vgic_state(hyp_vcpu); } From cf533b4d3630b229d78ad55567c5db2f365052a2 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:56:55 +0100 Subject: [PATCH 387/506] KVM: arm64: Validate the host vCPU's VM before reading it under pKVM On an MTE-capable host under pKVM, enter_exception64() reads the VM's MTE flag through vcpu->kvm, which for a host vCPU is a host-writable pointer nothing validates. The host can point it at any address in the hyp linear map and read back bit 1 of that word through PSR_TCO in the vCPU's CPSR, or panic the hypervisor with an unmapped one. Get the VM through a get/put pair around the read: a loaded vCPU's is the hyp VM, an unloaded host vCPU's is read once and pinned, and a pointer the host never shared leaves TCO clear. Fixes: ea7fc1bb1cd1b ("KVM: arm64: Introduce MTE VM feature") Reported-by: Sashiko Closes: https://lore.kernel.org/all/20260914070536.877D91F000FF@smtp.kernel.org/ Reviewed-by: Vincent Donnefort Reviewed-by: Joey Gouly Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-3-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/exception.c | 5 ++- arch/arm64/kvm/hyp/include/hyp/adjust_pc.h | 28 ++++++++++++++++ arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 2 ++ arch/arm64/kvm/hyp/nvhe/pkvm.c | 37 ++++++++++++++++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 754e2dc1df54..6e60d890afa4 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -70,6 +70,7 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, enum exception_type type) { unsigned long sctlr, vbar, old, new, mode; + struct kvm *kvm; u64 exc_offset; mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); @@ -109,8 +110,10 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, new |= (old & PSR_C_BIT); new |= (old & PSR_V_BIT); - if (kvm_has_mte(kern_hyp_va(vcpu->kvm))) + kvm = vcpu_get_kvm(vcpu); + if (kvm && kvm_has_mte(kvm)) new |= PSR_TCO_BIT; + vcpu_put_kvm(vcpu, kvm); new |= (old & PSR_DIT_BIT); diff --git a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h index 4fdfeabefeb4..f55950ee2a7e 100644 --- a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h +++ b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h @@ -13,6 +13,34 @@ #include #include +#ifdef __KVM_NVHE_HYPERVISOR__ +#include + +/* Under pKVM a host vCPU's ->kvm is host-writable. */ +static inline struct kvm *vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + if (is_protected_kvm_enabled()) + return pkvm_vcpu_get_kvm(vcpu); + + return kern_hyp_va(vcpu->kvm); +} + +static inline void vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ + if (is_protected_kvm_enabled()) + pkvm_vcpu_put_kvm(vcpu, kvm); +} +#else +static inline struct kvm *vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + return vcpu->kvm; +} + +static inline void vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ +} +#endif + static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) { if (vcpu_mode_is_32bit(vcpu)) { diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index c904647d2f76..5ddb40714966 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -83,6 +83,8 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, unsigned int vcpu_idx); void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu); struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void); +struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu); +void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm); struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle); struct pkvm_hyp_vm *get_np_pkvm_hyp_vm(pkvm_handle_t handle); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 459bd9eb7e4b..0e90de74778c 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -304,6 +304,43 @@ struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void) } +static struct pkvm_hyp_vm *pkvm_get_loaded_hyp_vm(struct kvm_vcpu *vcpu) +{ + struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + + if (hyp_vcpu && + (vcpu == &hyp_vcpu->vcpu || vcpu == hyp_vcpu->host_vcpu)) + return pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + + return NULL; +} + +/* + * A loaded vCPU's VM is the hyp VM. An unloaded host vCPU's is mapped at + * EL2 only while pinned, so it's read once and pinned; the pin fails for + * memory the host isn't sharing. + */ +struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_get_loaded_hyp_vm(vcpu); + struct kvm *kvm; + + if (hyp_vm) + return &hyp_vm->kvm; + + kvm = kern_hyp_va(READ_ONCE(vcpu->kvm)); + if (hyp_pin_shared_mem(kvm, kvm + 1)) + return NULL; + + return kvm; +} + +void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ + if (kvm && !pkvm_get_loaded_hyp_vm(vcpu)) + hyp_unpin_shared_mem(kvm, kvm + 1); +} + struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) { struct pkvm_hyp_vm *hyp_vm; From a2bb5f5c67a95ac97fcd6859dab51be421cf26fe Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:56:56 +0100 Subject: [PATCH 388/506] KVM: arm64: Pin the host vCPU before adjusting its PC under pKVM Under pKVM, a page the host shares with EL2 is mapped at EL2 only while it's pinned, and the host vCPU is pinned at its first KVM_RUN. KVM_SET_VCPU_EVENTS with ext_dabt_pending reaches __kvm_adjust_pc() at EL2 before that, and dereferencing the unmapped host vCPU panics the hypervisor. Any process holding /dev/kvm on a pKVM host can trigger it. Pin the host vCPU around the adjustment when no hyp vCPU is loaded for it. A loaded hyp vCPU implies it's pinned. A pin fails only for memory the host isn't sharing, and the request is then dropped like any other bad host pointer. Fixes: efa1368ba9f4b ("KVM: arm64: Commit exceptions from KVM_SET_VCPU_EVENTS immediately") Reviewed-by: Vincent Donnefort Reviewed-by: Joey Gouly Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-4-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index b6bfe502bcd0..8b9d76e5336b 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -546,8 +546,27 @@ static void handle___pkvm_host_mkyoung_guest(struct kvm_cpu_context *host_ctxt) static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; - __kvm_adjust_pc(kern_hyp_va(vcpu)); + host_vcpu = __get_host_hyp_vcpus(vcpu, &hyp_vcpu); + if (host_vcpu) { + __kvm_adjust_pc(host_vcpu); + return; + } + + /* + * With no hyp vCPU loaded for it, the host vCPU may be unpinned, + * and so unmapped at EL2: its first run pins it. A pin fails only + * for memory the host isn't sharing, a bad pointer, so the request + * is dropped. + */ + host_vcpu = kern_hyp_va(vcpu); + if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) + return; + + __kvm_adjust_pc(host_vcpu); + hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); } static void handle___kvm_flush_vm_context(struct kvm_cpu_context *host_ctxt) From a5b0f36a045e0e96f8e2e044403ff8e6464995e1 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:56:57 +0100 Subject: [PATCH 389/506] KVM: arm64: Disable steal time for protected VMs pKVM doesn't support steal time for protected guests, and KVM_CAP_STEAL_TIME already reads 0 for them on the VM file descriptor, but kvm_arm_pvtime_supported() takes no VM, so the host still accepts the KVM_ARM_VCPU_PVTIME_CTRL attribute, whose IPA would point kvm_update_stolen_time() at the guest's private memory on every vCPU load. Pass the VM in and return false for a protected one, so the attribute returns -ENXIO as it does where steal time isn't implemented. Reviewed-by: Vincent Donnefort Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-5-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/kvm/arm.c | 2 +- arch/arm64/kvm/pvtime.c | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..286489a69dff 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1346,7 +1346,7 @@ long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu); gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu); void kvm_update_stolen_time(struct kvm_vcpu *vcpu); -bool kvm_arm_pvtime_supported(void); +bool kvm_arm_pvtime_supported(struct kvm *kvm); int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu, diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..db3681563079 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -447,7 +447,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = system_supports_mte(); break; case KVM_CAP_STEAL_TIME: - r = kvm_arm_pvtime_supported(); + r = kvm_arm_pvtime_supported(kvm); break; case KVM_CAP_ARM_EL1_32BIT: r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1); diff --git a/arch/arm64/kvm/pvtime.c b/arch/arm64/kvm/pvtime.c index 4ceabaa4c30b..053fe831fa54 100644 --- a/arch/arm64/kvm/pvtime.c +++ b/arch/arm64/kvm/pvtime.c @@ -67,9 +67,9 @@ gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu) return base; } -bool kvm_arm_pvtime_supported(void) +bool kvm_arm_pvtime_supported(struct kvm *kvm) { - return !!sched_info_on(); + return !!sched_info_on() && (!kvm || !kvm_vm_is_protected(kvm)); } int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, @@ -81,7 +81,7 @@ int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, int ret = 0; int idx; - if (!kvm_arm_pvtime_supported() || + if (!kvm_arm_pvtime_supported(kvm) || attr->attr != KVM_ARM_VCPU_PVTIME_IPA) return -ENXIO; @@ -110,7 +110,7 @@ int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu, u64 __user *user = (u64 __user *)attr->addr; u64 ipa; - if (!kvm_arm_pvtime_supported() || + if (!kvm_arm_pvtime_supported(vcpu->kvm) || attr->attr != KVM_ARM_VCPU_PVTIME_IPA) return -ENXIO; @@ -126,7 +126,7 @@ int kvm_arm_pvtime_has_attr(struct kvm_vcpu *vcpu, { switch (attr->attr) { case KVM_ARM_VCPU_PVTIME_IPA: - if (kvm_arm_pvtime_supported()) + if (kvm_arm_pvtime_supported(vcpu->kvm)) return 0; } return -ENXIO; From 66a5e82d2a8d113d5953d04e008447d6b779e03e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 1 Oct 2026 14:56:58 +0100 Subject: [PATCH 390/506] KVM: arm64: Introduce per-EC entry handlers for pKVM Add an ESR_EL2.EC-indexed handler table, entry_hyp_vm_handlers[], read from flush_hyp_vcpu() on re-entry when the previous exit was a trap: sync_hyp_vcpu() records the exit reason in the hyp vCPU as exit_code, and the trap's exception class comes from its ESR_EL2. Its single generic handler stands in until the per-EC marshalling patch replaces the table with protected-only ones. Entry carries the host's requested PC updates to the hyp vCPU: add vcpu_copy_flag() to copy a masked set of flags between the two vCPU structures, with PC_UPDATE_REQ covering INCREMENT_PC and the pending-exception flags. The wholesale iflags copy moves into the non-protected branch; a protected vCPU takes only PC_UPDATE_REQ. Reviewed-by: Joey Gouly Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-6-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 19 +++++++++++++ arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 3 ++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 39 +++++++++++++++++++++++--- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 286489a69dff..37d0721d39a4 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1033,11 +1033,28 @@ struct kvm_vcpu_arch { set; \ }) +#define __vcpu_copy_flag(vt, vs, flagset, f, m) \ + do { \ + typeof(vs->arch.flagset) tmp, val; \ + \ + __build_check_flag(vs, flagset, f, m); \ + \ + val = READ_ONCE(vs->arch.flagset); \ + val &= (m); \ + __vcpu_flags_preempt_disable(); \ + tmp = READ_ONCE(vt->arch.flagset); \ + tmp &= ~(m); \ + tmp |= val; \ + WRITE_ONCE(vt->arch.flagset, tmp); \ + __vcpu_flags_preempt_enable(); \ + } while (0) + #define vcpu_get_flag(v, ...) __vcpu_get_flag((v), __VA_ARGS__) #define vcpu_set_flag(v, ...) __vcpu_set_flag((v), __VA_ARGS__) #define vcpu_clear_flag(v, ...) __vcpu_clear_flag((v), __VA_ARGS__) #define vcpu_test_and_clear_flag(v, ...) \ __vcpu_test_and_clear_flag((v), __VA_ARGS__) +#define vcpu_copy_flag(vt, vs, ...) __vcpu_copy_flag((vt), (vs), __VA_ARGS__) /* KVM_ARM_VCPU_INIT completed */ #define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0)) @@ -1055,6 +1072,8 @@ struct kvm_vcpu_arch { #define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1)) /* Target EL/MODE (not a single flag, but let's abuse the macro) */ #define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1)) +/* Cover both PENDING_EXCEPTION and EXCEPT_MASK for global operations */ +#define PC_UPDATE_REQ __vcpu_single_flag(iflags, GENMASK(3, 0)) /* Host-set: the hyp flushes the non-protected vCPU state in on entry */ #define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(4)) diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 5ddb40714966..ad42abaf31c5 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -26,6 +26,9 @@ struct pkvm_hyp_vcpu { * per-cpu pointer tracking us. Otherwise, NULL if not loaded. */ struct pkvm_hyp_vcpu **loaded_hyp_vcpu; + + /* The previous exit's ARM_EXCEPTION_* code. */ + u32 exit_code; }; /* diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 8b9d76e5336b..e3f4bbb62349 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -31,6 +31,17 @@ unsigned int hyp_gicv3_nr_lr; void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); +typedef void (*hyp_entry_exit_handler_fn)(struct pkvm_hyp_vcpu *); + +static void handle_vm_entry_generic(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + vcpu_copy_flag(&hyp_vcpu->vcpu, hyp_vcpu->host_vcpu, PC_UPDATE_REQ); +} + +static const hyp_entry_exit_handler_fn entry_hyp_vm_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = handle_vm_entry_generic, +}; + static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) { __vcpu_assign_sys_reg(vcpu, ZCR_EL1, read_sysreg_el1(SYS_ZCR)); @@ -216,6 +227,8 @@ static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + hyp_entry_exit_handler_fn ec_handler; + u8 esr_ec; fpsimd_sve_flush(); flush_debug_state(hyp_vcpu); @@ -228,6 +241,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) flush_hyp_vcpu_state(hyp_vcpu); + hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; } else { hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; } @@ -245,16 +259,31 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & (HCR_TWI | HCR_TWE | HCR_VSE); - hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; - hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; flush_hyp_vgic_state(hyp_vcpu); hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; + + switch (ARM_EXCEPTION_CODE(hyp_vcpu->exit_code)) { + case ARM_EXCEPTION_IRQ: + case ARM_EXCEPTION_EL1_SERROR: + case ARM_EXCEPTION_IL: + break; + case ARM_EXCEPTION_TRAP: + esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); + ec_handler = entry_hyp_vm_handlers[esr_ec]; + if (ec_handler) + ec_handler(hyp_vcpu); + break; + default: + BUG(); + } + + hyp_vcpu->exit_code = 0; } -static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) +static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; @@ -281,6 +310,8 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE; sync_hyp_vgic_state(hyp_vcpu); + + hyp_vcpu->exit_code = exit_reason; } static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) @@ -395,7 +426,7 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); - sync_hyp_vcpu(hyp_vcpu); + sync_hyp_vcpu(hyp_vcpu, ret); } else { /* The host is fully trusted, run its vCPU directly. */ fpsimd_lazy_switch_to_guest(host_vcpu); From 06499de8f998e0ec975cb73adde936c12a8a1200 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:56:59 +0100 Subject: [PATCH 391/506] KVM: arm64: Skip fixed-feature state flush for protected vCPUs flush_hyp_vcpu() copies the host's mdcr_el2, TWI/TWE and debug state into the hyp vCPU on every entry, so for a protected vCPU the host's trap configuration overrides the one pkvm_vcpu_init_traps() computed at EL2. Move the mdcr_el2 and TWI/TWE copies into the non-protected branch, and return early from flush_debug_state() and sync_debug_state() for a protected vCPU: its debug registers are hypervisor-owned, and it takes the host's TWI/TWE at vCPU load. MDCR_EL2.TDA traps only the guest's own accesses; it doesn't stop the world switch from loading host-supplied breakpoints and MDSCR_EL1. EL2's mdcr_el2 starts from 0, and HPMN == 0 is reserved without FEAT_HPMN0. Set HPMN at vCPU load to PMCR_EL0.N on a PE whose ID_AA64DFR0_EL1 implements PMUv3, the value the host's copy carried, read as kvm_init_host_debug_data() reads it: the PMUv3 cpucap is system-wide and off without CONFIG_HW_PERF_EVENTS. A protected guest's PMU accesses trap regardless. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-7-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 34 ++++++++++++++++++++++-------- arch/arm64/kvm/hyp/nvhe/pkvm.c | 2 +- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index e3f4bbb62349..b7b9f5afb673 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -200,6 +200,9 @@ static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + hyp_vcpu->vcpu.arch.debug_owner = host_vcpu->arch.debug_owner; if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) { @@ -218,6 +221,9 @@ static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) host_vcpu->arch.vcpu_debug_state = hyp_vcpu->vcpu.arch.vcpu_debug_state; else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) @@ -241,6 +247,12 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) flush_hyp_vcpu_state(hyp_vcpu); + + hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE); + hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & + (HCR_TWI | HCR_TWE); + + hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; } else { hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; @@ -249,17 +261,13 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) /* __hyp_running_vcpu must be NULL in a guest context. */ hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; - hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; /* - * HCR_EL2.VSE is host-owned (a pending virtual SError to inject), not a - * trap-control bit, so it must flow to the hyp vCPU alongside TWI/TWE - * for the vSError to be delivered. sync_hyp_vcpu() reflects it back. + * A host-injected vSError is masked by the guest's own PSTATE.A, so it + * applies to protected guests too. */ - hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE | HCR_VSE); - hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & - (HCR_TWI | HCR_TWE | HCR_VSE); - - hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; + hyp_vcpu->vcpu.arch.hcr_el2 &= ~HCR_VSE; + hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & HCR_VSE; + hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; flush_hyp_vgic_state(hyp_vcpu); @@ -326,9 +334,17 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) return; if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + u64 dfr0 = read_sysreg(id_aa64dfr0_el1); + /* Propagate WFx trapping flags */ hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI); hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI); + + /* HPMN == 0 is reserved without FEAT_HPMN0. */ + if (pmuv3_implemented(SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0))) + u64p_replace_bits(&hyp_vcpu->vcpu.arch.mdcr_el2, + FIELD_GET(ARMV8_PMU_PMCR_N, read_sysreg(pmcr_el0)), + MDCR_EL2_HPMN); } else { memcpy(&hyp_vcpu->vcpu.arch.fgt, hyp_vcpu->host_vcpu->arch.fgt, sizeof(hyp_vcpu->vcpu.arch.fgt)); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 0e90de74778c..8f3d9c293d18 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -102,7 +102,7 @@ static void pvm_init_traps_mdcr(struct kvm_vcpu *vcpu) if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) { val |= MDCR_EL2_TPM | MDCR_EL2_TPMCR; - val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME | MDCR_EL2_HPMN_MASK); + val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME); } if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, IMP)) From a8b2a7d0fe9c0eb493542bb1d306b68313e6a828 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 1 Oct 2026 14:57:00 +0100 Subject: [PATCH 392/506] KVM: arm64: Add {flush,sync}_hyp_timer_state() primitives A protected guest's virtual timer is programmed from EL2's own copy of the state and read back there, so the host can't change the guest's deadline. cntvoff_el2 is zeroed, so the guest's virtual time is the physical time. The flush also preserves the guest's CNTV and CNTP CVAL/CTL across the host's full-context copy into the hyp vCPU, until the per-EC marshalling patch removes that copy. Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-8-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index b7b9f5afb673..6f068d0f78d2 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -154,6 +154,35 @@ static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; } +static void flush_hyp_timer_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + + /* A hyp vcpu has no offset, and sees vtime == ptime. */ + write_sysreg(0, cntvoff_el2); + write_sysreg_el0(__vcpu_sys_reg(vcpu, CNTV_CVAL_EL0), SYS_CNTV_CVAL); + isb(); + write_sysreg_el0(__vcpu_sys_reg(vcpu, CNTV_CTL_EL0), SYS_CNTV_CTL); +} + +static void sync_hyp_timer_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + + /* + * Preserve the vtimer state so that it is always correct, + * even if the host tries to make a mess. + */ + __vcpu_assign_sys_reg(vcpu, CNTV_CVAL_EL0, read_sysreg_el0(SYS_CNTV_CVAL)); + __vcpu_assign_sys_reg(vcpu, CNTV_CTL_EL0, read_sysreg_el0(SYS_CNTV_CTL)); +} + static void __copy_vcpu_state(const struct kvm_vcpu *from_vcpu, struct kvm_vcpu *to_vcpu) { @@ -255,7 +284,17 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; } else { + u64 v_cval = hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTV_CVAL_EL0]; + u64 v_ctl = hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTV_CTL_EL0]; + u64 p_cval = hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTP_CVAL_EL0]; + u64 p_ctl = hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTP_CTL_EL0]; + hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; + + hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTV_CVAL_EL0] = v_cval; + hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTV_CTL_EL0] = v_ctl; + hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTP_CVAL_EL0] = p_cval; + hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTP_CTL_EL0] = p_ctl; } /* __hyp_running_vcpu must be NULL in a guest context. */ @@ -270,6 +309,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; flush_hyp_vgic_state(hyp_vcpu); + flush_hyp_timer_state(hyp_vcpu); hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; @@ -318,6 +358,7 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE; sync_hyp_vgic_state(hyp_vcpu); + sync_hyp_timer_state(hyp_vcpu); hyp_vcpu->exit_code = exit_reason; } From ceaf9e57f88711fa3c7775ee4fea4f6731f1a71c Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:01 +0100 Subject: [PATCH 393/506] KVM: arm64: Add system register reset framework for protected VMs Add kvm_reset_pvm_sys_regs() and the pvm_sys_reg_reset_vals[] table that drives it, and call it from init_pkvm_hyp_vcpu() for protected vCPUs. The table holds the registers KVM resets that EL2 loads for a protected vCPU: the ones __sysreg_restore_state_nvhe() restores, and CNTV_CTL_EL0, which flush_hyp_timer_state() loads from a copy the host's kvm_timer_vcpu_reset() doesn't write. The ptrauth keys, which entry.S loads, are not in the table: sys_regs.c resets them to UNKNOWN, and a vCPU that CPU_ON brings back keeps the keys the guest last set, one UNKNOWN value in place of another. A register that resets to 0 still takes an entry: a later patch calls kvm_reset_pvm_sys_regs() again for PSCI CPU_ON at EL2, on a context that has run. A poison value stands in where sys_regs.c resets to UNKNOWN, and for VBAR_EL1 and CONTEXTIDR_EL1 in place of the 0 it uses. MPIDR_EL1 is derived from vcpu_id, as for any KVM guest. AMAIR_EL1 resets to 0 where sys_regs.c reads the hardware: the host writes it freely before the hypercall, and it's loaded on every guest entry. Until the per-EC marshalling patch removes the host's full-context copy from flush_hyp_vcpu(), that copy overwrites these values on every entry. It preserves CNTV_CTL_EL0, whose entry writes the zero the page already holds. So this patch has no observable effect on its own. No functional change intended. Reviewed-by: Joey Gouly Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-9-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 1 + arch/arm64/kvm/hyp/nvhe/pkvm.c | 5 +++ arch/arm64/kvm/hyp/nvhe/sys_regs.c | 60 ++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index ad42abaf31c5..fcb13d8b4cb6 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -97,6 +97,7 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code); bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code); bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code); void kvm_init_pvm_id_regs(struct kvm_vcpu *vcpu); +void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu); int kvm_check_pvm_sysreg_table(void); #endif /* __ARM64_KVM_NVHE_PKVM_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 8f3d9c293d18..6ff08d195ff5 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -588,6 +588,11 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, goto done; ret = pkvm_vcpu_init_sve(hyp_vcpu, host_vcpu); + if (ret) + goto done; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); done: if (ret) unpin_host_vcpu(host_vcpu); diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8758c6801776..87557d1b3bf0 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -525,6 +525,66 @@ static const struct sys_reg_desc pvm_sys_reg_descs[] = { /* Performance Monitoring Registers are restricted. */ }; +struct sys_reg_desc_reset { + int reg; + void (*reset)(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *rd); + u64 value; +}; + +static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *r) +{ + __vcpu_assign_sys_reg(vcpu, r->reg, kvm_calculate_mpidr(vcpu)); +} + +static void reset_value(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *r) +{ + __vcpu_assign_sys_reg(vcpu, r->reg, r->value); +} + +#define RESET_VAL(REG, VAL) { REG, reset_value, VAL } + +#define RESET_ZERO(REG) RESET_VAL(REG, 0) + +#define RESET_UNKNOWN(REG) RESET_VAL(REG, 0x1de7ec7edbadc0deULL) + +#define RESET_FUNC(REG, FN) { REG, FN, 0 } + +static const struct sys_reg_desc_reset pvm_sys_reg_reset_vals[] = { + RESET_FUNC(MPIDR_EL1, reset_mpidr), + RESET_UNKNOWN(TPIDR_EL0), + RESET_UNKNOWN(TPIDRRO_EL0), + RESET_UNKNOWN(TPIDR_EL1), + RESET_ZERO(CNTKCTL_EL1), + RESET_UNKNOWN(PAR_EL1), + RESET_ZERO(DISR_EL1), + RESET_ZERO(CPACR_EL1), + RESET_VAL(CONTEXTIDR_EL1, 0x00000000dbadc0deULL), + RESET_VAL(SCTLR_EL1, 0x00C50078ULL), + RESET_ZERO(TCR_EL1), + RESET_UNKNOWN(AFSR0_EL1), + RESET_UNKNOWN(AFSR1_EL1), + RESET_UNKNOWN(ESR_EL1), + RESET_UNKNOWN(MAIR_EL1), + RESET_ZERO(AMAIR_EL1), + RESET_ZERO(MDSCR_EL1), + RESET_ZERO(CNTV_CTL_EL0), + RESET_UNKNOWN(TTBR0_EL1), + RESET_UNKNOWN(TTBR1_EL1), + RESET_UNKNOWN(FAR_EL1), + RESET_VAL(VBAR_EL1, 0x1de7ec7edbadc000ULL), +}; + +void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu) +{ + unsigned long i; + + for (i = 0; i < ARRAY_SIZE(pvm_sys_reg_reset_vals); i++) { + const struct sys_reg_desc_reset *r = &pvm_sys_reg_reset_vals[i]; + + r->reset(vcpu, r); + } +} + /* * Initializes feature registers for protected vms. */ From fe40ff87ead5c541ac4b42aa149b98052a1abca1 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:02 +0100 Subject: [PATCH 394/506] KVM: arm64: Implement HVC handling for protected guests at EL2 Extend kvm_handle_pvm_hvc64() to handle SMCCC_VERSION, SMCCC_ARCH_FEATURES and the vendor hypervisor call UID at EL2, so these queries don't reach the host. ARCH_FEATURES is mandatory from SMCCC 1.1, the version EL2 implements: it returns SUCCESS for itself and for SMCCC_VERSION, NOT_SUPPORTED for anything else, and handles the ARCH_WORKAROUND_1/2/3 queries as the host does, from a copy of the host's mitigation state taken at init with the sanitised ID registers. The workaround calls themselves are already handled on hyp entry. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-10-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hyp.h | 4 ++ arch/arm64/kvm/arm.c | 4 ++ arch/arm64/kvm/hyp/nvhe/pkvm.c | 68 ++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 4974492744cc..7609c78c68f3 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -10,6 +10,7 @@ #include #include #include +#include #include DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); @@ -150,6 +151,9 @@ extern u64 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val); extern unsigned long kvm_nvhe_sym(__icache_flags); extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits); +extern enum mitigation_state kvm_nvhe_sym(spectre_v2_state); +extern enum mitigation_state kvm_nvhe_sym(spectre_v4_state); +extern enum mitigation_state kvm_nvhe_sym(spectre_bhb_state); extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl); extern unsigned long kvm_nvhe_sym(hyp_nr_cpus); extern unsigned int kvm_nvhe_sym(hyp_gicv3_nr_lr); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index db3681563079..cceeeee26690 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2622,6 +2622,10 @@ static void kvm_hyp_init_symbols(void) kvm_nvhe_sym(__icache_flags) = __icache_flags; kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits; + kvm_nvhe_sym(spectre_v2_state) = arm64_get_spectre_v2_state(); + kvm_nvhe_sym(spectre_v4_state) = arm64_get_spectre_v4_state(); + kvm_nvhe_sym(spectre_bhb_state) = arm64_get_spectre_bhb_state(); + /* Propagate the FGT state to the nVHE side */ kvm_nvhe_sym(hfgrtr_masks) = hfgrtr_masks; kvm_nvhe_sym(hfgwtr_masks) = hfgwtr_masks; diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 6ff08d195ff5..10ff2ba6c6c1 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -10,12 +10,18 @@ #include #include +#include #include #include #include #include +/* The host's Spectre mitigation state, for SMCCC_ARCH_FEATURES. */ +enum mitigation_state spectre_v2_state; +enum mitigation_state spectre_v4_state; +enum mitigation_state spectre_bhb_state; + /* Used by icache_is_aliasing(). */ unsigned long __icache_flags; @@ -1181,8 +1187,70 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) { u64 val[4] = { SMCCC_RET_INVALID_PARAMETER }; bool handled = true; + u32 feature; + uuid_t uuid; switch (smccc_get_function(vcpu)) { + case ARM_SMCCC_VERSION_FUNC_ID: + /* Nothing to be handled by the host. Go back to the guest. */ + val[0] = ARM_SMCCC_VERSION_1_1; + break; + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + /* The workaround calls themselves are handled on hyp entry. */ + val[0] = SMCCC_RET_NOT_SUPPORTED; + feature = smccc_get_arg1(vcpu); + switch (feature) { + case ARM_SMCCC_VERSION_FUNC_ID: + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + val[0] = SMCCC_RET_SUCCESS; + break; + case ARM_SMCCC_ARCH_WORKAROUND_1: + switch (spectre_v2_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + val[0] = SMCCC_RET_SUCCESS; + break; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; + break; + } + break; + case ARM_SMCCC_ARCH_WORKAROUND_2: + switch (spectre_v4_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + /* With SSBS advertised the guest's default is safe. */ + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP)) + break; + fallthrough; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_RET_NOT_REQUIRED; + break; + } + break; + case ARM_SMCCC_ARCH_WORKAROUND_3: + switch (spectre_bhb_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + val[0] = SMCCC_RET_SUCCESS; + break; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; + break; + } + break; + } + break; + case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: + uuid = ARM_SMCCC_VENDOR_HYP_UID_KVM; + val[0] = smccc_uuid_to_reg(&uuid, 0); + val[1] = smccc_uuid_to_reg(&uuid, 1); + val[2] = smccc_uuid_to_reg(&uuid, 2); + val[3] = smccc_uuid_to_reg(&uuid, 3); + break; case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: val[0] = BIT(ARM_SMCCC_KVM_FUNC_FEATURES); val[0] |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO); From 9d4ec80be2eb4745d877c6989f1e59c62a2e4da4 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:03 +0100 Subject: [PATCH 395/506] KVM: arm64: Handle PSCI calls for protected VMs at EL2 EL2 implements PSCI 1.1 for protected VMs: CPU_ON, CPU_OFF, PSCI_VERSION and PSCI_FEATURES are resolved at EL2 (CPU_ON and CPU_OFF still exit to the host, which only schedules the target or stops scheduling it), AFFINITY_INFO, CPU_SUSPEND and the platform power operations are forwarded to the host, and anything else returns NOT_SUPPORTED, including the TRNG calls and the functions above 1.1, SYSTEM_OFF2 among them, that the host handled for a protected guest until now. TRNG for protected guests is a follow-up. AFFINITY_INFO stays with the host, which returns OFF only once it has stopped scheduling the target: the host is what a guest polls to see a CPU_OFF complete before it issues the next CPU_ON, as Linux does on hotplug. Three consequences follow: - A protected VM has one primary vCPU, the first whose hyp vCPU is created with mp_state RUNNABLE. A second one, or an mp_state other than RUNNABLE or STOPPED, fails that vCPU's first KVM_RUN with -EINVAL. - CPU_ON finds its target among the hyp vCPUs, which exist from the target's first KVM_RUN; before that the guest gets INVALID_PARAMETERS. - A vCPU EL2 holds powered off doesn't run: handle___kvm_vcpu_run() returns ARM_EXCEPTION_IL, reported as KVM_EXIT_FAIL_ENTRY. Its existing bail-outs return the same code, for every hyp vCPU, instead of an -EINVAL whose bit 31 the host read as a pending SError. Non-protected VMs keep power_state ON and accept any mp_state. Each protected vCPU is OFF, ON_PENDING or ON. CPU_ON moves the target to ON_PENDING, and the target's next run resets it and moves it to ON. The racing transitions are cmpxchg, and the reset state is published with a release/acquire pair, documented at each site. CPU_OFF clears the reset flag and stores OFF with a release, and CPU_ON's writes are control dependent on its cmpxchg reading OFF, so the target's accesses to reset_state are ordered before the winner's writes. Rolling a CPU_ON the host failed back to OFF requires the host's return value, which the per-EC marshalling patch delivers along with the rollback. Until then such a target stays ON_PENDING, and the reset has no observable effect: flush_hyp_vcpu() copies the host's context in on every entry until that patch removes the copy, so the target enters on the host's values rather than the ones EL2 reset. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-11-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 14 ++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 25 ++- arch/arm64/kvm/hyp/nvhe/pkvm.c | 293 ++++++++++++++++++++++++- 3 files changed, 321 insertions(+), 11 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index fcb13d8b4cb6..0623ec9d3a93 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -29,6 +29,12 @@ struct pkvm_hyp_vcpu { /* The previous exit's ARM_EXCEPTION_* code. */ u32 exit_code; + + /* + * PSCI_0_2_AFFINITY_LEVEL_{OFF, ON_PENDING, ON}. A non-protected + * vCPU is always ON. + */ + int power_state; }; /* @@ -46,6 +52,12 @@ struct pkvm_hyp_vm { struct hyp_pool pool; hyp_spinlock_t lock; + /* + * The vCPU initialised RUNNABLE: claimed under vm_table_lock, + * released only if its own init fails. + */ + struct pkvm_hyp_vcpu *primary_vcpu; + /* Array of the hyp vCPU structures for this VM. */ struct pkvm_hyp_vcpu *vcpus[]; }; @@ -100,4 +112,6 @@ void kvm_init_pvm_id_regs(struct kvm_vcpu *vcpu); void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu); int kvm_check_pvm_sysreg_table(void); +int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu); +struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *vm, unsigned long mpidr); #endif /* __ARM64_KVM_NVHE_PKVM_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 6f068d0f78d2..00038a162d09 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -458,14 +459,12 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - int ret; + int ret = ARM_EXCEPTION_IL; host_vcpu = get_host_hyp_vcpus(host_ctxt, 1, &hyp_vcpu); - if (!host_vcpu) { - ret = -EINVAL; + if (!host_vcpu) goto out; - } if (unlikely(hyp_vcpu)) { /* @@ -474,8 +473,22 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) * loading a vcpu. Therefore, if SME features enabled the host * is misbehaving. */ - if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) { - ret = -EINVAL; + if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) + goto out; + + /* + * ON has a single writer, pkvm_reset_vcpu() on this CPU, so + * READ_ONCE suffices. ON_PENDING takes the reset; -ECANCELED + * is a rollback that raced it. + */ + switch (READ_ONCE(hyp_vcpu->power_state)) { + case PSCI_0_2_AFFINITY_LEVEL_ON: + break; + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + if (pkvm_reset_vcpu(hyp_vcpu)) + goto out; + break; + default: goto out; } diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 10ff2ba6c6c1..dcc1db6201f1 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -5,6 +5,7 @@ */ #include +#include #include #include @@ -431,6 +432,40 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc allowed_features, KVM_VCPU_MAX_FEATURES); } +static int pkvm_vcpu_init_psci(struct pkvm_hyp_vcpu *hyp_vcpu, u32 mp_state) +{ + struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + struct kvm_vcpu *host_vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + /* The host manages a non-protected vCPU: always ON at EL2. */ + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON; + return 0; + } + + if (mp_state != KVM_MP_STATE_RUNNABLE && mp_state != KVM_MP_STATE_STOPPED) + return -EINVAL; + + if (mp_state == KVM_MP_STATE_STOPPED) { + reset_state->reset = false; + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_OFF; + return 0; + } + + hyp_assert_lock_held(&vm_table_lock); + if (hyp_vm->primary_vcpu) + return -EINVAL; + hyp_vm->primary_vcpu = hyp_vcpu; + + host_vcpu = hyp_vcpu->host_vcpu; + reset_state->pc = READ_ONCE(*vcpu_pc(host_vcpu)); + reset_state->r0 = READ_ONCE(vcpu_gp_regs(host_vcpu)->regs[0]); + reset_state->reset = true; + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; + return 0; +} + static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) { if (host_vcpu) @@ -445,6 +480,9 @@ static void unpin_host_sve_state(struct pkvm_hyp_vcpu *hyp_vcpu) return; sve_state = hyp_vcpu->vcpu.arch.sve_state; + if (!sve_state) + return; + hyp_unpin_shared_mem(sve_state, sve_state + vcpu_sve_state_size(&hyp_vcpu->vcpu)); } @@ -557,10 +595,12 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *host_vcpu) { int ret = 0; + u32 mp_state; if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) return -EBUSY; + mp_state = READ_ONCE(host_vcpu->arch.mp_state.mp_state); hyp_vcpu->host_vcpu = host_vcpu; hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; @@ -569,7 +609,6 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); - hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { /* @@ -599,9 +638,12 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); + ret = pkvm_vcpu_init_psci(hyp_vcpu, mp_state); done: - if (ret) + if (ret) { unpin_host_vcpu(host_vcpu); + unpin_host_sve_state(hyp_vcpu); + } return ret; } @@ -978,13 +1020,22 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu); if (ret) - goto unlock; + goto unclaim; ret = register_hyp_vcpu(hyp_vm, hyp_vcpu); if (ret) { unpin_host_vcpu(host_vcpu); unpin_host_sve_state(hyp_vcpu); + goto unclaim; } + goto unlock; +unclaim: + /* + * Under vm_table_lock, so no other claim can have landed: undo + * this one. + */ + if (hyp_vm->primary_vcpu == hyp_vcpu) + hyp_vm->primary_vcpu = NULL; unlock: hyp_spin_unlock(&vm_table_lock); @@ -1176,6 +1227,238 @@ static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu) ret[0] = SMCCC_RET_SUCCESS; } +/* + * Reset the vCPU to its power-on state and commit ON_PENDING -> ON, on the + * target's own CPU. Returns -ECANCELED, with no side effects, if a rollback + * raced the reset. + * + * The requestor's PSCI CPU_ON, in pvm_psci_vcpu_on(): + * + * cmpxchg_relaxed(power_state): OFF -> ON_PENDING + * + * write_reset_state(); + * smp_store_release(reset_state->reset, true); + * + * The target coming online here, running, then its PSCI CPU_OFF in + * pvm_psci_vcpu_off(): + * + * smp_load_acquire(reset_state->reset) == true; + * cmpxchg_relaxed(power_state): ON_PENDING -> ON; + * read_reset_state(); + * + * + * + * WRITE_ONCE(reset_state->reset, false); + * smp_store_release(power_state, OFF); + */ +int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; + + /* + * Pairs with smp_store_release(&reset_state->reset, true) in + * pvm_psci_vcpu_on(). The acquire must precede the cmpxchg: reversed, a + * winning cmpxchg with a false acquire would leave power_state == ON + * with the reset skipped. + */ + if (!smp_load_acquire(&reset_state->reset)) + return -ECANCELED; + + if (cmpxchg_relaxed(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, + PSCI_0_2_AFFINITY_LEVEL_ON) != PSCI_0_2_AFFINITY_LEVEL_ON_PENDING) + return -ECANCELED; + + kvm_reset_vcpu_core(&hyp_vcpu->vcpu); + kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); + + /* Must be done after resetting sys registers. */ + kvm_reset_vcpu_psci(&hyp_vcpu->vcpu, reset_state); + + hyp_vcpu->exit_code = 0; + return 0; +} + +struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, unsigned long mpidr) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + int i; + + mpidr &= MPIDR_HWID_BITMASK; + + for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { + /* Pairs with smp_store_release() in register_hyp_vcpu(). */ + hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[i]); + + if (!hyp_vcpu) + continue; + + if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu)) + return hyp_vcpu; + } + + return NULL; +} + +/* + * Returns true when handled at EL2, false when the host must wake the target + * vCPU. + */ +static bool pvm_psci_vcpu_on(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + struct vcpu_reset_state *reset_state; + struct pkvm_hyp_vcpu *target; + unsigned long cpu_id, ret; + int power_state; + + cpu_id = smccc_get_arg1(&hyp_vcpu->vcpu); + if (!kvm_psci_valid_affinity(&hyp_vcpu->vcpu, cpu_id)) { + ret = PSCI_RET_INVALID_PARAMS; + goto error; + } + + target = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); + if (!target) { + ret = PSCI_RET_INVALID_PARAMS; + goto error; + } + + /* + * vCPUs race to power on the same target. The writes below depend on + * the cmpxchg reading OFF, and that control dependency with the release + * of OFF in pvm_psci_vcpu_off() orders them after the target's accesses + * to reset_state, so relaxed suffices. + */ + power_state = cmpxchg_relaxed(&target->power_state, + PSCI_0_2_AFFINITY_LEVEL_OFF, + PSCI_0_2_AFFINITY_LEVEL_ON_PENDING); + switch (power_state) { + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + ret = PSCI_RET_ON_PENDING; + goto error; + case PSCI_0_2_AFFINITY_LEVEL_ON: + ret = PSCI_RET_ALREADY_ON; + goto error; + case PSCI_0_2_AFFINITY_LEVEL_OFF: + break; + default: + ret = PSCI_RET_INTERNAL_FAILURE; + goto error; + } + + reset_state = &target->vcpu.arch.reset_state; + WRITE_ONCE(reset_state->pc, smccc_get_arg2(&hyp_vcpu->vcpu)); + WRITE_ONCE(reset_state->r0, smccc_get_arg3(&hyp_vcpu->vcpu)); + WRITE_ONCE(reset_state->be, kvm_vcpu_is_be(&hyp_vcpu->vcpu)); + /* + * Publish reset_state.{pc, r0, be} to the target vCPU. Pairs with + * smp_load_acquire(&reset_state->reset) in pkvm_reset_vcpu(). + */ + smp_store_release(&reset_state->reset, true); + + /* The host requests KVM_REQ_VCPU_RESET and wakes the target. */ + return false; + +error: + smccc_set_retval(&hyp_vcpu->vcpu, ret, 0, 0, 0); + return true; +} + +/* + * Returns true when handled at EL2, false when the host must stop scheduling + * the vCPU. + */ +static bool pvm_psci_vcpu_off(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + /* No other writer runs while this vCPU is ON and executing. */ + WARN_ON(READ_ONCE(hyp_vcpu->power_state) != PSCI_0_2_AFFINITY_LEVEL_ON); + + WRITE_ONCE(hyp_vcpu->vcpu.arch.reset_state.reset, false); + + /* + * Orders pkvm_reset_vcpu()'s accesses to reset_state, and the clear + * above, before OFF. Pairs with the control dependency in + * pvm_psci_vcpu_on(). + */ + smp_store_release(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_OFF); + + /* Return to the host so that it can finish powering off the vcpu. */ + return false; +} + +static bool pvm_psci_version(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + /* Nothing to be handled by the host. Go back to the guest. */ + smccc_set_retval(&hyp_vcpu->vcpu, KVM_ARM_PSCI_1_1, 0, 0, 0); + return true; +} + +static bool pvm_psci_features(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u32 feature = smccc_get_arg1(vcpu); + unsigned long val; + + switch (feature) { + case PSCI_0_2_FN_PSCI_VERSION: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_1_0_FN_PSCI_FEATURES: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + case ARM_SMCCC_VERSION_FUNC_ID: + val = PSCI_RET_SUCCESS; + break; + default: + val = PSCI_RET_NOT_SUPPORTED; + break; + } + + /* Nothing to be handled by the host. Go back to the guest. */ + smccc_set_retval(vcpu, val, 0, 0, 0); + return true; +} + +static bool pkvm_handle_psci(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u32 psci_fn = smccc_get_function(vcpu); + + switch (psci_fn) { + case PSCI_0_2_FN_CPU_ON: + kvm_psci_narrow_to_32bit(vcpu); + fallthrough; + case PSCI_0_2_FN64_CPU_ON: + return pvm_psci_vcpu_on(hyp_vcpu); + case PSCI_0_2_FN_CPU_OFF: + return pvm_psci_vcpu_off(hyp_vcpu); + case PSCI_0_2_FN_PSCI_VERSION: + return pvm_psci_version(hyp_vcpu); + case PSCI_1_0_FN_PSCI_FEATURES: + return pvm_psci_features(hyp_vcpu); + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + return false; /* Handled by the host. */ + default: + /* Unknown PSCI calls are handled here, not forwarded. */ + smccc_set_retval(vcpu, PSCI_RET_NOT_SUPPORTED, 0, 0, 0); + return true; + } +} + /* * Handler for protected VM HVC calls. * @@ -1185,6 +1468,7 @@ static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu) */ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) { + struct pkvm_hyp_vcpu *hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); u64 val[4] = { SMCCC_RET_INVALID_PARAMETER }; bool handled = true; u32 feature; @@ -1283,8 +1567,7 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) pkvm_memunshare_call(val, vcpu); break; default: - /* Punt everything else back to the host, for now. */ - handled = false; + return pkvm_handle_psci(hyp_vcpu); } if (handled) From 0b7d843ef3bf0a791d9e92c8a85178fe2a5ee9ff Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:04 +0100 Subject: [PATCH 396/506] KVM: arm64: Restrict KVM_ARM_VCPU_INIT and PSCI version for protected VMs EL2 masks a protected VM's vCPU features down to the set pKVM allows, so a KVM_ARM_VCPU_INIT that requests more succeeds and produces a guest without them. Reject such an init with -EINVAL. The host and EL2 both take the allowed set from kvm_pkvm_vcpu_allowed_features(). EL2 returns 1.1 for a protected guest's PSCI_VERSION whatever the host is configured for, and forwards the platform calls to the host. Require KVM_ARM_VCPU_PSCI_0_2: without it the host dispatches the forwarded calls as PSCI 0.1. Reject a KVM_REG_ARM_PSCI_VERSION write below 1.1: below it the host rejects the SYSTEM_RESET2 that EL2 advertises. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-12-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pkvm.h | 24 ++++++++++++++++++++++++ arch/arm64/kvm/arm.c | 13 +++++++++++++ arch/arm64/kvm/hyp/nvhe/pkvm.c | 17 ++--------------- arch/arm64/kvm/hypercalls.c | 7 +++++++ 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index beea00e693a0..fbea052fa3e1 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -53,6 +53,30 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) } } +/* + * The vCPU features a protected VM may use: checked by the host at + * KVM_ARM_VCPU_INIT, applied by EL2 when the hyp VM is created. + */ +static inline void kvm_pkvm_vcpu_allowed_features(struct kvm *kvm, + unsigned long *allowed) +{ + bitmap_zero(allowed, KVM_VCPU_MAX_FEATURES); + + set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3)) + set_bit(KVM_ARM_VCPU_PMU_V3, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS)) + set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) + set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) + set_bit(KVM_ARM_VCPU_SVE, allowed); +} + /* * Check whether the KVM VM IOCTL is allowed in pKVM. * diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index cceeeee26690..ca6e109b3e5d 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1620,6 +1620,19 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, if (features & ~system_supported_vcpu_features()) return -EINVAL; + /* Reject features EL2 would drop when it creates the hyp VM. */ + if (vcpu_is_protected(vcpu)) { + DECLARE_BITMAP(allowed, KVM_VCPU_MAX_FEATURES); + + kvm_pkvm_vcpu_allowed_features(vcpu->kvm, allowed); + if (!bitmap_subset(&features, allowed, KVM_VCPU_MAX_FEATURES)) + return -EINVAL; + + /* EL2 implements PSCI 1.1; the host must not dispatch as 0.1. */ + if (!test_bit(KVM_ARM_VCPU_PSCI_0_2, &features)) + return -EINVAL; + } + /* * For now make sure that both address/generic pointer authentication * features are requested by the userspace together. diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index dcc1db6201f1..696f999996b1 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -410,23 +410,10 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE)) kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_MTE_ENABLED); - bitmap_zero(allowed_features, KVM_VCPU_MAX_FEATURES); + kvm_pkvm_vcpu_allowed_features(kvm, allowed_features); - set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3)) - set_bit(KVM_ARM_VCPU_PMU_V3, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS)) - set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) - set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) { - set_bit(KVM_ARM_VCPU_SVE, allowed_features); + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_GUEST_HAS_SVE); - } bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, allowed_features, KVM_VCPU_MAX_FEATURES); diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index b11b8821c9fb..e43ed13c82be 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -580,6 +580,13 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) wants_02 = vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2); + /* + * EL2 advertises PSCI 1.1; the host handles the forwarded calls + * by this version. + */ + if (vcpu_is_protected(vcpu) && val < KVM_ARM_PSCI_1_1) + return -EINVAL; + switch (val) { case KVM_ARM_PSCI_0_1: if (wants_02) From fc519dabd86b07b7f7e285ce68b2993d370f64d3 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:05 +0100 Subject: [PATCH 397/506] KVM: arm64: Prevent host PC adjustments for protected vCPUs __kvm_adjust_pc() lets the host advance a vCPU's PC or inject an exception, which for a protected vCPU would let the host redirect guest execution. Leave the request on the host copy there: the entry handlers apply the host's PC_UPDATE_REQ at the next entry, where EL2 allows it. __kvm_adjust_pc() adjusts the vCPU a get/put pair returns, the one it was given outside pKVM. Both host callers hold the vCPU mutex, so a hyp vCPU loaded for the vCPU is loaded on the calling CPU, and EL2's own calls pass the hyp vCPU itself. For a loaded protected vCPU, kvm_adjust_pc_get() returns NULL and the request stays on the host copy. For a loaded non-protected vCPU, PKVM_HOST_STATE_DIRTY selects the copy, since adjusting the hyp vCPU while the host copy is authoritative loses the update at the next flush. Adjusting the hyp vCPU copies PC_UPDATE_REQ in and back out again: without the copy back, INCREMENT_PC outlives the adjustment and the next KVM_SET_VCPU_EVENTS trips WARN_ON(INCREMENT_PC) in kvm_pend_exception(). With no hyp vCPU loaded, as under KVM_SET_VCPU_EVENTS, the host copy is adjusted as before. Until the marshalling patch clears PC_UPDATE_REQ on the host copy at exit, a KVM_RUN that returns to userspace with INCREMENT_PC set leaves it on the host copy of a loaded protected vCPU, and a KVM_SET_VCPU_EVENTS before the next run then trips that WARN_ON. Suggested-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-13-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/exception.c | 21 +++++++++----- arch/arm64/kvm/hyp/include/hyp/adjust_pc.h | 20 ++++++++++++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 32 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 6e60d890afa4..93d68b1d38be 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -353,12 +353,19 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu) */ void __kvm_adjust_pc(struct kvm_vcpu *vcpu) { - if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) { - kvm_inject_exception(vcpu); - vcpu_clear_flag(vcpu, PENDING_EXCEPTION); - vcpu_clear_flag(vcpu, EXCEPT_MASK); - } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) { - kvm_skip_instr(vcpu); - vcpu_clear_flag(vcpu, INCREMENT_PC); + struct kvm_vcpu *target = kvm_adjust_pc_get(vcpu); + + if (!target) + return; + + if (vcpu_get_flag(target, PENDING_EXCEPTION)) { + kvm_inject_exception(target); + vcpu_clear_flag(target, PENDING_EXCEPTION); + vcpu_clear_flag(target, EXCEPT_MASK); + } else if (vcpu_get_flag(target, INCREMENT_PC)) { + kvm_skip_instr(target); + vcpu_clear_flag(target, INCREMENT_PC); } + + kvm_adjust_pc_put(vcpu, target); } diff --git a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h index f55950ee2a7e..a07936f87af8 100644 --- a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h +++ b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h @@ -78,4 +78,24 @@ static inline void kvm_skip_host_instr(void) write_sysreg_el2(read_sysreg_el2(SYS_ELR) + 4, SYS_ELR); } +/* + * Under pKVM, the vCPU __kvm_adjust_pc() adjusts for @vcpu (NULL leaves the + * request on @vcpu for its next entry), and the copy of the consumed + * PC_UPDATE_REQ back to @vcpu. + */ +#ifdef __KVM_NVHE_HYPERVISOR__ +struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu); +void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, struct kvm_vcpu *target); +#else +static inline struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu) +{ + return vcpu; +} + +static inline void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, + struct kvm_vcpu *target) +{ +} +#endif + #endif diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 00038a162d09..2fcd3f8cc8fc 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -644,6 +644,38 @@ static void handle___pkvm_host_mkyoung_guest(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = ret; } +/* + * PKVM_HOST_STATE_DIRTY names the authoritative copy, the host's when set. + * A loaded protected vCPU takes the request at its next entry instead. + */ +struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + + if (!is_protected_kvm_enabled()) + return vcpu; + + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu || vcpu == &hyp_vcpu->vcpu) + return vcpu; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return NULL; + + if (vcpu_get_flag(vcpu, PKVM_HOST_STATE_DIRTY)) + return vcpu; + + vcpu_copy_flag(&hyp_vcpu->vcpu, vcpu, PC_UPDATE_REQ); + return &hyp_vcpu->vcpu; +} + +/* Reflect the consumed request back, otherwise it stays pending. */ +void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, struct kvm_vcpu *target) +{ + if (target != vcpu) + vcpu_copy_flag(vcpu, target, PC_UPDATE_REQ); +} + static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); From 3cfbad49ededd985555820d97b62a27f92506b52 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:06 +0100 Subject: [PATCH 398/506] KVM: arm64: Inject an UNDEF at EL2 for unhandled protected guest exits Make pvm_exit_handlers[] an allow-list: a protected guest's exit class is handled at EL2, forwarded where the host emulates it, or results in an UNDEF at EL2. Any other class, once forwarded, would re-execute forever. The host resolves such an exit by injecting an exception or incrementing the PC. The marshalling that follows allows neither: a protected vCPU takes only the UNDEF on a SYS64 trap and the external abort on a forwarded IABT or DABT that EL2 builds for it. WFxT takes the UNDEF, as it isn't advertised to protected guests. FP/SIMD keeps the lazy switch, and the watchpoint entry is dropped: a protected vCPU runs with TDE clear. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-14-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/switch.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 7318e3e6a5f3..91e1edf40bcd 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -203,15 +203,36 @@ static const exit_handler_fn hyp_exit_handlers[] = { [ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops, }; +/* WFI/WFE exit to the host, which emulates them. WFxT is not advertised. */ +static bool kvm_handle_pvm_wfx(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (kvm_vcpu_get_esr(vcpu) & ESR_ELx_WFx_ISS_WFxT) + return kvm_handle_pvm_restricted(vcpu, exit_code); + + return false; +} + +/* Lazy FP/SIMD switch, or the UNDEF that would otherwise be left to the host. */ +static bool kvm_handle_pvm_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (kvm_hyp_handle_fpsimd(vcpu, exit_code)) + return true; + + return kvm_handle_pvm_restricted(vcpu, exit_code); +} + +/* + * A class not listed takes an UNDEF at EL2: the host has no way to + * inject one into a protected vCPU. + */ static const exit_handler_fn pvm_exit_handlers[] = { - [0 ... ESR_ELx_EC_MAX] = NULL, + [0 ... ESR_ELx_EC_MAX] = kvm_handle_pvm_restricted, + [ESR_ELx_EC_WFx] = kvm_handle_pvm_wfx, [ESR_ELx_EC_HVC64] = kvm_handle_pvm_hvc64, [ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64, - [ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted, - [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, + [ESR_ELx_EC_FP_ASIMD] = kvm_handle_pvm_fpsimd, [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, - [ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low, [ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops, }; From 872383bd12e116fc79cb4487b1740c84f23c4c23 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:07 +0100 Subject: [PATCH 399/506] KVM: arm64: Add per-EC entry/exit state marshalling for protected guests Move a protected guest's state between the hyp vCPU and the host per exception class instead of copying the whole context. Add entry_hyp_pvm_handlers[] and exit_hyp_pvm_handlers[] for WFx, SYS64, IABT, DABT and HVC64, and route protected guests through them: on exit each handler copies out only what its class requires, and on re-entry only what the host may have changed: the PC update or exception it requested (PC_UPDATE_REQ), the value of a read it emulated, and a forwarded PSCI call's return value. entry_hyp_vm_handlers[] is removed: a non-protected vCPU's iflags are copied wholesale. The host's fault view is EL2's own syndrome with the guest register index withheld, the deferred SError syndrome (DISR_EL1), and the addresses each class requires. The value of a written register is passed in r0. MMIO data is clamped to the access width and, for a load, sign-extended at EL2 from EL2's syndrome. Endianness stays with the host. A data abort's PC update is taken for a completed MMIO access, and for a cache maintenance operation the host skips on unbacked memory, as it does for any guest. The host's copy of a protected vCPU's PSTATE is a view the guest never runs from: its reset value has PSTATE.A set, and EL2 sets PSTATE.A in the mode it copies out regardless of the guest's, so, unless the VMM wrote the host copy's PSTATE or SCTLR2_EL1 before the first run, serror_is_masked() is true and kvm_inject_serror_esr() pends a host-injected SError through HCR_EL2.VSE, masked by the guest's own PSTATE.A, instead of emulating the entry on that copy. Neither dispatch runs for a trap taken with an SError pending: EL2 doesn't handle it, and the guest replays it once the host has injected the SError. The exit handlers would otherwise marshal a trap EL2 never handled, and handle_pvm_exit_hvc64() would panic on an unfiltered function id. For HVC64, only the PSCI calls EL2 forwards reach the host: the exit handler passes the function id and the arguments each call takes, and the entry handler returns the host's result. A CPU_ON the host failed is rolled back to OFF and returned as INTERNAL_FAILURE, or as ALREADY_ON when that's what the host returned: PSCI defines it as the retry signal for a CPU_ON that reaches the implementation before the target's CPU_OFF has been processed (DEN0022 section 6.6), which a guest that doesn't poll AFFINITY_INFO first can do. A target that already ran returns SUCCESS. The rollback leaves the published reset state in place: clearing it races a fresh CPU_ON's publication and leaves the target stuck at ON_PENDING, and the entry point left behind is one the guest supplied. The rollback stores OFF with a release, as CPU_OFF does, so this vCPU's writes to the reset state are ordered before the next winner's. The rollback's cmpxchg carries no generation, so one that lands after a later CPU_ON has republished ON_PENDING cancels that cycle too. The target is then OFF at EL2 while the host has it runnable, and its CPU_ONs return ALREADY_ON until the VMM stops it again. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-15-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 435 +++++++++++++++++++++++++++-- 1 file changed, 412 insertions(+), 23 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 2fcd3f8cc8fc..eb73374bbf87 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -4,6 +4,8 @@ * Author: Andrew Scull */ +#include + #include #include @@ -34,13 +36,371 @@ void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); typedef void (*hyp_entry_exit_handler_fn)(struct pkvm_hyp_vcpu *); -static void handle_vm_entry_generic(struct pkvm_hyp_vcpu *hyp_vcpu) +static bool pvm_sys64_is_write(u64 esr) { - vcpu_copy_flag(&hyp_vcpu->vcpu, hyp_vcpu->host_vcpu, PC_UPDATE_REQ); + return (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_WRITE; } -static const hyp_entry_exit_handler_fn entry_hyp_vm_handlers[] = { - [0 ... ESR_ELx_EC_MAX] = handle_vm_entry_generic, +static void handle_pvm_entry_wfx(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + /* Exceptions have priority; the host injects none on WFx. */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) + return; + + if (vcpu_get_flag(host_vcpu, INCREMENT_PC)) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } +} + +static void handle_pvm_entry_sys64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + bool pc_update; + + /* Exceptions have priority over anything else */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) { + /* A host-requested exception on SYS64 is always an UNDEF. */ + u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); + return; + } + + /* Handle PC increment on a host-emulated access */ + pc_update = vcpu_get_flag(host_vcpu, INCREMENT_PC); + if (pc_update) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } + + /* If the host emulated a read access, update the register */ + if (pc_update && !pvm_sys64_is_write(kvm_vcpu_get_esr(vcpu))) { + /* r0 as transfer register between the guest and the host. */ + u64 rt_val = READ_ONCE(vcpu_gp_regs(host_vcpu)->regs[0]); + int rt = kvm_vcpu_sys_get_rt(vcpu); + + vcpu_set_reg(vcpu, rt, rt_val); + } +} + +static void handle_pvm_entry_iabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + unsigned long cpsr = *vcpu_cpsr(vcpu); + u32 esr = ESR_ELx_IL; + + if (!vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) + return; + + /* The host's only IABT injection: an external abort. */ + if ((cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) + esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT); + else + esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT); + + esr |= ESR_ELx_FSC_EXTABT; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + __vcpu_assign_sys_reg(vcpu, FAR_EL1, kvm_vcpu_get_hfar(vcpu)); + + /* Injected by __kvm_adjust_pc() on entry. */ + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); +} + +/* + * Clamp MMIO data to the access width, so a write does not leak the + * register's upper bits and a read takes no bits beyond the load. The + * host applies endianness. + */ +static inline u64 kvm_mmio_clamp_data(struct kvm_vcpu *vcpu, u64 val) +{ + unsigned int len = kvm_vcpu_dabt_get_as(vcpu); + + return val & GENMASK_U64(len * 8 - 1, 0); +} + +/* + * Complete an MMIO load: sign-extend from EL2's own syndrome, as the + * architecture does. + */ +static inline u64 kvm_mmio_read_data(struct kvm_vcpu *vcpu, u64 val) +{ + val = kvm_mmio_clamp_data(vcpu, val); + + if (kvm_vcpu_dabt_issext(vcpu)) + val = sign_extend64(val, kvm_vcpu_dabt_get_as(vcpu) * 8 - 1); + + if (!kvm_vcpu_dabt_issf(vcpu)) + val &= GENMASK_U64(31, 0); + + return val; +} + +static void handle_pvm_entry_dabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + bool pc_update; + + /* Exceptions have priority over anything else */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) { + unsigned long cpsr = *vcpu_cpsr(vcpu); + u32 esr = ESR_ELx_IL; + + if ((cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) + esr |= (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT); + else + esr |= (ESR_ELx_EC_DABT_CUR << ESR_ELx_EC_SHIFT); + + esr |= ESR_ELx_FSC_EXTABT; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + __vcpu_assign_sys_reg(vcpu, FAR_EL1, kvm_vcpu_get_hfar(vcpu)); + + /* Injected by __kvm_adjust_pc() on entry. */ + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); + + /* Cancel any in-flight MMIO */ + vcpu->mmio_needed = false; + return; + } + + /* Handle PC increment on MMIO, or on a CMO the host skipped */ + pc_update = vcpu_get_flag(host_vcpu, INCREMENT_PC) && + (vcpu->mmio_needed || kvm_vcpu_dabt_is_cm(vcpu)); + if (pc_update) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } + + /* If the host emulated an MMIO read, update the register */ + if (pc_update && vcpu->mmio_needed && !kvm_vcpu_dabt_iswrite(vcpu)) { + /* r0 as transfer register between the guest and the host. */ + u64 rd_val = READ_ONCE(vcpu_gp_regs(host_vcpu)->regs[0]); + int rd = kvm_vcpu_dabt_get_rd(vcpu); + + rd_val = kvm_mmio_read_data(vcpu, rd_val); + vcpu_set_reg(vcpu, rd, rd_val); + } + + vcpu->mmio_needed = false; +} + +static void handle_pvm_entry_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 ret = READ_ONCE(vcpu_gp_regs(host_vcpu)->regs[0]); + u32 psci_fn = smccc_get_function(vcpu); + + switch (psci_fn) { + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + /* + * Roll back a CPU_ON the host failed, unless the target + * already reached ON: it is running, and the guest sees + * SUCCESS. + */ + if (ret != PSCI_RET_SUCCESS) { + unsigned long cpu_id = smccc_get_arg1(vcpu); + struct pkvm_hyp_vcpu *target_vcpu; + struct pkvm_hyp_vm *hyp_vm; + int prev; + + hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + target_vcpu = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); + + /* + * pvm_psci_vcpu_on() resolved this MPIDR and vcpus[] + * entries are never removed, so the lookup cannot miss. + * The release orders this vCPU's reset_state writes + * before OFF, for the next CPU_ON. + */ + prev = cmpxchg_release(&target_vcpu->power_state, + PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, + PSCI_0_2_AFFINITY_LEVEL_OFF); + switch (prev) { + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + /* + * Leave reset_state.reset set: a clear races a + * fresh CPU_ON's publish. The stale pc/r0/be are + * the guest's own. ALREADY_ON is PSCI's retry + * signal for a CPU_ON that raced the CPU_OFF. + */ + if (ret != PSCI_RET_ALREADY_ON) + ret = PSCI_RET_INTERNAL_FAILURE; + break; + case PSCI_0_2_AFFINITY_LEVEL_ON: + case PSCI_0_2_AFFINITY_LEVEL_OFF: + /* Target already ran (and may have stopped). */ + ret = PSCI_RET_SUCCESS; + break; + default: + ret = PSCI_RET_INTERNAL_FAILURE; + break; + } + } + + break; + default: + break; + } + + vcpu_set_reg(vcpu, 0, ret); +} + +/* The host's view of a syndrome: the guest register index is withheld. */ +static u64 pvm_host_esr(u64 esr) +{ + switch (ESR_ELx_EC(esr)) { + case ESR_ELx_EC_WFx: + return esr & ~ESR_ELx_WFx_ISS_RN; + case ESR_ELx_EC_SYS64: + return esr & ~ESR_ELx_SYS64_ISS_RT_MASK; + case ESR_ELx_EC_DABT_LOW: + return esr & ~ESR_ELx_SRT_MASK; + default: + return esr; + } +} + +/* + * The host's view of PSTATE: the mode, with SErrors masked so that the host + * pends an SError through HCR_EL2.VSE rather than emulating the entry. + */ +static unsigned long pvm_host_pstate(unsigned long pstate) +{ + return (pstate & PSR_MODE_MASK) | PSR_A_BIT; +} + +static void handle_pvm_exit_wfx(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); +} + +static void handle_pvm_exit_sys64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 esr = kvm_vcpu_get_esr(vcpu); + + /* The mode is required for the host to emulate some sysregs */ + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); + + /* r0 as transfer register between the guest and the host. */ + if (pvm_sys64_is_write(esr)) { + int rt = kvm_vcpu_sys_get_rt(vcpu); + u64 rt_val = vcpu_get_reg(vcpu, rt); + + vcpu_set_reg(host_vcpu, 0, rt_val); + } +} + +static void handle_pvm_exit_iabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + host_vcpu->arch.fault.hpfar_el2 = vcpu->arch.fault.hpfar_el2; +} + +static void handle_pvm_exit_dabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 sctlr; + + /* + * EL2 has no memslot view: a decodable data abort is prepared as MMIO + * for the host to resolve. On unbacked memory the host injects an SEA + * for one with ISV clear (LDP/STP, atomics), which EL2 does not + * decode, and skips a cache maintenance operation, as for any guest. + */ + vcpu->mmio_needed = kvm_vcpu_dabt_isvalid(vcpu); + + /* r0 as transfer register between the guest and the host. */ + if (vcpu->mmio_needed && kvm_vcpu_dabt_iswrite(vcpu)) { + int rt = kvm_vcpu_dabt_get_rd(vcpu); + u64 rt_val = vcpu_get_reg(vcpu, rt); + + rt_val = kvm_mmio_clamp_data(vcpu, rt_val); + vcpu_set_reg(host_vcpu, 0, rt_val); + } + + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); + host_vcpu->arch.fault.far_el2 = vcpu->arch.fault.far_el2 & GENMASK(11, 0); + host_vcpu->arch.fault.hpfar_el2 = vcpu->arch.fault.hpfar_el2; + sctlr = __vcpu_sys_reg(vcpu, SCTLR_EL1) & (SCTLR_ELx_EE | SCTLR_EL1_E0E); + __vcpu_assign_sys_reg(host_vcpu, SCTLR_EL1, sctlr); +} + +static void handle_pvm_exit_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + int n, i; + + switch (smccc_get_function(vcpu)) { + /* + * CPU_ON: the host uses only the target MPIDR (x1). EL2 resets the + * target from its own copy of the entry point and context id. + */ + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + n = 2; + break; + + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + n = 1; + break; + + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + n = 3; + break; + + /* Unreachable: kvm_handle_pvm_hvc64() forwards only the calls above. */ + default: + hyp_panic(); + } + + /* Pass the HVC function id (r0) and its arguments. */ + for (i = 0; i < n; i++) + vcpu_set_reg(host_vcpu, i, vcpu_get_reg(vcpu, i)); +} + +static const hyp_entry_exit_handler_fn entry_hyp_pvm_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_WFx] = handle_pvm_entry_wfx, + [ESR_ELx_EC_SYS64] = handle_pvm_entry_sys64, + [ESR_ELx_EC_IABT_LOW] = handle_pvm_entry_iabt, + [ESR_ELx_EC_DABT_LOW] = handle_pvm_entry_dabt, + [ESR_ELx_EC_HVC64] = handle_pvm_entry_hvc64, +}; + +static const hyp_entry_exit_handler_fn exit_hyp_pvm_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_WFx] = handle_pvm_exit_wfx, + [ESR_ELx_EC_SYS64] = handle_pvm_exit_sys64, + [ESR_ELx_EC_IABT_LOW] = handle_pvm_exit_iabt, + [ESR_ELx_EC_DABT_LOW] = handle_pvm_exit_dabt, + [ESR_ELx_EC_HVC64] = handle_pvm_exit_hvc64, }; static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) @@ -284,18 +644,6 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; - } else { - u64 v_cval = hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTV_CVAL_EL0]; - u64 v_ctl = hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTV_CTL_EL0]; - u64 p_cval = hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTP_CVAL_EL0]; - u64 p_ctl = hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTP_CTL_EL0]; - - hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; - - hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTV_CVAL_EL0] = v_cval; - hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTV_CTL_EL0] = v_ctl; - hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTP_CVAL_EL0] = p_cval; - hyp_vcpu->vcpu.arch.ctxt.sys_regs[CNTP_CTL_EL0] = p_ctl; } /* __hyp_running_vcpu must be NULL in a guest context. */ @@ -320,10 +668,16 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) case ARM_EXCEPTION_IL: break; case ARM_EXCEPTION_TRAP: - esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); - ec_handler = entry_hyp_vm_handlers[esr_ec]; - if (ec_handler) - ec_handler(hyp_vcpu); + /* Nothing was marshalled for this trap, see sync_hyp_vcpu(). */ + if (ARM_SERROR_PENDING(hyp_vcpu->exit_code)) + break; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); + ec_handler = entry_hyp_pvm_handlers[esr_ec]; + if (ec_handler) + ec_handler(hyp_vcpu); + } break; default: BUG(); @@ -335,13 +689,26 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + hyp_entry_exit_handler_fn ec_handler; + u8 esr_ec; fpsimd_sve_sync(&hyp_vcpu->vcpu); sync_debug_state(hyp_vcpu); if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { - host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; + /* + * Protected: the host sees ESR_EL2 as EL2 took it, register + * index withheld; the fault addresses stay withheld unless the + * EC handler below adds them. + */ + host_vcpu->arch.fault = (struct kvm_vcpu_fault_info) { + .esr_el2 = pvm_host_esr(hyp_vcpu->vcpu.arch.fault.esr_el2), + .disr_el1 = hyp_vcpu->vcpu.arch.fault.disr_el1, + }; } else { + /* Non-protected: the host gets the full fault. */ + host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; + host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; /* * PC feeds trace_kvm_exit(), PSTATE.SS the host software-step * machine, and both run before the next on-demand ctxt sync. @@ -350,9 +717,28 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) host_vcpu->arch.ctxt.regs.pstate = hyp_vcpu->vcpu.arch.ctxt.regs.pstate; } - host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; + switch (ARM_EXCEPTION_CODE(exit_reason)) { + case ARM_EXCEPTION_IRQ: + break; + case ARM_EXCEPTION_TRAP: + /* SError pending: not handled at EL2, the guest replays it. */ + if (ARM_SERROR_PENDING(exit_reason)) + break; - host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; + /* Per-EC marshalling is for protected guests only. */ + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); + ec_handler = exit_hyp_pvm_handlers[esr_ec]; + if (ec_handler) + ec_handler(hyp_vcpu); + } + break; + case ARM_EXCEPTION_EL1_SERROR: + case ARM_EXCEPTION_IL: + break; + default: + BUG(); + } /* Cleared by hardware once the guest takes the vSError. */ host_vcpu->arch.hcr_el2 &= ~HCR_VSE; @@ -361,6 +747,9 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) sync_hyp_vgic_state(hyp_vcpu); sync_hyp_timer_state(hyp_vcpu); + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + vcpu_clear_flag(host_vcpu, PC_UPDATE_REQ); + hyp_vcpu->exit_code = exit_reason; } From ad4ebc3decaa19fad2c00868e237561b6ac004c6 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:08 +0100 Subject: [PATCH 400/506] KVM: arm64: Reject host access to protected VM private state A protected vCPU's register and debug state is no longer exposed to the host, and the host ioctls that would reach it fail instead: - KVM_GET_ONE_REG and KVM_SET_ONE_REG return -EPERM once the vCPU has run: the host copy then holds the boot state the VMM wrote plus what the exit handlers copy out, which the VMM can't tell apart. Pre-run access still builds the guest's boot state. - KVM_ARM_VCPU_INIT returns -EPERM once the vCPU has run: it would reset the host copy alone and rewrite mp_state, which EL2 reads only at hyp vCPU creation, so a vCPU the guest powered off would come back RUNNABLE. - KVM_SET_VCPU_EVENTS rejects external-abort injection with -EPERM; SError injection is forwarded and stays permitted. - KVM_SET_GUEST_DEBUG returns -EPERM: a protected guest's debug state is hypervisor-owned. The KVM_{GET,SET}_ONE_REG and KVM_ARM_VCPU_INIT checks are one filter on the ioctl number in kvm_arch_vcpu_ioctl(), the only caller of the three functions they were in. Its -EPERM now precedes the cases' -EFAULT and the ONE_REG case's pending-reset handling, which the next KVM_RUN performs. The external-abort check reads the payload, and KVM_SET_GUEST_DEBUG has its own case in kvm_vcpu_ioctl(), so it never reaches kvm_arch_vcpu_ioctl(): those two stay in their handlers. KVM_CHECK_EXTENSION returns 0 for KVM_CAP_ARM_INJECT_EXT_DABT and KVM_CAP_SET_GUEST_DEBUG on a protected VM: kvm_pkvm_ext_allowed() returns false on every capability it doesn't list, and the patch that advertises the capabilities protected VMs support leaves these two out. The two ioctl checks stay for a VMM that doesn't query KVM_CHECK_EXTENSION. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-16-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 22 ++++++++++++++++++++++ arch/arm64/kvm/guest.c | 11 +++++++++++ 2 files changed, 33 insertions(+) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index ca6e109b3e5d..14fe4e727579 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1865,6 +1865,24 @@ static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, return __kvm_arm_vcpu_set_events(vcpu, events); } +/* + * Once a protected vCPU has run, the host copy holds the boot state the VMM + * wrote plus what the exit handlers copy out, and EL2 has read mp_state, + * which it does only at hyp vCPU creation. + */ +static long pkvm_filter_vcpu_ioctl(struct kvm_vcpu *vcpu, unsigned int ioctl) +{ + switch (ioctl) { + case KVM_ARM_VCPU_INIT: + case KVM_SET_ONE_REG: + case KVM_GET_ONE_REG: + if (vcpu_is_protected(vcpu) && vcpu_has_run_once(vcpu)) + return -EPERM; + } + + return 0; +} + long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1873,6 +1891,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp, struct kvm_device_attr attr; long r; + r = pkvm_filter_vcpu_ioctl(vcpu, ioctl); + if (r) + return r; + switch (ioctl) { case KVM_ARM_VCPU_INIT: { struct kvm_vcpu_init init; diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index b01d6622b872..d5e2e08f0546 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -786,6 +786,13 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, u64 esr = events->exception.serror_esr; int ret = 0; + /* + * EL2 injects an external abort only to complete a forwarded abort. + * SError injection is forwarded. + */ + if (vcpu_is_protected(vcpu) && ext_dabt_pending) + return -EPERM; + /* * Immediately commit the pending SEA to the vCPU's architectural * state which is necessary since we do not return a pending SEA @@ -883,6 +890,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, { trace_kvm_set_guest_debug(vcpu, dbg->control); + /* A protected guest's debug state is not exposed to the host. */ + if (vcpu_is_protected(vcpu)) + return -EPERM; + if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) return -EINVAL; From 25d726dddc237043327aed9a2b51a5b6336bc31a Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:09 +0100 Subject: [PATCH 401/506] KVM: arm64: Reject host power-on of a vCPU that EL2 holds powered off A protected vCPU's power state belongs to EL2, which changes it only on the guest's own PSCI calls. KVM_SET_MP_STATE(RUNNABLE) on a vCPU EL2 holds powered off changes the host's copy alone, and the guest's next CPU_ON of that vCPU then fails: mp_state is no longer STOPPED, so the host returns ALREADY_ON, and the guest's retry fails the same way until the VMM stops the vCPU again. SUSPENDED has the same effect. Record a vCPU EL2 holds powered off as KVM_MP_STATE_UNINITIALIZED, an encoding arm64 didn't use: the guest's CPU_OFF writes it, as does hyp vCPU creation for a vCPU that starts STOPPED, and only the guest's CPU_ON writes RUNNABLE over it. RUNNABLE and SUSPENDED from the VMM return -EPERM on it, the VMM can't write the value itself, and KVM_ARM_VCPU_INIT, which would write STOPPED or RUNNABLE over it, is rejected once the vCPU has run. The VMM can still pause and resume a vCPU EL2 has powered on. Neither STOPPED nor kvm_prepare_system_event() overwrites the value: a vCPU at it is already off, and a STOPPED in its place would turn the guest's next CPU_ON into ALREADY_ON, after a SYSTEM_RESET exit the VMM runs on from. kvm_arm_vcpu_stopped() covers it, so the run loop sleeps the vCPU as before. The guest's CPU_ON is gated on the value rather than on STOPPED. A STOPPED written between the target's CPU_OFF exit and the host's power-off of it, which kvm_prepare_system_event() can do from another vCPU's SYSTEM_OFF or SYSTEM_RESET, would otherwise let the CPU_ON through, and the power-off would then record the target powered off while EL2 holds it ON_PENDING, with no way back before VM teardown. Gated on the value, that CPU_ON returns ALREADY_ON and the retry succeeds once the power-off has run. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-17-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/arm.c | 35 +++++++++++++++++++++++++------ arch/arm64/kvm/pkvm.c | 21 +++++++++++++++---- arch/arm64/kvm/psci.c | 11 ++++++++-- 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 37d0721d39a4..e99b706c879c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1564,6 +1564,7 @@ static inline void kvm_hyp_reserve(void) { } void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu); bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu); +bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu); static inline u64 *__vm_id_reg(struct kvm_arch *ka, u32 reg) { diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 14fe4e727579..a89087c83e27 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -770,23 +770,28 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) vcpu->cpu = -1; } -static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) +static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu, u32 mp_state) { - WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + WRITE_ONCE(vcpu->arch.mp_state.mp_state, mp_state); kvm_make_request(KVM_REQ_SLEEP, vcpu); kvm_vcpu_kick(vcpu); } +/* The guest's own CPU_OFF: EL2 has already powered the vCPU off. */ void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) { + u32 mp_state = vcpu_is_protected(vcpu) ? KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_STOPPED; + spin_lock(&vcpu->arch.mp_state_lock); - __kvm_arm_vcpu_power_off(vcpu); + __kvm_arm_vcpu_power_off(vcpu, mp_state); spin_unlock(&vcpu->arch.mp_state_lock); } bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu) { - return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED; + u32 mp_state = READ_ONCE(vcpu->arch.mp_state.mp_state); + + return mp_state == KVM_MP_STATE_STOPPED || mp_state == KVM_MP_STATE_UNINITIALIZED; } static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu) @@ -801,6 +806,12 @@ static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu) return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED; } +/* Only the guest's CPU_ON may start a vCPU EL2 holds powered off. */ +bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu) +{ + return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_UNINITIALIZED; +} + int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -818,12 +829,24 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, switch (mp_state->mp_state) { case KVM_MP_STATE_RUNNABLE: + if (kvm_pkvm_vcpu_is_powered_off(vcpu)) { + ret = -EPERM; + break; + } + WRITE_ONCE(vcpu->arch.mp_state, *mp_state); break; case KVM_MP_STATE_STOPPED: - __kvm_arm_vcpu_power_off(vcpu); + /* Keep EL2's off record: STOPPED over it would let RUNNABLE through. */ + if (!kvm_pkvm_vcpu_is_powered_off(vcpu)) + __kvm_arm_vcpu_power_off(vcpu, KVM_MP_STATE_STOPPED); break; case KVM_MP_STATE_SUSPENDED: + if (kvm_pkvm_vcpu_is_powered_off(vcpu)) { + ret = -EPERM; + break; + } + kvm_arm_vcpu_suspend(vcpu); break; default: @@ -1788,7 +1811,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, spin_lock(&vcpu->arch.mp_state_lock); if (power_off) - __kvm_arm_vcpu_power_off(vcpu); + __kvm_arm_vcpu_power_off(vcpu, KVM_MP_STATE_STOPPED); else WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4c6e4bec12..99a5d9a91312 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -118,12 +118,25 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) return -ENOMEM; ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, vcpu, hyp_vcpu); - if (!ret) - vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); - else + if (ret) { free_pages_exact(hyp_vcpu, hyp_vcpu_sz); + return ret; + } - return ret; + /* + * Mirror EL2's seeding of power_state from mp_state. The hyp vCPU is + * published, so take mp_state_lock against kvm_psci_vcpu_on(). + */ + if (vcpu_is_protected(vcpu)) { + spin_lock(&vcpu->arch.mp_state_lock); + if (kvm_arm_vcpu_stopped(vcpu)) + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_UNINITIALIZED); + spin_unlock(&vcpu->arch.mp_state_lock); + } + + vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); + + return 0; } /* diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index e3db84400d1f..b01f8d44033f 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -63,7 +63,12 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) return PSCI_RET_INVALID_PARAMS; spin_lock(&vcpu->arch.mp_state_lock); - if (!kvm_arm_vcpu_stopped(vcpu)) { + /* + * A protected vCPU is off once the host has acted on EL2's CPU_OFF. + * STOPPED before that is the VMM's pause. + */ + if (!kvm_arm_vcpu_stopped(vcpu) || + (vcpu_is_protected(vcpu) && !kvm_pkvm_vcpu_is_powered_off(vcpu))) { if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) ret = PSCI_RET_ALREADY_ON; else @@ -161,7 +166,9 @@ static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags) */ kvm_for_each_vcpu(i, tmp, vcpu->kvm) { spin_lock(&tmp->arch.mp_state_lock); - WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + /* Keep EL2's off record for the guest's next CPU_ON. */ + if (!kvm_pkvm_vcpu_is_powered_off(tmp)) + WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); spin_unlock(&tmp->arch.mp_state_lock); } kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP); From 4957c24559e90592cfb60b6d4425296c2a7cd0e0 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:10 +0100 Subject: [PATCH 402/506] KVM: arm64: Advertise the capabilities that protected VMs support For a protected VM, kvm_pkvm_ext_allowed() returns false on every capability it doesn't name, so KVM_CHECK_EXTENSION returns 0 for interfaces the VM implements: it denies KVM_CAP_ONE_REG while KVM_{GET,SET}_ONE_REG stage the guest's boot state. Allow the capabilities that work for a protected guest under the restrictions the preceding patches add, the vGIC and the I/O bus being host-managed. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-18-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pkvm.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index fbea052fa3e1..a165af0d7568 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -42,6 +42,15 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) case KVM_CAP_ARM_VM_IPA_SIZE: case KVM_CAP_ARM_PTRAUTH_ADDRESS: case KVM_CAP_ARM_PTRAUTH_GENERIC: + case KVM_CAP_ONE_REG: + case KVM_CAP_MP_STATE: + case KVM_CAP_VCPU_EVENTS: + case KVM_CAP_VCPU_ATTRIBUTES: + case KVM_CAP_IMMEDIATE_EXIT: + case KVM_CAP_IOEVENTFD: + case KVM_CAP_IRQFD_RESAMPLE: + case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2: + case KVM_CAP_ARM_INJECT_SERROR_ESR: return true; case KVM_CAP_ARM_MTE: return false; From 61ace86519fb830a7b271c2a1037c8db5f2f3333 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 1 Oct 2026 14:57:11 +0100 Subject: [PATCH 403/506] KVM: arm64: Document the protected VM userspace API EL2 owns a protected vCPU's state once it has run, and the ioctls that would reach it fail rather than operate on the host copy, which holds the boot state the VMM wrote plus what the exit handlers copy out. Describe the resulting API in pkvm.rst, point api.rst at it from each ioctl that behaves differently, and say beside the PVTIME attribute in devices/vcpu.rst and the PSCI version register in fw-pseudo-registers.rst how protected VMs differ. The overview also states the GIC support pKVM enforces: GICv2 is not supported in protected mode (vgic-v2.c) and GICv5-based guests aren't registered under pKVM (vgic-v5.c). Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261001135711.1640520-19-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- Documentation/virt/kvm/api.rst | 27 ++- .../virt/kvm/arm/fw-pseudo-registers.rst | 2 + Documentation/virt/kvm/arm/pkvm.rst | 164 +++++++++++++++++- Documentation/virt/kvm/devices/vcpu.rst | 3 +- 4 files changed, 185 insertions(+), 11 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc750c9..7fc0fed0edb1 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -1322,6 +1322,9 @@ guests, across different userspace implementations. Nevertheless, userspace can still emulate all Arm exceptions by manipulating individual registers using the KVM_SET_ONE_REG API. +For a protected VM, setting ext_dabt_pending returns -EPERM; see +Documentation/virt/kvm/arm/pkvm.rst. Injecting an SError is unaffected. + See KVM_GET_VCPU_EVENTS for the data structure. Calling this ioctl on a vCPU that hasn't been initialized will return @@ -1545,7 +1548,9 @@ Possible values are: KVM_MP_STATE_RUNNABLE the vcpu is currently running [x86,arm64,riscv,loongarch] KVM_MP_STATE_UNINITIALIZED the vcpu is an application processor (AP) - which has not yet received an INIT signal [x86] + which has not yet received an INIT signal + [x86], or a protected VM's vcpu that the + guest holds powered off [arm64] KVM_MP_STATE_INIT_RECEIVED the vcpu has received an INIT signal, and is now ready for a SIPI [x86] KVM_MP_STATE_HALTED the vcpu has executed a HLT instruction and @@ -1624,6 +1629,12 @@ For arm64/riscv: The only states that are valid are KVM_MP_STATE_STOPPED and KVM_MP_STATE_RUNNABLE which reflect if the vcpu should be paused or not. +On arm64, once a protected VM's vcpu has run, it reads as +KVM_MP_STATE_UNINITIALIZED while the guest holds it powered off, with +CPU_OFF or by not yet bringing it online with CPU_ON, and +KVM_MP_STATE_RUNNABLE and KVM_MP_STATE_SUSPENDED return -EPERM on it: only +the guest can power it on. See Documentation/virt/kvm/arm/pkvm.rst. + On LoongArch, only the KVM_MP_STATE_RUNNABLE state is used to reflect whether the vcpu is runnable. @@ -2324,7 +2335,8 @@ Errors: ENOENT no such register EINVAL invalid register ID, or no such register or used with VMs in protected virtualization mode on s390 - EPERM (arm64) register access not allowed before vcpu finalization + EPERM (arm64) register access not allowed before vcpu + finalization, or after a protected VM's vcpu has run EBUSY (riscv) changing register value not allowed after the vcpu has run at least once ====== ============================================================ @@ -2949,7 +2961,8 @@ Errors include: ENOENT no such register EINVAL invalid register ID, or no such register or used with VMs in protected virtualization mode on s390 - EPERM (arm64) register access not allowed before vcpu finalization + EPERM (arm64) register access not allowed before vcpu + finalization, or after a protected VM's vcpu has run ======== ============================================================ (These error codes are indicative only: do not rely on a specific error @@ -3484,6 +3497,7 @@ Errors: ====== ================================================================= EINVAL the target is unknown, or the combination of features is invalid. ENOENT a features bit specified is unknown. + EPERM the vcpu belongs to a protected VM and has already run. ====== ================================================================= This tells KVM what type of CPU to present to the guest, and what @@ -3512,6 +3526,10 @@ after the vcpu has been run. This will reset the vcpu to its initial state. All calls to this function after the initial call must use the same target and same set of feature flags, otherwise EINVAL will be returned. +For a protected VM this ioctl returns EPERM once the vcpu has run, and the +features it accepts are restricted: KVM_ARM_VCPU_PSCI_0_2 is required. See +Documentation/virt/kvm/arm/pkvm.rst. + Possible features: - KVM_ARM_VCPU_POWER_OFF: Starts the CPU in a power-off state. @@ -3790,6 +3808,9 @@ can be determined by querying the KVM_CAP_GUEST_DEBUG_HW_BPS and KVM_CAP_GUEST_DEBUG_HW_WPS capabilities which return a positive number indicating the number of supported registers. +On arm64, this ioctl returns -EPERM for a protected VM: debugging a +protected guest is not supported. See Documentation/virt/kvm/arm/pkvm.rst. + For ppc, the KVM_CAP_PPC_GUEST_DEBUG_SSTEP capability indicates whether the single-step debug event (KVM_GUESTDBG_SINGLESTEP) is supported. diff --git a/Documentation/virt/kvm/arm/fw-pseudo-registers.rst b/Documentation/virt/kvm/arm/fw-pseudo-registers.rst index d78b53b05dfc..c07471e258ea 100644 --- a/Documentation/virt/kvm/arm/fw-pseudo-registers.rst +++ b/Documentation/virt/kvm/arm/fw-pseudo-registers.rst @@ -33,6 +33,8 @@ The following registers are defined: highest PSCI version implemented by KVM and compatible with v0.2) - Allows any PSCI version implemented by KVM and compatible with v0.2 to be set with SET_ONE_REG + - Returns -EINVAL for a version below v1.1 on a protected VM (see + Documentation/virt/kvm/arm/pkvm.rst) - Affects the whole VM (even if the register view is per-vcpu) * KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: diff --git a/Documentation/virt/kvm/arm/pkvm.rst b/Documentation/virt/kvm/arm/pkvm.rst index 514992a79a83..b75bb8ad6f26 100644 --- a/Documentation/virt/kvm/arm/pkvm.rst +++ b/Documentation/virt/kvm/arm/pkvm.rst @@ -15,7 +15,9 @@ Overview Booting a host kernel with '``kvm-arm.mode=protected``' enables "Protected KVM" (pKVM). During boot, pKVM installs a stage-2 identity map page-table for the host and uses it to isolate the hypervisor -running at EL2 from the rest of the host running at EL1/0. +running at EL2 from the rest of the host running at EL1/0. pKVM supports +GICv3 only: GICv2 is not supported in protected mode, and neither are +GICv5-based guests. pKVM permits creation of protected virtual machines (pVMs) by passing the ``KVM_VM_TYPE_ARM_PROTECTED`` machine type identifier to the @@ -31,7 +33,7 @@ See hypercalls.rst for more details. Isolation mechanisms ==================== -pKVM relies on a number of mechanisms to isolate PVMs from the host: +pKVM relies on a number of mechanisms to isolate pVMs from the host: CPU memory isolation -------------------- @@ -50,8 +52,9 @@ stage-2 identity map as part of the donation procedure. This gives rise to some user-visible differences when compared to non-protected VMs, largely due to the lack of MMU notifiers: -* Memslots cannot be moved or deleted once the pVM has started running. -* Read-only memslots and dirty logging are not supported. +* Memslots cannot be moved or deleted once the pVM has started running + (``-EPERM``). +* Read-only memslots and dirty logging are not supported (``-EPERM``). * With the exception of swap, file-backed pages cannot be mapped into a pVM. * Donated pages are accounted against ``RLIMIT_MLOCK`` and so the VMM @@ -67,12 +70,19 @@ largely due to the lack of MMU notifiers: then it will either return ``-EFAULT`` or forcefully reclaim the memory pages. Reclaimed memory is zeroed by the hypervisor and a subsequent attempt to access it in the pVM will return ``-EFAULT`` - from the ``VCPU_RUN`` ioctl(). + from the ``KVM_RUN`` ioctl(). CPU state isolation ------------------- -Status: **Unimplemented.** +Status: CPU register state of protected vCPUs is managed entirely at EL2. + +pKVM performs the complete context switch for protected vCPUs within the +hypervisor. The hypervisor sets a protected vCPU's initial state, not the +host, and only what each exit needs is synchronised back. + +The user-visible consequences are described under `API behaviour for +protected VMs`_. DMA isolation using an IOMMU ---------------------------- @@ -89,13 +99,153 @@ The FF-A proxy ensures that the host cannot share pVM or hypervisor memory with Trustzone as part of a "confused deputy" attack. The PSCI proxy ensures that CPUs always have the stage-2 identity map -installed when they are executing in the host. +installed when they are executing in the host. This proxy is distinct +from the PSCI handling provided to protected guests, which is described +under `API behaviour for protected VMs`_. Protected VM firmware (pvmfw) ----------------------------- Status: **Unimplemented.** +API behaviour for protected VMs +=============================== + +Protected vCPU state is owned by EL2 (see `CPU state isolation`_). The VMM +configures a vCPU before its first ``KVM_RUN``; afterwards the state is +private to the guest and the ioctls that access it return ``-EPERM``. The +vCPU ioctls' errors follow one rule: ``-EPERM`` means the host asked for +state that the guest owns, ``-EINVAL`` means the request is not valid for a +protected VM, and a feature pKVM doesn't support for protected VMs is +rejected the way KVM rejects one that isn't implemented (``-ENXIO`` from +``KVM_ARM_VCPU_PVTIME_CTRL``, for example). The ioctls themselves are +described in Documentation/virt/kvm/api.rst. + +Boot +---- + +A protected VM boots from a single primary vCPU. Before the first +``KVM_RUN``, the VMM prepares the boot state: + +* Set ``KVM_MP_STATE_RUNNABLE`` on the primary vCPU and + ``KVM_MP_STATE_STOPPED`` on every other vCPU. EL2 allows only one + RUNNABLE primary per protected VM. A second RUNNABLE vCPU fails at its + first ``KVM_RUN``. +* Set the primary vCPU's boot state with ``KVM_SET_ONE_REG``: the kernel + entry address in ``PC`` and the DTB pointer in ``x0``. + +``PC`` and ``x0`` are the only registers EL2 takes from the host. Other +pre-run writes are accepted, but the guest starts from the state the +hypervisor resets it to. + +Secondary vCPUs are started by the guest itself through PSCI ``CPU_ON`` +(see `Power state`_), which supplies their entry point and context ID. +The VMM cannot choose where they boot. + +vCPU state +---------- + +* ``KVM_GET_ONE_REG`` and ``KVM_SET_ONE_REG`` return ``-EPERM`` once the + vCPU has run. Before that, they access the host-side copy from which + EL2 builds the guest's boot state (see `Boot`_). +* ``KVM_ARM_VCPU_INIT`` accepts only the vCPU features that a protected + guest supports and returns ``-EINVAL`` otherwise. + ``KVM_ARM_VCPU_PSCI_0_2`` is required, as EL2 implements PSCI 1.1 for + the guest (see `Power state`_). ``KVM_ARM_VCPU_EL1_32BIT`` is not + supported: protected guests run in AArch64 only and see no AArch32 + support in ``ID_AA64PFR0_EL1``. Once the vCPU has run, + ``KVM_ARM_VCPU_INIT`` returns ``-EPERM``, as re-initialising it would + reset the host-side copy alone. +* ``KVM_SET_VCPU_EVENTS`` returns ``-EPERM`` for external-abort injection + (``ext_dabt_pending``), and ``KVM_CAP_ARM_INJECT_EXT_DABT`` is not + advertised to a protected VM. SError injection is unaffected. +* ``KVM_SET_GUEST_DEBUG`` returns ``-EPERM``, and ``KVM_CAP_SET_GUEST_DEBUG`` + is not advertised to a protected VM (see `Debug`_). + +Power state +----------- + +EL2 implements PSCI 1.1 for a protected guest. The calls that move a +vCPU's power state, ``CPU_ON`` and ``CPU_OFF``, are handled at EL2, and +the host cannot change the outcome: for ``CPU_ON`` the host only +schedules the target, which EL2 has already reset to the entry point +the guest chose, and for ``CPU_OFF`` it only stops scheduling it. +``AFFINITY_INFO`` is handled by the host, which returns ``OFF`` once +it has stopped scheduling every vCPU the query covers, as for a +non-protected VM. A vCPU becomes a valid ``CPU_ON`` target at its first +``KVM_RUN``, and before that EL2 returns ``INVALID_PARAMETERS``. The +platform calls, ``CPU_SUSPEND``, ``SYSTEM_OFF``, ``SYSTEM_RESET`` and +``SYSTEM_RESET2``, are forwarded to the host and behave as for a +non-protected VM, with the ``SYSTEM_*`` calls exiting to the VMM as +``KVM_EXIT_SYSTEM_EVENT``; the ``SYSTEM_RESET2`` reset type and cookie +are in the guest's registers, which ``KVM_GET_ONE_REG`` rejects once +the vCPU has run. Any other function returns ``NOT_SUPPORTED``, and +``PSCI_FEATURES`` reports the same set. Because the host handles those +forwarded calls, ``KVM_SET_ONE_REG`` on ``KVM_REG_ARM_PSCI_VERSION`` +returns ``-EINVAL`` for a version below 1.1. + +Once a vCPU has run, its power state follows the guest's PSCI calls, not +the VMM's. A vCPU that the guest has powered off with ``CPU_OFF``, or has +not yet brought online with ``CPU_ON``, reads as +``KVM_MP_STATE_UNINITIALIZED``, and only an in-guest ``CPU_ON`` can power +it on: ``KVM_SET_MP_STATE`` returns ``-EPERM`` for ``KVM_MP_STATE_RUNNABLE`` +and ``KVM_MP_STATE_SUSPENDED`` on it, accepts ``KVM_MP_STATE_STOPPED`` and +leaves it as it is, and never takes ``KVM_MP_STATE_UNINITIALIZED`` from +the VMM. ``KVM_MP_STATE_STOPPED`` still stops a vCPU EL2 has powered on, +and ``KVM_MP_STATE_RUNNABLE`` resumes it, so the VMM can pause it. + +Other interface differences +--------------------------- + +* ``KVM_CHECK_EXTENSION`` reports only the arm64 capabilities that pKVM + supports for protected guests, and ``KVM_ENABLE_CAP`` accepts only + those. Query them on the VM file descriptor: the system file descriptor + has no VM to filter against. The capabilities the generic KVM code + answers itself (``KVM_CAP_SYNC_MMU`` or ``KVM_CAP_DEVICE_CTRL``, for + example) are reported as for any VM, and don't mean the feature works + for a protected one. The filter does not cover every interface either: + device-fd configuration (for example the VGIC after + ``KVM_CREATE_DEVICE``) and vCPU attributes are unfiltered, so a feature + the filter excludes is rejected by the attribute itself. + ``KVM_ARM_VCPU_PVTIME_CTRL`` returns ``-ENXIO``, for example, as it does + where steal time isn't implemented. +* The vGIC of a protected VM remains host-managed: device creation, + configuration and interrupt injection all work as they do for a + non-protected VM. +* ``KVM_ARM_SET_COUNTER_OFFSET`` and ``KVM_ARM_GET_REG_WRITABLE_MASKS`` + return ``-EINVAL``: their capabilities are not advertised to a protected + VM, whose counter offset and ID registers are set by EL2. A protected + guest sees the physical timebase. +* A protected guest's first access to each page of memory exits to the + host, since the hypervisor cannot tell memory from a device before the + page is mapped. For a store, the host sees the value of the register + the syndrome names, clamped to the access width, and nothing else from + the register file. For a load, the register the syndrome names takes + the value the host returns, as for any emulated access. +* A protected guest that uses a feature it was not given, or executes an + ``SMC``, takes an undefined instruction exception from the hypervisor; + the host is not involved. +* Apart from PSCI, the hypervisor handles a protected guest's SMCCC + calls itself and does not involve the host. A function it does not + implement returns ``NOT_SUPPORTED``, and the host's SMCCC filter does + not apply: the VM device attribute ioctls (``KVM_ARM_VM_SMCCC_CTRL``) + return ``-EINVAL`` for a protected VM. +* The hypervisor can decode a trapped guest access only from the CPU's + instruction syndrome, which is provided only for a load or store of a + single general-purpose register. An access without one (for example a + load/store pair or a SIMD/FP access) cannot be decoded. For a + non-protected VM it can exit to the VMM as ``KVM_EXIT_ARM_NISV``. For a + protected VM it cannot be emulated by the VMM, so the guest takes a + synchronous external abort instead. + +Debug +----- + +Hardware-assisted debugging is not available to protected guests. +``MDSCR_EL1``, ``OSLAR_EL1``, ``OSLSR_EL1``, ``OSDLR_EL1``, and breakpoint +and watchpoint 0 are RAZ/WI; any other debug register access takes an +undefined instruction exception. + Resources ========= diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst index deb5c51bc00c..916407835acd 100644 --- a/Documentation/virt/kvm/devices/vcpu.rst +++ b/Documentation/virt/kvm/devices/vcpu.rst @@ -231,7 +231,8 @@ Returns: Specifies the base address of the stolen time structure for this VCPU. The base address must be 64 byte aligned and exist within a valid guest memory region. See Documentation/virt/kvm/arm/pvtime.rst for more information -including the layout of the stolen time structure. +including the layout of the stolen time structure. Stolen time is not +supported for protected VMs (see Documentation/virt/kvm/arm/pkvm.rst). 4. GROUP: KVM_VCPU_TSC_CTRL =========================== From 88b23a2e8286b7016c3b58dc10093cf28c22aa3d Mon Sep 17 00:00:00 2001 From: Loc Nguyen Date: Fri, 2 Oct 2026 05:42:54 +0000 Subject: [PATCH 404/506] KVM: x86: Request event processing for exception VM-Exits Request event processing whenever kvm_queue_exception_vmexit() queues an exception VM-Exit. Most callers reach the helper through kvm_multiple_exception(), which already sets KVM_REQ_EVENT, but nested asynchronous page faults call it directly. Without another event request, vcpu_enter_guest() skips kvm_check_and_inject_events() and re-enters L2. A subsequent VM-Exit can clear the pending exception while reconstructing vectoring state, losing the synthetic #PF VM-Exit. The APF reason remains PAGE_NOT_PRESENT and a later regular #PF can consume the stale value. Setting KVM_REQ_EVENT in the common helper ensures that L1 receives the queued VM-Exit before KVM re-enters L2. Fixes: 7709aba8f716 ("KVM: x86: Morph pending exceptions to pending VM-Exits at queue time") Cc: stable@vger.kernel.org Signed-off-by: Loc Nguyen Link: https://patch.msgid.link/20261002053723.3275202-2-loc@ryzome.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5efd17b360de..727d68c1331d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -456,6 +456,8 @@ static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vecto { struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; + kvm_make_request(KVM_REQ_EVENT, vcpu); + ex->vector = vector; ex->injected = false; ex->pending = true; From 4b66a1941718ef1c945b32ab28e91330e4394de2 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:32 +0100 Subject: [PATCH 405/506] KVM: arm64: Add pkvm_private_va_range_pa Mappings in the pKVM private range are not identity mapped, making the standard __hyp_pa() unsuitable for translating these addresses. Introduce pkvm_private_va_range_pa() to resolve physical addresses for this range by walking the hypervisor page-table. This will be useful for the upcoming pKVM heap allocator. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-2-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/mm.h | 1 + arch/arm64/kvm/hyp/include/nvhe/spinlock.h | 4 ++++ arch/arm64/kvm/hyp/nvhe/mm.c | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 6e83ce35c2f2..85f1e3c711d9 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -30,5 +30,6 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, unsigned long *haddr); int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr); int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr); +phys_addr_t pkvm_private_va_range_pa(void *va); #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h index 7c7ea8c55405..260e14d6d200 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h +++ b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h @@ -17,6 +17,8 @@ #include #include +#include + typedef union hyp_spinlock { u32 __val; struct { @@ -122,4 +124,6 @@ static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) { } #endif +DEFINE_LOCK_GUARD_1(hyp_spinlock, hyp_spinlock_t, hyp_spin_lock(_T->lock), + hyp_spin_unlock(_T->lock)) #endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 3b0bee496bff..755630c0d15e 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -90,6 +90,24 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) return ret; } +/** + * pkvm_private_va_range_pa - Translate a private VA to a physical address. + * @va: The private virtual address to translate. + * + * Return: The physical address corresponding to @va. + */ +phys_addr_t pkvm_private_va_range_pa(void *va) +{ + kvm_pte_t pte; + s8 level; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + WARN_ON(kvm_pgtable_get_leaf(&pkvm_pgtable, (u64)va, &pte, &level)); + WARN_ON(!kvm_pte_valid(pte)); + + return kvm_pte_to_phys(pte) + ((u64)va & (kvm_granule_size(level) - 1)); +} + int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr) From 64a89873e0c60fd8a22ebacab6bc723415eeb4b2 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:33 +0100 Subject: [PATCH 406/506] KVM: arm64: Add pkvm_remove_mappings Add the counterpart to pkvm_create_mappings(), allowing previously mapped ranges to be removed. This will be useful for the upcoming pKVM heap allocator to manage its private mappings. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-3-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/mm.h | 1 + arch/arm64/kvm/hyp/nvhe/mm.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 85f1e3c711d9..00cadd13ee5d 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -25,6 +25,7 @@ int hyp_back_vmemmap(phys_addr_t back); int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot); int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot); int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot); +void pkvm_remove_mappings(void *from, void *to); int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr); diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 755630c0d15e..9ab7f613fa36 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -164,6 +164,24 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) return ret; } +/** + * pkvm_remove_mappings - Remove mappings from the hypervisor page-table + * @from: The starting virtual address of the range to remove + * @to: The ending virtual address of the range to remove + */ +void pkvm_remove_mappings(void *from, void *to) +{ + u64 size; + + WARN_ON(from > to); + to = PTR_ALIGN(to, PAGE_SIZE); + from = PTR_ALIGN_DOWN(from, PAGE_SIZE); + size = (u64)to - (u64)from; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, (u64)from, size) != size); +} + int hyp_back_vmemmap(phys_addr_t back) { unsigned long i, start, size, end = 0; From 06b705e789ed1a118331bc77f477a9b46c52dcfe Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:34 +0100 Subject: [PATCH 407/506] KVM: arm64: Add pkvm_map_private_va_range In preparation for the pKVM heap allocator, introduce pkvm_map_private_va_range() to map a physical range into the pKVM private range. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-4-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/mm.h | 1 + arch/arm64/kvm/hyp/nvhe/mm.c | 59 ++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 00cadd13ee5d..ea97ea49820e 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -31,6 +31,7 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, unsigned long *haddr); int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr); int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr); +int pkvm_map_private_va_range(void *haddr, phys_addr_t phys, size_t size); phys_addr_t pkvm_private_va_range_pa(void *va); #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 9ab7f613fa36..4a22fb74182b 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -25,6 +25,7 @@ struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS]; unsigned int hyp_memblock_nr; static u64 __io_map_base; +static u64 __io_map_next; struct hyp_fixmap_slot { u64 addr; @@ -50,7 +51,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) hyp_assert_lock_held(&pkvm_pgd_lock); - if (!start || start < __io_map_base) + if (!start || start < __io_map_next) return -EINVAL; /* The allocated size is always a multiple of PAGE_SIZE */ @@ -60,7 +61,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) if (cur > __hyp_vmemmap) return -ENOMEM; - __io_map_base = cur; + __io_map_next = cur; return 0; } @@ -70,7 +71,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) * @size: The size of the VA range to reserve. * @haddr: The hypervisor virtual start address of the allocation. * - * The private virtual address (VA) range is allocated above __io_map_base + * The private virtual address (VA) range is allocated above __io_map_next * and aligned based on the order of @size. * * Return: 0 on success or negative error code on failure. @@ -81,7 +82,7 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) int ret; hyp_spin_lock(&pkvm_pgd_lock); - addr = __io_map_base; + addr = __io_map_next; ret = __pkvm_alloc_private_va_range(addr, size); hyp_spin_unlock(&pkvm_pgd_lock); @@ -90,6 +91,41 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) return ret; } +/** + * pkvm_map_private_va_range() - Map a physical range into the private VA range + * @haddr: The virtual address in the private range. + * @phys: The physical address to map. + * @size: The size of the range to map. + * + * The hypervisor VA @haddr must have been first allocated with + * pkvm_alloc_private_va_range(). The created mapping can later be removed + * with pkvm_remove_mappings(). + * + * Return: 0 on success, negative error code on failure. + */ +int pkvm_map_private_va_range(void *haddr, phys_addr_t phys, size_t size) +{ + unsigned long addr = (unsigned long)haddr; + int ret; + + if (!PAGE_ALIGNED(addr | phys | size)) + return -EINVAL; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + + if (addr < __io_map_base || addr >= __io_map_next) + return -EINVAL; + + if (size > __io_map_next - addr) + return -EINVAL; + + ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, PAGE_HYP); + if (ret) + kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size); + + return ret; +} + /** * pkvm_private_va_range_pa - Translate a private VA to a physical address. * @va: The private virtual address to translate. @@ -168,6 +204,12 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) * pkvm_remove_mappings - Remove mappings from the hypervisor page-table * @from: The starting virtual address of the range to remove * @to: The ending virtual address of the range to remove + * + * The range [@from, @to) can be a subset of the range previously mapped by + * pkvm_create_mappings(), as the latter enforces PTE-level mappings. However, + * it cannot be a subset of a range mapped by pkvm_map_private_va_range() as + * that function allows block mappings which the hypervisor page-table does not + * split. */ void pkvm_remove_mappings(void *from, void *to) { @@ -377,7 +419,7 @@ static int create_fixblock(void) return -EINVAL; hyp_spin_lock(&pkvm_pgd_lock); - addr = ALIGN(__io_map_base, PMD_SIZE); + addr = ALIGN(__io_map_next, PMD_SIZE); ret = __pkvm_alloc_private_va_range(addr, PMD_SIZE); if (ret) goto unlock; @@ -462,6 +504,7 @@ int hyp_create_idmap(u32 hyp_va_bits) */ __io_map_base = start & BIT(hyp_va_bits - 2); __io_map_base ^= BIT(hyp_va_bits - 2); + __io_map_next = __io_map_base; __hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3); return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC); @@ -475,13 +518,13 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) hyp_spin_lock(&pkvm_pgd_lock); - prev_base = __io_map_base; + prev_base = __io_map_next; /* * Efficient stack verification using the NVHE_STACK_SHIFT bit implies * an alignment of our allocation on the order of the size. */ size = NVHE_STACK_SIZE * 2; - addr = ALIGN(__io_map_base, size); + addr = ALIGN(__io_map_next, size); ret = __pkvm_alloc_private_va_range(addr, size); if (!ret) { @@ -497,7 +540,7 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr + NVHE_STACK_SIZE, NVHE_STACK_SIZE, phys, PAGE_HYP); if (ret) - __io_map_base = prev_base; + __io_map_next = prev_base; } hyp_spin_unlock(&pkvm_pgd_lock); From c8d7834c9c219a9a0e3969986e90c5c444b9d98d Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:35 +0100 Subject: [PATCH 408/506] KVM: arm64: Add a heap allocator for the pKVM hyp Currently, memory used by the hypervisor comes from donations that are embedded within HVCs. e.g. __pkvm_init_vm()'s hyp_vm. This is cumbersome: the host needs to know the size of those struct, the memory must be page-aligned and physically contiguous which may be difficult to satisfy when host memory is highly fragmented. Create a heap allocator to manage VA-contiguous memory. This allocator grows upward, recycles unused chunks of memory and provides a simple API to allocate and free: hyp_alloc(size), hyp_free(addr) This heap allocator also manages the underlying physical memory, allowing the host to top up the allocator's pool and reclaim memory. hyp_alloc_topup(), hyp_alloc_reclaim(). Pages remain mapped in the allocator's VA-space as long as they are not reclaimed. When the allocator runs out of memory, hyp_alloc() fails and hyp_alloc_errno() returns -ENOMEM to signal that a top-up is required. Additionally, harden the allocator with a fast hash of the chunks meta-data. This intends to detect accidental corruptions and makes the allocator slightly more robust to potential attacks. Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-5-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/alloc.h | 17 + arch/arm64/kvm/hyp/nvhe/Makefile | 2 +- arch/arm64/kvm/hyp/nvhe/alloc.c | 1027 +++++++++++++++++++++++ arch/arm64/kvm/hyp/nvhe/setup.c | 6 + 4 files changed, 1051 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kvm/hyp/include/nvhe/alloc.h create mode 100644 arch/arm64/kvm/hyp/nvhe/alloc.c diff --git a/arch/arm64/kvm/hyp/include/nvhe/alloc.h b/arch/arm64/kvm/hyp/include/nvhe/alloc.h new file mode 100644 index 000000000000..8f87a63f8946 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/alloc.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __KVM_NVHE_ALLOC__ +#define __KVM_NVHE_ALLOC__ +#include + +#include + +void *hyp_alloc(size_t size); +int hyp_alloc_errno(void); +u32 hyp_alloc_topup_needed(void); +void hyp_free(void *addr); + +int hyp_alloc_init(size_t size); +int hyp_alloc_topup(struct kvm_hyp_memcache *host_mc); +unsigned long hyp_alloc_reclaimable(void); +void hyp_alloc_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target); +#endif diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index ccc1fe839409..ee0abb2e6af5 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -23,7 +23,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs)) CFLAGS_switch.nvhe.o += -Wno-override-init hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \ - hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ + hyp-main.o hyp-smp.o psci-relay.o alloc.o early_alloc.o page_alloc.o \ cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o diff --git a/arch/arm64/kvm/hyp/nvhe/alloc.c b/arch/arm64/kvm/hyp/nvhe/alloc.c new file mode 100644 index 000000000000..e0104d2dd6c0 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/alloc.c @@ -0,0 +1,1027 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2026 Google LLC + * Author: Vincent Donnefort + * + * This heap allocator manages a reserved VA space range, dynamically mapping + * and unmapping physical pages on-demand to minimise the pKVM hypervisor + * footprint. As memory is reclaimed and relinquished to the host, unmapped + * holes are introduced within the VA space. To prevent orphans mapped regions, + * neighboring unused chunks cannot be merged if they are separated by an + * unmapped region. + * + */ + +#include +#include +#include +#include + +#include +#include + +/* + * A minimum of 16 bytes allows us to comply with the alignment requirement for + * FPSIMD registers that are part of struct pkvm_hyp_vcpu. + * + * It also must be a power of two. + */ +#define MIN_ALLOC_SIZE 16UL + +/** + * struct chunk_hdr - Chunk header + * @next: offset from this chunk header to the next one. + * @prev: offset from this chunk header to the previous one. + * @__unmapped: Internal field containing the offset to the unmapped page + * boundary, multiplexed with the allocation state flag. + * @hash: Hash computed over the chunk header. + */ +struct chunk_hdr { + u32 next; + u32 prev; +#define USED_BIT_MASK 1U + u32 __unmapped; + u32 hash; + char data[]; +} __aligned(MIN_ALLOC_SIZE); + +/** + * struct hyp_allocator - Heap allocator + * @start: Start in the allocator's reserved virtual address range. + * @end: End in the allocator's reserved virtual address range. + * @last_used: Pointer to the end of the last used chunk. This is + * necessary for the last chunk in the list as the + * allocated size of a chunk is derived from the next one. + * @first_unmapped: Pointer to the first unmapped page in the + * allocator's range. This is only necessary and + * updated when no chunk is in the list. + * @head: Head of the chunk list. + * @tail: Tail of the chunk list. + * @mc: Memcache containing pre-allocated pages for mapping. + * @lock: Spinlock protecting the allocator state. + * @errno: Per-CPU error code for allocation failures. + * @topup_needed: Per-CPU page counter needed to top-up the memcache. + */ +struct hyp_allocator { + void *start; + void *end; + void *last_used; + void *first_unmapped; + struct chunk_hdr *head; + struct chunk_hdr *tail; + struct kvm_hyp_memcache mc; + hyp_spinlock_t lock; + int __percpu *errno; + u32 __percpu *topup_needed; +}; + +static u32 chunk_hash_compute(const struct chunk_hdr *chunk) +{ + u32 hash = 0; + + BUILD_BUG_ON(sizeof(*chunk) != 16); + + hash ^= hash_64(*(const u64 *)chunk, 32); + hash ^= hash_32(chunk->__unmapped, 32); + return hash; +} + +static void chunk_set_hash(struct chunk_hdr *chunk) +{ + if (chunk) + chunk->hash = chunk_hash_compute(chunk); +} + +static void chunk_check_hash(const struct chunk_hdr *chunk) +{ + if (chunk) + WARN_ON(chunk->hash != chunk_hash_compute(chunk)); +} + +static bool chunk_is_used(const struct chunk_hdr *chunk) +{ + return !!(chunk->__unmapped & USED_BIT_MASK); +} + +static void chunk_set_used(struct chunk_hdr *chunk) +{ + chunk->__unmapped |= USED_BIT_MASK; +} + +static void chunk_set_unused(struct chunk_hdr *chunk) +{ + chunk->__unmapped &= ~USED_BIT_MASK; +} + +static void *chunk_unmapped(const struct chunk_hdr *chunk) +{ + u32 offset = chunk->__unmapped & ~USED_BIT_MASK; + + if (!offset) + return NULL; + + return (void *)chunk + offset; +} + +static void __chunk_set_unmapped(struct chunk_hdr *chunk, u32 unmapped) +{ + chunk->__unmapped = unmapped | (chunk_is_used(chunk) ? USED_BIT_MASK : 0); +} + +static void chunk_set_unmapped(struct chunk_hdr *chunk, void *unmapped) +{ + WARN_ON(!PAGE_ALIGNED(unmapped)); + + if (unmapped) { + WARN_ON((void *)chunk > unmapped); + __chunk_set_unmapped(chunk, unmapped - (void *)chunk); + } else { + __chunk_set_unmapped(chunk, 0); + } +} + +static void *chunk_data(const struct chunk_hdr *chunk) +{ + return (void *)&chunk->data; +} + +static struct chunk_hdr *__chunk_next(const struct chunk_hdr *chunk) +{ + if (!chunk->next) + return NULL; + + return (struct chunk_hdr *)((void *)chunk + chunk->next); +} + +static struct chunk_hdr *__chunk_prev(const struct chunk_hdr *chunk) +{ + if (!chunk->prev) + return NULL; + + return (struct chunk_hdr *)((void *)chunk - chunk->prev); +} + +static void chunk_set_next(struct chunk_hdr *chunk, struct chunk_hdr *next) +{ + if (!chunk) + return; + + if (next) { + WARN_ON(chunk > next); + chunk->next = (void *)next - (void *)chunk; + } else { + chunk->next = 0; + } +} + +static void chunk_set_prev(struct chunk_hdr *chunk, struct chunk_hdr *prev) +{ + if (!chunk) + return; + + if (prev) { + WARN_ON(chunk < prev); + chunk->prev = (void *)chunk - (void *)prev; + } else { + chunk->prev = 0; + } +} + +static struct chunk_hdr *chunk_get_next(const struct chunk_hdr *chunk) +{ + struct chunk_hdr *next = __chunk_next(chunk); + + chunk_check_hash(next); + return next; +} + +static struct chunk_hdr *chunk_get_prev(const struct chunk_hdr *chunk) +{ + struct chunk_hdr *prev = __chunk_prev(chunk); + + chunk_check_hash(prev); + return prev; +} + +static struct chunk_hdr *chunk_get(struct chunk_hdr *chunk) +{ + chunk_check_hash(chunk); + return chunk; +} + +#define chunk_hdr_size() \ + offsetof(struct chunk_hdr, data) + +#define chunk_min_size() \ + (chunk_hdr_size() + MIN_ALLOC_SIZE) + +static size_t chunk_data_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + void *end; + + if (next) + end = (void *)next; + else + end = allocator->end; + + return end - chunk_data(chunk); +} + +static size_t chunk_mapped_data_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + void *unmapped = chunk_unmapped(chunk); + + if (!unmapped) + return chunk_data_size(chunk, allocator); + + return unmapped - chunk_data(chunk); +} + +static size_t chunk_used_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + + if (!chunk_is_used(chunk)) + return 0; + + if (next) + return chunk_mapped_data_size(chunk, allocator); + + return allocator->last_used - chunk_data(chunk); +} + +static void chunk_list_insert(struct chunk_hdr *chunk, struct chunk_hdr *prev) +{ + struct chunk_hdr *next = NULL; + + WARN_ON(!chunk); + + if (prev) { + next = chunk_get_next(prev); + chunk_set_next(prev, chunk); + chunk_set_hash(prev); + } + + if (next) { + chunk_set_prev(next, chunk); + chunk_set_hash(next); + } + + chunk_set_next(chunk, next); + chunk_set_prev(chunk, prev); +} + +static void chunk_list_del(struct chunk_hdr *chunk) +{ + struct chunk_hdr *prev, *next; + + WARN_ON(!chunk); + + prev = chunk_get_prev(chunk); + next = chunk_get_next(chunk); + + if (prev) { + chunk_set_next(prev, next); + chunk_set_hash(prev); + } + + if (next) { + chunk_set_prev(next, prev); + chunk_set_hash(next); + } +} + +/* + * Return a fixup start address for chunk creation. It makes sure the chunk + * header doesn't cross any page boundary and that it leaves enough space at the + * start of page. This is intended to prevent orphan mapped regions during chunk + * memory reclaim + */ +static void *chunk_start(void *start) +{ + void *page = PTR_ALIGN(start, PAGE_SIZE); + + if (page - start < chunk_hdr_size()) + return page; + + page = PTR_ALIGN_DOWN(start, PAGE_SIZE); + if (start - page < chunk_min_size()) + return page + chunk_min_size(); + + return start; +} + +static void hyp_allocator_set_errno(struct hyp_allocator *allocator, int errno) +{ + *this_cpu_ptr(allocator->errno) = errno; +} + +static int hyp_allocator_errno(struct hyp_allocator *allocator) +{ + int *errno = this_cpu_ptr(allocator->errno); + int ret = *errno; + + *errno = 0; + + return ret; +} + +static int hyp_allocator_map(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + struct chunk_hdr *next, + void *addr, void *end) +{ + void *unmapped = chunk ? chunk_unmapped(chunk) : allocator->first_unmapped; + void *unmapped_end = next ?: allocator->end; + + /* + * hyp_allocator_can_create_chunk() already validates addr/end + * belong to the chunk. + */ + WARN_ON(end <= addr); + + /* The chunk does not span an unmapped region */ + if (!unmapped) + return 0; + + while (unmapped < end) { + void *page = pop_hyp_memcache(&allocator->mc, hyp_phys_to_virt); + int ret; + + if (!page) { + end = PTR_ALIGN(end, PAGE_SIZE); + *this_cpu_ptr(allocator->topup_needed) = + (unsigned long)(end - unmapped) >> PAGE_SHIFT; + return -ENOMEM; + } + + ret = pkvm_map_private_va_range(unmapped, hyp_virt_to_phys(page), PAGE_SIZE); + if (ret) { + push_hyp_memcache(&allocator->mc, page, hyp_virt_to_phys); + return ret; + } + + unmapped += PAGE_SIZE; + + /* + * Reset the unmap field if we've reached the next chunk or the + * allocator boundary. Chunks are always installed to cover a + * mapped region followed by an unmapped one. Therefore, we + * expect next to be PAGE_ALIGNED. + */ + WARN_ON(!PAGE_ALIGNED(unmapped_end)); + if (unmapped == unmapped_end) + unmapped = NULL; + + if (chunk) { + chunk_set_unmapped(chunk, unmapped); + chunk_set_hash(chunk); + } else { + allocator->first_unmapped = unmapped; + } + + if (!unmapped) + break; + } + + return 0; +} + +static void hyp_allocator_unmap(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + void *addr, void *end) +{ + void *unmap = addr; + + /* + * hyp_allocator_chunk_reclaimable() already computes valid addr/end, no + * need to check them again + */ + WARN_ON(end <= addr); + + while (unmap < end) { + phys_addr_t pa = pkvm_private_va_range_pa((void *)unmap); + void *page = hyp_phys_to_virt(pa); + + push_hyp_memcache(&allocator->mc, page, hyp_virt_to_phys); + unmap += PAGE_SIZE; + } + + pkvm_remove_mappings(addr, end); + + if (chunk) { + chunk_set_unmapped(chunk, addr); + chunk_set_hash(chunk); + } else { + allocator->first_unmapped = addr; + } +} + +static bool hyp_allocator_can_create_chunk(struct hyp_allocator *allocator, + const struct chunk_hdr *prev, + const struct chunk_hdr *next, + void *addr, void *end) +{ + void *page, *unmapped; + + if (addr < allocator->start || end > allocator->end) + return false; + + /* First chunk created must be installed at allocator->start */ + if (!prev) + return addr == allocator->start; + + /* Must not overwrite the next chunk */ + if (next && end > (void *)next) + return false; + + /* Must not overwrite the previous chunk */ + if (addr < (chunk_data(prev) + chunk_used_size(prev, allocator))) + return false; + + /* Header must not cross page boundaries */ + page = PTR_ALIGN(addr, PAGE_SIZE); + if (page != addr && (page - addr) < chunk_hdr_size()) + return false; + + /* Must leave a minimum distance from a page-start to maximise reclaim */ + page = PTR_ALIGN_DOWN(addr, PAGE_SIZE); + if (page != addr && (addr - page) < chunk_min_size()) + return false; + + unmapped = chunk_unmapped(prev); + if (!unmapped) + return true; + + /* Must never create an orphan mapped region */ + if (addr > unmapped) + return false; + + return true; +} + +/* + * Tries to create a new chunk in the allocator whose header starts at @addr and + * whose data finishes at @end. + */ +static struct chunk_hdr *hyp_allocator_create_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *prev, void *addr, + void *end, bool used) +{ + struct chunk_hdr *next, *chunk = addr; + void *unmapped; + int ret; + + if (end > allocator->end) + return ERR_PTR(-E2BIG); + + next = prev ? chunk_get_next(prev) : NULL; + if (!hyp_allocator_can_create_chunk(allocator, prev, next, addr, end)) + return ERR_PTR(-EINVAL); + + ret = hyp_allocator_map(allocator, prev, next, addr, end); + if (ret) + return ERR_PTR(ret); + + memset(chunk, 0, sizeof(*chunk)); + if (used) + chunk_set_used(chunk); + else + chunk_set_unused(chunk); + + /* First chunk, first allocation */ + if (!prev) { + chunk_set_unmapped(chunk, allocator->first_unmapped); + chunk_list_insert(chunk, NULL); + chunk_set_hash(chunk); + + allocator->last_used = end; + allocator->head = allocator->tail = chunk; + return chunk; + } + + /* Last chunk in the list */ + if (!next) { + allocator->last_used = end; + allocator->tail = chunk; + } + + /* Inherit prev's unmapped region */ + unmapped = chunk_unmapped(prev); + chunk_set_unmapped(chunk, unmapped); + chunk_list_insert(chunk, prev); + chunk_set_hash(chunk); + + chunk_set_unmapped(prev, NULL); + chunk_set_hash(prev); + + return chunk; +} + +static bool hyp_allocator_can_destroy_chunk(const struct chunk_hdr *prev, + const struct chunk_hdr *next, + const struct chunk_hdr *chunk) +{ + if (chunk_is_used(chunk)) + return false; + + /* Can't merge down unless we are the last one in the list */ + if (next && chunk_is_used(prev)) + return false; + + /* Must never create an orphan mapped region */ + if (chunk_unmapped(prev)) + return false; + + return true; +} + +static int hyp_allocator_destroy_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *prev, + struct chunk_hdr *chunk) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + + if (!hyp_allocator_can_destroy_chunk(prev, next, chunk)) + return -EINVAL; + + /* Last chunk in the list */ + if (!next) { + allocator->last_used = chunk; + allocator->tail = prev; + } + + chunk_set_unmapped(prev, chunk_unmapped(chunk)); + chunk_set_hash(prev); + chunk_list_del(chunk); + + return 0; +} + +/* + * Return the best unused chunk for recycling, that is the smallest chunk + * fitting the allocation which needs to use the least unmapped region. + */ +static struct chunk_hdr *hyp_allocator_find_efficient_chunk(struct hyp_allocator *allocator, + size_t size) +{ + struct chunk_hdr *chunk, *best_chunk = NULL; + size_t best_data_size = SIZE_MAX; + size_t best_missing = SIZE_MAX; + + chunk = allocator->head; + while (chunk) { + size_t missing, mapped, data_size; + + if (chunk_is_used(chunk)) + goto next; + + data_size = chunk_data_size(chunk, allocator); + if (data_size < size) + goto next; + + mapped = chunk_mapped_data_size(chunk, allocator); + missing = (size > mapped) ? DIV_ROUND_UP(size - mapped, PAGE_SIZE) : 0; + if (missing > best_missing) + goto next; + + if (missing == best_missing && data_size >= best_data_size) + goto next; + + best_missing = missing; + best_data_size = data_size; + best_chunk = chunk; + +next: + chunk = chunk_get_next(chunk); + } + + return best_chunk; +} + +static struct chunk_hdr *hyp_allocator_reuse_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *chunk, size_t size) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + void *start, *end, *split, *split_end; + int ret; + + start = chunk_data(chunk); + end = start + size; + + /* Last chunk in the list, no need to split */ + if (!next) { + split = split_end = NULL; + allocator->last_used = chunk_data(chunk) + size; + } else { + split = chunk_start(end); + split_end = split + chunk_min_size(); + + if (!hyp_allocator_can_create_chunk(allocator, chunk, next, split, split_end)) + split = split_end = NULL; + } + + /* Batch the mapping of the reused chunk and the split */ + ret = hyp_allocator_map(allocator, chunk, next, chunk_data(chunk), split ? split_end : end); + if (ret) + return ERR_PTR(ret); + + if (split) + WARN_ON(IS_ERR_OR_NULL( + hyp_allocator_create_chunk(allocator, chunk, split, split_end, false))); + + chunk_set_used(chunk); + chunk_set_hash(chunk); + + return chunk; +} + +static struct chunk_hdr *hyp_allocator_alloc_chunk(struct hyp_allocator *allocator, size_t size) +{ + struct chunk_hdr *chunk; + void *start, *end; + + /* First allocation */ + if (!allocator->head) { + start = allocator->start; + end = start + chunk_hdr_size() + size; + return hyp_allocator_create_chunk(allocator, NULL, start, end, true); + } + + chunk = hyp_allocator_find_efficient_chunk(allocator, size); + + /* Nothing found, create a new chunk at the end in the list */ + if (!chunk) { + start = chunk_start(chunk_data(allocator->tail) + + chunk_used_size(allocator->tail, allocator)); + end = start + chunk_hdr_size() + size; + return hyp_allocator_create_chunk(allocator, allocator->tail, start, end, true); + } + + return hyp_allocator_reuse_chunk(allocator, chunk, size); +} + +static void *hyp_allocator_alloc(struct hyp_allocator *allocator, size_t size) +{ + struct chunk_hdr *chunk; + + size = max(size, MIN_ALLOC_SIZE); + + /* Ensure we do not overflow ALIGN(MIN_ALLOC_SIZE) */ + if (size > U32_MAX) { + hyp_allocator_set_errno(allocator, -E2BIG); + return NULL; + } + + size = ALIGN(size, MIN_ALLOC_SIZE); + if (size > (allocator->end - allocator->start - chunk_hdr_size())) { + hyp_allocator_set_errno(allocator, -E2BIG); + return NULL; + } + + scoped_guard(hyp_spinlock, &allocator->lock) { + chunk = hyp_allocator_alloc_chunk(allocator, size); + if (IS_ERR_OR_NULL(chunk)) { + hyp_allocator_set_errno(allocator, + IS_ERR(chunk) ? PTR_ERR(chunk) : -EINVAL); + return NULL; + } + } + + memset(chunk_data(chunk), 0, size); + return chunk_data(chunk); +} + +static void hyp_allocator_free(struct hyp_allocator *allocator, void *data) +{ + struct chunk_hdr *chunk, *next, *prev; + + if (!data) + return; + + WARN_ON(!IS_ALIGNED((unsigned long)data, MIN_ALLOC_SIZE)); + WARN_ON(data >= allocator->end || data < allocator->start + chunk_hdr_size()); + + guard(hyp_spinlock)(&allocator->lock); + + chunk = chunk_get(container_of(data, struct chunk_hdr, data)); + WARN_ON(!chunk_is_used(chunk)); + chunk_set_unused(chunk); + chunk_set_hash(chunk); + + next = chunk_get_next(chunk); + if (next) + hyp_allocator_destroy_chunk(allocator, chunk, next); + + prev = chunk_get_prev(chunk); + if (prev) + hyp_allocator_destroy_chunk(allocator, prev, chunk); +} + +static unsigned long hyp_allocator_chunk_reclaimable(struct hyp_allocator *allocator, + const struct chunk_hdr *chunk, + u64 *__addr, u64 *__end) +{ + struct chunk_hdr *next; + void *addr, *end; + + /* Last chunk in the allocator */ + if (chunk == allocator->head && chunk == allocator->tail && !chunk_is_used(chunk)) { + addr = (void *)chunk; + end = chunk_unmapped(chunk); + if (!end) + end = allocator->end; + goto end; + } + + next = chunk_get_next(chunk); + + /* Last chunk in the list we can reclaim, even if used */ + if (!next) { + addr = chunk_data(chunk) + chunk_used_size(chunk, allocator); + addr = PTR_ALIGN(addr, PAGE_SIZE); + end = chunk_unmapped(chunk); + if (!end) + end = allocator->end; + goto end; + } + + if (chunk_is_used(chunk)) + return 0; + + addr = PTR_ALIGN(chunk_data(chunk), PAGE_SIZE); + end = chunk_unmapped(chunk); + if (!end) + end = PTR_ALIGN_DOWN(next, PAGE_SIZE); + +end: + if (addr >= end) + return 0; + + if (__end) + *__end = (u64)end; + if (__addr) + *__addr = (u64)addr; + + return (end - addr) >> PAGE_SHIFT; +} + +static void hyp_allocator_reclaim_chunk(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + void *addr, void *end) +{ + struct chunk_hdr *next; + + WARN_ON(end <= addr); + + /* We are about to destroy the last chunk in the allocator */ + if (addr == allocator->start) { + allocator->tail = allocator->head = chunk = NULL; + goto unmap; + } + + next = chunk_get_next(chunk); + + /* + * Split the reclaimed chunk at the next page boundary, + * this ensures no orphan mapped region is created. Splitting at the page boundary is always + * possible because chunks always leave a minimum distance to the page start. + * + * +--------------+ + * |______________| + * |______________|<- Next chunk + * |_ _ _ __ _ _ _| + * | |<- Page-aligned split + * +--------------+ + * +--------------+ + * | | + * | |<- Page reclaimed + * | | + * | | + * +--------------+ + * +--------------+ + * | | + * |______________| + * |______________|<- Chunk to split + * | | + * +--------------+ + */ + if (next && !chunk_unmapped(chunk) && next != end) + WARN_ON(IS_ERR_OR_NULL(hyp_allocator_create_chunk(allocator, chunk, end, next, + false))); +unmap: + hyp_allocator_unmap(allocator, chunk, addr, end); +} + +/* + * Return the best reclaimable chunk which is the highest chunk in the list + * with the biggest reclaimable region. + */ +static struct chunk_hdr *hyp_allocator_find_reclaimable_chunk(struct hyp_allocator *allocator, + u64 *addr, u64 *end) +{ + struct chunk_hdr *chunk, *best_chunk = NULL; + unsigned long best_reclaimable = 0; + + chunk = allocator->head; + while (chunk) { + u64 __addr, __end; + unsigned long reclaimable = hyp_allocator_chunk_reclaimable(allocator, chunk, + &__addr, &__end); + + /* Favour the top biggest chunks */ + if (reclaimable && reclaimable >= best_reclaimable) { + best_reclaimable = reclaimable; + best_chunk = chunk; + *addr = __addr; + *end = __end; + } + + chunk = chunk_get_next(chunk); + } + + return best_chunk; +} + +static void hyp_allocator_drain_memcache(struct hyp_allocator *allocator, + struct kvm_hyp_memcache *host_mc, + unsigned long target) +{ + struct kvm_hyp_memcache *mc = &allocator->mc; + + while (target && mc->nr_pages) { + void *page = pop_hyp_memcache(mc, hyp_phys_to_virt); + + memset(page, 0, PAGE_SIZE); + kvm_flush_dcache_to_poc(page, PAGE_SIZE); + push_hyp_memcache(host_mc, page, hyp_virt_to_phys); + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(page), 1)); + + target--; + } +} + +static void hyp_allocator_reclaim(struct hyp_allocator *allocator, struct kvm_hyp_memcache *host_mc, + unsigned long target) +{ + unsigned long reclaimed = 0; + + if (!target) + return; + + guard(hyp_spinlock)(&allocator->lock); + + do { + struct chunk_hdr *chunk; + unsigned long r; + u64 addr, end; + + chunk = hyp_allocator_find_reclaimable_chunk(allocator, &addr, &end); + if (!chunk) + break; + + r = min((end - addr) >> PAGE_SHIFT, target - reclaimed); + addr = end - (r << PAGE_SHIFT); + hyp_allocator_reclaim_chunk(allocator, chunk, (void *)addr, (void *)end); + + reclaimed += r; + } while (reclaimed < target); + + hyp_allocator_drain_memcache(allocator, host_mc, target); +} + +static unsigned long hyp_allocator_reclaimable(struct hyp_allocator *allocator) +{ + unsigned long reclaimable; + struct chunk_hdr *chunk; + + guard(hyp_spinlock)(&allocator->lock); + + /* + * While allocator->mc typically contains transient pages, + * over-allocation or an aborted operation could leave pages behind. + * Account for allocator->mc so stalled topup pages can be reclaimed. + */ + reclaimable = allocator->mc.nr_pages; + + chunk = allocator->head; + while (chunk) { + reclaimable += hyp_allocator_chunk_reclaimable(allocator, chunk, NULL, NULL); + chunk = chunk_get_next(chunk); + } + + return reclaimable; +} + +static int hyp_allocator_topup(struct hyp_allocator *allocator, + struct kvm_hyp_memcache *host_mc) +{ + struct kvm_hyp_memcache *alloc_mc = &allocator->mc; + + guard(hyp_spinlock)(&allocator->lock); + return refill_memcache(alloc_mc, host_mc->nr_pages + alloc_mc->nr_pages, host_mc); +} + +static u32 hyp_allocator_topup_needed(struct hyp_allocator *allocator) +{ + u32 *topup_needed = this_cpu_ptr(allocator->topup_needed); + u32 ret = *topup_needed; + + *topup_needed = 0; + + return ret; +} + +static int hyp_allocator_init(struct hyp_allocator *allocator, size_t size) +{ + unsigned long start; + int ret; + + size = PAGE_ALIGN(size); + + /* constrained by chunk_hdr u32 types */ + if (size > U32_MAX || !size) + return -EINVAL; + + ret = pkvm_alloc_private_va_range(size, &start); + if (ret) + return ret; + + allocator->first_unmapped = allocator->start = (void *)start; + allocator->end = allocator->start + size; + hyp_spin_lock_init(&allocator->lock); + + return 0; +} + +static DEFINE_PER_CPU(int, __hyp_allocator_errno); +static DEFINE_PER_CPU(u32, __hyp_allocator_topup_needed); + +static struct hyp_allocator hyp_allocator = { + .errno = &__hyp_allocator_errno, + .topup_needed = &__hyp_allocator_topup_needed, +}; + +/** + * hyp_alloc() - Allocate memory from the heap allocator + * + * @size: Allocation size in bytes. + * + * Callers must ensure they do not hold pkvm_pgd_lock to avoid deadlocks, + * as the allocator may modify the hyp stage-1 page-table. + * + * Return: A pointer to the allocated memory on success, else NULL. + */ +void *hyp_alloc(size_t size) +{ + return hyp_allocator_alloc(&hyp_allocator, size); +} + +/** + * hyp_free() - Free memory allocated with hyp_alloc() + * + * @data: Address returned by the original hyp_alloc(). + * + * The use of any other address than one returned by hyp_alloc() will cause a + * hypervisor panic. + */ +void hyp_free(void *data) +{ + hyp_allocator_free(&hyp_allocator, data); +} + +/** + * hyp_alloc_errno() - Read the errno on allocation error + * + * Get the return code from an allocation failure. + * + * Return: -ENOMEM if the allocator needs a refill from the host, -E2BIG if + * there is no VA space left else 0. + */ +int hyp_alloc_errno(void) +{ + return hyp_allocator_errno(&hyp_allocator); +} + +int hyp_alloc_init(size_t size) +{ + return hyp_allocator_init(&hyp_allocator, size); +} + +void hyp_alloc_reclaim(struct kvm_hyp_memcache *mc, unsigned long target) +{ + hyp_allocator_reclaim(&hyp_allocator, mc, target); +} + +unsigned long hyp_alloc_reclaimable(void) +{ + return hyp_allocator_reclaimable(&hyp_allocator); +} + +int hyp_alloc_topup(struct kvm_hyp_memcache *host_mc) +{ + return hyp_allocator_topup(&hyp_allocator, host_mc); +} + +u32 hyp_alloc_topup_needed(void) +{ + return hyp_allocator_topup_needed(&hyp_allocator); +} diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 75b00c323310..9fd3dcde3557 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -363,6 +364,11 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_bas if (ret) return ret; + /* Arbitrary size, unmapped at boot and only consume VA-space */ + ret = hyp_alloc_init(SZ_128M); + if (ret) + return ret; + update_nvhe_init_params(); /* Jump in the idmap page to switch to the new page-tables */ From 11a59e2959e63546dd75c347d552193c38aeb37e Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:36 +0100 Subject: [PATCH 409/506] KVM: arm64: Allow kvm_hyp_memcache usage outside of stage-2 Although currently limited to guest stage-2 page-table allocations, struct kvm_hyp_memcache is a useful primitive for passing a list of discontiguous pages between host and hypervisor. Introduce init_hyp_memcache() to initialise a generic hyp memcache, and init_hyp_stage2_memcache() for stage-2 specific memcaches. The generic initialiser will be used to top up the upcoming pKVM heap allocator. Note that the generic kvm_hyp_memcache does not account for the allocated memory. This is expected: donations to the heap allocator are shared and recycled, they cannot be accounted for a specific VM. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-6-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 13 +++++++++++++ arch/arm64/kvm/mmu.c | 4 +++- arch/arm64/kvm/pkvm.c | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index fc1a82777ed4..64cd6233b439 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -92,9 +92,22 @@ struct kvm_hyp_memcache { struct pkvm_mapping *mapping; /* only used from EL1 */ #define HYP_MEMCACHE_ACCOUNT_STAGE2 BIT(1) +#define HYP_MEMCACHE_ACCOUNT_KMEMCG BIT(2) unsigned long flags; }; +static inline void init_hyp_memcache(struct kvm_hyp_memcache *mc) +{ + memset(mc, 0, sizeof(*mc)); + mc->mapping = ZERO_SIZE_PTR; /* Prevent allocation, solely useful for stage2 memcache */ +} + +static inline void init_hyp_stage2_memcache(struct kvm_hyp_memcache *mc) +{ + memset(mc, 0, sizeof(*mc)); + mc->flags = HYP_MEMCACHE_ACCOUNT_STAGE2 | HYP_MEMCACHE_ACCOUNT_KMEMCG; +} + static inline void push_hyp_memcache(struct kvm_hyp_memcache *mc, phys_addr_t *p, phys_addr_t (*to_pa)(void *virt)) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9ba86450fe4a..3163502cc7d2 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1160,8 +1160,10 @@ static void *hyp_mc_alloc_fn(void *mc) { struct kvm_hyp_memcache *memcache = mc; void *addr; + gfp_t gfp; - addr = (void *)__get_free_page(GFP_KERNEL_ACCOUNT); + gfp = memcache->flags & HYP_MEMCACHE_ACCOUNT_KMEMCG ? GFP_KERNEL_ACCOUNT : GFP_KERNEL; + addr = (void *)__get_free_page(gfp); if (addr && memcache->flags & HYP_MEMCACHE_ACCOUNT_STAGE2) kvm_account_pgtable_pages(addr, 1); diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4c6e4bec12..e5200731a45b 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -111,7 +111,7 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) void *hyp_vcpu; int ret; - vcpu->arch.pkvm_memcache.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2; + init_hyp_stage2_memcache(&vcpu->arch.pkvm_memcache); hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT); if (!hyp_vcpu) @@ -172,7 +172,7 @@ static int __pkvm_create_hyp_vm(struct kvm *kvm) goto free_vm; kvm->arch.pkvm.is_created = true; - kvm->arch.pkvm.stage2_teardown_mc.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2; + init_hyp_stage2_memcache(&kvm->arch.pkvm.stage2_teardown_mc); kvm_account_pgtable_pages(pgd, pgd_sz / PAGE_SIZE); return 0; From e868b8c35268a8cb786091c63f538c960b8807e5 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:37 +0100 Subject: [PATCH 410/506] KVM: arm64: Add pkvm_hyp_req infrastructure Introduce a struct pkvm_hyp_req to enable the pKVM hypervisor to request resources from the host. Provide serialisation helpers to transport these requests via SMCCC registers (starting from a2): pkvm_hyp_req_to_smccc() to encode into the SMCCC args. smccc_to_pkvm_hyp_req() to decode them. When the hypervisor raises a request, the host must handle it and retry the HVC. To automate this sequence, introduce the pkvm_call_hyp_req() macro. This intercepts pending requests, invokes the handler and retries the HVC. Additionally, introduce a trace event to track the handling of these requests. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-7-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pkvm.h | 103 ++++++++++++++++++++++++++++++ arch/arm64/kvm/pkvm.c | 25 ++++++++ arch/arm64/kvm/trace_pkvm.h | 43 +++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 arch/arm64/kvm/trace_pkvm.h diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index beea00e693a0..19bddf261c82 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -205,6 +205,109 @@ struct pkvm_mapping { u64 __subtree_last; /* Internal member for interval tree */ }; +enum pkvm_hyp_req_type { + PKVM_HYP_NO_REQ = 0, + __PKVM_HYP_REQ_TYPE_MAX, +}; + +#define PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX \ + (sizeof(struct arm_smccc_res) - offsetof(struct arm_smccc_res, a2) - 1) + +struct pkvm_hyp_req { + u8 type; + union { + struct { + u32 nr_pages; + } mem; + struct { + /* Helper for SMCCC encoding/decoding */ + u8 args[PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX]; + } args; + }; +}; + +static inline size_t pkvm_hyp_req_arg_size(u8 type) +{ + switch (type) { + case PKVM_HYP_NO_REQ: + return 0; + default: + WARN_ON(1); + } + + return 0; +} + +/* Encode the pending pkvm_hyp_req type into the SMCCC args */ +static inline void +pkvm_hyp_req_to_smccc(struct kvm_cpu_context *host_ctxt, struct pkvm_hyp_req *req) +{ + u8 *dst, type = req->type; + size_t size; + + if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) { + host_ctxt->regs.regs[2] = 0; + return; + } + + size = pkvm_hyp_req_arg_size(type); + if (WARN_ON(size > PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX)) + return; + + dst = (u8 *)&host_ctxt->regs.regs[2]; + *dst = type; + + memcpy(dst + 1, &req->args, size); +} + +/* Return true if a pkvm_hyp_req has been decoded from the SMCCC args */ +static inline bool smccc_to_pkvm_hyp_req(struct pkvm_hyp_req *req, struct arm_smccc_res *res) +{ + u8 *src = (u8 *)res + offsetof(struct arm_smccc_res, a2); + u8 type = *src; + + if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) + return false; + + req->type = type; + memcpy(&req->args, src + 1, pkvm_hyp_req_arg_size(type)); + + return true; +} + +int __pkvm_handle_smccc_req(struct arm_smccc_res *res); + +/** + * pkvm_call_hyp_req() - Issue an HVC that can return hypervisor requests + * @f: Hypervisor function symbol to call. + * @...: Arguments to pass to the hypercall. + * + * Re-issue an HVC and process any pending hypervisor request until completion + * or error. + * + * Only use this helper for HVCs whose hypervisor handlers format their return + * registers with pkvm_hyp_req_to_smccc(). + * + * Return: Result of the hypercall or a negative error if the hyp request + * handling failed. + */ +#define pkvm_call_hyp_req(f, ...) \ +({ \ + struct arm_smccc_res __res; \ + int __ret; \ + do { \ + __ret = -1; \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), ##__VA_ARGS__, &__res); \ + if (WARN_ON(__res.a0 != SMCCC_RET_SUCCESS)) \ + break; \ + __ret = __res.a1; \ + if (!__ret) \ + break; \ + __ret = __pkvm_handle_smccc_req(&__res); \ + } while (!__ret); \ + __ret; \ +}) + int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, struct kvm_pgtable_mm_ops *mm_ops); void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt, diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index e5200731a45b..11d9e2f70cd8 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -16,6 +16,9 @@ #include "hyp_constants.h" +#define CREATE_TRACE_POINTS +#include "trace_pkvm.h" + DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory); @@ -594,3 +597,25 @@ bool pkvm_force_reclaim_guest_page(phys_addr_t phys) return !ret || ret == -EAGAIN; } + +static int pkvm_handle_hyp_req(struct pkvm_hyp_req *req) +{ + int ret = -EINVAL; + + switch (req->type) { + } + + trace_kvm_handle_pkvm_hyp_req(req, ret); + + return ret; +} + +int __pkvm_handle_smccc_req(struct arm_smccc_res *res) +{ + struct pkvm_hyp_req req; + + if (smccc_to_pkvm_hyp_req(&req, res)) + return pkvm_handle_hyp_req(&req); + + return res->a1; +} diff --git a/arch/arm64/kvm/trace_pkvm.h b/arch/arm64/kvm/trace_pkvm.h new file mode 100644 index 000000000000..3966c111e3ad --- /dev/null +++ b/arch/arm64/kvm/trace_pkvm.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(_TRACE_PKVM_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_PKVM_ARM64_KVM_H + +#include +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +TRACE_DEFINE_ENUM(PKVM_HYP_NO_REQ); + +#define PKVM_HYP_REQ_TYPES \ + { PKVM_HYP_NO_REQ, "NO_REQ" } + +TRACE_EVENT(kvm_handle_pkvm_hyp_req, + TP_PROTO(struct pkvm_hyp_req *req, int ret), + TP_ARGS(req, ret), + + TP_STRUCT__entry( + __field(u8, type) + __field(int, ret) + ), + + TP_fast_assign( + __entry->type = req->type; + __entry->ret = ret; + ), + + TP_printk("type: %s ret: %d", + __print_symbolic(__entry->type, PKVM_HYP_REQ_TYPES), + __entry->ret) +); + +#endif /* _TRACE_PKVM_ARM64_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace_pkvm + +/* This part must be outside protection */ +#include From 7bf265fe68af9e1ef3af083d444a72bd3ce3d3e3 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:38 +0100 Subject: [PATCH 411/506] KVM: arm64: Add PKVM_HYP_REQ_HYP_ALLOC request Introduce a new pkvm_hyp_request type asking the host to top up the pKVM heap allocator. Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-8-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_hcall.h | 6 ++++++ arch/arm64/include/asm/kvm_pkvm.h | 5 +++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 26 ++++++++++++++++++++++++++ arch/arm64/kvm/pkvm.c | 30 ++++++++++++++++++++++++++++++ arch/arm64/kvm/trace_pkvm.h | 6 ++++-- 6 files changed, 72 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index e5b92ac09e69..de59e89318fa 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -89,6 +89,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_topup, MARKER(__KVM_HOST_SMCCC_FUNC_PKVM_ONLY), diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index 9f06af58a4b0..6d6f12400e94 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -22,6 +22,10 @@ typedef u16 pkvm_handle_t; +enum pkvm_topup_id { + PKVM_TOPUP_HYP_ALLOC, +}; + struct kvm; struct kvm_s2_mmu; struct kvm_vcpu; @@ -193,6 +197,8 @@ DECLARE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, struct vgic_v5_cpu_if __kern *, cpu_if) DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, struct vgic_v5_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, + enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) /* Hypercalls that are available only when pKVM has finalised. */ DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 19bddf261c82..c338178d6bb2 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -207,6 +207,7 @@ struct pkvm_mapping { enum pkvm_hyp_req_type { PKVM_HYP_NO_REQ = 0, + PKVM_HYP_REQ_HYP_ALLOC, __PKVM_HYP_REQ_TYPE_MAX, }; @@ -228,9 +229,13 @@ struct pkvm_hyp_req { static inline size_t pkvm_hyp_req_arg_size(u8 type) { + struct pkvm_hyp_req *req; + switch (type) { case PKVM_HYP_NO_REQ: return 0; + case PKVM_HYP_REQ_HYP_ALLOC: + return sizeof(req->mem); default: WARN_ON(1); } diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 91e74a1a334f..2b89eb274482 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -799,6 +800,30 @@ DEFINE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, return __pkvm_finalize_teardown_vm(handle); } +DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, + enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) +{ + struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); + struct kvm_hyp_memcache host_mc = { + .head = head, + .nr_pages = nr_pages, + }; + int ret; + + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + ret = hyp_alloc_topup(&host_mc); + break; + default: + ret = -EINVAL; + } + + cpu_reg(host_ctxt, 2) = host_mc.head; + cpu_reg(host_ctxt, 3) = host_mc.nr_pages; + + return ret; +} + DEFINE_KVM_HOST_HCALL(int, __tracing_load, void __kern *, desc_hva, size_t, desc_size) { @@ -893,6 +918,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), HANDLE_FUNC(__vgic_v5_save_apr), HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), + HANDLE_FUNC(__pkvm_hyp_topup), HANDLE_FUNC(__pkvm_host_share_hyp), HANDLE_FUNC(__pkvm_host_unshare_hyp), diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 11d9e2f70cd8..9cd23978481c 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -88,6 +88,33 @@ void __init kvm_hyp_reserve(void) hyp_mem_base); } +static int pkvm_hyp_topup(enum pkvm_topup_id id, unsigned long nr_pages) +{ + struct kvm_hyp_memcache mc; + struct arm_smccc_res res; + int ret; + + init_hyp_memcache(&mc); + ret = topup_hyp_memcache(&mc, nr_pages); + if (ret) + goto err; + + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_topup), id, mc.head, + mc.nr_pages, &res); + if (WARN_ON_ONCE(res.a0 != SMCCC_RET_SUCCESS)) { + ret = -EINVAL; + goto err; + } + + ret = res.a1; + mc.head = res.a2; + mc.nr_pages = res.a3; + +err: + free_hyp_memcache(&mc); + return ret; +} + static void __pkvm_destroy_hyp_vm(struct kvm *kvm) { if (pkvm_hyp_vm_is_created(kvm)) { @@ -603,6 +630,9 @@ static int pkvm_handle_hyp_req(struct pkvm_hyp_req *req) int ret = -EINVAL; switch (req->type) { + case PKVM_HYP_REQ_HYP_ALLOC: + ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC, req->mem.nr_pages); + break; } trace_kvm_handle_pkvm_hyp_req(req, ret); diff --git a/arch/arm64/kvm/trace_pkvm.h b/arch/arm64/kvm/trace_pkvm.h index 3966c111e3ad..493225e3282d 100644 --- a/arch/arm64/kvm/trace_pkvm.h +++ b/arch/arm64/kvm/trace_pkvm.h @@ -9,9 +9,11 @@ #define TRACE_SYSTEM kvm TRACE_DEFINE_ENUM(PKVM_HYP_NO_REQ); +TRACE_DEFINE_ENUM(PKVM_HYP_REQ_HYP_ALLOC); -#define PKVM_HYP_REQ_TYPES \ - { PKVM_HYP_NO_REQ, "NO_REQ" } +#define PKVM_HYP_REQ_TYPES \ + { PKVM_HYP_NO_REQ, "NO_REQ" }, \ + { PKVM_HYP_REQ_HYP_ALLOC, "HYP_ALLOC" } TRACE_EVENT(kvm_handle_pkvm_hyp_req, TP_PROTO(struct pkvm_hyp_req *req, int ret), From 43759e3c9631ffcec55e4b7396864af4a5b98f30 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:39 +0100 Subject: [PATCH 412/506] KVM: arm64: Add reclaim interface for the pKVM heap alloc Introduce a host interface to reclaim donated memory from the pKVM heap allocator back to the host. It specifically provides two helpers that will make it easier to create a shrinker for pKVM: pkvm_hyp_reclaimable() pkvm_hyp_relaim() Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-9-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 2 ++ arch/arm64/include/asm/kvm_hcall.h | 4 +++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 34 +++++++++++++++++++++++ arch/arm64/kvm/pkvm.c | 44 ++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index de59e89318fa..fc3c52b03154 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -90,6 +90,8 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, __KVM_HOST_SMCCC_FUNC___pkvm_hyp_topup, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaim, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaimable, MARKER(__KVM_HOST_SMCCC_FUNC_PKVM_ONLY), diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index 6d6f12400e94..bce6eb8b20e0 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -199,6 +199,10 @@ DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, struct vgic_v5_cpu_if __kern *, cpu_if) DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, + enum pkvm_topup_id, id, unsigned long, target) +DECLARE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, + enum pkvm_topup_id, id) /* Hypercalls that are available only when pKVM has finalised. */ DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 2b89eb274482..fdc39f064718 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -824,6 +824,38 @@ DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, return ret; } +DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, + enum pkvm_topup_id, id, unsigned long, target) +{ + struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); + struct kvm_hyp_memcache host_mc = {}; + int ret = 0; + + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + hyp_alloc_reclaim(&host_mc, target); + break; + default: + ret = -EINVAL; + } + + cpu_reg(host_ctxt, 2) = host_mc.head; + cpu_reg(host_ctxt, 3) = host_mc.nr_pages; + + return ret; +} + +DEFINE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, + enum pkvm_topup_id, id) +{ + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + return hyp_alloc_reclaimable(); + default: + return 0; + } +} + DEFINE_KVM_HOST_HCALL(int, __tracing_load, void __kern *, desc_hva, size_t, desc_size) { @@ -919,6 +951,8 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__vgic_v5_save_apr), HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), HANDLE_FUNC(__pkvm_hyp_topup), + HANDLE_FUNC(__pkvm_hyp_reclaim), + HANDLE_FUNC(__pkvm_hyp_reclaimable), HANDLE_FUNC(__pkvm_host_share_hyp), HANDLE_FUNC(__pkvm_host_unshare_hyp), diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 9cd23978481c..fec819a9dc78 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -115,6 +115,50 @@ static int pkvm_hyp_topup(enum pkvm_topup_id id, unsigned long nr_pages) return ret; } +static unsigned long __pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target) +{ + struct kvm_hyp_memcache mc; + struct arm_smccc_res res; + unsigned long reclaimed; + + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_reclaim), id, target, &res); + if (WARN_ON_ONCE(res.a0 != SMCCC_RET_SUCCESS) || WARN_ON_ONCE(res.a1)) + return 0; + + init_hyp_memcache(&mc); + mc.head = res.a2; + mc.nr_pages = reclaimed = res.a3; + free_hyp_memcache(&mc); + + return reclaimed; +} + +static __maybe_unused unsigned long pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target) +{ + unsigned long reclaimed = 0; + + while (reclaimed < target) { + /* Arbitrary limit to avoid blocking in EL2 for too long */ + unsigned long r = __pkvm_hyp_reclaim(id, min(target - reclaimed, 16)); + + if (!r) + break; + + reclaimed += r; + if (reclaimed >= target) + break; + + cond_resched(); + } + + return reclaimed; +} + +static __maybe_unused unsigned long pkvm_hyp_reclaimable(enum pkvm_topup_id id) +{ + return kvm_call_hyp_nvhe(__pkvm_hyp_reclaimable, id); +} + static void __pkvm_destroy_hyp_vm(struct kvm *kvm) { if (pkvm_hyp_vm_is_created(kvm)) { From bd93068340e9d770d6282ab425b686bef2042394 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:40 +0100 Subject: [PATCH 413/506] KVM: arm64: Add selftests for the pKVM heap allocator Introduce a comprehensive runtime selftest for the pKVM hypervisor heap allocator, executed during init when CONFIG_NVHE_EL2_DEBUG is enabled. The selftest runs entirely at EL2 and exercises allocator's core mechanisms: * over-sized allocations * basic allocation and alignment * chunk recycling, splitting, merging * memory reclaiming * memory topup Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-10-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_hcall.h | 2 + arch/arm64/include/asm/kvm_pkvm.h | 3 + arch/arm64/kvm/arm.c | 2 + arch/arm64/kvm/hyp/include/nvhe/alloc.h | 11 ++ arch/arm64/kvm/hyp/nvhe/alloc.c | 186 +++++++++++++++++++++++- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 24 +++ arch/arm64/kvm/pkvm.c | 19 +++ 8 files changed, 247 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index fc3c52b03154..1ac1990bbd68 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -62,6 +62,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs, __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs, __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_alloc_selftest, MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM), diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index bce6eb8b20e0..8a6c2506fdeb 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -24,6 +24,7 @@ typedef u16 pkvm_handle_t; enum pkvm_topup_id { PKVM_TOPUP_HYP_ALLOC, + PKVM_TOPUP_HYP_ALLOC_SELFTEST, }; struct kvm; @@ -153,6 +154,7 @@ DECLARE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, DECLARE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) DECLARE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) DECLARE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest) DECLARE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index c338178d6bb2..ec8060f220f4 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -17,6 +17,7 @@ #define HYP_MEMBLOCK_REGIONS 128 +void pkvm_selftests(void); int pkvm_init_host_vm(struct kvm *kvm, unsigned long type); int pkvm_create_hyp_vm(struct kvm *kvm); bool pkvm_hyp_vm_is_created(struct kvm *kvm); @@ -208,6 +209,7 @@ struct pkvm_mapping { enum pkvm_hyp_req_type { PKVM_HYP_NO_REQ = 0, PKVM_HYP_REQ_HYP_ALLOC, + PKVM_HYP_REQ_HYP_ALLOC_SELFTEST, __PKVM_HYP_REQ_TYPE_MAX, }; @@ -235,6 +237,7 @@ static inline size_t pkvm_hyp_req_arg_size(u8 type) case PKVM_HYP_NO_REQ: return 0; case PKVM_HYP_REQ_HYP_ALLOC: + case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST: return sizeof(req->mem); default: WARN_ON(1); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..09ae5431ce58 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2914,6 +2914,8 @@ static int __init init_hyp_mode(void) kvm_err("Failed to init hyp memory protection\n"); goto out_err; } + + pkvm_selftests(); } return 0; diff --git a/arch/arm64/kvm/hyp/include/nvhe/alloc.h b/arch/arm64/kvm/hyp/include/nvhe/alloc.h index 8f87a63f8946..bca0168e0046 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/alloc.h +++ b/arch/arm64/kvm/hyp/include/nvhe/alloc.h @@ -14,4 +14,15 @@ int hyp_alloc_init(size_t size); int hyp_alloc_topup(struct kvm_hyp_memcache *host_mc); unsigned long hyp_alloc_reclaimable(void); void hyp_alloc_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target); + +#ifdef CONFIG_NVHE_EL2_DEBUG +int hyp_allocator_selftest(void); +u32 hyp_alloc_selftest_topup_needed(void); +int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc); +void hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target); +#else +static inline int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc) { return 0; }; +static inline void +hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target) { }; +#endif #endif diff --git a/arch/arm64/kvm/hyp/nvhe/alloc.c b/arch/arm64/kvm/hyp/nvhe/alloc.c index e0104d2dd6c0..c1881c0011c9 100644 --- a/arch/arm64/kvm/hyp/nvhe/alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/alloc.c @@ -1001,9 +1001,17 @@ int hyp_alloc_errno(void) return hyp_allocator_errno(&hyp_allocator); } +static int selftest_init(void); + int hyp_alloc_init(size_t size) { - return hyp_allocator_init(&hyp_allocator, size); + int ret; + + ret = hyp_allocator_init(&hyp_allocator, size); + if (ret) + return ret; + + return selftest_init(); } void hyp_alloc_reclaim(struct kvm_hyp_memcache *mc, unsigned long target) @@ -1025,3 +1033,179 @@ u32 hyp_alloc_topup_needed(void) { return hyp_allocator_topup_needed(&hyp_allocator); } + +#ifdef CONFIG_NVHE_EL2_DEBUG +#define SELFTEST_MAX_PAGES 6 +#define SELFTEST_MAX_SIZE (PAGE_SIZE * SELFTEST_MAX_PAGES) + +static DEFINE_PER_CPU(int, __selftest_errno); +static DEFINE_PER_CPU(u32, __selftest_topup_needed); + +static struct hyp_allocator selftest_allocator = { + .errno = &__selftest_errno, + .topup_needed = &__selftest_topup_needed, + .lock = __HYP_SPIN_LOCK_UNLOCKED, +}; + +int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc) +{ + return hyp_allocator_topup(&selftest_allocator, host_mc); +} + +void hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target) +{ + hyp_allocator_reclaim(&selftest_allocator, host_mc, target); +} + +u32 hyp_alloc_selftest_topup_needed(void) +{ + return hyp_allocator_topup_needed(&selftest_allocator); +} + +static int selftest_init(void) +{ + return hyp_allocator_init(&selftest_allocator, SELFTEST_MAX_SIZE); +} + +static void *selftest_alloc(size_t size) +{ + return hyp_allocator_alloc(&selftest_allocator, size); +} + +static void selftest_free(void *addr) +{ + hyp_allocator_free(&selftest_allocator, addr); +} + +static int selftest_errno(void) +{ + return hyp_allocator_errno(&selftest_allocator); +} + +int hyp_allocator_selftest(void) +{ + struct hyp_allocator *allocator = &selftest_allocator; + static DEFINE_HYP_SPINLOCK(selftest_lock); + struct kvm_hyp_memcache host_mc = { }; + void *addr1, *addr2, *addr3, *addr4; + int ret; + + guard(hyp_spinlock)(&selftest_lock); + + if (allocator->mc.nr_pages < SELFTEST_MAX_PAGES) { + *this_cpu_ptr(allocator->topup_needed) = SELFTEST_MAX_PAGES - + allocator->mc.nr_pages; + return -ENOMEM; + } + + selftest_alloc(SELFTEST_MAX_SIZE); + if (selftest_errno() != -E2BIG) + return -EINVAL; + + selftest_alloc(SIZE_MAX); + if (selftest_errno() != -E2BIG) + return -EINVAL; + + /* Test first chunk */ + addr1 = selftest_alloc(0); + if (!addr1 || addr1 != (void *)allocator->start + chunk_hdr_size()) + return -EINVAL; + + /* Test second contiguous chunk with unaligned size */ + addr2 = selftest_alloc(MIN_ALLOC_SIZE + 1); + if (!addr2) + return -EINVAL; + addr3 = selftest_alloc(0); + if (!addr3 || + addr3 != addr2 + (2 * MIN_ALLOC_SIZE) + chunk_hdr_size()) + return -EINVAL; + + selftest_free(addr3); + + /* Test chunk recycling */ + selftest_free(addr1); + if (addr1 != selftest_alloc(0)) + return -EINVAL; + + /* Test chunk forward merging */ + addr3 = selftest_alloc(0); + selftest_free(addr2); + selftest_free(addr1); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 2)) + return -EINVAL; + + selftest_free(addr1); + + /* Test chunk splitting */ + if (addr1 != selftest_alloc(0)) + return -EINVAL; + if (addr2 != selftest_alloc(0)) + return -EINVAL; + + /* Test chunk backward merging */ + selftest_free(addr1); + selftest_free(addr2); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 2)) + return -EINVAL; + + selftest_free(addr1); + + /* Test chunk 3-way merging */ + addr1 = selftest_alloc(0); + addr2 = selftest_alloc(0); + addr4 = selftest_alloc(0); + selftest_free(addr1); + selftest_free(addr3); + selftest_free(addr2); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 3)) + return -EINVAL; + + selftest_free(addr4); + selftest_free(addr1); + + /* Test reclaiming */ + if (addr1 != selftest_alloc(0)) + return -EINVAL; + if (addr2 != selftest_alloc(PAGE_SIZE * 2)) + return -EINVAL; + addr3 = selftest_alloc(0); + addr4 = selftest_alloc(PAGE_SIZE); + + /* Test reclaiming the last chunk of the list */ + selftest_free(addr4); + hyp_allocator_reclaim(allocator, &host_mc, SELFTEST_MAX_PAGES); + if (host_mc.nr_pages != SELFTEST_MAX_PAGES - 3) + return -EINVAL; + + /* Test punching a hole in the middle of a free chunk ... */ + selftest_free(addr2); + hyp_allocator_reclaim(allocator, &host_mc, SELFTEST_MAX_PAGES); + if (host_mc.nr_pages != SELFTEST_MAX_PAGES - 2) + return -EINVAL; + + if (selftest_alloc(PAGE_SIZE)) + return -EINVAL; + if (selftest_errno() != -ENOMEM) + return -EINVAL; + + /* ... and to refill this hole */ + ret = hyp_allocator_topup(allocator, &host_mc); + if (ret) + return ret; + /* Chunk at addr2 was made smaller by the reclaim */ + if (addr2 != selftest_alloc(PAGE_SIZE)) + return -EINVAL; + + /* Test reclaiming the entire allocator from the host */ + selftest_free(addr3); + selftest_free(addr2); + selftest_free(addr1); + if (addr1 != selftest_alloc(SELFTEST_MAX_PAGES * PAGE_SIZE - chunk_hdr_size())) + return -EINVAL; + selftest_free(addr1); + + return 0; +} +#else +static int selftest_init(void) { return 0; } +#endif diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index fdc39f064718..a6e6c0941428 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -800,6 +800,23 @@ DEFINE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, return __pkvm_finalize_teardown_vm(handle); } +DEFINE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest) +{ + struct pkvm_hyp_req req = { .type = PKVM_HYP_NO_REQ }; + int ret = -EPERM; + +#ifdef CONFIG_NVHE_EL2_DEBUG + ret = hyp_allocator_selftest(); + if (ret == -ENOMEM) { + req.type = PKVM_HYP_REQ_HYP_ALLOC_SELFTEST; + req.mem.nr_pages = hyp_alloc_selftest_topup_needed(); + } +#endif + pkvm_hyp_req_to_smccc(host_data_ptr(host_ctxt), &req); + + return ret; +} + DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) { @@ -814,6 +831,9 @@ DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, case PKVM_TOPUP_HYP_ALLOC: ret = hyp_alloc_topup(&host_mc); break; + case PKVM_TOPUP_HYP_ALLOC_SELFTEST: + ret = hyp_alloc_selftest_topup(&host_mc); + break; default: ret = -EINVAL; } @@ -835,6 +855,9 @@ DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, case PKVM_TOPUP_HYP_ALLOC: hyp_alloc_reclaim(&host_mc, target); break; + case PKVM_TOPUP_HYP_ALLOC_SELFTEST: + hyp_alloc_selftest_reclaim(&host_mc, target); + break; default: ret = -EINVAL; } @@ -927,6 +950,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__kvm_enable_ssbs), HANDLE_FUNC(__vgic_v3_init_lrs), HANDLE_FUNC(__vgic_v3_get_gic_config), + HANDLE_FUNC(__pkvm_hyp_alloc_selftest), HANDLE_FUNC(__pkvm_prot_finalize), HANDLE_FUNC(__kvm_adjust_pc), diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index fec819a9dc78..7821f2592eba 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -344,6 +344,22 @@ static int __init pkvm_drop_host_privileges(void) return ret; } +void __init pkvm_selftests(void) +{ +#ifdef CONFIG_NVHE_EL2_DEBUG + int ret = pkvm_call_hyp_req(__pkvm_hyp_alloc_selftest); + unsigned long reclaimed; + + reclaimed = pkvm_hyp_reclaim(PKVM_TOPUP_HYP_ALLOC_SELFTEST, ULONG_MAX); + + /* On failure, not all the pages may be reclaimable */ + if (!ret) + WARN_ON(reclaimed != 6 /* SELFTEST_MAX_PAGES */); + else + kvm_err("pKVM hyp allocator selftest failed (%d)\n", ret); +#endif +} + static int __init finalize_pkvm(void) { int ret; @@ -677,6 +693,9 @@ static int pkvm_handle_hyp_req(struct pkvm_hyp_req *req) case PKVM_HYP_REQ_HYP_ALLOC: ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC, req->mem.nr_pages); break; + case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST: + ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC_SELFTEST, req->mem.nr_pages); + break; } trace_kvm_handle_pkvm_hyp_req(req, ret); From 909e70fd13567db8247c3850f27b65aaea2c9ad0 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:41 +0100 Subject: [PATCH 414/506] KVM: arm64: Add a shrinker for pKVM Integrate the pKVM memory reclaim interface with the host's memory management subsystem. This allows the host to automatically recover unused memory fom the hypervisor's heap allocator when the host is under memory pressure. Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-11-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hcall.h | 1 + arch/arm64/kvm/pkvm.c | 50 +++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index 8a6c2506fdeb..9f09eb11193f 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -24,6 +24,7 @@ typedef u16 pkvm_handle_t; enum pkvm_topup_id { PKVM_TOPUP_HYP_ALLOC, + NR_PKVM_TOPUP_HYP_IDS, PKVM_TOPUP_HYP_ALLOC_SELFTEST, }; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 7821f2592eba..ea96744f41fb 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -133,7 +133,7 @@ static unsigned long __pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long tar return reclaimed; } -static __maybe_unused unsigned long pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target) +static unsigned long pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target) { unsigned long reclaimed = 0; @@ -154,7 +154,7 @@ static __maybe_unused unsigned long pkvm_hyp_reclaim(enum pkvm_topup_id id, unsi return reclaimed; } -static __maybe_unused unsigned long pkvm_hyp_reclaimable(enum pkvm_topup_id id) +static unsigned long pkvm_hyp_reclaimable(enum pkvm_topup_id id) { return kvm_call_hyp_nvhe(__pkvm_hyp_reclaimable, id); } @@ -360,8 +360,39 @@ void __init pkvm_selftests(void) #endif } +static unsigned long pkvm_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long reclaimable = 0; + int id; + + for (id = 0; id < NR_PKVM_TOPUP_HYP_IDS; id++) + reclaimable += pkvm_hyp_reclaimable(id); + + return reclaimable ?: SHRINK_EMPTY; +} + +static unsigned long pkvm_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long reclaimed = 0; + int id; + + sc->nr_scanned = 0; + + for (id = 0; id < NR_PKVM_TOPUP_HYP_IDS; id++) { + unsigned long r = pkvm_hyp_reclaim(id, sc->nr_to_scan - sc->nr_scanned); + + reclaimed += r; + sc->nr_scanned += r; + if (sc->nr_scanned >= sc->nr_to_scan) + break; + } + + return reclaimed ?: SHRINK_STOP; +} + static int __init finalize_pkvm(void) { + struct shrinker *pkvm_shrinker; int ret; if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised()) @@ -377,10 +408,21 @@ static int __init finalize_pkvm(void) kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size); ret = pkvm_drop_host_privileges(); - if (ret) + if (ret) { pr_err("Failed to finalize Hyp protection: %d\n", ret); + return ret; + } - return ret; + pkvm_shrinker = shrinker_alloc(0, "pkvm"); + if (pkvm_shrinker) { + pkvm_shrinker->count_objects = pkvm_shrinker_count; + pkvm_shrinker->scan_objects = pkvm_shrinker_scan; + shrinker_register(pkvm_shrinker); + } else { + kvm_err("Failed to register shrinker for pKVM\n"); + } + + return 0; } device_initcall_sync(finalize_pkvm); From 0b27ba187dde3170a312904f18595f3ba266bcae Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:42 +0100 Subject: [PATCH 415/506] KVM: arm64: Filter out non-kernel addresses in kern_hyp_va kern_hyp_va() is idempotent for the hypervisor linear space. This is handy for nVHE hypervisor callers handling kvm_vcpu or kvm_arch pointers. Those pointers can originate from the hypervisor space (when protected mode is enabled, we don't trust the kernel and the hypervisor uses its own copy) or from the kernel space (we do trust the kernel in "non-protected" nVHE). This idempotence does not hold for addresses within the hypervisor private range, like the ones you get from the pKVM heap allocator (hyp_alloc()). To resolve this, filter out non-kernel addresses based on PAGE_OFFSET. Leave the assembly version untouched as it has no current users. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-12-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_mmu.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 57f65257bb56..04d9f59db976 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -128,6 +128,9 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) * replace the instructions with `nop`s. */ #ifndef __KVM_VHE_HYPERVISOR__ + if (is_protected_kvm_enabled() && !is_ttbr1_addr(v)) + return v; + asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n" /* mask with va_mask */ "ror %0, %0, #1\n" /* rotate to the first tag bit */ "add %0, %0, #0\n" /* insert the low 12 bits of the tag */ From 85406921dc81ea7ccfb78c92cf79b0a4bcd2439c Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:43 +0100 Subject: [PATCH 416/506] KVM: arm64: Move hyp_vm refcount into the structure In preparation for allocating hyp_vm using the pKVM heap allocator (hyp_alloc()), move its reference count out of the page metadata (vmemmap) and place it into the structure itself. This transition is necessary because hyp_alloc() allows multiple small objects to share the same physical page. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-13-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 13 +++++++++++++ arch/arm64/kvm/hyp/nvhe/pkvm.c | 10 +++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index c1171451e1be..b208544141c0 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -42,6 +42,7 @@ struct pkvm_hyp_vm { struct kvm_pgtable pgt; struct kvm_pgtable_mm_ops mm_ops; struct hyp_pool pool; + unsigned short refcount; hyp_spinlock_t lock; /* Array of the hyp vCPU structures for this VM. */ @@ -66,6 +67,18 @@ static inline bool pkvm_hyp_vm_is_protected(struct pkvm_hyp_vm *hyp_vm) return kvm_vm_is_protected(&hyp_vm->kvm); } +static inline void pkvm_hyp_vm_ref_inc(struct pkvm_hyp_vm *hyp_vm) +{ + BUG_ON(hyp_vm->refcount == USHRT_MAX); + hyp_vm->refcount++; +} + +static inline void pkvm_hyp_vm_ref_dec(struct pkvm_hyp_vm *hyp_vm) +{ + BUG_ON(!hyp_vm->refcount); + hyp_vm->refcount--; +} + void pkvm_hyp_vm_table_init(void *tbl); int __pkvm_reserve_vm(void); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 5cdb0318fa7b..98db5bd28dde 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -278,7 +278,7 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, } hyp_vcpu->loaded_hyp_vcpu = this_cpu_ptr(&loaded_hyp_vcpu); - hyp_page_ref_inc(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_inc(hyp_vm); unlock: hyp_spin_unlock(&vm_table_lock); @@ -294,7 +294,7 @@ void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_spin_lock(&vm_table_lock); hyp_vcpu->loaded_hyp_vcpu = NULL; __this_cpu_write(loaded_hyp_vcpu, NULL); - hyp_page_ref_dec(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_dec(hyp_vm); hyp_spin_unlock(&vm_table_lock); } @@ -311,7 +311,7 @@ struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) hyp_spin_lock(&vm_table_lock); hyp_vm = get_vm_by_handle(handle); if (hyp_vm) - hyp_page_ref_inc(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_inc(hyp_vm); hyp_spin_unlock(&vm_table_lock); return hyp_vm; @@ -320,7 +320,7 @@ struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) void put_pkvm_hyp_vm(struct pkvm_hyp_vm *hyp_vm) { hyp_spin_lock(&vm_table_lock); - hyp_page_ref_dec(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_dec(hyp_vm); hyp_spin_unlock(&vm_table_lock); } @@ -979,7 +979,7 @@ static struct pkvm_hyp_vm *get_pkvm_unref_hyp_vm_locked(pkvm_handle_t handle) hyp_assert_lock_held(&vm_table_lock); hyp_vm = get_vm_by_handle(handle); - if (!hyp_vm || hyp_page_count(hyp_vm)) + if (!hyp_vm || hyp_vm->refcount) return NULL; return hyp_vm; From 324397c85d1dde38e5d4805d8b79bfc549af4252 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:44 +0100 Subject: [PATCH 417/506] KVM: arm64: Alloc pkvm_hyp_vm using pKVM heap allocator Transition the allocation of the hypervisor VM state structure (pkvm_hyp_vm) from the host to the hypervisor using the new pKVM heap allocator (hyp_alloc()). Previously, the host was responsible for calculating the size of, allocating, and donating memory for pkvm_hyp_vm during VM creation. With the heap allocator in place, the hypervisor now allocates this structure dynamically at EL2. Use the pkvm_call_hyp_req() wrapper in the host to invoke __pkvm_init_vm, which automatically handles any top-up requests if the hypervisor runs out of heap memory during allocation. Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-14-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hcall.h | 3 +-- arch/arm64/kvm/hyp/hyp-constants.c | 1 - arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 3 +-- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 32 +++++++++++++++++++++++--- arch/arm64/kvm/hyp/nvhe/pkvm.c | 25 ++++++++++---------- arch/arm64/kvm/pkvm.c | 21 ++++------------- 6 files changed, 47 insertions(+), 38 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index 9f09eb11193f..c98243e79e31 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -230,8 +230,7 @@ DECLARE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, pkvm_handle_t, handle) DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm, - struct kvm __kern *, host_kvm, void __kern *, vm_hva, - void __kern *, pgd_hva) + struct kvm __kern *, host_kvm, void __kern *, pgd_hva) DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu, void __kern *, vcpu_hva) diff --git a/arch/arm64/kvm/hyp/hyp-constants.c b/arch/arm64/kvm/hyp/hyp-constants.c index b257a3b4bfc5..501ab35a3840 100644 --- a/arch/arm64/kvm/hyp/hyp-constants.c +++ b/arch/arm64/kvm/hyp/hyp-constants.c @@ -7,7 +7,6 @@ int main(void) { DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page)); - DEFINE(PKVM_HYP_VM_SIZE, sizeof(struct pkvm_hyp_vm)); DEFINE(PKVM_HYP_VCPU_SIZE, sizeof(struct pkvm_hyp_vcpu)); return 0; } diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index b208544141c0..c5fd7315c0e3 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -83,8 +83,7 @@ void pkvm_hyp_vm_table_init(void *tbl); int __pkvm_reserve_vm(void); void __pkvm_unreserve_vm(pkvm_handle_t handle); -int __pkvm_init_vm(struct kvm *host_kvm, void __kern *vm_hva, - void __kern *pgd_hva); +int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva); int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, void __kern *vcpu_hva); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index a6e6c0941428..7eecac7c7fbe 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -69,6 +69,33 @@ DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); } \ static ret __do_##name(void) +/* + * Encode a hypervisor request in the host SMCCC return registers for known + * error numbers. + * + * Must be paired with pkvm_call_hyp_req() on the host side. + */ +static int errno_to_smccc(int ret) +{ + struct pkvm_hyp_req req = { .type = PKVM_HYP_NO_REQ }; + + switch (ret) { + case -ENOMEM: { + u32 nr_pages = hyp_alloc_topup_needed(); + + if (nr_pages) { + req.type = PKVM_HYP_REQ_HYP_ALLOC; + req.mem.nr_pages = nr_pages; + } + break; + } + } + + pkvm_hyp_req_to_smccc(host_data_ptr(host_ctxt), &req); + + return ret; +} + /* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */ unsigned int hyp_gicv3_nr_lr; @@ -756,10 +783,9 @@ DEFINE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, } DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vm, - struct kvm __kern *, host_kvm, void __kern *, vm_hva, - void __kern *, pgd_hva) + struct kvm __kern *, host_kvm, void __kern *, pgd_hva) { - return __pkvm_init_vm(kern_hyp_va_host(host_kvm), vm_hva, pgd_hva); + return errno_to_smccc(__pkvm_init_vm(kern_hyp_va_host(host_kvm), pgd_hva)); } DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 98db5bd28dde..c29a25cae8e8 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -11,6 +11,7 @@ #include +#include #include #include #include @@ -810,16 +811,13 @@ void teardown_selftest_vm(void) * Unmap the donated memory from the host at stage 2. * * host_kvm: A pointer to the host's struct kvm. - * vm_hva: The host va of the area being donated for the VM state. - * Must be page aligned. * pgd_hva: The host va of the area being donated for the stage-2 PGD for * the VM. Must be page aligned. Its size is implied by the VM's * VTCR. * * Return 0 success, negative error code on failure. */ -int __pkvm_init_vm(struct kvm *host_kvm, void __kern *vm_hva, - void __kern *pgd_hva) +int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva) { struct pkvm_hyp_vm *hyp_vm = NULL; size_t vm_size, pgd_size; @@ -847,15 +845,17 @@ int __pkvm_init_vm(struct kvm *host_kvm, void __kern *vm_hva, vm_size = pkvm_get_hyp_vm_size(nr_vcpus); pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.mmu.vtcr); - ret = -ENOMEM; + hyp_vm = hyp_alloc(vm_size); + if (!hyp_vm) { + ret = hyp_alloc_errno(); + goto err_unpin_kvm; + } - hyp_vm = map_donated_memory(vm_hva, vm_size); - if (!hyp_vm) - goto err_remove_mappings; + ret = -ENOMEM; pgd = map_donated_memory_noclear(pgd_hva, pgd_size); if (!pgd) - goto err_remove_mappings; + goto err_free_hyp_vm; init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus, handle); @@ -873,8 +873,9 @@ int __pkvm_init_vm(struct kvm *host_kvm, void __kern *vm_hva, err_destroy_stage2: kvm_guest_destroy_stage2(hyp_vm); err_remove_mappings: - unmap_donated_memory(hyp_vm, vm_size); unmap_donated_memory(pgd, pgd_size); +err_free_hyp_vm: + hyp_free(hyp_vm); err_unpin_kvm: hyp_unpin_shared_mem(host_kvm, host_kvm + 1); return ret; @@ -1010,7 +1011,6 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) struct pkvm_hyp_vm *hyp_vm; struct kvm *host_kvm; unsigned int idx; - size_t vm_size; int err; hyp_spin_lock(&vm_table_lock); @@ -1053,8 +1053,7 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu)); } - vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus); - teardown_donated_memory(mc, hyp_vm, vm_size); + hyp_free(hyp_vm); hyp_unpin_shared_mem(host_kvm, host_kvm + 1); return 0; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index ea96744f41fb..8505965dbfb2 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -212,8 +212,8 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) */ static int __pkvm_create_hyp_vm(struct kvm *kvm) { - size_t pgd_sz, hyp_vm_sz; - void *pgd, *hyp_vm; + size_t pgd_sz; + void *pgd; int ret; if (kvm->created_vcpus < 1) @@ -230,28 +230,15 @@ static int __pkvm_create_hyp_vm(struct kvm *kvm) if (!pgd) return -ENOMEM; - /* Allocate memory to donate to hyp for vm and vcpu pointers. */ - hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE, - size_mul(sizeof(void *), - kvm->created_vcpus))); - hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT); - if (!hyp_vm) { - ret = -ENOMEM; - goto free_pgd; - } - - /* Donate the VM memory to hyp and let hyp initialize it. */ - ret = kvm_call_hyp_nvhe(__pkvm_init_vm, kvm, hyp_vm, pgd); + ret = pkvm_call_hyp_req(__pkvm_init_vm, kvm, pgd); if (ret) - goto free_vm; + goto free_pgd; kvm->arch.pkvm.is_created = true; init_hyp_stage2_memcache(&kvm->arch.pkvm.stage2_teardown_mc); kvm_account_pgtable_pages(pgd, pgd_sz / PAGE_SIZE); return 0; -free_vm: - free_pages_exact(hyp_vm, hyp_vm_sz); free_pgd: free_pages_exact(pgd, pgd_sz); return ret; From e188a901e4bc585a170ee4235c6b55ebcf07b592 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:45 +0100 Subject: [PATCH 418/506] KVM: arm64: Alloc pkvm_hyp_vcpu using pKVM heap allocator Transition the allocation of the hypervisor vCPU state structure (pkvm_hyp_vcpu) from the host to the hypervisor using the new pKVM heap allocator (hyp_alloc()). Previously, the host was responsible for calculating the size of, allocating, and donating memory for pkvm_hyp_vcpu during vCPU creation. With the heap allocator in place, the hypervisor now allocates this structure dynamically at EL2. Use the pkvm_call_hyp_req() wrapper in the host to invoke __pkvm_create_hyp_vcpu, which automatically handles any top-up requests if the hypervisor runs out of heap memory during allocation. Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-15-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hcall.h | 3 +- arch/arm64/include/asm/kvm_host.h | 1 - arch/arm64/kvm/hyp/hyp-constants.c | 1 - arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 3 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 5 ++-- arch/arm64/kvm/hyp/nvhe/pkvm.c | 38 +++++--------------------- arch/arm64/kvm/pkvm.c | 11 +------- 7 files changed, 12 insertions(+), 50 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h index c98243e79e31..7dd176b4bea4 100644 --- a/arch/arm64/include/asm/kvm_hcall.h +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -232,8 +232,7 @@ DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm, struct kvm __kern *, host_kvm, void __kern *, pgd_hva) DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, - pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu, - void __kern *, vcpu_hva) + pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu) DECLARE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) DECLARE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, phys_addr_t, phys) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 64cd6233b439..3cca9f3b2eeb 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -267,7 +267,6 @@ struct kvm_smccc_features { struct kvm_protected_vm { pkvm_handle_t handle; - struct kvm_hyp_memcache teardown_mc; struct kvm_hyp_memcache stage2_teardown_mc; bool is_protected; bool is_created; diff --git a/arch/arm64/kvm/hyp/hyp-constants.c b/arch/arm64/kvm/hyp/hyp-constants.c index 501ab35a3840..b2caae21f271 100644 --- a/arch/arm64/kvm/hyp/hyp-constants.c +++ b/arch/arm64/kvm/hyp/hyp-constants.c @@ -7,6 +7,5 @@ int main(void) { DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page)); - DEFINE(PKVM_HYP_VCPU_SIZE, sizeof(struct pkvm_hyp_vcpu)); return 0; } diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index c5fd7315c0e3..866961c3bacb 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -84,8 +84,7 @@ void pkvm_hyp_vm_table_init(void *tbl); int __pkvm_reserve_vm(void); void __pkvm_unreserve_vm(pkvm_handle_t handle); int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva); -int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - void __kern *vcpu_hva); +int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu); int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn); int __pkvm_start_teardown_vm(pkvm_handle_t handle); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 7eecac7c7fbe..281d1359606a 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -789,10 +789,9 @@ DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vm, } DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, - pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu, - void __kern *, vcpu_hva) + pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu) { - return __pkvm_init_vcpu(handle, kern_hyp_va_host(host_vcpu), vcpu_hva); + return errno_to_smccc(__pkvm_init_vcpu(handle, kern_hyp_va_host(host_vcpu))); } DEFINE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index c29a25cae8e8..5031ef9e6fe9 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -672,16 +672,6 @@ static void *map_donated_memory_noclear(void __kern *host_va, size_t size) return va; } -static void *map_donated_memory(void __kern *host_va, size_t size) -{ - void *va = map_donated_memory_noclear(host_va, size); - - if (va) - memset(va, 0, size); - - return va; -} - static void __unmap_donated_memory(void *va, size_t size) { kvm_flush_dcache_to_poc(va, size); @@ -910,16 +900,15 @@ static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, return 0; } -int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - void __kern *vcpu_hva) +int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu) { struct pkvm_hyp_vcpu *hyp_vcpu; struct pkvm_hyp_vm *hyp_vm; int ret; - hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu)); + hyp_vcpu = hyp_alloc(sizeof(*hyp_vcpu)); if (!hyp_vcpu) - return -ENOMEM; + return hyp_alloc_errno(); hyp_spin_lock(&vm_table_lock); @@ -940,24 +929,12 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, } unlock: hyp_spin_unlock(&vm_table_lock); - if (ret) - unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu)); + hyp_free(hyp_vcpu); + return ret; } -static void -teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size) -{ - size = PAGE_ALIGN(size); - memset(addr, 0, size); - - for (void *start = addr; start < addr + size; start += PAGE_SIZE) - push_hyp_memcache(mc, start, hyp_virt_to_phys); - - unmap_donated_memory_noclear(addr, size); -} - int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn) { struct pkvm_hyp_vm *hyp_vm = get_pkvm_hyp_vm(handle); @@ -1007,7 +984,7 @@ int __pkvm_start_teardown_vm(pkvm_handle_t handle) int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) { - struct kvm_hyp_memcache *mc, *stage2_mc; + struct kvm_hyp_memcache *stage2_mc; struct pkvm_hyp_vm *hyp_vm; struct kvm *host_kvm; unsigned int idx; @@ -1028,7 +1005,6 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) hyp_spin_unlock(&vm_table_lock); /* Reclaim guest pages (including page-table pages) */ - mc = &host_kvm->arch.pkvm.teardown_mc; stage2_mc = &host_kvm->arch.pkvm.stage2_teardown_mc; reclaim_pgtable_pages(hyp_vm, stage2_mc); unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus); @@ -1050,7 +1026,7 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) unmap_donated_memory_noclear(addr, PAGE_SIZE); } - teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu)); + hyp_free(hyp_vcpu); } hyp_free(hyp_vm); diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8505965dbfb2..36e677e51f7d 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -174,28 +174,19 @@ static void __pkvm_destroy_hyp_vm(struct kvm *kvm) kvm->arch.pkvm.handle = 0; kvm->arch.pkvm.is_created = false; - free_hyp_memcache(&kvm->arch.pkvm.teardown_mc); free_hyp_memcache(&kvm->arch.pkvm.stage2_teardown_mc); } static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) { - size_t hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE); pkvm_handle_t handle = vcpu->kvm->arch.pkvm.handle; - void *hyp_vcpu; int ret; init_hyp_stage2_memcache(&vcpu->arch.pkvm_memcache); - hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT); - if (!hyp_vcpu) - return -ENOMEM; - - ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, vcpu, hyp_vcpu); + ret = pkvm_call_hyp_req(__pkvm_init_vcpu, handle, vcpu); if (!ret) vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); - else - free_pages_exact(hyp_vcpu, hyp_vcpu_sz); return ret; } From 197dac8a9060c17988414086293aa8ed4c82c185 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:46 +0100 Subject: [PATCH 419/506] KVM: arm64: Rename vCPU pkvm_memcache to stage2_mc With the introduction of the pKVM hyp allocator, the memcache for vCPUs is now only used for carrying pages dedicated to the stage-2 page tables. Rename pkvm_memcache to stage2_mc to reflect its purpose. Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-16-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 4 ++-- arch/arm64/kvm/arm.c | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 6 +++--- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 10 +++++----- arch/arm64/kvm/hyp/nvhe/pkvm.c | 6 +++--- arch/arm64/kvm/mmu.c | 2 +- arch/arm64/kvm/pkvm.c | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 3cca9f3b2eeb..0c4498a2f30d 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -937,8 +937,8 @@ struct kvm_vcpu_arch { /* Cache some mmu pages needed inside spinlock regions */ struct kvm_mmu_memory_cache mmu_page_cache; - /* Pages to top-up the pKVM/EL2 guest pool */ - struct kvm_hyp_memcache pkvm_memcache; + /* Pages to top-up the pKVM/EL2 guest stage-2 pool */ + struct kvm_hyp_memcache stage2_mc; /* Virtual SError ESR to restore when HCR_EL2.VSE is set */ u64 vsesr_el2; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 09ae5431ce58..fa77abf8cff6 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -591,7 +591,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) if (!is_protected_kvm_enabled()) kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); else - free_hyp_memcache(&vcpu->arch.pkvm_memcache); + free_hyp_memcache(&vcpu->arch.stage2_mc); kvm_timer_vcpu_terminate(vcpu); kvm_pmu_vcpu_destroy(vcpu); kvm_vgic_vcpu_destroy(vcpu); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 281d1359606a..af348e008f52 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -464,9 +464,9 @@ static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; - return refill_memcache(&hyp_vcpu->vcpu.arch.pkvm_memcache, - host_vcpu->arch.pkvm_memcache.nr_pages, - &host_vcpu->arch.pkvm_memcache); + return refill_memcache(&hyp_vcpu->vcpu.arch.stage2_mc, + host_vcpu->arch.stage2_mc.nr_pages, + &host_vcpu->arch.stage2_mc); } DEFINE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 39aa8911f62c..dadd48431036 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -1023,7 +1023,7 @@ int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn) ret = 0; WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_SHARED_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); WARN_ON(__host_set_page_state_range(phys, PAGE_SIZE, PKVM_PAGE_SHARED_BORROWED)); unlock: guest_unlock_component(vm); @@ -1058,7 +1058,7 @@ int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn) PKVM_ID_GUEST, meta)); WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); host_unlock_component(); @@ -1422,7 +1422,7 @@ static int __guest_check_pgtable_memcache(struct pkvm_hyp_vcpu *vcpu) { struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); - if (vcpu->vcpu.arch.pkvm_memcache.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu)) + if (vcpu->vcpu.arch.stage2_mc.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu)) return -ENOMEM; return 0; @@ -1456,7 +1456,7 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu) PKVM_ID_GUEST, meta)); WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); @@ -1527,7 +1527,7 @@ int __pkvm_host_share_guest(u64 pfn, u64 gfn, u64 nr_pages, struct pkvm_hyp_vcpu WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, pkvm_mkstate(prot, PKVM_PAGE_SHARED_BORROWED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 5031ef9e6fe9..599488b6d124 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -774,7 +774,7 @@ struct pkvm_hyp_vcpu *init_selftest_vm(void *virt) continue; p[i].refcount = 1; if (seeded < min_pages) { - push_hyp_memcache(&selftest_vcpu.vcpu.arch.pkvm_memcache, + push_hyp_memcache(&selftest_vcpu.vcpu.arch.stage2_mc, hyp_page_to_virt(&p[i]), hyp_virt_to_phys); seeded++; } else { @@ -1009,7 +1009,7 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) reclaim_pgtable_pages(hyp_vm, stage2_mc); unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus); - /* Push the metadata pages to the teardown memcache */ + /* Push the stage-2 pages to the teardown memcache */ for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) { struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; struct kvm_hyp_memcache *vcpu_mc; @@ -1017,7 +1017,7 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) if (!hyp_vcpu) continue; - vcpu_mc = &hyp_vcpu->vcpu.arch.pkvm_memcache; + vcpu_mc = &hyp_vcpu->vcpu.arch.stage2_mc; while (vcpu_mc->nr_pages) { void *addr = pop_hyp_memcache(vcpu_mc, hyp_phys_to_virt); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 3163502cc7d2..afde0894ab15 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1553,7 +1553,7 @@ static void *get_mmu_memcache(struct kvm_vcpu *vcpu) if (!is_protected_kvm_enabled()) return &vcpu->arch.mmu_page_cache; else - return &vcpu->arch.pkvm_memcache; + return &vcpu->arch.stage2_mc; } static int topup_mmu_memcache(struct kvm_vcpu *vcpu, void *memcache) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 36e677e51f7d..83e18c0e30b5 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -182,7 +182,7 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) pkvm_handle_t handle = vcpu->kvm->arch.pkvm.handle; int ret; - init_hyp_stage2_memcache(&vcpu->arch.pkvm_memcache); + init_hyp_stage2_memcache(&vcpu->arch.stage2_mc); ret = pkvm_call_hyp_req(__pkvm_init_vcpu, handle, vcpu); if (!ret) From cf9cf059b8f30ffca78c16e6a51c905c95da47a8 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:47 +0100 Subject: [PATCH 420/506] KVM: arm64: Reject hyp trace descriptors with fewer CPUs than hyp_nr_cpus While a trace descriptor with fewer CPUs than hyp_nr_cpus currently causes no functional issue, such a configuration is invalid and would cause the hypervisor to skip ring buffer initialization when allocating backing pages in EL2. Enforce nr_cpus == hyp_nr_cpus during trace descriptor validation to guarantee that every possible CPU has a valid ring buffer descriptor. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-17-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/trace.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 47680a949f61..e9e584694506 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -181,6 +181,9 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz desc_end = (void *)desc + desc_size; nr_bpages = desc->bpages_backing_size / sizeof(struct simple_buffer_page); + if (desc->trace_buffer_desc.nr_cpus != hyp_nr_cpus) + return false; + for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) { /* Can we read nr_page_va? */ if ((void *)rb_desc + struct_size(rb_desc, page_va, 0) > desc_end) From 086f86286b5e060904bd372ad96edd04c366885a Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:48 +0100 Subject: [PATCH 421/506] KVM: arm64: Reject hyp trace descriptors with fewer than 3 pages A trace descriptor with 0 nr_page_va allocates a 0-size bpages backing region, which makes the ring buffer appear unloaded. Reject descriptors with nr_page_va < 3 in hyp_trace_desc_is_valid() as this is in any case the lower-limit for simple_ring_buffer. Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-18-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/trace.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index e9e584694506..7acb81ae0e6a 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -189,6 +189,10 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz if ((void *)rb_desc + struct_size(rb_desc, page_va, 0) > desc_end) return false; + /* simple_ring_buffer_init_mm() expects at least 3 pages */ + if (rb_desc->nr_page_va < 3) + return false; + /* Overflow desc? */ if ((void *)rb_desc + struct_size(rb_desc, page_va, rb_desc->nr_page_va) > desc_end) return false; From 35dc2dd00334555eb02428cff2302a1cfe60c15f Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:28:49 +0100 Subject: [PATCH 422/506] KVM: arm64: Alloc simple_buffer_page using pKVM hyp allocator In protected mode, transition the allocation of the simple_ring_buffer structures from the host to the hypervisor using the new pKVM heap allocator. Previously, the host allocated and donated a contiguous backing memory for these structures. In pKVM the hypervisor can now allocate them dynamically. Use the pkvm_call_hyp_req() wrapper in the host to invoke __tracing_load, which automatically handles any top-up requests if the hypervisor runs out of heap memory during allocation. Signed-off-by: Vincent Donnefort Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261001142849.3367614-19-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 2 +- arch/arm64/kvm/hyp/nvhe/trace.c | 63 +++++++++++++++++------------- arch/arm64/kvm/hyp_trace.c | 15 ++++++- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index af348e008f52..7a6435725dfe 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -907,7 +907,7 @@ DEFINE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, DEFINE_KVM_HOST_HCALL(int, __tracing_load, void __kern *, desc_hva, size_t, desc_size) { - return __tracing_load(desc_hva, desc_size); + return errno_to_smccc(__tracing_load(desc_hva, desc_size)); } DEFINE_KVM_HOST_HCALL0(void, __tracing_unload) diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 7acb81ae0e6a..0d7e4db0c3e1 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -4,6 +4,7 @@ * Author: Vincent Donnefort */ +#include #include #include #include @@ -62,18 +63,34 @@ static void __release_host_mem(void *start, u64 size) WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT)); } -static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_buffer, - struct hyp_trace_desc *desc) +static int hyp_trace_buffer_alloc_bpages(struct hyp_trace_buffer *trace_buffer, + struct hyp_trace_desc *desc) { - void *start = (void *)kern_hyp_va(desc->bpages_backing_start); - size_t size = desc->bpages_backing_size; + void *start; + size_t size; int ret; - ret = __admit_host_mem(start, size); - if (ret) - return ret; + if (is_protected_kvm_enabled()) { + struct ring_buffer_desc *rb_desc; + int cpu; - memset(start, 0, size); + size = 0; + for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) + size += rb_desc->nr_page_va * sizeof(struct simple_buffer_page); + + start = hyp_alloc(size); + if (!start) + return hyp_alloc_errno(); + } else { + start = (void *)kern_hyp_va(desc->bpages_backing_start); + size = desc->bpages_backing_size; + + ret = __admit_host_mem(start, size); + if (ret) + return ret; + + memset(start, 0, size); + } trace_buffer->bpages_backing_start = start; trace_buffer->bpages_backing_size = size; @@ -81,7 +98,7 @@ static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_bu return 0; } -static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace_buffer) +static void hyp_trace_buffer_free_bpages(struct hyp_trace_buffer *trace_buffer) { void *start = trace_buffer->bpages_backing_start; size_t size = trace_buffer->bpages_backing_size; @@ -89,9 +106,12 @@ static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace if (!size) return; - memset(start, 0, size); - - __release_host_mem(start, size); + if (is_protected_kvm_enabled()) { + hyp_free(start); + } else { + memset(start, 0, size); + __release_host_mem(start, size); + } trace_buffer->bpages_backing_start = NULL; trace_buffer->bpages_backing_size = 0; @@ -128,7 +148,7 @@ static void hyp_trace_buffer_unload(struct hyp_trace_buffer *trace_buffer) simple_ring_buffer_unload_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu), __unpin_shared_page); - hyp_trace_buffer_unload_bpage_backing(trace_buffer); + hyp_trace_buffer_free_bpages(trace_buffer); } static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer, @@ -143,7 +163,7 @@ static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer, if (hyp_trace_buffer_loaded(trace_buffer)) return -EINVAL; - ret = hyp_trace_buffer_load_bpage_backing(trace_buffer, desc); + ret = hyp_trace_buffer_alloc_bpages(trace_buffer, desc); if (ret) return ret; @@ -168,18 +188,16 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz { struct ring_buffer_desc *rb_desc; unsigned int cpu; - size_t nr_bpages; void *desc_end; if (!is_protected_kvm_enabled()) return true; /* - * Both desc_size and bpages_backing_size are untrusted host-provided - * values. We rely on __pkvm_host_donate_hyp() to enforce their validity. + * desc_size is an untrusted host-provided value. We rely on + * __pkvm_host_donate_hyp() to enforce its validity. */ desc_end = (void *)desc + desc_size; - nr_bpages = desc->bpages_backing_size / sizeof(struct simple_buffer_page); if (desc->trace_buffer_desc.nr_cpus != hyp_nr_cpus) return false; @@ -197,17 +215,8 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz if ((void *)rb_desc + struct_size(rb_desc, page_va, rb_desc->nr_page_va) > desc_end) return false; - /* Overflow bpages backing memory? */ - if (nr_bpages < rb_desc->nr_page_va) - return false; - - if (cpu >= hyp_nr_cpus) - return false; - if (cpu != rb_desc->cpu) return false; - - nr_bpages -= rb_desc->nr_page_va; } return true; diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 44937e257375..5bc881d6f343 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "hyp_trace.h" @@ -151,11 +152,18 @@ static void __unshare_page(unsigned long va) static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size) { - int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; size_t backing_size; + int nr_bpages; void *start; int ret; + if (is_protected_kvm_enabled()) { + trace_buffer->desc->bpages_backing_start = 0; + trace_buffer->desc->bpages_backing_size = 0; + return 0; + } + + nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * num_possible_cpus()); @@ -177,6 +185,9 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer) { + if (!trace_buffer->desc->bpages_backing_start) + return; + free_pages_exact((void *)trace_buffer->desc->bpages_backing_start, trace_buffer->desc->bpages_backing_size); } @@ -269,7 +280,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) if (ret) goto err_free_buffer; - ret = kvm_call_hyp_nvhe(__tracing_load, desc, desc_size); + ret = pkvm_call_hyp_req(__tracing_load, desc, desc_size); if (ret) goto err_unload_pages; From d35311c5d66095da3949787064d4738fee435d67 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:29:57 +0100 Subject: [PATCH 423/506] KVM: arm64: Allow early calls to pKVM host_share/unshare_hyp The hypervisor tracing for pKVM relies on the __pkvm_host_share_hyp and __pkvm_host_unshare_hyp HVCs. In order to start tracing as early as possible, allow those two HVCs before the host is deprivileged. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001143000.3371786-2-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 4 ++-- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index e5b92ac09e69..da8b6e3c01e3 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -89,12 +89,12 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, + __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp, + __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp, MARKER(__KVM_HOST_SMCCC_FUNC_PKVM_ONLY), /* Hypercalls that are available only when pKVM has finalised. */ - __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp, - __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp, __KVM_HOST_SMCCC_FUNC___pkvm_host_donate_guest, __KVM_HOST_SMCCC_FUNC___pkvm_host_share_guest, __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_guest, diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9a3b92e626ad..4f824b01465a 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -934,9 +934,9 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), HANDLE_FUNC(__vgic_v5_save_apr), HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), - HANDLE_FUNC(__pkvm_host_share_hyp), HANDLE_FUNC(__pkvm_host_unshare_hyp), + HANDLE_FUNC(__pkvm_host_donate_guest), HANDLE_FUNC(__pkvm_host_share_guest), HANDLE_FUNC(__pkvm_host_unshare_guest), From 955feee93e037af78caed7e98d29bbf0268420b0 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:29:58 +0100 Subject: [PATCH 424/506] KVM: arm64: Move kvm_define_hypevents.h to arch/arm64/kvm/ kvm_define_hypevents.h is used to define the kernel-side structures for hypervisor events. It doesn't need to be used anywhere else than in hyp_trace.c. Rename and move it to arch/arm64/kvm/ Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001143000.3371786-3-vdonnefort@google.com Signed-off-by: Marc Zyngier --- .../asm/kvm_define_hypevents.h => kvm/define_hypevents.h} | 0 arch/arm64/kvm/hyp_trace.c | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename arch/arm64/{include/asm/kvm_define_hypevents.h => kvm/define_hypevents.h} (100%) diff --git a/arch/arm64/include/asm/kvm_define_hypevents.h b/arch/arm64/kvm/define_hypevents.h similarity index 100% rename from arch/arm64/include/asm/kvm_define_hypevents.h rename to arch/arm64/kvm/define_hypevents.h diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 9644c424819b..8faaec899f92 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -402,7 +402,7 @@ static struct trace_remote_callbacks trace_remote_callbacks = { static const char *__hyp_enter_exit_reason_str(u8 reason); -#include +#include "define_hypevents.h" static const char *__hyp_enter_exit_reason_str(u8 reason) { From 604aaeb39521ef70f56c036dc065b133c85f06aa Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:29:59 +0100 Subject: [PATCH 425/506] tracing/remotes: Add REMOTE_EVENT_CUSTOM_PRINTK() helper The current REMOTE_EVENT() takes as a __printk argument a string format and a list of arguments, such as RE_PRINTK("foo=%d bar=%d", foo, bar). Add a REMOTE_EVENT_CUSTOM_PRINTK() where the __printk argument can be a function. This intends to support the creation of a "printk" event for the arm64 nVHE/pKVM hypervisor with a dynamic prototype and by extension a dynamic print format. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Acked-by: Steven Rostedt Link: https://patch.msgid.link/20261001143000.3371786-4-vdonnefort@google.com Signed-off-by: Marc Zyngier --- include/trace/define_remote_events.h | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/include/trace/define_remote_events.h b/include/trace/define_remote_events.h index 676e803dc144..78346ac77585 100644 --- a/include/trace/define_remote_events.h +++ b/include/trace/define_remote_events.h @@ -35,17 +35,26 @@ do { \ #define RE_PRINTK(__args...) __args -#define REMOTE_EVENT(__name, __id, __struct, __printk) \ - REMOTE_EVENT_FORMAT(__name, __struct); \ +#define REMOTE_EVENT_PRINT_FUNC(__name, __printk) \ static void remote_event_print_##__name(void *evt, struct trace_seq *seq) \ { \ struct remote_event_format_##__name __maybe_unused *__entry = evt; \ trace_seq_puts(seq, #__name); \ - remote_printk(__printk); \ + __printk; \ } + +#define REMOTE_EVENT(__name, __id, __struct, __printk) \ + REMOTE_EVENT_FORMAT(__name, __struct); \ + REMOTE_EVENT_PRINT_FUNC(__name, remote_printk(__printk)) + +#define REMOTE_EVENT_CUSTOM_PRINTK(__name, __id, __struct, __printk) \ + REMOTE_EVENT_FORMAT(__name, __struct); \ + REMOTE_EVENT_PRINT_FUNC(__name, __printk) + #include REMOTE_EVENT_INCLUDE(REMOTE_EVENT_INCLUDE_FILE) #undef REMOTE_EVENT +#undef REMOTE_EVENT_CUSTOM_PRINTK #undef RE_PRINTK #undef re_field #define re_field(__type, __field) \ @@ -70,4 +79,8 @@ do { \ .print_fmt = remote_event_print_fmt_##__name, \ .print = remote_event_print_##__name, \ } + +#define REMOTE_EVENT_CUSTOM_PRINTK(__name, __id, __struct, __printk) \ + REMOTE_EVENT(__name, __id, RE_STRUCT(__struct), "\"Unknown\"") + #include REMOTE_EVENT_INCLUDE(REMOTE_EVENT_INCLUDE_FILE) From 92866564020c2fbede7f77f5957a75f11f931d53 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 1 Oct 2026 15:30:00 +0100 Subject: [PATCH 426/506] KVM: arm64: Add hyp_printk event to nVHE/pKVM hyp Create an event to allow developers to log pretty much anything into the hypervisor tracing buffer with trace_hyp_printk(), just like the kernel tracing has the function trace_printk(). #include trace_hyp_printk("foobar"); trace_hyp_printk("foo=%d", foo); trace_hyp_printk("foo=%d bar=0x%016llx", foo, bar); To ensure writing into the tracing buffer is fast, store the string format into a kernel-accessible ELF section. The hypervisor only has to write into the event the string ID, which is the delta between the hyp_string_fmt and the start of the ELF section. The string format is parsed by the kernel. It is therefore not possible to dereference hypervisor pointers and the format specifier %s is not supported. Also the %p specifiers will fail to find the hypervisor function. To not waste tracing buffer data, use a dynamic size. Each argument is 8 bytes. Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261001143000.3371786-5-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hypevents.h | 14 ++++ arch/arm64/include/asm/kvm_hyptrace.h | 9 +++ arch/arm64/kernel/image-vars.h | 1 + arch/arm64/kernel/vmlinux.lds.S | 8 +++ .../kvm/hyp/include/nvhe/define_events.h | 2 - arch/arm64/kvm/hyp/include/nvhe/trace.h | 66 +++++++++++++++++++ arch/arm64/kvm/hyp/nvhe/events.c | 6 ++ arch/arm64/kvm/hyp_trace.c | 61 +++++++++++++++++ 8 files changed, 165 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hypevents.h b/arch/arm64/include/asm/kvm_hypevents.h index 5f6e6789d121..3396deb1a1f8 100644 --- a/arch/arm64/include/asm/kvm_hypevents.h +++ b/arch/arm64/include/asm/kvm_hypevents.h @@ -58,4 +58,18 @@ HYP_EVENT(selftest, ), RE_PRINTK("id=%llu", __entry->id) ); + +/* + * trace_hyp_printk() has too many specificities to be declared with HYP_EVENT(). + * However, we can use a REMOTE_EVENT macro to automatically do the declaration + * for the kernel side. + */ +REMOTE_EVENT_CUSTOM_PRINTK(hyp_printk, + 0, /* id will be overwritten during hyp event init */ + RE_STRUCT( + re_field(u16, fmt_id) + re_field(u64, args[]) + ), + __hyp_trace_printk(evt, seq) +); #endif diff --git a/arch/arm64/include/asm/kvm_hyptrace.h b/arch/arm64/include/asm/kvm_hyptrace.h index de133b735f72..16e8d60abba9 100644 --- a/arch/arm64/include/asm/kvm_hyptrace.h +++ b/arch/arm64/include/asm/kvm_hyptrace.h @@ -23,4 +23,13 @@ extern struct remote_event __hyp_events_end[]; extern struct hyp_event_id __hyp_event_ids_start[]; extern struct hyp_event_id __hyp_event_ids_end[]; +#define HYP_STRING_FMT_MAX_SIZE 127 + +struct hyp_string_fmt { + const char fmt[HYP_STRING_FMT_MAX_SIZE]; + u8 nr_args; +}; + +extern const struct hyp_string_fmt __hyp_string_fmts_start[]; +extern const struct hyp_string_fmt __hyp_string_fmts_end[]; #endif diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index 14beb7b9d304..ef9875a95f95 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -143,6 +143,7 @@ KVM_NVHE_ALIAS(__hyp_rodata_end); #ifdef CONFIG_NVHE_EL2_TRACING KVM_NVHE_ALIAS(__hyp_event_ids_start); KVM_NVHE_ALIAS(__hyp_event_ids_end); +KVM_NVHE_ALIAS(__hyp_string_fmts_start); #endif /* pKVM static key */ diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index af1d72020976..69ef676e659d 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -214,6 +214,14 @@ SECTIONS HYPERVISOR_RODATA_SECTIONS +#ifdef CONFIG_NVHE_EL2_TRACING + .rodata.hyp_string_fmts : { + __hyp_string_fmts_start = .; + *(_hyp_string_fmts) + __hyp_string_fmts_end = .; + } +#endif + .got : { *(.got) } /* * Make sure that the .got.plt is either completely empty or it diff --git a/arch/arm64/kvm/hyp/include/nvhe/define_events.h b/arch/arm64/kvm/hyp/include/nvhe/define_events.h index 776d4c6cb702..370e8c2d39fe 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/define_events.h +++ b/arch/arm64/kvm/hyp/include/nvhe/define_events.h @@ -10,5 +10,3 @@ #define HYP_EVENT_MULTI_READ #include #undef HYP_EVENT_MULTI_READ - -#undef HYP_EVENT diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h index 8813ff250f8e..d38e0fb284f8 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/trace.h +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h @@ -46,6 +46,70 @@ static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt) void *tracing_reserve_entry(unsigned long length); void tracing_commit_entry(void); +/* + * The trace_hyp_printk boilerplate is too fiddly to be declared with + * HYP_EVENT(): + * + * The string format is stored into a kernel-accessible ELF section. The + * hypervisor only writes the format ID. + * + * The function has a variadic prototype. We have no easy way to know each + * argument width so they must all cast to u64. + */ +#define REMOTE_EVENT_CUSTOM_PRINTK(...) + +#define __TO_U64_0() +#define __TO_U64_1(x) , (u64)(x) +#define __TO_U64_2(x, ...) , (u64)(x) __TO_U64_1(__VA_ARGS__) +#define __TO_U64_3(x, ...) , (u64)(x) __TO_U64_2(__VA_ARGS__) +#define __TO_U64_4(x, ...) , (u64)(x) __TO_U64_3(__VA_ARGS__) +#define __TO_U64_5(x, ...) , (u64)(x) __TO_U64_4(__VA_ARGS__) +#define __TO_U64_6(x, ...) , (u64)(x) __TO_U64_5(__VA_ARGS__) +#define __TO_U64_7(x, ...) , (u64)(x) __TO_U64_6(__VA_ARGS__) +#define __TO_U64_8(x, ...) , (u64)(x) __TO_U64_7(__VA_ARGS__) + +#define __TO_U64_X(N, ...) CONCATENATE(__TO_U64_, N)(__VA_ARGS__) +#define __TO_U64(...) __TO_U64_X(COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__) + +REMOTE_EVENT_FORMAT(hyp_printk, HE_STRUCT(he_field(u16, fmt_id) he_field(u64, args[]))); +extern struct hyp_event_id hyp_event_id_hyp_printk; + +static __always_inline void __trace_hyp_printk(const struct hyp_string_fmt *fmt, int nr_args, ...) +{ + struct remote_event_format_hyp_printk *entry; + va_list va; + int i; + + if (!atomic_read(&hyp_event_id_hyp_printk.enabled)) + return; + + entry = tracing_reserve_entry(struct_size(entry, args, nr_args)); + if (!entry) + return; + + entry->hdr.id = hyp_event_id_hyp_printk.id; + entry->fmt_id = fmt - __hyp_string_fmts_start; + + va_start(va, nr_args); + for (i = 0; i < nr_args; i++) + entry->args[i] = va_arg(va, u64); + va_end(va); + + tracing_commit_entry(); +} + +#define trace_hyp_printk(__fmt, __args...) \ +do { \ + static const struct hyp_string_fmt __used __section("_hyp_string_fmts") \ + __hyp_str_fmt = { \ + .fmt = __fmt, \ + .nr_args = COUNT_ARGS(__args), \ + }; \ + BUILD_BUG_ON(sizeof(__fmt) > HYP_STRING_FMT_MAX_SIZE); \ + /* __TO_U64 prepends a comma if there are arguments */ \ + __trace_hyp_printk(&__hyp_str_fmt, COUNT_ARGS(__args) __TO_U64(__args)); \ +} while (0) + int __tracing_load(unsigned long desc_va, size_t desc_size); void __tracing_unload(void); int __tracing_enable(bool enable); @@ -58,6 +122,8 @@ static inline void *tracing_reserve_entry(unsigned long length) { return NULL; } static inline void tracing_commit_entry(void) { } #define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \ static inline void trace_##__name(__proto) {} +#define REMOTE_EVENT_CUSTOM_PRINTK(...) +#define trace_hyp_printk(fmt, args...) do { } while (0) static inline int __tracing_load(unsigned long desc_va, size_t desc_size) { return -ENODEV; } static inline void __tracing_unload(void) { } diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/events.c index add9383aadb5..12223d2e3618 100644 --- a/arch/arm64/kvm/hyp/nvhe/events.c +++ b/arch/arm64/kvm/hyp/nvhe/events.c @@ -9,6 +9,12 @@ #include +/* + * The hyp_printk event is not declared with HYP_EVENT in kvm_hypevents.h, + * so we manually add the boilerplate here. + */ +HYP_EVENT(hyp_printk, 0, 0, 0, 0); + int __tracing_enable_event(unsigned short id, bool enable) { struct hyp_event_id *event_id = &__hyp_event_ids_start[id]; diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 8faaec899f92..c61488d16f4c 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -402,6 +402,9 @@ static struct trace_remote_callbacks trace_remote_callbacks = { static const char *__hyp_enter_exit_reason_str(u8 reason); +struct remote_event_format_hyp_printk; +static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq); + #include "define_hypevents.h" static const char *__hyp_enter_exit_reason_str(u8 reason) @@ -421,6 +424,61 @@ static const char *__hyp_enter_exit_reason_str(u8 reason) return strs[min(reason, HYP_REASON_UNKNOWN)]; } +static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq) +{ + const struct hyp_string_fmt *hyp_str_fmt; + const char *fmt; + + trace_seq_putc(seq, ' '); + + if (entry->fmt_id >= (__hyp_string_fmts_end - __hyp_string_fmts_start)) { + trace_seq_printf(seq, "Unknown hyp_string_fmt ID %u\n", entry->fmt_id); + return; + } + + hyp_str_fmt = &__hyp_string_fmts_start[entry->fmt_id]; + fmt = hyp_str_fmt->fmt; + + switch (hyp_str_fmt->nr_args) { + case 0: + trace_seq_puts(seq, fmt); + break; + case 1: + trace_seq_printf(seq, fmt, entry->args[0]); + break; + case 2: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1]); + break; + case 3: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2]); + break; + case 4: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3]); + break; + case 5: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4]); + break; + case 6: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4], entry->args[5]); + break; + case 7: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4], entry->args[5], entry->args[6]); + break; + default: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], + entry->args[2], entry->args[3], entry->args[4], entry->args[5], + entry->args[6], entry->args[7]); + break; + } + + if (trace_seq_used(seq) && seq->buffer[trace_seq_used(seq) - 1] != '\n') + trace_seq_putc(seq, '\n'); +} + static void __init hyp_trace_init_events(void) { struct hyp_event_id *hyp_event_id = __hyp_event_ids_start; @@ -430,6 +488,9 @@ static void __init hyp_trace_init_events(void) /* Events on both sides hypervisor are sorted */ for (; event < __hyp_events_end; event++, hyp_event_id++, id++) event->id = hyp_event_id->id = id; + + WARN(__hyp_string_fmts_end - __hyp_string_fmts_start > U16_MAX + 1, + "Too many trace_hyp_printk() callsites\n"); } int __init kvm_hyp_trace_init(void) From 8a88a46b86ca3dbbd24522760d29edfb3f24ac2a Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Mon, 31 Aug 2026 10:10:01 +0200 Subject: [PATCH 427/506] KVM: arm64: vgic-v3: Undo assignment on iodev registration failure vgic_register_redist_iodev() assigns a redistributor region and base address to the vCPU before adding its iodev to the MMIO bus. However, the region's free_index is advanced only after registration succeeds. If kvm_io_bus_register_dev() fails, the vCPU retains the assignment while free_index still identifies the same slot as free. A later registration can therefore reuse a slot that remains assigned to the vCPU. Reserve the slot before registering its iodev. The caller holds slots_lock, so a registration failure cannot race with a later assignment. Undo the reservation and clear the cached assignment on failure. Fixes: dbd9733ab674 ("KVM: arm/arm64: Replace the single rdist region by a list") Cc: stable@vger.kernel.org Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Karl Mehltretter Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260831081005.41346-2-kmehltretter@gmail.com [maz: moved helper into the sole caller] Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 5913a20d8301..8fb0a3eb3457 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -818,16 +818,21 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers); rd_dev->redist_vcpu = vcpu; + /* Protected by slots_lock */ + rdreg->free_index++; + mutex_unlock(&kvm->arch.config_lock); ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base, 2 * SZ_64K, &rd_dev->dev); - if (ret) - return ret; + if (ret) { + guard(mutex)(&vcpu->kvm->arch.config_lock); + vgic_cpu->rdreg->free_index--; + vgic_cpu->rdreg = NULL; + vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF; + } - /* Protected by slots_lock */ - rdreg->free_index++; - return 0; + return ret; out_unlock: mutex_unlock(&kvm->arch.config_lock); From aa41a7c3beb59f9dac4e60c9cf1d146f6c9025c4 Mon Sep 17 00:00:00 2001 From: Colton Lewis Date: Wed, 30 Sep 2026 21:18:33 +0000 Subject: [PATCH 428/506] KVM: arm64: Handle ID_AA64DFR0_EL1.PMUVer == NI in __kvm_pmu_event_mask() When userspace initializes a vCPU with KVM_ARM_VCPU_PMU_V3 but sets ID_AA64DFR0_EL1.PMUVer to NI (0) or IMP_DEF (0xf, which is sanitized to NI), resetting the vCPU invokes reset_pmevtyper() and calls kvm_pmu_evtyper_mask(). Because __kvm_pmu_event_mask() does not handle ID_AA64DFR0_EL1_PMUVer_NI, it triggers a WARN_ONCE("Unknown PMU version 0"). Explicitly handle ID_AA64DFR0_EL1_PMUVer_NI in __kvm_pmu_event_mask() by returning an event mask of 0, and drop the redundant kvm_vcpu_has_pmu() check in reset_pmevtyper(). Fixes: bc512d6a9b92 ("KVM: arm64: Make PMEVTYPER_EL0.NSH RES0 if EL2 isn't advertised") Suggested-by: Oliver Upton Assisted-by: LLM Signed-off-by: Colton Lewis Link: https://patch.msgid.link/20260930211834.2773546-2-coltonlewis@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/pmu-emul.c | 5 ++++- arch/arm64/kvm/sys_regs.c | 4 ---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 5b1af7e2176f..e2b28dbc0451 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -50,8 +50,11 @@ static u32 __kvm_pmu_event_mask(unsigned int pmuver) case ID_AA64DFR0_EL1_PMUVer_V3P5: case ID_AA64DFR0_EL1_PMUVer_V3P7: return GENMASK(15, 0); - default: /* Shouldn't be here, just for sanity */ + default: + /* Shouldn't be here, just for sanity */ WARN_ONCE(1, "Unknown PMU version %d\n", pmuver); + fallthrough; + case ID_AA64DFR0_EL1_PMUVer_NI: return 0; } } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 44aae52c473d..87f96c9bd2fe 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1058,10 +1058,6 @@ static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - /* This thing will UNDEF, who cares about the reset value? */ - if (!kvm_vcpu_has_pmu(vcpu)) - return 0; - reset_unknown(vcpu, r); __vcpu_rmw_sys_reg(vcpu, r->reg, &=, kvm_pmu_evtyper_mask(vcpu->kvm)); From 517cdec3d4b28ccf5dae2dd03dee2a0a1eef1981 Mon Sep 17 00:00:00 2001 From: Colton Lewis Date: Wed, 30 Sep 2026 21:18:34 +0000 Subject: [PATCH 429/506] KVM: arm64: Check ID_AA64DFR0_EL1.PMUVer in pmu_visibility() pmu_visibility() currently checks kvm_vcpu_has_pmu(), which only tests whether the vCPU was initialized with KVM_ARM_VCPU_PMU_V3. If userspace initializes a vCPU with KVM_ARM_VCPU_PMU_V3 and subsequently hides FEAT_PMUv3 via ID_AA64DFR0_EL1.PMUVer, KVM continues to expose the PMU system registers instead of emulating guest accesses as UNDEF. Check ID_AA64DFR0_EL1.PMUVer directly via kvm_has_feat() in pmu_visibility() so that PMU system registers are marked REG_HIDDEN whenever FEAT_PMUv3 is not advertised in the ID registers. Suggested-by: Oliver Upton Assisted-by: LLM Signed-off-by: Colton Lewis Link: https://patch.msgid.link/20260930211834.2773546-3-coltonlewis@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 87f96c9bd2fe..8315a188676f 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1028,7 +1028,7 @@ static unsigned int hidden_visibility(const struct kvm_vcpu *vcpu, static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - if (kvm_vcpu_has_pmu(vcpu)) + if (kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) return 0; return REG_HIDDEN; From a32c01324eb494dda811912eb847b0e907cd7270 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Mon, 31 Aug 2026 10:10:02 +0200 Subject: [PATCH 430/506] KVM: arm64: vgic-v3: Roll back assignments from the new region A failed REDIST_REGION write unregisters the redistributor iodevs of every vCPU processed before the failure. This includes vCPUs assigned to older regions. Their assignments remain cached, so a retry does not register the iodevs again. The vCPU whose registration fails has no assignment to roll back. For vCPUs processed earlier, limit rollback to those assigned to the new region. The region is removed on failure, so its free_index does not need restoring. Resetting assignments from older regions is unsafe because a vCPU can own one before kvm_for_each_vcpu() can see it. Rollback could then move free_index behind that live assignment. kvm_vgic_addr() holds slots_lock across the REDIST_REGION write. vCPU creation takes the same lock before assigning a redistributor. A concurrent vCPU cannot acquire an assignment from the new region before rollback completes. The lock also keeps rdreg valid across the config_lock drop required by MMIO-bus registration. Fixes: c011f4ea106b ("KVM: arm/arm64: Check vcpu redist base before registering an iodev") Cc: stable@vger.kernel.org Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Karl Mehltretter Link: https://patch.msgid.link/20260831081005.41346-3-kmehltretter@gmail.com [maz: moved helper into the sole caller] Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 8fb0a3eb3457..5ab789accc9d 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -846,7 +846,8 @@ void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu) kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev); } -static int vgic_register_all_redist_iodevs(struct kvm *kvm) +static int vgic_register_all_redist_iodevs(struct kvm *kvm, + struct vgic_redist_region *rdreg) { struct kvm_vcpu *vcpu; unsigned long c; @@ -866,7 +867,14 @@ static int vgic_register_all_redist_iodevs(struct kvm *kvm) for (i = 0; i < c; i++) { vcpu = kvm_get_vcpu(kvm, i); + if (vcpu->arch.vgic_cpu.rdreg != rdreg) + continue; + vgic_unregister_redist_iodev(vcpu); + + guard(mutex)(&vcpu->kvm->arch.config_lock); + vcpu->arch.vgic_cpu.rdreg = NULL; + vcpu->arch.vgic_cpu.rd_iodev.base_addr = VGIC_ADDR_UNDEF; } } @@ -975,10 +983,13 @@ void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdre int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count) { + struct vgic_redist_region *rdreg; int ret; mutex_lock(&kvm->arch.config_lock); ret = vgic_v3_alloc_redist_region(kvm, index, addr, count); + if (!ret) + rdreg = vgic_v3_rdist_region_from_index(kvm, index); mutex_unlock(&kvm->arch.config_lock); if (ret) return ret; @@ -987,12 +998,9 @@ int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count) * Register iodevs for each existing VCPU. Adding more VCPUs * afterwards will register the iodevs when needed. */ - ret = vgic_register_all_redist_iodevs(kvm); + ret = vgic_register_all_redist_iodevs(kvm, rdreg); if (ret) { - struct vgic_redist_region *rdreg; - mutex_lock(&kvm->arch.config_lock); - rdreg = vgic_v3_rdist_region_from_index(kvm, index); vgic_v3_free_redist_region(kvm, rdreg); mutex_unlock(&kvm->arch.config_lock); return ret; From 8d0ac33045370988a58a40a8a4a7d8b5c763eb09 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Mon, 31 Aug 2026 10:10:03 +0200 Subject: [PATCH 431/506] KVM: arm64: selftests: Pass guest code to vm_gic_create_with_vcpus() Pass the guest-code address to vm_gic_create_with_vcpus(), and use guest_code at the existing call sites. This lets tests with different guest code reuse the common VM and VGIC setup while retaining the void * interface of vm_create_with_vcpus(). Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Karl Mehltretter Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260831081005.41346-4-kmehltretter@gmail.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_init.c | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/arm64/vgic_init.c b/tools/testing/selftests/kvm/arm64/vgic_init.c index 47e34b43afb2..84aa97204b7f 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_init.c +++ b/tools/testing/selftests/kvm/arm64/vgic_init.c @@ -73,6 +73,7 @@ static int run_vcpu(struct kvm_vcpu *vcpu) static struct vm_gic vm_gic_create_with_vcpus(u32 gic_dev_type, u32 nr_vcpus, + void *guest_code, struct kvm_vcpu *vcpus[]) { struct vm_gic v; @@ -338,7 +339,7 @@ static void test_vgic_then_vcpus(u32 gic_dev_type) struct vm_gic v; int ret, i; - v = vm_gic_create_with_vcpus(gic_dev_type, 1, vcpus); + v = vm_gic_create_with_vcpus(gic_dev_type, 1, guest_code, vcpus); subtest_dist_rdist(&v); @@ -359,7 +360,8 @@ static void test_vcpus_then_vgic(u32 gic_dev_type) struct vm_gic v; int ret; - v = vm_gic_create_with_vcpus(gic_dev_type, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(gic_dev_type, NR_VCPUS, guest_code, + vcpus); subtest_dist_rdist(&v); @@ -411,7 +413,8 @@ static void test_v3_new_redist_regions(void) u64 addr; int ret; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); @@ -422,7 +425,8 @@ static void test_v3_new_redist_regions(void) /* step2 */ - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); addr = REDIST_REGION_ATTR_ADDR(1, 0x280000, 0, 2); @@ -436,7 +440,8 @@ static void test_v3_new_redist_regions(void) /* step 3 */ - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, @@ -608,7 +613,8 @@ static void test_v3_redist_ipa_range_check_at_vcpu_run(void) int ret, i; u64 addr; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, 1, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, 1, guest_code, + vcpus); /* Set space for 3 redists, we have 1 vcpu, so this succeeds. */ addr = max_phys_size - (3 * 2 * 0x10000); @@ -641,7 +647,8 @@ static void test_v3_its_region(void) u64 addr; int its_fd, ret; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); its_fd = kvm_create_device(v.vm, KVM_DEV_TYPE_ARM_VGIC_ITS); addr = 0x401000; @@ -684,7 +691,8 @@ static void test_v3_nassgicap(void) u32 typer2; int ret; - vm = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + vm = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); kvm_device_attr_get(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS, GICD_TYPER2, &typer2); has_nassgicap = typer2 & GICD_TYPER2_nASSGIcap; From d3db911d8dfb2ef9b02fdce548df792be253fa41 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Mon, 31 Aug 2026 10:10:04 +0200 Subject: [PATCH 432/506] KVM: arm64: selftests: Test VGICv3 redistributor region retry A failed REDIST_REGION write can remove redistributor iodevs without clearing their cached vCPU assignments. A retry then skips those assignments and leaves the redistributors absent from the MMIO bus. Create two partial regions, reject a third region that overlaps the distributor, and retry at a valid address. After initializing the VGIC, have the guest read GICR_TYPER from all four redistributors and check their processor numbers. This verifies the final configuration without assuming that assignments survive the failed write. Keep the redistributor IPAs outside the guest RAM memslot and install only the stage-1 mappings needed for the MMIO accesses. If an iodev is missing, KVM_RUN therefore exits with KVM_EXIT_MMIO. Without the fix, the first redistributor access exits at 0x8030008. Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Karl Mehltretter Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260831081005.41346-5-kmehltretter@gmail.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_init.c | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/vgic_init.c b/tools/testing/selftests/kvm/arm64/vgic_init.c index 84aa97204b7f..5a30f3cb039b 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_init.c +++ b/tools/testing/selftests/kvm/arm64/vgic_init.c @@ -5,6 +5,7 @@ * Copyright (C) 2020, Red Hat, Inc. */ #include +#include #include #include #include @@ -13,12 +14,21 @@ #include "test_util.h" #include "kvm_util.h" +#include "gic.h" #include "processor.h" #include "vgic.h" #include "gic_v3.h" #define NR_VCPUS 4 +#define REDIST_RETRY_REGION0_BASE GICR_BASE_GPA +#define REDIST_RETRY_REGION1_BASE \ + (REDIST_RETRY_REGION0_BASE + 2 * KVM_VGIC_V3_REDIST_SIZE) +#define REDIST_RETRY_DIST_BASE \ + (REDIST_RETRY_REGION1_BASE + KVM_VGIC_V3_REDIST_SIZE) +#define REDIST_RETRY_REGION2_BASE \ + (REDIST_RETRY_DIST_BASE + KVM_VGIC_V3_DIST_SIZE) + #define REG_OFFSET(vcpu, offset) (((u64)vcpu << 32) | offset) #define VGIC_DEV_IS_V2(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V2) @@ -65,6 +75,23 @@ static void guest_code(void) GUEST_DONE(); } +static void guest_check_redist_retry(void) +{ + unsigned int i; + + /* The first three redistributors span adjacent regions 0 and 1. */ + for (i = 0; i < NR_VCPUS; i++) { + u64 base = i < 3 ? REDIST_RETRY_REGION0_BASE + + i * KVM_VGIC_V3_REDIST_SIZE : + REDIST_RETRY_REGION2_BASE; + u64 typer = readq((void *)(unsigned long)(base + GICR_TYPER)); + + GUEST_ASSERT_EQ(GICR_TYPER_CPU_NUMBER(typer), i); + } + + GUEST_DONE(); +} + /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) { @@ -462,6 +489,70 @@ static void test_v3_new_redist_regions(void) vm_gic_destroy(&v); } +static void test_v3_redist_region_retry(void) +{ + struct kvm_vcpu *vcpus[NR_VCPUS]; + struct vm_gic v; + struct ucall uc; + u64 addr; + int ret; + + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_check_redist_retry, vcpus); + + addr = REDIST_REGION_ATTR_ADDR(2, REDIST_RETRY_REGION0_BASE, 0, 0); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_REGION1_BASE, 0, 1); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + addr = REDIST_RETRY_DIST_BASE; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_DIST, &addr); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_DIST_BASE, 0, 2); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, + &addr); + TEST_ASSERT(ret && errno == EINVAL, + "register redist region colliding with dist"); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_REGION2_BASE, 0, 2); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + virt_map(v.vm, REDIST_RETRY_REGION0_BASE, REDIST_RETRY_REGION0_BASE, + vm_calc_num_guest_pages(v.vm->mode, + 3 * KVM_VGIC_V3_REDIST_SIZE)); + virt_map(v.vm, REDIST_RETRY_REGION2_BASE, REDIST_RETRY_REGION2_BASE, + vm_calc_num_guest_pages(v.vm->mode, + KVM_VGIC_V3_REDIST_SIZE)); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + vcpu_run(vcpus[0]); + switch (get_ucall(vcpus[0], &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_NONE: + if (vcpus[0]->run->exit_reason == KVM_EXIT_MMIO) + TEST_FAIL("Unexpected MMIO exit at 0x%llx", + vcpus[0]->run->mmio.phys_addr); + fallthrough; + default: + TEST_FAIL("Unexpected ucall %lu, exit_reason %u", + uc.cmd, vcpus[0]->run->exit_reason); + } + + vm_gic_destroy(&v); +} + static void test_v3_typer_accesses(void) { struct vm_gic v; @@ -986,6 +1077,7 @@ void run_tests(u32 gic_dev_type) if (VGIC_DEV_IS_V3(gic_dev_type)) { test_v3_new_redist_regions(); + test_v3_redist_region_retry(); test_v3_typer_accesses(); test_v3_last_bit_redist_regions(); test_v3_last_bit_single_rdist(); From 6b1abd43b27a5ecac0672482dc8ca1e48c2c8d15 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Mon, 5 Oct 2026 08:20:46 +0100 Subject: [PATCH 433/506] KVM: arm64: pkvm: Drop keneldoc markers for the allocator structs chunk_hdr and hyp_allocator are private. Their description is not exposed to the API documentation. Drop the kerneldoc markers. While at it add the missing description for chunk_hdr::data. Signed-off-by: Vincent Donnefort Closes: https://lore.kernel.org/oe-kbuild-all/202610041002.Yvw5qUrt-lkp@intel.com/ Link: https://patch.msgid.link/20261005072046.2246900-1-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/alloc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/alloc.c b/arch/arm64/kvm/hyp/nvhe/alloc.c index c1881c0011c9..e1e6dd21cda1 100644 --- a/arch/arm64/kvm/hyp/nvhe/alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/alloc.c @@ -28,13 +28,14 @@ */ #define MIN_ALLOC_SIZE 16UL -/** +/* * struct chunk_hdr - Chunk header * @next: offset from this chunk header to the next one. * @prev: offset from this chunk header to the previous one. * @__unmapped: Internal field containing the offset to the unmapped page * boundary, multiplexed with the allocation state flag. * @hash: Hash computed over the chunk header. + * @data: Chunk payload. */ struct chunk_hdr { u32 next; @@ -45,7 +46,7 @@ struct chunk_hdr { char data[]; } __aligned(MIN_ALLOC_SIZE); -/** +/* * struct hyp_allocator - Heap allocator * @start: Start in the allocator's reserved virtual address range. * @end: End in the allocator's reserved virtual address range. From 8233000034a630432a7da0311d94e8725207a095 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 15 Sep 2026 09:44:39 +0100 Subject: [PATCH 434/506] KVM: Fix comments that refer to the non-existent VCPU_RUN ioctl Neither the VCPU_RUN in kvm_main.c nor the VCPU RUN in kvm_host.h names an ioctl that exists. The one that enters the guest is KVM_RUN, so refer to it by that name. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260915084441.3846947-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- include/linux/kvm_host.h | 2 +- virt/kvm/kvm_main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..acae5a61e7f8 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -562,7 +562,7 @@ static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) /* * The memory barrier ensures a previous write to vcpu->requests cannot * be reordered with the read of vcpu->mode. It pairs with the general - * memory barrier following the write of vcpu->mode in VCPU RUN. + * memory barrier following the write of vcpu->mode in KVM_RUN. */ smp_mb__before_atomic(); return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..cc391b0666a6 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -470,7 +470,7 @@ static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu) kvm_dirty_ring_free(&vcpu->dirty_ring); /* - * No need for rcu_read_lock as VCPU_RUN is the only place that changes + * No need for rcu_read_lock as KVM_RUN is the only place that changes * the vcpu->pid pointer, and at destruction time all file descriptors * are already gone. */ From 0ab1034969130967bda8e6eab5b8dd6745604f1b Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 15 Sep 2026 09:44:40 +0100 Subject: [PATCH 435/506] KVM: arm64: Fix references to the non-existent VCPU_RUN ioctl There is no VCPU_RUN ioctl. The one that enters the guest is KVM_RUN, so refer to it by that name. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260915084441.3846947-3-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- Documentation/virt/kvm/arm/pkvm.rst | 2 +- arch/arm64/kvm/arm.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/virt/kvm/arm/pkvm.rst b/Documentation/virt/kvm/arm/pkvm.rst index 514992a79a83..18d8afe36ebc 100644 --- a/Documentation/virt/kvm/arm/pkvm.rst +++ b/Documentation/virt/kvm/arm/pkvm.rst @@ -67,7 +67,7 @@ largely due to the lack of MMU notifiers: then it will either return ``-EFAULT`` or forcefully reclaim the memory pages. Reclaimed memory is zeroed by the hypervisor and a subsequent attempt to access it in the pVM will return ``-EFAULT`` - from the ``VCPU_RUN`` ioctl(). + from the ``KVM_RUN`` ioctl(). CPU state isolation ------------------- diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..a18ec13bd667 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1263,7 +1263,7 @@ static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu) * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code * @vcpu: The VCPU pointer * - * This function is called through the VCPU_RUN ioctl called from user space. It + * This function is called through the KVM_RUN ioctl called from user space. It * will execute VM code in a loop until the time slice for the process is used * or some emulation is needed from user space in which case the function will * return with return value 0 and with the kvm_run structure filled in with the From 6ce8b5c0effb227f5625b35965c0f74c0e953efb Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 15 Sep 2026 09:44:41 +0100 Subject: [PATCH 436/506] KVM: PPC: Book3S HV: Fix comment naming a non-existent KVM_VCPU_RUN ioctl There is no KVM_VCPU_RUN ioctl. The one that enters the guest is KVM_RUN, so refer to it by that name. Signed-off-by: Fuad Tabba Reviewed-by: Gautam Menghani Reviewed-by: Amit Machhiwal Link: https://patch.msgid.link/20260915084441.3846947-4-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/powerpc/kvm/book3s_hv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index dbac3573b2c8..db090f4c5e95 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -269,7 +269,7 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) * stolen time for a vcore when it is inactive, or for a vcpu * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of * a misnomer; it means that the vcpu task is not executing in - * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in + * the KVM_RUN ioctl, i.e. it is in userspace or elsewhere in * the kernel. We don't have any way of dividing up that time * between time that the vcpu is genuinely stopped, time that * the task is actively working on behalf of the vcpu, and time From 49930b1b557ceb698e0ab7f985ea41c99dc8bf22 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 7 Oct 2026 08:43:41 +0100 Subject: [PATCH 437/506] KVM: arm64: timers: Compute an offset-applied CVAL from the current count When the hardware won't apply a timer's offset, KVM programs CVAL + offset and lets it compare that against the raw counter. The compare is unsigned (IsTimerConditionMet, DDI0487M_c J1.4.3.29): condition_met = (UInt(PhysicalCountInt() - offset) - UInt(compare_value)) >= 0; so adding the offset to both sides holds only while both sums wrap or neither does. A guest whose counter is ahead of the host's by more than CVAL has an expired timer that never fires, and one behind it can have a far-future timer fire at once. On a CNTPOFF_EL2 host the first case livelocks: each entry reloads the guest's CVAL, the timer fires, and the exit rewrites it as CVAL + offset before the host can take the interrupt. Without CNTPOFF_EL2 the timer hangs until the vCPU is next loaded, which is what Zenghui reported on a Kunpeng920. Program the value from the current count instead: CVAL + offset while the deadline is ahead, 0 once the timer has expired for the guest, ~0 past the counter's wrap. KVM recomputes it at load, at exit and on a trapped CVAL write, and an expired timer stays expired in between. The value can't be inverted, so with an offset timer_save_state() keeps the memory copy, and kvm_hyp_handle_timer() returns that copy for a guest hypervisor's read of its physical CVAL rather than subtracting the offset back out. Fixes: c605ee245097 ("KVM: arm64: timers: Allow physical offset without CNTPOFF_EL2") Fixes: 9404673293b0 ("KVM: arm64: timers: Correctly handle TGE flip with CNTPOFF_EL2") Fixes: 0bc9a9e85fcf ("KVM: arm64: Work around x1e's CNTVOFF_EL2 bogosity") Reported-by: Zenghui Yu Closes: https://lore.kernel.org/r/460258be-0102-e922-c342-4e87cd94b9e5@huawei.com Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261007074343.324269-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arch_timer.c | 27 ++++++++++++--------------- arch/arm64/kvm/hyp/vhe/switch.c | 11 ++++++----- include/kvm/arm_arch_timer.h | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 6ac3321f4c57..9278b5383c04 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -528,17 +528,11 @@ static void timer_save_state(struct arch_timer_context *ctx) goto out; switch (index) { - u64 cval; - case TIMER_VTIMER: case TIMER_HVTIMER: timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL)); - cval = read_sysreg_el0(SYS_CNTV_CVAL); - - if (has_broken_cntvoff()) - cval -= timer_get_offset(ctx); - - timer_set_cval(ctx, cval); + if (!has_broken_cntvoff() || !timer_get_offset(ctx)) + timer_set_cval(ctx, read_sysreg_el0(SYS_CNTV_CVAL)); /* Disable the timer */ write_sysreg_el0(0, SYS_CNTV_CTL); @@ -564,11 +558,12 @@ static void timer_save_state(struct arch_timer_context *ctx) case TIMER_PTIMER: case TIMER_HPTIMER: timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL)); - cval = read_sysreg_el0(SYS_CNTP_CVAL); - - cval -= timer_get_offset(ctx); - - timer_set_cval(ctx, cval); + /* + * With an offset, memory already holds the guest's CVAL (the + * trap handler or __deactivate_traps() wrote it). + */ + if (!timer_get_offset(ctx)) + timer_set_cval(ctx, read_sysreg_el0(SYS_CNTP_CVAL)); /* Disable the timer */ write_sysreg_el0(0, SYS_CNTP_CTL); @@ -647,7 +642,8 @@ static void timer_restore_state(struct arch_timer_context *ctx) offset = timer_get_offset(ctx); if (has_broken_cntvoff()) { set_cntvoff(0); - cval += offset; + if (offset) + cval = timer_apply_offset(cval, offset, kvm_phys_timer_read()); } else { set_cntvoff(offset); } @@ -660,7 +656,8 @@ static void timer_restore_state(struct arch_timer_context *ctx) cval = timer_get_cval(ctx); offset = timer_get_offset(ctx); set_cntpoff(offset); - cval += offset; + if (offset) + cval = timer_apply_offset(cval, offset, kvm_phys_timer_read()); write_sysreg_el0(cval, SYS_CNTP_CVAL); isb(); write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL); diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 7875911c0506..14aada311bac 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -175,7 +175,8 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) offset = read_sysreg_s(SYS_CNTPOFF_EL2); if (map.direct_ptimer && offset) { - write_sysreg_el0(val + offset, SYS_CNTP_CVAL); + val = timer_apply_offset(val, offset, arch_timer_read_cntpct_el0()); + write_sysreg_el0(val, SYS_CNTP_CVAL); isb(); } } @@ -296,10 +297,10 @@ static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code) break; case SYS_CNTP_CVAL_EL0: if (vcpu_el2_e2h_is_set(vcpu)) { - val = read_sysreg_el0(SYS_CNTP_CVAL); - - if (!has_cntpoff()) - val -= timer_get_offset(vcpu_hptimer(vcpu)); + if (!has_cntpoff() && timer_get_offset(vcpu_hptimer(vcpu))) + val = __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2); + else + val = read_sysreg_el0(SYS_CNTP_CVAL); } else { val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); } diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index bc6f2fdd7ad3..80f96ea59f1a 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -195,4 +195,19 @@ static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset) WRITE_ONCE(*ctxt->offset.vm_offset, offset); } +/* + * CVAL to program when the hardware won't apply the timer's offset, so that + * its compare against the raw counter matches the guest's at 'now'. An + * expired timer stays expired; one past the wrap never fires. + */ +static inline u64 timer_apply_offset(u64 cval, u64 offset, u64 now) +{ + u64 hw = cval + offset; + + if (now - offset >= cval) + return 0; + + return hw > now ? hw : U64_MAX; +} + #endif From 1dbb5a2b5a21a62a223a2a2a0d775d417dca4242 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 7 Oct 2026 08:43:42 +0100 Subject: [PATCH 438/506] KVM: arm64: nv: Read a guest hypervisor's CNTV_CVAL_EL0 from memory on x1e On x1e KVM keeps CNTVOFF_EL2 at zero, traps the virtual timer whenever it has an offset and programs the hardware with the offset applied. kvm_hyp_handle_timer() returns that hardware value for a VHE guest hypervisor's read of its own CNTV_CVAL_EL0, instead of the CVAL it wrote. Its writes trap to the kernel and land in CNTHV_CVAL_EL2, so return that, as the physical timer's read does without CNTPOFF_EL2. Fixes: 0bc9a9e85fcf ("KVM: arm64: Work around x1e's CNTVOFF_EL2 bogosity") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261007074343.324269-3-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/vhe/switch.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 14aada311bac..4c965f9b1e1d 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -322,10 +322,14 @@ static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code) val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); break; case SYS_CNTV_CVAL_EL0: - if (vcpu_el2_e2h_is_set(vcpu)) - val = read_sysreg_el0(SYS_CNTV_CVAL); - else + if (vcpu_el2_e2h_is_set(vcpu)) { + if (has_broken_cntvoff() && timer_get_offset(vcpu_hvtimer(vcpu))) + val = __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2); + else + val = read_sysreg_el0(SYS_CNTV_CVAL); + } else { val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); + } break; case SYS_CNTVCT_EL0: case SYS_CNTVCTSS_EL0: From 46e0f2c9d7d6f99a8f16290d4d4e2f3129edfa54 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 7 Oct 2026 08:43:43 +0100 Subject: [PATCH 439/506] KVM: arm64: selftests: Test a timer set past the counter's wrap Program an unmasked timer whose cval sits within long_wait_ms of the counter's wrap while the guest's counter is at wait_ms, and expect no interrupt. The guest's counter can't reach it, but a host that applies the timer's offset by adding it to cval wraps the sum into the past, and the timer fires at once. The one existing case with a cval past the wrap, in the sanity checks, runs with the timer masked, so it gets no interrupt either way. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261007074343.324269-4-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- .../kvm/arm64/arch_timer_edge_cases.c | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c index d9c9377a6325..d46799ff91dc 100644 --- a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c +++ b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c @@ -787,6 +787,30 @@ static void test_timers_in_the_past(enum arch_timer timer) } } +/* + * A cval within long_wait_ms of the counter's wrap while the counter sits at + * wait_ms: a deadline further away than the counter can reach. Unmasked, so + * an early ISTATUS shows up as an IRQ. + */ +static void test_timers_past_counter_wrap(enum arch_timer timer) +{ + u64 cval = -msec_to_cycles(test_args.long_wait_ms); + int i; + + for (i = 0; i < ARRAY_SIZE(sleep_method); i++) { + sleep_method_t sm = sleep_method[i]; + + local_irq_disable(); + set_counter(timer, msec_to_cycles(test_args.wait_ms)); + set_cval_irq(timer, cval, CTL_ENABLE); + sm(timer, msecs_to_usecs(test_args.wait_ms) + TIMEOUT_NO_IRQ_US); + local_irq_enable(); + isb(); + assert_irqs_handled(0); + timer_set_ctl(timer, CTL_IMASK); + } +} + static void test_long_timer_delays(enum arch_timer timer) { s32 tval = (s32)msec_to_cycles(test_args.long_wait_ms); @@ -806,6 +830,7 @@ static void guest_run_iteration(enum arch_timer timer) test_basic_functionality(timer); test_timers_sanity_checks(timer); + test_timers_past_counter_wrap(timer); test_timers_above_tval_max(timer); test_timers_in_the_past(timer); From 3bd107ed6b23de45b861e7f0c151d7fba59d721a Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 7 Oct 2026 09:08:33 +0100 Subject: [PATCH 440/506] KVM: arm64: Only emulate an SError's entry for vCPUs with NV kvm_inject_serror_esr() emulates an SError's exception entry when the host's copy of PSTATE says SErrors are unmasked, although only NV needs this, for a nested context whose guest hypervisor owns the vSError. A protected vCPU's PSTATE lives at EL2, and the VMM can unmask SErrors in the host's copy before the first run, so KVM emulates the entry on that copy and the guest never takes the SError. Emulate the entry only for a vCPU with NV, which protected VMs don't support, and otherwise pend the SError through HCR_EL2.VSE, as KVM did before commit ce66109cec867 ("KVM: arm64: nv: Take "masked" aborts to EL2 when HCRX_EL2.TMEA is set"). Fixes: 872383bd12e11 ("KVM: arm64: Add per-EC entry/exit state marshalling for protected guests") Reported-by: Sashiko Closes: https://lore.kernel.org/all/20261001142109.794CA1F000FF@smtp.kernel.org/ Suggested-by: Oliver Upton Signed-off-by: Fuad Tabba Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20261007080833.371899-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/inject_fault.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index d6c4fc16f879..96bf926ca520 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -371,15 +371,15 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) } /* - * Emulate the exception entry if SErrors are unmasked. This is useful if - * the vCPU is in a nested context w/ vSErrors enabled then we've already - * delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2, - * VDISR_EL2) to the guest hypervisor. + * With NV, emulate the exception entry if SErrors are unmasked: in a + * nested context with vSErrors enabled, the hardware vSError context + * (i.e. HCR_EL2.VSE, VSESR_EL2, VDISR_EL2) is delegated to the guest + * hypervisor. Otherwise, HCR_EL2.VSE delivers it once it is unmasked. * * As we're emulating the SError injection we need to explicitly populate * ESR_ELx.EC because hardware will not do it on our behalf. */ - if (!serror_is_masked(vcpu)) { + if (vcpu_has_nv(vcpu) && !serror_is_masked(vcpu)) { pend_serror_exception(vcpu); esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); From 49664cd0b1db434b6bbf9e37c84b3f448a545dc5 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 6 Oct 2026 10:28:41 +0100 Subject: [PATCH 441/506] KVM: arm64: selftests: Check the pending SError state in external_aborts None of the external abort tests look at the vCPU events once the guest is done, so an SError left pending after the guest took it, or dropped while the guest had it masked, goes unnoticed. Read the vCPU events back at the end of every test that runs the guest. The SError must still be pending in the masked case, with the injected ESR when the vCPU has RAS, and no SError, injected or spurious, may be pending in any of the others. Signed-off-by: Fuad Tabba Tested-by: Itaru Kitayama Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20261006092841.2202109-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- .../selftests/kvm/arm64/external_aborts.c | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/external_aborts.c b/tools/testing/selftests/kvm/arm64/external_aborts.c index 7836756a38a6..1a858d0d8296 100644 --- a/tools/testing/selftests/kvm/arm64/external_aborts.c +++ b/tools/testing/selftests/kvm/arm64/external_aborts.c @@ -75,6 +75,17 @@ static void vcpu_inject_serror(struct kvm_vcpu *vcpu) vcpu_events_set(vcpu, &events); } +static void vcpu_assert_serror_pending(struct kvm_vcpu *vcpu, bool pending) +{ + struct kvm_vcpu_events events; + + vcpu_events_get(vcpu, &events); + TEST_ASSERT_EQ(events.exception.serror_pending, pending); + + if (pending && vcpu_has_ras(vcpu)) + TEST_ASSERT_EQ(events.exception.serror_esr, EXPECTED_SERROR_ISS); +} + static void __vcpu_run_expect(struct kvm_vcpu *vcpu, unsigned int cmd) { struct ucall uc; @@ -134,6 +145,7 @@ static void test_mmio_abort(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -162,6 +174,7 @@ static void test_mmio_nisv(void) TEST_ASSERT(_vcpu_run(vcpu), "Expected nonzero return code from KVM_RUN"); TEST_ASSERT_EQ(errno, ENOSYS); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -185,6 +198,7 @@ static void test_mmio_nisv_abort(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -212,6 +226,7 @@ static void test_serror_masked(void) vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, true); kvm_vm_free(vm); } @@ -247,6 +262,7 @@ static void test_serror(void) vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -288,6 +304,7 @@ static void test_s1ptw_abort(void) *ptep |= bad_pa; vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -313,6 +330,7 @@ static void test_serror_emulated(void) vcpu_run_expect_sync(vcpu); vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -356,6 +374,7 @@ static void test_mmio_ease(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); done: kvm_vm_free(vm); } @@ -395,6 +414,7 @@ static void test_serror_amo(void) vcpu_run_expect_sync(vcpu); vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } From 3d736a339e9cbeec3841da1a821f56fb3230fd68 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:24 +0100 Subject: [PATCH 442/506] KVM: arm64: protected VM: Handle user writes to CNTVCT_EL0/CNTPCT_EL0 Protected VMs doesn't allow setting offsets for virtual and physical counters, as the offset is always fixed to 0. The VM ioctl is filtered out based on the cap. However we don't prevent the userspace from trying to write to the CNTVCT/CNTPCT registers. This would lead to KVM triggering a WARN() in timer_set_offset() as the vm_offset pointer is set to NULL. Fix this by always "fixing" the timer offsets to 0 and marking that the timer offset is set in the kvm->arch.flags at KVM init time for protected VMs. This prevents the access to the VM specific vm_offset at low cost. A userspace writing to the CNT*CT_EL0 would observe success, without any real effect. This is cleaner over spilling "*_is_protected()" checks and "matches" what we really do in practise. i.e., always run with "fixed counter offset of 0". Reported by Sashiko Link: https://lore.kernel.org/all/20260908164641.416911F00A3A@smtp.kernel.org Fixes: f7d05ee84a6a ("KVM: arm64: Prevent host from managing timer offsets for protected VMs") Suggested-by: Marc Zyngier Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-2-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arch_timer.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 6ac3321f4c57..a45845f4ae4e 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -1110,8 +1110,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) timer_context_init(vcpu, i); /* Synchronize offsets across timers of a VM if not already provided */ - if (!vcpu_is_protected(vcpu) && - !test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { + if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read()); timer_set_offset(vcpu_ptimer(vcpu), 0); } @@ -1133,6 +1132,15 @@ void kvm_timer_init_vm(struct kvm *kvm) */ for (int i = 0; i < NR_KVM_TIMERS; i++) kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]); + + /* + * Protected VMs don't allow the userspace to set counter offsets, + * either set via counter register writes or the dedicated ioctls. + * Pretend the offset has already been set and rely on the default + * offset being 0. + */ + if (kvm_vm_is_protected(kvm)) + set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags); } void kvm_timer_cpu_up(void) From 22adb9c3543cc702516c18f0a28e3a8dfcdbeecc Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:26 +0100 Subject: [PATCH 443/506] KVM: arm64: Include kvm_emulate.h in kvm/arm_psci.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a potential build error (like below, when asm/kvm_emulate.h gets included after the kvm/arm_psci.h) by including the missing header file in kvm/arm_psci.h: ./include/kvm/arm_psci.h: In function ‘kvm_psci_version’: ./include/kvm/arm_psci.h:29:13: error: implicit declaration of function ‘vcpu_has_feature’; did you mean ‘cpu_have_feature’? [-Werror=implicit-function-declaration] 29 | if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2)) { | ^~~~~~~~~~~~~~~~ | cpu_have_feature Reviewed-by: Gavin Shan Reviewed-by: Fuad Tabba Reviewed-by: Jonathan Cameron Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-4-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- include/kvm/arm_psci.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h index f86a006d6713..06c20612e9e7 100644 --- a/include/kvm/arm_psci.h +++ b/include/kvm/arm_psci.h @@ -10,6 +10,8 @@ #include #include +#include + #define KVM_ARM_PSCI_0_1 PSCI_VERSION(0, 1) #define KVM_ARM_PSCI_0_2 PSCI_VERSION(0, 2) #define KVM_ARM_PSCI_1_0 PSCI_VERSION(1, 0) From 0c4063f89bfdf204112651f45620ef0c5a691e6b Mon Sep 17 00:00:00 2001 From: Steven Price Date: Wed, 7 Oct 2026 08:35:27 +0100 Subject: [PATCH 444/506] KVM: arm64: Avoid including linux/kvm_host.h in kvm_pgtable.h To avoid future include cycles, drop the linux/kvm_host.h include in kvm_pgtable.h and include the lightweight headers required for the types and inline helpers used there. Additionally provide a forward declaration for struct kvm_s2_mmu as it's only used as a pointer in this file. Both pgtable.c and kvm_pkvm.h relied on the indirect inclusion of kvm_host.h, so make that explicit. Reviewed-by: Fuad Tabba Reviewed-by: Gavin Shan Signed-off-by: Steven Price Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-5-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pgtable.h | 10 +++++++++- arch/arm64/include/asm/kvm_pkvm.h | 2 +- arch/arm64/kvm/hyp/pgtable.c | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 41a8687938eb..a211a5d87bf9 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -7,10 +7,18 @@ #ifndef __ARM64_KVM_PGTABLE_H__ #define __ARM64_KVM_PGTABLE_H__ +#include #include -#include +#include +#include +#include #include +#include +#include + +struct kvm_s2_mmu; + #define KVM_PGTABLE_FIRST_LEVEL -1 #define KVM_PGTABLE_LAST_LEVEL 3 diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index beea00e693a0..54a618d887fa 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -7,9 +7,9 @@ #define __ARM64_KVM_PKVM_H__ #include +#include #include #include -#include #include /* Maximum number of VMs that can co-exist under pKVM. */ diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index b74dd5ce1efd..f48253b9d88b 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -8,6 +8,7 @@ */ #include +#include #include #include From 762ad274c03a13686192d75ef9a354b96114b503 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:28 +0100 Subject: [PATCH 445/506] KVM: arm64: Track the type of VM in kvm_arch KVM arm64 has different types of VMs with all the different modes in which the hypervisor code can be run. e.g., VHE, nVHE, pKVM etc. Then there is protected VM and normal VMs with pKVM. We might soon add other types, e.g., Arm CCA Realm. So in an effort to make the handling of these different types of VMs a bit more friendly to the eyes, add a VM flavor to the kvm_arch and we could then add handlers for different operations based on the VM type. Keep the flavor initialisation at the beginning to allow for the detection early enough and fail out on any unsupported requests. With that, add wrappers for checking the "type" of a VM and replace the existing users with the new wrappers. Given we already have the construct of "kvm_vm_is_protected" in the core KVM code, use that for all confidential compute guests including Realms that we are about to add. Adds __VM_PROTECTED marker vm flavor to draw the boundary for "protected VMs". In later patches, we would add Realm VMs, which would also be classified as protected. While adding the vm_flavor, move the psci_version around to keep the structure packed. Suggested-by: Marc Zyngier Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-6-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 38 +++++++++++++++++++++++++++---- arch/arm64/include/asm/kvm_pkvm.h | 4 ++-- arch/arm64/kvm/arm.c | 33 ++++++++++++++++++++++----- arch/arm64/kvm/hyp/nvhe/pkvm.c | 6 ++++- arch/arm64/kvm/mmio.c | 1 + arch/arm64/kvm/pkvm.c | 6 ++--- 6 files changed, 70 insertions(+), 18 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..9ad6b5c215fd 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -257,7 +257,6 @@ struct kvm_protected_vm { pkvm_handle_t handle; struct kvm_hyp_memcache teardown_mc; struct kvm_hyp_memcache stage2_teardown_mc; - bool is_protected; bool is_created; /* @@ -306,9 +305,22 @@ enum fgt_group_id { __NR_FGT_GROUP_IDS__ }; +enum kvm_arm_vm_flavor { + VM_NVHE, + VM_VHE, + VM_PKVM, /* Normal guests on pKVM */ + MARKER(__VM_PROTECTED), + VM_PROTECTED_PKVM, /* Protected VM */ + VM_FLAVOR_MAX +}; + struct kvm_arch { struct kvm_s2_mmu mmu; + enum kvm_arm_vm_flavor vm_flavor; + /* Mandated version of PSCI */ + u32 psci_version; + /* * Fine-Grained UNDEF, mimicking the FGT layout defined by the * architecture. We track them globally, as we present the @@ -332,9 +344,6 @@ struct kvm_arch { /* Timers */ struct arch_timer_vm_data timer_data; - /* Mandated version of PSCI */ - u32 psci_version; - /* Protects VM-scoped configuration data */ struct mutex config_lock; @@ -1504,10 +1513,29 @@ struct kvm *kvm_arch_alloc_vm(void); #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE -#define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.is_protected) +#ifdef __KVM_NVHE_HYPERVISOR__ +#define kvm_vm_is_protected(kvm) \ + (is_protected_kvm_enabled() && ((kvm)->arch.vm_flavor == VM_PROTECTED_PKVM)) +/* + * Accessing vcpu->kvm from nVHE hyp stub is tricky, as we need to convert the + * pointer to the hyp VA. With pKVM, the nVHE code runs with the hyp_vcpu, + * which is populated correctly and is gated on is_protected_kvm_enabled(). + */ +#define vcpu_is_protected(vcpu) \ + ({ \ + struct kvm *__kvm = READ_ONCE((vcpu)->kvm); \ + \ + (__kvm && kvm_vm_is_protected(__kvm)); \ + }) + +#else + +#define kvm_vm_is_protected(kvm) ((kvm)->arch.vm_flavor >= __VM_PROTECTED) #define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm) +#endif /* __KVM_NVHE_HYPERVISOR__ */ + int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 54a618d887fa..2addc37c500e 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -17,7 +17,7 @@ #define HYP_MEMBLOCK_REGIONS 128 -int pkvm_init_host_vm(struct kvm *kvm, unsigned long type); +int pkvm_init_host_vm(struct kvm *kvm); int pkvm_create_hyp_vm(struct kvm *kvm); bool pkvm_hyp_vm_is_created(struct kvm *kvm); void pkvm_destroy_hyp_vm(struct kvm *kvm); @@ -49,7 +49,7 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: return false; default: - return !kvm || !kvm_vm_is_protected(kvm); + return !kvm || (kvm->arch.vm_flavor == VM_PKVM); } } diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..b6032a7a1fb8 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -214,6 +214,26 @@ static int kvm_arm_default_max_vcpus(void) return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; } +static int kvm_init_vm_flavor(struct kvm *kvm, unsigned long type) +{ + bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; + + if (is_protected_kvm_enabled()) { + if (protected) + kvm->arch.vm_flavor = VM_PROTECTED_PKVM; + else + kvm->arch.vm_flavor = VM_PKVM; + } else if (protected) { + return -EINVAL; + } else if (has_vhe()) { + kvm->arch.vm_flavor = VM_VHE; + } else { + kvm->arch.vm_flavor = VM_NVHE; + } + + return 0; +} + /** * kvm_arch_init_vm - initializes a VM data structure * @kvm: pointer to the KVM struct @@ -236,6 +256,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); #endif + ret = kvm_init_vm_flavor(kvm, type); + if (ret) + return ret; + kvm_init_nested(kvm); ret = kvm_share_hyp(kvm, kvm + 1); @@ -257,12 +281,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) * If any failures occur after this is successful, make sure to * call __pkvm_unreserve_vm to unreserve the VM in hyp. */ - ret = pkvm_init_host_vm(kvm, type); + ret = pkvm_init_host_vm(kvm); if (ret) goto err_uninit_mmu; - } else if (type & KVM_VM_TYPE_ARM_PROTECTED) { - ret = -EINVAL; - goto err_uninit_mmu; } kvm_vgic_early_init(kvm); @@ -751,7 +772,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_call_hyp_nvhe(__pkvm_vcpu_put); /* __pkvm_vcpu_put implies a sync of the state */ - if (!kvm_vm_is_protected(vcpu->kvm)) + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); } @@ -985,7 +1006,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) if (is_protected_kvm_enabled()) { /* Start with the vcpu in a dirty state */ - if (!kvm_vm_is_protected(vcpu->kvm)) + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); ret = pkvm_create_hyp_vm(kvm); if (ret) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 459bd9eb7e4b..ed51762aa4b5 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -432,7 +432,11 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, hyp_vm->host_kvm = host_kvm; hyp_vm->kvm.created_vcpus = nr_vcpus; - hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected); + if (READ_ONCE(host_kvm->arch.vm_flavor) == VM_PROTECTED_PKVM) + hyp_vm->kvm.arch.vm_flavor = VM_PROTECTED_PKVM; + else + hyp_vm->kvm.arch.vm_flavor = VM_PKVM; + hyp_vm->kvm.arch.flags = 0; pkvm_init_features_from_host(hyp_vm, host_kvm); diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index d1c3a352d5a2..ab1d2fef9a52 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -6,6 +6,7 @@ #include #include +#include #include #include "trace.h" diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4c6e4bec12..8e9176a70092 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -229,10 +229,9 @@ void pkvm_destroy_hyp_vm(struct kvm *kvm) mutex_unlock(&kvm->arch.config_lock); } -int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) +int pkvm_init_host_vm(struct kvm *kvm) { int ret; - bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; /* Reserve the VM in hyp and obtain a hyp handle for the VM. */ ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm); @@ -240,8 +239,7 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) return ret; kvm->arch.pkvm.handle = ret; - kvm->arch.pkvm.is_protected = protected; - if (protected) { + if (kvm_vm_is_protected(kvm)) { pr_warn_once("kvm: protected VMs are experimental and for development only, tainting kernel\n"); add_taint(TAINT_USER, LOCKDEP_STILL_OK); } From 42a4026b435e49af4d5eaae58afe7103ca740c30 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:29 +0100 Subject: [PATCH 446/506] KVM: arm64: Don't call vcpu_set_pauth_traps for pKVM host vcpu_set_pauth_traps() bails out and does nothing for pKVM hosts. Clean this up by moving the is_protected_kvm_enabled() check to the caller, in preparation for adding VM specific vcpu load/put callbacks. While at it, do an early return if the vcpu doesn't have ptrauth. No functional changes Reviewed-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-7-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 52 +++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index b6032a7a1fb8..7b09b59e2819 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -631,33 +631,34 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) { - if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) { - /* - * Either we're running an L2 guest, and the API/APK bits come - * from L1's HCR_EL2, or API/APK are both set. - */ - if (unlikely(is_nested_ctxt(vcpu))) { - u64 val; + if (!vcpu_has_ptrauth(vcpu)) + return; - val = __vcpu_sys_reg(vcpu, HCR_EL2); - val &= (HCR_API | HCR_APK); - vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); - vcpu->arch.hcr_el2 |= val; - } else { - vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); - } + /* + * Either we're running an L2 guest, and the API/APK bits come + * from L1's HCR_EL2, or API/APK are both set. + */ + if (unlikely(is_nested_ctxt(vcpu))) { + u64 val; - /* - * Save the host keys if there is any chance for the guest - * to use pauth, as the entry code will reload the guest - * keys in that case. - */ - if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { - struct kvm_cpu_context *ctxt; + val = __vcpu_sys_reg(vcpu, HCR_EL2); + val &= (HCR_API | HCR_APK); + vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); + vcpu->arch.hcr_el2 |= val; + } else { + vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); + } - ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); - ptrauth_save_keys(ctxt); - } + /* + * Save the host keys if there is any chance for the guest + * to use pauth, as the entry code will reload the guest + * keys in that case. + */ + if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { + struct kvm_cpu_context *ctxt; + + ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); + ptrauth_save_keys(ctxt); } } @@ -749,7 +750,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) else vcpu->arch.hcr_el2 |= HCR_TWI; - vcpu_set_pauth_traps(vcpu); + if (!is_protected_kvm_enabled()) + vcpu_set_pauth_traps(vcpu); if (is_protected_kvm_enabled()) { kvm_call_hyp_nvhe(__pkvm_vcpu_load, From 8252f74f9188b9148ec93438e53b8e1b865ffb7b Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:30 +0100 Subject: [PATCH 447/506] KVM: arm64: Refactor the vcpu_load to allow for VM specific callbacks To keep the VCPU load/put handling cleaner with the different kinds of VM types, we are about to introduce VM specific callbacks to do just the right thing. In preparation for that, make some refactoring to add the change easier by mainly feature specific configurations to individual wrappers, so that different callbacks could reuse the helpers. Adds vcpu_prepare_mmu()/vcpu_put_mmu() wrappers to load/put MMU related configurations for !pKVM guests. Additionally makes it explicit that kvm_arm_vmid_clear_active() is not required for pKVM host. Add vcpu_load_pvtime(), vcpu_set_wfx_traps() wrappers for handling the corresponding configurations. While at it, also make it clear that the timer loading constraints only apply for the VHE. No functional changes intended. Based on a work by Marc Zyngier. Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-8-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 65 +++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 7b09b59e2819..4f3f0d5423f9 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -684,14 +684,11 @@ static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu) return single_task_running(); } -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +static void vcpu_prepare_mmu(struct kvm_vcpu *vcpu) { struct kvm_s2_mmu *mmu; int *last_ran; - if (is_protected_kvm_enabled()) - goto nommu; - if (vcpu_has_nv(vcpu)) kvm_vcpu_load_hw_mmu(vcpu); @@ -721,25 +718,15 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_call_hyp(__kvm_flush_cpu_context, mmu); *last_ran = vcpu->vcpu_idx; } +} -nommu: - vcpu->cpu = cpu; - - /* - * The timer must be loaded before the vgic to correctly set up physical - * interrupt deactivation in nested state (e.g. timer interrupt). - */ - kvm_timer_vcpu_load(vcpu); - kvm_vgic_load(vcpu); - kvm_vcpu_load_debug(vcpu); - kvm_vcpu_load_fgt(vcpu); - if (has_vhe()) - kvm_vcpu_load_vhe(vcpu); - kvm_arch_vcpu_load_fp(vcpu); - kvm_vcpu_pmu_restore_guest(vcpu); - if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) - kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +static void vcpu_put_mmu(struct kvm_vcpu *vcpu) +{ + kvm_arm_vmid_clear_active(); +} +static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) +{ if (kvm_vcpu_should_clear_twe(vcpu)) vcpu->arch.hcr_el2 &= ~HCR_TWE; else @@ -749,6 +736,38 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.hcr_el2 &= ~HCR_TWI; else vcpu->arch.hcr_el2 |= HCR_TWI; +} + +static void vcpu_load_pvtime(struct kvm_vcpu *vcpu) +{ + if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +} + +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + vcpu->cpu = cpu; + + if (!is_protected_kvm_enabled()) + vcpu_prepare_mmu(vcpu); + /* + * For VHE, the timer must be loaded before the vgic to correctly + * set up physical interrupt deactivation in nested state (e.g. timer + * interrupt). + */ + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + + if (has_vhe()) + kvm_vcpu_load_vhe(vcpu); + + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); if (!is_protected_kvm_enabled()) vcpu_set_pauth_traps(vcpu); @@ -787,7 +806,9 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_vcpu_pmu_restore_host(vcpu); if (vcpu_has_nv(vcpu)) kvm_vcpu_put_hw_mmu(vcpu); - kvm_arm_vmid_clear_active(); + + if (!is_protected_kvm_enabled()) + vcpu_put_mmu(vcpu); vcpu_clear_on_unsupported_cpu(vcpu); vcpu->cpu = -1; From 0cb16544ccaceba4c5e8beb2e8ae896278dcf018 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:31 +0100 Subject: [PATCH 448/506] KVM: arm64: Add vcpu load/put call backs for flavors Add VM flavor specific handlers for VCPU load/put, in an effort to make it easier to follow the code. pauth traps were removed from VMs running PKVM as it is a no-op for them. Based on a patch by Marc Zyngier Suggested-by: Marc Zyngier Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-9-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 6 ++ arch/arm64/kvm/arm.c | 138 +++++++++++++++++++++++------- 2 files changed, 114 insertions(+), 30 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 9ad6b5c215fd..4c64d957ee06 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -150,6 +150,11 @@ struct kvm_vmid { atomic64_t id; }; +struct kvm_vcpu_ops { + void (*vcpu_load)(struct kvm_vcpu *vcpu); + void (*vcpu_put)(struct kvm_vcpu *vcpu); +}; + struct kvm_s2_mmu { struct kvm_vmid vmid; @@ -855,6 +860,7 @@ struct vncr_tlb; struct kvm_vcpu_arch { struct kvm_cpu_context ctxt; + const struct kvm_vcpu_ops *vcpu_ops; /* * Guest floating point state diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 4f3f0d5423f9..e99af4faa525 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -93,6 +93,7 @@ static const struct kvm_ioctl_cap_map vm_ioctl_caps[] = { { KVM_ARM_PREFERRED_TARGET, KVM_CAP_ARM_BASIC }, }; +static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu); /* * Set *ext to the capability. * Return 0 if found, or -EINVAL if no IOCTL matches. @@ -569,6 +570,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) mutex_unlock(&vcpu->mutex); #endif + kvm_init_vcpu_ops(vcpu); + /* Force users to call KVM_ARM_VCPU_INIT */ vcpu_clear_flag(vcpu, VCPU_INITIALIZED); @@ -744,12 +747,9 @@ static void vcpu_load_pvtime(struct kvm_vcpu *vcpu) kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); } -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +static void vhe_vcpu_load(struct kvm_vcpu *vcpu) { - vcpu->cpu = cpu; - - if (!is_protected_kvm_enabled()) - vcpu_prepare_mmu(vcpu); + vcpu_prepare_mmu(vcpu); /* * For VHE, the timer must be loaded before the vgic to correctly * set up physical interrupt deactivation in nested state (e.g. timer @@ -759,26 +759,53 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_vgic_load(vcpu); kvm_vcpu_load_debug(vcpu); kvm_vcpu_load_fgt(vcpu); + kvm_vcpu_load_vhe(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); - if (has_vhe()) - kvm_vcpu_load_vhe(vcpu); + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); + vcpu_set_pauth_traps(vcpu); +} +static void nvhe_vcpu_load(struct kvm_vcpu *vcpu) +{ + vcpu_prepare_mmu(vcpu); + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); + vcpu_set_pauth_traps(vcpu); +} + +static void pkvm_vcpu_load(struct kvm_vcpu *vcpu) +{ + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); kvm_arch_vcpu_load_fp(vcpu); kvm_vcpu_pmu_restore_guest(vcpu); vcpu_load_pvtime(vcpu); vcpu_set_wfx_traps(vcpu); - if (!is_protected_kvm_enabled()) - vcpu_set_pauth_traps(vcpu); + kvm_call_hyp_nvhe(__pkvm_vcpu_load, + vcpu->kvm->arch.pkvm.handle, + vcpu->vcpu_idx, vcpu->arch.hcr_el2); + kvm_call_hyp_nvhe(__vgic_v3_restore_vmcr_aprs, + &vcpu->arch.vgic_cpu.vgic_v3); +} - if (is_protected_kvm_enabled()) { - kvm_call_hyp_nvhe(__pkvm_vcpu_load, - vcpu->kvm->arch.pkvm.handle, - vcpu->vcpu_idx, vcpu->arch.hcr_el2); - kvm_call_hyp(__vgic_v3_restore_vmcr_aprs, - &vcpu->arch.vgic_cpu.vgic_v3); - } +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + vcpu->cpu = cpu; + vcpu->arch.vcpu_ops->vcpu_load(vcpu); if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus)) vcpu_set_on_unsupported_cpu(vcpu); @@ -786,30 +813,50 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.pid = pid_nr(vcpu->pid); } -void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +static void vhe_vcpu_put(struct kvm_vcpu *vcpu) { - if (is_protected_kvm_enabled()) { - kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); - kvm_call_hyp_nvhe(__pkvm_vcpu_put); - - /* __pkvm_vcpu_put implies a sync of the state */ - if (vcpu->kvm->arch.vm_flavor == VM_PKVM) - vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); - } - kvm_vcpu_put_debug(vcpu); kvm_arch_vcpu_put_fp(vcpu); - if (has_vhe()) - kvm_vcpu_put_vhe(vcpu); + kvm_vcpu_put_vhe(vcpu); kvm_timer_vcpu_put(vcpu); kvm_vgic_put(vcpu); kvm_vcpu_pmu_restore_host(vcpu); + if (vcpu_has_nv(vcpu)) kvm_vcpu_put_hw_mmu(vcpu); - if (!is_protected_kvm_enabled()) - vcpu_put_mmu(vcpu); + vcpu_put_mmu(vcpu); +} +static void nvhe_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_vcpu_put_debug(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + kvm_vcpu_pmu_restore_host(vcpu); + vcpu_put_mmu(vcpu); +} + +static void pkvm_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_call_hyp_nvhe(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); + kvm_call_hyp_nvhe(__pkvm_vcpu_put); + + /* __pkvm_vcpu_put implies a sync of the state */ + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); + + kvm_vcpu_put_debug(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + kvm_vcpu_pmu_restore_host(vcpu); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + vcpu->arch.vcpu_ops->vcpu_put(vcpu); vcpu_clear_on_unsupported_cpu(vcpu); vcpu->cpu = -1; } @@ -2149,6 +2196,37 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) } } +static const struct kvm_vcpu_ops vhe_vcpu_ops = { + .vcpu_load = vhe_vcpu_load, + .vcpu_put = vhe_vcpu_put, +}; + +static const struct kvm_vcpu_ops nvhe_vcpu_ops = { + .vcpu_load = nvhe_vcpu_load, + .vcpu_put = nvhe_vcpu_put, +}; + +static const struct kvm_vcpu_ops pkvm_vcpu_ops = { + .vcpu_load = pkvm_vcpu_load, + .vcpu_put = pkvm_vcpu_put, +}; + +#define KVM_VCPU_OPS(flavor, ops) \ + [(flavor)] = &(ops) + +static const struct kvm_vcpu_ops *arm64_vcpu_ops[] = { + KVM_VCPU_OPS(VM_NVHE, nvhe_vcpu_ops), + KVM_VCPU_OPS(VM_VHE, vhe_vcpu_ops), + KVM_VCPU_OPS(VM_PKVM, pkvm_vcpu_ops), + KVM_VCPU_OPS(VM_PROTECTED_PKVM, pkvm_vcpu_ops), +}; + +static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu) +{ + BUILD_BUG_ON(ARRAY_SIZE(arm64_vcpu_ops) != VM_FLAVOR_MAX); + vcpu->arch.vcpu_ops = arm64_vcpu_ops[vcpu->kvm->arch.vm_flavor]; +} + static unsigned long nvhe_percpu_size(void) { return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) - From fb3926af9d0d5e2e2f58a1f4b2252f529fbe23bf Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:32 +0100 Subject: [PATCH 449/506] KVM: arm64: Consolidate stage2 unmap range into kvm_stage2_unmap_range In preparation for adding VM specific backends for stage2 operations, nuke __unmap_stage2_range() and fold the logic into kvm_stage2_unmap_range(). Also, make kvm_unmap_gfn_range(), the only other user of the __unmap_stage2_range() call the kvm_stage2_unmap_range(). Also, while at it fix the comment to make it clear that mmu_lock must always be held while unmapping. The code already mandates that and the two call paths do have the write mmu_lock held. Later we would replace the logic in kvm_stage2_unmap_range() with VM specific backends. No functional changes intended. Reviewed-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-10-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9ba86450fe4a..3fef16393f56 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -314,36 +314,30 @@ static void invalidate_icache_guest_page(void *va, size_t size) * does. */ /** - * __unmap_stage2_range -- Clear stage2 page table entries to unmap a range + * kvm_stage2_unmap_range -- Clear stage2 page table entries to unmap a range * @mmu: The KVM stage-2 MMU pointer * @start: The intermediate physical base address of the range to unmap * @size: The size of the area to unmap * @may_block: Whether or not we are permitted to block * * Clear a range of stage-2 mappings, lowering the various ref-counts. Must - * be called while holding mmu_lock (unless for freeing the stage2 pgd before - * destroying the VM), otherwise another faulting VCPU may come in and mess - * with things behind our backs. + * be called while holding mmu_lock otherwise another faulting VCPU may + * come in and mess with things behind our backs. */ -static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, - bool may_block) -{ - struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - phys_addr_t end = start + size; - - lockdep_assert_held_write(&kvm->mmu_lock); - WARN_ON(size & ~PAGE_MASK); - WARN_ON(stage2_apply_range(mmu, start, end, KVM_PGT_FN(kvm_pgtable_stage2_unmap), - may_block)); -} - void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, bool may_block) { - if (kvm_vm_is_protected(kvm_s2_mmu_to_kvm(mmu))) + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + if (kvm_vm_is_protected(kvm)) return; - __unmap_stage2_range(mmu, start, size, may_block); + lockdep_assert_held_write(&kvm->mmu_lock); + WARN_ON(size & ~PAGE_MASK); + + WARN_ON(stage2_apply_range(mmu, start, start + size, + KVM_PGT_FN(kvm_pgtable_stage2_unmap), + may_block)); } void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end) @@ -2436,12 +2430,12 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) + if (!kvm->arch.mmu.pgt) return false; - __unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT, - (range->end - range->start) << PAGE_SHIFT, - range->may_block); + kvm_stage2_unmap_range(&kvm->arch.mmu, range->start << PAGE_SHIFT, + (range->end - range->start) << PAGE_SHIFT, + range->may_block); kvm_nested_s2_unmap(kvm, range->may_block); return false; From 68933203edddc092b30dd1039167b3262a40a326 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:33 +0100 Subject: [PATCH 450/506] KVM: arm64: Add VM specific callback for S2 MMU operations Add VM type specific S2 MMU operation backends which can be initialized per VM flavor, to keep the handling cleaner. Reviewed-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-11-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 14 +++ arch/arm64/kvm/mmu.c | 173 +++++++++++++++++++++++++----- 2 files changed, 160 insertions(+), 27 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 4c64d957ee06..7130ba5bef8a 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -155,6 +155,19 @@ struct kvm_vcpu_ops { void (*vcpu_put)(struct kvm_vcpu *vcpu); }; +struct kvm_gfn_range; + +struct kvm_vm_s2_ops { + bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + bool (*vm_test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + int (*vm_flush_remote_tlbs)(struct kvm *kvm); + int (*vm_flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, + u64 nr_pages); + void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu, + phys_addr_t start, u64 size, + bool may_block); +}; + struct kvm_s2_mmu { struct kvm_vmid vmid; @@ -321,6 +334,7 @@ enum kvm_arm_vm_flavor { struct kvm_arch { struct kvm_s2_mmu mmu; + const struct kvm_vm_s2_ops *vm_s2_ops; enum kvm_arm_vm_flavor vm_flavor; /* Mandated version of PSCI */ diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 3fef16393f56..65592feaa6a9 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -37,6 +37,8 @@ static unsigned long __ro_after_init io_map_base; #define KVM_PGT_FN(fn) (!is_protected_kvm_enabled() ? fn : p ## fn) +static int kvm_vm_init_vm_s2_ops(struct kvm *kvm); + static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end, phys_addr_t size) { @@ -166,6 +168,18 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); } +static int pkvm_flush_remote_tlbs(struct kvm *kvm) +{ + kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); + return 0; +} + +static int kvm_vm_flush_remote_tlbs(struct kvm *kvm) +{ + kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); + return 0; +} + /** * kvm_arch_flush_remote_tlbs() - flush all VM TLB entries for v7/8 * @kvm: pointer to kvm structure. @@ -174,26 +188,31 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) */ int kvm_arch_flush_remote_tlbs(struct kvm *kvm) { - if (is_protected_kvm_enabled()) - kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); - else - kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); - return 0; + return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs(kvm); } -int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, - gfn_t gfn, u64 nr_pages) +static int pkvm_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) +{ + return pkvm_flush_remote_tlbs(kvm); +} + +static int kvm_vm_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) { u64 size = nr_pages << PAGE_SHIFT; u64 addr = gfn << PAGE_SHIFT; - if (is_protected_kvm_enabled()) - kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); - else - kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size); + kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size); return 0; } +int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) +{ + return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs_range(kvm, gfn, nr_pages); +} + static void *stage2_memcache_zalloc_page(void *arg) { struct kvm_mmu_memory_cache *mc = arg; @@ -289,6 +308,27 @@ static void invalidate_icache_guest_page(void *va, size_t size) __invalidate_icache_guest_page(va, size); } +static void kvm_vm_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, + u64 size, bool may_block) +{ + WARN_ON(stage2_apply_range(mmu, start, start + size, + kvm_pgtable_stage2_unmap, may_block)); +} + +static void pkvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, + u64 size, bool may_block) +{ + WARN_ON(stage2_apply_range(mmu, start, start + size, + pkvm_pgtable_stage2_unmap, may_block)); +} + +static void no_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, u64 size, bool may_block) +{ +} + /* * Unmapping vs dcache management: * @@ -329,15 +369,10 @@ void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start, { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - if (kvm_vm_is_protected(kvm)) - return; - lockdep_assert_held_write(&kvm->mmu_lock); WARN_ON(size & ~PAGE_MASK); - WARN_ON(stage2_apply_range(mmu, start, start + size, - KVM_PGT_FN(kvm_pgtable_stage2_unmap), - may_block)); + kvm->arch.vm_s2_ops->vm_stage2_unmap_range(mmu, start, size, may_block); } void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end) @@ -977,6 +1012,12 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t int cpu, err; struct kvm_pgtable *pgt; + /* Initialize the VM ops for the VM instance for the first time */ + if (mmu == &kvm->arch.mmu) { + err = kvm_vm_init_vm_s2_ops(kvm); + if (err) + return err; + } /* * If we already have our page tables in place, and that the * MMU context is the canonical one, we have a bug somewhere, @@ -2441,34 +2482,68 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) return false; } -bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +static bool kvm_vm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { u64 size = (range->end - range->start) << PAGE_SHIFT; - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) - return false; - - return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt, + return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT, size, true); +} + +static bool pkvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, true); +} + +static bool no_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + /* The hypervisor doesn't support aging */ + return false; +} + +bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + if (!kvm->arch.mmu.pgt) + return false; + + return kvm->arch.vm_s2_ops->vm_age_gfn(kvm, range); /* * TODO: Handle nested_mmu structures here using the reverse mapping in * a later version of patch series. */ } -bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +static bool kvm_vm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { u64 size = (range->end - range->start) << PAGE_SHIFT; - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) - return false; - - return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt, + return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT, size, false); } +static bool pkvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, false); +} + +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + if (!kvm->arch.mmu.pgt) + return false; + + return kvm->arch.vm_s2_ops->vm_test_age_gfn(kvm, range); +} + phys_addr_t kvm_mmu_get_httbr(void) { return __pa(hyp_pgtable->pgd); @@ -2790,3 +2865,47 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); } + +static const struct kvm_vm_s2_ops protected_pkvm_vm_s2_ops = { + .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range, + .vm_age_gfn = no_age_gfn, + .vm_test_age_gfn = no_age_gfn, + .vm_stage2_unmap_range = no_stage2_unmap_range, +}; + +static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = { + .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range, + .vm_age_gfn = pkvm_age_gfn, + .vm_test_age_gfn = pkvm_test_age_gfn, + .vm_stage2_unmap_range = pkvm_stage2_unmap_range, +}; + +static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = { + .vm_flush_remote_tlbs = kvm_vm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = kvm_vm_flush_remote_tlbs_range, + .vm_age_gfn = kvm_vm_age_gfn, + .vm_test_age_gfn = kvm_vm_test_age_gfn, + .vm_stage2_unmap_range = kvm_vm_stage2_unmap_range, +}; + +#define KVM_VM_S2_OPS(flavor, ops) \ + [(flavor)] = &(ops) + +static const struct kvm_vm_s2_ops *arm64_vm_s2_ops[] = { + KVM_VM_S2_OPS(VM_VHE, kvm_default_vm_s2_ops), + KVM_VM_S2_OPS(VM_NVHE, kvm_default_vm_s2_ops), + KVM_VM_S2_OPS(VM_PKVM, pkvm_vm_s2_ops), + KVM_VM_S2_OPS(VM_PROTECTED_PKVM, protected_pkvm_vm_s2_ops), +}; + +static int kvm_vm_init_vm_s2_ops(struct kvm *kvm) +{ + BUILD_BUG_ON(ARRAY_SIZE(arm64_vm_s2_ops) != VM_FLAVOR_MAX); + + kvm->arch.vm_s2_ops = arm64_vm_s2_ops[kvm->arch.vm_flavor]; + if (WARN_ON(!kvm->arch.vm_s2_ops)) + return -EINVAL; + return 0; +} From 4ee1b583589781bfd47b28da3ce216b2fb51dac5 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:34 +0100 Subject: [PATCH 451/506] KVM: arm64: Use a local kvm pointer in kvm_handle_guest_abort() kvm_handle_guest_abort() repeatedly obtains the VM from vcpu->kvm. Introduce a local kvm pointer and use it throughout the function. While touching the code, fix the missing whitespace in the kvm_is_nested_s2_mmu() call. Suggested-by: Jonathan Cameron Reviewed-by: Fuad Tabba Reviewed-by: Gavin Shan Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-12-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 65592feaa6a9..46d23483db4b 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2284,6 +2284,7 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu) */ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) { + struct kvm *kvm = vcpu->kvm; struct kvm_s2_trans nested_trans, *nested = NULL; unsigned long esr; phys_addr_t fault_ipa; /* The address we faulted on */ @@ -2304,7 +2305,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * with an SEA. */ ipa = fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); - if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm)) + if (KVM_BUG_ON(ipa == INVALID_GPA, kvm)) return -EFAULT; is_iabt = kvm_vcpu_trap_is_iabt(vcpu); @@ -2339,7 +2340,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) return -EFAULT; } - idx = srcu_read_lock(&vcpu->kvm->srcu); + idx = srcu_read_lock(&kvm->srcu); /* * We may have faulted on a shadow stage 2 page table if we are @@ -2354,7 +2355,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * nothing to walk and we treat it as a 1:1 before going through the * canonical translation. */ - if (kvm_is_nested_s2_mmu(vcpu->kvm,vcpu->arch.hw_mmu) && + if (kvm_is_nested_s2_mmu(kvm, vcpu->arch.hw_mmu) && vcpu->arch.hw_mmu->nested_stage2_enabled) { u32 esr; @@ -2382,7 +2383,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } gfn = ipa >> PAGE_SHIFT; - memslot = gfn_to_memslot(vcpu->kvm, gfn); + memslot = gfn_to_memslot(kvm, gfn); hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); write_fault = kvm_is_write_fault(vcpu); if (kvm_is_error_hva(hva) || (write_fault && !writable)) { @@ -2446,7 +2447,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) .hva = hva, }; - if (kvm_vm_is_protected(vcpu->kvm)) { + if (kvm_vm_is_protected(kvm)) { ret = pkvm_mem_abort(&s2fd); } else { VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && @@ -2465,7 +2466,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) if (ret == -ENOEXEC) ret = kvm_inject_sea_iabt(vcpu, kvm_vcpu_get_hfar(vcpu)); out_unlock: - srcu_read_unlock(&vcpu->kvm->srcu, idx); + srcu_read_unlock(&kvm->srcu, idx); return ret; } From b44a8c32f0116aba0a934c3068628bacda3cd3a3 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:35 +0100 Subject: [PATCH 452/506] KVM: arm64: Abstract out memory abort handling Move the memory abort handling under VM specific s2 operation. Tested-by: Gavin Shan Reviewed-by: Gavin Shan Reviewed-by: Fuad Tabba Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-13-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/kvm/mmu.c | 33 ++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 7130ba5bef8a..c08b40a6d4cb 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -156,6 +156,7 @@ struct kvm_vcpu_ops { }; struct kvm_gfn_range; +struct kvm_s2_fault_desc; struct kvm_vm_s2_ops { bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); @@ -166,6 +167,7 @@ struct kvm_vm_s2_ops { void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, bool may_block); + int (*vm_mem_abort)(const struct kvm_s2_fault_desc *s2fd); }; struct kvm_s2_mmu { diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 46d23483db4b..9821cac0dce4 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1740,7 +1740,7 @@ struct kvm_s2_fault_vma_info { bool map_non_cacheable; }; -static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) +static int protected_pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) { unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; struct kvm_vcpu *vcpu = s2fd->vcpu; @@ -2178,6 +2178,20 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache); } +static int kvm_vm_mem_abort(const struct kvm_s2_fault_desc *s2fd) +{ + struct kvm_vcpu *vcpu = s2fd->vcpu; + + VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && + !kvm_is_write_fault(vcpu) && + !kvm_vcpu_trap_is_exec_fault(vcpu)); + + if (kvm_slot_has_gmem(s2fd->memslot)) + return gmem_abort(s2fd); + else + return user_mem_abort(s2fd); +} + /* Resolve the access fault by making the page young again. */ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) { @@ -2447,19 +2461,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) .hva = hva, }; - if (kvm_vm_is_protected(kvm)) { - ret = pkvm_mem_abort(&s2fd); - } else { - VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && - !write_fault && - !kvm_vcpu_trap_is_exec_fault(vcpu)); - - if (kvm_slot_has_gmem(memslot)) - ret = gmem_abort(&s2fd); - else - ret = user_mem_abort(&s2fd); - } - + ret = kvm->arch.vm_s2_ops->vm_mem_abort(&s2fd); if (ret == 0) ret = 1; out: @@ -2873,6 +2875,7 @@ static const struct kvm_vm_s2_ops protected_pkvm_vm_s2_ops = { .vm_age_gfn = no_age_gfn, .vm_test_age_gfn = no_age_gfn, .vm_stage2_unmap_range = no_stage2_unmap_range, + .vm_mem_abort = protected_pkvm_mem_abort, }; static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = { @@ -2881,6 +2884,7 @@ static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = { .vm_age_gfn = pkvm_age_gfn, .vm_test_age_gfn = pkvm_test_age_gfn, .vm_stage2_unmap_range = pkvm_stage2_unmap_range, + .vm_mem_abort = kvm_vm_mem_abort, }; static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = { @@ -2889,6 +2893,7 @@ static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = { .vm_age_gfn = kvm_vm_age_gfn, .vm_test_age_gfn = kvm_vm_test_age_gfn, .vm_stage2_unmap_range = kvm_vm_stage2_unmap_range, + .vm_mem_abort = kvm_vm_mem_abort, }; #define KVM_VM_S2_OPS(flavor, ops) \ From 37d7ea3f81bbe61f1e895b6c87decadbfe8799eb Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:36 +0100 Subject: [PATCH 453/506] KVM: arm64: Mandate VGIC v3 for pKVM VMs and Realms pKVM does not trust the host. Realm VMs follow a similar trust model, with the Realm Management Monitor owning the protected state instead of the host. Add a helper to identify VMs that run under a host-distrusting hypervisor. Use this for blocking ioremap of vgic-v2 into stage2 and prevent creation of VGIC other than v3. Reviewed-by: Jonathan Cameron Reviewed-by: Fuad Tabba Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-14-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 4 ++++ arch/arm64/kvm/mmu.c | 2 +- arch/arm64/kvm/vgic/vgic-init.c | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index c08b40a6d4cb..54d56bef3535 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -328,6 +328,8 @@ enum fgt_group_id { enum kvm_arm_vm_flavor { VM_NVHE, VM_VHE, + /* VMs running on a hyp that doesn't trust */ + MARKER(__VM_DISTRUSTING_HYP), VM_PKVM, /* Normal guests on pKVM */ MARKER(__VM_PROTECTED), VM_PROTECTED_PKVM, /* Protected VM */ @@ -1556,6 +1558,8 @@ struct kvm *kvm_arch_alloc_vm(void); #define kvm_vm_is_protected(kvm) ((kvm)->arch.vm_flavor >= __VM_PROTECTED) #define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm) +#define kvm_vm_hyp_is_distrusting(kvm) ((kvm)->arch.vm_flavor >= __VM_DISTRUSTING_HYP) + #endif /* __KVM_NVHE_HYPERVISOR__ */ int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9821cac0dce4..b0dacfa7e625 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1249,7 +1249,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, KVM_PGTABLE_PROT_R | (writable ? KVM_PGTABLE_PROT_W : 0); - if (is_protected_kvm_enabled()) + if (kvm_vm_hyp_is_distrusting(kvm)) return -EPERM; size += offset_in_page(guest_ipa); diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 4012df6002ea..874025513afc 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -84,6 +84,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) !kvm_vgic_global_state.can_emulate_gicv2) return -ENODEV; + if (kvm_vm_hyp_is_distrusting(kvm) && type != KVM_DEV_TYPE_ARM_VGIC_V3) + return -ENODEV; /* * Ensure mutual exclusion with vCPU creation and any vCPU ioctls by: * From a467f39afc2a149f9f5efbb7238fb531ade8a86e Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:37 +0100 Subject: [PATCH 454/506] KVM: arm64: Prevent unsupported vcpu features for VM types Prevent unsupported VCPU features for the protected VCPUs. Realms and pVMs not support 32bit EL1 or NV yet. pKVM doesn't rely on the host vcpu features for protected VMs and hand picks features while hyp_vcpu is initialised. Block the features early in the vcpu init if we detect incompatible features. Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-15-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 7 +++++++ arch/arm64/kvm/arm.c | 11 ++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 54d56bef3535..4175aa9dba47 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -42,6 +42,13 @@ #define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) +/* As dictated by kvm_pkvm_ext_allowed() */ +#define KVM_PROTECTED_VCPU_VALID_FEATURES \ + (BIT(KVM_ARM_VCPU_POWER_OFF) | \ + BIT(KVM_ARM_VCPU_PSCI_0_2) | \ + BIT(KVM_ARM_VCPU_PTRAUTH_ADDRESS) | \ + BIT(KVM_ARM_VCPU_PTRAUTH_GENERIC)) + #define KVM_REQ_SLEEP \ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e99af4faa525..90ce620a6ea4 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1668,9 +1668,14 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return -EINVAL; } -static unsigned long system_supported_vcpu_features(void) +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) { - unsigned long features = KVM_VCPU_VALID_FEATURES; + unsigned long features; + + if (vcpu->kvm->arch.vm_flavor == VM_PROTECTED_PKVM) + features = KVM_PROTECTED_VCPU_VALID_FEATURES; + else + features = KVM_VCPU_VALID_FEATURES; if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); @@ -1708,7 +1713,7 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, return -ENOENT; } - if (features & ~system_supported_vcpu_features()) + if (features & ~system_supported_vcpu_features(vcpu)) return -EINVAL; /* From 6164ce3491bcc3a27a2e2e9260a286eb2df2301e Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 8 Oct 2026 11:42:11 +0100 Subject: [PATCH 455/506] KVM: arm64: Prevent 64-bit overflow when computing hyp clock rate The hypervisor clock rate is delta_cycles * NSEC_PER_SEC / delta_boot. It is computed whenever a deviation occurs. If the delta between two rate updates is big enough, delta_cycles * NSEC_PER_SEC will overflow. Prevent overflow with mul_u64_u64_div_u64(). And as a bonus, simplify the clock estimation with mul_u64_u32_shr(). Fixes: b22888917fa4 ("KVM: arm64: Sync boot clock with the nVHE/pKVM hyp") Cc: stable@vger.kernel.org Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261008104212.3134550-2-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp_trace.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 5bc881d6f343..cd265f7c3873 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -60,14 +60,8 @@ static void __hyp_clock_work(struct work_struct *work) if (hyp_clock->mult) { u64 err, cur = delta_cycles; - if (WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64)) { - __uint128_t tmp = (__uint128_t)cur * hyp_clock->mult; - - cur = tmp >> hyp_clock->shift; - } else { - cur *= hyp_clock->mult; - cur >>= hyp_clock->shift; - } + WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64); + cur = mul_u64_u32_shr(cur, hyp_clock->mult, hyp_clock->shift); cur += hyp_clock->boot; err = abs_diff(cur, boot); @@ -85,7 +79,7 @@ static void __hyp_clock_work(struct work_struct *work) err / NSEC_PER_USEC); } - rate = div64_u64(delta_cycles * NSEC_PER_SEC, delta_boot); + rate = mul_u64_u64_div_u64(delta_cycles, NSEC_PER_SEC, delta_boot); clocks_calc_mult_shift(&hyp_clock->mult, &hyp_clock->shift, rate, NSEC_PER_SEC, CLOCK_MAX_CONVERSION_S); From aac4560e1287c36196fd2d78bf04d0c1c2559e3e Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 8 Oct 2026 11:42:12 +0100 Subject: [PATCH 456/506] KVM: arm64: Fix bpages backing allocation size in hyp_trace hyp_trace_buffer_alloc_bpages_backing() computes nr_bpages as (PAGE_ALIGN(size) / PAGE_SIZE) + 1, whereas trace_remote_alloc_buffer() takes into account the buffer page header. Align the bpages backing allocation by using __calc_nr_pages_ring_buffer_desc(). There's no need to validate nr_bpages again, it has been before with trace_buffer_desc_size(). Fixes: 3aed038aac8d ("KVM: arm64: Add trace remote for the nVHE/pKVM hyp") Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261008104212.3134550-3-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp_trace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index cd265f7c3873..86e52cac880a 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -146,8 +146,8 @@ static void __unshare_page(unsigned long va) static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size) { + unsigned long nr_bpages; size_t backing_size; - int nr_bpages; void *start; int ret; @@ -157,7 +157,7 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ return 0; } - nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; + nr_bpages = __calc_nr_pages_ring_buffer_desc(size); backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * num_possible_cpus()); From a6270ed4bec0caa25913082e71ce22254e9cd3ed Mon Sep 17 00:00:00 2001 From: Lisa Wang Date: Thu, 1 Oct 2026 19:37:40 +0000 Subject: [PATCH 457/506] KVM: selftests: Support guest_memfd in-place conversion With guest_memfd in-place conversion, the memory region can be set as shared, eliminating the need for dual-backed memory. By setting the region to shared initially, the KVM selftest framework can populate the mmap()-ed guest_memfd memory just like any standard memory provider. For CoCo VMs, pages that need to be private are explicitly set to private before executing the VM. Signed-off-by: Lisa Wang Link: https://patch.msgid.link/20261001-tdx-selftests-v15-13-7c62a5d8a992@google.com [sean: anchor comment to its relevant place, drop unnecessary code changes] Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/kvm_util.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 3f559f207408..0c67d964a9aa 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -494,6 +494,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, u64 nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus, nr_extra_pages); struct userspace_mem_region *slot0; + u64 gmem_flags = 0; struct kvm_vm *vm; int i, flags; @@ -509,10 +510,20 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, * for CoCo VMs that require GUEST_MEMFD backed private memory. */ flags = 0; - if (is_guest_memfd_required(shape)) + if (is_guest_memfd_required(shape)) { flags |= KVM_MEM_GUEST_MEMFD; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags); + /* + * Allow mmap() and initialize the gmem instance as SHARED if + * in-place conversions are supported, so that the memory can + * be populated/initialized like any other memory region. + */ + if (kvm_has_gmem_attributes) + gmem_flags |= GUEST_MEMFD_FLAG_INIT_SHARED | + GUEST_MEMFD_FLAG_MMAP; + } + + vm_mem_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags, -1, 0, gmem_flags); for (i = 0; i < NR_MEM_REGIONS; i++) vm->memslots[i] = 0; From 0d22ab95051473affa742c8a41e626b3f4141a16 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:16 +0200 Subject: [PATCH 458/506] KVM: Make device name configurable Allow KVM implementations to choose alternative device names. This is especially useful for architectures providing multiple KVM implementations simultaneously. Architectures providing multiple KVM implementations must compile the KVM common code once per implementation and mangle symbols. Suggested-by: Sean Christopherson Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-2-seiden@linux.ibm.com Signed-off-by: Paolo Bonzini --- arch/s390/kvm/gmap/trace_gmap.h | 4 +++- arch/s390/kvm/s390/Makefile | 2 +- include/linux/kvm_host.h | 4 ++++ include/trace/events/kvm.h | 8 +++++++- virt/kvm/Makefile.kvm | 5 +++++ virt/kvm/async_pf.c | 5 ++++- virt/kvm/eventfd.c | 3 ++- virt/kvm/kvm_main.c | 32 +++++++++++++++++--------------- 8 files changed, 43 insertions(+), 20 deletions(-) diff --git a/arch/s390/kvm/gmap/trace_gmap.h b/arch/s390/kvm/gmap/trace_gmap.h index 15ec88bdce55..78d26dae59f9 100644 --- a/arch/s390/kvm/gmap/trace_gmap.h +++ b/arch/s390/kvm/gmap/trace_gmap.h @@ -4,8 +4,10 @@ #include +#undef UNPACK_KVM +#define UNPACK_KVM(name) name #undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm +#define TRACE_SYSTEM UNPACK_KVM(KVM_DEV_NAME) #undef TRACE_INCLUDE_PATH #define TRACE_INCLUDE_PATH ../gmap #undef TRACE_INCLUDE_FILE diff --git a/arch/s390/kvm/s390/Makefile b/arch/s390/kvm/s390/Makefile index 762a63826423..b91334db90a2 100644 --- a/arch/s390/kvm/s390/Makefile +++ b/arch/s390/kvm/s390/Makefile @@ -4,7 +4,7 @@ KVM := ../../../../virt/kvm include $(srctree)/virt/kvm/Makefile.kvm include $(srctree)/arch/s390/kvm/gmap/Makefile -ccflags-y := -I$(src) -I$(srctree)/arch/s390/kvm/gmap +ccflags-y += -I$(src) -I$(srctree)/arch/s390/kvm/gmap kvm-y += s390.o intercept.o interrupt.o priv.o sigp.o kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index d8fa784e0b55..5624b572cb85 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -45,6 +45,10 @@ #include #include +#define __KVM_DEV_STRING(name, x) __stringify(name ## _ ## x) +#define _KVM_DEV_STRING(name, x) __KVM_DEV_STRING(name, x) +#define KVM_DEV_STRING(x) _KVM_DEV_STRING(KVM_DEV_NAME, x) + #ifndef KVM_MAX_VCPU_IDS #define KVM_MAX_VCPU_IDS KVM_MAX_VCPUS #endif diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h index b282e3a86769..f37582730ffc 100644 --- a/include/trace/events/kvm.h +++ b/include/trace/events/kvm.h @@ -4,8 +4,14 @@ #include +#undef UNPACK_KVM +#define UNPACK_KVM(name) name #undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm +#define TRACE_SYSTEM UNPACK_KVM(KVM_DEV_NAME) +#undef TRACE_SYSTEM_VAR +#define TRACE_SYSTEM_VAR kvm +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE kvm #define ERSN(x) { KVM_EXIT_##x, "KVM_EXIT_" #x } diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm index d047d4cf58c9..b39f7b2a5f6e 100644 --- a/virt/kvm/Makefile.kvm +++ b/virt/kvm/Makefile.kvm @@ -5,6 +5,11 @@ KVM ?= ../../../virt/kvm +KVM_DEV_NAME ?= kvm +KVM_DEV_MINOR ?= KVM_MINOR + +ccflags-y +=-DKVM_DEV_NAME=$(KVM_DEV_NAME) -DKVM_DEV_MINOR=$(KVM_DEV_MINOR) + kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o kvm-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o kvm-$(CONFIG_KVM_MMIO) += $(KVM)/coalesced_mmio.o diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index b8aaa96b799b..6b3b6353cb9d 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -21,7 +21,10 @@ static struct kmem_cache *async_pf_cache; int kvm_async_pf_init(void) { - async_pf_cache = KMEM_CACHE(kvm_async_pf, 0); + async_pf_cache = kmem_cache_create(KVM_DEV_STRING(async_pf), + sizeof(struct kvm_async_pf), + __alignof__(struct kvm_async_pf), + 0, NULL); if (!async_pf_cache) return -ENOMEM; diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 93ad2ebc963f..f43b8b25bc85 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -711,7 +711,8 @@ bool kvm_notify_irqfd_resampler(struct kvm *kvm, */ int kvm_irqfd_init(void) { - irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", WQ_PERCPU, 0); + irqfd_cleanup_wq = alloc_workqueue(__stringify(KVM_DEV_NAME) "-irqfd-cleanup", + WQ_PERCPU, 0); if (!irqfd_cleanup_wq) return -ENOMEM; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index dc3bca26375d..09ee08092ced 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -10,6 +10,8 @@ * Yaniv Kamay */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include #include @@ -1023,7 +1025,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname) mutex_lock(&kvm_debugfs_lock); dent = debugfs_lookup(dir_name, kvm_debugfs_dir); if (dent) { - pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name); + pr_warn_ratelimited("debugfs: duplicate directory %s\n", dir_name); dput(dent); mutex_unlock(&kvm_debugfs_lock); return 0; @@ -4125,9 +4127,9 @@ static struct file_operations kvm_vcpu_fops = { */ static int create_vcpu_fd(struct kvm_vcpu *vcpu) { - char name[8 + 1 + ITOA_MAX_LEN + 1]; + char name[sizeof(__stringify(KVM_DEV_NAME) "-vcpu:") + ITOA_MAX_LEN + 1]; - snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id); + snprintf(name, sizeof(name), __stringify(KVM_DEV_NAME) "-vcpu:%d", vcpu->vcpu_id); return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); } @@ -4340,9 +4342,9 @@ static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) { int fd; struct file *file; - char name[15 + ITOA_MAX_LEN + 1]; + char name[sizeof(__stringify(KVM_DEV_NAME) "-vcpu-stats:") + ITOA_MAX_LEN + 1]; - snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id); + snprintf(name, sizeof(name), __stringify(KVM_DEV_NAME) "-vcpu-stats:%d", vcpu->vcpu_id); fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) @@ -5155,7 +5157,7 @@ static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm) if (fd < 0) return fd; - file = anon_inode_getfile_fmode("kvm-vm-stats", + file = anon_inode_getfile_fmode(__stringify(KVM_DEV_NAME) "-vm-stats", &kvm_vm_stats_fops, kvm, O_RDONLY, FMODE_PREAD); if (IS_ERR(file)) { put_unused_fd(fd); @@ -5532,7 +5534,7 @@ static int kvm_dev_ioctl_create_vm(unsigned long type) goto put_fd; } - file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); + file = anon_inode_getfile(__stringify(KVM_DEV_NAME) "-vm", &kvm_vm_fops, kvm, O_RDWR); if (IS_ERR(file)) { r = PTR_ERR(file); goto put_kvm; @@ -5600,8 +5602,8 @@ static struct file_operations kvm_chardev_ops = { }; static struct miscdevice kvm_dev = { - KVM_MINOR, - "kvm", + KVM_DEV_MINOR, + __stringify(KVM_DEV_NAME), &kvm_chardev_ops, }; @@ -5635,7 +5637,7 @@ static int kvm_enable_virtualization_cpu(void) return 0; if (kvm_arch_enable_virtualization_cpu()) { - pr_info("kvm: enabling virtualization on CPU%d failed\n", + pr_info("enabling virtualization on CPU%d failed\n", raw_smp_processor_id()); return -EIO; } @@ -5682,7 +5684,7 @@ static void kvm_shutdown(void *data) * is used for both shutdown and reboot scenarios, i.e. neither name is * 100% comprehensive. */ - pr_info("kvm: exiting hardware virtualization\n"); + pr_info("exiting hardware virtualization\n"); on_each_cpu(kvm_disable_virtualization_cpu, NULL, 1); } @@ -6093,7 +6095,7 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, * attached devices. Otherwise, destroy the caller's device only. */ if (!new_bus) { - pr_err("kvm: failed to shrink bus, removing it completely\n"); + pr_err("failed to shrink bus, removing it completely\n"); kvm_io_bus_destroy(bus); return -ENOMEM; } @@ -6380,7 +6382,7 @@ static void kvm_init_debug(void) const struct kvm_stats_desc *pdesc; int i; - kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); + kvm_debugfs_dir = debugfs_create_dir(__stringify(KVM_DEV_NAME), NULL); for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) { pdesc = &kvm_vm_stats_desc[i]; @@ -6525,7 +6527,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) if (!vcpu_align) vcpu_align = __alignof__(struct kvm_vcpu); kvm_vcpu_cache = - kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align, + kmem_cache_create_usercopy(KVM_DEV_STRING(vcpu), vcpu_size, vcpu_align, SLAB_ACCOUNT, offsetof(struct kvm_vcpu, arch), offsetofend(struct kvm_vcpu, stats_id) @@ -6578,7 +6580,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) */ r = misc_register(&kvm_dev); if (r) { - pr_err("kvm: misc device register failed\n"); + pr_err("misc device register failed\n"); goto err_register; } From 297ce48f3d1d8ecbc57abe780508af781819011d Mon Sep 17 00:00:00 2001 From: Chaithanya Lagisetty Date: Tue, 13 Oct 2026 16:48:42 +0800 Subject: [PATCH 459/506] LoongArch: KVM: Return the actual error code in kvm_ipi_create() The kvm_ipi_create() function registers the IPI device on the IOCSR bus. The error path currently discards the actual error code returned by kvm_io_bus_register_dev() and returns -EFAULT instead: ret = kvm_io_bus_register_dev(kvm, KVM_IOCSR_BUS, IOCSR_IPI_BASE, IOCSR_IPI_SIZE, device); mutex_unlock(&kvm->slots_lock); if (ret < 0) { kvm_pr_unimpl("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret); goto err; } ... err: kfree(s); return -EFAULT; kvm_io_bus_register_dev() fails with -ENOMEM or -ENOSPC, so userspace creating a KVM_DEV_TYPE_LOONGARCH_IPI device is told that it passed a bad address when the real problem is that the host is out of memory or out of IOCSR bus slots. -EFAULT is reserved for faulting user addresses, and the remaining -EFAULT returns in this file are get_user() failures, where it is correct. Free the IPI state and propagate the error directly after cleanup. The error label has a single user, so handling the failure in place removes the label as well. This matches kvm_eiointc_create() which already frees its state and propagates the actual kvm_io_bus_register_dev() error code unchanged. Only compile-tested. Fixes: c532de5a67a7 ("LoongArch: KVM: Add IPI device support") Assisted-by: Cursor:claude-opus-5 Reviewed-by: Bibo Mao Suggested-by: Huacai Chen Signed-off-by: Chaithanya Lagisetty Signed-off-by: Huacai Chen --- arch/loongarch/kvm/intc/ipi.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/loongarch/kvm/intc/ipi.c b/arch/loongarch/kvm/intc/ipi.c index 7b333a4a0430..80faa770c0cd 100644 --- a/arch/loongarch/kvm/intc/ipi.c +++ b/arch/loongarch/kvm/intc/ipi.c @@ -423,15 +423,12 @@ static int kvm_ipi_create(struct kvm_device *dev, u32 type) mutex_unlock(&kvm->slots_lock); if (ret < 0) { kvm_pr_unimpl("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret); - goto err; + kfree(s); + return ret; } kvm->arch.ipi = s; return 0; - -err: - kfree(s); - return -EFAULT; } static void kvm_ipi_destroy(struct kvm_device *dev) From 5b5d40bff4206c15c7f9251cfb27acac557771f7 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 13 Oct 2026 16:48:42 +0800 Subject: [PATCH 460/506] LoongArch: KVM: Add vmid support for stage2 MMU LoongArch KVM hypervisor supports two-level MMU, vpid index (GSTAT.GID in HW) is used for stage1 MMU and vmid index (GTLBC.TGID in HW) is used for stage2 MMU. On Loongson-3A5000, vmid must be the same with vpid. On Loongson-3A6000 platform, vmid may separate from vpid. If a vCPU migrates to different physical CPUs, vpid need change however vmid can keep unchanged, so that TLB entries for stage2 MMU is valid still. Also vmid index of the whole VM machine is the same on one physical CPU, all vCPUs on the same VM can share the same vmid index on one physical CPU. Similar to ASID management, there are different ID management to support multiple VMs and vCPUs, the corresponding ID management functions are __kvm_check_vpid() and __kvm_check_vmid(). Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_host.h | 4 ++-- arch/loongarch/kernel/asm-offsets.c | 1 + arch/loongarch/kvm/main.c | 20 ++++++++++++++++++-- arch/loongarch/kvm/switch.S | 5 ++--- arch/loongarch/kvm/tlb.c | 5 ++++- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 65d91c3ce313..5616050b7416 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -233,8 +233,8 @@ struct kvm_vcpu_arch { /* Cache for pages needed inside spinlock regions */ struct kvm_mmu_memory_cache mmu_page_cache; - /* vcpu's vpid */ - u64 vpid; + /* vCPU's vpid & tgid */ + unsigned long vpid, tgid; gpa_t flush_gpa; /* Frequency of stable timer in Hz */ diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 1b861cbc5e10..7afeff852bfe 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -301,6 +301,7 @@ static void __used output_kvm_defines(void) OFFSET(KVM_ARCH_HTP, kvm_vcpu_arch, host_tp); OFFSET(KVM_ARCH_HPGD, kvm_vcpu_arch, host_pgd); OFFSET(KVM_ARCH_KVMPGD, kvm_vcpu_arch, kvm_pgd); + OFFSET(KVM_ARCH_TGID, kvm_vcpu_arch, tgid); OFFSET(KVM_ARCH_HANDLE_EXIT, kvm_vcpu_arch, handle_exit); OFFSET(KVM_ARCH_HEENTRY, kvm_vcpu_arch, host_eentry); OFFSET(KVM_ARCH_GEENTRY, kvm_vcpu_arch, guest_eentry); diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 236523d2449d..7f2969c43e30 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -225,7 +225,7 @@ static void kvm_update_vpid(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.vpid = vpid; } -void kvm_check_vpid(struct kvm_vcpu *vcpu) +static void __kvm_check_vpid(struct kvm_vcpu *vcpu) { int cpu; bool migrated; @@ -253,7 +253,6 @@ void kvm_check_vpid(struct kvm_vcpu *vcpu) kvm_update_vpid(vcpu, cpu); trace_kvm_vpid_change(vcpu, vcpu->arch.vpid); vcpu->cpu = cpu; - kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); /* * LLBCTL is a separated guest CSR register from host, a general @@ -279,6 +278,23 @@ void kvm_check_vpid(struct kvm_vcpu *vcpu) } } +static void __kvm_check_vmid(struct kvm_vcpu *vcpu) +{ + unsigned long vmid; + + vmid = vcpu->arch.vpid & vpid_mask; + if (vcpu->arch.tgid != vmid) { + vcpu->arch.tgid = vcpu->arch.vpid & vpid_mask; + kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + } +} + +void kvm_check_vpid(struct kvm_vcpu *vcpu) +{ + __kvm_check_vpid(vcpu); + __kvm_check_vmid(vcpu); +} + void kvm_init_vmcs(struct kvm *kvm) { kvm->arch.vmcs = vmcs; diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S index 936e4ae3e408..e17e9edb5dd4 100644 --- a/arch/loongarch/kvm/switch.S +++ b/arch/loongarch/kvm/switch.S @@ -66,9 +66,8 @@ ld.d t0, a2, KVM_ARCH_KVMPGD csrwr t0, LOONGARCH_CSR_PGDL - /* Mix GID and RID */ - csrrd t1, LOONGARCH_CSR_GSTAT - bstrpick.w t1, t1, CSR_GSTAT_GID_SHIFT_END, CSR_GSTAT_GID_SHIFT + /* Set VMID for gpa --> hpa mapping */ + ld.d t1, a2, KVM_ARCH_TGID csrrd t0, LOONGARCH_CSR_GTLBC bstrins.w t0, t1, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT csrwr t0, LOONGARCH_CSR_GTLBC diff --git a/arch/loongarch/kvm/tlb.c b/arch/loongarch/kvm/tlb.c index ebdbe9264e9c..afcacb62988a 100644 --- a/arch/loongarch/kvm/tlb.c +++ b/arch/loongarch/kvm/tlb.c @@ -23,7 +23,10 @@ void kvm_flush_tlb_all(void) void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) { + unsigned int vmid; + lockdep_assert_irqs_disabled(); gpa &= (PAGE_MASK << 1); - invtlb(INVTLB_GID_ADDR, read_csr_gstat() & CSR_GSTAT_GID, gpa); + vmid = (vcpu->arch.tgid << CSR_GSTAT_GID_SHIFT) & CSR_GSTAT_GID; + invtlb(INVTLB_GID_ADDR, vmid, gpa); } From 2339ae2c8a17021c24e015b28a1845066a152508 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 13 Oct 2026 16:48:42 +0800 Subject: [PATCH 461/506] LoongArch: KVM: Implement vmid updating logic VMID calculation method is the same as ASID on LoongArch which is a percpu vmid calculation method. For every physical CPU, the VMIDs of different VMs are different, and it is the same for different vCPUs of the same VM. When vCPU is scheduled on the physical CPU, it checked vmid of this VM and the global cached vmid, and judge whether it is valid or not. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_host.h | 7 ++++ arch/loongarch/kvm/main.c | 55 ++++++++++++++++++++++++--- arch/loongarch/kvm/mmu.c | 25 ++++++++++-- arch/loongarch/kvm/tlb.c | 14 +++++++ 4 files changed, 92 insertions(+), 9 deletions(-) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 5616050b7416..b045697dcce9 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -79,6 +79,7 @@ struct kvm_arch_memory_slot { #define HOST_MAX_PMNUM 16 struct kvm_context { unsigned long vpid_cache; + unsigned long vmid_cache; struct kvm_vcpu *last_vcpu; /* Host PMU CSR */ u64 perf_ctrl[HOST_MAX_PMNUM]; @@ -133,6 +134,8 @@ struct kvm_arch { unsigned long kvm_features; s64 time_offset; + cpumask_t tlb_flush_pending; + unsigned long vmid[NR_CPUS]; struct kvm_context __percpu *vmcs; struct loongarch_ipi *ipi; struct loongarch_dmsintc *dmsintc; @@ -318,6 +321,8 @@ int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu); /* MMU handling */ void kvm_flush_tlb_all(void); +void kvm_flush_tlb_all_stage1(void); +void kvm_flush_tlb_all_stage2(void); void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa); int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write, int ecode); @@ -349,6 +354,8 @@ static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS +int kvm_arch_flush_remote_tlbs(struct kvm *kvm); void kvm_check_vpid(struct kvm_vcpu *vcpu); enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer); void kvm_init_vmcs(struct kvm *kvm); diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 7f2969c43e30..2cbaed9b1663 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -218,7 +218,10 @@ static void kvm_update_vpid(struct kvm_vcpu *vcpu, int cpu) ++vpid; /* vpid 0 reserved for root */ /* start new vpid cycle */ - kvm_flush_tlb_all(); + if (!cpu_has_guestid) + kvm_flush_tlb_all(); + else + kvm_flush_tlb_all_stage1(); } context->vpid_cache = vpid; @@ -278,15 +281,54 @@ static void __kvm_check_vpid(struct kvm_vcpu *vcpu) } } -static void __kvm_check_vmid(struct kvm_vcpu *vcpu) +static void kvm_update_vmid(struct kvm_vcpu *vcpu, int cpu) { unsigned long vmid; + struct kvm_context *context; - vmid = vcpu->arch.vpid & vpid_mask; - if (vcpu->arch.tgid != vmid) { - vcpu->arch.tgid = vcpu->arch.vpid & vpid_mask; - kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + vmid = context->vmid_cache + 1; + if (!(vmid & vpid_mask)) { + /* finish round of vmid loop */ + if (unlikely(!vmid)) + vmid = vpid_mask + 1; + + ++vmid; /* vmid 0 reserved for root */ + + /* start new vmid cycle */ + kvm_flush_tlb_all_stage2(); } + + context->vmid_cache = vmid; + vcpu->kvm->arch.vmid[cpu] = vmid; +} + +static void __kvm_check_vmid(struct kvm_vcpu *vcpu) +{ + int cpu; + unsigned long ver, old, vmid; + + /* On CPUs like Loongson-3A5000, vmid must be the same as vpid */ + if (!cpu_has_guestid) { + vmid = vcpu->arch.vpid & vpid_mask; + if (vcpu->arch.tgid != vmid) { + vcpu->arch.tgid = vcpu->arch.vpid & vpid_mask; + kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + } + return; + } + + cpu = smp_processor_id(); + if (cpumask_test_and_clear_cpu(cpu, &vcpu->kvm->arch.tlb_flush_pending)) + vcpu->kvm->arch.vmid[cpu] = 0; + + /* Check if our vmid is of an older version */ + ver = vcpu->kvm->arch.vmid[cpu] & ~vpid_mask; + old = this_cpu_ptr(vcpu->kvm->arch.vmcs)->vmid_cache & ~vpid_mask; + if (ver != old) + kvm_update_vmid(vcpu, cpu); + + vcpu->arch.tgid = vcpu->kvm->arch.vmid[cpu] & vpid_mask; } void kvm_check_vpid(struct kvm_vcpu *vcpu) @@ -392,6 +434,7 @@ static int kvm_loongarch_env_init(void) for_each_possible_cpu(cpu) { context = per_cpu_ptr(vmcs, cpu); context->vpid_cache = vpid_mask + 1; + context->vmid_cache = vpid_mask + 1; context->last_vcpu = NULL; } diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c index 3e9a0b285fd2..64fddd9f0ada 100644 --- a/arch/loongarch/kvm/mmu.c +++ b/arch/loongarch/kvm/mmu.c @@ -903,7 +903,12 @@ static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, bool write) * there is invalid tlb with small page * need flush these invalid tlbs for current vcpu */ - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + if (!cpu_has_ptw) { + if (!cpu_has_guestid) + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + else + cpumask_set_cpu(vcpu->cpu, &vcpu->kvm->arch.tlb_flush_pending); + } ++kvm->stat.hugepages; } else if (kvm_pte_huge(*ptep) && write) ptep = kvm_split_huge(vcpu, ptep, gfn); @@ -939,8 +944,11 @@ int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long gpa, bool write, in * * With SW PTW, invalid TLB is added in TLB refill exception. */ - vcpu->arch.flush_gpa = gpa; - kvm_make_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + if (!cpu_has_guestid) { + vcpu->arch.flush_gpa = gpa; + kvm_make_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + } else + cpumask_set_cpu(vcpu->cpu, &vcpu->kvm->arch.tlb_flush_pending); } return 0; @@ -949,3 +957,14 @@ int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long gpa, bool write, in void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { } + +int kvm_arch_flush_remote_tlbs(struct kvm *kvm) +{ + /* Queue a TLB flush for each CPU to perform on next vCPU loading */ + if (cpu_has_guestid) { + cpumask_setall(&kvm->arch.tlb_flush_pending); + smp_wmb(); /* Be sure that other CPUS can watch the changes */ + } + + return 1; /* Return 1 continue to send IPI to running vCPUs */ +} diff --git a/arch/loongarch/kvm/tlb.c b/arch/loongarch/kvm/tlb.c index afcacb62988a..994216c98ea2 100644 --- a/arch/loongarch/kvm/tlb.c +++ b/arch/loongarch/kvm/tlb.c @@ -21,6 +21,20 @@ void kvm_flush_tlb_all(void) local_irq_restore(flags); } +/* Invalidate all stage1 TLB entries including GVA-->GPA mappings */ +void kvm_flush_tlb_all_stage1(void) +{ + lockdep_assert_irqs_disabled(); + invtlb_all(INVGTLB_ALLGID_GVA_TO_GPA, 0, 0); +} + +/* Invalidate all stage2 TLB entries including GPA-->HPA mappings */ +void kvm_flush_tlb_all_stage2(void) +{ + lockdep_assert_irqs_disabled(); + invtlb_all(INVTLB_ALLGID_GPA_TO_HPA, 0, 0); +} + void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) { unsigned int vmid; From 0f76a3d6abc1739017e1c37af97bc7d954741793 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 13 Oct 2026 16:48:42 +0800 Subject: [PATCH 462/506] LoongArch: KVM: Probe & enable separate vmid feature With CSR GTLBC shortname for Guest TLB Control Register, separate vmid feature will be enabled if bit 14 CSR_GTLBC_USES2TGID is set. Probe and enable this if S2TGID feature is detected from CPUCFG8. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/loongarch.h | 5 +++++ arch/loongarch/kernel/cpu-probe.c | 4 ++++ arch/loongarch/kvm/main.c | 6 ++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 2a6bc99177d8..88e3bdfd4971 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -138,6 +138,9 @@ #define CPUCFG6_PMBITS_SHIFT 8 #define CPUCFG6_UPM BIT(14) +#define LOONGARCH_CPUCFG8 0x8 +#define CPUCFG8_S2TGID BIT(8) + #define LOONGARCH_CPUCFG16 0x10 #define CPUCFG16_L1_IUPRE BIT(0) #define CPUCFG16_L1_IUUNIFY BIT(1) @@ -350,6 +353,8 @@ #define CSR_GTLBC_TGID_WIDTH 8 #define CSR_GTLBC_TGID_SHIFT_END (CSR_GTLBC_TGID_SHIFT + CSR_GTLBC_TGID_WIDTH - 1) #define CSR_GTLBC_TGID (_ULCAST_(0xff) << CSR_GTLBC_TGID_SHIFT) +#define CSR_GTLBC_USES2TGID_SHIFT 14 +#define CSR_GTLBC_USES2TGID (_ULCAST_(0x1) << CSR_GTLBC_USES2TGID_SHIFT) #define CSR_GTLBC_TOTI_SHIFT 13 #define CSR_GTLBC_TOTI (_ULCAST_(0x1) << CSR_GTLBC_TOTI_SHIFT) #define CSR_GTLBC_USETGID_SHIFT 12 diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 74d31f260dfd..7121e7483498 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c @@ -241,6 +241,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) if (config & CPUCFG6_PMP) c->options |= LOONGARCH_CPU_PMP; + config = read_cpucfg(LOONGARCH_CPUCFG8); + if (config & CPUCFG8_S2TGID) + c->options |= LOONGARCH_CPU_GUESTID; + config = csr_read32(LOONGARCH_CSR_ASID); config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT; asid_mask = GENMASK(config - 1, 0); diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 2cbaed9b1663..c65ae2fa4c4b 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -358,7 +358,7 @@ int kvm_arch_enable_virtualization_cpu(void) write_csr_gcfg(0); write_csr_gstat(0); write_csr_gintc(0); - clear_csr_gtlbc(CSR_GTLBC_USETGID | CSR_GTLBC_TOTI); + clear_csr_gtlbc(CSR_GTLBC_TOTI | CSR_GTLBC_USETGID | CSR_GTLBC_USES2TGID); /* * Enable virtualization features granting guest direct control of @@ -381,6 +381,8 @@ int kvm_arch_enable_virtualization_cpu(void) /* Enable using TGID */ set_csr_gtlbc(CSR_GTLBC_USETGID); + if (cpu_has_guestid) + set_csr_gtlbc(CSR_GTLBC_USES2TGID); kvm_debug("GCFG:%lx GSTAT:%lx GINTC:%lx GTLBC:%lx", read_csr_gcfg(), read_csr_gstat(), read_csr_gintc(), read_csr_gtlbc()); @@ -399,7 +401,7 @@ void kvm_arch_disable_virtualization_cpu(void) write_csr_gcfg(0); write_csr_gstat(0); write_csr_gintc(0); - clear_csr_gtlbc(CSR_GTLBC_USETGID | CSR_GTLBC_TOTI); + clear_csr_gtlbc(CSR_GTLBC_TOTI | CSR_GTLBC_USETGID | CSR_GTLBC_USES2TGID); /* Flush any remaining guest TLB entries */ kvm_flush_tlb_all(); From 3139c1c5fcbde9a30d403d69e9dee40d43352029 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 13 Oct 2026 16:48:50 +0800 Subject: [PATCH 463/506] LoongArch: KVM: Remove parameter "level" when injecting MSI With MSI interrupt, there is intterupt set injection, no interrupt clear action. DMSI interrupt controller is specially for MSI irqchip. And there is checking with parameter "level" in most upper functions such as kvm_set_msi() and kvm_arch_set_irq_inatomic() already. Here remove the duplicated "level" parameter checking in the MSI injection leaf functions. Reviewed-by: Tao Cui Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_dmsintc.h | 3 +-- arch/loongarch/include/asm/kvm_pch_pic.h | 2 +- arch/loongarch/kvm/intc/dmsintc.c | 9 +++------ arch/loongarch/kvm/intc/pch_pic.c | 6 +++--- arch/loongarch/kvm/irqfd.c | 4 ++-- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/arch/loongarch/include/asm/kvm_dmsintc.h b/arch/loongarch/include/asm/kvm_dmsintc.h index 7c0158764d84..e19b3f5b3c52 100644 --- a/arch/loongarch/include/asm/kvm_dmsintc.h +++ b/arch/loongarch/include/asm/kvm_dmsintc.h @@ -22,7 +22,6 @@ struct dmsintc_state { int kvm_loongarch_register_dmsintc_device(void); void kvm_loongarch_unregister_dmsintc_device(void); void dmsintc_inject_irq(struct kvm_vcpu *vcpu); -int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level); -int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 vector, int level); +int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data); #endif diff --git a/arch/loongarch/include/asm/kvm_pch_pic.h b/arch/loongarch/include/asm/kvm_pch_pic.h index 887b0431fd20..19ed1494c114 100644 --- a/arch/loongarch/include/asm/kvm_pch_pic.h +++ b/arch/loongarch/include/asm/kvm_pch_pic.h @@ -72,6 +72,6 @@ struct kvm_kernel_irq_routing_entry; int kvm_loongarch_register_pch_pic_device(void); void kvm_loongarch_unregister_pch_pic_device(void); void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level); -int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level); +int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e); #endif /* __ASM_KVM_PCH_PIC_H */ diff --git a/arch/loongarch/kvm/intc/dmsintc.c b/arch/loongarch/kvm/intc/dmsintc.c index 89f980d867be..79e56e330aeb 100644 --- a/arch/loongarch/kvm/intc/dmsintc.c +++ b/arch/loongarch/kvm/intc/dmsintc.c @@ -43,13 +43,10 @@ void dmsintc_inject_irq(struct kvm_vcpu *vcpu) } } -int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, - struct kvm_vcpu *vcpu, u32 vector, int level) +static int dmsintc_deliver_msi_to_vcpu(struct kvm_vcpu *vcpu, u32 vector) { struct dmsintc_state *ds = &vcpu->arch.dmsintc_state; - if (!level) - return 0; if (!vcpu || vector >= 256) return -EINVAL; if (!ds) @@ -65,7 +62,7 @@ int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, return 0; } -int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level) +int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data) { unsigned int irq, cpu; struct kvm_vcpu *vcpu; @@ -78,7 +75,7 @@ int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level) if (!vcpu) return -EINVAL; - return dmsintc_deliver_msi_to_vcpu(kvm, vcpu, irq, level); + return dmsintc_deliver_msi_to_vcpu(vcpu, irq); } static int kvm_dmsintc_ctrl_access(struct kvm_device *dev, diff --git a/arch/loongarch/kvm/intc/pch_pic.c b/arch/loongarch/kvm/intc/pch_pic.c index 2b63b0c2c7ce..220fc23ffe8c 100644 --- a/arch/loongarch/kvm/intc/pch_pic.c +++ b/arch/loongarch/kvm/intc/pch_pic.c @@ -68,20 +68,20 @@ void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level) } /* msi irq handler */ -int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level) +int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e) { u64 msg_addr = (((u64)e->msi.address_hi) << 32) | e->msi.address_lo; if (cpu_has_msgint && kvm->arch.dmsintc && msg_addr >= kvm->arch.dmsintc->msg_addr_base && msg_addr < (kvm->arch.dmsintc->msg_addr_base + kvm->arch.dmsintc->msg_addr_size)) { - return dmsintc_set_irq(kvm, msg_addr, e->msi.data, level); + return dmsintc_set_irq(kvm, msg_addr, e->msi.data); } if (e->msi.data >= EIOINTC_IRQS) return -EINVAL; - eiointc_set_irq(kvm->arch.eiointc, e->msi.data, level); + eiointc_set_irq(kvm->arch.eiointc, e->msi.data, 1); return 0; } diff --git a/arch/loongarch/kvm/irqfd.c b/arch/loongarch/kvm/irqfd.c index 40ed1081c4b6..2e9c32023752 100644 --- a/arch/loongarch/kvm/irqfd.c +++ b/arch/loongarch/kvm/irqfd.c @@ -29,7 +29,7 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, if (!level) return -1; - return pch_msi_set_irq(kvm, e, level); + return pch_msi_set_irq(kvm, e); } /* @@ -78,7 +78,7 @@ int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, pch_pic_set_irq(kvm->arch.pch_pic, e->irqchip.pin, level); return 0; case KVM_IRQ_ROUTING_MSI: - return pch_msi_set_irq(kvm, e, level); + return pch_msi_set_irq(kvm, e); default: return -EWOULDBLOCK; } From ef859f31c27edb2718e310deac6ad11923a9bce4 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 13 Oct 2026 16:48:50 +0800 Subject: [PATCH 464/506] LoongArch: KVM: Remove duplicated checking with DMSI irq injection There are some duplicated error checking in DMSI interrupt injection function, and vcpu::arch::dmsintc_state is an embeded structure rather than a pointer, NULL pointer checking for dmsintc_state is unnecessary. Here remove all these unnecessary checking. Reviewed-by: Tao Cui Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/kvm/intc/dmsintc.c | 33 ++++++++----------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/arch/loongarch/kvm/intc/dmsintc.c b/arch/loongarch/kvm/intc/dmsintc.c index 79e56e330aeb..ff327b4c05f7 100644 --- a/arch/loongarch/kvm/intc/dmsintc.c +++ b/arch/loongarch/kvm/intc/dmsintc.c @@ -14,9 +14,6 @@ void dmsintc_inject_irq(struct kvm_vcpu *vcpu) unsigned long vector[4], old; struct dmsintc_state *ds = &vcpu->arch.dmsintc_state; - if (!ds) - return; - for (i = 0; i < 4; i++) { old = atomic64_read(&(ds->vector_map[i])); vector[i] = old ? atomic64_xchg(&(ds->vector_map[i]), 0) : 0; @@ -43,25 +40,6 @@ void dmsintc_inject_irq(struct kvm_vcpu *vcpu) } } -static int dmsintc_deliver_msi_to_vcpu(struct kvm_vcpu *vcpu, u32 vector) -{ - struct dmsintc_state *ds = &vcpu->arch.dmsintc_state; - - if (!vcpu || vector >= 256) - return -EINVAL; - if (!ds) - return -ENODEV; - - if (!kvm_guest_has_msgint(&vcpu->arch)) - return -EINVAL; - - set_bit(vector, (unsigned long *)&ds->vector_map); - kvm_queue_irq(vcpu, INT_AVEC); - kvm_vcpu_kick(vcpu); - - return 0; -} - int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data) { unsigned int irq, cpu; @@ -69,13 +47,18 @@ int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data) irq = (addr >> AVEC_IRQ_SHIFT) & AVEC_IRQ_MASK; cpu = (addr >> AVEC_CPU_SHIFT) & kvm->arch.dmsintc->cpu_mask; - if (cpu >= KVM_MAX_VCPUS) - return -EINVAL; vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); if (!vcpu) return -EINVAL; - return dmsintc_deliver_msi_to_vcpu(vcpu, irq); + if (!kvm_guest_has_msgint(&vcpu->arch)) + return -EINVAL; + + set_bit(irq, (unsigned long *)&vcpu->arch.dmsintc_state.vector_map); + kvm_queue_irq(vcpu, INT_AVEC); + kvm_vcpu_kick(vcpu); + + return 0; } static int kvm_dmsintc_ctrl_access(struct kvm_device *dev, From 00289c235d4094b3beba123b163763f85f587247 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 13 Oct 2026 16:48:50 +0800 Subject: [PATCH 465/506] LoongArch: KVM: Add hypercall crash support Now generic pvpanic is supported in Linux kernel which will notify VMM with VM crash event. There is some limitations with pvpanic, such as it requires a good kernel stack or the CPU hardware context and panic() C function is in health state. Sometimes CPU hardware context or kernel/irq stack is corrupted. So hypercall code is a more direct and low-level crash method, which is useful in exception entry with assmble language. Here add a hypercall code to support VM crash function, so that VMM can capture CPU crash context in time. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_host.h | 1 + arch/loongarch/include/asm/kvm_para.h | 3 +++ arch/loongarch/include/uapi/asm/kvm.h | 1 + arch/loongarch/include/uapi/asm/kvm_para.h | 1 + arch/loongarch/kvm/exit.c | 11 +++++++++++ arch/loongarch/kvm/vm.c | 2 ++ 6 files changed, 19 insertions(+) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index b045697dcce9..1e888f5dc247 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -172,6 +172,7 @@ enum emulation_result { BIT(KVM_FEATURE_PREEMPT) | \ BIT(KVM_FEATURE_STEAL_TIME) | \ BIT(KVM_FEATURE_USER_HCALL) | \ + BIT(KVM_FEATURE_CRASH) | \ BIT(KVM_FEATURE_VIRT_EXTIOI)) struct kvm_vcpu_arch { diff --git a/arch/loongarch/include/asm/kvm_para.h b/arch/loongarch/include/asm/kvm_para.h index fb17ba0fa101..2f4b9ccf2afb 100644 --- a/arch/loongarch/include/asm/kvm_para.h +++ b/arch/loongarch/include/asm/kvm_para.h @@ -14,6 +14,7 @@ #define KVM_HCALL_CODE_SERVICE 0 #define KVM_HCALL_CODE_SWDBG 1 #define KVM_HCALL_CODE_USER_SERVICE 2 +#define KVM_HCALL_CODE_CRASH 3 #define KVM_HCALL_SERVICE HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_SERVICE) #define KVM_HCALL_FUNC_IPI 1 @@ -23,6 +24,8 @@ #define KVM_HCALL_USER_SERVICE HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_USER_SERVICE) +#define KVM_HCALL_CRASH HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_CRASH) + /* * LoongArch hypercall return code */ diff --git a/arch/loongarch/include/uapi/asm/kvm.h b/arch/loongarch/include/uapi/asm/kvm.h index cd0b5c11ca9c..b222bfc48042 100644 --- a/arch/loongarch/include/uapi/asm/kvm.h +++ b/arch/loongarch/include/uapi/asm/kvm.h @@ -106,6 +106,7 @@ struct kvm_fpu { #define KVM_LOONGARCH_VM_FEAT_PTW 8 #define KVM_LOONGARCH_VM_FEAT_MSGINT 9 #define KVM_LOONGARCH_VM_FEAT_PV_PREEMPT 10 +#define KVM_LOONGARCH_VM_FEAT_CRASH 11 /* Device Control API on vcpu fd */ #define KVM_LOONGARCH_VCPU_CPUCFG 0 diff --git a/arch/loongarch/include/uapi/asm/kvm_para.h b/arch/loongarch/include/uapi/asm/kvm_para.h index d28cbcadd276..bf1a0c826ca6 100644 --- a/arch/loongarch/include/uapi/asm/kvm_para.h +++ b/arch/loongarch/include/uapi/asm/kvm_para.h @@ -16,6 +16,7 @@ #define KVM_FEATURE_IPI 1 #define KVM_FEATURE_STEAL_TIME 2 #define KVM_FEATURE_PREEMPT 3 +#define KVM_FEATURE_CRASH 4 /* BIT 24 - 31 are features configurable by user space vmm */ #define KVM_FEATURE_VIRT_EXTIOI 24 #define KVM_FEATURE_USER_HCALL 25 diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index 4f58e6e2cf86..3b6daa946c76 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -911,6 +911,17 @@ static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode) vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE; ret = RESUME_HOST; break; + case KVM_HCALL_CRASH: + if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_CRASH)) { + kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE); + break; + } + + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; + vcpu->run->system_event.ndata = 0; + ret = RESUME_HOST; + break; case KVM_HCALL_SWDBG: /* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */ if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) { diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index 6dabb227a732..e4c1c65fe20b 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c @@ -50,7 +50,9 @@ static void kvm_vm_init_features(struct kvm *kvm) /* Enable all PV features by default */ kvm->arch.pv_features |= BIT(KVM_FEATURE_IPI); + kvm->arch.pv_features |= BIT(KVM_FEATURE_CRASH); kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PV_IPI); + kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_CRASH); if (kvm_pvtime_supported()) { kvm->arch.pv_features |= BIT(KVM_FEATURE_PREEMPT); kvm->arch.pv_features |= BIT(KVM_FEATURE_STEAL_TIME); From 4012f1f9ffe90d06e43a716859aa3be14d0c287b Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 13 Oct 2026 16:48:50 +0800 Subject: [PATCH 466/506] LoongArch: KVM: Allow to set pv_feature until vCPU run Now pv_feature can be set only once, there is problem with VM migration. When a vCPU is created and after migration, now pv_feature is allowed to set for many times, until vCPU starts to run. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/kvm_host.h | 3 ++- arch/loongarch/kvm/vcpu.c | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 1e888f5dc247..92bf94757ab8 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -167,7 +167,6 @@ enum emulation_result { #define KVM_LARCH_SWCSR_LATEST (0x1 << 3) #define KVM_LARCH_HWCSR_USABLE (0x1 << 4) -#define LOONGARCH_PV_FEAT_UPDATED BIT_ULL(63) #define LOONGARCH_PV_FEAT_MASK (BIT(KVM_FEATURE_IPI) | \ BIT(KVM_FEATURE_PREEMPT) | \ BIT(KVM_FEATURE_STEAL_TIME) | \ @@ -245,6 +244,8 @@ struct kvm_vcpu_arch { u64 timer_mhz; ktime_t expire; + /* vCPU has run at least once */ + bool has_run; /* Last CPU the vCPU state was loaded on */ int last_sched_cpu; /* mp state */ diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 8e028be3f0a9..4026b9bf9f2d 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -1166,12 +1166,17 @@ static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, /* All vCPUs need set the same PV features */ spin_lock(&kvm->arch.pv_setting_lock); - if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED) - && ((kvm->arch.pv_features & valid) != val)) { + if ((kvm->arch.pv_features & valid) == val) { + spin_unlock(&kvm->arch.pv_setting_lock); + return 0; + } + + if (vcpu->arch.has_run) { spin_unlock(&kvm->arch.pv_setting_lock); return -EINVAL; } - kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED; + + kvm->arch.pv_features = val; spin_unlock(&kvm->arch.pv_setting_lock); return 0; default: @@ -1858,6 +1863,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) int r = -EINTR; struct kvm_run *run = vcpu->run; + /* Mark vCPU has run at least once */ + if (!vcpu->arch.has_run) + vcpu->arch.has_run = true; + if (vcpu->mmio_needed) { if (!vcpu->mmio_is_write) kvm_complete_mmio_read(vcpu, run); From 34f4ce22cf25802d8026c60c13c2db81f8c8e3a0 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 13 Oct 2026 16:49:05 +0800 Subject: [PATCH 467/506] KVM: LoongArch: selftests: Enable HW PTW by auto detection method Add HW PTW feature when VM is created. This feature is detected firstly, and enabled if it is available on the host machine. With HW PTW feature enabled, refill exception will not happen and it is not necessary to set TLB refill exception entry. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- .../selftests/kvm/include/loongarch/processor.h | 3 +++ .../selftests/kvm/lib/loongarch/processor.c | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/include/loongarch/processor.h b/tools/testing/selftests/kvm/include/loongarch/processor.h index 0e59daa8024e..8d9fabdc89e3 100644 --- a/tools/testing/selftests/kvm/include/loongarch/processor.h +++ b/tools/testing/selftests/kvm/include/loongarch/processor.h @@ -79,6 +79,7 @@ #define CPUCFG2_LASX BIT(7) #define CPUCFG2_LLFTP BIT(14) #define CPUCFG2_LLFTPREV GENMASK(17, 15) +#define CPUCFG2_PTW BIT(24) /* LoongArch Basic CSR registers */ #define LOONGARCH_CSR_CRMD 0x0 /* Current mode info */ @@ -128,6 +129,8 @@ #define LOONGARCH_CSR_PGD 0x1b #define LOONGARCH_CSR_PWCTL0 0x1c #define LOONGARCH_CSR_PWCTL1 0x1d +#define CSR_PWCTL1_PTW_SHIFT 24 +#define CSR_PWCTL1_PTW BIT_ULL(CSR_PWCTL1_PTW_SHIFT) #define LOONGARCH_CSR_STLBPGSIZE 0x1e #define LOONGARCH_CSR_CPUID 0x20 #define LOONGARCH_CSR_KS0 0x30 diff --git a/tools/testing/selftests/kvm/lib/loongarch/processor.c b/tools/testing/selftests/kvm/lib/loongarch/processor.c index 47bbde3e205a..2b5a742a8939 100644 --- a/tools/testing/selftests/kvm/lib/loongarch/processor.c +++ b/tools/testing/selftests/kvm/lib/loongarch/processor.c @@ -278,6 +278,7 @@ static void loongarch_set_csr(struct kvm_vcpu *vcpu, u64 id, u64 val) void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) { + bool has_ptw; int ret, width; unsigned int cfg; unsigned long val; @@ -300,6 +301,12 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) ret = __kvm_has_device_attr(vm->fd, KVM_LOONGARCH_VM_FEAT_CTRL, KVM_LOONGARCH_VM_FEAT_LASX); if (!ret) cfg |= CPUCFG2_LASX; + ret = __kvm_has_device_attr(vm->fd, KVM_LOONGARCH_VM_FEAT_CTRL, KVM_LOONGARCH_VM_FEAT_PTW); + if (!ret) + cfg |= CPUCFG2_PTW; + + has_ptw = !ret; + loongarch_set_cpucfg(vcpu, LOONGARCH_CPUCFG2, cfg); cfg = read_cpucfg(LOONGARCH_CPUCFG6); loongarch_set_cpucfg(vcpu, LOONGARCH_CPUCFG6, cfg); @@ -338,6 +345,9 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) /* PGD page shift and width */ val = (vm->page_shift + width * (vm->mmu.pgtable_levels - 1)) | width << 6; + if (has_ptw) + val |= CSR_PWCTL1_PTW; + loongarch_set_csr(vcpu, LOONGARCH_CSR_PWCTL1, val); loongarch_set_csr(vcpu, LOONGARCH_CSR_PGDL, vm->mmu.pgd); @@ -345,8 +355,10 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) * Refill exception runs on real mode * Entry address should be physical address */ - val = addr_gva2gpa(vm, (unsigned long)handle_tlb_refill); - loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBRENTRY, val); + if (!has_ptw) { + val = addr_gva2gpa(vm, (unsigned long)handle_tlb_refill); + loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBRENTRY, val); + } /* * General exception runs on page-enabled mode From ab1d40006720a4857ab6f0d3cbc43a3be62e42dd Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Tue, 6 Oct 2026 15:58:35 +0530 Subject: [PATCH 468/506] KVM: selftests: Don't limit LE dirty-bitmap bitops to s390x dirty_log_test already has BITOP_LE_SWIZZLE helpers, but they were compiled in only for s390x, so every other big-endian host used native test_bit(). The test fails on big-endian powerpc because it treated dirty host pages as clean (because of the endianness issue) w/o the fix the test fails with: Clear page 1 value (1) >= iteration (1) Fixes: a049a377164c ("KVM: selftests: Enable dirty_log_test on s390x") Tested-by: Anushree Mathur Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Gautam Menghani Tested-by: Gautam Menghani Acked-by: Madhavan Srinivasan Link: https://patch.msgid.link/63c900860b5f7e49cfa7258602e94b58b3de73b3.1791280439.git.ritesh.list@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/dirty_log_test.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index af5eb0334a74..8189eaac713c 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -44,8 +44,11 @@ #define TEST_MIN_WRITES_PER_ITERATION 0x100 /* Dirty bitmaps are always little endian, so we need to swap on big endian */ -#if defined(__s390x__) +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ # define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) +#else +# define BITOP_LE_SWIZZLE 0 +#endif # define test_bit_le(nr, addr) \ test_bit((nr) ^ BITOP_LE_SWIZZLE, addr) # define __set_bit_le(nr, addr) \ @@ -56,13 +59,6 @@ __test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, addr) # define __test_and_clear_bit_le(nr, addr) \ __test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, addr) -#else -# define test_bit_le test_bit -# define __set_bit_le __set_bit -# define __clear_bit_le __clear_bit -# define __test_and_set_bit_le __test_and_set_bit -# define __test_and_clear_bit_le __test_and_clear_bit -#endif #define TEST_DIRTY_RING_COUNT 65536 From f51639c0ee7a0ed619a19aac970b3ac07714432b Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Tue, 6 Oct 2026 15:58:36 +0530 Subject: [PATCH 469/506] KVM: selftests: Move memslot_perf_test off the 256M ELF load address KVM selftests are built with -fno-PIE/-no-pie so the binary uses fixed load addresses. On powerpc64, GNU ld default links at 0x10000000 (256M) which is also the MEM_GPA, so the identity map trips with the error: PTE already present at level: 4 gva: 0x10000000 pte:0xc00000000002018f Put the test slots at 512M instead which is past the ELF. e.g. failure w/o this fix: ./memslot_perf_test -s 514 Random seed: 0x423558ef Testing map performance with 1 runs, 5 seconds each ==== Test Assertion Failure ==== lib/powerpc/processor.c:235: !pte pid=371 tid=371 errno=2 - No such file or directory 1 0x0000000010018293: virt_arch_pg_map at processor.c:235 2 0x0000000010008b3f: virt_pg_map at kvm_util.h:1325 3 (inlined by) virt_map at kvm_util.c:1571 4 0x0000000010001cab: prepare_vm at memslot_perf_test.c:353 5 (inlined by) test_execute at memslot_perf_test.c:816 6 (inlined by) test_loop at memslot_perf_test.c:1062 7 (inlined by) main at memslot_perf_test.c:1136 8 0x000000001001d713: __libc_start_call_main at libc-start.o:? 9 0x000000001001db47: __libc_start_main_impl at ??:? PTE already present at level: 4 gva: 0x10000000 pte:0xc00000000002018f Tested-by: Anushree Mathur Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Gautam Menghani Tested-by: Gautam Menghani Acked-by: Madhavan Srinivasan Link: https://patch.msgid.link/11b5712b213b7359bbf3bad32601e4373dac43bc.1791280439.git.ritesh.list@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/memslot_perf_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/memslot_perf_test.c b/tools/testing/selftests/kvm/memslot_perf_test.c index 1e241390e11e..db2fd3c59d49 100644 --- a/tools/testing/selftests/kvm/memslot_perf_test.c +++ b/tools/testing/selftests/kvm/memslot_perf_test.c @@ -30,7 +30,7 @@ #define MEM_EXTRA_SIZE SZ_64K #define MEM_SIZE (SZ_512M + MEM_EXTRA_SIZE) -#define MEM_GPA SZ_256M +#define MEM_GPA SZ_512M #define MEM_AUX_GPA MEM_GPA #define MEM_SYNC_GPA MEM_AUX_GPA #define MEM_TEST_GPA (MEM_AUX_GPA + MEM_EXTRA_SIZE) From 2bc2c4a8105f3e3bce457cc6446cf3a13e98a827 Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Tue, 6 Oct 2026 15:58:37 +0530 Subject: [PATCH 470/506] KVM: selftests: Skip the max vCPU ID test when KVM_CAP_PPC_SMT is set kvm_create_max_vcpus creates vCPUs from 0..MAX_VCPUS-1, then again with ids at the top of KVM_CAP_MAX_VCPU_ID. On PowerPC the vCPU id is not a flat index: it encodes SMT thread plus core. So on powerpc maximum permitted vCPU ID depends on the VM's SMT mode and the maximum reported by KVM_CAP_MAX_VCPU_ID exceeds a simple non-SMT VM's limit. So, skip that second loop when KVM_CAP_PPC_SMT is non-zero. The cap is 0 on other arches, so they still run it. The 0..MAX_VCPUS pass is unchanged. Suggested-by: Sean Christopherson Tested-by: Anushree Mathur Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Gautam Menghani Tested-by: Gautam Menghani Acked-by: Madhavan Srinivasan Link: https://patch.msgid.link/bbfbaa9fb2eca8c92f59dc980546c2587dcfc130.1791280439.git.ritesh.list@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/kvm_create_max_vcpus.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c index c5310736ed06..1b95027d41ba 100644 --- a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c +++ b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c @@ -58,7 +58,13 @@ int main(int argc, char *argv[]) test_vcpu_creation(0, kvm_max_vcpus); - if (kvm_max_vcpu_id > kvm_max_vcpus) + /* + * Skip the vCPU ID test when running on PowerPC with SMT support, in + * which case system topology is encoded into the vCPU ID and so a VM + * can't use the full advertised vCPU ID range without crafting a valid + * platform specific topology. + */ + if (kvm_max_vcpu_id > kvm_max_vcpus && !kvm_check_cap(KVM_CAP_PPC_SMT)) test_vcpu_creation( kvm_max_vcpu_id - kvm_max_vcpus, kvm_max_vcpus); From 5527310b4d199518eb61340db84107c653ed1abf Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Tue, 6 Oct 2026 15:58:39 +0530 Subject: [PATCH 471/506] KVM: selftests: Add helper for checking if running on powerpc pseries LPAR PowerPC's PAPR L0 does not guarantee H_GUEST_CREATE/H_GUEST_CREATE_VCPU hcalls to create upto MAX_VCPUS or hundreds of VMs, this is due to limited resouces at L0 for guest management space. This helper allows us to skip such tests which creates a lot of VMs/VCPUs, when we are running nested. Note that there is no easy way to differentiate between running nested on PowerNV v/s kvm-on-powervm case. So we use a common helper to detect any nested case. The helper is added to the common kvm_util header file to avoid sprinkling multiple #ifdef __powerpc64__ everywhere. Suggested-by: Sean Christopherson Tested-by: Anushree Mathur Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Gautam Menghani Tested-by: Gautam Menghani Acked-by: Madhavan Srinivasan Link: https://patch.msgid.link/294c0fcd19d6b00626e230985cf9906f98934d1c.1791280439.git.ritesh.list@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/kvm_util.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 8fff5b40a592..adab27b92311 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -14,6 +14,7 @@ #include #include "linux/rbtree.h" #include +#include #include #include @@ -1054,6 +1055,22 @@ const char *exit_reason_str(unsigned int exit_reason); bool kvm_arch_needs_naturally_aligned_page_tables(void); +/* + * True on powerpc when this host is a pseries LPAR (nested case). + * PowerPC's PAPR L0 does not guarantee H_GUEST_CREATE / H_GUEST_CREATE_VCPU up + * to MAX_VCPUS or creating hundreds of VMs - due to limited resouces at L0 for + * guest management space. This helper allows us to skip such tests when we are + * running nested. + */ +static inline bool kvm_is_ppc_pseries_lpar(void) +{ +#ifdef __powerpc64__ + return !access("/proc/device-tree/rtas/ibm,hypertas-functions", F_OK); +#else + return false; +#endif +} + gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, u32 memslot, bool protected, bool naturally_aligned); gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, From 9de211597e81954d4c981e04bdfa81fa7d3297b5 Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Tue, 6 Oct 2026 15:58:40 +0530 Subject: [PATCH 472/506] KVM: selftests: Skip kvm_create_max_vcpus on powerpc pseries LPAR On a PAPR LPAR, KVM_CREATE_VCPU is H_GUEST_CREATE_VCPU and KVM_CAP_MAX_VCPUS is only the vcpuId range but is not a guarantee from L0 i.e. because L0 Guest Management Space can be exhausted before that many vCPUs exist, so creating MAX_VCPUS is not a valid assert. So, skip the test when powerpc is pseries LPAR (running nested i.e. ibm,hypertas-functions is present). Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Gautam Menghani Tested-by: Gautam Menghani Acked-by: Madhavan Srinivasan Link: https://patch.msgid.link/c8856b73b06bd4b44ff31e7454c8fc56dd402428.1791280439.git.ritesh.list@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/kvm_create_max_vcpus.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c index 1b95027d41ba..22279c638d06 100644 --- a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c +++ b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c @@ -39,6 +39,12 @@ int main(int argc, char *argv[]) int kvm_max_vcpu_id = kvm_check_cap(KVM_CAP_MAX_VCPU_ID); int kvm_max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); + /* + * PowerPC's PAPR L0 does not guarantee KVM_CAP_MAX_VCPUS when running + * on pseries LPAR (nested kvm-on-powervm). + */ + TEST_REQUIRE(!kvm_is_ppc_pseries_lpar()); + pr_info("KVM_CAP_MAX_VCPU_ID: %d\n", kvm_max_vcpu_id); pr_info("KVM_CAP_MAX_VCPUS: %d\n", kvm_max_vcpus); From 1b53b581d4e8de0e48e33f19f131ab88987c4eac Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Tue, 6 Oct 2026 15:58:38 +0530 Subject: [PATCH 473/506] KVM: PPC: selftests: add support for powerpc Implement KVM selftests support for powerpc (Book3S-64). ucalls are implemented with an unsupported PAPR hcall number which will always cause KVM to exit to userspace. Virtual memory is implemented for the radix MMU, and only a base page size is supported (both 4K and 64K). Guest interrupts are taken in real-mode, so require a page allocated at gRA 0x0. Interrupt entry is complicated because gVA:gRA is not 1:1 mapped (like the kernel is), so the MMU can not just be switched on and off. This patch series also changes the following types as done in other selftests. These changes are squashed in this main patch as discussed here [1]: - Use u64 instead of uint64_t - Use s64 instead of int64_t - Use u32 instead of uint32_t - Use u8 instead of uint8_t - Replace u64 gpa, u64 gva|vaddr with gpa_t and gva_t - Adopt to the new page table allocator APIs - Start the guest at the entry point and not the descriptor in case of elfv1 [1]: https://lore.kernel.org/linuxppc-dev/cover.1781093720.git.ritesh.list@gmail.com/ [Rebased to latest mainline tree, "cc" to clobber list, fix the size calculation in kvm_arch_vm_post_create()] Co-developed-by: Ritesh Harjani (IBM) Signed-off-by: Nicholas Piggin Link: https://lore.kernel.org/linuxppc-dev/cover.1781093720.git.ritesh.list@gmail.com/ Tested-by: Anushree Mathur Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Gautam Menghani Tested-by: Gautam Menghani Acked-by: Madhavan Srinivasan Link: https://patch.msgid.link/062570e96982a9a1a82e95c7ed0a3458321fd569.1791280439.git.ritesh.list@gmail.com Signed-off-by: Sean Christopherson --- MAINTAINERS | 2 + tools/testing/selftests/kvm/Makefile | 2 +- tools/testing/selftests/kvm/Makefile.kvm | 7 + .../testing/selftests/kvm/include/kvm_util.h | 9 + .../selftests/kvm/include/powerpc/hcall.h | 17 + .../kvm/include/powerpc/kvm_util_arch.h | 22 + .../selftests/kvm/include/powerpc/ppc_asm.h | 32 ++ .../selftests/kvm/include/powerpc/processor.h | 39 ++ .../selftests/kvm/include/powerpc/ucall.h | 21 + tools/testing/selftests/kvm/lib/guest_modes.c | 20 +- tools/testing/selftests/kvm/lib/kvm_util.c | 8 + .../selftests/kvm/lib/powerpc/handlers.S | 93 ++++ .../testing/selftests/kvm/lib/powerpc/hcall.c | 45 ++ .../selftests/kvm/lib/powerpc/processor.c | 512 ++++++++++++++++++ .../testing/selftests/kvm/lib/powerpc/ucall.c | 22 + 15 files changed, 848 insertions(+), 3 deletions(-) create mode 100644 tools/testing/selftests/kvm/include/powerpc/hcall.h create mode 100644 tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h create mode 100644 tools/testing/selftests/kvm/include/powerpc/ppc_asm.h create mode 100644 tools/testing/selftests/kvm/include/powerpc/processor.h create mode 100644 tools/testing/selftests/kvm/include/powerpc/ucall.h create mode 100644 tools/testing/selftests/kvm/lib/powerpc/handlers.S create mode 100644 tools/testing/selftests/kvm/lib/powerpc/hcall.c create mode 100644 tools/testing/selftests/kvm/lib/powerpc/processor.c create mode 100644 tools/testing/selftests/kvm/lib/powerpc/ucall.c diff --git a/MAINTAINERS b/MAINTAINERS index 4dfc1fb14ef7..940eccd36dae 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14351,6 +14351,8 @@ F: arch/powerpc/include/asm/kvm* F: arch/powerpc/include/uapi/asm/kvm* F: arch/powerpc/kernel/kvm* F: arch/powerpc/kvm/ +F: tools/testing/selftests/kvm/*/powerpc/ +F: tools/testing/selftests/kvm/powerpc/ KERNEL VIRTUAL MACHINE FOR RISC-V (KVM/riscv) M: Anup Patel diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index f2b223072b62..03d91f00092f 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -3,7 +3,7 @@ top_srcdir = ../../../.. include $(top_srcdir)/scripts/subarch.include ARCH ?= $(SUBARCH) -ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch)) +ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch powerpc)) # Top-level selftests allows ARCH=x86_64 :-( ifeq ($(ARCH),x86_64) override ARCH := x86 diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 96bab7002d39..4b7f6ab0ee54 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -53,6 +53,11 @@ LIBKVM_loongarch += lib/loongarch/processor.c LIBKVM_loongarch += lib/loongarch/ucall.c LIBKVM_loongarch += lib/loongarch/exception.S +LIBKVM_powerpc += lib/powerpc/handlers.S +LIBKVM_powerpc += lib/powerpc/processor.c +LIBKVM_powerpc += lib/powerpc/ucall.c +LIBKVM_powerpc += lib/powerpc/hcall.c + # Non-compiled test targets TEST_PROGS_x86 += x86/nx_huge_pages_test.sh @@ -251,6 +256,8 @@ TEST_GEN_PROGS_loongarch += memslot_perf_test TEST_GEN_PROGS_loongarch += set_memory_region_test TEST_GEN_PROGS_loongarch += steal_time +TEST_GEN_PROGS_powerpc = $(TEST_GEN_PROGS_COMMON) + SPLIT_TESTS += arch_timer SPLIT_TESTS += get-reg-list diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index adab27b92311..f488208d65f7 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -242,6 +242,9 @@ enum vm_guest_mode { VM_MODE_P41V48_4K, VM_MODE_P41V39_4K, + VM_MODE_P52V52_4K, /* For powerpc64 */ + VM_MODE_P52V52_64K, + NUM_VM_MODES, }; @@ -301,6 +304,12 @@ extern enum vm_guest_mode vm_mode_default; #define MIN_PAGE_SHIFT 12U #define ptes_per_page(page_size) ((page_size) / 8) +#elif defined(__powerpc64__) + +#define VM_MODE_DEFAULT vm_mode_default +#define MIN_PAGE_SHIFT 12U +#define ptes_per_page(page_size) ((page_size) / 8) + #endif #define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) diff --git a/tools/testing/selftests/kvm/include/powerpc/hcall.h b/tools/testing/selftests/kvm/include/powerpc/hcall.h new file mode 100644 index 000000000000..21b2bf9180f1 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/hcall.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc hcall defines + */ +#ifndef SELFTEST_KVM_HCALL_H +#define SELFTEST_KVM_HCALL_H + +#include + +/* Ucalls use unimplemented PAPR hcall 0 which exits KVM */ +#define H_UCALL 0 + +s64 hcall0(u64 token); +s64 hcall1(u64 token, u64 arg1); +s64 hcall2(u64 token, u64 arg1, u64 arg2); + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h b/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h new file mode 100644 index 000000000000..5d45c25cd299 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_UTIL_ARCH_H +#define SELFTEST_KVM_UTIL_ARCH_H + +#include + +#include "kvm_util_types.h" + +struct kvm_mmu_arch {}; + +/* Page table fragment cache for guest page tables < page size */ +struct vm_pt_frag_cache { + gpa_t page; + size_t page_nr_used; +}; + +struct kvm_vm_arch { + gpa_t prtb; /* process table */ + struct vm_pt_frag_cache pt_frag_cache[2]; /* 256B and 4KB PT caches */ +}; + +#endif /* SELFTEST_KVM_UTIL_ARCH_H */ diff --git a/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h b/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h new file mode 100644 index 000000000000..b9df64659792 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc asm specific defines + */ +#ifndef SELFTEST_KVM_PPC_ASM_H +#define SELFTEST_KVM_PPC_ASM_H + +#define STACK_FRAME_MIN_SIZE 112 /* Could be 32 on ELFv2 */ +#define STACK_REDZONE_SIZE 512 + +#define INT_FRAME_SIZE (STACK_FRAME_MIN_SIZE + STACK_REDZONE_SIZE) + +#define SPR_SRR0 0x01a +#define SPR_SRR1 0x01b +#define SPR_CFAR 0x01c + +#define MSR_SF 0x8000000000000000ULL +#define MSR_HV 0x1000000000000000ULL +#define MSR_VEC 0x0000000002000000ULL +#define MSR_VSX 0x0000000000800000ULL +#define MSR_EE 0x0000000000008000ULL +#define MSR_PR 0x0000000000004000ULL +#define MSR_FP 0x0000000000002000ULL +#define MSR_ME 0x0000000000001000ULL +#define MSR_IR 0x0000000000000020ULL +#define MSR_DR 0x0000000000000010ULL +#define MSR_RI 0x0000000000000002ULL +#define MSR_LE 0x0000000000000001ULL + +#define LPCR_ILE 0x0000000002000000ULL + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/processor.h b/tools/testing/selftests/kvm/include/powerpc/processor.h new file mode 100644 index 000000000000..f0ee21a8e2d4 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/processor.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc processor specific defines + */ +#ifndef SELFTEST_KVM_PROCESSOR_H +#define SELFTEST_KVM_PROCESSOR_H + +#include +#include "ppc_asm.h" +#include "kvm_util_types.h" + +extern unsigned char __interrupts_start[]; +extern unsigned char __interrupts_end[]; + +struct kvm_vm; +struct kvm_vcpu; + +struct ex_regs { + u64 gprs[32]; + u64 nia; + u64 msr; + u64 cfar; + u64 lr; + u64 ctr; + u64 xer; + u32 cr; + u32 trap; + gva_t gva; /* gva of this struct */ +}; + +void vm_install_exception_handler(struct kvm_vm *vm, int vector, + void (*handler)(struct ex_regs *)); + +static inline void cpu_relax(void) +{ + asm volatile("" ::: "memory"); +} + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/ucall.h b/tools/testing/selftests/kvm/include/powerpc/ucall.h new file mode 100644 index 000000000000..e0dbe91e8848 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/ucall.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_UCALL_H +#define SELFTEST_KVM_UCALL_H + +#include "hcall.h" + +#define UCALL_EXIT_REASON KVM_EXIT_PAPR_HCALL + +#define UCALL_R4_UCALL 0x5715 /* regular ucall, r5 contains ucall pointer */ +#define UCALL_R4_SIMPLE 0x0000 /* simple exit usable by asm with no ucall data */ + +static inline void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa) +{ +} + +static inline void ucall_arch_do_ucall(gva_t uc) +{ + hcall2(H_UCALL, UCALL_R4_UCALL, (uintptr_t)(uc)); +} + +#endif diff --git a/tools/testing/selftests/kvm/lib/guest_modes.c b/tools/testing/selftests/kvm/lib/guest_modes.c index 7a96c43b5704..439766fad693 100644 --- a/tools/testing/selftests/kvm/lib/guest_modes.c +++ b/tools/testing/selftests/kvm/lib/guest_modes.c @@ -4,16 +4,20 @@ */ #include "guest_modes.h" -#if defined(__aarch64__) || defined(__riscv) +#if defined(__aarch64__) || defined(__riscv) || defined(__powerpc64__) #include "processor.h" enum vm_guest_mode vm_mode_default; #endif +#if defined(__powerpc64__) +#include +#endif + struct guest_mode guest_modes[NUM_VM_MODES]; void guest_modes_append_default(void) { -#if !defined(__aarch64__) && !defined(__riscv) +#if !defined(__aarch64__) && !defined(__riscv) && !defined(__powerpc64__) guest_mode_append(VM_MODE_DEFAULT, true); #endif @@ -108,6 +112,18 @@ void guest_modes_append_default(void) TEST_ASSERT(vm_mode_default != NUM_VM_MODES, "No supported mode!"); } #endif +#ifdef __powerpc64__ + { + TEST_REQUIRE(kvm_has_cap(KVM_CAP_PPC_MMU_RADIX)); + /* Radix guest EA and RA are 52-bit on POWER9 and POWER10 */ + if (sysconf(_SC_PAGESIZE) == 4096) + vm_mode_default = VM_MODE_P52V52_4K; + else + vm_mode_default = VM_MODE_P52V52_64K; + guest_mode_append(VM_MODE_P52V52_4K, true); + guest_mode_append(VM_MODE_P52V52_64K, true); + } +#endif } void for_each_guest_mode(void (*func)(enum vm_guest_mode, void *), void *arg) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index cb2e7a98820a..00ac3e9fc2c5 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -227,6 +227,8 @@ const char *vm_guest_mode_string(u32 i) [VM_MODE_P41V57_4K] = "PA-bits:41, VA-bits:57, 4K pages", [VM_MODE_P41V48_4K] = "PA-bits:41, VA-bits:48, 4K pages", [VM_MODE_P41V39_4K] = "PA-bits:41, VA-bits:39, 4K pages", + [VM_MODE_P52V52_4K] = "PA-bits:52, VA-bits:52, 4K pages", + [VM_MODE_P52V52_64K] = "PA-bits:52, VA-bits:52, 64K pages", }; _Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES, "Missing new mode strings?"); @@ -263,6 +265,8 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { [VM_MODE_P41V57_4K] = { 41, 57, 0x1000, 12 }, [VM_MODE_P41V48_4K] = { 41, 48, 0x1000, 12 }, [VM_MODE_P41V39_4K] = { 41, 39, 0x1000, 12 }, + [VM_MODE_P52V52_4K] = { 52, 52, 0x1000, 12 }, + [VM_MODE_P52V52_64K] = { 52, 52, 0x10000, 16 }, }; _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES, "Missing new mode params?"); @@ -383,6 +387,10 @@ struct kvm_vm *____vm_create(struct vm_shape shape) case VM_MODE_P41V39_4K: vm->mmu.pgtable_levels = 3; break; + case VM_MODE_P52V52_4K: + case VM_MODE_P52V52_64K: + vm->mmu.pgtable_levels = 4; + break; default: TEST_FAIL("Unknown guest mode: 0x%x", vm->mode); } diff --git a/tools/testing/selftests/kvm/lib/powerpc/handlers.S b/tools/testing/selftests/kvm/lib/powerpc/handlers.S new file mode 100644 index 000000000000..b860f6a520a1 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/handlers.S @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +.macro INTERRUPT vec +. = __interrupts_start + \vec + std %r0,(0*8)(%r13) + std %r3,(3*8)(%r13) + mfspr %r0,SPR_CFAR + li %r3,\vec + b handle_interrupt +.endm + +.balign 0x1000 +.global __interrupts_start +__interrupts_start: +INTERRUPT 0x100 +INTERRUPT 0x200 +INTERRUPT 0x300 +INTERRUPT 0x380 +INTERRUPT 0x400 +INTERRUPT 0x480 +INTERRUPT 0x500 +INTERRUPT 0x600 +INTERRUPT 0x700 +INTERRUPT 0x800 +INTERRUPT 0x900 +INTERRUPT 0xa00 +INTERRUPT 0xc00 +INTERRUPT 0xd00 +INTERRUPT 0xf00 +INTERRUPT 0xf20 +INTERRUPT 0xf40 +INTERRUPT 0xf60 + +virt_handle_interrupt: + stdu %r1,-INT_FRAME_SIZE(%r1) + mr %r3,%r31 + bl route_interrupt + ld %r4,(32*8)(%r31) /* NIA */ + ld %r5,(33*8)(%r31) /* MSR */ + ld %r6,(35*8)(%r31) /* LR */ + ld %r7,(36*8)(%r31) /* CTR */ + ld %r8,(37*8)(%r31) /* XER */ + lwz %r9,(38*8)(%r31) /* CR */ + mtspr SPR_SRR0,%r4 + mtspr SPR_SRR1,%r5 + mtlr %r6 + mtctr %r7 + mtxer %r8 + mtcr %r9 +reg=4 + ld %r0,(0*8)(%r31) + ld %r3,(3*8)(%r31) +.rept 28 + ld reg,(reg*8)(%r31) + reg=reg+1 +.endr + addi %r1,%r1,INT_FRAME_SIZE + rfid + +virt_handle_interrupt_p: + .llong virt_handle_interrupt + +handle_interrupt: +reg=4 +.rept 28 + std reg,(reg*8)(%r13) + reg=reg+1 +.endr + mfspr %r4,SPR_SRR0 + mfspr %r5,SPR_SRR1 + mflr %r6 + mfctr %r7 + mfxer %r8 + mfcr %r9 + std %r4,(32*8)(%r13) /* NIA */ + std %r5,(33*8)(%r13) /* MSR */ + std %r0,(34*8)(%r13) /* CFAR */ + std %r6,(35*8)(%r13) /* LR */ + std %r7,(36*8)(%r13) /* CTR */ + std %r8,(37*8)(%r13) /* XER */ + stw %r9,(38*8 + 0)(%r13) /* CR */ + stw %r3,(38*8 + 4)(%r13) /* TRAP */ + + ld %r31,(39*8)(%r13) /* vaddr */ + ld %r4,virt_handle_interrupt_p - __interrupts_start(0) + mtspr SPR_SRR0,%r4 + /* Reuse SRR1 */ + + rfid +.global __interrupts_end +__interrupts_end: diff --git a/tools/testing/selftests/kvm/lib/powerpc/hcall.c b/tools/testing/selftests/kvm/lib/powerpc/hcall.c new file mode 100644 index 000000000000..d5f5282654ee --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/hcall.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PAPR (pseries) hcall support. + */ +#include "kvm_util.h" +#include "hcall.h" + +s64 hcall0(u64 token) +{ + register uintptr_t r3 asm ("r3") = token; + + asm volatile("sc 1" : "+r"(r3) : + : "r0", "r4", "r5", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +} + +s64 hcall1(u64 token, u64 arg1) +{ + register uintptr_t r3 asm ("r3") = token; + register uintptr_t r4 asm ("r4") = arg1; + + asm volatile("sc 1" : "+r"(r3), "+r"(r4) : + : "r0", "r5", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +} + +s64 hcall2(u64 token, u64 arg1, u64 arg2) +{ + register uintptr_t r3 asm ("r3") = token; + register uintptr_t r4 asm ("r4") = arg1; + register uintptr_t r5 asm ("r5") = arg2; + + asm volatile("sc 1" : "+r"(r3), "+r"(r4), "+r"(r5) : + : "r0", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +} diff --git a/tools/testing/selftests/kvm/lib/powerpc/processor.c b/tools/testing/selftests/kvm/lib/powerpc/processor.c new file mode 100644 index 000000000000..301455024571 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/processor.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KVM selftest powerpc library code - CPU-related functions (page tables...) + */ + +#include + +#include "processor.h" +#include "kvm_util.h" +#include "ucall_common.h" +#include "guest_modes.h" +#include "hcall.h" + +#define RADIX_TREE_SIZE ((0x2UL << 61) | (0x5UL << 5)) /* 52-bits */ +#define RADIX_PGD_INDEX_SIZE 13 + +/* + * A radix table must be aligned to its size, and the tables are larger than + * the guest base page size for several level/page size combinations. + */ +bool kvm_arch_needs_naturally_aligned_page_tables(void) +{ + return true; +} + +static void set_proc_table(struct kvm_vm *vm, int pid, u64 dw0, u64 dw1) +{ + u64 *proc_table; + + proc_table = addr_gpa2hva(vm, vm->arch.prtb); + proc_table[pid * 2 + 0] = cpu_to_be64(dw0); + proc_table[pid * 2 + 1] = cpu_to_be64(dw1); +} + +static void set_radix_proc_table(struct kvm_vm *vm, int pid, gpa_t pgd) +{ + set_proc_table(vm, pid, pgd | RADIX_TREE_SIZE | RADIX_PGD_INDEX_SIZE, 0); +} + +void virt_arch_pgd_alloc(struct kvm_vm *vm) +{ + struct kvm_ppc_mmuv3_cfg mmu_cfg; + gpa_t prtb, pgtb; + size_t pgd_pages; + + TEST_ASSERT((vm->mode == VM_MODE_P52V52_4K) || + (vm->mode == VM_MODE_P52V52_64K), + "Unsupported guest mode, mode: 0x%x", vm->mode); + + if (vm->mmu.pgd_created) + return; + + prtb = vm_alloc_page_table(vm); + vm->arch.prtb = prtb; + + pgd_pages = (1UL << (RADIX_PGD_INDEX_SIZE + 3)) >> vm->page_shift; + if (!pgd_pages) + pgd_pages = 1; + pgtb = vm_alloc_page_table_pages(vm, pgd_pages); + vm->mmu.pgd = pgtb; + + /* Set the base page directory in the proc table */ + set_radix_proc_table(vm, 0, pgtb); + + if (vm->mode == VM_MODE_P52V52_4K) + mmu_cfg.process_table = prtb | 0x8000000000000000UL | 0x0; /* 4K size */ + else /* vm->mode == VM_MODE_P52V52_64K */ + mmu_cfg.process_table = prtb | 0x8000000000000000UL | 0x4; /* 64K size */ + mmu_cfg.flags = KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE; + + vm_ioctl(vm, KVM_PPC_CONFIGURE_V3_MMU, &mmu_cfg); + + vm->mmu.pgd_created = true; +} + +static int pt_shift(struct kvm_vm *vm, int level) +{ + switch (level) { + case 1: + return 13; + case 2: + case 3: + return 9; + case 4: + if (vm->mode == VM_MODE_P52V52_4K) + return 9; + else /* vm->mode == VM_MODE_P52V52_64K */ + return 5; + default: + TEST_ASSERT(false, "Invalid page table level %d\n", level); + return 0; + } +} + +static u64 pt_entry_coverage(struct kvm_vm *vm, int level) +{ + u64 size = vm->page_size; + + if (level == 4) + return size; + size <<= pt_shift(vm, 4); + if (level == 3) + return size; + size <<= pt_shift(vm, 3); + if (level == 2) + return size; + size <<= pt_shift(vm, 2); + return size; +} + +static int pt_idx(struct kvm_vm *vm, gva_t gva, int level, u64 *nls) +{ + switch (level) { + case 1: + if (nls) + *nls = 0x9; + return (gva >> 39) & 0x1fff; + case 2: + if (nls) + *nls = 0x9; + return (gva >> 30) & 0x1ff; + case 3: + if (vm->mode == VM_MODE_P52V52_4K) { + if (nls) + *nls = 0x9; + } else { /* vm->mode == VM_MODE_P52V52_64K */ + if (nls) + *nls = 0x5; + } + return (gva >> 21) & 0x1ff; + case 4: + if (vm->mode == VM_MODE_P52V52_4K) + return (gva >> 12) & 0x1ff; + else /* vm->mode == VM_MODE_P52V52_64K */ + return (gva >> 16) & 0x1f; + default: + TEST_ASSERT(false, "Invalid page table level %d\n", level); + return 0; + } +} + +static u64 *virt_get_pte(struct kvm_vm *vm, gpa_t pt, + gva_t gva, int level, u64 *nls) +{ + int idx = pt_idx(vm, gva, level, nls); + u64 *ptep = addr_gpa2hva(vm, pt + idx * 8); + + return ptep; +} + +#define PTE_VALID 0x8000000000000000ull +#define PTE_LEAF 0x4000000000000000ull +#define PTE_REFERENCED 0x0000000000000100ull +#define PTE_CHANGED 0x0000000000000080ull +#define PTE_PRIV 0x0000000000000008ull +#define PTE_READ 0x0000000000000004ull +#define PTE_RW 0x0000000000000002ull +#define PTE_EXEC 0x0000000000000001ull +#define PTE_PAGE_MASK 0x01fffffffffff000ull + +#define PDE_VALID PTE_VALID +#define PDE_NLS 0x0000000000000011ull +#define PDE_PT_MASK 0x0fffffffffffff00ull + +static gpa_t __vm_alloc_pt(struct kvm_vm *vm, u64 pt_shift) +{ + gpa_t pt; + + if (pt_shift >= vm->page_shift) { + size_t pt_pages = 1ULL << (pt_shift - vm->page_shift); + + pt = vm_alloc_page_table_pages(vm, pt_pages); + } else { + struct vm_pt_frag_cache *pt_frag_cache; + + if (pt_shift == 8) { + pt_frag_cache = &vm->arch.pt_frag_cache[0]; + } else if (pt_shift == 12) { + pt_frag_cache = &vm->arch.pt_frag_cache[1]; + } else { + TEST_ASSERT(0, "Invalid pt_shift:%lu\n", pt_shift); + return 0; + } + + if (!pt_frag_cache->page) + pt_frag_cache->page = vm_alloc_page_table(vm); + pt = pt_frag_cache->page + pt_frag_cache->page_nr_used; + pt_frag_cache->page_nr_used += (1 << pt_shift); + if (pt_frag_cache->page_nr_used == vm->page_size) { + pt_frag_cache->page = 0; + pt_frag_cache->page_nr_used = 0; + } + } + + return pt; +} + +void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa) +{ + gpa_t pt = vm->mmu.pgd; + u64 *ptep, pte; + int level; + + for (level = 1; level <= 3; level++) { + u64 nls; + u64 *pdep = virt_get_pte(vm, pt, gva, level, &nls); + u64 pde = be64_to_cpu(*pdep); + + if (pde) { + TEST_ASSERT((pde & PDE_VALID) && !(pde & PTE_LEAF), + "Invalid PDE at level: %u gva: 0x%lx pde:0x%lx\n", + level, gva, pde); + pt = pde & PDE_PT_MASK; + continue; + } + + pt = __vm_alloc_pt(vm, nls + 3); + pde = PDE_VALID | nls | pt; + *pdep = cpu_to_be64(pde); + } + + ptep = virt_get_pte(vm, pt, gva, level, NULL); + pte = be64_to_cpu(*ptep); + + TEST_ASSERT(!pte, "PTE already present at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + pte = PTE_VALID | PTE_LEAF | PTE_REFERENCED | PTE_CHANGED | PTE_PRIV | + PTE_READ | PTE_RW | PTE_EXEC | (gpa & PTE_PAGE_MASK); + *ptep = cpu_to_be64(pte); +} + +gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva) +{ + gpa_t pt = vm->mmu.pgd; + u64 *ptep, pte; + int level; + + for (level = 1; level <= 3; level++) { + u64 nls; + u64 *pdep = virt_get_pte(vm, pt, gva, level, &nls); + u64 pde = be64_to_cpu(*pdep); + + TEST_ASSERT((pde & PDE_VALID) && !(pde & PTE_LEAF), + "PDE not present at level: %u gva: 0x%lx pde:0x%lx\n", + level, gva, pde); + pt = pde & PDE_PT_MASK; + } + + ptep = virt_get_pte(vm, pt, gva, level, NULL); + pte = be64_to_cpu(*ptep); + + TEST_ASSERT(pte, + "PTE not present at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + TEST_ASSERT((pte & PTE_VALID) && (pte & PTE_LEAF), + "PTE not valid at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + return (pte & PTE_PAGE_MASK) + (gva & (vm->page_size - 1)); +} + +static void virt_dump_pt(FILE *stream, struct kvm_vm *vm, gpa_t pt, + gva_t va, int level, u8 indent) +{ + int size, idx; + + size = 1U << (pt_shift(vm, level) + 3); + + for (idx = 0; idx < size; idx += 8, va += pt_entry_coverage(vm, level)) { + u64 *page_table = addr_gpa2hva(vm, pt + idx); + u64 pte = be64_to_cpu(*page_table); + + if (!(pte & PTE_VALID)) + continue; + + if (pte & PTE_LEAF) { + fprintf(stream, + "%*s PTE[%d] gVA:0x%016lx -> gRA:0x%016llx\n", + indent, "", idx / 8, va, pte & PTE_PAGE_MASK); + } else { + fprintf(stream, "%*sPDE%d[%d] gVA:0x%016lx\n", + indent, "", level, idx / 8, va); + virt_dump_pt(stream, vm, pte & PDE_PT_MASK, va, + level + 1, indent + 2); + } + } + +} + +void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent) +{ + gpa_t pt = vm->mmu.pgd; + + if (!vm->mmu.pgd_created) + return; + + virt_dump_pt(stream, vm, pt, 0, 1, indent); +} + +static unsigned long get_r2(void) +{ + unsigned long r2; + + asm("mr %0,%%r2" : "=r"(r2)); + + return r2; +} + +/* + * Under ELFv1 a function pointer is the address of a descriptor holding the + * entry point and the TOC, not the entry point itself. ELFv2 dropped + * descriptors, i.e. the pointer is already the entry point. Dereference the + * descriptor as needed so that the guest starts executing code and not data. + */ +static uintptr_t function_entry(void *fn) +{ +#if defined(_CALL_ELF) && _CALL_ELF == 2 + return (uintptr_t)fn; +#else + return ((const u64 *)fn)[0]; +#endif +} + +void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) +{ + uintptr_t entry = function_entry(guest_code); + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + regs.pc = entry; + regs.gpr[12] = entry; + vcpu_regs_set(vcpu, ®s); +} + +struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) +{ + const size_t stack_size = SZ_64K; + gva_t stack_vaddr, ex_regs_vaddr; + gpa_t ex_regs_paddr; + struct ex_regs *ex_regs; + struct kvm_regs regs; + struct kvm_vcpu *vcpu; + u64 lpcr; + + stack_vaddr = __vm_alloc(vm, stack_size, + DEFAULT_GUEST_STACK_VADDR_MIN, + MEM_REGION_DATA); + + ex_regs_vaddr = __vm_alloc(vm, stack_size, + DEFAULT_GUEST_STACK_VADDR_MIN, + MEM_REGION_DATA); + ex_regs_paddr = addr_gva2gpa(vm, ex_regs_vaddr); + ex_regs = addr_gpa2hva(vm, ex_regs_paddr); + ex_regs->gva = ex_regs_vaddr; + + vcpu = __vm_vcpu_add(vm, vcpu_id); + + vcpu_enable_cap(vcpu, KVM_CAP_PPC_PAPR, 1); + + /* Setup guest registers */ + vcpu_regs_get(vcpu, ®s); + lpcr = vcpu_get_reg(vcpu, KVM_REG_PPC_LPCR_64); + + regs.gpr[1] = stack_vaddr + stack_size - 256; + regs.gpr[2] = (uintptr_t)get_r2(); + regs.gpr[13] = (uintptr_t)ex_regs_paddr; + + regs.msr = MSR_SF | MSR_VEC | MSR_VSX | MSR_FP | + MSR_ME | MSR_IR | MSR_DR | MSR_RI; + + if (BYTE_ORDER == LITTLE_ENDIAN) { + regs.msr |= MSR_LE; + lpcr |= LPCR_ILE; + } else { + lpcr &= ~LPCR_ILE; + } + + vcpu_regs_set(vcpu, ®s); + vcpu_set_reg(vcpu, KVM_REG_PPC_LPCR_64, lpcr); + + return vcpu; +} + +void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) +{ + va_list ap; + struct kvm_regs regs; + int i; + + TEST_ASSERT(num >= 1 && num <= 5, "Unsupported number of args: %u\n", + num); + + va_start(ap, num); + vcpu_regs_get(vcpu, ®s); + + for (i = 0; i < num; i++) + regs.gpr[i + 3] = va_arg(ap, u64); + + vcpu_regs_set(vcpu, ®s); + va_end(ap); +} + +void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) +{ + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + + fprintf(stream, "%*sNIA: 0x%016llx MSR: 0x%016llx\n", + indent, "", regs.pc, regs.msr); + fprintf(stream, "%*sLR: 0x%016llx CTR :0x%016llx\n", + indent, "", regs.lr, regs.ctr); + fprintf(stream, "%*sCR: 0x%08llx XER :0x%016llx\n", + indent, "", regs.cr, regs.xer); +} + +void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) +{ + size_t excp_size = __interrupts_end - __interrupts_start; + size_t excp_pages = (excp_size + vm->page_size - 1) / vm->page_size; + gpa_t excp_paddr; + void *mem; + + /* + * Guest interrupts are taken in real-mode, i.e. the vectors must live + * at gPA 0. Use the innermost allocator to override the minimum GPA + * that MEM_REGION_DATA would otherwise impose. + */ + excp_paddr = ____vm_phy_pages_alloc(vm, excp_pages, 0, + vm->memslots[MEM_REGION_DATA], + vm_arch_has_protected_memory(vm), + false); + + TEST_ASSERT(excp_paddr == 0, + "Interrupt vectors not allocated at gPA address 0: (0x%lx)", + excp_paddr); + + mem = addr_gpa2hva(vm, excp_paddr); + memcpy(mem, __interrupts_start, excp_size); +} + +void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) +{ + struct ucall uc; + + if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED) { + gpa_t ex_regs_paddr; + struct ex_regs *ex_regs; + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + ex_regs_paddr = (gpa_t)regs.gpr[13]; + ex_regs = addr_gpa2hva(vcpu->vm, ex_regs_paddr); + + TEST_FAIL("Unexpected interrupt in guest NIA:0x%016lx MSR:0x%016lx TRAP:0x%04x", + ex_regs->nia, ex_regs->msr, ex_regs->trap); + } +} + +struct handler { + void (*fn)(struct ex_regs *regs); + int trap; +}; + +#define NR_HANDLERS 10 +static struct handler handlers[NR_HANDLERS]; + +void route_interrupt(struct ex_regs *regs) +{ + int i; + + for (i = 0; i < NR_HANDLERS; i++) { + if (handlers[i].trap == regs->trap) { + handlers[i].fn(regs); + return; + } + } + + ucall(UCALL_UNHANDLED, 0); +} + +void vm_install_exception_handler(struct kvm_vm *vm, int trap, + void (*fn)(struct ex_regs *)) +{ + int i; + + for (i = 0; i < NR_HANDLERS; i++) { + if (!handlers[i].trap || handlers[i].trap == trap) { + if (fn == NULL) + trap = 0; /* Clear handler */ + handlers[i].trap = trap; + handlers[i].fn = fn; + sync_global_to_guest(vm, handlers[i]); + return; + } + } + + TEST_FAIL("Out of exception handlers"); +} + +void kvm_selftest_arch_init(void) +{ + TEST_REQUIRE(kvm_has_cap(KVM_CAP_PPC_MMU_RADIX)); + + /* + * powerpc default mode is set by host page size and not static, + * so start by computing that early. + */ + guest_modes_append_default(); +} diff --git a/tools/testing/selftests/kvm/lib/powerpc/ucall.c b/tools/testing/selftests/kvm/lib/powerpc/ucall.c new file mode 100644 index 000000000000..3481a7a0b850 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/ucall.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ucall support. A ucall is a "hypercall to host userspace". + */ +#include "kvm_util.h" +#include "ucall_common.h" +#include "hcall.h" + +void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + if (run->exit_reason == UCALL_EXIT_REASON && + run->papr_hcall.nr == H_UCALL) { + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + if (regs.gpr[4] == UCALL_R4_UCALL) + return (void *)regs.gpr[5]; + } + return NULL; +} From e5983ef19a6a4243ea9e6c7612dd7b59620c8745 Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Tue, 6 Oct 2026 15:58:41 +0530 Subject: [PATCH 474/506] KVM: selftests: Enable access tracking perf test for PPC (with LPAR clause) Enable the access tracking perf test for PPC, but skip the idle page count sanity check when running as a pSeries LPAR, which has the same nested TLB/access bit issue as x86. PowerPC nested KVM (pSeries LPAR) can hide accessed/idle, since KVM (L1) doesn't explicitly flush the TLB for access bit update, so L0 keeps translations cached. That also means that the L2 accesses do not refault and L1 thinks that the pages for L2 memory are still idle. Use kvm_is_ppc_pseries_lpar() check to detect when running nested on powerpc pseries LPAR and only warn on the idle-page count. Reported-by: Anushree Mathur Tested-by: Anushree Mathur Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Gautam Menghani Tested-by: Gautam Menghani Acked-by: Madhavan Srinivasan Link: https://patch.msgid.link/0227000da4e6fe69b278dc971b74b3f65b73ead9.1791280439.git.ritesh.list@gmail.com [sean: massage changelog] Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../testing/selftests/kvm/access_tracking_perf_test.c | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 4b7f6ab0ee54..692bbae44a65 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -257,6 +257,7 @@ TEST_GEN_PROGS_loongarch += set_memory_region_test TEST_GEN_PROGS_loongarch += steal_time TEST_GEN_PROGS_powerpc = $(TEST_GEN_PROGS_COMMON) +TEST_GEN_PROGS_powerpc += access_tracking_perf_test SPLIT_TESTS += arch_timer SPLIT_TESTS += get-reg-list diff --git a/tools/testing/selftests/kvm/access_tracking_perf_test.c b/tools/testing/selftests/kvm/access_tracking_perf_test.c index 4415c94b2866..eb6939065e79 100644 --- a/tools/testing/selftests/kvm/access_tracking_perf_test.c +++ b/tools/testing/selftests/kvm/access_tracking_perf_test.c @@ -471,6 +471,16 @@ static int access_tracking_unreliable(void) return 1; } #endif + /* + * PowerPC: Nested KVM (pSeries LPAR) can hide accessed/idle, since KVM + * doesn't explicitely flush the TLB for access bit update. L0 keeps + * translations cached so the guest won't see the idle bit cleared. + */ + if (kvm_is_ppc_pseries_lpar()) { + puts("Skipping idle page count sanity check, because the test is run nested"); + return 1; + } + /* * When NUMA balancing is enabled, guest memory will be unmapped to get * NUMA faults, dropping the Accessed bits. From 23140d6e54ac31e2f4ef3bcd7fb586a14360be34 Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Tue, 6 Oct 2026 15:58:42 +0530 Subject: [PATCH 475/506] KVM: selftests: Enable hardware disable test PPC (without pSeries LPAR) Enable the hardware disable test for PPC, but skip the test when running on pSeries LPAR. The test forks 512 children, each of which does KVM_CREATE_VM and then SIGKILLs without waiting. On a PAPR LPAR, that ioctl is H_GUEST_CREATE and unreaped guests may still occupy L0 Guest Management Space, which means create can fail with -ENOMEM error. Skip the test for now, as making it play nice with pSeries LPAR is non-trivial, while the test provides meaningful coverage for other flavors of PPC. Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Gautam Menghani Tested-by: Gautam Menghani Acked-by: Madhavan Srinivasan Link: https://patch.msgid.link/23f36aab5e9c849719ffeb796e1a74e69eb5d142.1791280439.git.ritesh.list@gmail.com [sean: massage changelog] Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 1 + tools/testing/selftests/kvm/hardware_disable_test.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 692bbae44a65..38c3900cac68 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -258,6 +258,7 @@ TEST_GEN_PROGS_loongarch += steal_time TEST_GEN_PROGS_powerpc = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_powerpc += access_tracking_perf_test +TEST_GEN_PROGS_powerpc += hardware_disable_test SPLIT_TESTS += arch_timer SPLIT_TESTS += get-reg-list diff --git a/tools/testing/selftests/kvm/hardware_disable_test.c b/tools/testing/selftests/kvm/hardware_disable_test.c index 43a36ef3ead8..0104b7e840b5 100644 --- a/tools/testing/selftests/kvm/hardware_disable_test.c +++ b/tools/testing/selftests/kvm/hardware_disable_test.c @@ -130,6 +130,9 @@ int main(int argc, char **argv) int s, r, cpu, i; pid_t pid; + /* PowerPC PAPR L0 cannot guarantee 512 overlapping H_GUEST_CREATE from fork+SIGKILL. */ + TEST_REQUIRE(!kvm_is_ppc_pseries_lpar()); + kvm_sched_getaffinity(0, sizeof(cpu_set_t), &allowed_cpu_set); for (i = 0; i < NR_VCPUS && CPU_COUNT(&allowed_cpu_set); i++) { From 71cf35e389b1b5a8851762ec7c74fded53342cc6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 12 Oct 2026 19:48:31 +0200 Subject: [PATCH 476/506] KVM: uapi: remove dead #defines These #defines were used for the old KVM-specific device assignment and have had no in-tree use for more than 10 years. Drop them. Link: https://patch.msgid.link/kvm/20261012174847.147915-1-pbonzini@redhat.com/ Signed-off-by: Paolo Bonzini --- arch/x86/include/uapi/asm/kvm.h | 2 -- tools/arch/x86/include/uapi/asm/kvm.h | 2 -- 2 files changed, 4 deletions(-) diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 1585ec804066..00ce265391e1 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -45,9 +45,7 @@ #define __KVM_HAVE_PIT #define __KVM_HAVE_IOAPIC #define __KVM_HAVE_IRQ_LINE -#define __KVM_HAVE_MSI #define __KVM_HAVE_USER_NMI -#define __KVM_HAVE_MSIX #define __KVM_HAVE_MCE #define __KVM_HAVE_PIT_STATE2 #define __KVM_HAVE_XEN_HVM diff --git a/tools/arch/x86/include/uapi/asm/kvm.h b/tools/arch/x86/include/uapi/asm/kvm.h index 1585ec804066..00ce265391e1 100644 --- a/tools/arch/x86/include/uapi/asm/kvm.h +++ b/tools/arch/x86/include/uapi/asm/kvm.h @@ -45,9 +45,7 @@ #define __KVM_HAVE_PIT #define __KVM_HAVE_IOAPIC #define __KVM_HAVE_IRQ_LINE -#define __KVM_HAVE_MSI #define __KVM_HAVE_USER_NMI -#define __KVM_HAVE_MSIX #define __KVM_HAVE_MCE #define __KVM_HAVE_PIT_STATE2 #define __KVM_HAVE_XEN_HVM From 1499fd7cd2e141c5cda4013fb4ed828d34efedcc Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Tue, 13 Oct 2026 14:13:43 +0200 Subject: [PATCH 477/506] KVM: Move architecture capability Kconfigs to header defines With s390 hosting multiple independent KVM implementations within the same kernel build (native s390 KVM and arm64 on s390), global Kconfig selections force both implementations to inherit options that only apply to one of them. Replace global Kconfig options which only apply to one KVM implementation with architecture-specific opt-in defines in the respective . Only remove Kconfig symbols that would be different on both s390 KVM implementations - __KVM_HAVE_INVALID_WAKEUPS (formerly HAVE_KVM_INVALID_WAKEUPS) - __KVM_HAVE_MSI (formerly HAVE_KVM_MSI) - __KVM_HAVE_NO_POLL (formerly HAVE_KVM_NO_POLL) - __KVM_HAVE_READONLY_MEM (formerly HAVE_KVM_READONLY_MEM) - __KVM_HAVE_VCPU_RUN_PID_CHANGE (formerly HAVE_KVM_VCPU_RUN_PID_CHANGE) This allows each architecture and guest implementation to configure its supported capabilities independently in its kvm_host header and eliminates adding unnecessary fallback stubs across different KVM targets. Acked-by: Hendrik Brueckner Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20261013121343.1997112-1-seiden@linux.ibm.com Signed-off-by: Paolo Bonzini --- arch/arm64/include/asm/kvm_host.h | 3 +++ arch/arm64/kvm/Kconfig | 3 --- arch/loongarch/include/asm/kvm_host.h | 2 ++ arch/loongarch/kvm/Kconfig | 2 -- arch/mips/include/asm/kvm_host.h | 1 + arch/mips/kvm/Kconfig | 1 - arch/powerpc/include/asm/kvm_host.h | 3 +++ arch/powerpc/kvm/Kconfig | 1 - arch/riscv/include/asm/kvm_host.h | 2 ++ arch/riscv/kvm/Kconfig | 2 -- arch/s390/include/asm/kvm_host_s390.h | 3 +++ arch/s390/kvm/Kconfig | 2 -- arch/x86/include/asm/kvm_host.h | 3 +++ arch/x86/kvm/Kconfig | 3 --- include/linux/kvm_host.h | 18 +++++++++++------- virt/kvm/Kconfig | 15 --------------- virt/kvm/kvm_main.c | 4 ++-- 17 files changed, 30 insertions(+), 38 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..41e75b8fbc3b 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -30,6 +30,9 @@ #include #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM +#define __KVM_HAVE_VCPU_RUN_PID_CHANGE #define KVM_HALT_POLL_NS_DEFAULT 500000 diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 449154f9a485..7329124deb2f 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -28,12 +28,9 @@ menuconfig KVM select KVM_VFIO select HAVE_KVM_DIRTY_RING_ACQ_REL select NEED_KVM_DIRTY_RING_WITH_BITMAP - select HAVE_KVM_MSI select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_IRQ_BYPASS - select HAVE_KVM_READONLY_MEM - select HAVE_KVM_VCPU_RUN_PID_CHANGE select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS select KVM_GUEST_MEMFD diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 65d91c3ce313..79d3ce7ad029 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -26,6 +26,8 @@ #include #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM /* Loongarch KVM register ids */ #define KVM_GET_IOC_CSR_IDX(id) ((id & KVM_CSR_IDX_MASK) >> LOONGARCH_REG_SHIFT) diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig index 15da2d88c0c1..a427429a2b05 100644 --- a/arch/loongarch/kvm/Kconfig +++ b/arch/loongarch/kvm/Kconfig @@ -23,8 +23,6 @@ config KVM select HAVE_KVM_DIRTY_RING_ACQ_REL select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_IRQCHIP - select HAVE_KVM_MSI - select HAVE_KVM_READONLY_MEM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index c14b10821817..d481275f38eb 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -893,5 +893,6 @@ static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS +#define __KVM_HAVE_READONLY_MEM #endif /* __MIPS_KVM_HOST_H__ */ diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index b1b9a1d67758..c24b872afbcb 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -24,7 +24,6 @@ config KVM select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_MMIO select KVM_GENERIC_HARDWARE_ENABLING - select HAVE_KVM_READONLY_MEM help Support for hosting Guest kernels. diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 2d139c807577..d8ef70c14aad 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -49,6 +49,9 @@ #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#ifdef CONFIG_KVM_MPIC +#define __KVM_HAVE_MSI +#endif #define KVM_HALT_POLL_NS_DEFAULT 10000 /* 10 us */ diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index b6bc2fc86dca..1fae2de8fa68 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -232,7 +232,6 @@ config KVM_MPIC depends on KVM && PPC_E500 select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_MSI help Enable support for emulating MPIC devices inside the host kernel, rather than relying on userspace to emulate. diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index a30600579231..e28f058a94d7 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -47,6 +47,8 @@ #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(6) #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM #define KVM_HAVE_MMU_RWLOCK diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index ec2cee0a39e0..3cb4f8037300 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -22,8 +22,6 @@ config KVM depends on RISCV_SBI && MMU select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_MSI - select HAVE_KVM_READONLY_MEM select HAVE_KVM_DIRTY_RING_ACQ_REL select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h index 9519b8028b10..a00353ba0343 100644 --- a/arch/s390/include/asm/kvm_host_s390.h +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -34,6 +34,9 @@ #define KVM_INTERNAL_MEM_SLOTS 1 +#define __KVM_HAVE_INVALID_WAKEUPS +#define __KVM_HAVE_NO_POLL + #define KVM_S390_MANAGES_S390_GUEST 1 /* diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 8d3ee17a1bcb..9045ae20fd89 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -25,8 +25,6 @@ config KVM select KVM_COMMON select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_INVALID_WAKEUPS - select HAVE_KVM_NO_POLL select KVM_VFIO select VIRT_XFER_TO_GUEST_WORK select KVM_MMU_LOCKLESS_AGING diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 0e827dcd0b88..a902b055c7f2 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -48,6 +48,9 @@ #define kvm_file_to_kvm_arch x86 #define __KVM_HAVE_ARCH_VCPU_DEBUGFS +#define __KVM_HAVE_MSI +#define __KVM_HAVE_NO_POLL +#define __KVM_HAVE_READONLY_MEM /* * CONFIG_KVM_MAX_NR_VCPUS is defined iff CONFIG_KVM!=n, provide a dummy max if diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 538ed1e80332..294684b217bb 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -28,7 +28,6 @@ config KVM_X86 select HAVE_KVM_DIRTY_RING_ACQ_REL select HAVE_KVM_IRQ_BYPASS select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_READONLY_MEM select VHOST_TASK select KVM_ASYNC_PF select USER_RETURN_NOTIFIER @@ -37,9 +36,7 @@ config KVM_X86 select PERF_EVENTS select GUEST_PERF_EVENTS select PERF_GUEST_MEDIATED_PMU - select HAVE_KVM_MSI select HAVE_KVM_CPU_RELAX_INTERCEPT - select HAVE_KVM_NO_POLL select VIRT_XFER_TO_GUEST_WORK select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_VFIO diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 5624b572cb85..e4ae201a1707 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -750,7 +750,11 @@ static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) #ifndef kvm_arch_has_readonly_mem static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm) { - return IS_ENABLED(CONFIG_HAVE_KVM_READONLY_MEM); +#ifdef __KVM_HAVE_READONLY_MEM + return true; +#else + return false; +#endif } #endif @@ -2483,7 +2487,7 @@ void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd, struct kvm_kernel_irq_routing_entry *new); #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ -#ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS +#ifdef __KVM_HAVE_INVALID_WAKEUPS /* If we wakeup during the poll time, was it a sucessful poll? */ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) { @@ -2495,9 +2499,9 @@ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) { return true; } -#endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */ +#endif /* __KVM_HAVE_INVALID_WAKEUPS */ -#ifdef CONFIG_HAVE_KVM_NO_POLL +#ifdef __KVM_HAVE_NO_POLL /* Callback that tells if we must not poll */ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu); #else @@ -2505,18 +2509,18 @@ static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) { return false; } -#endif /* CONFIG_HAVE_KVM_NO_POLL */ +#endif /* __KVM_HAVE_NO_POLL */ void kvm_arch_guest_memory_reclaimed(struct kvm *kvm); -#ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE +#ifdef __KVM_HAVE_VCPU_RUN_PID_CHANGE int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu); #else static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) { return 0; } -#endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */ +#endif /* __KVM_HAVE_VCPU_RUN_PID_CHANGE */ #ifdef CONFIG_VIRT_XFER_TO_GUEST_WORK static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu) diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index c3c0ee253fc7..f3d7f66746c8 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -50,21 +50,12 @@ config KVM_ASYNC_PF config KVM_ASYNC_PF_SYNC bool -config HAVE_KVM_MSI - bool - -config HAVE_KVM_READONLY_MEM - bool - config HAVE_KVM_CPU_RELAX_INTERCEPT bool config KVM_VFIO bool -config HAVE_KVM_INVALID_WAKEUPS - bool - config KVM_GENERIC_DIRTYLOG_READ_PROTECT bool @@ -79,12 +70,6 @@ config HAVE_KVM_IRQ_BYPASS tristate select IRQ_BYPASS_MANAGER -config HAVE_KVM_VCPU_RUN_PID_CHANGE - bool - -config HAVE_KVM_NO_POLL - bool - config VIRT_XFER_TO_GUEST_WORK bool diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 09ee08092ced..fe6dc0bf3bde 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4903,7 +4903,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: case KVM_CAP_INTERNAL_ERROR_DATA: -#ifdef CONFIG_HAVE_KVM_MSI +#ifdef __KVM_HAVE_MSI case KVM_CAP_SIGNAL_MSI: #endif #ifdef CONFIG_HAVE_KVM_IRQCHIP @@ -5293,7 +5293,7 @@ static long kvm_vm_ioctl(struct file *filp, r = kvm_ioeventfd(kvm, &data); break; } -#ifdef CONFIG_HAVE_KVM_MSI +#ifdef __KVM_HAVE_MSI case KVM_SIGNAL_MSI: { struct kvm_msi msi; From b62a0bf95825aab317cc09654602c7e934decee3 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:18 +0200 Subject: [PATCH 478/506] KVM: Replace CONFIG_KVM_MMIO with __KVM_NO_MMIO Defining KVM_MMIO is not flexible enough for multi-KVM systems where one KVM implements MMIO but others do not. Therefore, remove KVM_MMIO from the config space and use the macro __KVM_NO_MMIO instead. Define __KVM_NO_MMIO for KVM/s390 and for HV-only 64-bit Book3s (aka pSeries/sPAPR) configurations. In particular, for PowerPC configurations: * KVM_BOOK3S_32 got KVM_MMIO via KVM_BOOK3S_32_HANDLER and KVM_BOOK3S_PR_POSSIBLE * KVM_BOOK3S_64 got KVM_MMIO via KVM_BOOK3S_PR_POSSIBLE if !KVM_BOOK3S_HV_POSSIBLE * KVM_BOOK3S_64 + KVM_BOOK3S_64_PR got KVM_MMIO via KVM_BOOK3S_PR_POSSIBLE * likewise for KVM_BOOK3S_64 + KVM_BOOK3S_64_PR + KVM_BOOK3S_64_HV * KVM_E500V2 and KVM_E500MC directly selected KVM_MMIO but KVM_BOOK3S_64 + KVM_BOOK3S_64_HV didn't and therefore needs __KVM_NO_MMIO. coalesced_mmio.c is now always compiled and linked. However, for architectures/configurations defining __KVM_NO_MMIO this effectively collapses to an empty object with no mmio code. No functional changes. Suggested-by: Marc Zyngier Signed-off-by: Steffen Eiden Link: https://patch.msgid.link/20260930173441.66762-4-seiden@linux.ibm.com [Simplify powerpc condition, add underscores. - Paolo] Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/Kconfig | 1 - arch/loongarch/kvm/Kconfig | 1 - arch/mips/kvm/Kconfig | 1 - arch/powerpc/include/asm/kvm_host.h | 4 ++++ arch/powerpc/kvm/Kconfig | 4 ---- arch/riscv/kvm/Kconfig | 1 - arch/s390/include/asm/kvm_host_s390.h | 2 ++ arch/x86/kvm/Kconfig | 1 - include/linux/kvm_host.h | 2 +- virt/kvm/Kconfig | 3 --- virt/kvm/Makefile.kvm | 4 ++-- virt/kvm/coalesced_mmio.c | 4 ++++ virt/kvm/coalesced_mmio.h | 2 +- virt/kvm/kvm_main.c | 8 ++++---- 14 files changed, 18 insertions(+), 20 deletions(-) diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 7329124deb2f..8475f32ad9e6 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -22,7 +22,6 @@ menuconfig KVM select KVM_COMMON select KVM_GENERIC_HARDWARE_ENABLING select HAVE_KVM_CPU_RELAX_INTERCEPT - select KVM_MMIO select KVM_GENERIC_DIRTYLOG_READ_PROTECT select VIRT_XFER_TO_GUEST_WORK select KVM_VFIO diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig index a427429a2b05..fd387326d2f2 100644 --- a/arch/loongarch/kvm/Kconfig +++ b/arch/loongarch/kvm/Kconfig @@ -26,7 +26,6 @@ config KVM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING - select KVM_MMIO select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index c24b872afbcb..849ce7c7c4df 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -22,7 +22,6 @@ config KVM select EXPORT_UASM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT - select KVM_MMIO select KVM_GENERIC_HARDWARE_ENABLING help Support for hosting Guest kernels. diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index d8ef70c14aad..708a32f89776 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -29,6 +29,10 @@ #define __KVM_HAVE_ARCH_VCPU_DEBUGFS +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) && !defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) +#define __KVM_NO_MMIO +#endif + #define KVM_MAX_VCPUS NR_CPUS #define KVM_MAX_VCORES NR_CPUS diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 1fae2de8fa68..cacc22e34afd 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -30,7 +30,6 @@ config KVM_BOOK3S_HANDLER config KVM_BOOK3S_32_HANDLER bool select KVM_BOOK3S_HANDLER - select KVM_MMIO config KVM_BOOK3S_64_HANDLER bool @@ -38,7 +37,6 @@ config KVM_BOOK3S_64_HANDLER config KVM_BOOK3S_PR_POSSIBLE bool - select KVM_MMIO config KVM_BOOK3S_HV_POSSIBLE bool @@ -201,7 +199,6 @@ config KVM_E500V2 depends on PPC_E500 && !PPC_E500MC depends on !CONTEXT_TRACKING_USER select KVM - select KVM_MMIO help Support running unmodified E500 guest kernels in virtual machines on E500v2 host processors. @@ -216,7 +213,6 @@ config KVM_E500MC depends on PPC_E500MC depends on !CONTEXT_TRACKING_USER select KVM - select KVM_MMIO select KVM_BOOKE_HV help Support running unmodified E500MC/E5500/E6500 guest kernels in diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index 3cb4f8037300..2e3c4e6c1cd7 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -26,7 +26,6 @@ config KVM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING - select KVM_MMIO select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h index a00353ba0343..f7802600b3fb 100644 --- a/arch/s390/include/asm/kvm_host_s390.h +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -39,6 +39,8 @@ #define KVM_S390_MANAGES_S390_GUEST 1 +#define __KVM_NO_MMIO + /* * These seem to be used for allocating ->chip in the routing table, which we * don't use. 1 is as small as we can get to reduce the needed memory. If we diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 294684b217bb..42e79c3c87ff 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -31,7 +31,6 @@ config KVM_X86 select VHOST_TASK select KVM_ASYNC_PF select USER_RETURN_NOTIFIER - select KVM_MMIO select SCHED_INFO select PERF_EVENTS select GUEST_PERF_EVENTS diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index e4ae201a1707..c8a29162b3f9 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -847,7 +847,7 @@ struct kvm { struct kvm_vm_stat stat; struct kvm_arch arch; refcount_t users_count; -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; spinlock_t ring_lock; struct list_head coalesced_zones; diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index f3d7f66746c8..24d7dfceb828 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -40,9 +40,6 @@ config NEED_KVM_DIRTY_RING_WITH_BITMAP bool depends on HAVE_KVM_DIRTY_RING -config KVM_MMIO - bool - config KVM_ASYNC_PF bool diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm index b39f7b2a5f6e..ec7da5942354 100644 --- a/virt/kvm/Makefile.kvm +++ b/virt/kvm/Makefile.kvm @@ -10,9 +10,9 @@ KVM_DEV_MINOR ?= KVM_MINOR ccflags-y +=-DKVM_DEV_NAME=$(KVM_DEV_NAME) -DKVM_DEV_MINOR=$(KVM_DEV_MINOR) -kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o +kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o \ + $(KVM)/coalesced_mmio.o kvm-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o -kvm-$(CONFIG_KVM_MMIO) += $(KVM)/coalesced_mmio.o kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o diff --git a/virt/kvm/coalesced_mmio.c b/virt/kvm/coalesced_mmio.c index 6b1d90161099..3b5430036cae 100644 --- a/virt/kvm/coalesced_mmio.c +++ b/virt/kvm/coalesced_mmio.c @@ -15,6 +15,8 @@ #include #include +#ifndef __KVM_NO_MMIO + #include "coalesced_mmio.h" static inline struct kvm_coalesced_mmio_dev *to_mmio(struct kvm_io_device *dev) @@ -188,3 +190,5 @@ int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm, */ return 0; } + +#endif /* !__KVM_NO_MMIO */ diff --git a/virt/kvm/coalesced_mmio.h b/virt/kvm/coalesced_mmio.h index 36f84264ed25..77b811dee27c 100644 --- a/virt/kvm/coalesced_mmio.h +++ b/virt/kvm/coalesced_mmio.h @@ -11,7 +11,7 @@ * */ -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO #include diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index fe6dc0bf3bde..ba546c1a92f5 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4073,7 +4073,7 @@ static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf) else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) page = virt_to_page(vcpu->arch.pio_data); #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); #endif @@ -4914,7 +4914,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_ENABLE_CAP_VM: case KVM_CAP_HALT_POLL: return 1; -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO case KVM_CAP_COALESCED_MMIO: return KVM_COALESCED_MMIO_PAGE_OFFSET; case KVM_CAP_COALESCED_PIO: @@ -5255,7 +5255,7 @@ static long kvm_vm_ioctl(struct file *filp, break; } #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO case KVM_REGISTER_COALESCED_MMIO: { struct kvm_coalesced_mmio_zone zone; @@ -5584,7 +5584,7 @@ static long kvm_dev_ioctl(struct file *filp, #ifdef CONFIG_X86 r += PAGE_SIZE; /* pio data page */ #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO r += PAGE_SIZE; /* coalesced mmio ring page */ #endif break; From 22f74df7ac575c43c4dd1e2d4cbae65ff84a3595 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 15 Sep 2026 16:43:07 +0200 Subject: [PATCH 479/506] s390/uv: Rework page allocation for guest variable storage area Alexander Gordeev reported that the allocation of pages of the guest variable storage area via the uv_alloc_range_cb() callback of apply_to_page_range() will cause problems as soon as s390 fully supports lazy mmu mode. Problem is that uv_alloc_range_cb() allocates pages with GFP_KERNEL while preemption would be disabled. This would result in reports like this: BUG: sleeping function called from invalid context at ./include/linux/sched/mm.h:322 in_atomic(): 1, irqs_disabled(): 0, non_block: 0, pid: 6092, name: qemu-kvm preempt_count: 1, expected: 0 ... Call Trace: dump_stack_lvl+0xae/0x108 __might_resched+0x1de/0x2f0 prepare_alloc_pages+0x1ba/0x220 __alloc_frozen_pages_noprof+0xc6/0x390 alloc_pages_mpol+0xe6/0x220 alloc_frozen_pages_noprof+0x5c/0x80 alloc_pages_noprof+0x22/0x80 uv_alloc_range_cb+0x30/0x300 apply_to_pte_range+0x11a/0x3b0 apply_to_pmd_range+0x13a/0x250 __apply_to_page_range+0x232/0x4d0 apply_to_page_range+0x28/0x40 uv_alloc_stor_var+0x5e/0x90 kvm_s390_pv_alloc_vm+0x118/0x1e0 [kvm] kvm_s390_pv_init_vm+0x84/0x2e0 [kvm] kvm_s390_handle_pv+0x502/0xf70 [kvm] kvm_arch_vm_ioctl+0x234/0xdd0 [kvm] kvm_vm_ioctl+0x33a/0x890 [kvm] __s390x_sys_ioctl+0xfa/0x130 __do_syscall+0x1fc/0x700 system_call+0x72/0x90 Preallocate the pages and make use of vm_area_map_pages() to address this potential future bug. Reported-by: Alexander Gordeev Closes: https://lore.kernel.org/all/e9449c63-b8d2-4ed9-a184-7dcab3e8dfce-agordeev@linux.ibm.com Signed-off-by: Heiko Carstens Acked-by: Alexander Gordeev Signed-off-by: Janosch Frank Signed-off-by: Christian Borntraeger --- arch/s390/kernel/uv.c | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 8ea9dd7704ff..abb94720c79b 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -242,34 +243,38 @@ void uv_free_stor_var(void *stor_var) } EXPORT_SYMBOL_FOR_MODULES(uv_free_stor_var, "kvm"); -static int uv_alloc_range_cb(pte_t *ptep, unsigned long addr, void *data) -{ - struct page *page; - pte_t pte; - - page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!page) - return -ENOMEM; - pte = __pte(page_to_phys(page) | pgprot_val(PAGE_KERNEL)); - set_pte(ptep, pte); - return 0; -} - void *uv_alloc_stor_var(unsigned long size) { + unsigned long i, nr_pages, addr; struct vm_struct *area; - unsigned long addr; + struct page **pages; size = PAGE_ALIGN(size); + nr_pages = size >> PAGE_SHIFT; area = get_vm_area(size, VM_SPARSE); if (!area) return NULL; + pages = kvcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL_ACCOUNT); + if (!pages) + goto free_area; + for (i = 0; i < nr_pages; i++) { + pages[i] = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!pages[i]) + goto free_pages; + } addr = (unsigned long)area->addr; - if (apply_to_page_range(&init_mm, addr, size, uv_alloc_range_cb, NULL)) - goto out; + if (vm_area_map_pages(area, addr, addr + size, pages)) + goto free_pages; + kvfree(pages); return area->addr; -out: - uv_free_stor_var(area->addr); +free_pages: + for (i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kvfree(pages); +free_area: + free_vm_area(area); return NULL; } EXPORT_SYMBOL_FOR_MODULES(uv_alloc_stor_var, "kvm"); From b33cbce8a58383e127e44c552b0e7738e2afc091 Mon Sep 17 00:00:00 2001 From: Gabriella Seifert Date: Thu, 20 Aug 2026 12:20:35 +0000 Subject: [PATCH 480/506] KVM: s390: selftests: Add IRQ injection selftests The IRQ injection selftests haven't been added to the selftests yet so let's do that now. We start off with a couple simple error checks for invalid data. Signed-off-by: Gabriella Seifert [frankja@linux.ibm.com: Improved formatting and wording] Signed-off-by: Janosch Frank Signed-off-by: Christian Borntraeger --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/s390/irq_injection.c | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 tools/testing/selftests/kvm/s390/irq_injection.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index fa10eb9e8fa7..fc409ebbde38 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -225,6 +225,7 @@ TEST_GEN_PROGS_s390 += rseq_test TEST_GEN_PROGS_s390 += s390/irq_routing TEST_GEN_PROGS_s390 += mmu_stress_test TEST_GEN_PROGS_s390 += pre_fault_memory_test +TEST_GEN_PROGS_s390 += s390/irq_injection TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_riscv += riscv/sbi_pmu_test diff --git a/tools/testing/selftests/kvm/s390/irq_injection.c b/tools/testing/selftests/kvm/s390/irq_injection.c new file mode 100644 index 000000000000..75e0528f9b01 --- /dev/null +++ b/tools/testing/selftests/kvm/s390/irq_injection.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Test for KVM_S390_IRQ - s390x IRQ injection API + * + * Verifies that VM-level interrupt types are correctly rejected when + * injected via the vCPU file descriptor, and that invalid arguments + * to vCPU-level interrupts are also rejected. + * + * Copyright IBM Corp. 2026 + * Author: Gabriella Seifert + */ +#include +#include +#include +#include +#include +#include +#include "kvm_util.h" +#include "test_util.h" +#include + +static void test_vm_irq_on_vcpu_fails(struct kvm_vcpu *vcpu, __u64 type) +{ + struct kvm_s390_irq irq = { .type = type }; + int rc; + + rc = __vcpu_ioctl(vcpu, KVM_S390_IRQ, &irq); + ksft_test_result(rc == -1 && errno == EINVAL, + "VM on VCPU IRQ type 0x%llx: expected %d, got %d\n", + type, EINVAL, errno); +} + +static void test_vcpu_irq_bad_param_fails(struct kvm_vcpu *vcpu, + struct kvm_s390_irq *irq) +{ + int rc; + + rc = __vcpu_ioctl(vcpu, KVM_S390_IRQ, irq); + ksft_test_result(rc == -1 && errno == EINVAL, + "VCPU IRQ bad param type 0x%llx: expected %d got %d\n", + irq->type, EINVAL, errno); +} + +int main(void) +{ + struct kvm_s390_irq irq; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + + ksft_print_header(); + ksft_set_plan(6); + vm = vm_create_with_one_vcpu(&vcpu, NULL); + + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_VIRTIO); + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_SERVICE); + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_IO_MIN); + + memset(&irq, 0, sizeof(irq)); + irq.type = KVM_S390_INT_EMERGENCY; + irq.u.emerg.code = 0xffff; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + memset(&irq, 0, sizeof(irq)); + irq.type = KVM_S390_INT_EXTERNAL_CALL; + irq.u.extcall.code = 0xffff; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + memset(&irq, 0, sizeof(irq)); + irq.type = 0xdeadbeef; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + kvm_vm_free(vm); + ksft_finished(); +} From 6079170e9cb52e5cfa3efc589ef065290e3e87ad Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Sun, 6 Sep 2026 11:27:12 +0300 Subject: [PATCH 481/506] KVM: s390: Replace get_zeroed_page() with kzalloc() for sie_page2 and CMMA kvm_arch_init_vm() allocates struct sie_page2 that holds the facility list and the crypto control block of a VM and kvm_s390_vcpu_setup_cmma() allocates the CMMA collaborative memory management block referenced by the SIE control block. Both are passed to the hardware as physical addresses and must be page aligned. kmalloc() guarantees that a power of two sized allocation is aligned to its size, so a PAGE_SIZE allocation is page aligned as well. These buffers can be allocated with kmalloc() as there's nothing special about them to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. The sie_page allocated in kvm_arch_vcpu_create() is left alone because it is mapped to userspace with virt_to_page() in kvm_arch_vcpu_fault() and therefore really needs a struct page. The SIE hardware stores the offset of the next CBR entry in the low bits of cbrlo, so mask them out before converting cbrlo back to a virtual address. free_page() did not care, kfree() needs the address kzalloc() returned. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Claudio Imbrenda Reviewed-by: Janosch Frank Signed-off-by: Janosch Frank Signed-off-by: Christian Borntraeger --- arch/s390/kvm/s390/s390.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index b14706c70946..305a3ccaa25a 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -3273,8 +3273,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) goto out_err; BUILD_BUG_ON(sizeof(struct sie_page2) != 4096); - kvm->arch.sie_page2 = - (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA); + kvm->arch.sie_page2 = kzalloc_obj(*kvm->arch.sie_page2, + GFP_KERNEL_ACCOUNT | GFP_DMA); if (!kvm->arch.sie_page2) goto out_err; @@ -3369,7 +3369,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) return 0; out_err: - free_page((unsigned long)kvm->arch.sie_page2); + kfree(kvm->arch.sie_page2); debug_unregister(kvm->arch.dbf); sca_dispose(kvm); KVM_EVENT(3, "creation of vm failed: %d", rc); @@ -3427,7 +3427,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm); debug_unregister(kvm->arch.dbf); - free_page((unsigned long)kvm->arch.sie_page2); + kfree(kvm->arch.sie_page2); kvm_s390_destroy_adapters(kvm); kvm_s390_clear_float_irqs(kvm); kvm_s390_vsie_destroy(kvm); @@ -3657,13 +3657,13 @@ static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) { if (vcpu->arch.sie_block->cbrlo) - free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); + kfree(phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK)); vcpu->arch.sie_block->cbrlo = 0; } int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) { - void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + void *cbrlo_page = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!cbrlo_page) return -ENOMEM; From 603695afbeee031711cfa944519a5b5c00607e53 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Sun, 6 Sep 2026 11:27:11 +0300 Subject: [PATCH 482/506] KVM: s390: Replace get_zeroed_page() with kzalloc() for the GIB kvm_s390_gib_init() allocates the guest information block (GIB). The GIB is passed to the hardware as a physical address and must be page aligned. kmalloc() guarantees that a power of two sized allocation is aligned to its size, so a PAGE_SIZE allocation is page aligned as well. This buffer can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Claudio Imbrenda Reviewed-by: Janosch Frank Signed-off-by: Janosch Frank Signed-off-by: Christian Borntraeger --- arch/s390/kvm/s390/interrupt.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c index f892f4307883..27b5fdf7ee75 100644 --- a/arch/s390/kvm/s390/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -3690,7 +3690,7 @@ void kvm_s390_gib_destroy(void) } chsc_sgib(0); unregister_adapter_interrupt(&gib_alert_irq); - free_page((unsigned long)gib); + kfree(gib); gib = NULL; } @@ -3704,7 +3704,7 @@ int __init kvm_s390_gib_init(u8 nisc) goto out; } - gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA); + gib = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT | GFP_DMA); if (!gib) { rc = -ENOMEM; goto out; @@ -3723,7 +3723,7 @@ int __init kvm_s390_gib_init(u8 nisc) gib_origin = virt_to_phys(gib); if (chsc_sgib(gib_origin)) { pr_err("Associating the GIB with the AIV facility failed\n"); - free_page((unsigned long)gib); + kfree(gib); gib = NULL; rc = -EIO; goto out_unreg_gal; @@ -3743,7 +3743,7 @@ int __init kvm_s390_gib_init(u8 nisc) out_unreg_gal: unregister_adapter_interrupt(&gib_alert_irq); out_free_gib: - free_page((unsigned long)gib); + kfree(gib); gib = NULL; out: return rc; From f10a817b3ab1db7b7f912a6c57e60f93ceddfa12 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Sun, 6 Sep 2026 11:27:10 +0300 Subject: [PATCH 483/506] KVM: s390: Replace get_zeroed_page() with kzalloc() for the STSI buffer handle_stsi() allocates the buffer that receives the STSI response block before it is copied to the guest. This buffer can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. While on it, make the buffer a void pointer to get rid of the casts. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Janosch Frank Reviewed-by: Claudio Imbrenda Signed-off-by: Janosch Frank Signed-off-by: Christian Borntraeger --- arch/s390/kvm/s390/priv.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/arch/s390/kvm/s390/priv.c b/arch/s390/kvm/s390/priv.c index b3cb2c2c3aa7..856939acf012 100644 --- a/arch/s390/kvm/s390/priv.c +++ b/arch/s390/kvm/s390/priv.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -869,7 +870,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu) int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; int sel1 = vcpu->run->s.regs.gprs[0] & 0xff; int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff; - unsigned long mem = 0; + void *mem = NULL; u64 operand2; int rc = 0; u8 ar; @@ -911,19 +912,19 @@ static int handle_stsi(struct kvm_vcpu *vcpu) switch (fc) { case 1: /* same handling for 1 and 2 */ case 2: - mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); + mem = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!mem) goto out_no_data; - if (stsi((void *) mem, fc, sel1, sel2)) + if (stsi(mem, fc, sel1, sel2)) goto out_no_data; break; case 3: if (sel1 != 2 || sel2 != 2) goto out_no_data; - mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); + mem = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!mem) goto out_no_data; - handle_stsi_3_2_2(vcpu, (void *) mem); + handle_stsi_3_2_2(vcpu, mem); break; case 15: /* fc 15 is fully handled in userspace */ insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); @@ -931,10 +932,10 @@ static int handle_stsi(struct kvm_vcpu *vcpu) return -EREMOTE; } if (kvm_s390_pv_cpu_is_protected(vcpu)) { - memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE); + memcpy(sida_addr(vcpu->arch.sie_block), mem, PAGE_SIZE); rc = 0; } else { - rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); + rc = write_guest(vcpu, operand2, ar, mem, PAGE_SIZE); } if (rc) { rc = kvm_s390_inject_prog_cond(vcpu, rc); @@ -945,14 +946,14 @@ static int handle_stsi(struct kvm_vcpu *vcpu) rc = -EREMOTE; } trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); - free_page(mem); + kfree(mem); kvm_s390_set_psw_cc(vcpu, 0); vcpu->run->s.regs.gprs[0] = 0; return rc; out_no_data: kvm_s390_set_psw_cc(vcpu, 3); out: - free_page(mem); + kfree(mem); return rc; } From 60fbfe3af7a66e38b1ceeaa7722ba06be15e4fcc Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Sun, 6 Sep 2026 11:27:09 +0300 Subject: [PATCH 484/506] KVM: s390: Replace get_zeroed_page() with kzalloc() for the STHYI buffer handle_sthyi() allocates the buffer that receives the STHYI response block before it is copied to the guest. This buffer can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. The only goto in that function does not unwind anything, it merely skips to the exit when the function code is invalid. Turn it into an early return so that the buffer can be freed with __free(kfree). Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Janosch Frank Reviewed-by: Claudio Imbrenda Signed-off-by: Janosch Frank Signed-off-by: Christian Borntraeger --- arch/s390/kvm/s390/intercept.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/arch/s390/kvm/s390/intercept.c b/arch/s390/kvm/s390/intercept.c index ca1205dfac8b..f7ad111f259c 100644 --- a/arch/s390/kvm/s390/intercept.c +++ b/arch/s390/kvm/s390/intercept.c @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include @@ -411,9 +413,9 @@ static int handle_partial_execution(struct kvm_vcpu *vcpu) */ int handle_sthyi(struct kvm_vcpu *vcpu) { + struct sthyi_sctns *sctns __free(kfree) = NULL; int reg1, reg2, cc = 0, r = 0; u64 code, addr, rc = 0; - struct sthyi_sctns *sctns = NULL; if (!test_kvm_facility(vcpu->kvm, 74)) return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); @@ -430,37 +432,32 @@ int handle_sthyi(struct kvm_vcpu *vcpu) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); if (code & 0xffff) { - cc = 3; - rc = 4; - goto out; + vcpu->run->s.regs.gprs[reg2 + 1] = 4; + kvm_s390_set_psw_cc(vcpu, 3); + return 0; } if (!kvm_s390_pv_cpu_is_protected(vcpu) && (addr & ~PAGE_MASK)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); - sctns = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + sctns = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!sctns) return -ENOMEM; cc = sthyi_fill(sctns, &rc); - if (cc < 0) { - free_page((unsigned long)sctns); + if (cc < 0) return cc; - } -out: + if (!cc) { if (kvm_s390_pv_cpu_is_protected(vcpu)) { memcpy(sida_addr(vcpu->arch.sie_block), sctns, PAGE_SIZE); } else { r = write_guest(vcpu, addr, reg2, sctns, PAGE_SIZE); - if (r) { - free_page((unsigned long)sctns); + if (r) return kvm_s390_inject_prog_cond(vcpu, r); - } } } - free_page((unsigned long)sctns); vcpu->run->s.regs.gprs[reg2 + 1] = rc; kvm_s390_set_psw_cc(vcpu, cc); return r; From bbfbe09b281e2bf80914ea2850dbfb4f953796b0 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Fri, 18 Sep 2026 16:51:51 +0200 Subject: [PATCH 485/506] Documentation: Add missing KVM s390 crypto attributes Device attributes to enable/disable the AP interpretive execution support for crypto were added to KVM several years ago, but they weren't added to the documentation itself. Fixes: 37940fb0b6a2 ("KVM: s390: device attrs to enable/disable AP interpretation") Signed-off-by: Eric Farman Reviewed-by: Anthony Krowiak Signed-off-by: Janosch Frank Signed-off-by: Christian Borntraeger --- Documentation/virt/kvm/devices/vm.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Documentation/virt/kvm/devices/vm.rst b/Documentation/virt/kvm/devices/vm.rst index a4d39fa1b083..78e63eb9994e 100644 --- a/Documentation/virt/kvm/devices/vm.rst +++ b/Documentation/virt/kvm/devices/vm.rst @@ -280,6 +280,24 @@ Allows user space to disable dea key wrapping, clearing the wrapping key. :Parameters: none :Returns: 0 +4.5. ATTRIBUTE: KVM_S390_VM_CRYPTO_ENABLE_APIE (w/o) +------------------------------------------------------- + +Allows user space to enable AP interpretive execution. + +:Parameters: none +:Returns: 0 for success + -EOPNOTSUPP if AP instructions not available for guest + +4.6. ATTRIBUTE: KVM_S390_VM_CRYPTO_DISABLE_APIE (w/o) +------------------------------------------------------- + +Allows user space to disable AP interpretive execution. + +:Parameters: none +:Returns: 0 for success + -EOPNOTSUPP if AP instructions not available for guest + 5. GROUP: KVM_S390_VM_MIGRATION =============================== From 370b3a89cbd95173c57e2524477a471e69c0e682 Mon Sep 17 00:00:00 2001 From: Janosch Frank Date: Tue, 18 Aug 2026 08:36:06 +0000 Subject: [PATCH 486/506] KVM: s390: Improve floating IRQ injection behavior Floating IRQs can be handled by any VCPU that opened its masks. The current design does not check if the mask is open when a floating IRQ is injected via the FLIC. It will wakeup the last VCPU that went sleeping hoping it's the correct one. Improve this by checking if the VCPU can deliver that partiqular IRQ and if not try to find another VCPU which can take the IRQ. This is not the final fix around this topic but we'll eventually inject a pending IRQ. The current code can easily deadlock a VM and with this fix we work around that. Signed-off-by: Janosch Frank Acked-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/s390/interrupt.c | 86 ++++++++++++++++++++++++++-------- 1 file changed, 66 insertions(+), 20 deletions(-) diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c index 27b5fdf7ee75..0776e98a85ec 100644 --- a/arch/s390/kvm/s390/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -1918,49 +1918,93 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) return 0; } +static u64 inti_to_irq_pend_mask(u64 type, int isc) +{ + switch (type) { + case KVM_S390_MCHK: + /* Only repressible machine checks are floating */ + return BIT(IRQ_PEND_MCHK_REP); + case KVM_S390_INT_VIRTIO: + return BIT(IRQ_PEND_VIRTIO); + case KVM_S390_INT_SERVICE: + return BIT(IRQ_PEND_EXT_SERVICE) | + BIT(IRQ_PEND_EXT_SERVICE_EV); + case KVM_S390_INT_PFAULT_DONE: + return BIT(IRQ_PEND_PFAULT_DONE); + case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + return BIT(isc_to_irq_type(isc)); + default: + return 0; + } +} + +/* + * Setup intervention masks to catch running vcpus that hopefully open + * their masks soonish and kick sleeping vcpus to motivate them to + * take IRQs. + */ +static void vcpu_intervention_kick(struct kvm_vcpu *vcpu, u64 type) +{ + /* make the VCPU drop out of the SIE, or wake it up if sleeping */ + switch (type) { + case KVM_S390_MCHK: + kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT); + break; + case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + if (!(type & KVM_S390_INT_IO_AI_MASK && + vcpu->kvm->arch.gisa_int.origin) || + kvm_s390_pv_cpu_get_handle(vcpu)) + kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT); + break; + default: + kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); + break; + } + kvm_s390_vcpu_wakeup(vcpu); +} + /* * Find a destination VCPU for a floating irq and kick it. */ -static void __floating_irq_kick(struct kvm *kvm, u64 type) +static void __floating_irq_kick(struct kvm *kvm, u64 type, int isc) { struct kvm_vcpu *dst_vcpu; int sigcpu, online_vcpus, nr_tries = 0; + u64 irq_pend_mask; + unsigned long i; online_vcpus = atomic_read(&kvm->online_vcpus); if (!online_vcpus) return; + irq_pend_mask = inti_to_irq_pend_mask(type, isc); for (sigcpu = kvm->arch.float_int.last_sleep_cpu; ; sigcpu++) { sigcpu %= online_vcpus; dst_vcpu = kvm_get_vcpu(kvm, sigcpu); - if (!is_vcpu_stopped(dst_vcpu)) + if (!is_vcpu_stopped(dst_vcpu) && + deliverable_irqs(dst_vcpu) & irq_pend_mask) break; /* avoid endless loops if all vcpus are stopped */ - if (nr_tries++ >= online_vcpus) - return; + if (nr_tries++ >= online_vcpus) { + dst_vcpu = NULL; + break; + } } - /* make the VCPU drop out of the SIE, or wake it up if sleeping */ - switch (type) { - case KVM_S390_MCHK: - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT); - break; - case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: - if (!(type & KVM_S390_INT_IO_AI_MASK && - kvm->arch.gisa_int.origin) || - kvm_s390_pv_cpu_get_handle(dst_vcpu)) - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT); - break; - default: - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT); - break; + /* Nobody was enabled, time to wake all of them */ + if (!dst_vcpu) { + kvm_for_each_vcpu(i, dst_vcpu, kvm) + vcpu_intervention_kick(dst_vcpu, type); + return; } - kvm_s390_vcpu_wakeup(dst_vcpu); + + vcpu_intervention_kick(dst_vcpu, type); } static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { u64 type = READ_ONCE(inti->type); + int isc = -1; int rc; switch (type) { @@ -1977,6 +2021,8 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) rc = __inject_pfault_done(kvm, inti); break; case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + /* Grab isc here since __inject_io() might free inti */ + isc = int_word_to_isc(inti->io.io_int_word); rc = __inject_io(kvm, inti); break; default: @@ -1985,7 +2031,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) if (rc) return rc; - __floating_irq_kick(kvm, type); + __floating_irq_kick(kvm, type, isc); return 0; } From c03717417232c690534315a428e019a006502031 Mon Sep 17 00:00:00 2001 From: Janosch Frank Date: Wed, 19 Aug 2026 08:19:26 +0000 Subject: [PATCH 487/506] KVM: s390: Kick PV cpus at the right time for service irqs Service call handling is a two stage process for PV vms. First we receive the secure instruction intercept and then the secure instruction notification intercept. The secure instruction intercept (104) is analogous to the non-pv instruction intercept (4) with the difference that we're not allowed to inject an IRQ when re-entering SIE. We have to wait for the notification intercept (108) which tells us that we're allowed to inject. Unfortunately we never considered this difference and hence the IRQ injection code will try to kick on the secure instruction intercept where service IRQs are masked. It's time to move the kick to the instruction notification and skip kicking on the instruction interception path. Signed-off-by: Janosch Frank Acked-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/s390/intercept.c | 9 ++++++ arch/s390/kvm/s390/interrupt.c | 50 ++++++++++++++++++++++++++++------ arch/s390/kvm/s390/s390.h | 1 + 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/arch/s390/kvm/s390/intercept.c b/arch/s390/kvm/s390/intercept.c index f7ad111f259c..adfde2a656e7 100644 --- a/arch/s390/kvm/s390/intercept.c +++ b/arch/s390/kvm/s390/intercept.c @@ -533,6 +533,15 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu) set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); spin_unlock_irqrestore(&fi->lock, flags); + + /* + * We missed the floating IRQ kick since we can only inject + * when we end up here and not when the irq was injected via + * the FLIC. + * + * Now that we have cleared the masking we can kick cpus. + */ + kvm_s390_pv_sclp_kick(vcpu); return 0; } diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c index 0776e98a85ec..31a6074feffc 100644 --- a/arch/s390/kvm/s390/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -1963,10 +1963,7 @@ static void vcpu_intervention_kick(struct kvm_vcpu *vcpu, u64 type) kvm_s390_vcpu_wakeup(vcpu); } -/* - * Find a destination VCPU for a floating irq and kick it. - */ -static void __floating_irq_kick(struct kvm *kvm, u64 type, int isc) +static void kick_cpu_irq(struct kvm *kvm, u64 type, u64 parm) { struct kvm_vcpu *dst_vcpu; int sigcpu, online_vcpus, nr_tries = 0; @@ -1977,7 +1974,7 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type, int isc) if (!online_vcpus) return; - irq_pend_mask = inti_to_irq_pend_mask(type, isc); + irq_pend_mask = inti_to_irq_pend_mask(type, parm); for (sigcpu = kvm->arch.float_int.last_sleep_cpu; ; sigcpu++) { sigcpu %= online_vcpus; dst_vcpu = kvm_get_vcpu(kvm, sigcpu); @@ -2001,10 +1998,46 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type, int isc) vcpu_intervention_kick(dst_vcpu, type); } +void kvm_s390_pv_sclp_kick(struct kvm_vcpu *vcpu) +{ + /* + * The cpu that called sclp likely will also take the IRQ, no + * need to kick anyone. + */ + if (deliverable_irqs(vcpu) & BIT(IRQ_PEND_EXT_SERVICE)) + return; + + /* + * For the other cases we might have sleeping cpus with open + * masks. Time to find and kick them. + */ + kick_cpu_irq(vcpu->kvm, KVM_S390_INT_SERVICE, -1); +} + +/* + * Find a destination VCPU for a floating irq and kick it. + */ +static void __floating_irq_kick(struct kvm *kvm, u64 type, u64 parm) +{ + struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + + /* + * No need to kick on non-ev service IRQs for PV VMs, we're + * not allowed to inject anyway. We need to wait for the sclp + * instruction notification AFTER re-entry of the vcpu that + * handled the instruction intercept. + */ + if (type == KVM_S390_INT_SERVICE && !(parm & SCCB_EVENT_PENDING) && + test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs)) + return; + + kick_cpu_irq(kvm, type, parm); +} + static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { u64 type = READ_ONCE(inti->type); - int isc = -1; + u64 parm = -1; int rc; switch (type) { @@ -2015,6 +2048,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) rc = __inject_virtio(kvm, inti); break; case KVM_S390_INT_SERVICE: + parm = inti->ext.ext_params & SCCB_EVENT_PENDING; rc = __inject_service(kvm, inti); break; case KVM_S390_INT_PFAULT_DONE: @@ -2022,7 +2056,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) break; case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: /* Grab isc here since __inject_io() might free inti */ - isc = int_word_to_isc(inti->io.io_int_word); + parm = int_word_to_isc(inti->io.io_int_word); rc = __inject_io(kvm, inti); break; default: @@ -2031,7 +2065,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) if (rc) return rc; - __floating_irq_kick(kvm, type, isc); + __floating_irq_kick(kvm, type, parm); return 0; } diff --git a/arch/s390/kvm/s390/s390.h b/arch/s390/kvm/s390/s390.h index aa0d1d062f8d..9ed1af5583e3 100644 --- a/arch/s390/kvm/s390/s390.h +++ b/arch/s390/kvm/s390/s390.h @@ -375,6 +375,7 @@ enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer); int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu); void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu); void kvm_s390_clear_float_irqs(struct kvm *kvm); +void kvm_s390_pv_sclp_kick(struct kvm_vcpu *vcpu); int __must_check kvm_s390_inject_vm(struct kvm *kvm, struct kvm_s390_interrupt *s390int, struct kvm_s390_interrupt_info *inti); From b290b5bff8505b0eafa04684b10c50cccfc64013 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 19:34:27 +0200 Subject: [PATCH 488/506] s390/Kconfig: remove PCI dependency from HAS_IOMEM HAS_IOMEM was defined as def_bool PCI on s390, making it false on kernels built without PCI support. This made sense until now as the IOMEM code was only required if PCI was enabled. This will change as KVM will require IOMEM in the future. To simplify the config space, turn HAS_IOMEM on unconditionally by default. All other architectures either do not override HAS_IOMEM or set it unconditionally. s390 was the only architecture where HAS_IOMEM could be false on a real system. Since HAS_IOMEM=y now causes kernel/iomem.o and lib/devres.o to be built unconditionally, ioremap()/iounmap() must always be available. Select GENERIC_IOREMAP unconditionally to provide the generic implementations for the PCI=n case. Previously GENERIC_IOREMAP was conditioned on CONFIG_PCI in lockstep with HAS_IOMEM, so mm/ioremap.c was never built without PCI. Therefore, guard the PCI override macros ioremap_prot, iounmap, and ioremap_wc with CONFIG_PCI. Reviewed-by: Ilya Leoshkevich Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/Kconfig | 5 +---- arch/s390/include/asm/io.h | 12 ++++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 4b51bc6e8948..5a01fbd8e32b 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -181,7 +181,7 @@ config S390 select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY select GENERIC_SMP_IDLE_THREAD - select GENERIC_IOREMAP if PCI + select GENERIC_IOREMAP select GLOB select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL @@ -768,9 +768,6 @@ config PCI_NR_FUNCTIONS endif # PCI -config HAS_IOMEM - def_bool PCI - config CHSC_SCH def_tristate m prompt "Support for CHSC subchannels" diff --git a/arch/s390/include/asm/io.h b/arch/s390/include/asm/io.h index faddb9aef3b8..ef0490edeccd 100644 --- a/arch/s390/include/asm/io.h +++ b/arch/s390/include/asm/io.h @@ -27,14 +27,8 @@ void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr); /* * I/O memory mapping functions. */ -#define ioremap_prot ioremap_prot -#define iounmap iounmap - #define _PAGE_IOREMAP pgprot_val(PAGE_KERNEL) -#define ioremap_wc(addr, size) \ - ioremap_prot((addr), (size), pgprot_writecombine(PAGE_KERNEL)) - static inline void __iomem *ioport_map(unsigned long port, unsigned int nr) { return NULL; @@ -46,6 +40,12 @@ static inline void ioport_unmap(void __iomem *p) #ifdef CONFIG_PCI +#define ioremap_prot ioremap_prot +#define iounmap iounmap + +#define ioremap_wc(addr, size) \ + ioremap_prot((addr), (size), pgprot_writecombine(PAGE_KERNEL)) + /* * s390 needs a private implementation of pci_iomap since ioremap with its * offset parameter isn't sufficient. That's because BAR spaces are not From 2687655bef3524bb158f9b592e9cb2397d9556b9 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Wed, 30 Sep 2026 14:27:35 +0200 Subject: [PATCH 489/506] KVM: s390: Use dedicated function for migration mode Use kvm_s390_is_migration_mode() instead of kvm.arch->migration_mode. The function is independent of the actual kvm_arch implementation which is different for the second parallel KVM module.. Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/kvm/gmap/kvm_mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c index c2ffb5e59ec6..24e618f77be1 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.c +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -76,7 +76,7 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, return -EINVAL; } - if (kvm->arch.migration_mode) { + if (kvm_s390_is_migration_mode(kvm)) { /* * Turn off migration mode when: * - userspace creates a new memslot with dirty logging off, From b70113c00604ee52c1c18a0b1b498c9bcf5ddfbb Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 22 Jun 2026 14:47:10 +0200 Subject: [PATCH 490/506] s390/tools: Use arm64 headers Introduce a flexible system for sharing ARM64 headers with s390 KVM. Selected arm64 headers or snippet of headers will be reachable by s390 code through asm/header.h. Add copy-arm64h.awk that detects ARM64_S390_COMMON markers and extracts marked section into a generated header consumable by s390. Falls back to full file copy when no markers are found. Transform copied includes by replacing '#include ' with '#include ' and similar for uapi headers to ensure correct path resolution on s390 and allow for a clear separation of s390 asm headers and arm64 asm headers. Integrate this extraction into the s390/kapi target, conditional on CONFIG_KVM. Reviewed-by: Hendrik Brueckner Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/include/uapi/Kbuild | 6 ++ arch/s390/tools/Makefile | 6 +- arch/s390/tools/Makefile.arm64h | 55 ++++++++++++++++++ arch/s390/tools/copy-arm64h-full.awk | 50 +++++++++++++++++ arch/s390/tools/copy-arm64h.awk | 83 ++++++++++++++++++++++++++++ 5 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 arch/s390/include/uapi/Kbuild create mode 100644 arch/s390/tools/Makefile.arm64h create mode 100644 arch/s390/tools/copy-arm64h-full.awk create mode 100644 arch/s390/tools/copy-arm64h.awk diff --git a/arch/s390/include/uapi/Kbuild b/arch/s390/include/uapi/Kbuild new file mode 100644 index 000000000000..81dcaaf42c40 --- /dev/null +++ b/arch/s390/include/uapi/Kbuild @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 + +# The ARM64 uapi headers under generated/uapi/arm64/ are private copies used +# internally by the s390 KVM ARM64 implementation. They contain #line +# directives and rewritten include paths and must not be shipped to userspace. +no-export-headers += arm64/% diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index f2862364fb42..d0aa7e076bad 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -3,12 +3,16 @@ # Makefile for s390 specific build tools # +include $(srctree)/arch/s390/tools/Makefile.arm64h + kapi := arch/$(ARCH)/include/generated/asm kapi-hdrs-y := $(kapi)/facility-defs.h $(kapi)/dis-defs.h +kapi-hdrs-$(CONFIG_KVM) += $(ARM64_TARGETS) + PHONY += kapi -kapi: $(kapi-hdrs-y) +kapi: $(kapi-hdrs-y) $(kapi-hdrs-m) hostprogs += gen_facilities hostprogs += gen_opcode_table diff --git a/arch/s390/tools/Makefile.arm64h b/arch/s390/tools/Makefile.arm64h new file mode 100644 index 000000000000..f3f8e9d9bdd8 --- /dev/null +++ b/arch/s390/tools/Makefile.arm64h @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Share ARM64 headers with s390 KVM +# + +arm64api := $(objtree)/arch/$(ARCH)/include/generated/arm64 +arm64uapi := $(objtree)/arch/$(ARCH)/include/generated/uapi/arm64 + +# List of ARM64 headers to share with s390 KVM +# Format: [uapi:]header.h or header-gen.h +# +# * header.h - from arch/arm64/include/asm/ to arm64/ +# * uapi:header.h - from arch/arm64/include/uapi/asm/ to uapi/arm64/ +# * header-gen.h - generated as header-gen.h from ARM64 header header.h +# If a header contains partial markers (#if ARM64_S390_COMMON == 1) only this part will be copied. +# If the file does not contain any markers the whole file will be copied. +ARM64_SHARED_HEADERS := \ + brk-imm.h \ + esr.h \ + kvm_arm.h \ + kvm_emulate-gen.h \ + kvm_host.h \ + ptrace-gen.h \ + sysreg-gen.h \ + uapi:hwcap.h \ + uapi:kvm.h \ + uapi:ptrace.h \ + uapi:sve_context.h + +filechk_gen_arm_hdr = \ + $(AWK) -f $(src)/copy-arm64h.awk $< || \ + $(AWK) -f $(src)/copy-arm64h-full.awk $< || \ + { echo "Error: Unbalanced ARM64_S390_COMMON markers in $<" >&2; exit 1; } + +arm64-type = $(firstword $(subst :, ,$(1))) +arm64-base = $(or $(word 2,$(subst :, ,$(1))),$(1)) +arm64-src-base = $(patsubst %-gen.h,%.h,$(call arm64-base,$(1))) + +arm64-src = $(srctree)/arch/arm64/include/$(if $(filter uapi,$(call arm64-type,$(1))),uapi/)asm/$(call arm64-src-base,$(1)) +arm64-dst = $(if $(filter uapi,$(call arm64-type,$(1))),$(arm64uapi),$(arm64api))/$(call arm64-base,$(1)) + +$(foreach hdr,$(ARM64_SHARED_HEADERS),\ + $(eval $(call arm64-dst,$(hdr)): $(call arm64-src,$(hdr)) $(src)/copy-arm64h.awk $(src)/copy-arm64h-full.awk FORCE)) + +ARM64_TARGETS := $(sort $(foreach hdr,$(ARM64_SHARED_HEADERS),$(call arm64-dst,$(hdr)))) +$(ARM64_TARGETS): + $(call filechk,gen_arm_hdr) + +filechk_gen_sysreg_arm64 = \ + $(AWK) -f $(srctree)/arch/arm64/tools/gen-sysreg.awk $(srctree)/arch/arm64/tools/sysreg + +$(arm64api)/sysreg-defs.h: $(srctree)/arch/arm64/tools/gen-sysreg.awk $(srctree)/arch/arm64/tools/sysreg FORCE + $(call filechk,gen_sysreg_arm64) + +ARM64_TARGETS += $(arm64api)/sysreg-defs.h diff --git a/arch/s390/tools/copy-arm64h-full.awk b/arch/s390/tools/copy-arm64h-full.awk new file mode 100644 index 000000000000..ad7bb98dc4a6 --- /dev/null +++ b/arch/s390/tools/copy-arm64h-full.awk @@ -0,0 +1,50 @@ +#!/usr/bin/awk -f +# SPDX-License-Identifier: GPL-2.0 +# +# Process entire ARM64 headers for sharing with s390 KVM +# +# Usage: copy-arm64h-full.awk +# +# This processes the entire file (unlike copy-arm64h.awk which only extracts marked sections) + +BEGIN { + max_guard_line = 25 + guard_found = 0 + header_added = 0 +} + +# Find and process the include guard in the first few lines +NR <= max_guard_line && !guard_found && /^#ifndef [A-Z_0-9]+$/ { + guard_name = $2 + guard_found = 1 + print "/* This header was copied from " FILENAME " */" + print "" + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " NR " \"" FILENAME "\"" + print + header_added = 1 + next +} + +# Transform include directives +/^#include +# +# Extracts all sections between start/end markers. If no markers found, signals for fallback. +BEGIN { + # Constants + start_pattern = "^#if ARM64_S390_COMMON == 1$" + end_pattern = "^#endif \\/\\* ARM64_S390_COMMON == 1 \\*\\/$" + max_guard_line = 25 + + # State variables + copying = found_marker = error = 0 + guard_name = "" + file_header_done = 0 + depth = 0 +} + +NR <= max_guard_line && !guard_name && /^#ifndef [A-Za-z0-9_]+$/ { + guard_name = $2 + next +} + +NR > max_guard_line && !guard_name && !file_header_done { + print "error: no include guard found in first " max_guard_line " lines" > "/dev/stderr" + file_header_done = 1 + error = 1; exit +} + +$0 ~ start_pattern { + if (copying) { error = 1; exit } + if (!found_marker) { + print "/*" + print " * This header was automatically generated from " FILENAME + print " * Do not modify this file directly." + print " */" + print "#ifndef " guard_name + print "#define " guard_name + print "" + } + depth = copying= found_marker = 1 + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " (NR + 1) " \"" FILENAME "\"" + next +} + +guard_name && !found_marker { + next +} + +$0 ~ /^[ \t]*#[ \t]*endif/ { + depth-- + if (depth == 0) { + if ($0 ~ end_pattern ) { + copying = 0 + next + } + error = 1; exit + } +} + +/ARM64_S390_COMMON/ { error = 1; exit } + +copying { + if ($0 ~ /^[ \t]*#[ \t]*if(def|ndef)?([ \t]|$)/) depth++ + gsub(/#include Date: Thu, 25 Jun 2026 13:56:16 +0200 Subject: [PATCH 491/506] KVM: s390: Use arm64 code Add the infrastructure to extract KVM/arm64 code into s390 at build time similar to the arm64 header sharing. Add copy-arm64c.awk that detects ARM64_S390_COMMON markers and extracts marked section into an .inc file that can be consumed by arm on s390 host code. If no marker is found make will fail. To ensure that no code is consumed twice by accident a guard is added during file generation. A s390 C file wanting to consume such an inc file must first define __INCL_GEN_ARM_FILE otherwise the inc file will emit a compile error. Example: #define __INCL_GEN_ARM_FILE #include #undef __INCL_GEN_ARM_FILE Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/kvm/arm64/.gitignore | 2 + arch/s390/kvm/arm64/Makefile.gen | 28 ++++++++++ arch/s390/kvm/arm64/copy-arm64c.awk | 82 +++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 arch/s390/kvm/arm64/.gitignore create mode 100644 arch/s390/kvm/arm64/Makefile.gen create mode 100644 arch/s390/kvm/arm64/copy-arm64c.awk diff --git a/arch/s390/kvm/arm64/.gitignore b/arch/s390/kvm/arm64/.gitignore new file mode 100644 index 000000000000..bd9d90694676 --- /dev/null +++ b/arch/s390/kvm/arm64/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +generated/* diff --git a/arch/s390/kvm/arm64/Makefile.gen b/arch/s390/kvm/arm64/Makefile.gen new file mode 100644 index 000000000000..9c33b068fab5 --- /dev/null +++ b/arch/s390/kvm/arm64/Makefile.gen @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Extracts ARM64 marked sections to .inc files that are included by main source +# files + +# List of ARM64 C files to extract +ARM64_CFILES := \ + arm.c \ + guest.c \ + mmio.c + +quiet_cmd_extract_inc = GEN $@ + cmd_extract_inc = mkdir -p $(dir $@); \ + $(AWK) -f $(src)/copy-arm64c.awk $< > $@ || \ + { echo "Error: No or unbalanced ARM64_S390_COMMON markers in $<" >&2; rm -f $@; exit 1; } + +ARM64_INC_FILES := $(foreach cfile,$(ARM64_CFILES),generated/$(basename $(cfile)).inc) + +$(addprefix $(obj)/,$(ARM64_INC_FILES)): $(obj)/generated/%.inc: $(srctree)/arch/arm64/kvm/%.c $(src)/copy-arm64c.awk FORCE + $(call if_changed,extract_inc) + +targets += $(ARM64_INC_FILES) +clean-files += generated/*.inc + +$(foreach cfile,$(basename $(ARM64_CFILES)),\ + $(eval $(obj)/$(cfile).o $(obj)/$(cfile).i $(obj)/$(cfile).s \ + $(obj)/$(cfile).ll $(obj)/$(cfile).lst: \ + $(obj)/generated/$(cfile).inc)) diff --git a/arch/s390/kvm/arm64/copy-arm64c.awk b/arch/s390/kvm/arm64/copy-arm64c.awk new file mode 100644 index 000000000000..640a7bcb7e23 --- /dev/null +++ b/arch/s390/kvm/arm64/copy-arm64c.awk @@ -0,0 +1,82 @@ +#!/usr/bin/awk -f +# SPDX-License-Identifier: GPL-2.0 +# +# Extract marked sections from ARM64 C files for sharing with s390 KVM +# +# Usage: copy-arm64c.awk +# +# Extracts all sections between start/end markers. If no markers found, signals failure. + +BEGIN { + # Constants + start_pattern = "^#if ARM64_S390_COMMON == 1$" + end_pattern = "^#endif /\\* ARM64_S390_COMMON == 1 \\*/$" + + # State variables + copying = found_marker = 0 + file_header_done = 0 + depth = 0 +} + +!file_header_done { + if (/^\/\*/ || /^\/\/ SPDX-License-Identifier:/) { + print + next + } + if (/[[:space:]]\*([[:space:]]|$)/) { + print + next + } + if (/\*\//) { + print " *" + } else { + print "/*" + } + + print " * This file was automatically generated from " FILENAME + print " * Do not modify this file directly." + print " */" + print "" + print "#ifndef __INCL_GEN_ARM_FILE" + print "#error do not include this file directly." + print "#endif /* __INCL_GEN_ARM_FILE */" + print "#undef __INCL_GEN_ARM_FILE" + print "" + + file_header_done = 1 +} + +$0 ~ start_pattern { + if (copying) { error = 1; exit } + depth = copying = found_marker = 1 + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " (NR + 1) " \"" FILENAME "\"" + next +} + +copying && /^[ \t]*#[ \t]*if(def|ndef)?([ \t]|$)/ { depth++ } + +$0 ~ /^[ \t]*#[ \t]*endif/ { + depth-- + if (depth == 0) { + if ($0 ~ end_pattern) { + copying = 0 + next + } + error = 1; exit + } +} + +/ARM64_S390_COMMON/ { error = 1; exit } + +copying { + gsub(/#include Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 492/506] s390: Introduce Start Arm Execution instruction The Start Arm Execution (SAE) instruction is the centerpiece for executing arm64 (KVM) guests on s390. Its purpose is, similar to SIE, to enable accelerated execution of arm64 virtual machines. SAE expects the physical address of a control block as the only argument. The host is responsible to save & restore - GPRs 0-13 - access registers 0-15 - breaking event register (BEAR) - vector/floating point registers between SAE executions to guarantee host consistency. GPRs and BEAR are saved and restored in the asm functions. The other registers are handled within C code. Access registers are handled in a later patch and SVEs will be handled when they are introduced in a future series. Most arm64 registers are handled by a satellite block called save_area. Some registers, frequently used by hypervisors, are placed into the SAE control block itself. Enlighten asm/kvm_host_types.h for the new header variant. The new header is chosen instead of asm/kvm_host_s390_types.h if KVM_S390_ARM64 is defined. Co-developed-by: Andreas Grapentin Signed-off-by: Andreas Grapentin Co-developed-by: Nina Schoetterl-Glausch Signed-off-by: Nina Schoetterl-Glausch Reviewed-by: Ilya Leoshkevich Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/asm-prototypes.h | 1 + arch/s390/include/asm/kvm_host_arm64_types.h | 124 +++++++++++++++++++ arch/s390/include/asm/kvm_host_types.h | 4 + arch/s390/include/asm/sae.h | 49 ++++++++ arch/s390/include/asm/stacktrace.h | 5 + arch/s390/kernel/asm-offsets.c | 1 + arch/s390/kernel/entry.S | 22 ++++ arch/s390/tools/opcodes.txt | 3 + 8 files changed, 209 insertions(+) create mode 100644 arch/s390/include/asm/kvm_host_arm64_types.h create mode 100644 arch/s390/include/asm/sae.h diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index d4da4436d02b..faa2afb259a1 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #endif /* _ASM_S390_PROTOTYPES_H */ diff --git a/arch/s390/include/asm/kvm_host_arm64_types.h b/arch/s390/include/asm/kvm_host_arm64_types.h new file mode 100644 index 000000000000..8783a66d3029 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_arm64_types.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_KVM_HOST_ARM64_TYPES_H +#define ASM_KVM_HOST_ARM64_TYPES_H + +#include +#include +#include +#include +#include + +struct kvm_sae_block { + u64 _0000[16]; /* 0x0000 */ +#define SAE_ICPTR_SPURIOUS 0x00 +#define SAE_ICPTR_VALIDITY 0x01 +#define SAE_ICPTR_HOST_ACCESS_EXCEPTION 0x02 +#define SAE_ICPTR_SYNCHRONOUS_EXCEPTION 0x03 +#define SAE_ICPTR_TIMER 0x04 +#define SAE_ICPTR_PE_INTERCOMM 0x05 +#define SAE_ICPTR_GUEST_ADDRESS_SIZE 0x06 +#define SAE_ICPTR_STOP 0x07 +#define SAE_ICPTR_MIO_ADDRESS 0x08 +#define SAE_ICPTR_PMU 0x09 +#define SAE_ICPTR_MAINTENANCE 0x0a + u8 icptr; /* 0x0080 */ + u8 _0081[7]; /* 0x0081 */ + u64 scad; /* 0x0088 */ + u64 _0090[16]; /* 0x0090 */ + u32 cntp_ctl; /* 0x0110 */ + u32 cntv_ctl; /* 0x0114 */ + u8 irq_ctl; /* 0x0118 */ + u8 _0119[7]; /* 0x0119 */ + struct { + u64 ich_hcr_el2; /* 0x0120 */ + u64 ich_vmcr_el2; /* 0x0128 */ + u64 ich_ap0r0_el2; /* 0x0130 */ + u64 ich_ap1r0_el2; /* 0x0138 */ + u64 _0140[2]; /* 0x0140 */ + u64 ich_lrn_el2[4]; /* 0x0150 */ + u64 _0170[4]; /* 0x0170 */ + } ic_regs; + u64 _0190[12]; /* 0x0190 */ + u64 contextidr_el1; /* 0x01f0 */ + u32 wip; /* 0x01f8 */ + u32 _01fc; /* 0x01fc */ +#define SAE_SD_FORMAT_0 0x00 + u8 sdf; /* 0x0200 */ + u8 _0201[7]; /* 0x0201 */ + u64 mso; /* 0x0208 */ + u64 msl; /* 0x0210 */ + u64 hbasce; /* 0x0218 */ + u64 _0220; /* 0x0220 */ + u64 gpto; /* 0x0228 */ + u64 ic; /* 0x0230 */ + u64 ec; /* 0x0238 */ + u64 save_area; /* 0x0240 */ + u64 _0248[7]; /* 0x0248 */ + u8 _0280[6]; /* 0x0280 */ + u16 lrcpua; /* 0x0286 */ + u64 pstate; /* 0x0288 */ + u64 pc; /* 0x0290 */ + u64 sp_el0; /* 0x0298 */ + u64 sp_el1; /* 0x02a0 */ + u64 _02a8; /* 0x02a8 */ + struct { /* 0x02b0 */ + u32 _fpcr_res0; + u32 fpcr; + }; + struct { /* 0x02b8 */ + u32 _fpsr_res0; + u32 fpsr; + }; + u16 sve_pregs[16]; /* 0x02c0 */ + u16 sve_ffr; /* 0x02e0 */ + u8 _02e2[6]; /* 0x02e2 */ + u64 _02e8[3]; /* 0x02e8 */ + + u64 gpr[31]; /* 0x0300 */ + u64 _03f8; /* 0x03f8 */ + + union { + u64 icptd[8]; /* 0x0400 */ + /* validity-interception reason; icptr 0x01 */ + u16 vir; /* 0x0400 */ + /* host access interception details; icptr 0x02 */ + struct { + u64 esr_el2; /* 0x0400 */ + u8 _0408[6]; /* 0x0408 */ + u16 pic; /* 0x040e */ + union teid teid; /* 0x0410 */ + gva_t far_el2; /* 0x0418 */ + gva_t vaddr; /* 0x0420 */ + u64 suppl; /* 0x0428 */ + u8 gltl; /* 0x0430 */ + u8 _0431[7]; /* 0x0431 */ + u64 _0438; /* 0x0438 */ + } hai; + /* exception-interception details; icptr 0x03 */ + struct { + u64 esr_el2; /* 0x0400 */ + u64 _0408[2]; /* 0x0408 */ + gva_t far_el2; /* 0x0418 */ + } trap; + /* timer-interception reason; icptr 0x04 */ +#define SAE_IR_TIMER_ID_VIRT BIT(6) +#define SAE_IR_TIMER_ID_PHYS BIT(7) + u8 tir; /* 0x0400 */ + }; + u64 _0440[376]; /* 0x0440 */ +} __packed __aligned(PAGE_SIZE); +static_assert(sizeof(struct kvm_sae_block) == PAGE_SIZE); + +struct kvm_sae_save_area { +#define SAE_SAVE_AREA_FORMAT_0 0x00 + u8 saf; /* 0x0000 */ + u8 _0001[5]; /* 0x0001 */ +#define SAE_SAS_VALID BIT(0) + u16 sas; /* 0x0006 */ + u64 sdo; /* 0x0008 */ + u64 _0010[2]; /* 0x0010 */ + u64 regs[508]; /* 0x0020 */ +} __packed __aligned(PAGE_SIZE); +static_assert(sizeof(struct kvm_sae_save_area) == PAGE_SIZE); + +#endif /* ASM_KVM_HOST_ARM64_TYPES_H */ diff --git a/arch/s390/include/asm/kvm_host_types.h b/arch/s390/include/asm/kvm_host_types.h index e5bdba07cab0..d308739ebcf5 100644 --- a/arch/s390/include/asm/kvm_host_types.h +++ b/arch/s390/include/asm/kvm_host_types.h @@ -3,6 +3,10 @@ #ifndef ASM_KVM_HOST_TYPES_H #define ASM_KVM_HOST_TYPES_H +#ifdef KVM_S390_ARM64 +#include +#else #include +#endif /* KVM_S390_ARM64 */ #endif /* ASM_KVM_HOST_TYPES_H */ diff --git a/arch/s390/include/asm/sae.h b/arch/s390/include/asm/sae.h new file mode 100644 index 000000000000..26d89fae0836 --- /dev/null +++ b/arch/s390/include/asm/sae.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_S390_SAE_H +#define __ASM_S390_SAE_H + +#include +#include + +#ifndef __ASSEMBLER__ + +/* defined in arch/s390/kernel/entry.S */ +asmlinkage void sae(phys_addr_t sae_block_phys); + +#include +#include + +/** + * stiasrm() - STore and Invalidate Arm System Register Multiple + * @save_area: Pointer to SAE save area + * + * Store the guest system registers to the save area. + * The values in the guest are no longer valid. + */ +static __always_inline void stiasrm(struct kvm_sae_save_area *save_area) +{ + asm_inline volatile( + " .insn rre,0xb9a70000,%[r1],0\n" + : "+m" (*save_area) + : [r1] "a" (save_area) + ); +} + +/** + * lasrm() - Load Arm System Register Multiple + * + * @save_area: Pointer to SAE save area + * + * Load the system registers from save_area into the guest. + */ +static __always_inline void lasrm(struct kvm_sae_save_area *save_area) +{ + asm_inline volatile( + " .insn rre,0xb9a60000,%[r1],0\n" + : + : "m" (*save_area), [r1] "a" (save_area) + ); +} + +#endif /* !__ASSEMBLER__ */ +#endif /* __ASM_S390_SAE_H */ diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h index ac3606c3babe..2d332d7c8145 100644 --- a/arch/s390/include/asm/stacktrace.h +++ b/arch/s390/include/asm/stacktrace.h @@ -59,6 +59,7 @@ static inline bool on_stack(struct stack_info *info, struct stack_frame { union { unsigned long empty[9]; + /* SIE stack frame */ struct { unsigned long sie_control_block; unsigned long sie_savearea; @@ -68,6 +69,10 @@ struct stack_frame { unsigned long sie_guest_asce; unsigned long sie_irq; }; + /* SAE stack frame */ + struct { + unsigned long sae_bear; + }; }; unsigned long gprs[10]; unsigned long back_chain; diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index f6dd2b67dcee..f9f9c2246074 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -70,6 +70,7 @@ int main(void) OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys); OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce); OFFSET(__SF_SIE_IRQ, stack_frame, sie_irq); + OFFSET(__SF_SAE_BEAR, stack_frame, sae_bear); DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame)); BLANK(); OFFSET(__SFUSER_BACKCHAIN, stack_frame_user, back_chain); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 10dd9bbdf985..2004fd7642d7 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -245,6 +245,28 @@ EXPORT_SYMBOL(__sie64a) EXPORT_SYMBOL(sie_exit) #endif +#if IS_ENABLED(CONFIG_KVM) +/* + * SAE calling convention: + * %r2 physical address of the SAE control block + */ +SYM_FUNC_START(sae) + stmg %r6,%r14,__SF_GPRS(%r15) # store kernel registers + STBEAR __SF_SAE_BEAR(%r15) # save breaking event address register + .insn rre,0xb9a50000,%r2,0 # Start Arm Execution + LBEAR __SF_SAE_BEAR(%r15) # restore breaking event address register + lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers + xgr %r0,%r0 # clear guest registers to + xgr %r1,%r1 # prevent speculative use + xgr %r2,%r2 + xgr %r3,%r3 + xgr %r4,%r4 + xgr %r5,%r5 + BR_EX %r14 +SYM_FUNC_END(sae) +EXPORT_SYMBOL(sae) +#endif + /* * SVC interrupt handler routine. System calls are synchronous events and * are entered with interrupts disabled. diff --git a/arch/s390/tools/opcodes.txt b/arch/s390/tools/opcodes.txt index def2659f6602..0e4773c94af0 100644 --- a/arch/s390/tools/opcodes.txt +++ b/arch/s390/tools/opcodes.txt @@ -594,6 +594,9 @@ b9a0 clp RRF_U0RR b9a1 tpei RRE_RR b9a2 ptf RRE_R0 b9a4 uvc RRF_URR +b9a5 sae RRE_R0 +b9a6 lasrm RRE_R0 +b9a7 stiasrm RRE_R0 b9aa lptea RRF_RURR2 b9ab essa RRF_U0RR b9ac irbm RRE_RR From e6c329180d1d59766a651ec92397a417953b5239 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 493/506] KVM: s390: arm64: Introduce host definitions Add all basic definitions the arm on s390 KVM host requires. Including, but not limited to, struct kvm*arch definitions, various functions (to be implemented in the following patches), and various defines required to run arm64 guests. Enlighten asm/kvm_host.h for the new header variant. The new header is chosen instead of asm/kvm_host_s390.h if KVM_S390_ARM64 is defined. Co-developed-by: Andreas Grapentin Signed-off-by: Andreas Grapentin Co-developed-by: Nina Schoetterl-Glausch Signed-off-by: Nina Schoetterl-Glausch Signed-off-by: Steffen Eiden [borntraeger: Rebased onto latest changes regarding KVM configuration] Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/kvm.h | 6 + arch/s390/include/asm/kvm_host.h | 4 + arch/s390/include/asm/kvm_host_arm64.h | 209 +++++++++++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 arch/s390/include/asm/kvm.h create mode 100644 arch/s390/include/asm/kvm_host_arm64.h diff --git a/arch/s390/include/asm/kvm.h b/arch/s390/include/asm/kvm.h new file mode 100644 index 000000000000..aeb11dc631d5 --- /dev/null +++ b/arch/s390/include/asm/kvm.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef KVM_S390_ARM64 +#include +#else +#include +#endif diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 147325978225..ac3440a617c1 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -3,7 +3,11 @@ #ifndef ASM_KVM_HOST_H #define ASM_KVM_HOST_H +#ifdef KVM_S390_ARM64 +#include +#else #include +#endif #define PGM_OPERATION 0x01 #define PGM_PRIVILEGED_OP 0x02 diff --git a/arch/s390/include/asm/kvm_host_arm64.h b/arch/s390/include/asm/kvm_host_arm64.h new file mode 100644 index 000000000000..82343e723ebf --- /dev/null +++ b/arch/s390/include/asm/kvm_host_arm64.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_KVM_HOST_ARM64_H +#define ASM_KVM_HOST_ARM64_H + +#include + +#include +#include + +#define vcpu_gp_regs(v) ((v)->arch.sae_block.gpr) + +#include +#include + +#include + +#define KVM_HAVE_MMU_RWLOCK +#define KVM_MAX_VCPUS 1 +#define __KVM_HAVE_VCPU_RUN_PID_CHANGE + +#define KVM_S390_ARM64_IMPL_FEATURES 0 +#define KVM_HALT_POLL_NS_DEFAULT 50000 + +#define KVM_S390_MANAGES_S390_GUEST 0 + +/* Minimal (=no) vgic definitions */ +#define KVM_IRQCHIP_NUM_PINS 1 +#define irqchip_in_kernel(_k) false + +#define __ctxt_sys_reg(ctx, reg) NULL +struct kvm_cpu_context { + /* + * These are just for 32 bit, which we don't have, making them RES0. + * They are exposed to user space. + */ + u64 spsr_abt; + u64 spsr_und; + u64 spsr_irq; + u64 spsr_fiq; + + __vector128 __aligned(16) vregs[32]; +}; + +struct kvm_vcpu_arch { + struct kvm_sae_block sae_block; + struct kvm_sae_save_area save_area; + struct kvm_cpu_context ctxt; + + u32 host_acrs[NUM_ACRS]; + + /* Hypervisor Configuration Register */ + u64 hcr_el2; + + /* Configuration flags, set once and for all before the vcpu can run */ + u8 cflags; + + /* Input flags to the hypervisor code, potentially cleared after use */ + u8 iflags; + + /* State flags for kernel bookkeeping, unused by the hypervisor code */ + u8 sflags; + + /* + * Don't run the guest (internal implementation need). + * + * Contrary to the flags above, this is set/cleared outside of + * a vcpu context, and thus cannot be mixed with the flags + * themselves (or the flag accesses need to be made atomic). + */ + bool pause; + + /* vcpu power state */ + struct kvm_mp_state mp_state; + /* lock for mp_state & reset_state.reset */ + spinlock_t mp_state_lock; + + /* vcpu reset state */ + struct vcpu_reset_state reset_state; + + struct kvm_s390_mmu_cache *mc; + + void *debugfs_state_data; +}; + +struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; + /* ARM64 stats */ + u64 hvc_exit_stat; + u64 wfe_exit_stat; + u64 wfi_exit_stat; + u64 mmio_exit_user; + u64 mmio_exit_kernel; + u64 signal_exits; + u64 exits; + /* GMAP stats */ + u64 pfault_sync; +}; + +#define kvm_has_mte(_kvm) false +#define vcpu_has_sve(_vcpu) false +#define vcpu_has_ptrauth(_vcpu) false + +struct kvm_arch_memory_slot { +}; + +struct kvm_arch { + struct gmap *gmap; + u64 guest_phys_size; + + /* VM-wide vCPU feature set */ + unsigned long flags; + + /* Protects VM-scoped configuration data */ + struct mutex config_lock; + + debug_info_t *dbf; + + DECLARE_BITMAP(vcpu_features, KVM_VCPU_MAX_FEATURES); + + unsigned long mem_limit; +}; + +static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature) +{ + return test_bit(feature, ka->vcpu_features); +} + +struct kvm_vm_stat { + struct kvm_vm_stat_generic generic; +}; + +static inline bool system_has_full_ptr_auth(void) +{ + return true; +} + +#define kvm_vm_is_protected(_kvm) false +#define vcpu_is_protected(_vcpu) false + +#define vcpu_is_loaded(_vcpu) ((_vcpu)->cpu != -1) + +#define KVM_HVA_ERR_BAD (-1UL) +#define KVM_HVA_ERR_RO_BAD (-2UL) + +static inline bool kvm_is_error_hva(unsigned long addr) +{ + return IS_ERR_VALUE(addr); +} + +u32 get_kvm_ipa_limit(void); + +/* unused, but required functions */ +static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} +static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_sync_events(struct kvm *kvm) {} +static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} +static inline void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, + unsigned long mask) {} + +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu); +void kvm_reset_vcpu(struct kvm_vcpu *vcpu); + +/* arm64 guests do not use async-pf. Defined because Kbuild requires it as s390 kvm turns it on. */ +#define ASYNC_PF_PER_VCPU 0 +struct kvm_arch_async_pf { + unsigned long pfault_token; +}; + +#define __unsupp_async_call(fn) WARN_ONCE(true, "async not supported on kvm-arm64 %s", fn) + +static inline bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) +{ + __unsupp_async_call(__func__); + return false; +}; + +static inline void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); +}; + +static inline bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); + return false; +}; + +static inline void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); +}; + +static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) +{ + __unsupp_async_call(__func__); +}; + +#define kvm_supports_32bit_el0() false + +#endif /* ASM_KVM_HOST_ARM64_H */ From dcd1a3a29250a971e6dc86a6f5e4e16749d6006d Mon Sep 17 00:00:00 2001 From: Hendrik Brueckner Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 494/506] s390/hwcaps: Report SAE support as hwcap Report SAE support as hwcap (and /proc/cpuinfo) Signed-off-by: Hendrik Brueckner Reviewed-by: Steffen Eiden Reviewed-by: Janosch Frank Reviewed-by: Ilya Leoshkevich Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/elf.h | 2 ++ arch/s390/include/asm/sclp.h | 5 ++++- arch/s390/kernel/processor.c | 3 +++ drivers/s390/char/sclp_early.c | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index bb63fa4d20bb..ad3108ecfb07 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h @@ -123,6 +123,7 @@ enum { HWCAP_NR_NNPA = 20, HWCAP_NR_PCI_MIO = 21, HWCAP_NR_SIE = 22, + HWCAP_NR_SAE = 23, HWCAP_NR_MAX }; @@ -150,6 +151,7 @@ enum { #define HWCAP_NNPA BIT(HWCAP_NR_NNPA) #define HWCAP_PCI_MIO BIT(HWCAP_NR_PCI_MIO) #define HWCAP_SIE BIT(HWCAP_NR_SIE) +#define HWCAP_SAE BIT(HWCAP_NR_SAE) /* * These are used to set parameters in the core dumps. diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 44066550b9b1..68f61b71a2d5 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -56,7 +56,9 @@ struct sclp_core_entry { u8 siif : 1; u8 sigpif : 1; u8 : 3; - u8 reserved2[3]; + u8 aef: 1; + u8 : 7; + u8 reserved2[2]; u8 : 2; u8 ib : 1; u8 cei : 1; @@ -109,6 +111,7 @@ struct sclp_info { unsigned char has_aeni : 1; unsigned char has_aisi : 1; unsigned char has_astfleie2 : 1; + unsigned char has_aef : 1; unsigned int ibc; unsigned int mtid; unsigned int mtid_cp; diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index e33a3eccda56..6da55a158027 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -150,6 +150,7 @@ static void show_cpu_summary(struct seq_file *m, void *v) [HWCAP_NR_NNPA] = "nnpa", [HWCAP_NR_PCI_MIO] = "pcimio", [HWCAP_NR_SIE] = "sie", + [HWCAP_NR_SAE] = "sae", }; int i, cpu; @@ -254,6 +255,8 @@ static int __init setup_hwcaps(void) /* virtualization support */ if (sclp.has_sief2) elf_hwcap |= HWCAP_SIE; + if (sclp.has_aef) + elf_hwcap |= HWCAP_SAE; return 0; } diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 22dd797e6229..e58d24b95bd1 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -96,6 +96,7 @@ static void __init sclp_early_facilities_detect(void) sclp.has_ib = cpue->ib; sclp.has_cei = cpue->cei; sclp.has_skey = cpue->skey; + sclp.has_aef = cpue->aef; break; } From 761404d7acfe8c58f9bd938332c6cc9beed6bafd Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 495/506] KVM: s390: Add basic arm64 kvm module Add basic code for the new arm64 on s390 KVM implementation. Add kernel module boilerplate code and trivial functions. Add tracing scaffolding. Signed-off-by: Steffen Eiden [borntraeger@linux.ibm.com: add struct kvm_vcpu *vcpu to system_supported_vcpu_features] Signed-off-by: Christian Borntraeger --- arch/s390/kvm/arm64/arm.c | 147 ++++++++++++++++++++++++++++++++++++ arch/s390/kvm/arm64/arm.h | 44 +++++++++++ arch/s390/kvm/arm64/guest.c | 90 ++++++++++++++++++++++ arch/s390/kvm/arm64/trace.h | 18 +++++ 4 files changed, 299 insertions(+) create mode 100644 arch/s390/kvm/arm64/arm.c create mode 100644 arch/s390/kvm/arm64/arm.h create mode 100644 arch/s390/kvm/arm64/guest.c create mode 100644 arch/s390/kvm/arm64/trace.h diff --git a/arch/s390/kvm/arm64/arm.c b/arch/s390/kvm/arm64/arm.c new file mode 100644 index 000000000000..3e70e3c3dcbc --- /dev/null +++ b/arch/s390/kvm/arm64/arm.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include + +#include "arm.h" + +#define CREATE_TRACE_POINTS +#include "trace.h" + +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu); + +#define __INCL_GEN_ARM_FILE +#include "generated/arm.inc" +#undef __INCL_GEN_ARM_FILE + +int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) +{ + int ret; + + switch (ext) { + case KVM_CAP_NR_VCPUS: + case KVM_CAP_MAX_VCPUS: + case KVM_CAP_MAX_VCPU_ID: + ret = KVM_MAX_VCPUS; + break; + case KVM_CAP_ARM_VM_IPA_SIZE: + ret = get_kvm_ipa_limit(); + break; + case KVM_CAP_IOEVENTFD: + case KVM_CAP_PRE_FAULT_MEMORY: + ret = 1; + break; + default: + ret = 0; + } + + return ret; +} + +static u64 kvm_max_guest_address(void) +{ + u64 max_addr; + + max_addr = min_t(u64, TASK_SIZE_MAX, sclp.hamax); + max_addr = max_t(u64, max_addr, SZ_1G - 1); + return ALIGN_DOWN(max_addr + 1, SZ_1G) - 1; +} + +vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) +{ + return VM_FAULT_SIGBUS; +} + +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + return -EINVAL; +} + +u32 get_kvm_ipa_limit(void) +{ + return fls64(kvm_max_guest_address() + 1) - 1; +} + +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) +{ + return 0; +} + +void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) +{ +} + +int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + *mp_state = READ_ONCE(vcpu->arch.mp_state); + return 0; +} + +int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return 0; +} + +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) +{ + unsigned long features = KVM_S390_ARM64_IMPL_FEATURES; + + return features; +} + +bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) +{ + return false; +} + +int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, + int irq_source_id, int level, bool line_status) +{ + return -EINVAL; +} + +int kvm_set_routing_entry(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) +{ + return -EINVAL; +} + +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return 0; +} + +long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + return -ENOIOCTLCMD; +} + +static int __init kvm_s390_arm64_init(void) +{ + if (!sclp.has_aef) + return -ENODEV; + + return kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); +} + +static __exit void kvm_s390_arm64_exit(void) +{ + kvm_exit(); +} + +module_init(kvm_s390_arm64_init); +module_exit(kvm_s390_arm64_exit); diff --git a/arch/s390/kvm/arm64/arm.h b/arch/s390/kvm/arm64/arm.h new file mode 100644 index 000000000000..9d903916c896 --- /dev/null +++ b/arch/s390/kvm/arm64/arm.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_S390_KVM_ARM64_H +#define ARCH_S390_KVM_ARM64_H + +static __always_inline int kvm_is_ucontrol(struct kvm *kvm) +{ + return 0; +} + +static __always_inline int __kvm_s390_pv_destroy_page(struct page *page) +{ + return 0; +} + +static __always_inline void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end) +{ +} + +static __always_inline int kvm_s390_pv_get_handle(struct kvm *kvm) +{ + return 0; +} + +static __always_inline int kvm_s390_is_migration_mode(struct kvm *kvm) +{ + return false; +} + +static __always_inline bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) +{ + return false; +} + +static __always_inline void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old) +{ +} + +/* should never be called */ +static __always_inline int kvm_s390_vm_stop_migration(struct kvm *kvm) +{ + return -EINVAL; +} + +#endif /* ARCH_S390_KVM_ARM64_H */ diff --git a/arch/s390/kvm/arm64/guest.c b/arch/s390/kvm/arm64/guest.c new file mode 100644 index 000000000000..1bf546bded81 --- /dev/null +++ b/arch/s390/kvm/arm64/guest.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include +#include + +#define __INCL_GEN_ARM_FILE +#include "generated/guest.inc" +#undef __INCL_GEN_ARM_FILE + +const struct kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS() +}; + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + /* ARM64 stats */ + STATS_DESC_COUNTER(VCPU, hvc_exit_stat), + STATS_DESC_COUNTER(VCPU, wfe_exit_stat), + STATS_DESC_COUNTER(VCPU, wfi_exit_stat), + STATS_DESC_COUNTER(VCPU, mmio_exit_user), + STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), + STATS_DESC_COUNTER(VCPU, signal_exits), + STATS_DESC_COUNTER(VCPU, exits), + /* GMAP stats */ + STATS_DESC_COUNTER(VCPU, pfault_sync), +}; + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), +}; + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, + struct kvm_guest_debug *dbg) +{ + return -EINVAL; +} diff --git a/arch/s390/kvm/arm64/trace.h b/arch/s390/kvm/arm64/trace.h new file mode 100644 index 000000000000..3a5dcd039c13 --- /dev/null +++ b/arch/s390/kvm/arm64/trace.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_ARM64_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define KVM_ARM64_TRACE_KVM_H + +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +#endif /* KVM_ARM64_TRACE_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include From 3fb328f76b0f78b27ce9f4c3c1687c2c5bd15134 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 496/506] KVM: s390: arm64: Implement required functions Implement the mostly trivial functions that the shared arm64 (kvm) code & headers oblige s390 to implement. MAINTAINERS: Add arch/s390/include/arm64/* to KVM/s390 Implement a very basic smccc handler that (non-compliantly) is just able to stop a vcpu. Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- MAINTAINERS | 1 + arch/s390/include/arm64/kvm_emulate.h | 130 ++++++++++++++++++++++++++ arch/s390/include/arm64/kvm_nested.h | 11 +++ arch/s390/include/arm64/ptrace.h | 11 +++ arch/s390/include/arm64/sysreg.h | 13 +++ arch/s390/kvm/arm64/handle_exit.c | 127 +++++++++++++++++++++++++ arch/s390/kvm/arm64/inject_fault.c | 21 +++++ 7 files changed, 314 insertions(+) create mode 100644 arch/s390/include/arm64/kvm_emulate.h create mode 100644 arch/s390/include/arm64/kvm_nested.h create mode 100644 arch/s390/include/arm64/ptrace.h create mode 100644 arch/s390/include/arm64/sysreg.h create mode 100644 arch/s390/kvm/arm64/handle_exit.c create mode 100644 arch/s390/kvm/arm64/inject_fault.c diff --git a/MAINTAINERS b/MAINTAINERS index c2414447892c..1d45ccf863c1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14372,6 +14372,7 @@ L: kvm@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git F: Documentation/virt/kvm/s390* +F: arch/s390/include/arm64/* F: arch/s390/include/asm/gmap_helpers.h F: arch/s390/include/asm/kvm* F: arch/s390/include/uapi/asm/kvm* diff --git a/arch/s390/include/arm64/kvm_emulate.h b/arch/s390/include/arm64/kvm_emulate.h new file mode 100644 index 000000000000..416c4f96db05 --- /dev/null +++ b/arch/s390/include/arm64/kvm_emulate.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_KVM_EMULATE_H__ +#define __S390_ARM64_KVM_EMULATE_H__ + +#include +#include + +#include +#include +#include +#include + +static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.pc; +} + +static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.pstate; +} + +static __always_inline unsigned long *vcpu_sp_el0(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.sp_el0; +} + +static __always_inline u64 *vcpu_sp_el1(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.sp_el1; +} + +static __always_inline __vector128 *vcpu_vreg(struct kvm_vcpu *vcpu, int off) +{ + return &vcpu->arch.ctxt.vregs[off]; +} + +static __always_inline u32 *vcpu_fpsr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.fpsr; +} + +static __always_inline u32 *vcpu_fpcr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.fpcr; +} + +static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) +{ + return false; +} + +static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.esr_el2; +} + +static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.hcr_el2; +} + +static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.far_el2; +} + +static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu) +{ + return gfn_to_gpa(vcpu->arch.sae_block.hai.teid.addr); +} + +static inline u16 kvm_vcpu_fault_pic(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.pic & PGM_INT_CODE_MASK; +} + +static __always_inline +bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_fault_pic(vcpu) == PGM_PROTECTION; +} + +static __always_inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu) +{ + return true; +} + +static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) +{ + return false; +} + +static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) +{ + return false; +} + +static inline int kvm_vcpu_abt_gltl(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.gltl; +} + +static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu) +{ + u32 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK; + + return mode != PSR_MODE_EL0t; +} + +static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) +{ + *vcpu_pc(vcpu) += 4; + *vcpu_cpsr(vcpu) &= ~PSR_BTYPE_MASK; + + /* advance the singlestep state machine */ + *vcpu_cpsr(vcpu) &= ~SPSR_ELx_SS; +} + +static inline void kvm_reset_fpsimd(struct kvm_vcpu *vcpu) +{ + memset(vcpu->arch.ctxt.vregs, 0, sizeof(vcpu->arch.ctxt.vregs)); + vcpu->arch.sae_block.fpsr = 0; + vcpu->arch.sae_block.fpcr = 0; +} + +#include + +#endif /* __S390_ARM64_KVM_EMULATE_H__ */ diff --git a/arch/s390/include/arm64/kvm_nested.h b/arch/s390/include/arm64/kvm_nested.h new file mode 100644 index 000000000000..e950b1a10c41 --- /dev/null +++ b/arch/s390/include/arm64/kvm_nested.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef ASM_KVM_NESTED_H +#define ASM_KVM_NESTED_H + +static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) +{ + return false; +} + +#endif /* ASM_KVM_NESTED_H */ diff --git a/arch/s390/include/arm64/ptrace.h b/arch/s390/include/arm64/ptrace.h new file mode 100644 index 000000000000..449b85582c97 --- /dev/null +++ b/arch/s390/include/arm64/ptrace.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_PTRACE_H__ +#define __S390_ARM64_PTRACE_H__ + +#include +#include + +#include + +#endif /* __S390_ARM64_PTRACE_H__ */ diff --git a/arch/s390/include/arm64/sysreg.h b/arch/s390/include/arm64/sysreg.h new file mode 100644 index 000000000000..ac6b638fef56 --- /dev/null +++ b/arch/s390/include/arm64/sysreg.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_SYSREG_H__ +#define __S390_ARM64_SYSREG_H__ + +#include +#include +#include +#include + +#include + +#endif /* __S390_ARM64_SYSREG_H__ */ diff --git a/arch/s390/kvm/arm64/handle_exit.c b/arch/s390/kvm/arm64/handle_exit.c new file mode 100644 index 000000000000..07d8d7a53702 --- /dev/null +++ b/arch/s390/kvm/arm64/handle_exit.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include +#include + +typedef int (*exit_handle_fn)(struct kvm_vcpu *); + +#define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 +#define PSCI_RET_NOT_SUPPORTED -1 +#define PSCI_RET_INTERNAL_FAILURE -6 +/* + * Temporary smc/hvc handler. Non-compliant implementation (features missing). + * Implements only system off so that test programs are able to end their execution. + * Multi-vCPU power state management and KVM_REQ_SLEEP handling are deferred + * to future SMP series. + */ +static int kvm_smccc_call_handler(struct kvm_vcpu *vcpu) +{ + u32 func_id = vcpu_get_reg(vcpu, 0); + u64 val = PSCI_RET_NOT_SUPPORTED; + int ret = 1; + + if (func_id == PSCI_0_2_FN_SYSTEM_OFF) { + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + } + memset(&vcpu->run->system_event, 0, + sizeof(vcpu->run->system_event)); + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SHUTDOWN; + vcpu->run->system_event.ndata = 1; + vcpu->run->system_event.data[0] = 0; + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + val = PSCI_RET_INTERNAL_FAILURE; + ret = 0; + } + vcpu_set_reg(vcpu, 0, val); + + return ret; +} + +static int handle_hvc(struct kvm_vcpu *vcpu) +{ + vcpu->stat.hvc_exit_stat++; + return kvm_smccc_call_handler(vcpu); +} + +static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu) +{ + u64 esr = kvm_vcpu_get_esr(vcpu); + + kvm_pr_unimpl("Unknown exception class: esr: %#016llx -- %s\n", + esr, esr_get_class_string(esr)); + + kvm_inject_undefined(vcpu); + return 1; +} + +static exit_handle_fn arm_exit_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = kvm_handle_unknown_ec, + [ESR_ELx_EC_HVC64] = handle_hvc, +}; + +static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) +{ + u64 esr = kvm_vcpu_get_esr(vcpu); + u8 esr_ec = ESR_ELx_EC(esr); + + return arm_exit_handlers[esr_ec]; +} + +/* + * We may be single-stepping an emulated instruction. If the emulation + * has been completed in the kernel, we can return to userspace with a + * KVM_EXIT_DEBUG, otherwise userspace needs to complete its + * emulation first. + */ +static int handle_trap_exceptions(struct kvm_vcpu *vcpu) +{ + exit_handle_fn exit_handler; + + exit_handler = kvm_get_exit_handler(vcpu); + return exit_handler(vcpu); +} + +/* manually copied from arch/arm64/kernel/traps.c */ +static const char * const esr_class_str[] = { + [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", + [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", + [ESR_ELx_EC_WFx] = "WFI/WFE", + [ESR_ELx_EC_FP_ASIMD] = "ASIMD", + [ESR_ELx_EC_PAC] = "PAC", + [ESR_ELx_EC_BTI] = "BTI", + [ESR_ELx_EC_ILL] = "PSTATE.IL", + [ESR_ELx_EC_SVC64] = "SVC (AArch64)", + [ESR_ELx_EC_HVC64] = "HVC (AArch64)", + [ESR_ELx_EC_SMC64] = "SMC (AArch64)", + [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", + [ESR_ELx_EC_SVE] = "SVE", + [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB", + [ESR_ELx_EC_FPAC] = "FPAC", + [ESR_ELx_EC_SME] = "SME", + [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", + [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", + [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", + [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", + [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", + [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", + [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", + [ESR_ELx_EC_MOPS] = "MOPS", + [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", + [ESR_ELx_EC_GCS] = "Guarded Control Stack", + [ESR_ELx_EC_SERROR] = "SError", + [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", + [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", + [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", + [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", + [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", + [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", + [ESR_ELx_EC_BRK64] = "BRK (AArch64)", +}; + +const char *esr_get_class_string(unsigned long esr) +{ + return esr_class_str[ESR_ELx_EC(esr)]; +} diff --git a/arch/s390/kvm/arm64/inject_fault.c b/arch/s390/kvm/arm64/inject_fault.c new file mode 100644 index 000000000000..425dbeaa421c --- /dev/null +++ b/arch/s390/kvm/arm64/inject_fault.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/** + * kvm_inject_undefined - inject an undefined instruction into the guest + * @vcpu: The vCPU in which to inject the exception + * + * It is assumed that this code is called from the VCPU thread and that the + * VCPU therefore is not currently executing guest code. + */ +void kvm_inject_undefined(struct kvm_vcpu *vcpu) +{ + /* Stub until s390 supports arm64 sysregs TODO sysregs*/ +} + +int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr) +{ + /* Stub until s390 supports arm64 sysregs TODO sysregs*/ + return 1; +} From 90246e7d1476b5471e7340d971e6b1800f182b05 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 497/506] KVM: s390: arm64: Implement vm/vcpu create destroy Implement init and destroy IOCTLS for vcpu and vm. Implement arch vm IOCTL. Use s390 gmap and gmap plumbing. Implement function (stubs) required by gmap. Co-developed-by: Janosch Frank Signed-off-by: Janosch Frank Co-developed-by: Andreas Grapentin Signed-off-by: Andreas Grapentin Co-developed-by: Nina Schoetterl-Glausch Signed-off-by: Nina Schoetterl-Glausch Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/kvm/arm64/arm.c | 190 ++++++++++++++++++++++++++++++++ arch/s390/kvm/arm64/arm.h | 14 +++ arch/s390/kvm/gmap/faultin.c | 5 + arch/s390/kvm/gmap/gmap.c | 4 + arch/s390/kvm/gmap/kvm_mmu.c | 4 + arch/s390/kvm/gmap/trace_gmap.h | 13 +++ 6 files changed, 230 insertions(+) diff --git a/arch/s390/kvm/arm64/arm.c b/arch/s390/kvm/arm64/arm.c index 3e70e3c3dcbc..d5bae17d9c79 100644 --- a/arch/s390/kvm/arm64/arm.c +++ b/arch/s390/kvm/arm64/arm.c @@ -7,6 +7,9 @@ #include #include +#include +#include + #include "arm.h" #define CREATE_TRACE_POINTS @@ -18,6 +21,8 @@ static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu); #include "generated/arm.inc" #undef __INCL_GEN_ARM_FILE +#define KVM_PHYS_SHIFT (40) + int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) { int ret; @@ -51,6 +56,83 @@ static u64 kvm_max_guest_address(void) return ALIGN_DOWN(max_addr + 1, SZ_1G) - 1; } +static int kvm_gmap_init(struct kvm *kvm) +{ + struct crst_table *table; + + kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.guest_phys_size)); + + if (!kvm->arch.gmap) + return -ENOMEM; + + /* arm64 (on s390) do not have pfault */ + clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags); + set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); + + table = dereference_asce(kvm->arch.gmap->asce); + crst_table_init((void *)table, _CRSTE_HOLE(table->crstes[0].h.tt).val); + + return 0; +} + +static int kvm_vm_type_ipa_size_shift(unsigned long type) +{ + int phys_shift; + + phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type); + if (phys_shift) { + if (phys_shift > get_kvm_ipa_limit() || + phys_shift < ARM64_MIN_PARANGE_BITS) + return -EINVAL; + } else { + phys_shift = KVM_PHYS_SHIFT; + if (phys_shift > get_kvm_ipa_limit()) { + pr_warn_once("%s using unsupported default IPA limit\n", + current->comm); + return -EINVAL; + } + } + + return phys_shift; +} + +int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) +{ + char debug_name[32]; + int ret; + + if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) + return -EINVAL; + + ret = kvm_vm_type_ipa_size_shift(type); + if (ret < 0) + return ret; + kvm->arch.guest_phys_size = 1UL << ret; + + mutex_init(&kvm->arch.config_lock); + bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES); + + snprintf(debug_name, sizeof(debug_name), "kvm-arm64-%u-%u", + current->pid, get_random_u32()); + kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long)); + if (!kvm->arch.dbf) + return -ENOMEM; + debug_register_view(kvm->arch.dbf, &debug_sprintf_view); + + ret = kvm_gmap_init(kvm); + if (ret) + goto out_err; + kvm->arch.mem_limit = kvm->arch.guest_phys_size; + + VM_EVENT(kvm, 3, "vm created with type %lu", type); + return 0; + +out_err: + debug_unregister(kvm->arch.dbf); + + return ret; +} + vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) { return VM_FAULT_SIGBUS; @@ -62,6 +144,13 @@ long kvm_arch_dev_ioctl(struct file *filp, return -EINVAL; } +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + kvm_destroy_vcpus(kvm); + debug_unregister(kvm->arch.dbf); + kvm->arch.gmap = gmap_put(kvm->arch.gmap); +} + u32 get_kvm_ipa_limit(void) { return fls64(kvm_max_guest_address() + 1) - 1; @@ -72,10 +161,45 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) return 0; } +int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) +{ + struct kvm_sae_block *sae_block = &vcpu->arch.sae_block; + struct kvm_sae_save_area *save_area = &vcpu->arch.save_area; + + spin_lock_init(&vcpu->arch.mp_state_lock); + + /* Force users to call KVM_ARM_VCPU_INIT */ + vcpu_clear_flag(vcpu, VCPU_INITIALIZED); + + vcpu->arch.mc = kvm_s390_new_mmu_cache(); + if (!vcpu->arch.mc) + return -ENOMEM; + + save_area->sdo = virt_to_phys(sae_block); + + sae_block->hbasce = vcpu->kvm->arch.gmap->asce.val; + sae_block->mso = 0L; + sae_block->msl = vcpu->kvm->arch.mem_limit - 1; + sae_block->save_area = virt_to_phys(save_area); + + VM_EVENT(vcpu->kvm, 3, + "create cpu %d at 0x%p, sae block at 0x%p, satellite at 0x%p", + vcpu->vcpu_id, vcpu, &vcpu->arch.sae_block, + &vcpu->arch.save_area); + return 0; +} + void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) { } +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + kvm_s390_free_mmu_cache(vcpu->arch.mc); + + VCPU_EVENT(vcpu, 3, "%s", "free cpu"); +} + int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -101,6 +225,72 @@ static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) return features; } +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log) +{ + return s390_kvm_mmu_get_dirty_log(kvm, log); +} + +bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + scoped_guard(read_lock, &kvm->mmu_lock) + return gmap_age_gfn(kvm->arch.gmap, range->start, range->end); +} + +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + gfn_t last_gfn = memslot->base_gfn + memslot->npages; + + scoped_guard(read_lock, &kvm->mmu_lock) + gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn); +} + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change); +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ +} + +bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end); +} + +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + scoped_guard(read_lock, &kvm->mmu_lock) + return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end); +} + +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + + switch (ioctl) { + case KVM_ARM_PREFERRED_TARGET: { + struct kvm_vcpu_init init = { + .target = KVM_ARM_TARGET_GENERIC_V8, + }; + + if (copy_to_user(argp, &init, sizeof(init))) + return -EFAULT; + + return 0; + } + + default: + return -EINVAL; + } +} + bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) { return false; diff --git a/arch/s390/kvm/arm64/arm.h b/arch/s390/kvm/arm64/arm.h index 9d903916c896..760230685ca1 100644 --- a/arch/s390/kvm/arm64/arm.h +++ b/arch/s390/kvm/arm64/arm.h @@ -2,6 +2,20 @@ #ifndef ARCH_S390_KVM_ARM64_H #define ARCH_S390_KVM_ARM64_H +#include +#include + +#define VM_EVENT(d_kvm, d_loglevel, d_string, d_args...) \ + debug_sprintf_event((d_kvm)->arch.dbf, d_loglevel, \ + __stringify(KVM_DEV_NAME) ": " d_string "\n", \ + d_args) + +#define VCPU_EVENT(d_vcpu, d_loglevel, d_string, d_args...) \ + debug_sprintf_event((d_vcpu)->kvm->arch.dbf, d_loglevel, \ + __stringify(KVM_DEV_NAME) " %02d[%016llx-%016llx]: " d_string "\n", \ + (d_vcpu)->vcpu_id, (d_vcpu)->arch.sae_block.pstate, \ + (d_vcpu)->arch.sae_block.pc, d_args) + static __always_inline int kvm_is_ucontrol(struct kvm *kvm) { return 0; diff --git a/arch/s390/kvm/gmap/faultin.c b/arch/s390/kvm/gmap/faultin.c index 1dc79807012c..2e93bd563d5c 100644 --- a/arch/s390/kvm/gmap/faultin.c +++ b/arch/s390/kvm/gmap/faultin.c @@ -10,6 +10,11 @@ #include "gmap.h" #include "faultin.h" +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else +#include "s390.h" +#endif bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); #define CREATE_TRACE_POINTS diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c index 3f3fa864cc36..bf91e3d5013b 100644 --- a/arch/s390/kvm/gmap/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c @@ -21,7 +21,11 @@ #include "dat.h" #include "gmap.h" +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else #include "s390.h" +#endif #include "faultin.h" static int gmap_limit_to_type(gfn_t limit) diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c index 24e618f77be1..4e2da60b9859 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.c +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -3,7 +3,11 @@ #include #include +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else #include "s390.h" +#endif #include "gmap.h" #include "dat.h" #include "kvm_mmu.h" diff --git a/arch/s390/kvm/gmap/trace_gmap.h b/arch/s390/kvm/gmap/trace_gmap.h index 78d26dae59f9..1dce3ef3b0d9 100644 --- a/arch/s390/kvm/gmap/trace_gmap.h +++ b/arch/s390/kvm/gmap/trace_gmap.h @@ -13,6 +13,18 @@ #undef TRACE_INCLUDE_FILE #define TRACE_INCLUDE_FILE trace_gmap +#ifdef KVM_S390_ARM64 +#define __KVM_FIELDS \ + __field(unsigned long, pstate) \ + __field(unsigned long, pc) +#define __KVM_ASSIGN ({\ + __entry->pstate = vcpu->arch.sae_block.pstate; \ + __entry->pc = vcpu->arch.sae_block.pc; \ + }) +#define __KVM_PRINT \ + __entry->pstate, \ + __entry->pc +#else #define __KVM_FIELDS \ __field(unsigned long, pswmask) \ __field(unsigned long, pswaddr) @@ -23,6 +35,7 @@ #define __KVM_PRINT \ __entry->pswmask,\ __entry->pswaddr +#endif TRACE_EVENT(kvm_s390_major_guest_pfault, TP_PROTO(struct kvm_vcpu *vcpu), From 0630ce34a00dc65ec102c6def207397537bed52c Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 498/506] KVM: s390: arm64: Implement vCPU IOCTLs Implement all required vCPU (arch) IOCTLs. Co-developed-by: Andreas Grapentin Signed-off-by: Andreas Grapentin Co-developed-by: Nina Schoetterl-Glausch Signed-off-by: Nina Schoetterl-Glausch Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/kvm/arm64/arm.c | 417 ++++++++++++++++++++++++++++++ arch/s390/kvm/arm64/guest.c | 45 ++++ arch/s390/kvm/arm64/handle_exit.c | 32 +++ arch/s390/kvm/arm64/handle_exit.h | 9 + arch/s390/kvm/arm64/reset.c | 71 +++++ arch/s390/kvm/arm64/trace.h | 36 ++- 6 files changed, 609 insertions(+), 1 deletion(-) create mode 100644 arch/s390/kvm/arm64/handle_exit.h create mode 100644 arch/s390/kvm/arm64/reset.c diff --git a/arch/s390/kvm/arm64/arm.c b/arch/s390/kvm/arm64/arm.c index d5bae17d9c79..bd9d85235f43 100644 --- a/arch/s390/kvm/arm64/arm.c +++ b/arch/s390/kvm/arm64/arm.c @@ -6,11 +6,19 @@ #include #include #include +#include +#include + +#include + +#include +#include #include #include #include "arm.h" +#include "handle_exit.h" #define CREATE_TRACE_POINTS #include "trace.h" @@ -200,6 +208,22 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) VCPU_EVENT(vcpu, 3, "%s", "free cpu"); } +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + save_access_regs(&vcpu->arch.host_acrs[0]); + vcpu->cpu = cpu; + + lasrm(&vcpu->arch.save_area); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + stiasrm(&vcpu->arch.save_area); + + vcpu->cpu = -1; + restore_access_regs(&vcpu->arch.host_acrs[0]); +} + int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -225,6 +249,399 @@ static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) return features; } +bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) +{ + return vcpu_mode_priv(vcpu); +} + +int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + if (!kvm_arm_vcpu_is_finalized(vcpu)) + return -EPERM; + + if (likely(READ_ONCE(vcpu->pid))) + return 0; + + scoped_guard(mutex, &kvm->arch.config_lock) + set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags); + + return 0; +} + +/** + * check_vcpu_requests - check and handle pending vCPU requests + * @vcpu: the VCPU pointer + * + * Return: 1 if we should enter the guest + * 0 if we should exit to userspace + * < 0 if we should exit to userspace, where the return value indicates + * an error + */ +static int check_vcpu_requests(struct kvm_vcpu *vcpu) +{ + if (kvm_request_pending(vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) + return -EIO; + if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) + kvm_reset_vcpu(vcpu); + /* + * Clear IRQ_PENDING requests that were made to guarantee + * that a VCPU sees new virtual interrupts. + */ + kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu); + + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) + asce_flush_tlb(vcpu->kvm->arch.gmap->asce); + } + + return 1; +} + +static int kvm_vcpu_initialize(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + unsigned long features = init->features[0]; + struct kvm *kvm = vcpu->kvm; + + guard(mutex)(&kvm->arch.config_lock); + if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) && + kvm_vcpu_init_changed(vcpu, init)) + return -EINVAL; + + bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES); + + kvm_reset_vcpu(vcpu); + + set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags); + vcpu_set_flag(vcpu, VCPU_INITIALIZED); + + return 0; +} + +static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + int ret; + + if (init->target != KVM_ARM_TARGET_GENERIC_V8) + return -EINVAL; + + ret = kvm_vcpu_init_check_features(vcpu, init); + if (ret) + return ret; + + if (!kvm_vcpu_initialized(vcpu)) + return kvm_vcpu_initialize(vcpu, init); + + if (kvm_vcpu_init_changed(vcpu, init)) + return -EINVAL; + + kvm_reset_vcpu(vcpu); + + return 0; +} + +static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, + struct kvm_vcpu_init *init) +{ + struct kvm_sae_save_area *save_area = &vcpu->arch.save_area; + struct kvm_sae_block *sae_block = &vcpu->arch.sae_block; + int ret; + + sae_block->save_area = virt_to_phys(save_area); + save_area->sdo = virt_to_phys(sae_block); + + ret = kvm_vcpu_set_target(vcpu, init); + if (ret) + return ret; + + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); + } + + return 0; +} + +int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, + bool line_status) +{ + /* stub for now*/ + return -EINVAL; +} + +static void adjust_pc(struct kvm_vcpu *vcpu) +{ + if (vcpu_get_flag(vcpu, INCREMENT_PC)) { + kvm_skip_instr(vcpu); + vcpu_clear_flag(vcpu, INCREMENT_PC); + } +} + +static noinstr void arm_vcpu_enter_exit(phys_addr_t sae_block) +{ + guest_timing_enter_irqoff(); + guest_state_enter_irqoff(); + + /* + * External interrupts may occur while the SAE instruction is executing + * and must preempt guest execution to maintain responsiveness of the + * host. + * + * As a temporary workaround, run SAE with interrupts enabled so that any + * interrupt is taken via the normal host interrupt path rather than + * being subsumed inside the SAE window. + */ + raw_local_irq_enable(); + sae(sae_block); + raw_local_irq_disable(); + + guest_state_exit_irqoff(); + guest_timing_exit_irqoff(); +} + +/** kvm_arch_vcpu_ioctl_run() - run arm64 vCPU + * + * Execute arm64 guest instructions using SAE. + * + * Returns: + * 1 enter the guest (should not be observed by userspace) + * 0 exit to userspace + * < 0 exit to userspace, where the return value indicates an error + * + * + */ +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) +{ + DECLARE_KERNEL_FPU_ONSTACK32(fpu_save); + struct kvm_run *kvm_run = vcpu->run; + int ret; + + if (kvm_run->exit_reason == KVM_EXIT_MMIO) { + ret = kvm_handle_mmio_return(vcpu); + if (ret <= 0) + return ret; + } + + vcpu_load(vcpu); + + kernel_fpu_begin(&fpu_save, KERNEL_FPC | KERNEL_VXR); + load_vx_regs((vcpu->arch.ctxt.vregs)); + + if (!vcpu->wants_to_run) { + ret = -EINTR; + goto out; + } + + kvm_sigset_activate(vcpu); + + might_fault(); + + ret = 1; + kvm_run->exit_reason = KVM_EXIT_UNKNOWN; + while (ret > 0) { + ret = kvm_xfer_to_guest_mode_handle_work(vcpu); + if (!ret) + ret = 1; + + if (ret > 0) + ret = check_vcpu_requests(vcpu); + + vcpu->arch.sae_block.icptr = 0; + + local_irq_disable(); + xfer_to_guest_mode_prepare(); + /* + * The full memory barrier in smp_store_mb() orders the mode + * store against the kvm_request_pending() check below, so + * any pending kick is visible before we decide to enter the + * guest. + */ + smp_store_mb(vcpu->mode, IN_GUEST_MODE); + + if (ret <= 0 || kvm_request_pending(vcpu) || + xfer_to_guest_mode_work_pending()) { + vcpu->mode = OUTSIDE_GUEST_MODE; + local_irq_enable(); + continue; + } + + trace_kvm_entry(*vcpu_pc(vcpu)); + + adjust_pc(vcpu); + + arm_vcpu_enter_exit(virt_to_phys(&vcpu->arch.sae_block)); + + vcpu->mode = OUTSIDE_GUEST_MODE; + vcpu->stat.exits++; + + local_irq_enable(); + trace_kvm_exit(kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); + + ret = handle_exit(vcpu); + } + + kvm_sigset_deactivate(vcpu); +out: + if (unlikely(vcpu_get_flag(vcpu, INCREMENT_PC))) + adjust_pc(vcpu); + + save_vx_regs(vcpu->arch.ctxt.vregs); + kernel_fpu_end(&fpu_save, KERNEL_FPC | KERNEL_VXR); + vcpu_put(vcpu); + + return ret; +} + +static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + struct kvm_device_attr attr; + int ret; + + switch (ioctl) { + case KVM_ARM_VCPU_INIT: { + struct kvm_vcpu_init init; + + ret = -EFAULT; + if (copy_from_user(&init, argp, sizeof(init))) + break; + + ret = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init); + break; + } + case KVM_SET_ONE_REG: + case KVM_GET_ONE_REG: { + struct kvm_one_reg reg; + + ret = -ENOEXEC; + if (unlikely(!kvm_vcpu_initialized(vcpu))) + break; + + ret = -EFAULT; + if (copy_from_user(®, argp, sizeof(reg))) + break; + + if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) + kvm_reset_vcpu(vcpu); + + if (ioctl == KVM_SET_ONE_REG) + ret = kvm_arm_set_reg(vcpu, ®); + else + ret = kvm_arm_get_reg(vcpu, ®); + break; + } + case KVM_GET_REG_LIST: { + struct kvm_reg_list __user *user_list = argp; + struct kvm_reg_list reg_list; + unsigned int n; + + ret = -ENOEXEC; + if (unlikely(!kvm_vcpu_initialized(vcpu))) + break; + ret = -EPERM; + if (!kvm_arm_vcpu_is_finalized(vcpu)) + break; + ret = -EFAULT; + if (copy_from_user(®_list, user_list, sizeof(reg_list))) + break; + n = reg_list.n; + reg_list.n = kvm_arm_num_regs(vcpu); + if (copy_to_user(user_list, ®_list, sizeof(reg_list))) + break; + ret = -E2BIG; + if (n < reg_list.n) + break; + ret = kvm_arm_copy_reg_indices(vcpu, user_list->reg); + break; + } + case KVM_ARM_VCPU_FINALIZE: { + int what; + + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + if (get_user(what, (const int __user *)argp)) + return -EFAULT; + + ret = kvm_arm_vcpu_finalize(vcpu, what); + break; + } + case KVM_SET_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_set_attr(vcpu, &attr); + break; + } + case KVM_GET_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_get_attr(vcpu, &attr); + break; + } + case KVM_HAS_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_has_attr(vcpu, &attr); + break; + } + default: + ret = -EINVAL; + } + + return ret; +} + int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) { diff --git a/arch/s390/kvm/arm64/guest.c b/arch/s390/kvm/arm64/guest.c index 1bf546bded81..4ea6fe2c270d 100644 --- a/arch/s390/kvm/arm64/guest.c +++ b/arch/s390/kvm/arm64/guest.c @@ -45,6 +45,51 @@ const struct kvm_stats_header kvm_vcpu_stats_header = { sizeof(kvm_vcpu_stats_desc), }; +int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) +{ + int ret; + + ret = copy_core_reg_indices(vcpu, uindices); + if (ret < 0) + return ret; + uindices += ret; + + return 0; +} + +unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) +{ + return num_core_regs(vcpu); +} + +int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + /* We currently use nothing arch-specific in upper 32 bits */ + if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) + return -EINVAL; + + switch (reg->id & KVM_REG_ARM_COPROC_MASK) { + case KVM_REG_ARM_CORE: + return get_core_reg(vcpu, reg); + default: + return -EINVAL; + } +} + +int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + /* We currently use nothing arch-specific in upper 32 bits */ + if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) + return -EINVAL; + + switch (reg->id & KVM_REG_ARM_COPROC_MASK) { + case KVM_REG_ARM_CORE: + return set_core_reg(vcpu, reg); + default: + return -EINVAL; + } +} + int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) { return -EINVAL; diff --git a/arch/s390/kvm/arm64/handle_exit.c b/arch/s390/kvm/arm64/handle_exit.c index 07d8d7a53702..dbf6edaeefaf 100644 --- a/arch/s390/kvm/arm64/handle_exit.c +++ b/arch/s390/kvm/arm64/handle_exit.c @@ -5,6 +5,8 @@ #include #include +#include "handle_exit.h" + typedef int (*exit_handle_fn)(struct kvm_vcpu *); #define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 @@ -84,6 +86,36 @@ static int handle_trap_exceptions(struct kvm_vcpu *vcpu) return exit_handler(vcpu); } +/* + * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on + * proper exit to userspace. + */ +int handle_exit(struct kvm_vcpu *vcpu) +{ + u8 icptr = vcpu->arch.sae_block.icptr; + int ret = 1; + + switch (icptr) { + case SAE_ICPTR_SPURIOUS: + break; + case SAE_ICPTR_VALIDITY: + WARN_ONCE(true, "SAE: validity intercept. vir: 0x%04x", + vcpu->arch.sae_block.vir); + vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; + ret = 0; + break; + case SAE_ICPTR_GUEST_ADDRESS_SIZE: + case SAE_ICPTR_HOST_ACCESS_EXCEPTION: + case SAE_ICPTR_SYNCHRONOUS_EXCEPTION: + ret = handle_trap_exceptions(vcpu); + break; + default: + vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; + ret = 0; + } + return ret; +} + /* manually copied from arch/arm64/kernel/traps.c */ static const char * const esr_class_str[] = { [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", diff --git a/arch/s390/kvm/arm64/handle_exit.h b/arch/s390/kvm/arm64/handle_exit.h new file mode 100644 index 000000000000..65ae58721537 --- /dev/null +++ b/arch/s390/kvm/arm64/handle_exit.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef KVM_ARM64_HANDLE_EXIT_H +#define KVM_ARM64_HANDLE_EXIT_H + +#include + +int handle_exit(struct kvm_vcpu *vcpu); + +#endif /* KVM_ARM64_HANDLE_EXIT_H */ diff --git a/arch/s390/kvm/arm64/reset.c b/arch/s390/kvm/arm64/reset.c new file mode 100644 index 000000000000..e4cdce1dbf9d --- /dev/null +++ b/arch/s390/kvm/arm64/reset.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include + +bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu) +{ + return true; +} + +static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) +{ + vcpu->arch.hcr_el2 = HCR_EL2_E2H | HCR_EL2_RW | HCR_EL2_AMO | + HCR_EL2_IMO | HCR_EL2_FMO | HCR_EL2_PTW; + /* traps */ + vcpu->arch.hcr_el2 |= HCR_EL2_TSC | HCR_EL2_TID1 | HCR_EL2_TID2 | + HCR_EL2_TID3 | HCR_EL2_TID4 | HCR_EL2_TID5 | + HCR_EL2_TIDCP | HCR_EL2_TLOR; +} + +void kvm_reset_vcpu(struct kvm_vcpu *vcpu) +{ + struct vcpu_reset_state reset_state; + bool loaded; + + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + reset_state = vcpu->arch.reset_state; + vcpu->arch.reset_state.reset = false; + } + + /* + * Disable preemption around the vcpu reset as we might otherwise race with + * preempt notifiers which call stiasrm/lasrm from put/load + */ + preempt_disable(); + + /* The reset must run with an unloaded save area */ + loaded = vcpu_is_loaded(vcpu); + if (loaded) + vcpu_put(vcpu); + + kvm_reset_vcpu_core(vcpu); + + /* Reset special registers */ + vcpu_reset_hcr(vcpu); + + if (reset_state.reset) { + *vcpu_pc(vcpu) = reset_state.pc; + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); + vcpu_clear_flag(vcpu, EXCEPT_MASK); + vcpu_clear_flag(vcpu, INCREMENT_PC); + vcpu_set_reg(vcpu, 0, reset_state.r0); + } + + /* Load new vx-regs into HW if they are currently loaded */ + if (current->thread.kfpu_flags) + load_vx_regs(vcpu->arch.ctxt.vregs); + + if (loaded) + vcpu_load(vcpu); + + preempt_enable(); +} + +int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature) +{ + return -EINVAL; +} diff --git a/arch/s390/kvm/arm64/trace.h b/arch/s390/kvm/arm64/trace.h index 3a5dcd039c13..0380e45fba28 100644 --- a/arch/s390/kvm/arm64/trace.h +++ b/arch/s390/kvm/arm64/trace.h @@ -5,7 +5,41 @@ #include #undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm +#define TRACE_SYSTEM kvm_arm64 +#undef TRACE_SYSTEM_VAR +#define TRACE_SYSTEM_VAR kvm_arm64 + +TRACE_EVENT(kvm_entry, + TP_PROTO(unsigned long vcpu_pc), + TP_ARGS(vcpu_pc), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + ), + + TP_fast_assign(__entry->vcpu_pc = vcpu_pc;), + + TP_printk("PC: 0x%016lx", __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_exit, + TP_PROTO(unsigned int esr_ec, unsigned long vcpu_pc), + TP_ARGS(esr_ec, vcpu_pc), + + TP_STRUCT__entry( + __field(unsigned int, esr_ec ) + __field(unsigned long, vcpu_pc ) + ), + + TP_fast_assign( + __entry->esr_ec = esr_ec; + __entry->vcpu_pc = vcpu_pc; + ), + + TP_printk("HSR_EC: 0x%04x, PC: 0x%016lx", + __entry->esr_ec, + __entry->vcpu_pc) +); #endif /* KVM_ARM64_TRACE_KVM_H */ From 4265225bbeb46e1d63b5cc55205ab6fd2a19f6c6 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 499/506] KVM: s390: arm64: Implement basic page fault handler Add host functionality to page in guest memory. If the guest does something unexpected or illegal exit to userspace abort guest execution. This behaviour will be changed to guest error injects once all sysregs are accessible for the host. Add pre-fault handler. Co-developed-by: Nina Schoetterl-Glausch Signed-off-by: Nina Schoetterl-Glausch Signed-off-by: Steffen Eiden [borntraeger@linux.ibm.com: fixup for recent arm changes] Signed-off-by: Christian Borntraeger --- arch/s390/kvm/arm64/handle_exit.c | 2 + arch/s390/kvm/arm64/mmio.c | 15 +++ arch/s390/kvm/arm64/mmu.c | 210 ++++++++++++++++++++++++++++++ arch/s390/kvm/arm64/trace.h | 49 +++++++ 4 files changed, 276 insertions(+) create mode 100644 arch/s390/kvm/arm64/mmio.c create mode 100644 arch/s390/kvm/arm64/mmu.c diff --git a/arch/s390/kvm/arm64/handle_exit.c b/arch/s390/kvm/arm64/handle_exit.c index dbf6edaeefaf..e0c7773c2165 100644 --- a/arch/s390/kvm/arm64/handle_exit.c +++ b/arch/s390/kvm/arm64/handle_exit.c @@ -62,6 +62,8 @@ static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu) static exit_handle_fn arm_exit_handlers[] = { [0 ... ESR_ELx_EC_MAX] = kvm_handle_unknown_ec, [ESR_ELx_EC_HVC64] = handle_hvc, + [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort, + [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort, }; static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) diff --git a/arch/s390/kvm/arm64/mmio.c b/arch/s390/kvm/arm64/mmio.c new file mode 100644 index 000000000000..908a6b43ddf7 --- /dev/null +++ b/arch/s390/kvm/arm64/mmio.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include + +#include +#include + +#include "trace.h" + +#define __INCL_GEN_ARM_FILE +#include "generated/mmio.inc" +#undef __INCL_GEN_ARM_FILE diff --git a/arch/s390/kvm/arm64/mmu.c b/arch/s390/kvm/arm64/mmu.c new file mode 100644 index 000000000000..31e0d036896f --- /dev/null +++ b/arch/s390/kvm/arm64/mmu.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include + +#include + +#include "trace.h" + +static inline bool kvm_s390_cur_gmap_fault_is_write(struct kvm_vcpu *vcpu) +{ + return (vcpu->arch.sae_block.hai.pic & PGM_INT_CODE_MASK) == PGM_PROTECTION || + vcpu->arch.sae_block.hai.teid.fsi == TEID_FSI_STORE; +} + +/* + * user_mem_abort() - handle a dat fault for the gmap of a vcpu + * + * Return: 0 on success, < 0 in case of error. + * Context: The mm lock must not be held before calling. May sleep. + */ +static int user_mem_abort(struct kvm_vcpu *vcpu, gpa_t fault_ipa, + struct kvm_memory_slot *slot, hva_t hva) +{ + struct guest_fault f = { }; + int ret; + + if (kvm_s390_cur_gmap_fault_is_write(vcpu)) + f.write_attempt = FOLL_WRITE; + f.gfn = gpa_to_gfn(fault_ipa); + + ret = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (ret <= 0) + return ret; + if (ret == PGM_ADDRESSING) + /* + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + return -ENOEXEC; + KVM_BUG_ON(ret, vcpu->kvm); + return -EINVAL; +} + +static int kvm_handle_pic(struct kvm_vcpu *vcpu, bool *translation) +{ + switch (kvm_vcpu_fault_pic(vcpu)) { + /* expected cases: */ + case PGM_ASCE_TYPE: + case PGM_REGION_FIRST_TRANS: + case PGM_REGION_SECOND_TRANS: + case PGM_REGION_THIRD_TRANS: + case PGM_SEGMENT_TRANSLATION: + case PGM_PAGE_TRANSLATION: + *translation = true; + break; + case PGM_PROTECTION: + break; + /* unexpected cases: */ + case 0: + KVM_BUG(1, vcpu->kvm, "On MMU fault path but no fault occurred"); + return -EFAULT; + default: + KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx", + vcpu->arch.sae_block.hai.pic, vcpu->arch.sae_block.hai.teid.val); + send_sig(SIGSEGV, current, 0); + return -EFAULT; + } + + return 0; +} + +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) +{ + struct kvm_memory_slot *memslot; + bool translation = false; + phys_addr_t fault_ipa; + unsigned long hva; + unsigned long esr; + bool write_fault; + bool guest_size_err; + bool writable; + bool is_iabt; + int ret; + gfn_t gfn; + + esr = kvm_vcpu_get_esr(vcpu); + fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); + is_iabt = kvm_vcpu_trap_is_iabt(vcpu); + guest_size_err = vcpu->arch.sae_block.icptr == SAE_ICPTR_GUEST_ADDRESS_SIZE; + + if (guest_size_err) { + translation = true; + } else { + ret = kvm_handle_pic(vcpu, &translation); + if (ret) + return ret; + } + + if (translation) { + /* + * For both cases: + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + if (fault_ipa >= BIT_ULL(get_kvm_ipa_limit())) + return -ENOEXEC; + + if (fault_ipa >= vcpu->kvm->arch.guest_phys_size) + return -ENOEXEC; + } + + trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu), + kvm_vcpu_get_hfar(vcpu), fault_ipa); + + kvm_vcpu_srcu_read_lock(vcpu); + + gfn = fault_ipa >> PAGE_SHIFT; + + memslot = gfn_to_memslot(vcpu->kvm, gfn); + hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); + write_fault = kvm_s390_cur_gmap_fault_is_write(vcpu); + if (kvm_is_error_hva(hva) || (write_fault && !writable)) { + ret = -ENOEXEC; + /* + * The guest has put either its instructions or its page-tables + * somewhere it shouldn't have. Userspace won't be able to do + * anything about this (there's no syndrome for a start). + * + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + if (is_iabt) + goto out_unlock; + + if (kvm_vcpu_abt_iss1tw(vcpu)) { + /* + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + goto out_unlock; + } + + /* + * Check for a cache maintenance operation. Assume the guest is + * cautious and skip instruction + */ + if (kvm_is_error_hva(hva) && esr_dabt_is_cm(esr)) { + kvm_incr_pc(vcpu); + ret = 1; + goto out_unlock; + } + + /* + * The IPA is reported as [MAX:12], so we need to + * complement it with the bottom 12 bits from the + * faulting VA. This is always 12 bits, irrespective + * of the page size. + */ + fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1); + ret = io_mem_abort(vcpu, fault_ipa); + goto out_unlock; + } + + /* Userspace should not be able to register out-of-bounds IPAs */ + VM_BUG_ON(fault_ipa >= vcpu->kvm->arch.guest_phys_size); + /* + * Proper guest size faults have been injected. + * In theory it's fine to have device memory higher than MSL, + * even if not currently possible, but that would have been handled above. + * So if we get here with a guest size intercept, we have a bug somewhere. + */ + VM_BUG_ON(guest_size_err); + + ret = user_mem_abort(vcpu, fault_ipa, memslot, hva); + if (!ret) + ret = 1; +out_unlock: + kvm_vcpu_srcu_read_unlock(vcpu); + return ret; +} + +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) +{ + struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), }; + gpa_t end; + int rc; + + rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (rc == PGM_ADDRESSING) + return -ENOENT; + if (rc > 0) + return -EIO; + if (rc < 0) + return rc; + + if (f.ptep) + return PAGE_SIZE; + + end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE); + return min(range->size, end - range->gpa); +} + +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + return 0; +} diff --git a/arch/s390/kvm/arm64/trace.h b/arch/s390/kvm/arm64/trace.h index 0380e45fba28..ee232c4f5a6d 100644 --- a/arch/s390/kvm/arm64/trace.h +++ b/arch/s390/kvm/arm64/trace.h @@ -41,6 +41,55 @@ TRACE_EVENT(kvm_exit, __entry->vcpu_pc) ); +TRACE_EVENT(kvm_mmio_nisv, + TP_PROTO(unsigned long vcpu_pc, unsigned long esr, + unsigned long far, unsigned long ipa), + TP_ARGS(vcpu_pc, esr, far, ipa), + + TP_STRUCT__entry( + __field( unsigned long, vcpu_pc ) + __field( unsigned long, esr ) + __field( unsigned long, far ) + __field( unsigned long, ipa ) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->esr = esr; + __entry->far = far; + __entry->ipa = ipa; + ), + + TP_printk("ipa %#016lx, esr %#016lx, far %#016lx, pc %#016lx", + __entry->ipa, __entry->esr, + __entry->far, __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_guest_fault, + TP_PROTO(unsigned long vcpu_pc, unsigned long hsr, + unsigned long hxfar, + unsigned long long ipa), + TP_ARGS(vcpu_pc, hsr, hxfar, ipa), + + TP_STRUCT__entry( + __field( unsigned long, vcpu_pc ) + __field( unsigned long, hsr ) + __field( unsigned long, hxfar ) + __field( unsigned long long, ipa ) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->hsr = hsr; + __entry->hxfar = hxfar; + __entry->ipa = ipa; + ), + + TP_printk("ipa %#llx, hsr %#08lx, hxfar %#08lx, pc %#016lx", + __entry->ipa, __entry->hsr, + __entry->hxfar, __entry->vcpu_pc) +); + #endif /* KVM_ARM64_TRACE_KVM_H */ #undef TRACE_INCLUDE_PATH From 91b87e6eea4d09a4d80f6a66515e8bc2821303ad Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 500/506] KVM: s390: arm64: Integrate arm on s390 Add all Kbuild/Makefile configurations to build a second KVM module on s390 implementing the arm64 KVM API. The new module is named kvm_arm64. Symbol conflicts with kvm_s390 are prevented by localising all internal symbols in kvm_arm64 when compiled as built-in (=y): kvm-unnamespaced.o holds all constituent objects. objcopy --localize-symbols then produces kvm-namespaced.o which is what kvm-arm64.o resolves to. For =m the normal module symbol scoping applies and no localisation step is needed. Because the %-arm64.o pattern targets do not match any kvm-arm64-y entry, kbuild would derive a wrong KBUILD_MODNAME (e.g. "kvm_main_arm64") and a wrong KBUILD_MODFILE for the virt/kvm and gmap copies. Co-developed-by: Andreas Grapentin Signed-off-by: Andreas Grapentin Co-developed-by: Nina Schoetterl-Glausch Signed-off-by: Nina Schoetterl-Glausch Co-developed-by: Gautam Gala Signed-off-by: Gautam Gala Signed-off-by: Steffen Eiden Signed-off-by: Christian Borntraeger --- arch/s390/kvm/Kconfig | 21 ++++--- arch/s390/kvm/Makefile | 1 + arch/s390/kvm/arm64/Makefile | 110 +++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 arch/s390/kvm/arm64/Makefile diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 9045ae20fd89..e7b525d04c6d 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -30,15 +30,22 @@ config KVM select KVM_MMU_LOCKLESS_AGING select KVM_GENERIC_PRE_FAULT_MEMORY help - Support hosting paravirtualized guest machines using the SIE - virtualization capability on the mainframe. This should work - on any 64bit machine. + Enable support for hosting virtualized, hardware-accelerated + virtual machines via two modules: kvm and kvm_arm64. - This module provides access to the hardware capabilities through - a character device node named /dev/kvm. + The kvm module provides access to host hardware-accelerated s390 + guests using the Start Interpretive Execution (SIE) instruction via a + character device node named /dev/kvm. This requires a 64-bit machine + (hardware model). - To compile this as a module, choose M here: the module - will be called kvm. + The kvm_arm64 module provides access to host hardware-accelerated + arm64 guests using the Start ARM Execution (SAE) instruction via a + character device node named /dev/kvm_arm64. This requires hardware + models that support the Arm execution facility (AEF). + + To compile this driver as a module, choose M here. The module will be + called kvm for the SIE acceleration and kvm_arm64 for the SAE + acceleration. If unsure, say N. diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index a4e3875b5bdd..380db443a0e9 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -4,3 +4,4 @@ # Copyright IBM Corp. 2008 obj-$(CONFIG_KVM) += s390/ +obj-$(CONFIG_KVM) += arm64/ diff --git a/arch/s390/kvm/arm64/Makefile b/arch/s390/kvm/arm64/Makefile new file mode 100644 index 000000000000..3f0fe5c9f0a1 --- /dev/null +++ b/arch/s390/kvm/arm64/Makefile @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: GPL-2.0 + +KVM := ../../../../virt/kvm +KVM_DEV_NAME = kvm_arm64 +KVM_DEV_MINOR = MISC_DYNAMIC_MINOR +include $(srctree)/virt/kvm/Makefile.kvm +include $(srctree)/arch/$(SRCARCH)/kvm/gmap/Makefile +include $(src)/Makefile.gen + +ccflags-y += -I $(src) -I $(objtree)/$(obj) -I$(srctree)/arch/$(SRCARCH)/kvm/gmap -DKVM_S390_ARM64 + +kvm-arm64-local := \ + arm.o \ + guest.o \ + handle_exit.o \ + inject_fault.o \ + mmio.o \ + mmu.o \ + reset.o + +kvm-arm64-obj := $(kvm-arm64-local) +kvm-arm64-obj += $(patsubst %.o,%-arm64.o,$(kvm-y)) +kvm-arm64-obj += $(patsubst %.o,%-arm64.o,$(gmap-y)) + +targets += $(kvm-arm64-obj) + +$(obj)/%-arm64.o: $(src)/%.c FORCE + @mkdir -p $(dir $@) + $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) + +# %-arm64.o stems don't match any kvm-arm64-y entry so Kbuild derives wrong +# KBUILD_MODNAME/KBUILD_MODFILE. Force include a generated header to override. +KVM_ARM64_MODNAME_H := $(obj)/generated/kvm_arm64_modname.h +ccflags-y += -include $(KVM_ARM64_MODNAME_H) + +$(addprefix $(obj)/,$(kvm-arm64-obj)): $(KVM_ARM64_MODNAME_H) +$(foreach sfx, i s ll lst, \ + $(eval $(patsubst %.o,%.$(sfx),$(addprefix $(obj)/,$(kvm-arm64-local))): $(KVM_ARM64_MODNAME_H))) + +targets += generated/kvm_arm64_modname.h + +define filechk_kvm_arm64_modname + echo '#ifndef __KVM_ARM64_MODNAME_H'; \ + echo '#define __KVM_ARM64_MODNAME_H'; \ + echo '#undef KBUILD_MODNAME'; \ + echo '#define KBUILD_MODNAME "$(KVM_DEV_NAME)"'; \ + echo '#undef __KBUILD_MODNAME'; \ + echo '#define __KBUILD_MODNAME $(KVM_DEV_NAME)'; \ + echo '#undef KBUILD_MODFILE'; \ + echo '#define KBUILD_MODFILE "$(obj)/kvm-arm64"'; \ + echo '#endif /* __KVM_ARM64_MODNAME_H */' +endef + +$(KVM_ARM64_MODNAME_H): FORCE + $(call filechk,kvm_arm64_modname) + +# Members are not in real-obj-y. Set part-of-builtin to enable instrumentation. +$(addprefix $(obj)/,$(kvm-arm64-obj)): private part-of-builtin := y + +ifeq ($(CONFIG_KVM),m) + +kvm-arm64-y = $(kvm-arm64-obj) + +else ifeq ($(CONFIG_KVM),y) + +prereq-o-cmd = $(foreach o, $(filter %.o, $^), $(dir $(o)).$(notdir $(o)).cmd) +cmd_gen_symversions_o = \ + grep --no-filename "^\#SYMVER" $(prereq-o-cmd) >> $(dot-target).cmd || true + +cmd_ld_o_o = $(cmd_ld) +define rule_ld_o_o + $(call cmd_and_savecmd,ld) + $(call cmd,gen_symversions_o) +endef + +LDFLAGS_kvm-unnamespaced.o := -r +$(obj)/kvm-unnamespaced.o: $(addprefix $(obj)/,$(kvm-arm64-obj)) FORCE + $(call if_changed_rule,ld_o_o) + +quiet_cmd_nm_filter = NMFLTR $@ + cmd_nm_filter = $(NM) --defined-only --format=just-symbols $< | sort -u > $@ + +$(obj)/kvm_symbol_list: $(obj)/kvm-unnamespaced.o FORCE + $(call if_changed,nm_filter) + +# cmd_oc_o_o carries -DKBUILD_MODFILE so generate_builtin_ranges.awk +# can attribute kvm-namespaced.o to kvm_arm64 via its .cmd file. +cmd_oc_o_o = $(cmd_objcopy) -DKBUILD_MODFILE='"$(obj)/kvm-arm64"' +define rule_oc_o_o + $(call cmd,objcopy) + @printf '%s\n' 'savedcmd_$@ := $(call make-cmd,oc_o_o)' > $(dot-target).cmd + $(call cmd,gen_objtooldep) + $(call cmd,gen_symversions_o) +endef + +OBJCOPYFLAGS_kvm-namespaced.o := -O $(LD_BFD) --localize-symbols=$(obj)/kvm_symbol_list +$(obj)/kvm-namespaced.o: $(obj)/kvm-unnamespaced.o $(obj)/kvm_symbol_list FORCE + $(call if_changed_rule,oc_o_o) + +kvm-arm64-y = kvm-namespaced.o + +targets += kvm-unnamespaced.o kvm_symbol_list kvm-namespaced.o + +endif + +clean-files += kvm-unnamespaced.o kvm_symbol_list kvm-namespaced.o + +obj-$(CONFIG_KVM) += kvm-arm64.o + From ffae233b4d536c7eb6bbad21bc8afa41f6be2929 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 12 Oct 2026 17:57:25 +0200 Subject: [PATCH 501/506] MAINTAINERS: Add entry for ARM on s390 KVM Add a dedicated MAINTAINERS section for KVM ARM64 virtualization support on s390. Adding only reviewers. arm-on-s390 is primarily maintained by the KVM/s390 maintainers through inheritance. Signed-off-by: Steffen Eiden Acked-by: Andreas Grapentin Acked-by: Marc Zyngier Acked-by: Janosch Frank Signed-off-by: Christian Borntraeger --- MAINTAINERS | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 1d45ccf863c1..d9d212d06ef2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14385,6 +14385,22 @@ F: tools/testing/selftests/drivers/s390x/uvdevice/ F: tools/testing/selftests/kvm/*/s390/ F: tools/testing/selftests/kvm/s390/ +KERNEL VIRTUAL MACHINE for arm-on-s390 (KVM/s390) +R: Steffen Eiden +R: Andreas Grapentin +R: Marc Zyngier +L: kvm@vger.kernel.org +L: linux-s390@vger.kernel.org +S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git +F: arch/s390/include/arm64/ +F: arch/s390/include/asm/aef.h +F: arch/s390/include/asm/kvm_host_arm64* +F: arch/s390/include/asm/sae* +F: arch/s390/kernel/aef.c +F: arch/s390/kvm/arm64/ +F: arch/s390/tools/*arm64* + KERNEL VIRTUAL MACHINE FOR X86 (KVM/x86) M: Sean Christopherson M: Paolo Bonzini From e93fb70b748982fda5e6d97721df35194cacfc4a Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 24 Sep 2026 12:57:18 +0200 Subject: [PATCH 502/506] arm64: Use proper include variant Use <> for including the sysreg header as it is not discoverable from the current directory. Acked-by: Marc Zyngier Signed-off-by: Steffen Eiden Acked-by: Mark Rutland Signed-off-by: Catalin Marinas Signed-off-by: Christian Borntraeger --- arch/arm64/include/asm/sysreg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index b4f5dedff5ae..aa65603455a2 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -179,7 +179,7 @@ * come from here. The header relies on the definition of sys_reg() * earlier in this file. */ -#include "asm/sysreg-defs.h" +#include /* * System registers, organised loosely by encoding but grouped together From 3a74cf568b9afd0eaa138bc40308c3ddf0f547bc Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 20 Oct 2026 08:59:35 -0400 Subject: [PATCH 503/506] selftests: kvm: remove useless comma operator This is the result of an incorrect conflict resolution in merge commit 994489f04ebb ("Merge tag 'kvm-x86-selftests-7.4' of https://github.com/kvm-x86/linux into HEAD", 2026-10-16). Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/x86/nested_x2apic_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c index bba301cfc0b5..2ae698c9aa1c 100644 --- a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c @@ -86,7 +86,7 @@ static void l1_vmx_code(struct vmx_pages *vmx, struct hyperv_test_pages *hv_page SECONDARY_EXEC_APIC_REGISTER_VIRT | SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY; control &= (rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32); - vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0; + vmwrite(SECONDARY_VM_EXEC_CONTROL, control); vmlaunch(); GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_CPUID); From 28d5df7b727471439ca343208e5be1fd55338128 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 19 Oct 2026 13:54:07 -0400 Subject: [PATCH 504/506] KVM: split CONFIG_KVM_PRIVATE_MEMATTR_VM out of KVM_VM_MEMORY_ATTRIBUTES With Linux 7.4 the "private" memory attribute was moved from the VM to the individual guest_memfd file descriptor. The old semantics are available for backwards compatibility via CONFIG_KVM_VM_MEMORY_ATTRIBUTES. However, this ties support for VM memory attributes entirely to a backwards compatibility symbol, which is a problem when adding VM memory attributes other than PRIVATE. Thus, make CONFIG_KVM_VM_MEMORY_ATTRIBUTES the equivalent of the old CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES, and use a more descriptive name for the private memory attribute. Do this now, instead of waiting for more attributes to come, before distros start adjusting their build configuration. Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini --- Documentation/admin-guide/kernel-parameters.txt | 4 ++-- arch/x86/kvm/Kconfig | 3 ++- include/linux/kvm_host.h | 4 +++- virt/kvm/Kconfig | 3 +++ virt/kvm/kvm_main.c | 6 +++--- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 410a3eed3a3d..42ff19defe16 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3178,8 +3178,8 @@ Kernel parameters userspace VMM design. Note, this parameter is only available when - CONFIG_KVM_VM_MEMORY_ATTRIBUTES=y. When - CONFIG_KVM_VM_MEMORY_ATTRIBUTES is not set, in-place + CONFIG_KVM_PRIVATE_MEMATTR_VM=y. When + CONFIG_KVM_PRIVATE_MEMATTR_VM is not set, in-place conversion is enabled (and cannot be disabled) if KVM is built with private memory support. diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 796d33f629d3..01a683a9042a 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -76,9 +76,10 @@ config KVM_WERROR If in doubt, say "N". -config KVM_VM_MEMORY_ATTRIBUTES +config KVM_PRIVATE_MEMATTR_VM bool "Enable per-VM PRIVATE vs. SHARED attributes (for CoCo VMs)" depends on KVM_SW_PROTECTED_VM || KVM_INTEL_TDX || KVM_AMD_SEV + select KVM_VM_MEMORY_ATTRIBUTES help Enable support for tracking PRIVATE vs. SHARED memory using per-VM memory attributes. Using per-VM attributes is deprecated in favor of diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 4d4e76a50d77..09c57d31d153 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2582,12 +2582,14 @@ bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, struct kvm_gfn_range *range); bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, struct kvm_gfn_range *range); +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM static inline bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; } -#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_PRIVATE_MEMATTR_VM */ #ifdef kvm_arch_has_private_mem extern bool gmem_in_place_conversion; diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index c5bfaa1f7749..99c9969e77a4 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -101,3 +101,6 @@ config HAVE_KVM_ARCH_GMEM_INVALIDATE config HAVE_KVM_ARCH_GMEM_POPULATE bool depends on KVM_GUEST_MEMFD + +config KVM_VM_MEMORY_ATTRIBUTES + bool diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index a97aa5b92616..4a06d8678289 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -105,8 +105,8 @@ static bool __ro_after_init allow_unsafe_mappings; module_param(allow_unsafe_mappings, bool, 0444); #ifdef kvm_arch_has_private_mem -bool __ro_after_init gmem_in_place_conversion = !IS_ENABLED(CONFIG_KVM_VM_MEMORY_ATTRIBUTES); -#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +bool __ro_after_init gmem_in_place_conversion = !IS_ENABLED(CONFIG_KVM_PRIVATE_MEMATTR_VM); +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM module_param(gmem_in_place_conversion, bool, 0444); #endif EXPORT_SYMBOL_FOR_KVM_INTERNAL(gmem_in_place_conversion); @@ -2713,7 +2713,7 @@ static void kvm_init_memory_attributes(void) { if (gmem_in_place_conversion) static_call_update(__kvm_is_private_gfn, kvm_gmem_is_private_gfn); -#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM else static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn); #endif From 9bcd6af09e2a3493ee702c9aa1110002ed4e83e8 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 20 Oct 2026 06:12:22 -0400 Subject: [PATCH 505/506] KVM: x86: leave the legacy private attribute in for TDX and SEV As of today, and presumably as of Linux 7.4's release date, there is hardly any VMM that supports guest_memfd attributes. Support the legacy VM attribute for private memory if SEV or TDX are requested; default it to disabled if only the software-protected VM is requested, since it is only used by guest_memfd selftests. While in-place conversion is the best option, it is way too early to enable it by default. Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini --- arch/x86/kvm/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 01a683a9042a..1719059570fb 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -80,6 +80,7 @@ config KVM_PRIVATE_MEMATTR_VM bool "Enable per-VM PRIVATE vs. SHARED attributes (for CoCo VMs)" depends on KVM_SW_PROTECTED_VM || KVM_INTEL_TDX || KVM_AMD_SEV select KVM_VM_MEMORY_ATTRIBUTES + default KVM_INTEL_TDX || KVM_AMD_SEV help Enable support for tracking PRIVATE vs. SHARED memory using per-VM memory attributes. Using per-VM attributes is deprecated in favor of @@ -87,7 +88,7 @@ config KVM_PRIVATE_MEMATTR_VM CoCo VMs using a VMM that doesn't support guest_memfd memory attributes. - If unsure, say N. + If unsure, say Y. config KVM_SW_PROTECTED_VM bool "Enable support for KVM software-protected VMs" From 164405e8e9a0378c7e1aff2254625a6553a291f1 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 20 Oct 2026 06:38:31 -0400 Subject: [PATCH 506/506] KVM: always make in-place conversion a compile-time constant when possible Remove the unnecessary static call indirection and make gmem_in_place_conversion a constant if CONFIG_KVM_PRIVATE_MEMATTR_VM is false. Require that CONFIG_KVM_PRIVATE_MEMATTR_VM be enabled together with CONFIG_KVM_GUEST_MEMFD and with a definition of kvm_arch_has_private_mem. This way, the three possible cases can be expressed without nested #ifs. Reviewed-by: Xiaoyao Li Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 13 +++++-------- virt/kvm/kvm_main.c | 13 +++++++------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 09c57d31d153..e7fc94e2910a 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2585,15 +2585,7 @@ bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, #endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ #ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM -static inline bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn) -{ - return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; -} -#endif /* CONFIG_KVM_PRIVATE_MEMATTR_VM */ - -#ifdef kvm_arch_has_private_mem extern bool gmem_in_place_conversion; - typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn); DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t); @@ -2601,6 +2593,11 @@ static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { return static_call(__kvm_is_private_gfn)(kvm, gfn); } + +#elif defined kvm_arch_has_private_mem +#define gmem_in_place_conversion true +#define kvm_is_private_gfn kvm_gmem_is_private_gfn + #else #define gmem_in_place_conversion false diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 4a06d8678289..cf4cb586f140 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -104,11 +104,9 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink); static bool __ro_after_init allow_unsafe_mappings; module_param(allow_unsafe_mappings, bool, 0444); -#ifdef kvm_arch_has_private_mem -bool __ro_after_init gmem_in_place_conversion = !IS_ENABLED(CONFIG_KVM_PRIVATE_MEMATTR_VM); #ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM +bool __ro_after_init gmem_in_place_conversion = false; module_param(gmem_in_place_conversion, bool, 0444); -#endif EXPORT_SYMBOL_FOR_KVM_INTERNAL(gmem_in_place_conversion); #endif @@ -2705,18 +2703,21 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, } #endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ -#ifdef kvm_arch_has_private_mem +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM DEFINE_STATIC_CALL_RET0(__kvm_is_private_gfn, kvm_is_private_gfn_t); EXPORT_STATIC_CALL_GPL(__kvm_is_private_gfn); +static bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; +} + static void kvm_init_memory_attributes(void) { if (gmem_in_place_conversion) static_call_update(__kvm_is_private_gfn, kvm_gmem_is_private_gfn); -#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM else static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn); -#endif } #else static void kvm_init_memory_attributes(void) { }