From 3d736a339e9cbeec3841da1a821f56fb3230fd68 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:24 +0100 Subject: [PATCH 01/13] KVM: arm64: protected VM: Handle user writes to CNTVCT_EL0/CNTPCT_EL0 Protected VMs doesn't allow setting offsets for virtual and physical counters, as the offset is always fixed to 0. The VM ioctl is filtered out based on the cap. However we don't prevent the userspace from trying to write to the CNTVCT/CNTPCT registers. This would lead to KVM triggering a WARN() in timer_set_offset() as the vm_offset pointer is set to NULL. Fix this by always "fixing" the timer offsets to 0 and marking that the timer offset is set in the kvm->arch.flags at KVM init time for protected VMs. This prevents the access to the VM specific vm_offset at low cost. A userspace writing to the CNT*CT_EL0 would observe success, without any real effect. This is cleaner over spilling "*_is_protected()" checks and "matches" what we really do in practise. i.e., always run with "fixed counter offset of 0". Reported by Sashiko Link: https://lore.kernel.org/all/20260908164641.416911F00A3A@smtp.kernel.org Fixes: f7d05ee84a6a ("KVM: arm64: Prevent host from managing timer offsets for protected VMs") Suggested-by: Marc Zyngier Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-2-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arch_timer.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 6ac3321f4c57..a45845f4ae4e 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -1110,8 +1110,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) timer_context_init(vcpu, i); /* Synchronize offsets across timers of a VM if not already provided */ - if (!vcpu_is_protected(vcpu) && - !test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { + if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read()); timer_set_offset(vcpu_ptimer(vcpu), 0); } @@ -1133,6 +1132,15 @@ void kvm_timer_init_vm(struct kvm *kvm) */ for (int i = 0; i < NR_KVM_TIMERS; i++) kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]); + + /* + * Protected VMs don't allow the userspace to set counter offsets, + * either set via counter register writes or the dedicated ioctls. + * Pretend the offset has already been set and rely on the default + * offset being 0. + */ + if (kvm_vm_is_protected(kvm)) + set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags); } void kvm_timer_cpu_up(void) From 22adb9c3543cc702516c18f0a28e3a8dfcdbeecc Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:26 +0100 Subject: [PATCH 02/13] KVM: arm64: Include kvm_emulate.h in kvm/arm_psci.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a potential build error (like below, when asm/kvm_emulate.h gets included after the kvm/arm_psci.h) by including the missing header file in kvm/arm_psci.h: ./include/kvm/arm_psci.h: In function ‘kvm_psci_version’: ./include/kvm/arm_psci.h:29:13: error: implicit declaration of function ‘vcpu_has_feature’; did you mean ‘cpu_have_feature’? [-Werror=implicit-function-declaration] 29 | if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2)) { | ^~~~~~~~~~~~~~~~ | cpu_have_feature Reviewed-by: Gavin Shan Reviewed-by: Fuad Tabba Reviewed-by: Jonathan Cameron Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-4-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- include/kvm/arm_psci.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h index f86a006d6713..06c20612e9e7 100644 --- a/include/kvm/arm_psci.h +++ b/include/kvm/arm_psci.h @@ -10,6 +10,8 @@ #include #include +#include + #define KVM_ARM_PSCI_0_1 PSCI_VERSION(0, 1) #define KVM_ARM_PSCI_0_2 PSCI_VERSION(0, 2) #define KVM_ARM_PSCI_1_0 PSCI_VERSION(1, 0) From 0c4063f89bfdf204112651f45620ef0c5a691e6b Mon Sep 17 00:00:00 2001 From: Steven Price Date: Wed, 7 Oct 2026 08:35:27 +0100 Subject: [PATCH 03/13] KVM: arm64: Avoid including linux/kvm_host.h in kvm_pgtable.h To avoid future include cycles, drop the linux/kvm_host.h include in kvm_pgtable.h and include the lightweight headers required for the types and inline helpers used there. Additionally provide a forward declaration for struct kvm_s2_mmu as it's only used as a pointer in this file. Both pgtable.c and kvm_pkvm.h relied on the indirect inclusion of kvm_host.h, so make that explicit. Reviewed-by: Fuad Tabba Reviewed-by: Gavin Shan Signed-off-by: Steven Price Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-5-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pgtable.h | 10 +++++++++- arch/arm64/include/asm/kvm_pkvm.h | 2 +- arch/arm64/kvm/hyp/pgtable.c | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 41a8687938eb..a211a5d87bf9 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -7,10 +7,18 @@ #ifndef __ARM64_KVM_PGTABLE_H__ #define __ARM64_KVM_PGTABLE_H__ +#include #include -#include +#include +#include +#include #include +#include +#include + +struct kvm_s2_mmu; + #define KVM_PGTABLE_FIRST_LEVEL -1 #define KVM_PGTABLE_LAST_LEVEL 3 diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index beea00e693a0..54a618d887fa 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -7,9 +7,9 @@ #define __ARM64_KVM_PKVM_H__ #include +#include #include #include -#include #include /* Maximum number of VMs that can co-exist under pKVM. */ diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index b74dd5ce1efd..f48253b9d88b 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -8,6 +8,7 @@ */ #include +#include #include #include From 762ad274c03a13686192d75ef9a354b96114b503 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:28 +0100 Subject: [PATCH 04/13] KVM: arm64: Track the type of VM in kvm_arch KVM arm64 has different types of VMs with all the different modes in which the hypervisor code can be run. e.g., VHE, nVHE, pKVM etc. Then there is protected VM and normal VMs with pKVM. We might soon add other types, e.g., Arm CCA Realm. So in an effort to make the handling of these different types of VMs a bit more friendly to the eyes, add a VM flavor to the kvm_arch and we could then add handlers for different operations based on the VM type. Keep the flavor initialisation at the beginning to allow for the detection early enough and fail out on any unsupported requests. With that, add wrappers for checking the "type" of a VM and replace the existing users with the new wrappers. Given we already have the construct of "kvm_vm_is_protected" in the core KVM code, use that for all confidential compute guests including Realms that we are about to add. Adds __VM_PROTECTED marker vm flavor to draw the boundary for "protected VMs". In later patches, we would add Realm VMs, which would also be classified as protected. While adding the vm_flavor, move the psci_version around to keep the structure packed. Suggested-by: Marc Zyngier Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-6-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 38 +++++++++++++++++++++++++++---- arch/arm64/include/asm/kvm_pkvm.h | 4 ++-- arch/arm64/kvm/arm.c | 33 ++++++++++++++++++++++----- arch/arm64/kvm/hyp/nvhe/pkvm.c | 6 ++++- arch/arm64/kvm/mmio.c | 1 + arch/arm64/kvm/pkvm.c | 6 ++--- 6 files changed, 70 insertions(+), 18 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..9ad6b5c215fd 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -257,7 +257,6 @@ struct kvm_protected_vm { pkvm_handle_t handle; struct kvm_hyp_memcache teardown_mc; struct kvm_hyp_memcache stage2_teardown_mc; - bool is_protected; bool is_created; /* @@ -306,9 +305,22 @@ enum fgt_group_id { __NR_FGT_GROUP_IDS__ }; +enum kvm_arm_vm_flavor { + VM_NVHE, + VM_VHE, + VM_PKVM, /* Normal guests on pKVM */ + MARKER(__VM_PROTECTED), + VM_PROTECTED_PKVM, /* Protected VM */ + VM_FLAVOR_MAX +}; + struct kvm_arch { struct kvm_s2_mmu mmu; + enum kvm_arm_vm_flavor vm_flavor; + /* Mandated version of PSCI */ + u32 psci_version; + /* * Fine-Grained UNDEF, mimicking the FGT layout defined by the * architecture. We track them globally, as we present the @@ -332,9 +344,6 @@ struct kvm_arch { /* Timers */ struct arch_timer_vm_data timer_data; - /* Mandated version of PSCI */ - u32 psci_version; - /* Protects VM-scoped configuration data */ struct mutex config_lock; @@ -1504,10 +1513,29 @@ struct kvm *kvm_arch_alloc_vm(void); #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE -#define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.is_protected) +#ifdef __KVM_NVHE_HYPERVISOR__ +#define kvm_vm_is_protected(kvm) \ + (is_protected_kvm_enabled() && ((kvm)->arch.vm_flavor == VM_PROTECTED_PKVM)) +/* + * Accessing vcpu->kvm from nVHE hyp stub is tricky, as we need to convert the + * pointer to the hyp VA. With pKVM, the nVHE code runs with the hyp_vcpu, + * which is populated correctly and is gated on is_protected_kvm_enabled(). + */ +#define vcpu_is_protected(vcpu) \ + ({ \ + struct kvm *__kvm = READ_ONCE((vcpu)->kvm); \ + \ + (__kvm && kvm_vm_is_protected(__kvm)); \ + }) + +#else + +#define kvm_vm_is_protected(kvm) ((kvm)->arch.vm_flavor >= __VM_PROTECTED) #define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm) +#endif /* __KVM_NVHE_HYPERVISOR__ */ + int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 54a618d887fa..2addc37c500e 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -17,7 +17,7 @@ #define HYP_MEMBLOCK_REGIONS 128 -int pkvm_init_host_vm(struct kvm *kvm, unsigned long type); +int pkvm_init_host_vm(struct kvm *kvm); int pkvm_create_hyp_vm(struct kvm *kvm); bool pkvm_hyp_vm_is_created(struct kvm *kvm); void pkvm_destroy_hyp_vm(struct kvm *kvm); @@ -49,7 +49,7 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: return false; default: - return !kvm || !kvm_vm_is_protected(kvm); + return !kvm || (kvm->arch.vm_flavor == VM_PKVM); } } diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..b6032a7a1fb8 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -214,6 +214,26 @@ static int kvm_arm_default_max_vcpus(void) return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; } +static int kvm_init_vm_flavor(struct kvm *kvm, unsigned long type) +{ + bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; + + if (is_protected_kvm_enabled()) { + if (protected) + kvm->arch.vm_flavor = VM_PROTECTED_PKVM; + else + kvm->arch.vm_flavor = VM_PKVM; + } else if (protected) { + return -EINVAL; + } else if (has_vhe()) { + kvm->arch.vm_flavor = VM_VHE; + } else { + kvm->arch.vm_flavor = VM_NVHE; + } + + return 0; +} + /** * kvm_arch_init_vm - initializes a VM data structure * @kvm: pointer to the KVM struct @@ -236,6 +256,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); #endif + ret = kvm_init_vm_flavor(kvm, type); + if (ret) + return ret; + kvm_init_nested(kvm); ret = kvm_share_hyp(kvm, kvm + 1); @@ -257,12 +281,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) * If any failures occur after this is successful, make sure to * call __pkvm_unreserve_vm to unreserve the VM in hyp. */ - ret = pkvm_init_host_vm(kvm, type); + ret = pkvm_init_host_vm(kvm); if (ret) goto err_uninit_mmu; - } else if (type & KVM_VM_TYPE_ARM_PROTECTED) { - ret = -EINVAL; - goto err_uninit_mmu; } kvm_vgic_early_init(kvm); @@ -751,7 +772,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_call_hyp_nvhe(__pkvm_vcpu_put); /* __pkvm_vcpu_put implies a sync of the state */ - if (!kvm_vm_is_protected(vcpu->kvm)) + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); } @@ -985,7 +1006,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) if (is_protected_kvm_enabled()) { /* Start with the vcpu in a dirty state */ - if (!kvm_vm_is_protected(vcpu->kvm)) + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); ret = pkvm_create_hyp_vm(kvm); if (ret) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 459bd9eb7e4b..ed51762aa4b5 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -432,7 +432,11 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, hyp_vm->host_kvm = host_kvm; hyp_vm->kvm.created_vcpus = nr_vcpus; - hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected); + if (READ_ONCE(host_kvm->arch.vm_flavor) == VM_PROTECTED_PKVM) + hyp_vm->kvm.arch.vm_flavor = VM_PROTECTED_PKVM; + else + hyp_vm->kvm.arch.vm_flavor = VM_PKVM; + hyp_vm->kvm.arch.flags = 0; pkvm_init_features_from_host(hyp_vm, host_kvm); diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index d1c3a352d5a2..ab1d2fef9a52 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -6,6 +6,7 @@ #include #include +#include #include #include "trace.h" diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4c6e4bec12..8e9176a70092 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -229,10 +229,9 @@ void pkvm_destroy_hyp_vm(struct kvm *kvm) mutex_unlock(&kvm->arch.config_lock); } -int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) +int pkvm_init_host_vm(struct kvm *kvm) { int ret; - bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; /* Reserve the VM in hyp and obtain a hyp handle for the VM. */ ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm); @@ -240,8 +239,7 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) return ret; kvm->arch.pkvm.handle = ret; - kvm->arch.pkvm.is_protected = protected; - if (protected) { + if (kvm_vm_is_protected(kvm)) { pr_warn_once("kvm: protected VMs are experimental and for development only, tainting kernel\n"); add_taint(TAINT_USER, LOCKDEP_STILL_OK); } From 42a4026b435e49af4d5eaae58afe7103ca740c30 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:29 +0100 Subject: [PATCH 05/13] KVM: arm64: Don't call vcpu_set_pauth_traps for pKVM host vcpu_set_pauth_traps() bails out and does nothing for pKVM hosts. Clean this up by moving the is_protected_kvm_enabled() check to the caller, in preparation for adding VM specific vcpu load/put callbacks. While at it, do an early return if the vcpu doesn't have ptrauth. No functional changes Reviewed-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-7-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 52 +++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index b6032a7a1fb8..7b09b59e2819 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -631,33 +631,34 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) { - if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) { - /* - * Either we're running an L2 guest, and the API/APK bits come - * from L1's HCR_EL2, or API/APK are both set. - */ - if (unlikely(is_nested_ctxt(vcpu))) { - u64 val; + if (!vcpu_has_ptrauth(vcpu)) + return; - val = __vcpu_sys_reg(vcpu, HCR_EL2); - val &= (HCR_API | HCR_APK); - vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); - vcpu->arch.hcr_el2 |= val; - } else { - vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); - } + /* + * Either we're running an L2 guest, and the API/APK bits come + * from L1's HCR_EL2, or API/APK are both set. + */ + if (unlikely(is_nested_ctxt(vcpu))) { + u64 val; - /* - * Save the host keys if there is any chance for the guest - * to use pauth, as the entry code will reload the guest - * keys in that case. - */ - if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { - struct kvm_cpu_context *ctxt; + val = __vcpu_sys_reg(vcpu, HCR_EL2); + val &= (HCR_API | HCR_APK); + vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); + vcpu->arch.hcr_el2 |= val; + } else { + vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); + } - ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); - ptrauth_save_keys(ctxt); - } + /* + * Save the host keys if there is any chance for the guest + * to use pauth, as the entry code will reload the guest + * keys in that case. + */ + if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { + struct kvm_cpu_context *ctxt; + + ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); + ptrauth_save_keys(ctxt); } } @@ -749,7 +750,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) else vcpu->arch.hcr_el2 |= HCR_TWI; - vcpu_set_pauth_traps(vcpu); + if (!is_protected_kvm_enabled()) + vcpu_set_pauth_traps(vcpu); if (is_protected_kvm_enabled()) { kvm_call_hyp_nvhe(__pkvm_vcpu_load, From 8252f74f9188b9148ec93438e53b8e1b865ffb7b Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:30 +0100 Subject: [PATCH 06/13] KVM: arm64: Refactor the vcpu_load to allow for VM specific callbacks To keep the VCPU load/put handling cleaner with the different kinds of VM types, we are about to introduce VM specific callbacks to do just the right thing. In preparation for that, make some refactoring to add the change easier by mainly feature specific configurations to individual wrappers, so that different callbacks could reuse the helpers. Adds vcpu_prepare_mmu()/vcpu_put_mmu() wrappers to load/put MMU related configurations for !pKVM guests. Additionally makes it explicit that kvm_arm_vmid_clear_active() is not required for pKVM host. Add vcpu_load_pvtime(), vcpu_set_wfx_traps() wrappers for handling the corresponding configurations. While at it, also make it clear that the timer loading constraints only apply for the VHE. No functional changes intended. Based on a work by Marc Zyngier. Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-8-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 65 +++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 7b09b59e2819..4f3f0d5423f9 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -684,14 +684,11 @@ static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu) return single_task_running(); } -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +static void vcpu_prepare_mmu(struct kvm_vcpu *vcpu) { struct kvm_s2_mmu *mmu; int *last_ran; - if (is_protected_kvm_enabled()) - goto nommu; - if (vcpu_has_nv(vcpu)) kvm_vcpu_load_hw_mmu(vcpu); @@ -721,25 +718,15 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_call_hyp(__kvm_flush_cpu_context, mmu); *last_ran = vcpu->vcpu_idx; } +} -nommu: - vcpu->cpu = cpu; - - /* - * The timer must be loaded before the vgic to correctly set up physical - * interrupt deactivation in nested state (e.g. timer interrupt). - */ - kvm_timer_vcpu_load(vcpu); - kvm_vgic_load(vcpu); - kvm_vcpu_load_debug(vcpu); - kvm_vcpu_load_fgt(vcpu); - if (has_vhe()) - kvm_vcpu_load_vhe(vcpu); - kvm_arch_vcpu_load_fp(vcpu); - kvm_vcpu_pmu_restore_guest(vcpu); - if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) - kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +static void vcpu_put_mmu(struct kvm_vcpu *vcpu) +{ + kvm_arm_vmid_clear_active(); +} +static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) +{ if (kvm_vcpu_should_clear_twe(vcpu)) vcpu->arch.hcr_el2 &= ~HCR_TWE; else @@ -749,6 +736,38 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.hcr_el2 &= ~HCR_TWI; else vcpu->arch.hcr_el2 |= HCR_TWI; +} + +static void vcpu_load_pvtime(struct kvm_vcpu *vcpu) +{ + if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +} + +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + vcpu->cpu = cpu; + + if (!is_protected_kvm_enabled()) + vcpu_prepare_mmu(vcpu); + /* + * For VHE, the timer must be loaded before the vgic to correctly + * set up physical interrupt deactivation in nested state (e.g. timer + * interrupt). + */ + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + + if (has_vhe()) + kvm_vcpu_load_vhe(vcpu); + + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); if (!is_protected_kvm_enabled()) vcpu_set_pauth_traps(vcpu); @@ -787,7 +806,9 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_vcpu_pmu_restore_host(vcpu); if (vcpu_has_nv(vcpu)) kvm_vcpu_put_hw_mmu(vcpu); - kvm_arm_vmid_clear_active(); + + if (!is_protected_kvm_enabled()) + vcpu_put_mmu(vcpu); vcpu_clear_on_unsupported_cpu(vcpu); vcpu->cpu = -1; From 0cb16544ccaceba4c5e8beb2e8ae896278dcf018 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:31 +0100 Subject: [PATCH 07/13] KVM: arm64: Add vcpu load/put call backs for flavors Add VM flavor specific handlers for VCPU load/put, in an effort to make it easier to follow the code. pauth traps were removed from VMs running PKVM as it is a no-op for them. Based on a patch by Marc Zyngier Suggested-by: Marc Zyngier Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-9-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 6 ++ arch/arm64/kvm/arm.c | 138 +++++++++++++++++++++++------- 2 files changed, 114 insertions(+), 30 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 9ad6b5c215fd..4c64d957ee06 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -150,6 +150,11 @@ struct kvm_vmid { atomic64_t id; }; +struct kvm_vcpu_ops { + void (*vcpu_load)(struct kvm_vcpu *vcpu); + void (*vcpu_put)(struct kvm_vcpu *vcpu); +}; + struct kvm_s2_mmu { struct kvm_vmid vmid; @@ -855,6 +860,7 @@ struct vncr_tlb; struct kvm_vcpu_arch { struct kvm_cpu_context ctxt; + const struct kvm_vcpu_ops *vcpu_ops; /* * Guest floating point state diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 4f3f0d5423f9..e99af4faa525 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -93,6 +93,7 @@ static const struct kvm_ioctl_cap_map vm_ioctl_caps[] = { { KVM_ARM_PREFERRED_TARGET, KVM_CAP_ARM_BASIC }, }; +static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu); /* * Set *ext to the capability. * Return 0 if found, or -EINVAL if no IOCTL matches. @@ -569,6 +570,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) mutex_unlock(&vcpu->mutex); #endif + kvm_init_vcpu_ops(vcpu); + /* Force users to call KVM_ARM_VCPU_INIT */ vcpu_clear_flag(vcpu, VCPU_INITIALIZED); @@ -744,12 +747,9 @@ static void vcpu_load_pvtime(struct kvm_vcpu *vcpu) kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); } -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +static void vhe_vcpu_load(struct kvm_vcpu *vcpu) { - vcpu->cpu = cpu; - - if (!is_protected_kvm_enabled()) - vcpu_prepare_mmu(vcpu); + vcpu_prepare_mmu(vcpu); /* * For VHE, the timer must be loaded before the vgic to correctly * set up physical interrupt deactivation in nested state (e.g. timer @@ -759,26 +759,53 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_vgic_load(vcpu); kvm_vcpu_load_debug(vcpu); kvm_vcpu_load_fgt(vcpu); + kvm_vcpu_load_vhe(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); - if (has_vhe()) - kvm_vcpu_load_vhe(vcpu); + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); + vcpu_set_pauth_traps(vcpu); +} +static void nvhe_vcpu_load(struct kvm_vcpu *vcpu) +{ + vcpu_prepare_mmu(vcpu); + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); + vcpu_set_pauth_traps(vcpu); +} + +static void pkvm_vcpu_load(struct kvm_vcpu *vcpu) +{ + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); kvm_arch_vcpu_load_fp(vcpu); kvm_vcpu_pmu_restore_guest(vcpu); vcpu_load_pvtime(vcpu); vcpu_set_wfx_traps(vcpu); - if (!is_protected_kvm_enabled()) - vcpu_set_pauth_traps(vcpu); + kvm_call_hyp_nvhe(__pkvm_vcpu_load, + vcpu->kvm->arch.pkvm.handle, + vcpu->vcpu_idx, vcpu->arch.hcr_el2); + kvm_call_hyp_nvhe(__vgic_v3_restore_vmcr_aprs, + &vcpu->arch.vgic_cpu.vgic_v3); +} - if (is_protected_kvm_enabled()) { - kvm_call_hyp_nvhe(__pkvm_vcpu_load, - vcpu->kvm->arch.pkvm.handle, - vcpu->vcpu_idx, vcpu->arch.hcr_el2); - kvm_call_hyp(__vgic_v3_restore_vmcr_aprs, - &vcpu->arch.vgic_cpu.vgic_v3); - } +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + vcpu->cpu = cpu; + vcpu->arch.vcpu_ops->vcpu_load(vcpu); if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus)) vcpu_set_on_unsupported_cpu(vcpu); @@ -786,30 +813,50 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.pid = pid_nr(vcpu->pid); } -void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +static void vhe_vcpu_put(struct kvm_vcpu *vcpu) { - if (is_protected_kvm_enabled()) { - kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); - kvm_call_hyp_nvhe(__pkvm_vcpu_put); - - /* __pkvm_vcpu_put implies a sync of the state */ - if (vcpu->kvm->arch.vm_flavor == VM_PKVM) - vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); - } - kvm_vcpu_put_debug(vcpu); kvm_arch_vcpu_put_fp(vcpu); - if (has_vhe()) - kvm_vcpu_put_vhe(vcpu); + kvm_vcpu_put_vhe(vcpu); kvm_timer_vcpu_put(vcpu); kvm_vgic_put(vcpu); kvm_vcpu_pmu_restore_host(vcpu); + if (vcpu_has_nv(vcpu)) kvm_vcpu_put_hw_mmu(vcpu); - if (!is_protected_kvm_enabled()) - vcpu_put_mmu(vcpu); + vcpu_put_mmu(vcpu); +} +static void nvhe_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_vcpu_put_debug(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + kvm_vcpu_pmu_restore_host(vcpu); + vcpu_put_mmu(vcpu); +} + +static void pkvm_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_call_hyp_nvhe(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); + kvm_call_hyp_nvhe(__pkvm_vcpu_put); + + /* __pkvm_vcpu_put implies a sync of the state */ + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); + + kvm_vcpu_put_debug(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + kvm_vcpu_pmu_restore_host(vcpu); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + vcpu->arch.vcpu_ops->vcpu_put(vcpu); vcpu_clear_on_unsupported_cpu(vcpu); vcpu->cpu = -1; } @@ -2149,6 +2196,37 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) } } +static const struct kvm_vcpu_ops vhe_vcpu_ops = { + .vcpu_load = vhe_vcpu_load, + .vcpu_put = vhe_vcpu_put, +}; + +static const struct kvm_vcpu_ops nvhe_vcpu_ops = { + .vcpu_load = nvhe_vcpu_load, + .vcpu_put = nvhe_vcpu_put, +}; + +static const struct kvm_vcpu_ops pkvm_vcpu_ops = { + .vcpu_load = pkvm_vcpu_load, + .vcpu_put = pkvm_vcpu_put, +}; + +#define KVM_VCPU_OPS(flavor, ops) \ + [(flavor)] = &(ops) + +static const struct kvm_vcpu_ops *arm64_vcpu_ops[] = { + KVM_VCPU_OPS(VM_NVHE, nvhe_vcpu_ops), + KVM_VCPU_OPS(VM_VHE, vhe_vcpu_ops), + KVM_VCPU_OPS(VM_PKVM, pkvm_vcpu_ops), + KVM_VCPU_OPS(VM_PROTECTED_PKVM, pkvm_vcpu_ops), +}; + +static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu) +{ + BUILD_BUG_ON(ARRAY_SIZE(arm64_vcpu_ops) != VM_FLAVOR_MAX); + vcpu->arch.vcpu_ops = arm64_vcpu_ops[vcpu->kvm->arch.vm_flavor]; +} + static unsigned long nvhe_percpu_size(void) { return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) - From fb3926af9d0d5e2e2f58a1f4b2252f529fbe23bf Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:32 +0100 Subject: [PATCH 08/13] KVM: arm64: Consolidate stage2 unmap range into kvm_stage2_unmap_range In preparation for adding VM specific backends for stage2 operations, nuke __unmap_stage2_range() and fold the logic into kvm_stage2_unmap_range(). Also, make kvm_unmap_gfn_range(), the only other user of the __unmap_stage2_range() call the kvm_stage2_unmap_range(). Also, while at it fix the comment to make it clear that mmu_lock must always be held while unmapping. The code already mandates that and the two call paths do have the write mmu_lock held. Later we would replace the logic in kvm_stage2_unmap_range() with VM specific backends. No functional changes intended. Reviewed-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-10-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9ba86450fe4a..3fef16393f56 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -314,36 +314,30 @@ static void invalidate_icache_guest_page(void *va, size_t size) * does. */ /** - * __unmap_stage2_range -- Clear stage2 page table entries to unmap a range + * kvm_stage2_unmap_range -- Clear stage2 page table entries to unmap a range * @mmu: The KVM stage-2 MMU pointer * @start: The intermediate physical base address of the range to unmap * @size: The size of the area to unmap * @may_block: Whether or not we are permitted to block * * Clear a range of stage-2 mappings, lowering the various ref-counts. Must - * be called while holding mmu_lock (unless for freeing the stage2 pgd before - * destroying the VM), otherwise another faulting VCPU may come in and mess - * with things behind our backs. + * be called while holding mmu_lock otherwise another faulting VCPU may + * come in and mess with things behind our backs. */ -static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, - bool may_block) -{ - struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - phys_addr_t end = start + size; - - lockdep_assert_held_write(&kvm->mmu_lock); - WARN_ON(size & ~PAGE_MASK); - WARN_ON(stage2_apply_range(mmu, start, end, KVM_PGT_FN(kvm_pgtable_stage2_unmap), - may_block)); -} - void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, bool may_block) { - if (kvm_vm_is_protected(kvm_s2_mmu_to_kvm(mmu))) + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + if (kvm_vm_is_protected(kvm)) return; - __unmap_stage2_range(mmu, start, size, may_block); + lockdep_assert_held_write(&kvm->mmu_lock); + WARN_ON(size & ~PAGE_MASK); + + WARN_ON(stage2_apply_range(mmu, start, start + size, + KVM_PGT_FN(kvm_pgtable_stage2_unmap), + may_block)); } void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end) @@ -2436,12 +2430,12 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) + if (!kvm->arch.mmu.pgt) return false; - __unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT, - (range->end - range->start) << PAGE_SHIFT, - range->may_block); + kvm_stage2_unmap_range(&kvm->arch.mmu, range->start << PAGE_SHIFT, + (range->end - range->start) << PAGE_SHIFT, + range->may_block); kvm_nested_s2_unmap(kvm, range->may_block); return false; From 68933203edddc092b30dd1039167b3262a40a326 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:33 +0100 Subject: [PATCH 09/13] KVM: arm64: Add VM specific callback for S2 MMU operations Add VM type specific S2 MMU operation backends which can be initialized per VM flavor, to keep the handling cleaner. Reviewed-by: Gavin Shan Signed-off-by: Suzuki K Poulose Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261007073537.2454351-11-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 14 +++ arch/arm64/kvm/mmu.c | 173 +++++++++++++++++++++++++----- 2 files changed, 160 insertions(+), 27 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 4c64d957ee06..7130ba5bef8a 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -155,6 +155,19 @@ struct kvm_vcpu_ops { void (*vcpu_put)(struct kvm_vcpu *vcpu); }; +struct kvm_gfn_range; + +struct kvm_vm_s2_ops { + bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + bool (*vm_test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + int (*vm_flush_remote_tlbs)(struct kvm *kvm); + int (*vm_flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, + u64 nr_pages); + void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu, + phys_addr_t start, u64 size, + bool may_block); +}; + struct kvm_s2_mmu { struct kvm_vmid vmid; @@ -321,6 +334,7 @@ enum kvm_arm_vm_flavor { struct kvm_arch { struct kvm_s2_mmu mmu; + const struct kvm_vm_s2_ops *vm_s2_ops; enum kvm_arm_vm_flavor vm_flavor; /* Mandated version of PSCI */ diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 3fef16393f56..65592feaa6a9 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -37,6 +37,8 @@ static unsigned long __ro_after_init io_map_base; #define KVM_PGT_FN(fn) (!is_protected_kvm_enabled() ? fn : p ## fn) +static int kvm_vm_init_vm_s2_ops(struct kvm *kvm); + static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end, phys_addr_t size) { @@ -166,6 +168,18 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); } +static int pkvm_flush_remote_tlbs(struct kvm *kvm) +{ + kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); + return 0; +} + +static int kvm_vm_flush_remote_tlbs(struct kvm *kvm) +{ + kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); + return 0; +} + /** * kvm_arch_flush_remote_tlbs() - flush all VM TLB entries for v7/8 * @kvm: pointer to kvm structure. @@ -174,26 +188,31 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) */ int kvm_arch_flush_remote_tlbs(struct kvm *kvm) { - if (is_protected_kvm_enabled()) - kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); - else - kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); - return 0; + return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs(kvm); } -int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, - gfn_t gfn, u64 nr_pages) +static int pkvm_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) +{ + return pkvm_flush_remote_tlbs(kvm); +} + +static int kvm_vm_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) { u64 size = nr_pages << PAGE_SHIFT; u64 addr = gfn << PAGE_SHIFT; - if (is_protected_kvm_enabled()) - kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); - else - kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size); + kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size); return 0; } +int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) +{ + return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs_range(kvm, gfn, nr_pages); +} + static void *stage2_memcache_zalloc_page(void *arg) { struct kvm_mmu_memory_cache *mc = arg; @@ -289,6 +308,27 @@ static void invalidate_icache_guest_page(void *va, size_t size) __invalidate_icache_guest_page(va, size); } +static void kvm_vm_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, + u64 size, bool may_block) +{ + WARN_ON(stage2_apply_range(mmu, start, start + size, + kvm_pgtable_stage2_unmap, may_block)); +} + +static void pkvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, + u64 size, bool may_block) +{ + WARN_ON(stage2_apply_range(mmu, start, start + size, + pkvm_pgtable_stage2_unmap, may_block)); +} + +static void no_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, u64 size, bool may_block) +{ +} + /* * Unmapping vs dcache management: * @@ -329,15 +369,10 @@ void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start, { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - if (kvm_vm_is_protected(kvm)) - return; - lockdep_assert_held_write(&kvm->mmu_lock); WARN_ON(size & ~PAGE_MASK); - WARN_ON(stage2_apply_range(mmu, start, start + size, - KVM_PGT_FN(kvm_pgtable_stage2_unmap), - may_block)); + kvm->arch.vm_s2_ops->vm_stage2_unmap_range(mmu, start, size, may_block); } void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end) @@ -977,6 +1012,12 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t int cpu, err; struct kvm_pgtable *pgt; + /* Initialize the VM ops for the VM instance for the first time */ + if (mmu == &kvm->arch.mmu) { + err = kvm_vm_init_vm_s2_ops(kvm); + if (err) + return err; + } /* * If we already have our page tables in place, and that the * MMU context is the canonical one, we have a bug somewhere, @@ -2441,34 +2482,68 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) return false; } -bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +static bool kvm_vm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { u64 size = (range->end - range->start) << PAGE_SHIFT; - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) - return false; - - return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt, + return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT, size, true); +} + +static bool pkvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, true); +} + +static bool no_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + /* The hypervisor doesn't support aging */ + return false; +} + +bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + if (!kvm->arch.mmu.pgt) + return false; + + return kvm->arch.vm_s2_ops->vm_age_gfn(kvm, range); /* * TODO: Handle nested_mmu structures here using the reverse mapping in * a later version of patch series. */ } -bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +static bool kvm_vm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { u64 size = (range->end - range->start) << PAGE_SHIFT; - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) - return false; - - return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt, + return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT, size, false); } +static bool pkvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, false); +} + +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + if (!kvm->arch.mmu.pgt) + return false; + + return kvm->arch.vm_s2_ops->vm_test_age_gfn(kvm, range); +} + phys_addr_t kvm_mmu_get_httbr(void) { return __pa(hyp_pgtable->pgd); @@ -2790,3 +2865,47 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); } + +static const struct kvm_vm_s2_ops protected_pkvm_vm_s2_ops = { + .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range, + .vm_age_gfn = no_age_gfn, + .vm_test_age_gfn = no_age_gfn, + .vm_stage2_unmap_range = no_stage2_unmap_range, +}; + +static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = { + .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range, + .vm_age_gfn = pkvm_age_gfn, + .vm_test_age_gfn = pkvm_test_age_gfn, + .vm_stage2_unmap_range = pkvm_stage2_unmap_range, +}; + +static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = { + .vm_flush_remote_tlbs = kvm_vm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = kvm_vm_flush_remote_tlbs_range, + .vm_age_gfn = kvm_vm_age_gfn, + .vm_test_age_gfn = kvm_vm_test_age_gfn, + .vm_stage2_unmap_range = kvm_vm_stage2_unmap_range, +}; + +#define KVM_VM_S2_OPS(flavor, ops) \ + [(flavor)] = &(ops) + +static const struct kvm_vm_s2_ops *arm64_vm_s2_ops[] = { + KVM_VM_S2_OPS(VM_VHE, kvm_default_vm_s2_ops), + KVM_VM_S2_OPS(VM_NVHE, kvm_default_vm_s2_ops), + KVM_VM_S2_OPS(VM_PKVM, pkvm_vm_s2_ops), + KVM_VM_S2_OPS(VM_PROTECTED_PKVM, protected_pkvm_vm_s2_ops), +}; + +static int kvm_vm_init_vm_s2_ops(struct kvm *kvm) +{ + BUILD_BUG_ON(ARRAY_SIZE(arm64_vm_s2_ops) != VM_FLAVOR_MAX); + + kvm->arch.vm_s2_ops = arm64_vm_s2_ops[kvm->arch.vm_flavor]; + if (WARN_ON(!kvm->arch.vm_s2_ops)) + return -EINVAL; + return 0; +} From 4ee1b583589781bfd47b28da3ce216b2fb51dac5 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:34 +0100 Subject: [PATCH 10/13] KVM: arm64: Use a local kvm pointer in kvm_handle_guest_abort() kvm_handle_guest_abort() repeatedly obtains the VM from vcpu->kvm. Introduce a local kvm pointer and use it throughout the function. While touching the code, fix the missing whitespace in the kvm_is_nested_s2_mmu() call. Suggested-by: Jonathan Cameron Reviewed-by: Fuad Tabba Reviewed-by: Gavin Shan Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-12-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 65592feaa6a9..46d23483db4b 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2284,6 +2284,7 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu) */ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) { + struct kvm *kvm = vcpu->kvm; struct kvm_s2_trans nested_trans, *nested = NULL; unsigned long esr; phys_addr_t fault_ipa; /* The address we faulted on */ @@ -2304,7 +2305,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * with an SEA. */ ipa = fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); - if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm)) + if (KVM_BUG_ON(ipa == INVALID_GPA, kvm)) return -EFAULT; is_iabt = kvm_vcpu_trap_is_iabt(vcpu); @@ -2339,7 +2340,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) return -EFAULT; } - idx = srcu_read_lock(&vcpu->kvm->srcu); + idx = srcu_read_lock(&kvm->srcu); /* * We may have faulted on a shadow stage 2 page table if we are @@ -2354,7 +2355,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * nothing to walk and we treat it as a 1:1 before going through the * canonical translation. */ - if (kvm_is_nested_s2_mmu(vcpu->kvm,vcpu->arch.hw_mmu) && + if (kvm_is_nested_s2_mmu(kvm, vcpu->arch.hw_mmu) && vcpu->arch.hw_mmu->nested_stage2_enabled) { u32 esr; @@ -2382,7 +2383,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } gfn = ipa >> PAGE_SHIFT; - memslot = gfn_to_memslot(vcpu->kvm, gfn); + memslot = gfn_to_memslot(kvm, gfn); hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); write_fault = kvm_is_write_fault(vcpu); if (kvm_is_error_hva(hva) || (write_fault && !writable)) { @@ -2446,7 +2447,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) .hva = hva, }; - if (kvm_vm_is_protected(vcpu->kvm)) { + if (kvm_vm_is_protected(kvm)) { ret = pkvm_mem_abort(&s2fd); } else { VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && @@ -2465,7 +2466,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) if (ret == -ENOEXEC) ret = kvm_inject_sea_iabt(vcpu, kvm_vcpu_get_hfar(vcpu)); out_unlock: - srcu_read_unlock(&vcpu->kvm->srcu, idx); + srcu_read_unlock(&kvm->srcu, idx); return ret; } From b44a8c32f0116aba0a934c3068628bacda3cd3a3 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:35 +0100 Subject: [PATCH 11/13] KVM: arm64: Abstract out memory abort handling Move the memory abort handling under VM specific s2 operation. Tested-by: Gavin Shan Reviewed-by: Gavin Shan Reviewed-by: Fuad Tabba Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-13-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/kvm/mmu.c | 33 ++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 7130ba5bef8a..c08b40a6d4cb 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -156,6 +156,7 @@ struct kvm_vcpu_ops { }; struct kvm_gfn_range; +struct kvm_s2_fault_desc; struct kvm_vm_s2_ops { bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); @@ -166,6 +167,7 @@ struct kvm_vm_s2_ops { void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, bool may_block); + int (*vm_mem_abort)(const struct kvm_s2_fault_desc *s2fd); }; struct kvm_s2_mmu { diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 46d23483db4b..9821cac0dce4 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1740,7 +1740,7 @@ struct kvm_s2_fault_vma_info { bool map_non_cacheable; }; -static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) +static int protected_pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) { unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; struct kvm_vcpu *vcpu = s2fd->vcpu; @@ -2178,6 +2178,20 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache); } +static int kvm_vm_mem_abort(const struct kvm_s2_fault_desc *s2fd) +{ + struct kvm_vcpu *vcpu = s2fd->vcpu; + + VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && + !kvm_is_write_fault(vcpu) && + !kvm_vcpu_trap_is_exec_fault(vcpu)); + + if (kvm_slot_has_gmem(s2fd->memslot)) + return gmem_abort(s2fd); + else + return user_mem_abort(s2fd); +} + /* Resolve the access fault by making the page young again. */ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) { @@ -2447,19 +2461,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) .hva = hva, }; - if (kvm_vm_is_protected(kvm)) { - ret = pkvm_mem_abort(&s2fd); - } else { - VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && - !write_fault && - !kvm_vcpu_trap_is_exec_fault(vcpu)); - - if (kvm_slot_has_gmem(memslot)) - ret = gmem_abort(&s2fd); - else - ret = user_mem_abort(&s2fd); - } - + ret = kvm->arch.vm_s2_ops->vm_mem_abort(&s2fd); if (ret == 0) ret = 1; out: @@ -2873,6 +2875,7 @@ static const struct kvm_vm_s2_ops protected_pkvm_vm_s2_ops = { .vm_age_gfn = no_age_gfn, .vm_test_age_gfn = no_age_gfn, .vm_stage2_unmap_range = no_stage2_unmap_range, + .vm_mem_abort = protected_pkvm_mem_abort, }; static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = { @@ -2881,6 +2884,7 @@ static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = { .vm_age_gfn = pkvm_age_gfn, .vm_test_age_gfn = pkvm_test_age_gfn, .vm_stage2_unmap_range = pkvm_stage2_unmap_range, + .vm_mem_abort = kvm_vm_mem_abort, }; static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = { @@ -2889,6 +2893,7 @@ static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = { .vm_age_gfn = kvm_vm_age_gfn, .vm_test_age_gfn = kvm_vm_test_age_gfn, .vm_stage2_unmap_range = kvm_vm_stage2_unmap_range, + .vm_mem_abort = kvm_vm_mem_abort, }; #define KVM_VM_S2_OPS(flavor, ops) \ From 37d7ea3f81bbe61f1e895b6c87decadbfe8799eb Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:36 +0100 Subject: [PATCH 12/13] KVM: arm64: Mandate VGIC v3 for pKVM VMs and Realms pKVM does not trust the host. Realm VMs follow a similar trust model, with the Realm Management Monitor owning the protected state instead of the host. Add a helper to identify VMs that run under a host-distrusting hypervisor. Use this for blocking ioremap of vgic-v2 into stage2 and prevent creation of VGIC other than v3. Reviewed-by: Jonathan Cameron Reviewed-by: Fuad Tabba Reviewed-by: Gavin Shan Tested-by: Gavin Shan Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-14-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 4 ++++ arch/arm64/kvm/mmu.c | 2 +- arch/arm64/kvm/vgic/vgic-init.c | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index c08b40a6d4cb..54d56bef3535 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -328,6 +328,8 @@ enum fgt_group_id { enum kvm_arm_vm_flavor { VM_NVHE, VM_VHE, + /* VMs running on a hyp that doesn't trust */ + MARKER(__VM_DISTRUSTING_HYP), VM_PKVM, /* Normal guests on pKVM */ MARKER(__VM_PROTECTED), VM_PROTECTED_PKVM, /* Protected VM */ @@ -1556,6 +1558,8 @@ struct kvm *kvm_arch_alloc_vm(void); #define kvm_vm_is_protected(kvm) ((kvm)->arch.vm_flavor >= __VM_PROTECTED) #define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm) +#define kvm_vm_hyp_is_distrusting(kvm) ((kvm)->arch.vm_flavor >= __VM_DISTRUSTING_HYP) + #endif /* __KVM_NVHE_HYPERVISOR__ */ int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9821cac0dce4..b0dacfa7e625 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1249,7 +1249,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, KVM_PGTABLE_PROT_R | (writable ? KVM_PGTABLE_PROT_W : 0); - if (is_protected_kvm_enabled()) + if (kvm_vm_hyp_is_distrusting(kvm)) return -EPERM; size += offset_in_page(guest_ipa); diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 4012df6002ea..874025513afc 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -84,6 +84,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) !kvm_vgic_global_state.can_emulate_gicv2) return -ENODEV; + if (kvm_vm_hyp_is_distrusting(kvm) && type != KVM_DEV_TYPE_ARM_VGIC_V3) + return -ENODEV; /* * Ensure mutual exclusion with vCPU creation and any vCPU ioctls by: * From a467f39afc2a149f9f5efbb7238fb531ade8a86e Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Wed, 7 Oct 2026 08:35:37 +0100 Subject: [PATCH 13/13] KVM: arm64: Prevent unsupported vcpu features for VM types Prevent unsupported VCPU features for the protected VCPUs. Realms and pVMs not support 32bit EL1 or NV yet. pKVM doesn't rely on the host vcpu features for protected VMs and hand picks features while hyp_vcpu is initialised. Block the features early in the vcpu init if we detect incompatible features. Signed-off-by: Suzuki K Poulose Link: https://patch.msgid.link/20261007073537.2454351-15-suzuki.poulose@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 7 +++++++ arch/arm64/kvm/arm.c | 11 ++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 54d56bef3535..4175aa9dba47 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -42,6 +42,13 @@ #define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) +/* As dictated by kvm_pkvm_ext_allowed() */ +#define KVM_PROTECTED_VCPU_VALID_FEATURES \ + (BIT(KVM_ARM_VCPU_POWER_OFF) | \ + BIT(KVM_ARM_VCPU_PSCI_0_2) | \ + BIT(KVM_ARM_VCPU_PTRAUTH_ADDRESS) | \ + BIT(KVM_ARM_VCPU_PTRAUTH_GENERIC)) + #define KVM_REQ_SLEEP \ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e99af4faa525..90ce620a6ea4 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1668,9 +1668,14 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return -EINVAL; } -static unsigned long system_supported_vcpu_features(void) +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) { - unsigned long features = KVM_VCPU_VALID_FEATURES; + unsigned long features; + + if (vcpu->kvm->arch.vm_flavor == VM_PROTECTED_PKVM) + features = KVM_PROTECTED_VCPU_VALID_FEATURES; + else + features = KVM_VCPU_VALID_FEATURES; if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); @@ -1708,7 +1713,7 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, return -ENOENT; } - if (features & ~system_supported_vcpu_features()) + if (features & ~system_supported_vcpu_features(vcpu)) return -EINVAL; /*