From 007e0f0fd9b77b9fa8f0d86253bac690f523460e Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Mon, 13 Jul 2026 14:13:20 +0000 Subject: [PATCH 01/89] KVM: arm64: Avoid naming collision in tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the hypervisor tracing (CONFIG_NVHE_EL2_TRACING) is disabled, it defines a static inline stub for trace_clock(). However, trace_clock() is already declared as an extern function in linux/trace_clock.h which is pulled in EL2 compilation. If the file is included when CONFIG_NVHE_EL2_TRACING is disabled (by including it manually in setup.c) it will cause: In file included from arch/arm64/kvm/hyp/nvhe/setup.c:22: ./arch/arm64/kvm/hyp/include/nvhe/clock.h:14:19: error: static declaration of ‘trace_clock’ follows non-static declaration 14 | static inline u64 trace_clock(void) { return 0; } | ^~~~~~~~~~~ on GCC and a linker error on LLVM (it seems to change the linkage to global) Although that is not a problem at the moment, as no other files include . That does not seem to be the intent of this code and that will cause issues with more users as the SMMUv3 driver. Signed-off-by: Mostafa Saleh Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Reviewed-by: Vincent Donnefort Link: https://patch.msgid.link/20260713141320.4065600-1-smostafa@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/nvhe/clock.h | 8 ++++---- arch/arm64/kvm/hyp/nvhe/clock.c | 4 ++-- arch/arm64/kvm/hyp/nvhe/trace.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/clock.h b/arch/arm64/kvm/hyp/include/nvhe/clock.h index 9f429f5c0664..ae03ec6965af 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/clock.h +++ b/arch/arm64/kvm/hyp/include/nvhe/clock.h @@ -6,11 +6,11 @@ #include #ifdef CONFIG_NVHE_EL2_TRACING -void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc); -u64 trace_clock(void); +void trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc); +u64 trace_hyp_clock(void); #else static inline void -trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { } -static inline u64 trace_clock(void) { return 0; } +trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { } +static inline u64 trace_hyp_clock(void) { return 0; } #endif #endif diff --git a/arch/arm64/kvm/hyp/nvhe/clock.c b/arch/arm64/kvm/hyp/nvhe/clock.c index a7fc61976fd0..f3e2619db4e4 100644 --- a/arch/arm64/kvm/hyp/nvhe/clock.c +++ b/arch/arm64/kvm/hyp/nvhe/clock.c @@ -30,7 +30,7 @@ static u64 __clock_mult_uint128(u64 cyc, u32 mult, u32 shift) } /* Does not guarantee no reader on the modified bank. */ -void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) +void trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { struct clock_data *clock = &trace_clock_data; u64 bank = clock->cur ^ 1; @@ -48,7 +48,7 @@ void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) } /* Use untrusted host data */ -u64 trace_clock(void) +u64 trace_hyp_clock(void) { struct clock_data *clock = &trace_clock_data; u64 bank = smp_load_acquire(&clock->cur); diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index e7e150ab265f..65be6c9fb379 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -35,7 +35,7 @@ static bool hyp_trace_buffer_loaded(struct hyp_trace_buffer *trace_buffer) void *tracing_reserve_entry(unsigned long length) { return simple_ring_buffer_reserve(this_cpu_ptr(trace_buffer.simple_rbs), length, - trace_clock()); + trace_hyp_clock()); } void tracing_commit_entry(void) @@ -290,7 +290,7 @@ void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) } /* ...we can now override the old one and swap. */ - trace_clock_update(mult, shift, epoch_ns, epoch_cyc); + trace_hyp_clock_update(mult, shift, epoch_ns, epoch_cyc); } int __tracing_reset(unsigned int cpu) From 581829501360bda487b93ec7d7eff0da6296f187 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 6 Jul 2026 11:31:27 +0100 Subject: [PATCH 02/89] KVM: arm64: Drop the unused EL2-side is_created write init_pkvm_hyp_vm() sets is_created on the EL2-private VM struct, but the hypervisor never reads it: pkvm_hyp_vm_is_created() and every other consumer operate on the host's struct kvm, a distinct allocation from the EL2-private copy. The field is write-only at EL2. Remove the store; host-side is_created tracking is unaffected. Reviewed-by: Keir Fraser Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260706103129.706974-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 3b2c4fbc34d8..c54aa5031336 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -433,7 +433,6 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, hyp_vm->host_kvm = host_kvm; hyp_vm->kvm.created_vcpus = nr_vcpus; hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected); - hyp_vm->kvm.arch.pkvm.is_created = true; hyp_vm->kvm.arch.flags = 0; pkvm_init_features_from_host(hyp_vm, host_kvm); From 9b36eda4dfe0bcd76233b3f1318b66478c1d0698 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 6 Jul 2026 11:31:28 +0100 Subject: [PATCH 03/89] KVM: arm64: Remove unreachable early checks in pkvm_init_host_vm() pkvm_init_host_vm() runs once from kvm_arch_init_vm(), while the VM is still being allocated and is not yet reachable by another thread. Both early checks therefore test impossible state: is_created is still false (it is only set on first vCPU run) and the handle is still zero (this function is what reserves it). Neither branch can be taken. Remove them. Reviewed-by: Keir Fraser Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260706103129.706974-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/pkvm.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 053e4f733e4b..67b90a58fbea 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -230,13 +230,6 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) int ret; bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; - if (pkvm_hyp_vm_is_created(kvm)) - return -EINVAL; - - /* VM is already reserved, no need to proceed. */ - if (kvm->arch.pkvm.handle) - return 0; - /* Reserve the VM in hyp and obtain a hyp handle for the VM. */ ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm); if (ret < 0) From 008ea7a6aa01877a2439bc911426c27a03a6d9e3 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 6 Jul 2026 11:31:29 +0100 Subject: [PATCH 04/89] KVM: arm64: Drop redundant READ_ONCE() in pkvm_hyp_vm_is_created() is_created is written under config_lock. Every concurrent reader is serialised against that write: pkvm_create_hyp_vm() under config_lock, and the memslot path (kvm_arch_prepare_memory_region) via slots_lock, which the creation writer also holds. The teardown-path accesses have no concurrent writer. The read is therefore serialised, and the READ_ONCE() is unnecessary. Reviewed-by: Keir Fraser Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260706103129.706974-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/pkvm.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 67b90a58fbea..008766273912 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -185,7 +185,11 @@ static int __pkvm_create_hyp_vm(struct kvm *kvm) bool pkvm_hyp_vm_is_created(struct kvm *kvm) { - return READ_ONCE(kvm->arch.pkvm.is_created); + /* + * Serialised by config_lock/slots_lock, or by VM lifecycle at + * teardown, so a plain read suffices. + */ + return kvm->arch.pkvm.is_created; } int pkvm_create_hyp_vm(struct kvm *kvm) From 719e67103c5e1486e06e340a03f33f1493fae914 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:49 +0100 Subject: [PATCH 05/89] KVM: arm64: vgic-v3: Make vtr_to_* helpers use architectural field symbols vgic-v3-sr.c still contains some hardcoded constants for some of the ICH)VTR_EL2 fields. Bring them into the modern world by using the named symbols from the sysreg file. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-2-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/vgic-v3-sr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index c4d2f1feea8b..8da7a40ced58 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -16,9 +16,9 @@ #include "../../vgic/vgic.h" -#define vtr_to_max_lr_idx(v) ((v) & 0xf) -#define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1) -#define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5)) +#define vtr_to_max_lr_idx(v) FIELD_GET(ICH_VTR_EL2_ListRegs, (v)) +#define vtr_to_nr_pre_bits(v) (FIELD_GET(ICH_VTR_EL2_PREbits, (v)) + 1) +#define vtr_to_nr_apr_regs(v) BIT(vtr_to_nr_pre_bits(v) - 5) u64 __gic_v3_get_lr(unsigned int lr) { From 27a263e1a0fa23374066696aa7732a01bfde7a4e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:50 +0100 Subject: [PATCH 06/89] KVM: arm64: Move GICv3 broken SEIS implementation detection to a CPU errrata Using is_midr_in_range_list() in a noinstr function isn't a good idea. And yet kvm_compute_ich_hcr_trap_bits() is doing that by calling vgic_v3_broken_seis(). Move all the broken SEIS detection logic to the errata detection framework, and use a new ARM64_WORKAROUND_GICv3_BROKEN_SEIS cap to indicate that we're running on broken CPUs. This reuses the MIDR list used for IMPDEF PMU detection, which has a 100% overlap with the SEIS stuff... Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-3-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kernel/cpu_errata.c | 17 +++++++++++++++-- arch/arm64/kvm/vgic/vgic-v3.c | 26 +++----------------------- arch/arm64/tools/cpucaps | 1 + 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 1995e1198648..78e4e105f6ab 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -272,7 +272,7 @@ has_neoverse_n1_erratum_1542419(const struct arm64_cpu_capabilities *entry, return is_midr_in_range(&range) && has_dic; } -static const struct midr_range impdef_pmuv3_cpus[] = { +static const struct midr_range apple_cpus[] = { MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM), MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM), MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO), @@ -301,7 +301,14 @@ static bool has_impdef_pmuv3(const struct arm64_cpu_capabilities *entry, int sco if (pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF) return false; - return is_midr_in_range_list(impdef_pmuv3_cpus); + return is_midr_in_range_list(apple_cpus); +} + +static bool has_broken_gic_v3_seis(const struct arm64_cpu_capabilities *entry, int scope) +{ + return (is_kernel_in_hyp_mode() && + is_midr_in_range_list(apple_cpus) && + (read_sysreg_s(SYS_ICH_VTR_EL2) & ICH_VTR_EL2_SEIS)); } static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *__unused) @@ -1009,6 +1016,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .matches = has_impdef_pmuv3, .cpu_enable = cpu_enable_impdef_pmuv3_traps, }, + { + .desc = "Known broken GICv3 SEIS implementation", + .capability = ARM64_WORKAROUND_GICv3_BROKEN_SEIS, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_broken_gic_v3_seis, + }, { } }; diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 9e841e7afd4a..0991a649a5d1 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -809,27 +809,9 @@ static int __init early_gicv4_enable(char *buf) } early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable); -static const struct midr_range broken_seis[] = { - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_PRO), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_MAX), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_MAX), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_PRO), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_PRO), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_MAX), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_MAX), - {}, -}; - -static bool vgic_v3_broken_seis(void) +static __always_inline bool vgic_v3_broken_seis(void) { - return (is_kernel_in_hyp_mode() && - is_midr_in_range_list(broken_seis) && - (read_sysreg_s(SYS_ICH_VTR_EL2) & ICH_VTR_EL2_SEIS)); + return cpus_have_cap(ARM64_WORKAROUND_GICv3_BROKEN_SEIS); } void noinstr kvm_compute_ich_hcr_trap_bits(struct alt_instr *alt, @@ -959,10 +941,8 @@ int vgic_v3_probe(const struct gic_kvm_info *info) if (has_v2) static_branch_enable(&vgic_v3_has_v2_compat); - if (vgic_v3_broken_seis()) { - kvm_info("GICv3 with broken locally generated SEI\n"); + if (vgic_v3_broken_seis()) kvm_vgic_global_state.ich_vtr_el2 &= ~ICH_VTR_EL2_SEIS; - } vgic_v3_enable_cpuif_traps(); diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9b85a84f6fd4..50373264ecde 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -121,6 +121,7 @@ WORKAROUND_CAVIUM_TX2_219_TVM WORKAROUND_CLEAN_CACHE WORKAROUND_DEVICE_LOAD_ACQUIRE WORKAROUND_DISABLE_CNP +WORKAROUND_GICv3_BROKEN_SEIS WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF From 4edfdc8ddebdac3243592f14cfec529304a9666a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:51 +0100 Subject: [PATCH 07/89] KVM: arm64: Add a helper providing an inlined literal value for ICH_VTR_EL2 We already have two ways to deal with ICH_VTR_EL2: - directly read the system register - read a cached copy in the vgic global state Add a third way, in the form of an inlined literal value that will eventually replace all of the above. This literal value is computed at boot time, and patched in the relevant code locations. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-4-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kernel/image-vars.h | 1 + arch/arm64/kvm/vgic/vgic-v3.c | 55 ++++++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.h | 17 +++++++++++ 3 files changed, 73 insertions(+) diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index d4c7d45ae6bc..459c33634ebd 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -92,6 +92,7 @@ KVM_NVHE_ALIAS(spectre_bhb_patch_wa3); KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb); KVM_NVHE_ALIAS(alt_cb_patch_nops); KVM_NVHE_ALIAS(kvm_compute_ich_hcr_trap_bits); +KVM_NVHE_ALIAS(kvm_patch_ich_vtr_el2); /* Global kernel state accessed by nVHE hyp code. */ KVM_NVHE_ALIAS(kvm_vgic_global_state); diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 0991a649a5d1..098187e34c58 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -858,6 +858,61 @@ void noinstr kvm_compute_ich_hcr_trap_bits(struct alt_instr *alt, *updptr = cpu_to_le32(insn); } +void noinstr kvm_patch_ich_vtr_el2(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, + int nr_inst) +{ + struct arm_smccc_res res = {}; + u32 insn, oinsn, rd, vtr; + + /* No KVM? Nothing to do */ + if (!is_hyp_mode_available()) + return; + + /* No v3, compat, nor the fruity erzatz of a GIC? Bugger off */ + if (!cpus_have_cap(ARM64_HAS_GICV5_LEGACY) && + !cpus_have_cap(ARM64_HAS_GICV3_CPUIF) && + !vgic_v3_broken_seis()) + return; + + /* + * At the point where this is called, we are guaranteed that if + * we're running at EL1, then the EL2 stubs are still in place. + */ + if (is_kernel_in_hyp_mode()) + res.a1 = read_sysreg_s(SYS_ICH_VTR_EL2); + else + arm_smccc_1_1_hvc(HVC_GET_ICH_VTR_EL2, &res); + + if (res.a0 == HVC_STUB_ERR) + return; + + vtr = res.a1; + + if (vgic_v3_broken_seis()) + vtr &= ~ICH_VTR_EL2_SEIS; + + /* Compute target register */ + oinsn = le32_to_cpu(*origptr); + rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn); + + /* movz rd, #(vtr & 0xffff) */ + insn = aarch64_insn_gen_movewide(rd, + (u16)vtr, + 0, + AARCH64_INSN_VARIANT_64BIT, + AARCH64_INSN_MOVEWIDE_ZERO); + *updptr++ = cpu_to_le32(insn); + + /* movk rd, #((vtr >> 16) & 0xffff), lsl #16 */ + insn = aarch64_insn_gen_movewide(rd, + (u16)(vtr >> 16), + 16, + AARCH64_INSN_VARIANT_64BIT, + AARCH64_INSN_MOVEWIDE_KEEP); + *updptr++ = cpu_to_le32(insn); +} + void vgic_v3_enable_cpuif_traps(void) { u64 traps = vgic_ich_hcr_trap_bits(); diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index f45f7e3ec4d6..b76bb6e56de3 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -71,6 +71,23 @@ ICH_VTR_EL2_IDbits) #define KVM_ICH_VTR_EL2_RES1 ICH_VTR_EL2_nV4 +void kvm_patch_ich_vtr_el2(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst); + +static inline u64 vgic_ich_vtr(void) +{ + u64 vtr; + + /* All non-RES0 bits are in the bottom 32bits */ + asm volatile(ALTERNATIVE_CB("movz %0, #0\n" + "movk %0, #0, lsl #16\n", + ARM64_ALWAYS_SYSTEM, + kvm_patch_ich_vtr_el2) + : "=r" (vtr)); + + return vtr; +} + static inline u64 kvm_get_guest_vtr_el2(void) { u64 vtr; From 2bba3d2c60926f19e1d0da931513fead3af48f36 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:52 +0100 Subject: [PATCH 08/89] KVM: arm64: Convert most ICH_VTR_EL2 accesses to inlined literal value Now that we have a new helper that inlines the value of ICH_VTR_EL2 into the code, use this to replace most of the sysreg accesses to that register. This avoids a lot of traps when running KVM under NV. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-5-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/vgic-v3-sr.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 8da7a40ced58..3e5d5ddf35a1 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -367,7 +367,7 @@ void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if) u64 val; u32 nr_pre_bits; - val = read_gicreg(ICH_VTR_EL2); + val = vgic_ich_vtr(); nr_pre_bits = vtr_to_nr_pre_bits(val); switch (nr_pre_bits) { @@ -400,7 +400,7 @@ static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) u64 val; u32 nr_pre_bits; - val = read_gicreg(ICH_VTR_EL2); + val = vgic_ich_vtr(); nr_pre_bits = vtr_to_nr_pre_bits(val); switch (nr_pre_bits) { @@ -430,7 +430,7 @@ static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) void __vgic_v3_init_lrs(void) { - int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2)); + int max_lr_idx = vtr_to_max_lr_idx(vgic_ich_vtr()); int i; for (i = 0; i <= max_lr_idx; i++) @@ -455,7 +455,7 @@ u64 __vgic_v3_get_gic_config(void) * system, so we first check if we have GICv5 support. */ if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) - return read_gicreg(ICH_VTR_EL2); + return vgic_ich_vtr(); sre = read_gicreg(ICC_SRE_EL1); /* @@ -498,7 +498,7 @@ u64 __vgic_v3_get_gic_config(void) } val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); - val |= read_gicreg(ICH_VTR_EL2); + val |= vgic_ich_vtr(); return val; } @@ -540,7 +540,7 @@ void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) static int __vgic_v3_bpr_min(void) { /* See Pseudocode for VPriorityGroup */ - return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2)); + return 8 - vtr_to_nr_pre_bits(vgic_ich_vtr()); } static int __vgic_v3_get_group(struct kvm_vcpu *vcpu) @@ -614,7 +614,7 @@ static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid, static int __vgic_v3_get_highest_active_priority(void) { - u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); + u8 nr_apr_regs = vtr_to_nr_apr_regs(vgic_ich_vtr()); u32 hap = 0; int i; @@ -707,7 +707,7 @@ static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp) static int __vgic_v3_clear_highest_active_priority(void) { - u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); + u8 nr_apr_regs = vtr_to_nr_apr_regs(vgic_ich_vtr()); u32 hap = 0; int i; @@ -1039,7 +1039,7 @@ static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) { u32 vtr, val; - vtr = read_gicreg(ICH_VTR_EL2); + vtr = vgic_ich_vtr(); /* PRIbits */ val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT; /* IDbits */ From bea608299d28ee4e10db3468985dde440adb09b7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:53 +0100 Subject: [PATCH 09/89] KVM: arm64: vgic-v3: Simplify initial GICv3 configuration sampling Now that we have our magic inline helper for ICH_VTR_EL2, we can get rid of the hack that was reporting a combination of that register and of the indication of the CPU interface supporting GICv2 compatibility. We now only report the latter. As a small benefit, GICv5 is not involved in this stuff anymore, since it never has GICv2 compatibility.. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-6-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_asm.h | 2 +- arch/arm64/kvm/hyp/vgic-v3-sr.c | 23 +++-------------------- arch/arm64/kvm/vgic/vgic-v3.c | 9 +++------ arch/arm64/kvm/vgic/vgic-v5.c | 2 +- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 043495f7fc78..cb354a037fc5 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -281,7 +281,7 @@ extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); extern void __kvm_adjust_pc(struct kvm_vcpu *vcpu); -extern u64 __vgic_v3_get_gic_config(void); +extern bool __vgic_v3_get_gic_config(void); extern void __vgic_v3_init_lrs(void); #define __KVM_EXTABLE(from, to) \ diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 3e5d5ddf35a1..74d4a509f1de 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -437,26 +437,12 @@ void __vgic_v3_init_lrs(void) __gic_v3_set_lr(0, i); } -/* - * Return the GIC CPU configuration: - * - [31:0] ICH_VTR_EL2 - * - [62:32] RES0 - * - [63] MMIO (GICv2) capable - */ -u64 __vgic_v3_get_gic_config(void) +/* Return true if GICv3 is MMIO (GICv2) capable, false otherwise */ +bool __vgic_v3_get_gic_config(void) { u64 val, sre; unsigned long flags = 0; - /* - * In compat mode, we cannot access ICC_SRE_EL1 at any EL - * other than EL1 itself; just return the - * ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5 - * system, so we first check if we have GICv5 support. - */ - if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) - return vgic_ich_vtr(); - sre = read_gicreg(ICC_SRE_EL1); /* * To check whether we have a MMIO-based (GICv2 compatible) @@ -497,10 +483,7 @@ u64 __vgic_v3_get_gic_config(void) isb(); } - val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); - val |= vgic_ich_vtr(); - - return val; + return !(val & ICC_SRE_EL1_SRE); } static void __vgic_v3_compat_mode_enable(void) diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 098187e34c58..734fa3138723 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -936,12 +936,12 @@ void vgic_v3_enable_cpuif_traps(void) */ int vgic_v3_probe(const struct gic_kvm_info *info) { - u64 ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config); + u64 ich_vtr_el2; bool has_v2; int ret; - has_v2 = ich_vtr_el2 >> 63; - ich_vtr_el2 = (u32)ich_vtr_el2; + has_v2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config); + ich_vtr_el2 = vgic_ich_vtr(); /* * The ListRegs field is 5 bits, but there is an architectural @@ -996,9 +996,6 @@ int vgic_v3_probe(const struct gic_kvm_info *info) if (has_v2) static_branch_enable(&vgic_v3_has_v2_compat); - if (vgic_v3_broken_seis()) - kvm_vgic_global_state.ich_vtr_el2 &= ~ICH_VTR_EL2_SEIS; - vgic_v3_enable_cpuif_traps(); kvm_vgic_global_state.vctrl_base = NULL; diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index d4789ff3e740..16bc0a670d3e 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -83,7 +83,7 @@ int vgic_v5_probe(const struct gic_kvm_info *info) } kvm_vgic_global_state.has_gcie_v3_compat = true; - ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config); + ich_vtr_el2 = vgic_ich_vtr(); kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2; /* From 36d32222fcab099cf07b032aa5f022136bfa91be Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:54 +0100 Subject: [PATCH 10/89] KVM: arm64: vgic-v3: Kill kvm_vgic_global_state.ich_vtr_el2 kvm_vgic_global_state.ich_vtr_el2 is the last bit of caching that we can get rid of. Not as bad as a sysreg access, but still worse than a constant. Move over to the inlined stuff and remove the cached value. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-7-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 3 ++- arch/arm64/kvm/vgic-sys-reg-v3.c | 8 ++++---- arch/arm64/kvm/vgic/vgic-v3.c | 5 ++--- arch/arm64/kvm/vgic/vgic-v5.c | 5 +---- arch/arm64/kvm/vgic/vgic.h | 2 +- include/kvm/arm_vgic.h | 2 -- 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index fb54f6dad995..54defcd8240d 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -16,6 +16,7 @@ #include #include "sys_regs.h" +#include "vgic/vgic.h" struct vncr_tlb { /* The guest's VNCR_EL2 */ @@ -1904,7 +1905,7 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) /* ICH_HCR_EL2 */ resx.res0 = ICH_HCR_EL2_RES0; resx.res1 = ICH_HCR_EL2_RES1; - if (!(kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_EL2_TDS)) + if (!(vgic_ich_vtr() & ICH_VTR_EL2_TDS)) resx.res0 |= ICH_HCR_EL2_TDIR; /* No GICv4 is presented to the guest */ resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; diff --git a/arch/arm64/kvm/vgic-sys-reg-v3.c b/arch/arm64/kvm/vgic-sys-reg-v3.c index bdc2d57370b2..89315336b9d3 100644 --- a/arch/arm64/kvm/vgic-sys-reg-v3.c +++ b/arch/arm64/kvm/vgic-sys-reg-v3.c @@ -35,12 +35,12 @@ static int set_gic_ctlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, vgic_v3_cpu->num_id_bits = host_id_bits; - host_seis = FIELD_GET(ICH_VTR_EL2_SEIS, kvm_vgic_global_state.ich_vtr_el2); + host_seis = FIELD_GET(ICH_VTR_EL2_SEIS, vgic_ich_vtr()); seis = FIELD_GET(ICC_CTLR_EL1_SEIS_MASK, val); if (host_seis != seis) return -EINVAL; - host_a3v = FIELD_GET(ICH_VTR_EL2_A3V, kvm_vgic_global_state.ich_vtr_el2); + host_a3v = FIELD_GET(ICH_VTR_EL2_A3V, vgic_ich_vtr()); a3v = FIELD_GET(ICC_CTLR_EL1_A3V_MASK, val); if (host_a3v != a3v) return -EINVAL; @@ -69,9 +69,9 @@ static int get_gic_ctlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, val |= FIELD_PREP(ICC_CTLR_EL1_ID_BITS_MASK, vgic_v3_cpu->num_id_bits); val |= FIELD_PREP(ICC_CTLR_EL1_SEIS_MASK, FIELD_GET(ICH_VTR_EL2_SEIS, - kvm_vgic_global_state.ich_vtr_el2)); + vgic_ich_vtr())); val |= FIELD_PREP(ICC_CTLR_EL1_A3V_MASK, - FIELD_GET(ICH_VTR_EL2_A3V, kvm_vgic_global_state.ich_vtr_el2)); + FIELD_GET(ICH_VTR_EL2_A3V, vgic_ich_vtr())); /* * The VMCR.CTLR value is in ICC_CTLR_EL1 layout. * Extract it directly using ICC_CTLR_EL1 reg definitions. diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 734fa3138723..91514d6faf74 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -490,9 +490,9 @@ void vgic_v3_reset(struct kvm_vcpu *vcpu) } vcpu->arch.vgic_cpu.num_id_bits = FIELD_GET(ICH_VTR_EL2_IDbits, - kvm_vgic_global_state.ich_vtr_el2); + vgic_ich_vtr()); vcpu->arch.vgic_cpu.num_pri_bits = FIELD_GET(ICH_VTR_EL2_PRIbits, - kvm_vgic_global_state.ich_vtr_el2) + 1; + vgic_ich_vtr()) + 1; } void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu) @@ -949,7 +949,6 @@ int vgic_v3_probe(const struct gic_kvm_info *info) */ kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1; kvm_vgic_global_state.can_emulate_gicv2 = false; - kvm_vgic_global_state.ich_vtr_el2 = ich_vtr_el2; /* GICv4 support? */ if (info->has_v4) { diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 16bc0a670d3e..4e3e4db24e77 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -40,7 +40,6 @@ static void vgic_v5_get_implemented_ppis(void) int vgic_v5_probe(const struct gic_kvm_info *info) { bool v5_registered = false; - u64 ich_vtr_el2; int ret; kvm_vgic_global_state.type = VGIC_V5; @@ -83,14 +82,12 @@ int vgic_v5_probe(const struct gic_kvm_info *info) } kvm_vgic_global_state.has_gcie_v3_compat = true; - ich_vtr_el2 = vgic_ich_vtr(); - kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2; /* * The ListRegs field is 5 bits, but there is an architectural * maximum of 16 list registers. Just ignore bit 4... */ - kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1; + kvm_vgic_global_state.nr_lr = (vgic_ich_vtr() & 0xf) + 1; ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3); if (ret) { diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index b76bb6e56de3..b71d486ae514 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -92,7 +92,7 @@ static inline u64 kvm_get_guest_vtr_el2(void) { u64 vtr; - vtr = kvm_vgic_global_state.ich_vtr_el2; + vtr = vgic_ich_vtr(); vtr &= ~KVM_ICH_VTR_EL2_RES0; vtr |= KVM_ICH_VTR_EL2_RES1; diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index fe49fb56dc3c..bd1bb03500b3 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -176,8 +176,6 @@ struct vgic_global { /* GICv3 compat mode on a GICv5 host */ bool has_gcie_v3_compat; - u32 ich_vtr_el2; - /* GICv5 PPI capabilities */ struct { DECLARE_BITMAP(impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); From 596289fe4836067b0320c80a9ab0101b9f7f9328 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 21 Jul 2026 17:16:24 +0100 Subject: [PATCH 11/89] KVM: arm64: Sync SCTLR_EL1 when injecting an exception into a pVM When pKVM injects a synchronous exception into a protected guest, enter_exception64() reads SCTLR_EL1 to set the new PSTATE's PAN and SSBS bits. pKVM refreshes VBAR_EL1 from the live value before injecting but not SCTLR_EL1, which is untrapped for protected guests, so a guest that updates it and then traps takes the exception with stale PAN/SSBS. Sync SCTLR_EL1 alongside VBAR_EL1. Fixes: 798eb5978700 ("KVM: arm64: Sync protected guest VBAR_EL1 on injecting an undef exception") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260721161624.3983041-1-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/sys_regs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8c3fbb413a06..7608cc4030b8 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -257,10 +257,11 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr) *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); /* - * Make sure we have the latest update to VBAR_EL1, as pKVM - * handles traps very early, before sysregs are resync'ed + * Sync VBAR_EL1 and SCTLR_EL1, both read by enter_exception64(), + * as pKVM handles traps before sysregs are resync'ed. */ __vcpu_assign_sys_reg(vcpu, VBAR_EL1, read_sysreg_el1(SYS_VBAR)); + __vcpu_assign_sys_reg(vcpu, SCTLR_EL1, read_sysreg_el1(SYS_SCTLR)); kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); From 453f6ff04b8d28991f94ccac89350951252aa947 Mon Sep 17 00:00:00 2001 From: Congkai Tan Date: Wed, 22 Jul 2026 20:26:59 +0000 Subject: [PATCH 12/89] KVM: arm64: Expose PMMIR_EL1.SLOTS under strict PMUv3 UAPI Introduce a new field pmmir_slots in struct kvm_arch to store PMMIR_EL1.SLOTS. It only saves the actual hardware PMU value when the VMM explicitly selects a PMU under KVM_ARM_VCPU_PMU_V3_STRICT. Otherwise, it stays 0 after allocation. Use this field to implement guest access, userspace get, and userspace set for PMMIR_EL1: - access_pmmir(): uses the value in kvm->arch.pmmir_slots directly. If the VMM selected a PMU and KVM_ARM_VCPU_PMU_V3_STRICT is set, the guest can correctly read the underlying core's SLOTS. Otherwise, it continues to read 0 since the true SLOTS value can be nondeterministic. - get_pmmir(): same as access_pmmir(). - set_pmmir(): only the SLOTS field is writable; a value setting any other bit is rejected with -EINVAL, since get_pmmir() returns SLOTS zero-extended. A value of 0 resets kvm->arch.pmmir_slots to 0 for backward compatibility, as the register is RAZ in older KVM, a value matching the current SLOTS is accepted as a no-op, and anything else is rejected with -EINVAL. Once the VM has run PMMIR_EL1 is immutable, so a mismatching write then returns -EBUSY. The register is now exposed via KVM_GET_REG_LIST for PMUv3 vCPUs, so add it to the get-reg-list selftest's PMU register list. Signed-off-by: Congkai Tan Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Reviewed-by: Stanislav Spassov Co-developed-by: Oliver Upton Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-2-congkai@amazon.com Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 3 + arch/arm64/include/uapi/asm/kvm.h | 1 + arch/arm64/kvm/pmu-emul.c | 11 ++++ arch/arm64/kvm/sys_regs.c | 63 ++++++++++++++++++- include/kvm/arm_pmu.h | 4 ++ .../selftests/kvm/arm64/get-reg-list.c | 1 + 6 files changed, 81 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..9e035d587970 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -387,6 +387,9 @@ struct kvm_arch { /* Maximum number of counters for the guest */ u8 nr_pmu_counters; + /* PMMIR_EL1.SLOTS value exposed to the guest. */ + u8 pmmir_slots; + /* Hypercall features firmware registers' descriptor */ struct kvm_smccc_features smccc_feat; struct maple_tree smccc_filter; diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 1c13bfa2d38a..019e5e3d892e 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -106,6 +106,7 @@ struct kvm_regs { #define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */ #define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */ #define KVM_ARM_VCPU_HAS_EL2_E2H0 8 /* Limit NV support to E2H RES0 */ +#define KVM_ARM_VCPU_PMU_V3_STRICT 9 /* No default PMU creation */ struct kvm_vcpu_init { __u32 target; diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 98305bbfc095..7ac00423fa8d 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -1092,6 +1092,17 @@ static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id) kvm_arm_set_pmu(kvm, arm_pmu); cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus); + + /* + * Since a specific PMU is explicitly selected, + * PMMIR_EL1.SLOTS is deterministic to the guest. + * If KVM_ARM_VCPU_PMU_V3_STRICT is set, snapshot + * the value to allow the guest to read it. + */ + if (kvm_vcpu_has_pmuv3_strict(vcpu)) + kvm->arch.pmmir_slots = + FIELD_GET(ARMV8_PMU_SLOTS, + arm_pmu->reg_pmmir); ret = 0; break; } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..4c6f608c4fe5 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1367,6 +1367,64 @@ static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p, return true; } +static bool access_pmmir(struct kvm_vcpu *vcpu, struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + if (p->is_write) + return write_to_read_only(vcpu, p, r); + + /* + * If KVM_ARM_VCPU_PMU_V3_STRICT is set and PMU was explicitly + * selected, the underlying hardware SLOTS value was read into this + * field. Otherwise, it stays 0. All other PMMIR_EL1 fields are RAZ. + */ + p->regval = FIELD_PREP(ARMV8_PMU_SLOTS, vcpu->kvm->arch.pmmir_slots); + return true; +} + +static int get_pmmir(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + *val = FIELD_PREP(ARMV8_PMU_SLOTS, vcpu->kvm->arch.pmmir_slots); + return 0; +} + +static int set_pmmir(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct kvm *kvm = vcpu->kvm; + u8 slots = FIELD_GET(ARMV8_PMU_SLOTS, val); + + /* + * Only the SLOTS field is exposed (get_pmmir returns just that field), + * so reject a write that sets any other bit rather than silently + * masking it. + */ + if (val & ~(u64)ARMV8_PMU_SLOTS) + return -EINVAL; + + guard(mutex)(&kvm->arch.config_lock); + + /* + * Once the VM has started PMMIR_EL1 is immutable. Reject any write + * that does not match the current value. + */ + if (kvm_vm_has_ran_once(kvm)) + return slots == kvm->arch.pmmir_slots ? 0 : -EBUSY; + + /* + * Only SLOTS = 0 is honored for backwards compatibility with the + * old RAZ behavior. Reject any non-zero write that does not match + * the current value. + */ + if (!slots) + kvm->arch.pmmir_slots = 0; + else if (slots != kvm->arch.pmmir_slots) + return -EINVAL; + + return 0; +} + static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) { @@ -3448,7 +3506,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { PMU_SYS_REG(PMINTENCLR_EL1), .access = access_pminten, .reg = PMINTENSET_EL1, .get_user = get_pmreg, .set_user = set_pmreg }, - { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi }, + { PMU_SYS_REG(PMMIR_EL1), .access = access_pmmir, .reset = NULL, + .get_user = get_pmmir, .set_user = set_pmmir }, { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 }, { SYS_DESC(SYS_PIRE0_EL1), NULL, reset_unknown, PIRE0_EL1, @@ -4593,7 +4652,7 @@ static const struct sys_reg_desc cp15_regs[] = { { CP15_PMU_SYS_REG(HI, 0, 9, 14, 4), .access = access_pmceid }, { CP15_PMU_SYS_REG(HI, 0, 9, 14, 5), .access = access_pmceid }, /* PMMIR */ - { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = trap_raz_wi }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = access_pmmir }, /* PRRR/MAIR0 */ { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 }, diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h index b5e5942204fc..6b4a118d17ca 100644 --- a/include/kvm/arm_pmu.h +++ b/include/kvm/arm_pmu.h @@ -75,6 +75,9 @@ void kvm_vcpu_pmu_resync_el0(void); #define kvm_vcpu_has_pmu(vcpu) \ (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3)) +#define kvm_vcpu_has_pmuv3_strict(vcpu) \ + (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3_STRICT)) + /* * Updates the vcpu's view of the pmu events for this cpu. * Must be called before every vcpu run after disabling interrupts, to ensure @@ -160,6 +163,7 @@ static inline u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) } #define kvm_vcpu_has_pmu(vcpu) ({ false; }) +#define kvm_vcpu_has_pmuv3_strict(vcpu) ({ false; }) static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {} static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {} static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {} diff --git a/tools/testing/selftests/kvm/arm64/get-reg-list.c b/tools/testing/selftests/kvm/arm64/get-reg-list.c index 0a3a94c4cca1..cfa99979d57c 100644 --- a/tools/testing/selftests/kvm/arm64/get-reg-list.c +++ b/tools/testing/selftests/kvm/arm64/get-reg-list.c @@ -532,6 +532,7 @@ static __u64 base_regs[] = { static __u64 pmu_regs[] = { ARM64_SYS_REG(3, 0, 9, 14, 1), /* PMINTENSET_EL1 */ ARM64_SYS_REG(3, 0, 9, 14, 2), /* PMINTENCLR_EL1 */ + ARM64_SYS_REG(3, 0, 9, 14, 6), /* PMMIR_EL1 */ ARM64_SYS_REG(3, 3, 9, 12, 0), /* PMCR_EL0 */ ARM64_SYS_REG(3, 3, 9, 12, 1), /* PMCNTENSET_EL0 */ ARM64_SYS_REG(3, 3, 9, 12, 2), /* PMCNTENCLR_EL0 */ From 203b22f76dc3025d3fa96830a7cf6eadbf2c8407 Mon Sep 17 00:00:00 2001 From: Congkai Tan Date: Wed, 22 Jul 2026 20:27:00 +0000 Subject: [PATCH 13/89] KVM: arm64: Advertise STALL_SLOT* in PMCEID1 under strict PMUv3 UAPI Skip masking STALL_SLOT, STALL_SLOT_FRONTEND and STALL_SLOT_BACKEND out of PMCEID1 when KVM_ARM_VCPU_PMU_V3_STRICT is set, because this is when PMMIR_EL1.SLOTS is exposed to guests, making these events meaningful for collection. Change the parameter of compute_pmceid1() from arm_pmu to kvm_vcpu, to check if KVM_ARM_VCPU_PMU_V3_STRICT is set. Also updated the signature of compute_pmceid0() for consistency. Signed-off-by: Congkai Tan Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Reviewed-by: Stanislav Spassov Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-3-congkai@amazon.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/pmu-emul.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 7ac00423fa8d..bcfd0a91e114 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -838,9 +838,9 @@ static u64 __compute_pmceid(struct arm_pmu *pmu, bool pmceid1) return ((u64)hi[pmceid1] << 32) | lo[pmceid1]; } -static u64 compute_pmceid0(struct arm_pmu *pmu) +static u64 compute_pmceid0(struct kvm_vcpu *vcpu) { - u64 val = __compute_pmceid(pmu, 0); + u64 val = __compute_pmceid(vcpu->kvm->arch.arm_pmu, 0); /* always support SW_INCR */ val |= BIT(ARMV8_PMUV3_PERFCTR_SW_INCR); @@ -849,32 +849,33 @@ static u64 compute_pmceid0(struct arm_pmu *pmu) return val; } -static u64 compute_pmceid1(struct arm_pmu *pmu) +static u64 compute_pmceid1(struct kvm_vcpu *vcpu) { - u64 val = __compute_pmceid(pmu, 1); + u64 val = __compute_pmceid(vcpu->kvm->arch.arm_pmu, 1); /* - * Don't advertise STALL_SLOT*, as PMMIR_EL0 is handled - * as RAZ + * If KVM_ARM_VCPU_PMU_V3_STRICT is not set, PMMIR_EL1 is + * unconditionally RAZ, so don't advertise STALL_SLOT* events. */ - val &= ~(BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32) | - BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND - 32) | - BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND - 32)); + if (!kvm_vcpu_has_pmuv3_strict(vcpu)) + val &= ~(BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32) | + BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND - 32) | + BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND - 32)); + return val; } u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) { - struct arm_pmu *cpu_pmu = vcpu->kvm->arch.arm_pmu; unsigned long *bmap = vcpu->kvm->arch.pmu_filter; u64 val, mask = 0; int base, i, nr_events; if (!pmceid1) { - val = compute_pmceid0(cpu_pmu); + val = compute_pmceid0(vcpu); base = 0; } else { - val = compute_pmceid1(cpu_pmu); + val = compute_pmceid1(vcpu); base = 32; } From bbef6317f97b8612a694ddc2ff6e7ca81780768d Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 22 Jul 2026 20:27:01 +0000 Subject: [PATCH 14/89] KVM: arm64: Ignore writes to PMCR_EL0.N when using strict UAPI KVM allows userspace to configure the number of event counters by writing to PMCR_EL0.N. While this makes a bit of sense prior to NV, the value of the field is context-dependent whether or not the vCPU is at EL2 or EL1. As such, the vCPU attribute was added afterwards and made mandatory for NV. As we're in the process of papering over old mistakes, force userspace to use the vCPU attribute if KVM_ARM_VCPU_PMU_V3_STRICT is set. Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Signed-off-by: Congkai Tan Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-4-congkai@amazon.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/sys_regs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 4c6f608c4fe5..e93f11b90813 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1502,6 +1502,7 @@ static int set_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, */ if (!kvm_vm_has_ran_once(kvm) && !vcpu_has_nv(vcpu) && + !kvm_vcpu_has_pmuv3_strict(vcpu) && new_n <= kvm_arm_pmu_get_max_counters(kvm)) kvm->arch.nr_pmu_counters = new_n; From ad220e275c3987aea865d4c05ad66ba0295c494d Mon Sep 17 00:00:00 2001 From: Congkai Tan Date: Wed, 22 Jul 2026 20:27:02 +0000 Subject: [PATCH 15/89] KVM: arm64: Add KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature Introduce a new vCPU feature KVM_ARM_VCPU_PMU_V3_STRICT. When set, KVM does not create a default PMU when initializing the vCPU, and userspace must select one explicitly via KVM_ARM_VCPU_PMU_V3_SET_PMU before the first KVM_RUN. The flag forces the VMM to be aware of the PMU implementation of the guest to be created, so that certain information about the PMU becomes deterministic (if on a heterogeneous system) and becomes safe to be exposed to the guest. It can be used as an umbrella flag to gate future PMUv3 UAPI changes. When no default PMU is created, kvm->arch.arm_pmu stays NULL until SET_PMU runs, so kvm_arm_pmu_v3_init() now refuses to run if kvm->arch.arm_pmu is NULL. Signed-off-by: Congkai Tan Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Reviewed-by: Stanislav Spassov Co-developed-by: Oliver Upton Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-5-congkai@amazon.com Signed-off-by: Oliver Upton --- Documentation/virt/kvm/api.rst | 11 +++++++++++ Documentation/virt/kvm/devices/vcpu.rst | 11 +++++++++-- arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/kvm/arm.c | 19 +++++++++++++++---- arch/arm64/kvm/pmu-emul.c | 18 +++++++++++++++++- include/uapi/linux/kvm.h | 1 + 6 files changed, 54 insertions(+), 8 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index a5f9ee92f43e..4988c32df4bf 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -3515,6 +3515,17 @@ Possible features: Depends on KVM_CAP_ARM_PSCI_0_2. - KVM_ARM_VCPU_PMU_V3: Emulate PMUv3 for the CPU. Depends on KVM_CAP_ARM_PMU_V3. + - KVM_ARM_VCPU_PMU_V3_STRICT: Enable strict PMUv3 UAPI. + Requires KVM_ARM_VCPU_PMU_V3. Depends on KVM_CAP_ARM_PMU_V3_STRICT. + When enabled: + + * Userspace must explicitly select a PMU implementation before + initializing the PMU or configuring a PMU event filter + + * If the PMU implements FEAT_PMUv3p4, PMMIR_EL1.SLOTS provides the + hardware value of the underlying implementation + + * Writes to PMCR_EL0.N via KVM_SET_ONE_REG are ignored - KVM_ARM_VCPU_PTRAUTH_ADDRESS: Enables Address Pointer authentication for arm64 only. diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst index 66e714f2fcfa..deb5c51bc00c 100644 --- a/Documentation/virt/kvm/devices/vcpu.rst +++ b/Documentation/virt/kvm/devices/vcpu.rst @@ -53,8 +53,9 @@ Returns: ======= ====================================================== -EEXIST Interrupt number already used -ENODEV PMUv3 not supported or GIC not initialized - -ENXIO PMUv3 not supported, missing VCPU feature or interrupt - number not set (non-GICv5 guests, only) + -ENXIO PMUv3 not supported, missing VCPU feature, missing + hardware PMU, or interrupt number not set (non-GICv5 + guests, only) -EBUSY PMUv3 already initialized ======= ====================================================== @@ -62,6 +63,9 @@ Request the initialization of the PMUv3. If using the PMUv3 with an in-kernel virtual GIC implementation, this must be done after initializing the in-kernel irqchip. +When the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature is enabled this must be done +after selecting a hardware PMU. + 1.3 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_FILTER ----------------------------------------- @@ -108,6 +112,9 @@ hardware event. Filtering event 0x1E (CHAIN) has no effect either, as it isn't strictly speaking an event. Filtering the cycle counter is possible using event 0x11 (CPU_CYCLES). +When the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature is enabled this must be done +after selecting a hardware PMU. + 1.4 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_SET_PMU ------------------------------------------ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 9e035d587970..39f7fc740d07 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -39,7 +39,7 @@ #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS -#define KVM_VCPU_MAX_FEATURES 9 +#define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) #define KVM_REQ_SLEEP \ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 50adfff75be8..7dbefdd846aa 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -452,6 +452,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = get_num_wrps(); break; case KVM_CAP_ARM_PMU_V3: + case KVM_CAP_ARM_PMU_V3_STRICT: r = kvm_supports_guest_pmuv3(); break; case KVM_CAP_ARM_INJECT_SERROR_ESR: @@ -1563,8 +1564,10 @@ static unsigned long system_supported_vcpu_features(void) if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); - if (!kvm_supports_guest_pmuv3()) + if (!kvm_supports_guest_pmuv3()) { clear_bit(KVM_ARM_VCPU_PMU_V3, &features); + clear_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features); + } if (!system_supports_sve()) clear_bit(KVM_ARM_VCPU_SVE, &features); @@ -1605,6 +1608,11 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features)) return -EINVAL; + /* Strict PMUv3 UAPI requires PMUv3. */ + if (test_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features) && + !test_bit(KVM_ARM_VCPU_PMU_V3, &features)) + return -EINVAL; + if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features)) return 0; @@ -1634,10 +1642,13 @@ static int kvm_setup_vcpu(struct kvm_vcpu *vcpu) int ret = 0; /* - * When the vCPU has a PMU, but no PMU is set for the guest - * yet, set the default one. + * When the vCPU has a PMU, but no PMU is set for the guest yet, set + * the default one. If KVM_ARM_VCPU_PMU_V3_STRICT is set, no default + * PMU is created, and userspace must select a PMU via + * KVM_ARM_VCPU_PMU_V3_SET_PMU. */ - if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu) + if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu && + !kvm_vcpu_has_pmuv3_strict(vcpu)) ret = kvm_arm_set_default_pmu(kvm); /* Prepare for nested if required */ diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index bcfd0a91e114..5b1af7e2176f 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -939,6 +939,10 @@ int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu) { + /* Only possible when using KVM_ARM_VCPU_PMU_V3_STRICT */ + if (!vcpu->kvm->arch.arm_pmu) + return -ENXIO; + if (irqchip_in_kernel(vcpu->kvm)) { int ret; @@ -1009,6 +1013,14 @@ u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm) { struct arm_pmu *arm_pmu = kvm->arch.arm_pmu; + /* + * Under KVM_ARM_VCPU_PMU_V3_STRICT no PMU exists until userspace sets + * one, so this can be reached before arm_pmu is set. Report no + * counters in that case. + */ + if (!arm_pmu) + return 0; + /* * PMUv3 requires that all event counters are capable of counting any * event, though the same may not be true of non-PMUv3 hardware. @@ -1050,7 +1062,8 @@ static void kvm_arm_set_pmu(struct kvm *kvm, struct arm_pmu *arm_pmu) } /** - * kvm_arm_set_default_pmu - No PMU set, get the default one. + * kvm_arm_set_default_pmu - No PMU set and KVM_ARM_VCPU_PMU_V3_STRICT not + * set, get the default one. * @kvm: The kvm pointer * * The observant among you will notice that the supported_cpus @@ -1190,6 +1203,9 @@ int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) if (kvm_vm_has_ran_once(kvm)) return -EBUSY; + if (!kvm->arch.arm_pmu) + return -ENXIO; + if (!kvm->arch.pmu_filter) { kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT); if (!kvm->arch.pmu_filter) diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 419011097fa8..9fc8dfdfd65f 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -997,6 +997,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_KEYOP 247 #define KVM_CAP_S390_VSIE_ESAMODE 248 #define KVM_CAP_S390_HPAGE_2G 249 +#define KVM_CAP_ARM_PMU_V3_STRICT 250 struct kvm_irq_routing_irqchip { __u32 irqchip; From 81b2d03f8c86a1956606640794cafa6949fbc295 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 23 Jul 2026 16:20:02 +0000 Subject: [PATCH 16/89] KVM: arm64: Optimize protected mode with FWB and DIC KVM opportunistically enables FWB if supported by the system for guest VMs, which allows it to elude cache maintenance for data as they are forced to be cacheable from stage-2. In that case, __clean_dcache_guest_page() will immediately return. However in protected mode, before calling __clean_dcache_guest_page() it loops over the range and fix_map/unmap it, issuing TLB invalidations, dsb() and isb() unnecessarily. This can be optimized by returning early if FWB is supported, kvm_pgtable_stage2_map() already issues dsb() and tlb invalidation functions issue dsb() for the unmap path. Similarly for DIC, we can omit the call in invalidate_icache_guest_page() No isb() needed as ERET is context synchronization event. Signed-off-by: Mostafa Saleh Acked-by: Marc Zyngier Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723162002.3848054-1-smostafa@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 4e329e39a695..39aa8911f62c 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -261,11 +261,18 @@ static void __apply_guest_page(void *va, size_t size, static void clean_dcache_guest_page(void *va, size_t size) { + /* See comment in __clean_dcache_guest_page() */ + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) + return; + __apply_guest_page(va, size, __clean_dcache_guest_page); } static void invalidate_icache_guest_page(void *va, size_t size) { + if (alternative_has_cap_unlikely(ARM64_HAS_CACHE_DIC)) + return; + __apply_guest_page(va, size, __invalidate_icache_guest_page); } From 119a9a23fbae4fefbb5cd4776547b3b6803a7eaa Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Sun, 26 Jul 2026 15:36:42 +0100 Subject: [PATCH 17/89] KVM: arm64: Flush external_mdscr_el1 to the pKVM hyp vCPU flush_debug_state() propagates the guest's debug_owner and the owner-selected debug register state to the hyp vCPU, but not external_mdscr_el1. While the host owns the debug registers, the world switch loads MDSCR_EL1 from external_mdscr_el1 (ctxt_mdscr_el1()), where the host's KDE/MDE/SS bits live. A non-protected guest under KVM_GUESTDBG_USE_HW or single-step therefore runs with MDSCR_EL1.MDE/SS clear in hardware, and its watchpoints, breakpoints and single-step never fire. Propagate external_mdscr_el1 to the hyp vCPU alongside the host-owned debug state. Fixes: 4ad3a0b87f2ec ("KVM: arm64: Don't hijack guest context MDSCR_EL1") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260726143643.303456-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index d3c69de698f4..104026ee70e8 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -108,10 +108,16 @@ static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.debug_owner = host_vcpu->arch.debug_owner; - if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) + if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) { hyp_vcpu->vcpu.arch.vcpu_debug_state = host_vcpu->arch.vcpu_debug_state; - else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) + } else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) { hyp_vcpu->vcpu.arch.external_debug_state = host_vcpu->arch.external_debug_state; + /* + * The world switch loads MDSCR_EL1 from external_mdscr_el1 + * (ctxt_mdscr_el1()). + */ + hyp_vcpu->vcpu.arch.external_mdscr_el1 = host_vcpu->arch.external_mdscr_el1; + } } static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) From 3be99f8751c9eed017f437644fd7ecbdfdc66fbe Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Sun, 26 Jul 2026 15:36:43 +0100 Subject: [PATCH 18/89] KVM: arm64: selftests: Add a userspace watchpoint test debug-exceptions covers guest self-hosted debug and userspace single-step, but not a userspace (KVM_GUESTDBG_USE_HW) watchpoint, whose KVM_EXIT_DEBUG reports the accessed address in debug.arch.far. Add a test that installs a host-directed write watchpoint and checks that the reported address matches the accessed variable. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260726143643.303456-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- .../selftests/kvm/arm64/debug-exceptions.c | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/debug-exceptions.c b/tools/testing/selftests/kvm/arm64/debug-exceptions.c index 3eb4b1b6682d..7dc5f0b4f6ad 100644 --- a/tools/testing/selftests/kvm/arm64/debug-exceptions.c +++ b/tools/testing/selftests/kvm/arm64/debug-exceptions.c @@ -527,6 +527,46 @@ void test_single_step_from_userspace(int test_cnt) kvm_vm_free(vm); } +static void guest_code_wp(void) +{ + write_data = 'x'; + GUEST_DONE(); +} + +/* + * A userspace hardware watchpoint (KVM_GUESTDBG_USE_HW) must fire and report + * the accessed address in debug.arch.far, exercising the watchpoint exit path. + */ +static void test_watchpoint_from_userspace(void) +{ + struct kvm_guest_debug debug = {}; + struct kvm_vcpu *vcpu; + struct kvm_run *run; + struct kvm_vm *vm; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_wp); + run = vcpu->run; + + debug.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW; + debug.arch.dbg_wcr[0] = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | + DBGWCR_EL1 | DBGWCR_E; + /* + * BAS = 0xff (LEN8) requires a doubleword-aligned DBGWVR; FAR still + * reports the exact accessed byte. + */ + debug.arch.dbg_wvr[0] = PC(write_data) & ~7UL; + vcpu_guest_debug_set(vcpu, &debug); + + vcpu_run(vcpu); + TEST_ASSERT(run->exit_reason == KVM_EXIT_DEBUG, + "Expected KVM_EXIT_DEBUG, got %u", run->exit_reason); + TEST_ASSERT((u64)run->debug.arch.far == PC(write_data), + "Watchpoint FAR 0x%lx != accessed address 0x%lx", + (u64)run->debug.arch.far, PC(write_data)); + + kvm_vm_free(vm); +} + /* * Run debug testing using the various breakpoint#, watchpoint# and * context-aware breakpoint# with the given ID_AA64DFR0_EL1 configuration. @@ -600,6 +640,7 @@ int main(int argc, char *argv[]) test_guest_debug_exceptions_all(aa64dfr0); test_single_step_from_userspace(ss_iteration); + test_watchpoint_from_userspace(); return 0; } From c0646b96473340c71cd133ac4056efc9f78141f7 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:16 +0100 Subject: [PATCH 19/89] KVM: arm64: Extract MPIDR computation into a shared header Extract the vCPU MPIDR computation embedded in reset_mpidr() into a kvm_calculate_mpidr() inline in sys_regs.h, so it can be computed without duplicating the logic. A follow-up series reuses it to reset protected vCPUs at EL2. No functional change intended. Reviewed-by: Vincent Donnefort Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/sys_regs.c | 14 +------------- arch/arm64/kvm/sys_regs.h | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..08ba882799d4 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -976,21 +976,9 @@ static u64 reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - u64 mpidr; + u64 mpidr = kvm_calculate_mpidr(vcpu); - /* - * Map the vcpu_id into the first three affinity level fields of - * the MPIDR. We limit the number of VCPUs in level 0 due to a - * limitation to 16 CPUs in that level in the ICC_SGIxR registers - * of the GICv3 to be able to address each CPU directly when - * sending IPIs. - */ - mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0); - mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1); - mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2); - mpidr |= (1ULL << 31); vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1); - return mpidr; } diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index 2a983664220c..bd56a45abbf9 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -222,6 +222,25 @@ find_reg(const struct sys_reg_params *params, const struct sys_reg_desc table[], return __inline_bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg); } +static inline u64 kvm_calculate_mpidr(const struct kvm_vcpu *vcpu) +{ + u64 mpidr; + + /* + * Map the vcpu_id into the first three affinity level fields of + * the MPIDR. We limit the number of VCPUs in level 0 due to a + * limitation to 16 CPUs in that level in the ICC_SGIxR registers + * of the GICv3 to be able to address each CPU directly when + * sending IPIs. + */ + mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0); + mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1); + mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2); + mpidr |= (1ULL << 31); + + return mpidr; +} + const struct sys_reg_desc *get_reg_by_id(u64 id, const struct sys_reg_desc table[], unsigned int num); From f88099ad9fc2fc5c7d75db848fcc2bb55a83133a Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:17 +0100 Subject: [PATCH 20/89] KVM: arm64: Make vcpu_{read,write}_sys_reg available to HYP code The vcpu_{read,write}_sys_reg() accessors abstract where a system register lives, but their VHE implementation cannot be linked into the nVHE hypervisor, so exception.c open-codes has_vhe() wrappers instead. Redirect the accessors to the raw context accessors for nVHE hyp builds, where registers are always in memory, and drop the local wrappers. This lets hyp code reuse helpers built on the canonical accessors. No functional change intended. Suggested-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_emulate.h | 6 +++++ arch/arm64/kvm/hyp/exception.c | 34 ++++++++-------------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 5bf3d7e1d92c..fb4062b3ce55 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -506,6 +506,12 @@ static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; } +/* In nVHE hyp code, registers are always in memory: use the raw accessors. */ +#if defined(__KVM_NVHE_HYPERVISOR__) +#define vcpu_read_sys_reg(v, r) __vcpu_sys_reg(v, r) +#define vcpu_write_sys_reg(v, x, r) __vcpu_assign_sys_reg(v, r, x) +#endif + static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) { if (vcpu_mode_is_32bit(vcpu)) { diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index bef40ddb16db..754e2dc1df54 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -20,22 +20,6 @@ #error Hypervisor code only! #endif -static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) -{ - if (has_vhe()) - return vcpu_read_sys_reg(vcpu, reg); - - return __vcpu_sys_reg(vcpu, reg); -} - -static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) -{ - if (has_vhe()) - vcpu_write_sys_reg(vcpu, val, reg); - else - __vcpu_assign_sys_reg(vcpu, reg, val); -} - static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long target_mode, u64 val) { @@ -101,14 +85,14 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, switch (target_mode) { case PSR_MODE_EL1h: - vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1); - sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); - __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1); + vbar = vcpu_read_sys_reg(vcpu, VBAR_EL1); + sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); + vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1); break; case PSR_MODE_EL2h: - vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL2); - sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL2); - __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2); + vbar = vcpu_read_sys_reg(vcpu, VBAR_EL2); + sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL2); + vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2); break; default: /* Don't do that */ @@ -185,7 +169,7 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, */ static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode) { - u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); + u32 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); unsigned long old, new; old = *vcpu_cpsr(vcpu); @@ -281,7 +265,7 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) { unsigned long spsr = *vcpu_cpsr(vcpu); bool is_thumb = (spsr & PSR_AA32_T_BIT); - u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); + u32 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); u32 return_address; *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode); @@ -305,7 +289,7 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) if (sctlr & (1 << 13)) vect_offset += 0xffff0000; else /* always have security exceptions */ - vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1); + vect_offset += vcpu_read_sys_reg(vcpu, VBAR_EL1); *vcpu_pc(vcpu) = vect_offset; } From 29b2e2ab7fa7a9938b09e0756d90805107c965e1 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:18 +0100 Subject: [PATCH 21/89] KVM: arm64: Factor out reusable vCPU reset helpers Pull the reusable pieces out of kvm_reset_vcpu(): expose the reset PSTATE values in kvm_arm.h, and split the core register reset and the PSCI-driven reset into kvm_reset_vcpu_core() and kvm_reset_vcpu_psci(). A follow-up series reuses these to reset protected vCPUs at EL2. No functional change intended. Reviewed-by: Vincent Donnefort Reviewed-by: Steffen Eiden Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_arm.h | 12 ++++++ arch/arm64/include/asm/kvm_emulate.h | 57 ++++++++++++++++++++++++++ arch/arm64/kvm/reset.c | 60 ++-------------------------- 3 files changed, 72 insertions(+), 57 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 3f9233b5a130..aba4ec09acd2 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -348,4 +348,16 @@ { PSR_AA32_MODE_UND, "32-bit UND" }, \ { PSR_AA32_MODE_SYS, "32-bit SYS" } +/* + * ARMv8 Reset Values + */ +#define VCPU_RESET_PSTATE_EL1 (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \ + PSR_F_BIT | PSR_D_BIT) + +#define VCPU_RESET_PSTATE_EL2 (PSR_MODE_EL2h | PSR_A_BIT | PSR_I_BIT | \ + PSR_F_BIT | PSR_D_BIT) + +#define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \ + PSR_AA32_I_BIT | PSR_AA32_F_BIT) + #endif /* __ARM64_KVM_ARM_H__ */ diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index fb4062b3ce55..dbc77d059d7e 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -694,4 +694,61 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR; } } + +/* Reset a vcpu's core registers. */ +static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) +{ + u32 pstate; + + if (vcpu_el1_is_32bit(vcpu)) + pstate = VCPU_RESET_PSTATE_SVC; + else if (vcpu_has_nv(vcpu)) + pstate = VCPU_RESET_PSTATE_EL2; + else + pstate = VCPU_RESET_PSTATE_EL1; + + /* Reset core registers */ + memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); + memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); + vcpu->arch.ctxt.spsr_abt = 0; + vcpu->arch.ctxt.spsr_und = 0; + vcpu->arch.ctxt.spsr_irq = 0; + vcpu->arch.ctxt.spsr_fiq = 0; + vcpu_gp_regs(vcpu)->pstate = pstate; +} + +/* PSCI reset handling for a vcpu. */ +static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu, + struct vcpu_reset_state *reset_state) +{ + unsigned long target_pc = reset_state->pc; + + /* Gracefully handle Thumb2 entry point */ + if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { + target_pc &= ~1UL; + vcpu_set_thumb(vcpu); + } + + /* Propagate caller endianness */ + if (reset_state->be) + kvm_vcpu_set_be(vcpu); + + *vcpu_pc(vcpu) = target_pc; + + /* + * We may come from a state where either a PC update was + * pending (SMC call resulting in PC being increpented to + * skip the SMC) or a pending exception. Make sure we get + * rid of all that, as this cannot be valid out of reset. + * + * Note that clearing the exception mask also clears PC + * updates, but that's an implementation detail, and we + * really want to make it explicit. + */ + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); + vcpu_clear_flag(vcpu, EXCEPT_MASK); + vcpu_clear_flag(vcpu, INCREMENT_PC); + vcpu_set_reg(vcpu, 0, reset_state->r0); +} + #endif /* __ARM64_KVM_EMULATE_H__ */ diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index b963fd975aac..10eb7249aa9e 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -34,18 +34,6 @@ static u32 __ro_after_init kvm_ipa_limit; unsigned int __ro_after_init kvm_host_sve_max_vl; -/* - * ARMv8 Reset Values - */ -#define VCPU_RESET_PSTATE_EL1 (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \ - PSR_F_BIT | PSR_D_BIT) - -#define VCPU_RESET_PSTATE_EL2 (PSR_MODE_EL2h | PSR_A_BIT | PSR_I_BIT | \ - PSR_F_BIT | PSR_D_BIT) - -#define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \ - PSR_AA32_I_BIT | PSR_AA32_F_BIT) - unsigned int __ro_after_init kvm_sve_max_vl; int __init kvm_arm_init_sve(void) @@ -191,7 +179,6 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) { struct vcpu_reset_state reset_state; bool loaded; - u32 pstate; spin_lock(&vcpu->arch.mp_state_lock); reset_state = vcpu->arch.reset_state; @@ -210,21 +197,8 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) kvm_vcpu_reset_sve(vcpu); } - if (vcpu_el1_is_32bit(vcpu)) - pstate = VCPU_RESET_PSTATE_SVC; - else if (vcpu_has_nv(vcpu)) - pstate = VCPU_RESET_PSTATE_EL2; - else - pstate = VCPU_RESET_PSTATE_EL1; - /* Reset core registers */ - memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); - memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); - vcpu->arch.ctxt.spsr_abt = 0; - vcpu->arch.ctxt.spsr_und = 0; - vcpu->arch.ctxt.spsr_irq = 0; - vcpu->arch.ctxt.spsr_fiq = 0; - vcpu_gp_regs(vcpu)->pstate = pstate; + kvm_reset_vcpu_core(vcpu); /* Reset system registers */ kvm_reset_sys_regs(vcpu); @@ -233,36 +207,8 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) * Additional reset state handling that PSCI may have imposed on us. * Must be done after all the sys_reg reset. */ - if (reset_state.reset) { - unsigned long target_pc = reset_state.pc; - - /* Gracefully handle Thumb2 entry point */ - if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { - target_pc &= ~1UL; - vcpu_set_thumb(vcpu); - } - - /* Propagate caller endianness */ - if (reset_state.be) - kvm_vcpu_set_be(vcpu); - - *vcpu_pc(vcpu) = target_pc; - - /* - * We may come from a state where either a PC update was - * pending (SMC call resulting in PC being increpented to - * skip the SMC) or a pending exception. Make sure we get - * rid of all that, as this cannot be valid out of reset. - * - * Note that clearing the exception mask also clears PC - * updates, but that's an implementation detail, and we - * really want to make it explicit. - */ - vcpu_clear_flag(vcpu, PENDING_EXCEPTION); - vcpu_clear_flag(vcpu, EXCEPT_MASK); - vcpu_clear_flag(vcpu, INCREMENT_PC); - vcpu_set_reg(vcpu, 0, reset_state.r0); - } + if (reset_state.reset) + kvm_reset_vcpu_psci(vcpu, &reset_state); /* Reset timer */ kvm_timer_vcpu_reset(vcpu); From 86db2bd7f5f07e4b169345d291bdd5c72730061c Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:19 +0100 Subject: [PATCH 22/89] KVM: arm64: Move PSCI helper functions to a shared header Move kvm_psci_valid_affinity() and kvm_psci_narrow_to_32bit() from psci.c to include/kvm/arm_psci.h, and move psci_affinity_mask() there too, renaming it kvm_psci_affinity_mask() now that it is no longer file-local. A follow-up series handles some protected-guest PSCI calls at EL2 using these helpers. No functional change intended. Reviewed-by: Vincent Donnefort Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/psci.c | 30 +----------------------------- include/kvm/arm_psci.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 3b5dbe9a0a0e..e3db84400d1f 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -21,16 +21,6 @@ * as described in ARM document number ARM DEN 0022A. */ -#define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1) - -static unsigned long psci_affinity_mask(unsigned long affinity_level) -{ - if (affinity_level <= 3) - return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level); - - return 0; -} - static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) { /* @@ -51,12 +41,6 @@ static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) return PSCI_RET_SUCCESS; } -static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, - unsigned long affinity) -{ - return !(affinity & ~MPIDR_HWID_BITMASK); -} - static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) { struct vcpu_reset_state *reset_state; @@ -135,7 +119,7 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) return PSCI_RET_INVALID_PARAMS; /* Determine target affinity mask */ - target_affinity_mask = psci_affinity_mask(lowest_affinity_level); + target_affinity_mask = kvm_psci_affinity_mask(lowest_affinity_level); if (!target_affinity_mask) return PSCI_RET_INVALID_PARAMS; @@ -220,18 +204,6 @@ static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu) run->exit_reason = KVM_EXIT_SYSTEM_EVENT; } -static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) -{ - int i; - - /* - * Zero the input registers' upper 32 bits. They will be fully - * zeroed on exit, so we're fine changing them in place. - */ - for (i = 1; i < 4; i++) - vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); -} - static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn) { /* diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h index cbaec804eb83..f86a006d6713 100644 --- a/include/kvm/arm_psci.h +++ b/include/kvm/arm_psci.h @@ -38,6 +38,33 @@ static inline int kvm_psci_version(struct kvm_vcpu *vcpu) return KVM_ARM_PSCI_0_1; } +/* Narrow the PSCI register arguments (r1 to r3) to 32 bits. */ +static inline void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) +{ + int i; + + /* + * Zero the input registers' upper 32 bits. They will be fully + * zeroed on exit, so we're fine changing them in place. + */ + for (i = 1; i < 4; i++) + vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); +} + +static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, + unsigned long affinity) +{ + return !(affinity & ~MPIDR_HWID_BITMASK); +} + +static inline unsigned long kvm_psci_affinity_mask(unsigned long affinity_level) +{ + if (affinity_level <= 3) + return MPIDR_HWID_BITMASK & + ~((0x1UL << (affinity_level * MPIDR_LEVEL_BITS)) - 1); + + return 0; +} int kvm_psci_call(struct kvm_vcpu *vcpu); From e9f88aa0b16cc95e9c69d5d9fc02fe506c6a0df3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 29 Jul 2026 14:18:20 +0100 Subject: [PATCH 23/89] KVM: arm64: Add host and hypervisor vCPU lookup primitives The nVHE hypervisor repeatedly resolves a host vCPU into the EL2 address space and validates that the loaded hyp vCPU matches it, with that logic open-coded in each handler. Add __get_host_hyp_vcpus() and the get_host_hyp_vcpus() macro, which translate the host vCPU into the hypervisor's address space and, when pKVM is enabled, also return the loaded hyp vCPU if it matches. If pKVM is enabled but the loaded hyp vCPU does not correspond to the requested host vCPU, both the host and hyp vCPU are returned as NULL. Convert handle___kvm_vcpu_run() to use it. No functional change intended. Reviewed-by: Vincent Donnefort Signed-off-by: Marc Zyngier Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-6-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 52 ++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 104026ee70e8..f859c8aca544 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -220,14 +220,45 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) pkvm_put_hyp_vcpu(hyp_vcpu); } -static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) +static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, + struct pkvm_hyp_vcpu **hyp_vcpup) { - DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 1); - int ret; + struct kvm_vcpu *host_vcpu = kern_hyp_va(arg); + struct pkvm_hyp_vcpu *hyp_vcpu = NULL; if (unlikely(is_protected_kvm_enabled())) { - struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu || hyp_vcpu->host_vcpu != host_vcpu) { + hyp_vcpu = NULL; + host_vcpu = NULL; + } + } + + *hyp_vcpup = hyp_vcpu; + return host_vcpu; +} + +#define get_host_hyp_vcpus(ctxt, regnr, hyp_vcpup) \ + ({ \ + DECLARE_REG(struct kvm_vcpu *, __vcpu, ctxt, regnr); \ + __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ + }) + +static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; + int ret; + + host_vcpu = get_host_hyp_vcpus(host_ctxt, 1, &hyp_vcpu); + + if (!host_vcpu) { + ret = -EINVAL; + goto out; + } + + if (unlikely(hyp_vcpu)) { /* * KVM (and pKVM) doesn't support SME guests for now, and * ensures that SME features aren't enabled in pstate when @@ -239,23 +270,16 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) goto out; } - if (!hyp_vcpu) { - ret = -EINVAL; - goto out; - } - flush_hyp_vcpu(hyp_vcpu); ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); sync_hyp_vcpu(hyp_vcpu); } else { - struct kvm_vcpu *vcpu = kern_hyp_va(host_vcpu); - /* The host is fully trusted, run its vCPU directly. */ - fpsimd_lazy_switch_to_guest(vcpu); - ret = __kvm_vcpu_run(vcpu); - fpsimd_lazy_switch_to_host(vcpu); + fpsimd_lazy_switch_to_guest(host_vcpu); + ret = __kvm_vcpu_run(host_vcpu); + fpsimd_lazy_switch_to_host(host_vcpu); } out: cpu_reg(host_ctxt, 1) = ret; From 79c3122a1e142166b9254809115ef26ba7797769 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 29 Jul 2026 14:18:21 +0100 Subject: [PATCH 24/89] KVM: arm64: Minimise EL2's exposure of host VGIC state during world switch The host passes a vgic_v3_cpu_if pointer to the __vgic_v3_save_aprs and __vgic_v3_restore_vmcr_aprs hypercalls, which EL2 dereferences wholesale. That exposes the host's full VGIC emulation state to the hypervisor, against pKVM's isolation goals. Recover the host vCPU from the supplied cpu_if via container_of() and copy only vgic_vmcr and the active priority registers between EL2's hyp-side state and the host vCPU, so EL2 no longer dereferences the host's vgic_v3_cpu_if directly. Reviewed-by: Vincent Donnefort Signed-off-by: Marc Zyngier Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-7-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 67 ++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index f859c8aca544..415bbd7f9452 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -7,6 +7,8 @@ #include #include +#include + #include #include #include @@ -245,6 +247,16 @@ static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ }) +#define get_host_hyp_vcpus_from_vgic_v3_cpu_if(ctxt, regnr, hyp_vcpup) \ + ({ \ + DECLARE_REG(struct vgic_v3_cpu_if *, cif, ctxt, regnr);\ + struct kvm_vcpu *__vcpu = container_of(cif, \ + struct kvm_vcpu, \ + arch.vgic_cpu.vgic_v3); \ + \ + __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ + }) + static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) { struct pkvm_hyp_vcpu *hyp_vcpu; @@ -514,16 +526,63 @@ static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt) static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) { - DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1); + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; - __vgic_v3_save_aprs(kern_hyp_va(cpu_if)); + host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, + &hyp_vcpu); + if (!host_vcpu) + return; + + if (unlikely(hyp_vcpu)) { + struct vgic_v3_cpu_if *hyp_cpu_if, *host_cpu_if; + int i; + + hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; + __vgic_v3_save_aprs(hyp_cpu_if); + + host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; + host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; + for (i = 0; i < ARRAY_SIZE(host_cpu_if->vgic_ap0r); i++) { + host_cpu_if->vgic_ap0r[i] = hyp_cpu_if->vgic_ap0r[i]; + host_cpu_if->vgic_ap1r[i] = hyp_cpu_if->vgic_ap1r[i]; + } + } else { + __vgic_v3_save_aprs(&host_vcpu->arch.vgic_cpu.vgic_v3); + } } static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt) { - DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1); + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; - __vgic_v3_restore_vmcr_aprs(kern_hyp_va(cpu_if)); + host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, + &hyp_vcpu); + if (!host_vcpu) + return; + + if (unlikely(hyp_vcpu)) { + struct vgic_v3_cpu_if *hyp_cpu_if, *host_cpu_if; + int i; + + hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; + host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; + + hyp_cpu_if->vgic_vmcr = host_cpu_if->vgic_vmcr; + /* Should be a one-off */ + hyp_cpu_if->vgic_sre = (ICC_SRE_EL1_DIB | + ICC_SRE_EL1_DFB | + ICC_SRE_EL1_SRE); + for (i = 0; i < ARRAY_SIZE(host_cpu_if->vgic_ap0r); i++) { + hyp_cpu_if->vgic_ap0r[i] = host_cpu_if->vgic_ap0r[i]; + hyp_cpu_if->vgic_ap1r[i] = host_cpu_if->vgic_ap1r[i]; + } + + __vgic_v3_restore_vmcr_aprs(hyp_cpu_if); + } else { + __vgic_v3_restore_vmcr_aprs(&host_vcpu->arch.vgic_cpu.vgic_v3); + } } static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt) From 42fe26543e66c27932b4c7f4c5c149e38f4e14b9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 29 Jul 2026 14:18:22 +0100 Subject: [PATCH 25/89] KVM: arm64: Add primitives to flush/sync the VGIC state at EL2 pKVM performs its own world switch for protected VMs but has no primitives to move the per-vCPU VGIC state between the host and hypervisor vCPU contexts. Add flush_hyp_vgic_state() and sync_hyp_vgic_state(). Flush copies vgic_hcr, the in-use list registers and used_lrs from the host into the hyp vCPU and pins vgic_sre to a fixed value; sync copies vgic_hcr, vgic_vmcr and the in-use list registers back. The active priority registers are handled separately by the save/restore-aprs path. Bound used_lrs by hyp_gicv3_nr_lr, the cached implemented-LR count, instead of reading ICH_VTR_EL2 on each entry. That clamps the host-supplied value and avoids a per-entry sysreg read that is costly under NV. Reviewed-by: Vincent Donnefort Signed-off-by: Marc Zyngier Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-8-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 55 ++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 415bbd7f9452..ac36257c8da2 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -104,6 +104,45 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; } +static void flush_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct vgic_v3_cpu_if *host_cpu_if, *hyp_cpu_if; + unsigned int used_lrs, i; + + host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; + hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; + + used_lrs = host_cpu_if->used_lrs; + used_lrs = min(used_lrs, hyp_gicv3_nr_lr); + + hyp_cpu_if->vgic_hcr = host_cpu_if->vgic_hcr; + /* Should be a one-off */ + hyp_cpu_if->vgic_sre = (ICC_SRE_EL1_DIB | + ICC_SRE_EL1_DFB | + ICC_SRE_EL1_SRE); + hyp_cpu_if->used_lrs = used_lrs; + + for (i = 0; i < used_lrs; i++) + hyp_cpu_if->vgic_lr[i] = host_cpu_if->vgic_lr[i]; +} + +static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct vgic_v3_cpu_if *host_cpu_if, *hyp_cpu_if; + unsigned int i; + + host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; + hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; + + host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr; + host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; + + for (i = 0; i < hyp_cpu_if->used_lrs; i++) + host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; +} + static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; @@ -158,13 +197,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; - hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3; - - /* Bound used_lrs by the number of implemented list registers. */ - hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs = - min_t(unsigned int, - hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs, - hyp_gicv3_nr_lr); + flush_hyp_vgic_state(hyp_vcpu); hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; } @@ -172,9 +205,6 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; - struct vgic_v3_cpu_if *hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; - struct vgic_v3_cpu_if *host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; - unsigned int i; fpsimd_sve_sync(&hyp_vcpu->vcpu); sync_debug_state(hyp_vcpu); @@ -187,10 +217,7 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; - host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr; - host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; - for (i = 0; i < hyp_cpu_if->used_lrs; ++i) - host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; + sync_hyp_vgic_state(hyp_vcpu); } static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) From 734dc8c01c838779ecda7d0d113839c2b68910be Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:23 +0100 Subject: [PATCH 26/89] KVM: arm64: Implement lazy vCPU state sync for non-protected guests pKVM copies a non-protected guest's register context between the host and the hypervisor on every world switch, even when the host never inspects it. Defer the copy: on entry, flush the host context into the hyp vCPU only when the host marked it dirty (PKVM_HOST_STATE_DIRTY); on exit, leave it in the hyp vCPU and copy it back only when the host needs it, via a __pkvm_vcpu_sync_state hypercall or at vcpu put. A protected guest's context is copied as before, since lazy sync only helps where the host is trusted to see the guest's registers. PC and PSTATE are the exception, copied back on every exit: the kvm_exit tracepoint reports the guest's real exit PC, and vcpu_mode_is_bad_32bit(), the SError-masking checks and the host software-step state machine all read PSTATE before the next on-demand sync. Without it an ARM_EXCEPTION_IRQ exit syncs nothing, so flush_debug_state() pushes a stale PSTATE.SS back on the next entry and single-stepping runs an extra instruction before the step is reported. The host needs the full context when it is about to read it (trap handling) or write it (the SError injection that writes ESR_EL1). Sync both from handle_exit_early(), which runs non-preemptible so the loaded hyp vCPU is stable without a preempt guard. Reviewed-by: Vincent Donnefort Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-9-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_host.h | 2 + arch/arm64/kvm/arm.c | 7 +++ arch/arm64/kvm/handle_exit.c | 23 ++++++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 89 ++++++++++++++++++++++++++++-- 5 files changed, 117 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 043495f7fc78..6e1135b3ded4 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -113,6 +113,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_finalize_teardown_vm, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, + __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state, __KVM_HOST_SMCCC_FUNC___pkvm_tlb_flush_vmid, MARKER(__KVM_HOST_SMCCC_FUNC_MAX) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..86d45bc8f495 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1051,6 +1051,8 @@ struct kvm_vcpu_arch { #define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1)) /* Target EL/MODE (not a single flag, but let's abuse the macro) */ #define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1)) +/* Host-set: the hyp flushes the non-protected vCPU state in on entry */ +#define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(4)) /* Helpers to encode exceptions with minimum fuss */ #define __EXCEPT_MASK_VAL unpack_vcpu_flag(EXCEPT_MASK) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 50adfff75be8..ace7b67fe1cc 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -735,6 +735,10 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) if (is_protected_kvm_enabled()) { kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); kvm_call_hyp_nvhe(__pkvm_vcpu_put); + + /* __pkvm_vcpu_put implies a sync of the state */ + if (!kvm_vm_is_protected(vcpu->kvm)) + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); } kvm_vcpu_put_debug(vcpu); @@ -966,6 +970,9 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; if (is_protected_kvm_enabled()) { + /* Start with the vcpu in a dirty state */ + if (!kvm_vm_is_protected(vcpu->kvm)) + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); ret = pkvm_create_hyp_vm(kvm); if (ret) return ret; diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 54aedf93c78b..29108e5c0206 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -486,9 +486,32 @@ int handle_exit(struct kvm_vcpu *vcpu, int exception_index) } } +static void handle_exit_pkvm_state(struct kvm_vcpu *vcpu, int exception_index) +{ + int exception_code = ARM_EXCEPTION_CODE(exception_index); + + if (!is_protected_kvm_enabled() || kvm_vm_is_protected(vcpu->kvm)) + return; + + /* + * Sync the context back when the host will read (trap) or write + * (SError) it. Preempt-off here, so the loaded hyp vCPU is stable. + */ + if (exception_code == ARM_EXCEPTION_TRAP || + exception_code == ARM_EXCEPTION_EL1_SERROR || + ARM_SERROR_PENDING(exception_index)) { + kvm_call_hyp_nvhe(__pkvm_vcpu_sync_state); + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); + } else { + vcpu_clear_flag(vcpu, PKVM_HOST_STATE_DIRTY); + } +} + /* For exit types that need handling before we can be preempted */ void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index) { + handle_exit_pkvm_state(vcpu, exception_index); + if (ARM_SERROR_PENDING(exception_index)) { if (this_cpu_has_cap(ARM64_HAS_RAS_EXTN)) { u64 disr = kvm_vcpu_get_disr(vcpu); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index ac36257c8da2..65a7c735aa39 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -143,6 +143,48 @@ static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; } +static void __copy_vcpu_state(const struct kvm_vcpu *from_vcpu, + struct kvm_vcpu *to_vcpu) +{ + int i; + + to_vcpu->arch.ctxt.regs = from_vcpu->arch.ctxt.regs; + to_vcpu->arch.ctxt.spsr_abt = from_vcpu->arch.ctxt.spsr_abt; + to_vcpu->arch.ctxt.spsr_und = from_vcpu->arch.ctxt.spsr_und; + to_vcpu->arch.ctxt.spsr_irq = from_vcpu->arch.ctxt.spsr_irq; + to_vcpu->arch.ctxt.spsr_fiq = from_vcpu->arch.ctxt.spsr_fiq; + to_vcpu->arch.ctxt.fp_regs = from_vcpu->arch.ctxt.fp_regs; + + /* + * Copy the sysregs, but don't mess with the timer state which + * is directly handled by EL1 and is expected to be preserved. + * enum vcpu_sysreg is sparse: VNCR-mapped registers take values + * derived from their VNCR page offset, so the timer registers do + * not form a contiguous numeric range and must be skipped by name. + */ + for (i = 1; i < NR_SYS_REGS; i++) { + switch (i) { + case CNTVOFF_EL2: + case CNTV_CVAL_EL0: + case CNTV_CTL_EL0: + case CNTP_CVAL_EL0: + case CNTP_CTL_EL0: + continue; + } + to_vcpu->arch.ctxt.sys_regs[i] = from_vcpu->arch.ctxt.sys_regs[i]; + } +} + +static void sync_hyp_vcpu_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + __copy_vcpu_state(&hyp_vcpu->vcpu, hyp_vcpu->host_vcpu); +} + +static void flush_hyp_vcpu_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + __copy_vcpu_state(hyp_vcpu->host_vcpu, &hyp_vcpu->vcpu); +} + static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; @@ -178,7 +220,17 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) fpsimd_sve_flush(); flush_debug_state(hyp_vcpu); - hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; + /* + * If we deal with a non-protected guest and the state is potentially + * dirty (from a host perspective), copy the state back into the hyp + * vcpu. + */ + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) + flush_hyp_vcpu_state(hyp_vcpu); + } else { + hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; + } /* __hyp_running_vcpu must be NULL in a guest context. */ hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; @@ -209,9 +261,16 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) fpsimd_sve_sync(&hyp_vcpu->vcpu); sync_debug_state(hyp_vcpu); - host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; - - host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; + } else { + /* + * PC feeds trace_kvm_exit(), PSTATE.SS the host software-step + * machine, and both run before the next on-demand ctxt sync. + */ + host_vcpu->arch.ctxt.regs.pc = hyp_vcpu->vcpu.arch.ctxt.regs.pc; + host_vcpu->arch.ctxt.regs.pstate = hyp_vcpu->vcpu.arch.ctxt.regs.pstate; + } host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; @@ -245,8 +304,27 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) { struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - if (hyp_vcpu) + if (hyp_vcpu) { + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu) && + !vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) { + sync_hyp_vcpu_state(hyp_vcpu); + } + pkvm_put_hyp_vcpu(hyp_vcpu); + } +} + +static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + + sync_hyp_vcpu_state(hyp_vcpu); } static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, @@ -877,6 +955,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_finalize_teardown_vm), HANDLE_FUNC(__pkvm_vcpu_load), HANDLE_FUNC(__pkvm_vcpu_put), + HANDLE_FUNC(__pkvm_vcpu_sync_state), HANDLE_FUNC(__pkvm_tlb_flush_vmid), }; From fff4eb33ffa082f1a106a5d696d035eac8d7842c Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Fri, 17 Jul 2026 14:03:11 +0100 Subject: [PATCH 27/89] KVM: arm64: Skip cache maintenance for non-cacheable pKVM mappings The pKVM flush path walks its own pkvm_mappings list and cleans the data cache for every mapping, unlike the generic stage-2 walker it shadows, which skips non-cacheable leaves. Cleaning the cacheable alias of a non-cacheable mapping is pointless and can corrupt a device endpoint. Record whether a mapping is non-cacheable in spare bits of nr_pages and skip cache maintenance for it. Fixes: e912efed485a ("KVM: arm64: Introduce the EL1 pKVM MMU") Suggested-by: Marc Zyngier Signed-off-by: Bradley Morgan [tabba: use Marc's anonymous bitfield in place of the open-coded mask and helpers] Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_pkvm.h | 5 ++++- arch/arm64/kvm/pkvm.c | 15 +++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 74fedd9c5ff0..57afb07d6b13 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -195,7 +195,10 @@ struct pkvm_mapping { struct rb_node node; u64 gfn; u64 pfn; - u64 nr_pages; + struct { + u64 nr_pages:48; + u64 nc:1; + }; u64 __subtree_last; /* Internal member for interval tree */ }; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 008766273912..a66105db8b11 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -366,7 +366,7 @@ static int __pkvm_pgtable_stage2_unshare(struct kvm_pgtable *pgt, u64 start, u64 for_each_mapping_in_range_safe(pgt, start, end, mapping) { ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_guest, handle, mapping->gfn, - mapping->nr_pages); + (u64)mapping->nr_pages); if (WARN_ON(ret)) return ret; pkvm_mapping_remove(mapping, &pgt->pkvm_mappings); @@ -470,6 +470,7 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, mapping->gfn = gfn; mapping->pfn = pfn; mapping->nr_pages = size / PAGE_SIZE; + mapping->nc = !!(prot & (KVM_PGTABLE_PROT_DEVICE | KVM_PGTABLE_PROT_NORMAL_NC)); pkvm_mapping_insert(mapping, &pgt->pkvm_mappings); return ret; @@ -500,7 +501,7 @@ int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size) lockdep_assert_held(&kvm->mmu_lock); for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) { ret = kvm_call_hyp_nvhe(__pkvm_host_wrprotect_guest, handle, mapping->gfn, - mapping->nr_pages); + (u64)mapping->nr_pages); if (WARN_ON(ret)) break; } @@ -514,9 +515,11 @@ int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) struct pkvm_mapping *mapping; lockdep_assert_held(&kvm->mmu_lock); - for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) - __clean_dcache_guest_page(pfn_to_kaddr(mapping->pfn), - PAGE_SIZE * mapping->nr_pages); + for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) { + if (!mapping->nc) + __clean_dcache_guest_page(pfn_to_kaddr(mapping->pfn), + PAGE_SIZE * mapping->nr_pages); + } return 0; } @@ -534,7 +537,7 @@ bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 lockdep_assert_held(&kvm->mmu_lock); for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) young |= kvm_call_hyp_nvhe(__pkvm_host_test_clear_young_guest, handle, mapping->gfn, - mapping->nr_pages, mkold); + (u64)mapping->nr_pages, mkold); return young; } From 6bf0215c565ee122ffff7241b297a7b88f71005f Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:12 +0100 Subject: [PATCH 28/89] KVM: arm64: Top up the memcache for pKVM permission faults A permission fault normally only relaxes a leaf, so user_mem_abort() skips the memcache top-up. Under pKVM such a fault can instead collapse pages into a block. That needs a fresh pkvm_mapping object, and without it cache->mapping is NULL, so pkvm_pgtable_stage2_map() dereferences NULL and faults the host under mmu_lock. Staging only the object is not enough: the hypervisor requires kvm_mmu_cache_min_pages in the memcache even for the allocation-free install, so under memcache pressure the collapse returns -ENOMEM and trips the WARN_ON(ret) in pkvm_pgtable_stage2_map() where a non-pKVM guest succeeds. Top up the full memcache for pKVM permission faults so both the mapping object and the min-pages are staged before mmu_lock. Fixes: db14091d8f75 ("KVM: arm64: Stage-2 huge mappings for np-guests") Reported-by: Bradley Morgan Link: https://lore.kernel.org/all/20260623161545.EA08E1F000E9@smtp.kernel.org/ Signed-off-by: Fuad Tabba Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 6c941aaa10c6..4d7c9bdcef00 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2114,10 +2114,14 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) * and so normally don't require allocations from the memcache. The * only exception to this is when dirty logging is enabled at runtime * and a write fault needs to collapse a block entry into a table. + * Under pKVM a permission fault can also collapse pages into a block, + * which needs a fresh mapping object, and the hypervisor requires the + * min-pages memcache even when the install allocates nothing. */ memcache = get_mmu_memcache(s2fd->vcpu); if (!perm_fault || (memslot_is_logging(s2fd->memslot) && - kvm_is_write_fault(s2fd->vcpu))) { + kvm_is_write_fault(s2fd->vcpu)) || + is_protected_kvm_enabled()) { ret = topup_mmu_memcache(s2fd->vcpu, memcache); if (ret) return ret; From 56fd1fc526e6413781a6980aa13ed9db104cbfea Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Fri, 17 Jul 2026 14:03:13 +0100 Subject: [PATCH 29/89] KVM: arm64: Top up stage-2 memcache for dirty logging faults Dirty logging forces new stage-2 mappings to page size but does not always split an existing block first (eager splitting is best effort and off by default). A non-write permission fault on such a block, an instruction fetch, still needs a page-table page to split it, but the top-up is gated on write faults. With the cache empty, kvm_mmu_memory_cache_alloc() hits its guest-triggerable WARN_ON(!nobjs) and falls back to a GFP_ATOMIC allocation under mmu_lock, with a BUG_ON() if that fails. Top up the memcache for any permission fault while dirty logging is active. Fixes: 6f745f1bb5bf ("KVM: arm64: Convert user_mem_abort() to generic page-table API") Link: https://lore.kernel.org/all/20260623165634.699011F000E9@smtp.kernel.org/ Signed-off-by: Bradley Morgan [tabba: reword the commit message for the generic, non-pKVM failure mode] Signed-off-by: Fuad Tabba Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 4d7c9bdcef00..74e7e7f7564c 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2113,14 +2113,13 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) * Permission faults just need to update the existing leaf entry, * and so normally don't require allocations from the memcache. The * only exception to this is when dirty logging is enabled at runtime - * and a write fault needs to collapse a block entry into a table. + * and a fault needs to collapse a block entry into a table. * Under pKVM a permission fault can also collapse pages into a block, * which needs a fresh mapping object, and the hypervisor requires the * min-pages memcache even when the install allocates nothing. */ memcache = get_mmu_memcache(s2fd->vcpu); - if (!perm_fault || (memslot_is_logging(s2fd->memslot) && - kvm_is_write_fault(s2fd->vcpu)) || + if (!perm_fault || memslot_is_logging(s2fd->memslot) || is_protected_kvm_enabled()) { ret = topup_mmu_memcache(s2fd->vcpu, memcache); if (ret) From 1feb9f6af895c1ec0caed269108d6ffa1f9a31c4 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:14 +0100 Subject: [PATCH 30/89] KVM: arm64: Skip pKVM stage-2 flush when FWB is enabled pkvm_pgtable_stage2_flush() cleans the D-cache for every mapping in the range even on hardware with stage-2 Force Write-Back, where FWB keeps guest memory coherent to the PoC and the maintenance is unnecessary. The generic kvm_pgtable_stage2_flush() returns early in that case, but the pKVM MMU does not, so it needlessly cleans the whole range on, e.g., every set/way trap. Return early when FWB is enabled, matching the generic walker. Fixes: e912efed485a ("KVM: arm64: Introduce the EL1 pKVM MMU") Signed-off-by: Fuad Tabba Reviewed-by: Bradley Morgan Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/pkvm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index a66105db8b11..b28d700c649e 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -515,6 +515,10 @@ int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) struct pkvm_mapping *mapping; lockdep_assert_held(&kvm->mmu_lock); + + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) + return 0; + for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) { if (!mapping->nc) __clean_dcache_guest_page(pfn_to_kaddr(mapping->pfn), From c243222701690da30c010d6b98545684f5d78b15 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:15 +0100 Subject: [PATCH 31/89] KVM: arm64: Don't WARN on pKVM stage-2 map failures pkvm_pgtable_stage2_map() wraps the __pkvm_host_share_guest() and __pkvm_host_donate_guest() return in WARN_ON(), but those hypercalls fail for reasons that are not EL1 invariant violations: -EINVAL for a pfn that is not memblock RAM (check_range_allowed_memory() rejects a device page mapped into a non-protected guest) and -ENOMEM under memcache pressure. Both are reachable from a guest fault, so the WARN splats on host input. Return the error without warning. The unshare and write-protect WARNs stay, since a failure there does signal a broken EL1 invariant. Fixes: 3669ddd8fa8b5 ("KVM: arm64: Add a range to pkvm_mappings") Signed-off-by: Fuad Tabba Reviewed-by: Bradley Morgan Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-6-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/pkvm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index b28d700c649e..ac1a5e2dc09f 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -463,7 +463,7 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, size / PAGE_SIZE, prot); } - if (WARN_ON(ret)) + if (ret) return ret; swap(mapping, cache->mapping); From c6f308b4d341a239c1ef7423b82d62f50fd87745 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:16 +0100 Subject: [PATCH 32/89] KVM: arm64: Don't advertise eager page splitting under pKVM Under pKVM the stage-2 walker resolves to pkvm_pgtable_stage2_split(), a WARN_ON_ONCE(1) stub, yet KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE is still enabled and reported for non-protected guests: the capability check keys on the per-VM protected state while the walker dispatch keys on the host-global mode. Enabling the cap and then dirty-logging the guest reaches the stub, splatting a userspace-reachable WARN. Reject the capability, and stop reporting a chunk size and the supported block sizes, for every VM once pKVM is enabled, keyed on the host-global mode like the split dispatch. Gating only protected VMs would leave the non-protected guests that reach the stub still able to enable it. Userspace decides whether eager splitting is available from the block-size bitmap (QEMU falls back to no eager splitting when it reads 0), so leaving it advertised steers an explicit request into the enable failure instead of the fallback. Fixes: e912efed485a ("KVM: arm64: Introduce the EL1 pKVM MMU") Signed-off-by: Fuad Tabba Reviewed-by: Bradley Morgan Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-7-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_pkvm.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 57afb07d6b13..beea00e693a0 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -45,6 +45,9 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) return true; case KVM_CAP_ARM_MTE: return false; + case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: + case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: + return false; default: return !kvm || !kvm_vm_is_protected(kvm); } From 979ae12a2d090a2a858c72779d222baad78697a6 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:17 +0100 Subject: [PATCH 33/89] KVM: arm64: selftests: Add stage-2 block transition test Add a test for the two stage-2 granularity changes dirty logging forces at fault time, asserting the guest completes with no KVM_RUN error. The first scenario collapses a page into a hugetlb-backed block: it writes under logging, re-write-protects the page via GET_DIRTY_LOG, then writes again with logging off. The second splits blocks: it faults in several non-executable 2M blocks, enables logging, then executes in each block so an execute permission fault splits it. It is skipped when CTR_EL0.DIC is set, since mappings are then executable and no execute fault occurs. Both paths make the fault handler allocate under mmu_lock, so a backend that fails to stage that memory returns a KVM_RUN error or crashes the host. The property holds on any host. On a pKVM host, where a non-protected guest uses the pkvm_pgtable_*() backend, it also guards that backend's fault-time staging. Assisted-by: Antigravity:gemini-3.1-pro Signed-off-by: Fuad Tabba Tested-by: Bradley Morgan # On QEMU arm64 host Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-8-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../kvm/arm64/stage2_block_transitions.c | 226 ++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/stage2_block_transitions.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index d28a057fa6c2..ed2877825eee 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -179,6 +179,7 @@ TEST_GEN_PROGS_arm64 += arm64/psci_test TEST_GEN_PROGS_arm64 += arm64/sea_to_user TEST_GEN_PROGS_arm64 += arm64/set_id_regs TEST_GEN_PROGS_arm64 += arm64/smccc_filter +TEST_GEN_PROGS_arm64 += arm64/stage2_block_transitions TEST_GEN_PROGS_arm64 += arm64/vcpu_width_config TEST_GEN_PROGS_arm64 += arm64/vgic_init TEST_GEN_PROGS_arm64 += arm64/vgic_irq diff --git a/tools/testing/selftests/kvm/arm64/stage2_block_transitions.c b/tools/testing/selftests/kvm/arm64/stage2_block_transitions.c new file mode 100644 index 000000000000..5fd47f4ada1f --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/stage2_block_transitions.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2026 Google LLC + * Author: Fuad Tabba + * + * stage2_block_transitions - Exercise stage-2 block/page granularity changes + * that dirty logging forces at fault time, and assert the guest completes. + * + * Both scenarios need the fault handler to allocate at fault time (a fresh + * mapping and/or page-table pages while holding mmu_lock), so a fault path + * that fails to stage that memory manifests as a KVM_RUN error or, worse, a + * host crash. The asserted property is host-agnostic: the guest runs the + * sequence to completion and every KVM_RUN succeeds. On a pKVM host, where a + * non-protected guest's stage-2 faults are serviced by the pkvm_pgtable_*() + * backend, the same sequences also guard that backend's fault-time staging. + * + * Scenario 1 - block collapse on dirty-logging disable: + * A write under dirty logging installs a 4K page; GET_DIRTY_LOG + * re-write-protects it; logging is disabled; a second write takes a + * permission fault that collapses the page into a hugetlb-backed block, + * which requires a fresh mapping object under mmu_lock. + * + * Scenario 2 - block split under dirty logging: + * Several hugetlb-backed blocks are faulted in as non-executable blocks, + * dirty logging is enabled (write-protect only), then the guest executes + * into each block. Each instruction fetch takes an execute permission + * fault that must split the block into pages during logging, draining + * page-table pages. Skipped on CTR_EL0.DIC hardware, where mappings are + * made executable eagerly and the execute fault never occurs. + */ +#include +#include +#include +#include +#include + +#include + +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" +#include "ucall.h" + +#define DATA_SLOT 1 +#define TEST_GVA 0xc0000000UL +#define BLOCK_SIZE SZ_2M + +/* AArch64 "ret" (ret x30): a self-contained, returnable executable payload. */ +#define RET_INSN 0xd65f03c0U + +/* + * A non-protected guest's per-VM stage-2 pool is seeded only with the PGD + * donation, which stage-2 init immediately consumes, so the page-table budget + * for a fault that does not top up is just the handful (~2x the stage-2 min + * pages) of memcache leftovers. Executing into this many distinct blocks + * demands far more than that budget: a fault path that tops up on every fault + * completes all of them, one that skips non-write faults runs out mid-sequence. + */ +#define NR_BLOCKS 16 + +/* Scenario 2 guest -> host sync stages. */ +#define STAGE_SKIP_DIC 1 +#define STAGE_BLOCKS_READY 2 + +static void collapse_guest_code(u64 gva) +{ + u64 *data = (u64 *)gva; + + /* Under dirty logging: install a 4K writable page. */ + WRITE_ONCE(*data, 0x1); + GUEST_SYNC(1); + + /* Logging disabled: a permission fault collapses the page into a block. */ + WRITE_ONCE(*data, 0x2); + GUEST_SYNC(2); + + GUEST_DONE(); +} + +static void test_block_collapse(void) +{ + struct kvm_vcpu *vcpu; + unsigned long *bmap; + struct kvm_vm *vm; + struct ucall uc; + size_t npages; + u64 gpa; + + vm = vm_create_with_one_vcpu(&vcpu, collapse_guest_code); + npages = BLOCK_SIZE / vm->page_size; + + gpa = (vm_compute_max_gfn(vm) * vm->page_size) - BLOCK_SIZE; + gpa = align_down(gpa, BLOCK_SIZE); + + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_HUGETLB_2MB, gpa, + DATA_SLOT, npages, KVM_MEM_LOG_DIRTY_PAGES); + virt_map(vm, TEST_GVA, gpa, npages); + vcpu_args_set(vcpu, 1, TEST_GVA); + + bmap = bitmap_zalloc(BLOCK_SIZE / getpagesize()); + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && uc.args[1] == 1, + "Expected first sync, got cmd %lu arg %lu", uc.cmd, uc.args[1]); + + /* GET_DIRTY_LOG re-write-protects the dirtied page; then stop logging. */ + kvm_vm_get_dirty_log(vm, DATA_SLOT, bmap); + vm_mem_region_set_flags(vm, DATA_SLOT, 0); + + /* The collapsing permission fault: a broken fault path faults here. */ + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && uc.args[1] == 2, + "Expected second sync, got cmd %lu arg %lu", uc.cmd, uc.args[1]); + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_DONE, + "Expected done, got cmd %lu", uc.cmd); + + free(bmap); + kvm_vm_free(vm); +} + +static void guest_sync_insn(u64 va) +{ + /* Make the just-written instruction coherent for execution (!DIC). */ + asm volatile("dc cvau, %0\n" + "dsb ish\n" + "ic ivau, %0\n" + "dsb ish\n" + "isb\n" + :: "r" (va) : "memory"); +} + +static void split_guest_code(u64 base_gva, u64 nblocks) +{ + u64 i, va; + + if (FIELD_GET(CTR_EL0_DIC_MASK, read_sysreg(ctr_el0))) { + GUEST_SYNC(STAGE_SKIP_DIC); + GUEST_DONE(); + return; + } + + /* Fault in each block (non-executable) and stage an executable payload. */ + for (i = 0; i < nblocks; i++) { + va = base_gva + i * BLOCK_SIZE; + WRITE_ONCE(*(u32 *)va, RET_INSN); + guest_sync_insn(va); + } + GUEST_SYNC(STAGE_BLOCKS_READY); + + /* Logging is now on: executing into each block splits it into pages. */ + for (i = 0; i < nblocks; i++) { + va = base_gva + i * BLOCK_SIZE; + ((void (*)(void))va)(); + } + + GUEST_DONE(); +} + +static void test_exec_split_drain(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + size_t npages; + u64 gpa; + + vm = vm_create_with_one_vcpu(&vcpu, split_guest_code); + npages = NR_BLOCKS * (BLOCK_SIZE / vm->page_size); + + gpa = (vm_compute_max_gfn(vm) * vm->page_size) - NR_BLOCKS * BLOCK_SIZE; + gpa = align_down(gpa, BLOCK_SIZE); + + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_HUGETLB_2MB, gpa, + DATA_SLOT, npages, 0); + virt_map(vm, TEST_GVA, gpa, npages); + vcpu_args_set(vcpu, 2, TEST_GVA, (u64)NR_BLOCKS); + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC, + "Expected sync, got cmd %lu", uc.cmd); + if (uc.args[1] == STAGE_SKIP_DIC) { + ksft_print_msg("SKIP block split: CTR_EL0.DIC == 1\n"); + kvm_vm_free(vm); + return; + } + TEST_ASSERT(uc.args[1] == STAGE_BLOCKS_READY, + "Expected blocks-ready sync, got arg %lu", uc.args[1]); + + /* Write-protect the blocks; the guest then splits them by executing. */ + vm_mem_region_set_flags(vm, DATA_SLOT, KVM_MEM_LOG_DIRTY_PAGES); + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_DONE, + "Expected done, got cmd %lu", uc.cmd); + + kvm_vm_free(vm); +} + +/* + * The explicit-size hugetlb backing hard-fails region creation if the pages + * are not already reserved, so probe here and skip rather than abort. The + * peak reservation is scenario 2's; the two scenarios run and free in turn. + */ +static void require_hugepages(size_t bytes) +{ + void *mem = mmap(NULL, bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, + -1, 0); + + if (mem == MAP_FAILED) + ksft_exit_skip("Need %zu bytes of reserved 2M hugepages\n", bytes); + munmap(mem, bytes); +} + +int main(void) +{ + require_hugepages(NR_BLOCKS * BLOCK_SIZE); + + test_block_collapse(); + test_exec_split_drain(); + + ksft_print_msg("All ok!\n"); + return 0; +} From 6ae7fab7d9d33b5b552d1eb901d2594d0999ba3d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:09:54 +0100 Subject: [PATCH 34/89] arm64: sysreg: Emit RESx/UNKN values for Mapping/Fields definitions The sysreg file is using the Mapping or Fields qualifiers to indicate that a given encoding is only a mapping to a particular register (or an instance of a more generic register definition). As a result, we don't output any definition, and instead expect the canonical definitions to be used. This works rather well for individual fields, but creates problems for macros that refer to more generic classes of bits such as RESx. Relax the above rule by emitting the RESx and UNKN values for Mapping and Fields qualifiers as well. Signed-off-by: Marc Zyngier Acked-by: Mark Rutland Link: https://patch.msgid.link/20260730071022.296811-2-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/tools/gen-sysreg.awk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/tools/gen-sysreg.awk b/arch/arm64/tools/gen-sysreg.awk index 86860ab672dc..53844f5c5e52 100755 --- a/arch/arm64/tools/gen-sysreg.awk +++ b/arch/arm64/tools/gen-sysreg.awk @@ -228,7 +228,7 @@ $1 == "EndSysreg" && block_current() == "Sysreg" { } # Currently this is effectivey a comment, in future we may want to emit -# defines for the fields. +# defines for the fields. We do emit RESx and UNKN values in any case. ($1 == "Fields" || $1 == "Mapping") && block_current() == "Sysreg" { expect_fields(2) @@ -239,9 +239,9 @@ $1 == "EndSysreg" && block_current() == "Sysreg" { print "" next_bit = -1 - res0 = null - res1 = null - unkn = null + res0 = $2 "_RES0" + res1 = $2 "_RES1" + unkn = $2 "_UNKN" next } From 1441f717ab5c9b6b476effc8b62a57a39c27b809 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:09:55 +0100 Subject: [PATCH 35/89] arm64: Update ID_AA64MMFR4_EL1 description to 2026-03 JSON release ID_AA64MMFR4_EL1 has gained a few fields and enum values in the past few months, so resync its definition with the 2026-03 JSON release. Signed-off-by: Marc Zyngier Reviewed-by: Joey Gouly Link: https://patch.msgid.link/20260730071022.296811-3-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/tools/sysreg | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 7cb61aca3797..3171c87eb244 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2386,17 +2386,40 @@ EndEnum EndSysreg Sysreg ID_AA64MMFR4_EL1 3 0 0 7 4 -Res0 63:48 -UnsignedEnum 47:44 SRMASK +UnsignedEnum 63:60 MTEFGT + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 59:56 SCRX + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 55:52 TEV + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 51:48 TPS + 0b0000 VAL_0000 + 0b0001 VAL_0001 + 0b0010 VAL_0010 +EndEnum +UnsignedEnum 47:44 SRMASK + 0b0000 NI + 0b0001 IMP + 0b0010 SRMASK2 +EndEnum +UnsignedEnum 43:40 TLBID 0b0000 NI 0b0001 IMP EndEnum -Res0 43:40 UnsignedEnum 39:36 E3DSE 0b0000 NI 0b0001 IMP EndEnum -Res0 35:32 +UnsignedEnum 35:32 EAESR + 0b0000 NI + 0b0001 IMP +EndEnum UnsignedEnum 31:28 RMEGDI 0b0000 NI 0b0001 IMP @@ -2410,6 +2433,7 @@ UnsignedEnum 23:20 NV_frac 0b0000 NV_NV2 0b0001 NV2_ONLY 0b0010 NV2P1 + 0b0011 NV3 EndEnum UnsignedEnum 19:16 FGWTE3 0b0000 NI From 4422d8b7502db1b910c519763135a99836efb5e5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:09:56 +0100 Subject: [PATCH 36/89] KVM: arm64: Merge guest's HCRX_EL2 using NV_HCRX_GUEST_EXCLUDE The way we merge the guest-provided HCRX_EL2 value with the host's is bonkers. We try to make it look like the FGT registers by using positive and negative polarities for traps, but most of these bits are not strictly about trapping, as they actively change the way some architectural state is managed. It would be far better to deal with these bits like we do for HCR_EL2, by enumerating the list of bits we don't allow the guest to override. This is simplified by the fact that HCRX_EL2 only affects EL1, and not EL2. Re-jig the HCRX_EL2 handling with a macro that list the bits excluded from the merge (TMEA, PTTWI, EnIDCP128). Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Reviewed-by: Joey Gouly Link: https://patch.msgid.link/20260730071022.296811-4-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/hyp/switch.h | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 18131e395e24..0903e08f4b6a 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -325,6 +325,24 @@ static inline void __deactivate_traps_mpam(void) write_sysreg_s(MPAMHCR_HOST_FLAGS, SYS_MPAMHCR_EL2); } +/* + * Just like for HCR_EL2, we can't let the guest mess with some of the + * basics we rely on in HCRX_EL2. However, the major difference is that + * HCRX_EL2 only affects EL1, and never EL2 (sudden outburst of sanity, I + * guess). So it is always the guest inflicting it on its own guestx. + * + * Things we don't want to let the guest control are: + * + * - TMEA: That's for us to decide how an SEA is routed, not the guest. + * + * - PTTWI: Similarly, it is for us to decide whether Reduced Coherency for + * the PTW is a thing. It really isn't. + * + * - EnIDCP128: We don't allow IMPDEF sysregs -- full stop. + */ +#define NV_HCRX_GUEST_EXCLUDE (HCRX_EL2_TMEA | HCRX_EL2_PTTWI | \ + HCRX_EL2_EnIDCP128) + static inline void __activate_traps_common(struct kvm_vcpu *vcpu) { struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); @@ -350,8 +368,8 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) u64 hcrx = vcpu->arch.hcrx_el2; if (is_nested_ctxt(vcpu)) { u64 val = __vcpu_sys_reg(vcpu, HCRX_EL2); - hcrx |= val & __HCRX_EL2_MASK; - hcrx &= ~(~val & __HCRX_EL2_nMASK); + hcrx |= (val & ~NV_HCRX_GUEST_EXCLUDE); + hcrx &= ~(~val & ~NV_HCRX_GUEST_EXCLUDE); } ctxt_sys_reg(hctxt, HCRX_EL2) = read_sysreg_s(SYS_HCRX_EL2); From 34a7514b87ac66bd2bbfb955c889b661a074593a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:09:57 +0100 Subject: [PATCH 37/89] KVM: arm64: Drop __HCRX_EL2_* masks The __HCRX_EL2_* masks are a leftover from a time where we didn't have much sanitisation for the system registers. Since we are now in a better place, rely on the existing checks to detect unhandled bits in HCRX_EL2. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-5-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_arm.h | 15 --------------- arch/arm64/kvm/config.c | 5 +++-- arch/arm64/kvm/emulate-nested.c | 5 ----- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 3f9233b5a130..f6cd85104794 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -287,21 +287,6 @@ GENMASK(19, 18) | \ GENMASK(15, 0)) -/* - * Polarity masks for HCRX_EL2, limited to the bits that we know about - * at this point in time. It doesn't mean that we actually *handle* - * them, but that at least those that are not advertised to a guest - * will be RES0 for that guest. - */ -#define __HCRX_EL2_MASK (BIT_ULL(6)) -#define __HCRX_EL2_nMASK (GENMASK_ULL(24, 14) | \ - GENMASK_ULL(11, 7) | \ - GENMASK_ULL(5, 0)) -#define __HCRX_EL2_RES0 ~(__HCRX_EL2_nMASK | __HCRX_EL2_MASK) -#define __HCRX_EL2_RES1 ~(__HCRX_EL2_nMASK | \ - __HCRX_EL2_MASK | \ - __HCRX_EL2_RES0) - /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ #define HPFAR_MASK (~UL(0xf)) /* diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index 0622162b089e..96ddcefc71eb 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -930,10 +930,12 @@ static const struct reg_bits_to_feat_map hcrx_feat_map[] = { NEEDS_FEAT(HCRX_EL2_EnASR, FEAT_LS64_V), NEEDS_FEAT(HCRX_EL2_EnALS, FEAT_LS64), NEEDS_FEAT(HCRX_EL2_EnAS0, FEAT_LS64_ACCDATA), + FORCE_RES0(HCRX_EL2_RES0), + FORCE_RES1(HCRX_EL2_RES1), }; -static const DECLARE_FEAT_MAP(hcrx_desc, __HCRX_EL2, +static const DECLARE_FEAT_MAP(hcrx_desc, HCRX_EL2, hcrx_feat_map, FEAT_HCX); static const struct reg_bits_to_feat_map hcr_feat_map[] = { @@ -1579,7 +1581,6 @@ struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg) break; case HCRX_EL2: resx = compute_reg_resx_bits(kvm, &hcrx_desc, 0, 0); - resx.res1 |= __HCRX_EL2_RES1; break; case HCR_EL2: resx = compute_reg_resx_bits(kvm, &hcr_desc, 0, 0); diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index e688bc5139c1..a6d1a265841f 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2320,7 +2320,6 @@ int __init populate_nv_trap_config(void) BUILD_BUG_ON(__NR_CGT_GROUP_IDS__ > BIT(TC_CGT_BITS)); BUILD_BUG_ON(__NR_FGT_GROUP_IDS__ > BIT(TC_FGT_BITS)); BUILD_BUG_ON(__NR_FG_FILTER_IDS__ > BIT(TC_FGF_BITS)); - BUILD_BUG_ON(__HCRX_EL2_MASK & __HCRX_EL2_nMASK); for (int i = 0; i < ARRAY_SIZE(encoding_to_cgt); i++) { const struct encoding_to_trap_config *cgt = &encoding_to_cgt[i]; @@ -2346,10 +2345,6 @@ int __init populate_nv_trap_config(void) } } - if (__HCRX_EL2_RES0 != HCRX_EL2_RES0) - kvm_info("Sanitised HCR_EL2_RES0 = %016llx, expecting %016llx\n", - __HCRX_EL2_RES0, HCRX_EL2_RES0); - kvm_info("nv: %ld coarse grained trap handlers\n", ARRAY_SIZE(encoding_to_cgt)); From fed0a1e87c283be0334f939879e890419a648f19 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:09:58 +0100 Subject: [PATCH 38/89] KVM: arm64: Plumb HCRX_EL2.SRMASKEn in HCRX_EL2 sanitisation HCRX_EL2.SRMASKEn is a new bit enabling FEAT_SRMASK for a guest. We don't plan to support it any time soon, but it doesn't hurt to actively document it, specially as we are going to add more bits we actually care about. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-6-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/config.c | 1 + arch/arm64/tools/sysreg | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index 96ddcefc71eb..c61d62b79602 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -904,6 +904,7 @@ static const DECLARE_FEAT_MAP_FGT(hdfgwtr2_desc, hdfgwtr2_masks, static const struct reg_bits_to_feat_map hcrx_feat_map[] = { + NEEDS_FEAT(HCRX_EL2_SRMASKEn, FEAT_SRMASK), NEEDS_FEAT(HCRX_EL2_PACMEn, feat_pauth_lr), NEEDS_FEAT(HCRX_EL2_EnFPM, FEAT_FPMR), NEEDS_FEAT(HCRX_EL2_GCSEn, FEAT_GCS), diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 3171c87eb244..28755f70f0bf 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -4545,7 +4545,9 @@ Fields ZCR_ELx EndSysreg Sysreg HCRX_EL2 3 4 1 2 2 -Res0 63:25 +Res0 63:27 +Field 26 SRMASKEn +Res0 25 Field 24 PACMEn Field 23 EnFPM Field 22 GCSEn From 0b72c406005767cf9ee11eb2b989dda9f51c7fa8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:09:59 +0100 Subject: [PATCH 39/89] KVM: arm64: Classify CPTR_EL2 as a SR_LOC_SPECIAL register It may not be obvious unless you look at it closely, but CPTR_EL2 is treated very differently from other registers. It is one the registers that, despite looking very similar between EL1 and EL2 when E2H==1, have RES0 bits that get in the way. Make it clear that CPTR_EL2 is odd by classifying it as SR_LOC_SPECIAL, just like CNTHCTL_EL2 (and for the same reasons). This makes it possible to use vcpu_read_sys_reg() with it, and will be necessary once we support FEAT_NV2P1. Reviewed-by: Joey Gouly Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-7-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_emulate.h | 2 +- arch/arm64/kvm/sys_regs.c | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 5bf3d7e1d92c..983116669518 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -617,7 +617,7 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu) */ static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu) { - u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2); + u64 cptr = vcpu_read_sys_reg(vcpu, CPTR_EL2); if (!vcpu_el2_e2h_is_set(vcpu)) cptr = translate_cptr_el2_to_cpacr_el1(cptr); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..6b47d936efb3 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -183,8 +183,6 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg, switch (reg) { MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1, translate_sctlr_el2_to_sctlr_el1 ); - MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1, - translate_cptr_el2_to_cpacr_el1 ); MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1, translate_ttbr0_el2_to_ttbr0_el1 ); MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL ); @@ -210,6 +208,19 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg, loc->loc = ((is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu)) ? SR_LOC_SPECIAL : SR_LOC_MEMORY); break; + case CPTR_EL2: + /* + * CPTR_EL2 is just as special, and needs a certain amount + * of handholding. It always lives in memory, due to being + * heavily trapped thanks to CPACR_EL1.TCPAC being RES0. + * FEAT_NV2p1 fixes this. + */ + locate_mapped_el2_register(vcpu, CPTR_EL2, CPACR_EL1, + translate_cptr_el2_to_cpacr_el1, + loc); + if (is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu)) + loc->loc = SR_LOC_SPECIAL; + break; default: loc->loc = locate_direct_register(vcpu, reg); } @@ -314,6 +325,8 @@ u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg) val &= CNTKCTL_VALID_BITS; val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS; return val; + case CPTR_EL2: + return __vcpu_sys_reg(vcpu, reg); default: WARN_ON_ONCE(1); } @@ -359,6 +372,9 @@ void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, enum vcpu_sysreg reg) */ write_sysreg_el1(val, SYS_CNTKCTL); break; + case CPTR_EL2: + write_sysreg_el1(val, SYS_CPACR); + break; default: WARN_ON_ONCE(1); } From c938c826c1efd70267ab4cf882c938e10abf0152 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:00 +0100 Subject: [PATCH 40/89] KVM: arm64: Don't evaluate HCR_EL2.NV nor HFGITR_EL2.ERET on ERET fast path We currently avoid using the ERET fast path if the guest has HCR_EL2.NV set. This is an odd check, as NV doesn't mean much if HCR_EL2.TGE==1. Similarly, evaluating HFGITR_EL2.ERET makes little sense, as this only applies to the nested context, while the ERET fast-path is purely for the benefit of L1. Replace these bizarre checks with is_nested_ctxt() which makes a lot more sense: if we are running an L2, the ERET trap must go to L1. Fixes: dd0717a998f77 ("KVM: arm64: nv: Fast-track 'InHost' exception returns") Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-8-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/vhe/switch.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index bbe9cebd3d9d..684cebf23aa0 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -344,13 +344,15 @@ static bool kvm_hyp_handle_eret(struct kvm_vcpu *vcpu, u64 *exit_code) * if this is a VHE guest hypervisor returning to its own * userspace, or the hypervisor performing a local exception * return. No need to save/restore registers, no need to - * switch S2 MMU. Just do the canonical ERET. + * switch S2 MMU. Just do the canonical ERET unless we are in + * nested context. * - * Unless the trap has to be forwarded further down the line, - * of course... + * Note that this is made possible because KVM itself never traps + * ERET when running an L2. The consequence is that any ERET trap is + * the result of HCR_EL2 or HFGITR_EL2 programming by L1 for its own + * guest, and the exception must be forwarded to L1. */ - if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) || - (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_ERET)) + if (is_nested_ctxt(vcpu)) return false; spsr = read_sysreg_el1(SYS_SPSR); From 0dbb49dcdbbe9ea1546a80548b95e5199328b181 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:01 +0100 Subject: [PATCH 41/89] arm64: Add ARM64_HAS_NV2P1 capability As we're about to deal with FEAT_NV2P1, add a new capability that will be used to key any support for it. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-9-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/tools/cpucaps | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9b85a84f6fd4..242dc211d8ef 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -51,6 +51,7 @@ HAS_LS64_V HAS_LSUI HAS_MOPS HAS_NESTED_VIRT +HAS_NV2P1 HAS_BBML2_NOABORT HAS_PAN HAS_PMUV3 From 649ba043262cd710eccbdaf138d0aa30b07c5a58 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:02 +0100 Subject: [PATCH 42/89] KVM: arm64: Relax CPTR_EL2 handling when FEAT_NV2p1 is present With FEAT_NV2P1, it is no longer necessary to trap CPTR_EL2 accesses via CPACR_EL1, as CPACR_EL1.TCPAC is guaranteed to be stateful. Prevent such trapping and context switch CPACTR_EL1 in NV contexts when NV2P1 is present. Reviewed-by: Joey Gouly Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-10-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/hyp/switch.h | 5 +++-- arch/arm64/kvm/hyp/vhe/switch.c | 3 +++ arch/arm64/kvm/hyp/vhe/sysreg-sr.c | 8 +++++--- arch/arm64/kvm/sys_regs.c | 5 ++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 0903e08f4b6a..38cdf041c70b 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -108,9 +108,10 @@ static inline void __activate_cptr_traps_vhe(struct kvm_vcpu *vcpu) * The architecture is a bit crap (what a surprise): an EL2 guest * writing to CPTR_EL2 via CPACR_EL1 can't set any of TCPAC or TTA, * as they are RES0 in the guest's view. To work around it, trap the - * sucker using the very same bit it can't set... + * sucker using the very same bit it can't set. FEAT_NV2p1 fixes it. */ - if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu)) + if (!cpus_have_final_cap(ARM64_HAS_NV2P1) && + vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu)) val |= CPTR_EL2_TCPAC; /* diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 684cebf23aa0..a8da40568a04 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -443,6 +443,9 @@ static bool kvm_hyp_handle_cpacr_el1(struct kvm_vcpu *vcpu, u64 *exit_code) u64 esr = kvm_vcpu_get_esr(vcpu); int rt; + if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + return false; + if (!is_hyp_ctxt(vcpu) || esr_sys64_to_sysreg(esr) != SYS_CPACR_EL1) return false; diff --git a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c index be685b63e8cf..6f0f046e4ca4 100644 --- a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c @@ -42,10 +42,12 @@ static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu) u64 val; /* - * We don't save CPTR_EL2, as accesses to CPACR_EL1 - * are always trapped, ensuring that the in-memory - * copy is always up-to-date. A small blessing... + * Without FEAT_NV2p1, we don't save CPTR_EL2, as accesses + * to CPACR_EL1 are always trapped, ensuring that the + * in-memory copy is always up-to-date. A small blessing... */ + if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + __vcpu_assign_sys_reg(vcpu, CPTR_EL2, read_sysreg_el1(SYS_CPACR)); __vcpu_assign_sys_reg(vcpu, SCTLR_EL2, read_sysreg_el1(SYS_SCTLR)); __vcpu_assign_sys_reg(vcpu, TTBR0_EL2, read_sysreg_el1(SYS_TTBR0)); __vcpu_assign_sys_reg(vcpu, TTBR1_EL2, read_sysreg_el1(SYS_TTBR1)); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 6b47d936efb3..1dfc1f88bec8 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -326,7 +326,10 @@ u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg) val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS; return val; case CPTR_EL2: - return __vcpu_sys_reg(vcpu, reg); + if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + return read_sysreg_el1(SYS_CPACR); + else + return __vcpu_sys_reg(vcpu, reg); default: WARN_ON_ONCE(1); } From 7c614e169826d8adeededb3fc8681bf1b6b84fce Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:03 +0100 Subject: [PATCH 43/89] KVM: arm64: Relax CNTHCTL_EL2 handling when FEAT_NV2p1 is present With NV2p1, it is no longer necessary to use the split approach where bits of CNTHCTL_EL2 cannot be accessed via CNTKCTL_EL1, and we can treat the CNTKCTL_EL1 accessor as if it was "normal". Key the special casing on FEAT_NV2P1 not being implemented. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-11-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/arch_timer.c | 10 ++++++++-- arch/arm64/kvm/hyp/vhe/sysreg-sr.c | 13 ++++++++++--- arch/arm64/kvm/sys_regs.c | 6 ++++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 4155fe89b58a..db60facad9f3 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -876,8 +876,14 @@ static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map) assign_clear_set_bit(tvt02, CNTHCTL_EL1NVVCT, clr, set); assign_clear_set_bit(tpt02, CNTHCTL_EL1NVPCT, clr, set); - /* This only happens on VHE, so use the CNTHCTL_EL2 accessor. */ - sysreg_clear_set(cnthctl_el2, clr, set); + /* + * This only happens on VHE, so use the CNTHCTL_EL2 accessor, unless + * we are sure CNTKCTL_EL1 is completely stateful with FEAT_NV2p1. + */ + if (!cpus_have_final_cap(ARM64_HAS_NV2P1)) + sysreg_clear_set(cnthctl_el2, clr, set); + else + sysreg_clear_set(cntkctl_el1, clr, set); } void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c index 6f0f046e4ca4..0c4ef1ce32ae 100644 --- a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c @@ -69,11 +69,18 @@ static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu) * The EL1 view of CNTKCTL_EL1 has a bunch of RES0 bits where * the interesting CNTHCTL_EL2 bits live. So preserve these * bits when reading back the guest-visible value. + * + * While NV2p1 fixes some of that, it makes CNTHCTL_EL2.ECV + * even more broken than it already was with NV2. */ val = read_sysreg_el1(SYS_CNTKCTL); - val &= CNTKCTL_VALID_BITS; - __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS); - __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val); + if (!cpus_have_final_cap(ARM64_HAS_NV2P1)) { + val &= CNTKCTL_VALID_BITS; + __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS); + __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val); + } else { + __vcpu_assign_sys_reg(vcpu, CNTHCTL_EL2, val); + } } __vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1)); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 1dfc1f88bec8..9439c5b2b1fe 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -322,8 +322,10 @@ u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg) switch (reg) { case CNTHCTL_EL2: val = read_sysreg_el1(SYS_CNTKCTL); - val &= CNTKCTL_VALID_BITS; - val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS; + if (!cpus_have_final_cap(ARM64_HAS_NV2P1)) { + val &= CNTKCTL_VALID_BITS; + val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS; + } return val; case CPTR_EL2: if (cpus_have_final_cap(ARM64_HAS_NV2P1)) From 4eb776d0b98436c726db2d0066f253d98c2758d2 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:04 +0100 Subject: [PATCH 44/89] KVM: arm64: Expose FEAT_NV2p1 to NV guests Since NV2p1 is reducing the number of traps, it is valuable to expose it to NV guests. Do so. Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-12-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index fb54f6dad995..82be8bcb0703 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1726,7 +1726,7 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) * You get EITHER * * - FEAT_VHE without FEAT_E2H0 - * - FEAT_NV limited to FEAT_NV2 + * - FEAT_NV limited to FEAT_NV2(p1) * - HCR_EL2.NV1 being RES0 * * OR @@ -1738,7 +1738,11 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) { val = 0; } else { - val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); + val &= ID_AA64MMFR4_EL1_NV_frac; + if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV2P1); + else + val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1); } break; From 1f4177ea49b5cd9511db5d95557f89f2d236985e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:05 +0100 Subject: [PATCH 45/89] arm64: Add FEAT_NV2p1 detection Add the necessary NV2p1 probing to the cpufeature infrastructure. Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-13-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kernel/cpufeature.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9a22df0c5120..c9c124b0ccc8 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2620,6 +2620,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { { /* Sentinel */ } }, }, + { + .desc = "FEAT_NV2p1", + .capability = ARM64_HAS_NV2P1, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV2P1) + }, { .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, .type = ARM64_CPUCAP_SYSTEM_FEATURE, From 7c81d7c4ef6e3d9d3c8796c1f0947a2716140f5f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:06 +0100 Subject: [PATCH 46/89] arm64: sysreg: Add NVHCR_EL2 description as a mirror of HCR_EL2 FEAT_NV3 introduces a new register that contains the HCR_EL2 value exposed to a NV guest. As such, it has the exact same layout as HCR_EL2. Describe NVHCR_EL2 as a mapping to HCR_EL2. Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-14-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/tools/sysreg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 28755f70f0bf..42839d2b15ae 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -4266,6 +4266,9 @@ Field 1 E2TRE Field 0 E0HTRE EndSysreg +Sysreg NVHCR_EL2 3 4 1 5 0 +Mapping HCR_EL2 +EndSysreg Sysreg HDFGRTR2_EL2 3 4 3 1 0 Res0 63:25 From 9809bdf151e2e92981ee2bd45bde1e0477c933a5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:07 +0100 Subject: [PATCH 47/89] arm64: sysreg: Add HCRX_EL2 bits related to FEAT_NV3 FEAT_NV3 introduces 4 new HCRX_EL2 control bits. Describe them in the sysreg file. Signed-off-by: Marc Zyngier Reviewed-by: Joey Gouly Link: https://patch.msgid.link/20260730071022.296811-15-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/tools/sysreg | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 42839d2b15ae..67d013b376a3 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -4548,7 +4548,12 @@ Fields ZCR_ELx EndSysreg Sysreg HCRX_EL2 3 4 1 2 2 -Res0 63:27 +Res0 63:35 +Field 34 NVnTTLBOS +Field 33 NVnTTLBIS +Field 32 NVnTTLB +Res0 31:28 +Field 27 NVTGE Field 26 SRMASKEn Res0 25 Field 24 PACMEn From 5be1b8de27dce70af8cd0bf7fa87f7c4c592aec2 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:08 +0100 Subject: [PATCH 48/89] arm64: Add ARM64_HAS_NV3 capability As a bunch of KVM code is going to depend on FEAT_NV3 being detected on the host, add a new capability that will describe it. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-16-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/tools/cpucaps | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 242dc211d8ef..7e0414509e89 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -52,6 +52,7 @@ HAS_LSUI HAS_MOPS HAS_NESTED_VIRT HAS_NV2P1 +HAS_NV3 HAS_BBML2_NOABORT HAS_PAN HAS_PMUV3 From dbb45adfcdef8b9a5811b75fd7037517da5f66e3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:09 +0100 Subject: [PATCH 49/89] KVM: arm64: Split NV-specific exit fixups from the non-NV handling In order to facilitate further changes, move the NV handling of early fixups in its own helper. This also makes the code slightly simpler to parse. Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-17-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/vhe/switch.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index a8da40568a04..53af19212895 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -539,18 +539,15 @@ static const exit_handler_fn hyp_exit_handlers[] = { [0x3F] = kvm_hyp_handle_impdef, }; -static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) +static void fixup_nv_guest_exit(struct kvm_vcpu *vcpu) { - synchronize_vcpu_pstate(vcpu); - /* * If we were in HYP context on entry, adjust the PSTATE view * so that the usual helpers work correctly. This enforces our * invariant that the guest's HYP context status is preserved * across a run. */ - if (vcpu_has_nv(vcpu) && - unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) { + if (unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) { u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); switch (mode) { @@ -567,8 +564,15 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) } /* Apply extreme paranoia! */ - BUG_ON(vcpu_has_nv(vcpu) && - !!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu)); + BUG_ON(!!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu)); +} + +static bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + synchronize_vcpu_pstate(vcpu); + + if (vcpu_has_nv(vcpu)) + fixup_nv_guest_exit(vcpu); return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers); } From 6ec15c963e536e73b9322ed578b7b45957d3bbdf Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:10 +0100 Subject: [PATCH 50/89] KVM: arm64: Add NV3 control bits to HCRX_EL2 sanitisation Expose the FEAT_NV3 control bits to the sanitisation code so that KVM stops moaning about the unattributed bits. Signed-off-by: Marc Zyngier Reviewed-by: Joey Gouly Link: https://patch.msgid.link/20260730071022.296811-18-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/config.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index c61d62b79602..467f57eb8e6b 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -225,6 +225,7 @@ struct reg_feat_map_desc { #define FEAT_HCX ID_AA64MMFR1_EL1, HCX, IMP #define FEAT_S2PIE ID_AA64MMFR3_EL1, S2PIE, IMP #define FEAT_GCIE ID_AA64PFR2_EL1, GCIE, IMP +#define FEAT_NV3 ID_AA64MMFR4_EL1, NV_frac, NV3 static bool not_feat_aa64el3(struct kvm *kvm) { @@ -904,6 +905,11 @@ static const DECLARE_FEAT_MAP_FGT(hdfgwtr2_desc, hdfgwtr2_masks, static const struct reg_bits_to_feat_map hcrx_feat_map[] = { + NEEDS_FEAT(HCRX_EL2_NVTGE | + HCRX_EL2_NVnTTLB | + HCRX_EL2_NVnTTLBIS | + HCRX_EL2_NVnTTLBOS, + FEAT_NV3), NEEDS_FEAT(HCRX_EL2_SRMASKEn, FEAT_SRMASK), NEEDS_FEAT(HCRX_EL2_PACMEn, feat_pauth_lr), NEEDS_FEAT(HCRX_EL2_EnFPM, FEAT_FPMR), From 3908e302723501648105ab2d086b4f0b869a22bc Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:11 +0100 Subject: [PATCH 51/89] KVM: arm64: Add kvm_has_nv{2,3}() predicates Add a new set of predicates indicating whether VM is capable of NV2, NV3, and is in a nested NV3 context. This is going to become useful as we start dealing with a mix of behaviours (NV2, NV3, NV2 on NV3...). Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-19-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_emulate.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 983116669518..bdb7dfaa271c 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -266,6 +266,25 @@ static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu) (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA); } +static inline bool kvm_has_nv2(struct kvm *kvm) +{ + return (cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && + kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)); +} + +static inline bool kvm_has_nv3(struct kvm *kvm) +{ + return (cpus_have_final_cap(ARM64_HAS_NV3) && + kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV3)); +} + +static inline bool is_nested_nv3_ctxt(struct kvm_vcpu *vcpu) +{ + return (has_vhe() && kvm_has_nv3(vcpu->kvm) && is_nested_ctxt(vcpu) && + (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) && + (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_NVTGE)); +} + /* * The layout of SPSR for an AArch32 state is different when observed from an * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32 From 3fc7867a95c01de51f9c7d22adc117c3202e7bb7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:12 +0100 Subject: [PATCH 52/89] KVM: arm64: Make HCR_EL2 a non-VNCR register FEAT_NV3 makes a fundamental change to the architecture, by moving guest-initiated HCR_EL2 accesses to the NVHCR_EL2 register. As the names suggests, this is HCR_EL2 for a NV guest. But where do NVHCR_EL2 accesses from a guest go? The are redirected to the VNCR page, right where HCR_EL2 is stored in the NV2 case. Does it hurt? Good. There's more coming. The challenge here is to make KVM work seamlessly, without rewriting everything. Which implies that things such as __vcpu_sys_reg(HCR_EL2) must work, no matter the underlying NV implementation. A simple way to deal with it is to move HCR_EL2's canonical storage outside of VNCR for the vast majority of the KVM code, and only have a copy at entry/exit times. Given that we don't really support NV3 yet, this is pretty simple. In the process, advertise NVHCR_EL2 as the register that now holds offset 0x78 in the VNCR page. Reviewed-by: Joey Gouly Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-20-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 3 ++- arch/arm64/include/asm/vncr_mapping.h | 2 +- arch/arm64/kvm/hyp/vhe/switch.c | 9 +++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..2648c8a717ba 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -543,6 +543,7 @@ enum vcpu_sysreg { MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */ CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */ ZCR_EL2, /* SVE Control Register (EL2) */ + HCR_EL2, /* Hypervisor Control Register */ /* Any VNCR-capable reg goes after this point */ MARKER(__VNCR_START__), @@ -571,7 +572,7 @@ enum vcpu_sysreg { VNCR(TFSR_EL1), /* Tag Fault Status Register (EL1) */ VNCR(VPIDR_EL2),/* Virtualization Processor ID Register */ VNCR(VMPIDR_EL2),/* Virtualization Multiprocessor ID Register */ - VNCR(HCR_EL2), /* Hypervisor Configuration Register */ + VNCR(NVHCR_EL2),/* NV Hypervisor Configuration Register */ VNCR(HSTR_EL2), /* Hypervisor System Trap Register */ VNCR(VTTBR_EL2),/* Virtualization Translation Table Base Register */ VNCR(VTCR_EL2), /* Virtualization Translation Control Register */ diff --git a/arch/arm64/include/asm/vncr_mapping.h b/arch/arm64/include/asm/vncr_mapping.h index 14366d35ce82..9e8a49fa8b63 100644 --- a/arch/arm64/include/asm/vncr_mapping.h +++ b/arch/arm64/include/asm/vncr_mapping.h @@ -11,7 +11,7 @@ #define VNCR_VTCR_EL2 0x040 #define VNCR_VMPIDR_EL2 0x050 #define VNCR_CNTVOFF_EL2 0x060 -#define VNCR_HCR_EL2 0x078 +#define VNCR_NVHCR_EL2 0x078 #define VNCR_HSTR_EL2 0x080 #define VNCR_VPIDR_EL2 0x088 #define VNCR_TPIDR_EL2 0x090 diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 53af19212895..e5a51dc3cbe5 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -70,6 +70,9 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) if (!vcpu_el2_e2h_is_set(vcpu)) hcr |= HCR_NV1; + /* Publish the guest's view of HCR_EL2 to the HW */ + __vcpu_assign_sys_reg(vcpu, NVHCR_EL2, __vcpu_sys_reg(vcpu, HCR_EL2)); + /* * Nothing in HCR_EL2 should impact running in hypervisor * context, apart from bits we have defined as RESx (E2H, @@ -549,6 +552,7 @@ static void fixup_nv_guest_exit(struct kvm_vcpu *vcpu) */ if (unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) { u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); + u64 hcr; switch (mode) { case PSR_MODE_EL1t: @@ -561,6 +565,11 @@ static void fixup_nv_guest_exit(struct kvm_vcpu *vcpu) *vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT); *vcpu_cpsr(vcpu) |= mode; + + /* Publish the latest HCR_EL2 to the emulation */ + hcr = __vcpu_sys_reg(vcpu, NVHCR_EL2); + + __vcpu_assign_sys_reg(vcpu, HCR_EL2, hcr); } /* Apply extreme paranoia! */ From 3b3ae6bfaca0ca2ec82cde9f21fc7e4cb4ed4b9a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:13 +0100 Subject: [PATCH 53/89] KVM: arm64: Add sanitisation for NVHCR_EL2 Just like any other VNCR-based register, NVHCR_EL2 requires some level of sanitisation. Being specified as a live copy of HCR_EL2, it adopts the exact same format, but depends on FEAT_NV3 instead. A subtle aspect is that we only want to apply the sanitisation if FEAT_NV3 is actually present, as the VNCR location is otherwise used to back accesses to HCR_EL2. Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-21-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/config.c | 15 +++++++++++++++ arch/arm64/kvm/nested.c | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index 467f57eb8e6b..1053676551af 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -1019,6 +1019,9 @@ static const struct reg_bits_to_feat_map hcr_feat_map[] = { static const DECLARE_FEAT_MAP(hcr_desc, HCR_EL2, hcr_feat_map, FEAT_AA64EL2); +static const DECLARE_FEAT_MAP(nvhcr_desc, NVHCR_EL2, + hcr_feat_map, FEAT_NV3); + static const struct reg_bits_to_feat_map sctlr2_feat_map[] = { NEEDS_FEAT(SCTLR2_EL1_NMEA | SCTLR2_EL1_EASE, @@ -1393,6 +1396,7 @@ void __init check_feature_map(void) check_reg_desc(&hdfgwtr2_desc); check_reg_desc(&hcrx_desc); check_reg_desc(&hcr_desc); + check_reg_desc(&nvhcr_desc); check_reg_desc(&sctlr2_desc); check_reg_desc(&tcr2_el2_desc); check_reg_desc(&sctlr_el1_desc); @@ -1592,6 +1596,17 @@ struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg) case HCR_EL2: resx = compute_reg_resx_bits(kvm, &hcr_desc, 0, 0); break; + case NVHCR_EL2: + /* + * Only apply sanitisation if we do have FEAT_NV3. + * Otherwise, the register aliases with HCR_EL2 in VNCR, + * and we're better off relying on data transfers between + * NVHCR_EL2 and HCR_EL2 to sanitise things. + */ + resx = (kvm_has_nv3(kvm) ? + compute_reg_resx_bits(kvm, &nvhcr_desc, 0, 0) : + (typeof(resx)){}); + break; case SCTLR2_EL1: case SCTLR2_EL2: resx = compute_reg_resx_bits(kvm, &sctlr2_desc, 0, 0); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 82be8bcb0703..34e63480886c 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1828,6 +1828,10 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) resx = get_reg_fixed_bits(kvm, HCR_EL2); set_sysreg_masks(kvm, HCR_EL2, resx); + /* NVHCR_EL2 */ + resx = get_reg_fixed_bits(kvm, NVHCR_EL2); + set_sysreg_masks(kvm, NVHCR_EL2, resx); + /* HCRX_EL2 */ resx = get_reg_fixed_bits(kvm, HCRX_EL2); set_sysreg_masks(kvm, HCRX_EL2, resx); From 0c0b2ad26b368ef679393c7e75c09c1f542716ac Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:14 +0100 Subject: [PATCH 54/89] KVM: arm64: Add NVHCR_EL2 handling to the sysreg array Expose NVHCR_EL2 to userspace, and treat the direct access as UNDEF, as that would only outline a bug in our exception routing. The generic accessors are also updated to deal with the relatively uncommon location of that register. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-22-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/sys_regs.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 9439c5b2b1fe..0aeb2e736fde 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -221,6 +221,20 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg, if (is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu)) loc->loc = SR_LOC_SPECIAL; break; + case NVHCR_EL2: + /* + * Yes, NVHCR_EL2 maps to itself when loaded in nested + * context. If you feel like the architecture is double + * backing on itself upside down, you're not alone. + */ + WARN_ON_ONCE(!kvm_has_nv3(vcpu->kvm)); + if (is_hyp_ctxt(vcpu)) { + loc->loc = SR_LOC_MEMORY; + } else { + loc->loc = SR_LOC_LOADED | SR_LOC_MAPPED; + loc->map_reg = NVHCR_EL2; + } + break; default: loc->loc = locate_direct_register(vcpu, reg); } @@ -260,6 +274,7 @@ static u64 read_sr_from_cpu(enum vcpu_sysreg reg) case DACR32_EL2: val = read_sysreg_s(SYS_DACR32_EL2); break; case IFSR32_EL2: val = read_sysreg_s(SYS_IFSR32_EL2); break; case DBGVCR32_EL2: val = read_sysreg_s(SYS_DBGVCR32_EL2); break; + case NVHCR_EL2: val = read_sysreg_s(SYS_NVHCR_EL2); break; default: WARN_ON_ONCE(1); } @@ -298,6 +313,7 @@ static void write_sr_to_cpu(enum vcpu_sysreg reg, u64 val) case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break; case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break; case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break; + case NVHCR_EL2: write_sysreg_s(val, SYS_NVHCR_EL2); break; default: WARN_ON_ONCE(1); } } @@ -2861,6 +2877,16 @@ static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu, return REG_HIDDEN; } +static unsigned int nvhcr_el2_visibility(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + if (el2_visibility(vcpu, rd) == 0 && + kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV3)) + return 0; + + return REG_HIDDEN; +} + static unsigned int sctlr2_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) { @@ -3774,6 +3800,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { sve_el2_visibility), EL2_REG_VNCR(HCRX_EL2, reset_val, 0), + EL2_REG_FILTERED(NVHCR_EL2, undef_access, reset_val, 0, + nvhcr_el2_visibility), EL2_REG(TTBR0_EL2, access_rw, reset_val, 0), EL2_REG(TTBR1_EL2, access_rw, reset_val, 0), From d6debf95f1aa8100d5d5a4feacee244f9b230087 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:15 +0100 Subject: [PATCH 55/89] KVM: arm64: Add routing for NVHCR_EL2 trap NVHCR_EL2 accesses from EL1 are taken to EL2 when HCRX_EL2.NVTGE==0 and HCR_EL2.NV==1. Describe this in the exception routing tables. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-23-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/emulate-nested.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index a6d1a265841f..ffe307f222e6 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -136,6 +136,8 @@ enum cgt_group_id { CGT_CPTR_TTA, CGT_MDCR_HPMN, + CGT_HCR_NV_HCRX_nNVTGE, + /* Must be last */ __NR_CGT_GROUP_IDS__ }; @@ -588,6 +590,15 @@ static enum trap_behaviour check_mdcr_hpmn(struct kvm_vcpu *vcpu) return BEHAVE_HANDLE_LOCALLY; } +static enum trap_behaviour check_hcr_nv_hcrx_nnvtge(struct kvm_vcpu *vcpu) +{ + if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) && + !(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_NVTGE)) + return BEHAVE_FORWARD_RW; + + return BEHAVE_HANDLE_LOCALLY; +} + #define CCC(id, fn) \ [id - __COMPLEX_CONDITIONS__] = fn @@ -598,6 +609,7 @@ static const complex_condition_check ccc[] = { CCC(CGT_CNTHCTL_EL1NVVCT, check_cnthctl_el1nvvct), CCC(CGT_CPTR_TTA, check_cptr_tta), CCC(CGT_MDCR_HPMN, check_mdcr_hpmn), + CCC(CGT_HCR_NV_HCRX_nNVTGE, check_hcr_nv_hcrx_nnvtge), }; /* @@ -853,6 +865,7 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = { SR_TRAP(SYS_SCTLR2_EL2, CGT_HCR_NV), SR_RANGE_TRAP(SYS_HCR_EL2, SYS_HCRX_EL2, CGT_HCR_NV), + SR_TRAP(SYS_NVHCR_EL2, CGT_HCR_NV_HCRX_nNVTGE), SR_TRAP(SYS_SMPRIMAP_EL2, CGT_HCR_NV), SR_TRAP(SYS_SMCR_EL2, CGT_HCR_NV), SR_RANGE_TRAP(SYS_TTBR0_EL2, From 62ed180231f419e2367e243fbfbc4ce83eb59d56 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:16 +0100 Subject: [PATCH 56/89] KVM: arm64: Add NVHCR_EL2 context switching Since NVHCR_EL2 represents the HCR_EL2 state of the EL1 guest, it must be dealt with in some particular way: - for a guest in hyp context (an L1 by definition), NVHCR_EL2 directly reflects HCR_EL2 as read and written by the guest itself. It must therefore be eagerly synced back with the emulation code which only knows about HCR_EL2. This is unconditional if NV3 is available on the host. - For an L2 guest, NVHCR_EL2 is controlled by the L1 guest, and we just context switch it like any other EL1 register. Yes, EL1, as that's where this thing runs from the PoV of L1. This is conditioned on the guest using NV3. Reviewed-by: Joey Gouly Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-24-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 11 +++++++++++ arch/arm64/kvm/hyp/vhe/switch.c | 10 ++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h index a17cbe7582de..c382848d3194 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -172,6 +172,10 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) if (ctxt_has_sctlr2(ctxt)) ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2); + + /* Retrieve L2's HCR_EL2, and save it for future use */ + if (is_nested_nv3_ctxt(ctxt_to_vcpu(ctxt))) + ctxt_sys_reg(ctxt, NVHCR_EL2) = read_sysreg_s(SYS_NVHCR_EL2); } static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt) @@ -285,6 +289,13 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt, if (ctxt_has_sctlr2(ctxt)) write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR2_EL1), SYS_SCTLR2); + + /* + * Publish the L2 view of HCR_EL2 to the HW if L1 is using NV3. + * Otherwise, the data is already in place in the L1's own VNCR. + */ + if (is_nested_nv3_ctxt(ctxt_to_vcpu(ctxt))) + write_sysreg_s(ctxt_sys_reg(ctxt, NVHCR_EL2), SYS_NVHCR_EL2); } /* Read the VCPU state's PSTATE, but translate (v)EL2 to EL1. */ diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index e5a51dc3cbe5..cbda9e087db3 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -71,7 +71,10 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) hcr |= HCR_NV1; /* Publish the guest's view of HCR_EL2 to the HW */ - __vcpu_assign_sys_reg(vcpu, NVHCR_EL2, __vcpu_sys_reg(vcpu, HCR_EL2)); + if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu)) + write_sysreg_s(__vcpu_sys_reg(vcpu, HCR_EL2), SYS_NVHCR_EL2); + else + __vcpu_assign_sys_reg(vcpu, NVHCR_EL2, __vcpu_sys_reg(vcpu, HCR_EL2)); /* * Nothing in HCR_EL2 should impact running in hypervisor @@ -567,7 +570,10 @@ static void fixup_nv_guest_exit(struct kvm_vcpu *vcpu) *vcpu_cpsr(vcpu) |= mode; /* Publish the latest HCR_EL2 to the emulation */ - hcr = __vcpu_sys_reg(vcpu, NVHCR_EL2); + hcr = (cpus_have_final_cap(ARM64_HAS_NV3) && + vcpu_el2_e2h_is_set(vcpu)) ? + read_sysreg_s(SYS_NVHCR_EL2) : + __vcpu_sys_reg(vcpu, NVHCR_EL2); __vcpu_assign_sys_reg(vcpu, HCR_EL2, hcr); } From c962853bf5e14bf95dcb316e93d7f18aa2739b5d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:17 +0100 Subject: [PATCH 57/89] KVM: arm64: Engage NV3 ERET trap elision When running on NV3 HW, always engage ERET trap elision when running the L1 context, as there is no benefit in not doing so. An L1 can itself engage trap elision by setting its own view of HCRX_EL2.NVTGE==1, which will subsequently be honored. Reviewed-by: Joey Gouly Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-25-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_emulate.h | 10 ++++++++++ arch/arm64/kvm/hyp/vhe/switch.c | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index bdb7dfaa271c..ee340d329447 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -705,6 +705,16 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR; + + /* + * NV3 is a host-specific extension, and we always use + * it when present and that the guest uses NV. It may + * be hidden from the guest though. + */ + if (cpus_have_final_cap(ARM64_HAS_NV3) && + vcpu_has_nv(vcpu) && vcpu_el2_e2h_is_set(vcpu)) { + vcpu->arch.hcrx_el2 |= HCRX_EL2_NVTGE; + } } } #endif /* __ARM64_KVM_EMULATE_H__ */ diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index cbda9e087db3..4ed745bdde41 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -345,6 +345,10 @@ static bool kvm_hyp_handle_eret(struct kvm_vcpu *vcpu, u64 *exit_code) u64 esr = kvm_vcpu_get_esr(vcpu); u64 spsr, elr, mode; + /* With NV3, the fast path is handled in HW */ + if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu)) + return false; + /* * Going through the whole put/load motions is a waste of time * if this is a VHE guest hypervisor returning to its own From 328ab9719ee0d39b77c1cc5708198d6177f23af9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:18 +0100 Subject: [PATCH 58/89] KVM: arm64: Engage NV3 TLBI trap elision Similarly to the ERET elision mechanism, FEAT_NV3 can elide TLBIs that only affects the guest's S1 translation. Enable this, with the express condition that the guest isn't NV2 aware, as we otherwise need to trap these TLBIs to deal with VNCR mappings. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-26-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_emulate.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index ee340d329447..67ad4523a1e6 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -714,6 +714,19 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_has_nv(vcpu) && vcpu_el2_e2h_is_set(vcpu)) { vcpu->arch.hcrx_el2 |= HCRX_EL2_NVTGE; + + /* + * If the guest is NV2-capable, then we need to see + * all the TLBIs, as configured in HCR_EL2. + * Otherwise, relax the TLBI traps to only TGE=0. + */ + if (!kvm_has_nv2(vcpu->kvm)) { + vcpu->arch.hcrx_el2 |= (HCRX_EL2_NVnTTLB | + HCRX_EL2_NVnTTLBIS); + + if (kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) + vcpu->arch.hcrx_el2 |= HCRX_EL2_NVnTTLBOS; + } } } } From a64183418a80f5819761146d109abdd2e0c08e7a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:19 +0100 Subject: [PATCH 59/89] KVM: arm64: Add FEAT_NV3 detection Now that everything is in place to engage the FEAT_NV3 fast-path, add the detection code to cpufeature.c. Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-27-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kernel/cpufeature.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index c9c124b0ccc8..6ae1c816e201 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2627,6 +2627,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV2P1) }, + { + .desc = "FEAT_NV3", + .capability = ARM64_HAS_NV3, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV3) + }, { .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, .type = ARM64_CPUCAP_SYSTEM_FEATURE, From 89cdbeb183829e1e75e29e8c830650d60099aedd Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:20 +0100 Subject: [PATCH 60/89] KVM: arm64: Expose FEAT_NV3 to guests Further enable FEAT_NV3 by making it visible to NV guests. Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-28-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 34e63480886c..cad2e672cff1 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1726,7 +1726,7 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) * You get EITHER * * - FEAT_VHE without FEAT_E2H0 - * - FEAT_NV limited to FEAT_NV2(p1) + * - FEAT_NV limited to FEAT_NV2(p1)/NV3 * - HCR_EL2.NV1 being RES0 * * OR @@ -1739,7 +1739,9 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) val = 0; } else { val &= ID_AA64MMFR4_EL1_NV_frac; - if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + if (cpus_have_final_cap(ARM64_HAS_NV3)) + val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV3); + else if (cpus_have_final_cap(ARM64_HAS_NV2P1)) val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV2P1); else val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); From 1f4a10ff9090068cfd78f6c76e0f2871e5953461 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:21 +0100 Subject: [PATCH 61/89] KVM: arm64: selftest: Add NVHCR_EL2 to get-reg-list Since we now expose a new system register on NV3-capable HW when the guest is exposed to it, add it to the existing checker. FWIW. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-29-maz@kernel.org Signed-off-by: Oliver Upton --- tools/testing/selftests/kvm/arm64/get-reg-list.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/get-reg-list.c b/tools/testing/selftests/kvm/arm64/get-reg-list.c index 0a3a94c4cca1..3cecc6d5dbd5 100644 --- a/tools/testing/selftests/kvm/arm64/get-reg-list.c +++ b/tools/testing/selftests/kvm/arm64/get-reg-list.c @@ -67,6 +67,7 @@ static struct feature_id_reg feat_id_regs[] = { REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP), REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP), REG_FEAT(VNCR_EL2, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY), + REG_FEAT(NVHCR_EL2, ID_AA64MMFR4_EL1, NV_frac, NV3), REG_FEAT(CNTHV_CTL_EL2, ID_AA64MMFR1_EL1, VH, IMP), REG_FEAT(CNTHV_CVAL_EL2,ID_AA64MMFR1_EL1, VH, IMP), REG_FEAT(ZCR_EL2, ID_AA64PFR0_EL1, SVE, IMP), @@ -770,6 +771,7 @@ static __u64 el2_regs[] = { SYS_REG(SP_EL2), SYS_REG(VDISR_EL2), SYS_REG(VSESR_EL2), + SYS_REG(NVHCR_EL2), }; static __u64 el2_e2h0_regs[] = { From 13c164acef3625ea03d839300b2960d045a48369 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 30 Jul 2026 08:10:22 +0100 Subject: [PATCH 62/89] arm64: Add override for ID_AA64MMFR4_EL1.NV_frac In a very unsurprising turn of events, there is a large class of firmware that is totally unable to deal with FEAT_NV3, and doesn't set the required SCR2_EL3.NV3En bit, leading to an UNDEF exception or an unhandled trap to EL3, depending on the implementation. Allow the unfortunate user to override ID_AA64MMFR4_EL1.NV_frac and get a working system. Hopefully firmware will be fixed before actually HW ships, but I have been there before... :-/ Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260730071022.296811-30-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/cpufeature.h | 1 + arch/arm64/kernel/cpufeature.c | 4 +++- arch/arm64/kernel/image-vars.h | 1 + arch/arm64/kernel/pi/idreg-override.c | 10 ++++++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index a57870fa96db..a42683af79fb 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -968,6 +968,7 @@ struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); extern struct arm64_ftr_override id_aa64mmfr0_override; extern struct arm64_ftr_override id_aa64mmfr1_override; extern struct arm64_ftr_override id_aa64mmfr2_override; +extern struct arm64_ftr_override id_aa64mmfr4_override; extern struct arm64_ftr_override id_aa64pfr0_override; extern struct arm64_ftr_override id_aa64pfr1_override; extern struct arm64_ftr_override id_aa64zfr0_override; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6ae1c816e201..14fbfa8e6b7b 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -785,6 +785,7 @@ static const struct arm64_ftr_bits ftr_raz[] = { struct arm64_ftr_override __read_mostly id_aa64mmfr0_override; struct arm64_ftr_override __read_mostly id_aa64mmfr1_override; struct arm64_ftr_override __read_mostly id_aa64mmfr2_override; +struct arm64_ftr_override __read_mostly id_aa64mmfr4_override; struct arm64_ftr_override __read_mostly id_aa64pfr0_override; struct arm64_ftr_override __read_mostly id_aa64pfr1_override; struct arm64_ftr_override __read_mostly id_aa64zfr0_override; @@ -858,7 +859,8 @@ static const struct __ftr_reg_entry { ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2, &id_aa64mmfr2_override), ARM64_FTR_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3), - ARM64_FTR_REG(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4), + ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4, + &id_aa64mmfr4_override), /* Op1 = 0, CRn = 10, CRm = 4 */ ARM64_FTR_REG(SYS_MPAMIDR_EL1, ftr_mpamidr), diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index d4c7d45ae6bc..d15c2cb1b0f2 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -51,6 +51,7 @@ PI_EXPORT_SYM(id_aa64isar2_override); PI_EXPORT_SYM(id_aa64mmfr0_override); PI_EXPORT_SYM(id_aa64mmfr1_override); PI_EXPORT_SYM(id_aa64mmfr2_override); +PI_EXPORT_SYM(id_aa64mmfr4_override); PI_EXPORT_SYM(id_aa64pfr0_override); PI_EXPORT_SYM(id_aa64pfr1_override); PI_EXPORT_SYM(id_aa64smfr0_override); diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index bc57b290e5e7..4e47616bcac2 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -106,6 +106,15 @@ static const struct ftr_set_desc mmfr2 __prel64_initconst = { }, }; +static const struct ftr_set_desc mmfr4 __prel64_initconst = { + .name = "id_aa64mmfr4", + .override = &id_aa64mmfr4_override, + .fields = { + FIELD("nv_frac", ID_AA64MMFR4_EL1_NV_frac_SHIFT, NULL), + {} + }, +}; + static bool __init pfr0_sve_filter(u64 val) { /* @@ -220,6 +229,7 @@ PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = { { &mmfr0 }, { &mmfr1 }, { &mmfr2 }, + { &mmfr4 }, { &pfr0 }, { &pfr1 }, { &isar1 }, From c6c156d931c33b92362383cf76f6d6e1291dcbfe Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 7 Aug 2026 11:40:59 +0100 Subject: [PATCH 63/89] KVM: arm64: vgic-its: Don't dereference a NULL collection on ITT save MAPC with V=0 drops ite->collection but leaves the ITE on the device's ITT list, and vgic_its_save_ite() dereferences it unconditionally. A guest that issues MAPD, MAPTI and then MAPC(V=0) therefore oopses the host when the VMM issues KVM_DEV_ARM_ITS_SAVE_TABLES to migrate it. That sequence is UNPREDICTABLE per the architecture, but KVM already handles the resulting state in the translate, MOVI and DISCARD paths. Save a zeroed entry, which vgic_its_restore_ite() reads back as invalid. Skipping the ITE instead would leave the ITT slot holding whatever is in guest memory, and restore rejects an entry naming a collection the restored collection table does not have. Fixes: eff484e0298da ("KVM: arm64: vgic-its: ITT save and restore") Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260807104102.2410744-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 4477f870c7b3..6f3ea75b4c23 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2108,6 +2108,14 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev, u32 next_offset; u64 val; + /* + * MAPC with V=0 keeps the ITEs mapped but drops their collection, + * and with it the ICID. Save a zeroed entry, which the restore path + * reads back as invalid. + */ + if (!ite->collection) + return vgic_its_write_entry_lock(its, gpa, 0ULL, ite); + next_offset = compute_next_eventid_offset(&dev->itt_head, ite); val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) | ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) | From 52d044d6e76fcd48ee384f0628a39d56b04de95b Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 7 Aug 2026 11:41:00 +0100 Subject: [PATCH 64/89] KVM: arm64: vgic: Don't leak the SPI array when init is retried Nothing latches a failed vgic_init(), so userspace can retry KVM_DEV_ARM_VGIC_CTRL_INIT after a failure past kvm_vgic_dist_init(). kvm_vgic_setup_default_irq_routing() is the reachable case, running on every configuration. Each retry overwrites dist->spis and only the last allocation is freed at teardown, leaking up to 960 struct vgic_irq, about 90KB, per attempt. Return early when the array is already allocated, as vgic_allocate_private_irqs_locked() and vgic_v4_init() do. Fixes: ad275b8bb1e65 ("KVM: arm/arm64: vgic-new: vgic_init: implement vgic_init") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260807104102.2410744-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-init.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 907057881b26..d4cf143f3ae6 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -210,6 +210,9 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis) struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); int i; + if (dist->spis) + return 0; + dist->active_spis = (atomic_t)ATOMIC_INIT(0); dist->spis = kzalloc_objs(struct vgic_irq, nr_spis, GFP_KERNEL_ACCOUNT); if (!dist->spis) From 9b10fb74e4b661543d188701bd4d024fc5c18f58 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 7 Aug 2026 11:41:01 +0100 Subject: [PATCH 65/89] KVM: arm64: vgic-its: Don't save collections the table cannot hold A guest that disables the ITS and rewrites GITS_BASER with fewer pages, VALID still set, keeps every collection it mapped against the larger table: KVM stores the new BASER unconditionally and frees the list only when VALID is cleared. vgic_its_save_collection_table() then walks the whole list, writing up to 448K past the end of the table, and saves collection IDs that vgic_its_restore_cte() rejects, so the save succeeds and the restore fails with -EINVAL on the destination. The overrun stays in guest memory, as vgic_write_guest_lock() validates every gfn. Validate each collection against the current table with vgic_its_check_id() and return -EINVAL, as vgic_its_save_device_tables() does for devices. Collection IDs are unique and the collection table is never indirect, so the check also bounds the walk. Fixes: ea1ad53e1e31a ("KVM: arm64: vgic-its: Collection table save/restore") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260807104102.2410744-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 6f3ea75b4c23..d8b784f69abc 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2529,6 +2529,9 @@ static int vgic_its_save_collection_table(struct vgic_its *its) max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K; list_for_each_entry(collection, &its->collection_list, coll_list) { + if (!vgic_its_check_id(its, baser, collection->collection_id, NULL)) + return -EINVAL; + ret = vgic_its_save_cte(its, collection, gpa); if (ret) return ret; From ad1e686e2378d7afb0e16b065c877be626eaaaf5 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 7 Aug 2026 11:41:02 +0100 Subject: [PATCH 66/89] KVM: arm64: vgic-its: Point saved ITEs at the next valid entry An ITE whose collection was dropped is saved as an invalid entry, and vgic_its_restore_ite() has no offset to follow from one, so the scan steps a single entry at a time until it reaches a valid entry or the end of the ITT. Compute the offset to the next ITE that is saved as valid instead. Suggested-by: Oliver Upton Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260807104102.2410744-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index d8b784f69abc..5882fc859c69 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2024,15 +2024,16 @@ static u32 compute_next_devid_offset(struct list_head *h, static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite) { - struct its_ite *next; - u32 next_offset; + struct its_ite *next = ite; - if (list_is_last(&ite->ite_list, h)) - return 0; - next = list_next_entry(ite, ite_list); - next_offset = next->event_id - ite->event_id; + /* Point at the next ITE that vgic_its_save_ite() stores as valid. */ + list_for_each_entry_continue(next, h, ite_list) { + if (next->collection) + return min_t(u32, next->event_id - ite->event_id, + VITS_ITE_MAX_EVENTID_OFFSET); + } - return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET); + return 0; } /** From f5b8f203bfc07a5a257dff859e66d2c500f9f509 Mon Sep 17 00:00:00 2001 From: Qihang Date: Fri, 7 Aug 2026 10:55:34 +0800 Subject: [PATCH 67/89] KVM: arm64: vgic-v3: take an LPI reference in vgic_v3_save_pending_tables vgic_v3_save_pending_tables() iterates dist->lpi_xa using xa_for_each() and dereferences the returned struct vgic_irq in the loop body without holding a reference on the LPI. The xarray iterator only provides temporary RCU coverage while looking up the current entry. That is not sufficient for this loop body, which reads fields from struct vgic_irq and performs guest memory accesses before the iteration completes. A concurrent path can trigger this race: the irqfd cached injection path (vgic_its_inject_cached_translation) obtains a transient LPI reference via vgic_its_check_cache() without holding kvm->lock, vcpu->mutex, config_lock, or its_lock. If guest ITS DISCARD then drops the cache and ITE references under its_lock, the transient inject reference may become the final one. When vgic_put_irq() drops it, the LPI is erased from lpi_xa and freed via kfree_rcu(). Meanwhile, vgic_v3_save_pending_tables() may still hold a stale pointer obtained from the xarray iterator and dereference it after the RCU grace period completes. Fix this by re-fetching each iterated LPI via vgic_get_irq(), which takes a stable reference, and dropping it with vgic_put_irq() on all paths. This matches the pattern already used by other lpi_xa iterators in the vgic ITS code. Cc: stable@vger.kernel.org Signed-off-by: Qihang Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260807025534.34125-1-q.h.hack.winter@gmail.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-v3.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 9e841e7afd4a..f42aeba4c696 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -611,9 +611,13 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) bool is_pending; bool stored; + irq = vgic_get_irq(kvm, index); + if (!irq) + continue; + vcpu = irq->target_vcpu; if (!vcpu) - continue; + goto put_irq; pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser); @@ -624,7 +628,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) if (ptr != last_ptr) { ret = kvm_read_guest_lock(kvm, ptr, &val, 1); if (ret) - goto out; + goto put_irq; last_ptr = ptr; } @@ -636,7 +640,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) vgic_v4_get_vlpi_state(irq, &is_pending); if (stored == is_pending) - continue; + goto put_irq; if (is_pending) val |= 1 << bit_nr; @@ -644,6 +648,8 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) val &= ~(1 << bit_nr); ret = vgic_write_guest_lock(kvm, ptr, &val, 1); +put_irq: + vgic_put_irq(kvm, irq); if (ret) goto out; } From c55bc773b6e814406658fae7dc5c15f639ed816e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:19 +0100 Subject: [PATCH 68/89] KVM: arm64: Remove VM-wide VNCR mapping counter The global VNCR mapping counter is used to decide whether an L1 provided VNCR page is mapped in L0 on any CPU at the point of dealing with a TLB invalidation. It is incremented when a mapping is made in the fixmap, and decremented when unmapped. As it turns out, this tracking has several flaws: - we are trying to invalidate TLBs, and the mapping is only an opportunistic consequence of the TLB. Checking this counter to decide whether a TLB needs to be invalidated may result in missed invalidations. - an L1 vcpu invalidating its own TLB (a very likely case) will not succeed in invalidating the VNCR pseudo TLB because that page is not mapped in L0 at this stage. Given that this tracking fails at delivering the minimum guarantees that are required and is only a performance optimisation, remove it completely. Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Acked-by: Lorenzo Stoakes (ARM) Link: https://patch.msgid.link/20260806091026.620700-2-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 3 --- arch/arm64/kvm/hyp/vhe/switch.c | 3 +-- arch/arm64/kvm/nested.c | 3 --- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..ac16f96c878d 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -411,9 +411,6 @@ struct kvm_arch { /* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; - /* Count the number of VNCR_EL2 currently mapped */ - atomic_t vncr_map_count; - /* * For an untrusted host VM, 'pkvm.handle' is used to lookup * the associated pKVM instance in the hypervisor. diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index bbe9cebd3d9d..c09b1d411c58 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -427,8 +427,7 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) * If we have to check for any VNCR mapping being invalidated, * go back to the slow path for further processing. */ - if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && - atomic_read(&vcpu->kvm->arch.vncr_map_count)) + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) return false; __kvm_skip_instr(vcpu); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index dfb96edbdc43..f3c75954cf36 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -48,7 +48,6 @@ void kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = NULL; kvm->arch.nested_mmus_size = 0; - atomic_set(&kvm->arch.vncr_map_count, 0); } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -890,7 +889,6 @@ static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); vcpu->arch.vncr_tlb->cpu = -1; host_data_clear_flag(L1_VNCR_MAPPED); - atomic_dec(&vcpu->kvm->arch.vncr_map_count); } void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) @@ -1592,7 +1590,6 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); host_data_set_flag(L1_VNCR_MAPPED); - atomic_inc(&vcpu->kvm->arch.vncr_map_count); } } From 8053393680d4fa3eb962667d2be95dd39f0940e5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:20 +0100 Subject: [PATCH 69/89] KVM: arm64: Handle negative S1 walk levels in VNCR TLB size evaluation Computing the effects of a TLB invalidation involves looking at the size of the mapping cached by the TLB. For S1 mappings such as VNCR, this is deducted from the combination of the base granule size and the mapping level. However, this implies that the S1 MMU is *on*. When the MMU is off, we indicate this with the level being set to a "creative" value of -127 (S1_MMU_DISABLED). This ends-up being misinterpreted by pgshift_level_to_ttl() as it doesn't handle negative levels at all (the level is immediately cast to a u8 and only the bottom two bits considered), leading to an invalidation size of 0. Not helpful. Tidy-up pgshift_level_to_ttl() to handle these negative levels, and ttl_to_size() to always return SZ_1G when no valid TTL is present. This allows the removal of open-coded checks for similar situations. Note that the check for a negative value not explicitely checking for S1_MMU_DISABLED is deliberate, so that actual negative levels introduced with LVA2 and D128 can take the same path if we ever support them. Fixes: 7270cc9157f47 ("KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers") Reported-by: Hyunwoo Kim Link: https://lore.kernel.org/r/ameGoxbn2wzBq2kL@v4bel Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-3-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index f3c75954cf36..035cda256e2a 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -505,7 +505,7 @@ int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, return ret; } -static unsigned int ttl_to_size(u8 ttl) +static unsigned int __ttl_to_size(u8 ttl) { int level = ttl & 3; int gran = (ttl >> 2) & 3; @@ -561,10 +561,22 @@ static unsigned int ttl_to_size(u8 ttl) return max_size; } -static u8 pgshift_level_to_ttl(u16 shift, u8 level) +static unsigned int ttl_to_size(u8 ttl) +{ + return __ttl_to_size(ttl) ?: SZ_1G; +} + +static u8 pgshift_level_to_ttl(u16 shift, s8 level) { u8 ttl; + /* + * If we don't have a proper level, fallback to the maximum + * size. + */ + if (level < 0) + return 0; + switch(shift) { case 12: ttl = TLBI_TTL_TG_4K; @@ -675,7 +687,11 @@ unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) ttl = get_guest_mapping_ttl(mmu, addr); } - max_size = ttl_to_size(ttl); + /* + * Don't use the default 1GB fallback, as we can adapt to the + * max mapping size we allow at S2. + */ + max_size = __ttl_to_size(ttl); if (!max_size) { /* Compute the maximum extent of the invalidation */ @@ -1124,8 +1140,6 @@ static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, case OP_TLBI_VALE1OSNXS: scope->type = TLBI_VA; scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); - if (!scope->size) - scope->size = SZ_1G; scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); break; @@ -1152,8 +1166,6 @@ static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, case OP_TLBI_VAALE1OSNXS: scope->type = TLBI_VAA; scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); - if (!scope->size) - scope->size = SZ_1G; scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); break; case OP_TLBI_RVAE2: From 8c774604b6ecaca495fa3d27c21593269627f48f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:21 +0100 Subject: [PATCH 70/89] KVM: arm64: Consider SCTLR_EL2.M when mapping the L1 VNCR page We record a VNCR TLB even when SCTLR_EL2.M is 0 in order to make our life easier. But this is not something that the architecture anticipate. As a consequence, a hypervisor is free to set VNCR_EL2 to some PA when SCTLR_EL2.M==0, use it to run a guest which indirectly accesses the VNCR page, then eventually set SCTLR_EL2.M==1 with the same VA. Yes, this is odd, but apparently legal. A common trick in HW is to invalidate the TLBs on SCTLR_ELx.M being flipped. But doing this is a not a good idea for us (we'd need to trap SCTLR accesses), and wouldn't scale as we nest deeper. Instead, use the fact that the S1 MMU being off at the point of translation is cached in our TLB, and if it doesn't match the current MMU state, leave the VNCR unmapped. Fixes: 2a359e072596f ("KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2") Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-4-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_nested.h | 7 +++++++ arch/arm64/kvm/at.c | 2 -- arch/arm64/kvm/nested.c | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 012d711034d1..bfed664d823b 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -388,6 +388,8 @@ struct s1_walk_result { bool failed; }; +#define S1_MMU_DISABLED (-127) + static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) { wr->fst = fst; @@ -396,6 +398,11 @@ static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) wr->failed = true; } +static inline bool s1_walk_translated(struct s1_walk_result *wr) +{ + return wr->level != S1_MMU_DISABLED; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 640f2dc00a8b..0926426b8798 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -11,8 +11,6 @@ #include #include -#define S1_MMU_DISABLED (-127) - static int get_ia_size(struct s1_walk_info *wi) { return 64 - wi->txsz; diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 035cda256e2a..27bc7ee4b338 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1578,6 +1578,10 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) if (!vt->valid) return; + /* We cache the MMU state in the TLB. Check that it matches. */ + if (!!(vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_M) != s1_walk_translated(&vt->wr)) + return; + if (read_vncr_el2(vcpu) != vt->gva) return; From 34af2c3e31f91a739dc175459fdbd99ed952b457 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:22 +0100 Subject: [PATCH 71/89] KVM: arm64: Correctly handle end of VA space TLBI invalidation Our TLB invalidation by VA code is based on comparing two ranges, one defined by the TLB, and one defined by the TLBI instruction. Each range is defined by a start and a size. However, the way the comparison is done doesn't account for address rollover, as it compares an address with (base + size). This works nicely until this expression represent the last page/block in the TTBR1 VA space, as the result is a big fat 0. And a failed TLB invalidation. Rewrite the comparison in a way that is immune to the address rollover (making the end address inclusive instead of exclusive), and move this into a common helper that is used by both VA and IPA invalidations, as suggested by Hyunwoo Kim (although the IPA version didn't suffer from this particular problem, obviously). Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reviewed-by: Yuan Yao Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-5-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 43 ++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 27bc7ee4b338..8a602d074dbb 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -999,6 +999,20 @@ static void invalidate_vncr(struct vncr_tlb *vt) clear_fixmap(vncr_fixmap(vt->cpu)); } +static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, + u64 scope_start, u64 scope_size) +{ + u64 tlb_size, tlb_start, tlb_end, scope_end; + + tlb_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); + + tlb_start = addr & ~(tlb_size - 1); + tlb_end = tlb_start + tlb_size - 1; + scope_end = scope_start + scope_size - 1; + + return !(tlb_end < scope_start || tlb_start > scope_end); +} + /* * VNCR TLB invalidation occurs from MMU notifiers or TLBI instructions, and * either can race against a vcpu not being onlined yet (no pseudo-TLB @@ -1021,19 +1035,9 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; - kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 ipa_start, ipa_end, ipa_size; - - ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - ipa_start = vt->wr.pa & ~(ipa_size - 1); - ipa_end = ipa_start + ipa_size; - - if (ipa_end <= start || ipa_start >= end) - continue; - - invalidate_vncr(vt); - } + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) + if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) + invalidate_vncr(vt); } struct s1e2_tlbi_scope { @@ -1059,28 +1063,19 @@ static void invalidate_vncr_va(struct kvm *kvm, lockdep_assert_held_write(&kvm->mmu_lock); kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 va_start, va_end, va_size; - - va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - va_start = vt->gva & ~(va_size - 1); - va_end = va_start + va_size; - switch (scope->type) { case TLBI_ALL: break; case TLBI_VA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; if (vt->wr.nG && vt->wr.asid != scope->asid) continue; break; case TLBI_VAA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; break; From 38640bc32be3fcf9526d477155bc19d3f146231f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:23 +0100 Subject: [PATCH 72/89] KVM: arm64: Handle VNCR TLB invalidation race with vcpu_put() VNCR unmapping While VNCR TLB invalidation always occurs under the MMU lock, vcpu_put() doesn't, while it unmaps the VNCR page. The problem is that the invalidation evaluates vncr_tlb::cpu to decide whether an unmapping needs to take place (cpu != -1) before performing it. On the other hand, this_cpu_reset_vncr_fixmap() unconditionally unmaps if L1_VNCR_MAPPED is set. These two obviously can race, with a TOCTOU pattern on the TLBI path, and a BUG_ON() on the vcpu_put() path. And the two can end-up calling vncr_fixmap(-1), with extra lethal effects. Move the reset of vncr_tlb::cpu to -1 to a common function, and make this update atomic so that only a single thread can reset the field and perform the corresponding unmap. The vcpu_put() still need to unconditionally unmap the current VNCR to close another ugly race. Finally, the assignment of vncr_tlb::cpu is moved to be kept in sync with the actual mapping, similar to L1_VNCR_MAPPED being set. Fixes: 7270cc9157f47 ("KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers") Reported-by: sashiko-bot@kernel.org Link: https://lore.kernel.org/r/20260801130237.0FD8F1F00ACA@smtp.kernel.org Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Reviewed-by: Yuan Yao Link: https://patch.msgid.link/20260806091026.620700-6-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 42 +++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 8a602d074dbb..cf0d45059edb 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -27,7 +27,7 @@ struct vncr_tlb { bool hpa_writable; /* -1 when not mapped on a CPU */ - int cpu; + atomic_t cpu; /* * true if the TLB is valid. Can only be changed with the @@ -894,16 +894,40 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) } } +/* + * Unmapping an L1 VNCR can happen concurrently without the mmu lock being + * effective (vcpu_put() vs TLBI handling). The atomic_xchg below ensures + * that only one CPU sets it to -1 while getting a valid CPU number back. + */ +static int unmap_l1_vncr(struct vncr_tlb *vt) +{ + int cpu = atomic_xchg_relaxed(&vt->cpu, -1); + + if (cpu != -1) + clear_fixmap(vncr_fixmap(cpu)); + + return cpu; +} + static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) { if (!host_data_test_flag(L1_VNCR_MAPPED)) return; - BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); BUG_ON(is_hyp_ctxt(vcpu)); - clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); - vcpu->arch.vncr_tlb->cpu = -1; + /* + * Unconditionally unmap the local VNCR if we have lost the race + * against a concurrent TLBI. Otherwise we could end-up running + * another vcpu with VNCR still mapped if the TLBI thread is + * preempted between the exchange and the clear_fixmap(). + * + * Note that we do not care about the TLBI nuking the fixmap behind + * the back of an running vcpu. This will only generate a fault and + * possibly a retranslation. + */ + if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1) + clear_fixmap(vncr_fixmap(smp_processor_id())); host_data_clear_flag(L1_VNCR_MAPPED); } @@ -995,8 +1019,7 @@ u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) static void invalidate_vncr(struct vncr_tlb *vt) { vt->valid = false; - if (vt->cpu != -1) - clear_fixmap(vncr_fixmap(vt->cpu)); + unmap_l1_vncr(vt); } static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, @@ -1452,7 +1475,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) vt->hpa = pfn << PAGE_SHIFT; vt->hpa_writable = writable; vt->valid = true; - vt->cpu = -1; + atomic_set(&vt->cpu, -1); kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); @@ -1583,8 +1606,6 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid) return; - vt->cpu = smp_processor_id(); - if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) prot = PAGE_KERNEL; else if (vt->wr.pr) @@ -1599,7 +1620,8 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) * FIXME: WO doesn't work at all, need POE support in the kernel. */ if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { - __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); + atomic_set(&vt->cpu, smp_processor_id()); + __set_fixmap(vncr_fixmap(atomic_read(&vt->cpu)), vt->hpa, prot); host_data_set_flag(L1_VNCR_MAPPED); } } From 2393470085649f0b973ecceb26fe8fc71edde0c1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:24 +0100 Subject: [PATCH 73/89] KVM: arm64: Sign-extend VA for range-based TLBI invalidation When the decode_range_tlbi() helper was moved to be used for S1 TLBIs, the required sign extension was omitted. Add it. As a result, special care must be taken to not overflow PA bits when this is used for S2 invalidation. Fixes: 85bba00425ae0 ("KVM: arm64: nv: Move TLBI range decoding to a helper") Reported-by: sashiko-bot@kernel.org Link: https://lore.kernel.org/r/20260801130337.EB2BA1F00AC4@smtp.kernel.org Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-7-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_nested.h | 7 +++++++ arch/arm64/kvm/sys_regs.c | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index bfed664d823b..c83be6d0e79a 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -291,6 +291,13 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) base = (val & GENMASK(36, 0)) << shift; + /* + * We only deal with at most 48bit VA/IPA, so 48 is where we + * sign-extend from. Should we support FEAT_L{VP}A* at some point, + * this will need to be revisited. + */ + base = (u64)sign_extend64(base, 48); + if (asid) *asid = FIELD_GET(TLBIR_ASID_MASK, val); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..797e888bf939 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -4057,6 +4057,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); u64 base, range; + int pa_bits; if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding)) return undef_access(vcpu, p, r); @@ -4068,6 +4069,16 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, */ base = decode_range_tlbi(p->regval, &range, NULL); + /* + * Ignore TLBIs that start out of PA_bits range, and cap the + * invalidation to the [base:bit(PA_bits)] interval. + */ + pa_bits = kvm_get_pa_bits(vcpu->kvm); + if (fls64(base) > pa_bits) + return true; + + range = min(range, BIT_ULL(pa_bits) - base); + kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), &(union tlbi_info) { .range = { From 2b7324f3a0c1072b9d578b8d42f199506753f26e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:25 +0100 Subject: [PATCH 74/89] KVM: arm64: Make VNCR invalidation participate in MMU invalidation retry A VNCR TLB invalidation can occur on one vcpu while another vcpu is faulting in this same page. Without correctly handling this, we can end up with the following scenario: - vcpu A walks the PTs to translate VNCR - before vcpu A is able to grab the MMU lock to insert the TLB, vcpu B updates the S1 PTs with an invalid entry, and issues a TLBI S1E2 for this VA - vcpu A inserts the TLB for something that is now invalid This isn't a new problem, and we manage S2 by having the MMU notifier to bump up mmu_invalidate_seq on invalidation so that the fault can be replayed. We can perform something similar here, and extend invalidate_vncr_va() to update the same counter, clearly indicating that the context has changed under our feet. This is safe as the invalidation always happen while holding the MMU lock for write, and that we sample the sequence number before walking S1. Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reported-by: sashiko-bot@kernel.org Link: https://lore.kernel.org/r/20260801130454.5D9F11F00AC4@smtp.kernel.org Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260806091026.620700-8-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index cf0d45059edb..550c9bd3dbe7 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1058,6 +1058,12 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; + /* + * Note that invalidating the VNCR on the back of an MMU notifier + * doesn't require messing with the invalidation counter for a + * parallel walk. The notifier itself will have bumped the counter, + * making sure we rewalk. + */ kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) invalidate_vncr(vt); @@ -1085,6 +1091,15 @@ static void invalidate_vncr_va(struct kvm *kvm, lockdep_assert_held_write(&kvm->mmu_lock); + /* + * We might be performing a parallel S1 walk, so bump up the + * invalidation counter even in the absence of an actual VNCR TLB + * invalidation, as this could indicate that the guest has gone + * through a BBM sequence. + */ + kvm->mmu_invalidate_seq++; + smp_wmb(); + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { switch (scope->type) { case TLBI_ALL: @@ -1419,15 +1434,15 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) va = read_vncr_el2(vcpu); + mmu_seq = vcpu->kvm->mmu_invalidate_seq; + smp_rmb(); + ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); if (ret) return ret; write_fault = kvm_is_write_fault(vcpu); - mmu_seq = vcpu->kvm->mmu_invalidate_seq; - smp_rmb(); - gfn = vt->wr.pa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); if (!memslot) { From 62e3bb894a6e926a26132bc4f558d80c2997d249 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 6 Aug 2026 10:10:26 +0100 Subject: [PATCH 75/89] KVM: arm64: Add VNCR TLB tracking again Having established that our VNCR TLB tracking was flawed and having dropped it from KVM, it is time to replace it with something that actually works in order to get some performance back, as the TLBI overhead is otherwise pretty high. The goal of such tracking is to hit the TLBI slow path if there are any VNCR TLBs in the guest, irrespective of their mapping state. For this purpose, we introduce an VM wide counter (vncr_tlb_count) that tracks how many valid VNCR TLB are present. This means that creating such TLB must increment the counter, and invalidation decrement it, and both these operations must be done with the MMU lock held for write. On TLBI handling affecting EL2 S1, a non-zero counter forces the handling to take the slow path to consider the VNCR TLBs. Note that the bumping up is done "speculatively" in order to make sure that a concurrent invalidation will be taken on the slow path. Not exactly rocket science. Hopefully I got it right this time. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260806091026.620700-9-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 3 +++ arch/arm64/kvm/hyp/vhe/switch.c | 11 +++++++--- arch/arm64/kvm/nested.c | 36 +++++++++++++++++++++++++------ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index ac16f96c878d..108966a9db12 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -411,6 +411,9 @@ struct kvm_arch { /* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; + /* Count the number of VNCR_EL2 TLBs */ + atomic_t vncr_tlb_count; + /* * For an untrusted host VM, 'pkvm.handle' is used to lookup * the associated pKVM instance in the hypervisor. diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index c09b1d411c58..fa776b18a32c 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -424,10 +424,15 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) return false; /* - * If we have to check for any VNCR mapping being invalidated, - * go back to the slow path for further processing. + * If we have to check for any VNCR TLB being invalidated, go back + * to the slow path for further processing. + * + * The synchronisation betweem TLBI and walk is provided by the + * speculative increment of the TLB counter on walk, and the + * invalidation counter. Yes, this is fiddly. */ - if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && + atomic_read(&vcpu->kvm->arch.vncr_tlb_count)) return false; __kvm_skip_instr(vcpu); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 550c9bd3dbe7..20af94197a8a 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -48,6 +48,7 @@ void kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = NULL; kvm->arch.nested_mmus_size = 0; + atomic_set(&kvm->arch.vncr_tlb_count, 0); } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -1016,10 +1017,12 @@ u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) return asid; } -static void invalidate_vncr(struct vncr_tlb *vt) +static void invalidate_vncr(struct kvm *kvm, struct vncr_tlb *vt) { + BUG_ON(!vt->valid); vt->valid = false; unmap_l1_vncr(vt); + atomic_dec(&kvm->arch.vncr_tlb_count); } static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, @@ -1066,7 +1069,7 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) */ kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) - invalidate_vncr(vt); + invalidate_vncr(kvm, vt); } struct s1e2_tlbi_scope { @@ -1123,7 +1126,7 @@ static void invalidate_vncr_va(struct kvm *kvm, break; } - invalidate_vncr(vt); + invalidate_vncr(kvm, vt); } } @@ -1359,13 +1362,20 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm) * intersects with the TLBI request, invalidate it, and unmap the page * from the fixmap. Because we need to look at all the vcpu-private TLBs, * this requires some wide-ranging locking to ensure that nothing races - * against it. This may require some refcounting to avoid the search when - * no such TLB is present. + * against it. This requires some refcounting to avoid the search when + * no such TLB is present (see below). * * - On MMU notifiers, we must invalidate our TLB in a similar way, but * looking at the IPA instead. The funny part is that there may not be a * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST * instructions. + * + * - vncr_tlb_count tracks the number of valid VNCR TLBs VM-wide. This isn't + * the number of *mapped* L1 VNCR pages, which is likely be a subset (and + * by definition, a TLBI handled from L1 runs with the canonical VNCR + * page, not the L1's). The innermost trap handling code checks this to + * find out whether to return to the guest ASAP (no L1 TLBs) or to visit + * this part of the world for some extra invalidation work. */ int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) @@ -1420,7 +1430,8 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) */ scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { this_cpu_reset_vncr_fixmap(vcpu); - vt->valid = false; + if (vt->valid) + invalidate_vncr(vcpu->kvm, vt); vt->wi = (struct s1_walk_info) { .regime = TR_EL20, @@ -1545,7 +1556,20 @@ int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) return -EIO; } + /* + * Speculatively increment the TLB count to make sure concurrent + * TLBIs will take the slow path, and will interact with the retry + * mechanism. Drop it again on error. + */ + atomic_inc(&vcpu->kvm->arch.vncr_tlb_count); + smp_mb__after_atomic(); + ret = kvm_translate_vncr(vcpu, &is_gmem); + if (ret) { + smp_mb__before_atomic(); + atomic_dec(&vcpu->kvm->arch.vncr_tlb_count); + } + switch (ret) { case -EAGAIN: /* Let's try again... */ From 2858600ecd014c4465008b7c39a9799d7e37722b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 8 Aug 2026 08:58:22 +0000 Subject: [PATCH 76/89] KVM: arm64: Make timer_get_offset() work in all contexts We currently have two implementations of get_timer offset(), one in arm_arch_timer.h, and another one in switch.h. These two only differ by a pair of kern_hyp_va(), which seems a pretty weak reason to open-code it. Turn this function into a macro to avoid the include dependency hell on kern_hyp_va(), and make it work correctly in all contexts. Signed-off-by: Marc Zyngier Signed-off-by: Mostafa Saleh Link: https://patch.msgid.link/20260808085824.732659-2-smostafa@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/hyp/switch.h | 15 +---------- include/kvm/arm_arch_timer.h | 34 +++++++++++++++---------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 18131e395e24..4057773aa40b 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -705,22 +705,9 @@ static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu) return true; } -/* Open-coded version of timer_get_offset() to allow for kern_hyp_va() */ -static inline u64 hyp_timer_get_offset(struct arch_timer_context *ctxt) -{ - u64 offset = 0; - - if (ctxt->offset.vm_offset) - offset += *kern_hyp_va(ctxt->offset.vm_offset); - if (ctxt->offset.vcpu_offset) - offset += *kern_hyp_va(ctxt->offset.vcpu_offset); - - return offset; -} - static inline u64 compute_counter_value(struct arch_timer_context *ctxt) { - return arch_timer_read_cntpct_el0() - hyp_timer_get_offset(ctxt); + return arch_timer_read_cntpct_el0() - timer_get_offset(ctxt); } static bool kvm_handle_cntxct(struct kvm_vcpu *vcpu) diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index 15a4f97f8105..bc6f2fdd7ad3 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -162,20 +162,28 @@ static inline bool has_cntpoff(void) return (has_vhe() && cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF)); } -static inline u64 timer_get_offset(struct arch_timer_context *ctxt) -{ - u64 offset = 0; +#ifdef __KVM_NVHE_HYPERVISOR__ +#define KERN_HYP_VA(x) kern_hyp_va(x) +#else +#define KERN_HYP_VA(x) x +#endif - if (!ctxt) - return 0; - - if (ctxt->offset.vm_offset) - offset += *ctxt->offset.vm_offset; - if (ctxt->offset.vcpu_offset) - offset += *ctxt->offset.vcpu_offset; - - return offset; -} +#define timer_get_offset(ctxt) \ + ({ \ + struct arch_timer_context *__ctxt = (ctxt); \ + u64 off = 0; \ + \ + if (__ctxt) { \ + struct arch_timer_offset *ato = &__ctxt->offset;\ + \ + if (ato->vm_offset) \ + off += *KERN_HYP_VA(ato->vm_offset); \ + if (ato->vcpu_offset) \ + off += *KERN_HYP_VA(ato->vcpu_offset); \ + } \ + \ + off; \ + }) static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset) { From 47d3eef780e30cf94fa5968e49b47cef77834ab9 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Sat, 8 Aug 2026 08:58:23 +0000 Subject: [PATCH 77/89] KVM: arm64: Fix timer offsets for non-protected VMs With pKVM, protected VMs always have offset of zero. However, timer offsets for non-protected guests fail to take effect for two reasons: 1) In __timer_enable_traps(), enabling of traps check for is_protected_kvm_enabled() rather than vcpu_is_protected(vcpu) 2) The vcpu timer offsets were never initialised and kept as NULL. This is problematic for cases when the timer is trapped in the hypervisor as the with the case of broken CNTVOFF_EL2, which leads to the hypervisor and host using different offsets and causing VM hangs. This can be confirmed by running the arch_timer selftest which fails: ./arch_timer -o 100000000 Random seed: 0x6b8b4567 Guest assert failed, vcpu 0; stage; 3; iter: 0 ==== Test Assertion Failure ==== arm64/arch_timer.c:137: config_iter + 1 == irq_iter pid=310 tid=312 errno=4 - Interrupted system call Guest assert failed, vcpu 3; stage; 3; iter: 0 Guest assert failed, vcpu 1; stage; 3; iter: 0 ==== Test Assertion Failure ==== arm64/arch_timer.c:137: config_iter + 1 == irq_iter pid=310 tid=313 errno=4 - Interrupted system call Guest assert failed, vcpu 2; stage; 3; iter: 0 ==== Test Assertion Failure ==== arm64/arch_timer.c:137: config_iter + 1 == irq_iter pid=310 tid=314 errno=4 - Interrupted system call [...] After the fix: ./arch_timer -o 100000000 Random seed: 0x6b8b4567 PASS(vCPU-1). PASS(vCPU-3). PASS(vCPU-0). PASS(vCPU-2) Reported-by: Sashiko Fixes: cb0c272acebd ("KVM: arm64: Initialize the hypervisor's VM state at EL2") Signed-off-by: Mostafa Saleh Link: https://patch.msgid.link/20260808085824.732659-3-smostafa@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 14 ++++++++++++++ arch/arm64/kvm/hyp/nvhe/timer-sr.c | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index c54aa5031336..5a2c7f7d956b 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -528,6 +528,20 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + /* + * Timer offsets are pointing to the untrusted KVM copy, + * which is pinned in __pkvm_init_vm() for the VM life time. + * It is worth noting that hyp_vm->host_kvm points to an EL2 + * linear map address and timer_get_offset() will use + * kern_hyp_va() which is safe as it is idempotent. + */ + vcpu_vtimer(&hyp_vcpu->vcpu)->offset.vm_offset = + &hyp_vm->host_kvm->arch.timer_data.voffset; + vcpu_ptimer(&hyp_vcpu->vcpu)->offset.vm_offset = + &hyp_vm->host_kvm->arch.timer_data.poffset; + } + ret = pkvm_vcpu_init_sysregs(hyp_vcpu); if (ret) goto done; diff --git a/arch/arm64/kvm/hyp/nvhe/timer-sr.c b/arch/arm64/kvm/hyp/nvhe/timer-sr.c index ff176f4ce7de..51b4f5010b66 100644 --- a/arch/arm64/kvm/hyp/nvhe/timer-sr.c +++ b/arch/arm64/kvm/hyp/nvhe/timer-sr.c @@ -45,11 +45,11 @@ void __timer_enable_traps(struct kvm_vcpu *vcpu) /* * Disallow physical timer access for the guest * Physical counter access is allowed if no offset is enforced - * or running protected (we don't offset anything in this case). + * or running a protected VM (we don't offset anything in this case). */ clr = CNTHCTL_EL1PCEN; - if (is_protected_kvm_enabled() || - !kern_hyp_va(vcpu->kvm)->arch.timer_data.poffset) + if (vcpu_is_protected(vcpu) || + !timer_get_offset(vcpu_ptimer(vcpu))) set |= CNTHCTL_EL1PCTEN; else clr |= CNTHCTL_EL1PCTEN; From 2e813a6e8ebe142a7f40e39cebb924223914c130 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Sat, 8 Aug 2026 08:58:24 +0000 Subject: [PATCH 78/89] KVM: arm64: Fix hvhe and broken CNTVOFF_EL2 When running on a setup affected with broken CNTVOFF_EL2 (has_broken_cntvoff()) Booting with VHE or protected mode(nvhe) (id_aa64mmfr1.vh=0 and arm64_sw.hvhe=0) works fine. However launching a protected VM with protected hvhe mode panics the guest kernel: [ 0.000000] Internal error: Oops - Undefined instruction: 0000000000000000 [#1] SMP [ 0.000000] Modules linked in: [ 0.000000] CPU: 0 UID: 0 PID: 0 Comm: swapper/0 Not tainted 7.2.0-rc3-g05f75bd71e0e-dirty #29 PREEMPT [ 0.000000] Hardware name: linux,dummy-virt (DT) [ 0.000000] pstate: 000003c5 (nzcv DAIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 0.000000] pc : arch_timer_shutdown_virt+0x4/0x1c [ 0.000000] lr : arch_timer_starting_cpu+0x1c4/0x2d4 [ 0.000000] sp : ffffa6bd9a193c00 [ 0.000000] x29: ffffa6bd9a193c20 x28: ffffa6bd9a1bcf88 x27: 0000000000000000 [ 0.000000] x26: ffff00001be70dd8 x25: ffffa6bd99d85000 x24: ffffa6bd99d85ee4 [ 0.000000] x23: ffffa6bd99d85000 x22: ffffa6bd9a1499c0 x21: ffffa6bd9a1ab900 [ 0.000000] x20: 00ffffffffffffff x19: ffff00001be8b600 x18: 000000000000028c [ 0.000000] x17: 00000000510f0010 x16: 00000000510f0010 x15: 00000000500f0000 [ 0.000000] x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000018 [ 0.000000] x11: ffffa6bd9a8ac000 x10: 0000000000f0000f x9 : ffffffffffffffff [ 0.000000] x8 : ffffa6bd98822e18 x7 : 0070752d65746174 x6 : 00111ff76e007261 [ 0.000000] x5 : ffffa6bd9ad68078 x4 : 0000000000000000 x3 : ffffa6bd98822a0c [ 0.000000] x2 : 0000000000000073 x1 : 0000000000000001 x0 : ffff00001be8b600 [ 0.000000] Call trace: [ 0.000000] arch_timer_shutdown_virt+0x4/0x1c (P) [ 0.000000] cpuhp_invoke_callback+0x11c/0x280 [ 0.000000] cpuhp_issue_call+0x1e8/0x224 [ 0.000000] __cpuhp_setup_state_cpuslocked+0x1d8/0x2b8 [ 0.000000] __cpuhp_setup_state+0x50/0x74 [ 0.000000] arch_timer_register+0xc0/0x148 [ 0.000000] arch_timer_of_init+0x148/0x170 [ 0.000000] timer_probe+0x74/0x124 [ 0.000000] time_init+0x18/0x58 [ 0.000000] start_kernel+0x1c0/0x3ac [ 0.000000] __primary_switched+0x88/0x90 [ 0.000000] Code: c80b7d2a 35ffffab 17ffffeb d503245f (d53be328) The workaround avoids setting non-zero CNTVOFF_EL2 and trapping the virtual counter to emulate the offset. In the VHE path (timer_set_traps()), traps are only enabled when the guest actually has a non-zero virtual timer offset. However, __timer_enable_traps() in hyp/nvhe/timer-sr.c unconditionally set CNTHCTL_EL1TVT and CNTHCTL_EL1TVCT whenever has_broken_cntvoff() was true. Which causes 2 issues: 1) Protected VMs: kvm_handle_pvm_sysreg() does not find "cntv_ctl_el0" in pvm_sys_reg_descs and injects undefined instruction exceptions. 2) non-protected guests are trapped all the time even with offset of zero. Fix this by adding a check in __timer_enable_traps() similar to the one in timer_set_traps() Fixes: 0bc9a9e85fcf ("KVM: arm64: Work around x1e's CNTVOFF_EL2 bogosity") Reviewed-by: Yuan Yao Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Mostafa Saleh Link: https://patch.msgid.link/20260808085824.732659-4-smostafa@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/timer-sr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/timer-sr.c b/arch/arm64/kvm/hyp/nvhe/timer-sr.c index 51b4f5010b66..993065716913 100644 --- a/arch/arm64/kvm/hyp/nvhe/timer-sr.c +++ b/arch/arm64/kvm/hyp/nvhe/timer-sr.c @@ -61,9 +61,9 @@ void __timer_enable_traps(struct kvm_vcpu *vcpu) /* * Trap the virtual counter/timer if we have a broken cntvoff - * implementation. + * implementation and non zero offset as in timer_set_traps() */ - if (has_broken_cntvoff()) + if (has_broken_cntvoff() && timer_get_offset(vcpu_vtimer(vcpu))) set |= CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT; sysreg_clear_set(cnthctl_el2, clr, set); From 47746d4b7436ba802f6d7627118649ae108ee24a Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sun, 2 Aug 2026 20:22:22 +0200 Subject: [PATCH 79/89] KVM: arm64: Preserve GPRs for AArch32 CP64 reads generating an UNDEF kvm_handle_cp_64() only seeds params.regval for writes. If a CP64 read is decoded but UNDEFs, emulate_cp() still returns handled and the caller writes params.regval back to Rt/Rt2. This can happen for PMU counter read accesses generating an UNDEF. KVM injects the exception into the guest, so the MRRC GPRs must remain unchanged. Instead, the uninitialised regval is copied into the guest GPRs. With stack auto-initialisation this is a deterministic zero or pattern value. With CONFIG_INIT_STACK_NONE it may be stale host stack data. Match kvm_handle_cp_32() and kvm_handle_sys_reg() by seeding regval from the GPRs before emulation. Fixes: 62a89c44954f0 ("arm64: KVM: 32bit handling of coprocessor traps") Assisted-by: Claude:claude-opus-5 Signed-off-by: Karl Mehltretter Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260802182222.2239-1-kmehltretter@gmail.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/sys_regs.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..ac15b5dc0fe8 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -4861,10 +4861,8 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, * Make a 64-bit value out of Rt and Rt2. As we use the same trap * backends between AArch32 and AArch64, we get away with it. */ - if (params.is_write) { - params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff; - params.regval |= vcpu_get_reg(vcpu, Rt2) << 32; - } + params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff; + params.regval |= vcpu_get_reg(vcpu, Rt2) << 32; /* * If the table contains a handler, handle the From c6d9c8ac6521d3049ec90ac58bebd23ed03ac496 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sun, 26 Jul 2026 19:48:03 +0200 Subject: [PATCH 80/89] KVM: arm64: GICv2: Don't WARN on out-of-range GICV_DIR INTID vgic_v2_deactivate() passes the INTID a guest wrote to GICV_DIR straight to vgic_get_vcpu_irq(), and treats a failed lookup as a "can't happen" condition with WARN_ON_ONCE(). The guest can make it happen at will, though: for any INTID outside of the implemented SGI, PPI and SPI ranges the lookup returns NULL, since GICv2 has no LPIs. A guest running with EOImode==1 writing such an INTID to GICV_DIR triggers the WARN, and panics hosts running with panic_on_warn. Drop the WARN and ignore failed lookups. Fixes: 255de897e7fb ("KVM: arm64: GICv2: Handle deactivation via GICV_DIR traps") Cc: stable@vger.kernel.org Suggested-by: Marc Zyngier Signed-off-by: Karl Mehltretter Link: https://patch.msgid.link/20260726174803.5880-1-kmehltretter@gmail.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-v2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index cafa3cb32bda..7182f63fc938 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -170,8 +170,9 @@ void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val) /* Make sure we're in the same context as LR handling */ local_irq_save(flags); + /* Guest-supplied INTID: out of range yields no irq, so ignore it */ irq = vgic_get_vcpu_irq(vcpu, val); - if (WARN_ON_ONCE(!irq)) + if (!irq) goto out; /* See the corresponding v3 code for the rationale */ From 43347154e7ab642474c886bc54ad090166c0d9c9 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Sun, 2 Aug 2026 16:08:45 +0100 Subject: [PATCH 81/89] KVM: arm64: vgic: Reset in_kernel on private IRQ allocation failure kvm_vgic_create() sets vgic.in_kernel before allocating the per-vCPU private IRQs, but the allocation-failure path resets only vgic_model and leaves in_kernel set. As irqchip_in_kernel() is !!in_kernel, the VM is left with an in-kernel irqchip but no model, and the -EEXIST guard at the top of kvm_vgic_create() rejects every retry, so userspace cannot recover from a transient -ENOMEM. Reset in_kernel alongside vgic_model on the failure path. Fixes: 9435c1e1431003 ("KVM: arm64: gic: Set vgic_model before initing private IRQs") Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Acked-by: Marc Zyngier Link: https://patch.msgid.link/20260802150845.3485757-1-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-init.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index d4cf143f3ae6..0a3df6d3a691 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -176,6 +176,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) } kvm->arch.vgic.vgic_model = 0; + kvm->arch.vgic.in_kernel = false; goto out_unlock; } From 2ad524c5e257ea1054422da26d646dd777c2a08d Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Mon, 3 Aug 2026 10:39:06 +0100 Subject: [PATCH 82/89] KVM: arm64: Drop %pB on nVHE panic when stage-2 is active With pKVM, the host stage-2 is lazily mapped. kallsyms, accessed via the modifier %pB is therefore potentially unmapped. If the hyp panic occurred while the host stage-2 lock was held, handling this fault deadlocks. Skip %pB formatting unless the host stage-2 is disabled. Fixes: 6ccf9cb557bd ("KVM: arm64: Symbolize the nVHE HYP addresses") Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260803093906.3531699-1-vdonnefort@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/handle_exit.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 54aedf93c78b..88dcb8cb36f3 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -507,10 +507,20 @@ void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index) kvm_handle_guest_serror(vcpu, kvm_vcpu_get_esr(vcpu)); } +static bool nvhe_hyp_panic_host_s2_disabled(void) +{ + return !is_protected_kvm_enabled() || + IS_ENABLED(CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC); +} + static void print_nvhe_hyp_panic(const char *name, u64 panic_addr) { - kvm_err("nVHE hyp %s at: [<%016llx>] %pB!\n", name, panic_addr, - (void *)(panic_addr + kaslr_offset())); + /* Kallsyms might not be mapped in the host stage-2 */ + if (nvhe_hyp_panic_host_s2_disabled()) + kvm_err("nVHE hyp %s at: [<%016llx>] %pB!\n", name, panic_addr, + (void *)(panic_addr + kaslr_offset())); + else + kvm_err("nVHE hyp %s at: %016llx!\n", name, panic_addr); } static void kvm_nvhe_report_cfi_failure(u64 panic_addr) @@ -538,8 +548,7 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, unsigned int line = 0; /* All hyp bugs, including warnings, are treated as fatal. */ - if (!is_protected_kvm_enabled() || - IS_ENABLED(CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC)) { + if (nvhe_hyp_panic_host_s2_disabled()) { struct bug_entry *bug = find_bug(elr_in_kimg); if (bug) From a342faadc5acbd5d9fd894fd4499d4fd614dfcf6 Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Mon, 10 Aug 2026 11:29:24 +0100 Subject: [PATCH 83/89] KVM: arm64: vgic: Fix detection of MI on no pending LR As per the ARM GICv3 spec, the maintenance interrupt identified by ICH_MISR_EL2.NP is asserted when it is enabled and no List register is in pending state. This is further described in the document as "no List registers with the State field set to 0b01 (pending)". By checking only the pending bit of the LR (bit 62), KVM currently asserts the MI when there are no LRs in "pending" or "pending and active" states. Fix the detection logic to consider only the "pending" state. Cc: stable@vger.kernel.org Fixes: 96c2f03311de ("KVM: arm64: nv: Plumb handling of GICv3 EL2 accesses") Signed-off-by: Kajetan Puchalski Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260810102923.2426475-2-kajetan.puchalski@arm.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-v3-nested.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic-v3-nested.c b/arch/arm64/kvm/vgic/vgic-v3-nested.c index 5c69fa615823..e3e84d7f5ad5 100644 --- a/arch/arm64/kvm/vgic/vgic-v3-nested.c +++ b/arch/arm64/kvm/vgic/vgic-v3-nested.c @@ -152,7 +152,7 @@ static void vgic_compute_mi_state(struct kvm_vcpu *vcpu, struct mi_state *mi_sta eisr |= BIT(i); if (!(lr & ICH_LR_STATE)) elrsr |= BIT(i); - pend |= (lr & ICH_LR_PENDING_BIT); + pend |= (lr & ICH_LR_STATE) == ICH_LR_PENDING_BIT; } mi_state->eisr = eisr; From 69a598288195947a1662b53de702eb6976af96b7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 10 Aug 2026 18:06:16 +0100 Subject: [PATCH 84/89] KVM: arm64: Correctly cap TLBI Range to the architural limit TLB Invalidation by Range has a fairly powerful way of encoding pretty large ranges in a small number of bits. This range can be based on an arbitrary VA, which means it is pretty easy for a guest to generate an overflow should the hypervisor be naive enough to add the range to the base... Make sure the range is capped to the limit dictated by the address bit that determines the VA range. For an IPA invalidation, this is further corrected down the line to ignore the upper range. Fixes: 4ffa72ad8f37e ("KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2") Reported-by: Wei-Lin Chang Link: https://lore.kernel.org/r/yifz3wn5gk5sr6mapi32trgk5m5kp33bquctsjmkifebnsnndt@fix6u4rthx4g Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260810170616.746100-1-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_nested.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index c83be6d0e79a..1ed708335809 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -305,6 +305,12 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) num = FIELD_GET(GENMASK(43, 39), val); *range = __TLBI_RANGE_PAGES(num, scale) << shift; + /* Cap the range to the correct half of the address space */ + if (!(base & BIT(48))) + *range = min(*range, (BIT(48) - base)); + else + *range = min(*range, ~base + 1); + return base; } From 028d8df0a1e376c6a87409302d9b6131ea4374f6 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 13 Aug 2026 14:17:16 +0100 Subject: [PATCH 85/89] KVM: arm64: Avoid mismatched accesses to 'struct kvm_nvhe_init_params' When running with hVHE enabled, ___kvm_hyp_init() calls __kvm_init_el2_state() on the CPU initialisation path during onlining and resume from suspend. In order to avoid clobbering the link register across this call, it is stashed away in the 'tmp' member of 'struct kvm_nvhe_init_params', however this save/restore operation is performed with the stage-1 MMU disabled at EL2 and therefore gives rise to coherency problems because the field is not aligned or padded to the CWG. For example, a cacheable write to a physically-adjacent structure sharing the same cacheline could lead to an eviction and subsequent write-back, overwriting the saved LR while the incoming CPU is executing __kvm_init_el2_state(). Save the lr in far_el2 and remove the 'tmp' member from 'struct kvm_nvhe_init_params' altogether. Cc: Oliver Upton Cc: Marc Zyngier Fixes: afa9b48f327c ("KVM: arm64: Shave a few bytes from the EL2 idmap code") Signed-off-by: Will Deacon Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260813131717.5885-1-will@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_asm.h | 1 - arch/arm64/kernel/asm-offsets.c | 1 - arch/arm64/kvm/hyp/nvhe/hyp-init.S | 11 +++++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 043495f7fc78..dae9b1d34c8a 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -214,7 +214,6 @@ struct kvm_nvhe_init_params { unsigned long hcr_el2; unsigned long vttbr; unsigned long vtcr; - unsigned long tmp; }; /* diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index b6367ff3a49c..9c853ed3ceab 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -124,7 +124,6 @@ int main(void) DEFINE(NVHE_INIT_HCR_EL2, offsetof(struct kvm_nvhe_init_params, hcr_el2)); DEFINE(NVHE_INIT_VTTBR, offsetof(struct kvm_nvhe_init_params, vttbr)); DEFINE(NVHE_INIT_VTCR, offsetof(struct kvm_nvhe_init_params, vtcr)); - DEFINE(NVHE_INIT_TMP, offsetof(struct kvm_nvhe_init_params, tmp)); #endif #ifdef CONFIG_CPU_PM DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 89cb553be1e5..0b3e0b28dfc7 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -106,16 +106,19 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init) and x2, x1, x2 cbz x2, 1f - // hVHE: Replay the EL2 setup to account for the E2H bit - // TPIDR_EL2 is used to preserve x0 across the macro maze... + /* + * hVHE: Replay the EL2 setup to account for the E2H bit + * TPIDR_EL2 and FAR_EL2 are used to preserve x0 and LR across + * the macro maze... + */ isb msr tpidr_el2, x0 - str lr, [x0, #NVHE_INIT_TMP] + msr far_el2, lr bl __kvm_init_el2_state + mrs lr, far_el2 mrs x0, tpidr_el2 - ldr lr, [x0, #NVHE_INIT_TMP] 1: ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] From d3a1b28ea25d4c1899a05833e1a9bc79d7c642c3 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 11 Aug 2026 15:10:17 +0000 Subject: [PATCH 86/89] KVM: arm64: vgic: Free gic_kvm_info on initialization failure vgic_set_kvm_info() allocates gic_kvm_info for use by kvm_vgic_hyp_init(). When a maintenance interrupt is mandatory but not provided, kvm_vgic_hyp_init() returns -ENXIO before reaching the common cleanup path, leaking said allocation. Route this error through that cleanup path so that gic_kvm_info is freed and the global pointer is cleared. Fixes: 0e5cb7770684 ("irqchip/gic: Split vGIC probing information from the GIC code") Link: https://sashiko.dev/#/patchset/20260724104819.1296803-1-sascha.bischoff@arm.com?part=1 Signed-off-by: Sascha Bischoff Reviewed-by: Joey Gouly Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260811150941.941295-2-sascha.bischoff@arm.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-init.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 0a3df6d3a691..4012df6002ea 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -791,7 +791,8 @@ int kvm_vgic_hyp_init(void) if (has_mask && !gic_kvm_info->maint_irq) { kvm_err("No vgic maintenance irq\n"); - return -ENXIO; + ret = -ENXIO; + goto out_free; } /* @@ -824,6 +825,7 @@ int kvm_vgic_hyp_init(void) kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq; +out_free: kfree(gic_kvm_info); gic_kvm_info = NULL; From 93cfad8da7164b095b2402ec9b2067fa15d6f250 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 11 Aug 2026 15:10:47 +0000 Subject: [PATCH 87/89] KVM: arm64: vgic: Prevent speculative SPI array underflow For a non-GICv5 VM, SPI interrupt IDs include the private-interrupt offset, while KVM's SPI array is indexed from zero. The lookup applies array_index_nospec() to the absolute interrupt ID and subtracts the private-interrupt offset afterwards. On a speculative bypass of the range check for an interrupt ID below the private range, the clamp preserves the small absolute value and the subtraction underflows to an out-of-bounds SPI array index. Convert the interrupt ID to a zero-based index into the SPI array before applying array_index_nospec(). This way, we ensure that we clamp to a reachable SPI ID, rather than an out-of-range SPI index. Fixes: 41b87599c743 ("KVM: arm/arm64: vgic: fix possible spectre-v1 in vgic_get_irq()") Link: https://sashiko.dev/#/patchset/20260724104819.1296803-1-sascha.bischoff@arm.com?part=27 Signed-off-by: Sascha Bischoff Reviewed-by: Joey Gouly Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260811150941.941295-3-sascha.bischoff@arm.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5a4768d8cd4f..3077cfdaa146 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -93,8 +93,9 @@ struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid) /* SPIs */ if (intid >= VGIC_NR_PRIVATE_IRQS && intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) { - intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS); - return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS]; + intid -= VGIC_NR_PRIVATE_IRQS; + intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); + return &kvm->arch.vgic.spis[intid]; } /* LPIs */ From 2962174fdfa63e272082ea7ae989a8fdef29bd0e Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 11 Aug 2026 15:11:18 +0000 Subject: [PATCH 88/89] KVM: arm64: vgic: Reject out-of-range GICv5 PPI IDs GICv5 supports up to 128 PPIs, but KVM currently implements only the first 64, which contain the architected PPIs it supports. An encoded PPI with an ID outside that range passes irq_is_ppi(), which only checks the encoded interrupt type. vgic_get_vcpu_irq() therefore looks it up in private_irqs[], where array_index_nospec() clamps the out-of-range index to zero and aliases PPI 0. Include the supported PPI range in irq_is_ppi() so that KVM interfaces reject unsupported PPIs. Also reject an out-of-range PPI in the lookup as a safeguard against callers bypassing the predicate. Fixes: 4d591252bacb ("KVM: arm64: gic-v5: Implement PPI interrupt injection") Fixes: eb8bce08ecb1 ("KVM: arm64: gic: Introduce interrupt type helpers") Link: https://sashiko.dev/#/patchset/20260724104819.1296803-1-sascha.bischoff@arm.com?part=27 Signed-off-by: Sascha Bischoff Reviewed-by: Joey Gouly Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260811150941.941295-4-sascha.bischoff@arm.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic.c | 2 ++ include/kvm/arm_vgic.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 3077cfdaa146..d0c91f0c5269 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -118,6 +118,8 @@ struct vgic_irq *vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid) switch (type) { case KVM_DEV_TYPE_ARM_VGIC_V5: intid = vgic_v5_get_hwirq_id(intid); + if (intid >= VGIC_V5_NR_PRIVATE_IRQS) + return NULL; intid = array_index_nospec(intid, VGIC_V5_NR_PRIVATE_IRQS); break; default: diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index fe49fb56dc3c..41012457841c 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -65,6 +65,8 @@ switch (t) { \ case KVM_DEV_TYPE_ARM_VGIC_V5: \ __ret = is_v5_type(GICV5_HWIRQ_TYPE_PPI, (i)); \ + __ret &= FIELD_GET(GICV5_HWIRQ_ID, (i)) < \ + VGIC_V5_NR_PRIVATE_IRQS; \ break; \ default: \ __ret = (i) >= VGIC_NR_SGIS; \ From 302f94dc3f8c21c97d5e91a61a8987810803be6f Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 11 Aug 2026 15:11:48 +0000 Subject: [PATCH 89/89] KVM: arm64: Validate GICv5 timer PPIs before claiming ownership With GICv5, timer PPIs use architecturally defined IDs, and therefore we require userspace to use the architected IDs too. We perform this validation in timer_irqs_are_valid(), which validates the configured PPI after claiming it with kvm_vgic_set_owner(). This is obviously the wrong order, and could both result in a NULL pointer dereference for an ID >= 64 and in an incorrect ownership claim. Switch the order to first validate the PPI before setting the owner. This both prevents an invalid GICv5 timer PPI from reaching the IRQ lookup, and avoids claiming an IRQ for a timer configuration that will be rejected. Fixes: 06c85b58e0b1 ("KVM: arm64: Move GICv5 timer PPI validation into timer_irqs_are_valid()") Link: https://sashiko.dev/#/message/20260810111614.DDC731F000E9%40smtp.kernel.org Signed-off-by: Sascha Bischoff Reviewed-by: Joey Gouly Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260811150941.941295-5-sascha.bischoff@arm.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/arch_timer.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 4155fe89b58a..af7e2894cd69 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -1529,13 +1529,14 @@ static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu) ctx = vcpu_get_timer(vcpu, i); irq = timer_irq(ctx); - if (kvm_vgic_set_owner(vcpu, irq, ctx)) - break; /* With GICv5, the default PPI is what you get -- nothing else */ if (vgic_is_v5(vcpu->kvm) && irq != get_vgic_ppi(vcpu->kvm, default_ppi[i])) break; + if (kvm_vgic_set_owner(vcpu, irq, ctx)) + break; + /* * We know by construction that we only have PPIs, so all values * are less than 32 for non-GICv5 VGICs. On GICv5, they are