From 165a50429e9f319e4a3bbc396cbec7c3f336ae5f Mon Sep 17 00:00:00 2001 From: jinrui Date: Thu, 13 Aug 2026 10:29:30 +0000 Subject: [PATCH 01/42] KVM: selftests: riscv: Add lazy V extension enablement for guests When the cross-compiler defaults to an -march that includes the V (vector) extension, -O2 auto-vectorization generates vector instructions (e.g. vsetvli, vadd.vv) in guest code. Executing such an instruction with sstatus.VS Off raises EXC_INST_ILLEGAL (scause=2); KVM's hedeleg forwards it to the guest, but the bare-metal selftest cannot handle it, so all guest tests fail. A real kernel handles this via riscv_v_first_use_handler(), which enables V and re-executes the instruction. Fix it in processor.c: 1. Delete the now-unused guest_unexp_trap() handler, replaced by the full exception vector table. 2. In vm_arch_vcpu_add(), advertise V to KVM via __vcpu_set_reg(V, 1) (best-effort, errors ignored on hardware without V) and install the full exception vector table instead of a raw stvec handler. 3. In route_exception(), decode the faulting instruction (stval) with insn_is_vector() and, when it is a vector instruction while sstatus.VS is Off, set VS to Initial and sret to re-execute it, before any test-registered handler. Genuinely illegal instructions still reach the unexpected-exception path. 4. Make vm_init_vector_tables() idempotent by checking vm->handlers before allocating, so tests that call it directly (ebreak_test, arch_timer, sbi_pmu_test) do not leak memory. Tested on a riscv64 host with KVM enabled. Signed-off-by: jinrui Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20EA6EE00884FB87+20260813102930.3892251-1-jinrui@haiwei.tech Signed-off-by: Anup Patel --- .../selftests/kvm/include/riscv/processor.h | 13 +++ .../selftests/kvm/lib/riscv/processor.c | 99 ++++++++++++++++--- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h index e3acf2ae9881..685baefebdb1 100644 --- a/tools/testing/selftests/kvm/include/riscv/processor.h +++ b/tools/testing/selftests/kvm/include/riscv/processor.h @@ -25,6 +25,19 @@ #define GET_RM(insn) (((insn) & INSN_MASK_FUNCT3) >> INSN_SHIFT_FUNCT3) #define GET_CSR_NUM(insn) (((insn) & INSN_CSR_MASK) >> INSN_CSR_SHIFT) +/* Vector (V) instruction decoding, matching arch/riscv/include/asm/insn.h */ +#define RV_INSN_OPCODE_MASK 0x7f +#define RVG_OPCODE_SYSTEM 0x73 +#define RVV_OPCODE_VECTOR 0x57 +#define RVV_OPCODE_VL 0x07 +#define RVV_OPCODE_VS 0x27 +#define RVV_VL_VS_WIDTH_8 0 +#define RVV_VL_VS_WIDTH_16 5 +#define RVV_VL_VS_WIDTH_32 6 +#define RVV_VL_VS_WIDTH_64 7 +#define RVV_EXTRACT_VL_VS_WIDTH(insn) (((insn) >> 12) & 0x7) +#define RVG_EXTRACT_SYSTEM_CSR(insn) (((insn) >> 20) & 0xfff) + static inline u64 __kvm_reg_id(u64 type, u64 subtype, u64 idx, u64 size) { return KVM_REG_RISCV | type | subtype | idx | size; diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index ded5429f3448..d00ac997291c 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -17,6 +17,11 @@ static gva_t exception_handlers; +struct handlers { + exception_handler_fn exception_handlers[NR_VECTORS][NR_EXCEPTIONS]; + bool v_available; +}; + bool __vcpu_has_ext(struct kvm_vcpu *vcpu, u64 ext) { unsigned long value = 0; @@ -298,13 +303,6 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) core.regs.t3, core.regs.t4, core.regs.t5, core.regs.t6); } -static void __aligned(16) guest_unexp_trap(void) -{ - sbi_ecall(KVM_RISCV_SELFTESTS_SBI_EXT, - KVM_RISCV_SELFTESTS_SBI_UNEXP, - 0, 0, 0, 0, 0, 0); -} - void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) { vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.pc), (unsigned long)guest_code); @@ -348,8 +346,26 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) /* Setup sscratch for guest_get_vcpuid() */ vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(sscratch), vcpu_id); - /* Setup default exception vector of guest */ - vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(stvec), (unsigned long)guest_unexp_trap); + /* + * Advertise V to KVM so -O2 auto-vectorization in guest code is valid; + * ignore errors since the tests work without V too. Use the full + * exception vector table (which lazily enables V in route_exception()) + * as the default handler; vm_init_vector_tables() is idempotent. + */ + __vcpu_set_reg(vcpu, RISCV_ISA_EXT_REG(KVM_RISCV_ISA_EXT_V), 1); + vm_init_vector_tables(vm); + vcpu_init_vector_tables(vcpu); + + /* + * Record V availability for route_exception(), which runs in guest + * context. V is enabled uniformly for every vCPU, so this is a + * VM-wide property. + */ + { + struct handlers *h = addr_gva2hva(vm, vm->handlers); + + h->v_available = __vcpu_has_isa_ext(vcpu, KVM_RISCV_ISA_EXT_V); + } return vcpu; } @@ -408,19 +424,43 @@ void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) struct ucall uc; if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED) { + vcpu_dump(stderr, vcpu, 2); TEST_FAIL("Unexpected exception (vector:0x%lx, ec:0x%lx)", uc.args[0], uc.args[1]); } } -struct handlers { - exception_handler_fn exception_handlers[NR_VECTORS][NR_EXCEPTIONS]; -}; +static bool insn_is_vector(u32 insn) +{ + u32 opcode = insn & RV_INSN_OPCODE_MASK; + u32 width, csr; + + /* All V-related instructions are 4-byte, i.e. not compressed. */ + if ((insn & 0x3) != 0x3) + return false; + + switch (opcode) { + case RVV_OPCODE_VECTOR: + return true; + case RVV_OPCODE_VL: + case RVV_OPCODE_VS: + width = RVV_EXTRACT_VL_VS_WIDTH(insn); + return width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || + width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64; + case RVG_OPCODE_SYSTEM: + csr = RVG_EXTRACT_SYSTEM_CSR(insn); + return (csr >= CSR_VSTART && csr <= CSR_VCSR) || + (csr >= CSR_VL && csr <= CSR_VLENB); + } + + return false; +} void route_exception(struct pt_regs *regs) { struct handlers *handlers = (struct handlers *)exception_handlers; - int vector = 0, ec; + int vector = 0; + unsigned long ec; ec = regs->cause & ~CAUSE_IRQ_FLAG; if (ec >= NR_EXCEPTIONS) @@ -432,6 +472,36 @@ void route_exception(struct pt_regs *regs) ec = 0; } + /* + * Lazily enable V on the first vector instruction: if the faulting + * instruction decodes as vector while VS is off, set VS to Initial + * and re-execute it, like the kernel's riscv_v_first_use_handler(). + * Genuinely illegal instructions continue to the unexpected-exception + * path. + */ + if (!(regs->cause & CAUSE_IRQ_FLAG) && ec == EXC_INST_ILLEGAL && + handlers && handlers->v_available && !(regs->status & SR_VS)) { + u32 insn = (u32)regs->badaddr; + + /* + * stval is not guaranteed to hold the faulting instruction. + * Vector instructions are always 32-bit, so check the first + * 16-bit halfword and, for a 32-bit instruction, assemble it + * from two halfwords to avoid an unaligned 32-bit access. + */ + if (!insn) { + u16 half = *(u16 *)regs->epc; + + if ((half & 0x3) == 0x3) + insn = (u32)half | ((u32)*(u16 *)(regs->epc + 2) << 16); + } + + if (insn_is_vector(insn)) { + regs->status |= SR_VS_INITIAL; + return; + } + } + if (handlers && handlers->exception_handlers[vector][ec]) return handlers->exception_handlers[vector][ec](regs); @@ -448,6 +518,9 @@ void vcpu_init_vector_tables(struct kvm_vcpu *vcpu) void vm_init_vector_tables(struct kvm_vm *vm) { + if (vm->handlers) + return; + vm->handlers = __vm_alloc(vm, sizeof(struct handlers), vm->page_size, MEM_REGION_DATA); From a84c5e0c31462588d2ec0fba17d9380ed9be6fcf Mon Sep 17 00:00:00 2001 From: Jinyu Tang Date: Fri, 14 Aug 2026 08:35:41 -0400 Subject: [PATCH 02/42] KVM: riscv: Avoid redundant interrupt check for pending timers The kvm_vcpu_check_block() calls kvm_arch_vcpu_runnable() before checking whether a timer is pending. On RISC-V, kvm_arch_vcpu_runnable() already checks all pending guest interrupts through kvm_riscv_vcpu_has_interrupts(vcpu, -1UL), including IRQ_VS_TIMER. Drop the repeated IRQ_VS_TIMER pending-interrupt check from kvm_riscv_vcpu_timer_pending() and keep the helper focused on the timer deadline state. If the timer becomes pending after the runnable check, the timer path still injects IRQ_VS_TIMER and kicks the vCPU through the normal wakeup path. Signed-off-by: Jinyu Tang Reviewed-by: Nutty Liu Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260814123541.593613-1-jinyu.tang@linux.dev Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_timer.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c index a2cd277a4059..adbd1617b78a 100644 --- a/arch/riscv/kvm/vcpu_timer.c +++ b/arch/riscv/kvm/vcpu_timer.c @@ -135,11 +135,7 @@ bool kvm_riscv_vcpu_timer_pending(struct kvm_vcpu *vcpu) struct kvm_vcpu_timer *t = &vcpu->arch.timer; struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer; - if (!kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t) || - kvm_riscv_vcpu_has_interrupts(vcpu, 1UL << IRQ_VS_TIMER)) - return true; - else - return false; + return !kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t); } static void kvm_riscv_vcpu_timer_blocking(struct kvm_vcpu *vcpu) From e8b32e3779513e9ceed73f09c3f53db423140193 Mon Sep 17 00:00:00 2001 From: Jinyu Tang Date: Fri, 14 Aug 2026 07:45:05 -0400 Subject: [PATCH 03/42] KVM: riscv: Implement KVM_PRE_FAULT_MEMORY The generic KVM_PRE_FAULT_MEMORY ioctl lets userspace populate KVM page tables before running a vCPU over a GPA range. x86 already supports the ioctl, but RISC-V does not expose the capability and has no arch hook. Add the RISC-V arch hook and reuse the existing G-stage fault mapping path with a read access. Report progress using the G-stage mapping returned by the map path, so the ioctl can advance by the actual leaf size that covers the requested GPA. Retry until a mapping is installed or a signal, VM-dead request, or real error is observed. Signed-off-by: Jinyu Tang Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260814114507.583424-2-jinyu.tang@linux.dev Signed-off-by: Anup Patel --- arch/riscv/kvm/Kconfig | 1 + arch/riscv/kvm/gstage.c | 6 +++--- arch/riscv/kvm/mmu.c | 42 +++++++++++++++++++++++++++++++++++++++++ arch/riscv/kvm/vm.c | 1 + 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index ec2cee0a39e0..8ac209e8ac87 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -28,6 +28,7 @@ config KVM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING + select KVM_GENERIC_PRE_FAULT_MEMORY select KVM_MMIO select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c index e5002cb9cbef..be70607a33ea 100644 --- a/arch/riscv/kvm/gstage.c +++ b/arch/riscv/kvm/gstage.c @@ -216,12 +216,10 @@ int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, pte_t *ptep; int ret; - out_map->addr = gpa; - out_map->level = 0; - ret = gstage_page_size_to_level(gstage, page_size, &out_map->level); if (ret) return ret; + out_map->addr = ALIGN_DOWN(gpa, page_size); /* * A RISC-V implementation can choose to either: @@ -280,6 +278,8 @@ int kvm_riscv_gstage_map_page(struct kvm_gstage *gstage, out_map->level, true); } else if (ALIGN_DOWN(PFN_PHYS(pte_pfn(ptep_get(ptep))), page_size) == hpa) { kvm_riscv_gstage_update_pte_prot(gstage, ptep_level, gpa, ptep, prot); + out_map->level = ptep_level; + out_map->pte = ptep_get(ptep); return 0; } } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 3e955d808743..78a1826c466d 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -754,6 +754,48 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, return ret; } +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, + struct kvm_pre_fault_memory *range) +{ + struct kvm_gstage_mapping out_map = { 0 }; + struct kvm_memory_slot *memslot; + unsigned long map_size; + unsigned long hva; + gpa_t end; + gfn_t gfn; + int ret; + + gfn = gpa_to_gfn(range->gpa); + memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); + if (!memslot) + return -ENOENT; + + hva = gfn_to_hva_memslot_prot(memslot, gfn, NULL); + if (kvm_is_error_hva(hva)) + return -ENOENT; + + for (;;) { + if (signal_pending(current)) + return -EINTR; + + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) + return -EIO; + + cond_resched(); + ret = kvm_riscv_mmu_map(vcpu, memslot, range->gpa, hva, false, &out_map); + if (ret) + return ret; + + if (pte_val(out_map.pte)) + break; + } + + map_size = PAGE_SIZE << (out_map.level * kvm_riscv_gstage_index_bits); + end = ALIGN_DOWN(range->gpa, map_size) + map_size; + + return min_t(u64, range->size, end - range->gpa); +} + int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm) { struct page *pgd_page; diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index a9f083feeb76..58500a19b33b 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -187,6 +187,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_MP_STATE: case KVM_CAP_IMMEDIATE_EXIT: case KVM_CAP_SET_GUEST_DEBUG: + case KVM_CAP_PRE_FAULT_MEMORY: r = 1; break; case KVM_CAP_NR_VCPUS: From 0971048f467ef2f52f8e05ab4488705e27f9024e Mon Sep 17 00:00:00 2001 From: Jinyu Tang Date: Fri, 14 Aug 2026 07:45:06 -0400 Subject: [PATCH 04/42] KVM: selftests: Add RISC-V Sv57 page table indexing RISC-V selftests can create guests with five page-table levels, for example when the selected guest mode is Sv57. The RISC-V page-table walker only had index arrays for levels 0 through 3, so virt_arch_pg_map() indexed past the end of the arrays when level 4 was used. Add the missing L4 index mask and shift so selftests can build guest page tables for Sv57 VMs. Reviewed-by: Nutty Liu Reviewed-by: Anup Patel Signed-off-by: Jinyu Tang Link: https://lore.kernel.org/r/20260814114507.583424-3-jinyu.tang@linux.dev Signed-off-by: Anup Patel --- tools/testing/selftests/kvm/include/riscv/processor.h | 3 +++ tools/testing/selftests/kvm/lib/riscv/processor.c | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h index 685baefebdb1..5d242f063147 100644 --- a/tools/testing/selftests/kvm/include/riscv/processor.h +++ b/tools/testing/selftests/kvm/include/riscv/processor.h @@ -140,6 +140,9 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector, exception_handl void vm_install_interrupt_handler(struct kvm_vm *vm, exception_handler_fn handler); +/* L4 index Bit[56:48] */ +#define PGTBL_L4_INDEX_MASK 0x01FF000000000000ULL +#define PGTBL_L4_INDEX_SHIFT 48 /* L3 index Bit[47:39] */ #define PGTBL_L3_INDEX_MASK 0x0000FF8000000000ULL #define PGTBL_L3_INDEX_SHIFT 39 diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index d00ac997291c..8f0b619fcded 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -48,6 +48,7 @@ static u64 pte_index_mask[] = { PGTBL_L1_INDEX_MASK, PGTBL_L2_INDEX_MASK, PGTBL_L3_INDEX_MASK, + PGTBL_L4_INDEX_MASK, }; static u32 pte_index_shift[] = { @@ -55,6 +56,7 @@ static u32 pte_index_shift[] = { PGTBL_L1_INDEX_SHIFT, PGTBL_L2_INDEX_SHIFT, PGTBL_L3_INDEX_SHIFT, + PGTBL_L4_INDEX_SHIFT, }; static u64 pte_index(struct kvm_vm *vm, gva_t gva, int level) From eb28015d47fd06e55a3bd84cf70d0e4362c0efe6 Mon Sep 17 00:00:00 2001 From: Jinyu Tang Date: Fri, 14 Aug 2026 07:45:07 -0400 Subject: [PATCH 05/42] KVM: selftests: Enable pre_fault_memory_test for RISC-V RISC-V now supports KVM_PRE_FAULT_MEMORY, so include the generic pre_fault_memory_test in the RISC-V KVM selftest build. The test uses PAGE_SIZE from the architecture processor header. Define the normal 4K RISC-V selftest page size so the generic test can build for RISC-V. Reviewed-by: Nutty Liu Reviewed-by: Anup Patel Signed-off-by: Jinyu Tang Link: https://lore.kernel.org/r/20260814114507.583424-4-jinyu.tang@linux.dev Signed-off-by: Anup Patel --- tools/testing/selftests/kvm/Makefile.kvm | 1 + tools/testing/selftests/kvm/include/riscv/processor.h | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 6a1482e3a286..4cfb63854955 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -232,6 +232,7 @@ TEST_GEN_PROGS_riscv += coalesced_io_test TEST_GEN_PROGS_riscv += dirty_log_perf_test TEST_GEN_PROGS_riscv += get-reg-list TEST_GEN_PROGS_riscv += mmu_stress_test +TEST_GEN_PROGS_riscv += pre_fault_memory_test TEST_GEN_PROGS_riscv += rseq_test TEST_GEN_PROGS_riscv += steal_time diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h index 5d242f063147..a5f101039a21 100644 --- a/tools/testing/selftests/kvm/include/riscv/processor.h +++ b/tools/testing/selftests/kvm/include/riscv/processor.h @@ -12,6 +12,9 @@ #include #include "kvm_util.h" +#define PAGE_SHIFT 12 +#define PAGE_SIZE BIT_ULL(PAGE_SHIFT) + #define INSN_OPCODE_MASK 0x007c #define INSN_OPCODE_SHIFT 2 #define INSN_OPCODE_SYSTEM 28 From 3da5289d8d19e5a5398e725ad6186e5686e18737 Mon Sep 17 00:00:00 2001 From: Yong-Xuan Wang Date: Mon, 31 Aug 2026 20:57:31 -0700 Subject: [PATCH 06/42] KVM: riscv: selftests: Dump sepc/scause/stval on vcpu_arch_dump() vcpu_arch_dump() omits trap-related CSRs, making it hard to diagnose unexpected guest exceptions from the dump output alone. Print sepc, scause, and stval alongside the existing register dump. Signed-off-by: Yong-Xuan Wang Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260831-kvm_kselftest_dump-v1-1-55b022c6ef8e@sifive.com Signed-off-by: Anup Patel --- tools/testing/selftests/kvm/lib/riscv/processor.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index 8f0b619fcded..bd753e9a76cd 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -242,6 +242,7 @@ void riscv_vcpu_mmu_setup(struct kvm_vcpu *vcpu) void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) { struct kvm_riscv_core core; + struct kvm_riscv_csr general_csr; core.mode = vcpu_get_reg(vcpu, RISCV_CORE_REG(mode)); core.regs.pc = vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.pc)); @@ -277,6 +278,10 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) core.regs.t5 = vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t5)); core.regs.t6 = vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t6)); + general_csr.sepc = vcpu_get_reg(vcpu, RISCV_GENERAL_CSR_REG(sepc)); + general_csr.scause = vcpu_get_reg(vcpu, RISCV_GENERAL_CSR_REG(scause)); + general_csr.stval = vcpu_get_reg(vcpu, RISCV_GENERAL_CSR_REG(stval)); + fprintf(stream, " MODE: 0x%lx\n", core.mode); fprintf(stream, @@ -303,6 +308,9 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) fprintf(stream, " T3: 0x%016lx T4: 0x%016lx T5: 0x%016lx T6: 0x%016lx\n", core.regs.t3, core.regs.t4, core.regs.t5, core.regs.t6); + fprintf(stream, + "SEPC: 0x%016lx SCAUSE: 0x%016lx STVAL: 0x%016lx\n", + general_csr.sepc, general_csr.scause, general_csr.stval); } void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) From a01f4c27a6ff5133408952aefd6289584a67ca67 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Fri, 25 Sep 2026 18:03:00 +0200 Subject: [PATCH 07/42] RISC-V: KVM: Reject vCPU creation after AIA initialization AIA initialization sets up per-vCPU IMSIC state by iterating over all online vCPUs. Creating another vCPU afterwards leaves it without initialized IMSIC state, but KVM currently permits that sequence. Reject KVM_CREATE_VCPU once AIA initialization succeeds. Both paths are serialized by kvm->lock, so checking the initialized state from kvm_arch_vcpu_precreate() freezes the vCPU set used by AIA. Fixes: 89d01306e34d ("RISC-V: KVM: Implement device interface for AIA irqchip") Signed-off-by: Andrew Jones Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260925160300.467884-1-andrew.jones@oss.qualcomm.com Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 7f1636c44d41..dd786f758eb7 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -138,6 +138,9 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) { + if (kvm_riscv_aia_initialized(kvm)) + return -EBUSY; + return 0; } From 2ddd9c19780f60a4251f8bfd447efe811e62811b Mon Sep 17 00:00:00 2001 From: Zhanpeng Zhang Date: Tue, 22 Sep 2026 10:18:24 +0800 Subject: [PATCH 08/42] RISC-V: KVM: Fix build with vector support disabled The kernel-mode vector context handling added unconditional calls to get_cpu_vector_context(), put_cpu_vector_context() and __kvm_riscv_vector_restore() in vcpu.c. With CONFIG_RISCV_ISA_V=n, these calls fail to build due to missing declarations. The get/put helpers already have no-op definitions in asm/vector.h, but kvm_vcpu_vector.h only includes that header when CONFIG_RISCV_ISA_V is enabled. Include it unconditionally and provide a no-op implementation of __kvm_riscv_vector_restore() for the disabled configuration, matching the other vector context helpers. The paths that set RISCV_V_VCPU_NEED_RESTORE are compiled out when vector support is disabled, so the restore stub does not discard any live vector state. Keep it __always_inline for its noinstr caller. The vector enabled implementation is unchanged. Fixes: b5060a4aa33d ("RISC-V: KVM: fix vcpu vector context handling for kernel-mode vector") Signed-off-by: Zhanpeng Zhang Reviewed-by: Charlie Jenkins Tested-by: Charlie Jenkins Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260922021823.44251-2-zhangzhanpeng.jasper@bytedance.com Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_vcpu_vector.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/kvm_vcpu_vector.h b/arch/riscv/include/asm/kvm_vcpu_vector.h index 6371d5ea5392..50d4789b9cac 100644 --- a/arch/riscv/include/asm/kvm_vcpu_vector.h +++ b/arch/riscv/include/asm/kvm_vcpu_vector.h @@ -11,9 +11,9 @@ #define __KVM_VCPU_RISCV_VECTOR_H #include +#include #ifdef CONFIG_RISCV_ISA_V -#include #include static __always_inline void __kvm_riscv_vector_save(struct kvm_cpu_context *context) @@ -55,6 +55,10 @@ static inline void kvm_riscv_v_exit(void) struct kvm_cpu_context; +static __always_inline void __kvm_riscv_vector_restore(struct kvm_cpu_context *context) +{ +} + static inline void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) { } From c367d3bf721f2bc391d6cfb47f6884e99d563df0 Mon Sep 17 00:00:00 2001 From: Xiaofeng Yuan Date: Thu, 17 Sep 2026 20:20:12 +0800 Subject: [PATCH 09/42] RISC-V: KVM: Fix IPI delivery for out-of-order vcpu_id The SBI IPI handler walks vCPUs with kvm_for_each_vcpu(), which iterates by vcpu_idx (creation order) rather than vcpu_id order. Since vcpu_id can be assigned out of order by userspace, a vCPU whose hart_bit falls outside the XLEN-bit hart_mask range may be reached before vCPUs the mask actually targets. In that case the handler jumps to "done" and stops sending IPIs, leaving valid target vCPUs without an interrupt. Replace the early "goto done" with "continue" so vCPUs outside the hart_mask range are skipped without aborting the loop. Reproduced with a minimal userspace VMM driving KVM inside a QEMU (RISC-V virt) guest: three vCPUs are created with ids 100, 0, 1 (in creation order) and sbi_send_ipi(hart_mask=bit0, hbase=0) is called from vcpu_id 1. Before this change the IPI is dropped and the ecall returns SBI_ERR_INVALID_PARAM; with this change the IPI is delivered and the ecall returns SBI_SUCCESS. Fixes: 0611f78f83c9 ("riscv: KVM: Fix SBI IPI error generation") Cc: stable@vger.kernel.org Suggested-by: Andrew Jones Signed-off-by: Xiaofeng Yuan Reviewed-by: Nutty Liu Reviewed-by: Troy Mitchell Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260917122012.2577-1-yuanxiaofeng@eswincomputing.com Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_sbi_replace.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kvm/vcpu_sbi_replace.c b/arch/riscv/kvm/vcpu_sbi_replace.c index 506a510b6bff..c2f1f7f26442 100644 --- a/arch/riscv/kvm/vcpu_sbi_replace.c +++ b/arch/riscv/kvm/vcpu_sbi_replace.c @@ -64,8 +64,17 @@ static int kvm_sbi_ext_ipi_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, if (tmp->vcpu_id < hbase) continue; hart_bit = tmp->vcpu_id - hbase; + /* + * kvm_for_each_vcpu() walks kvm->vcpus[] by + * vcpu_idx, i.e. the creation order, which has + * nothing to do with the vcpu_id (hart id) space + * that the SBI IPI operates on. vcpu_ids need not + * increase along the iteration, so harts outside + * the hart_mask window must be skipped instead of + * aborting the loop. + */ if (hart_bit >= __riscv_xlen) - goto done; + continue; if (!(hmask & (1UL << hart_bit))) continue; } @@ -76,7 +85,6 @@ static int kvm_sbi_ext_ipi_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, kvm_riscv_vcpu_pmu_incr_fw(tmp, SBI_PMU_FW_IPI_RCVD); } -done: if (hbase != -1UL && (hmask ^ sentmask)) retdata->err_val = SBI_ERR_INVALID_PARAM; From bae99073549aba7c3d79685a4640a46aa5c55340 Mon Sep 17 00:00:00 2001 From: Xiaofeng Yuan Date: Thu, 17 Sep 2026 20:14:11 +0800 Subject: [PATCH 10/42] RISC-V: KVM: Replace imsic switch-case macros with range checks The imsic_eix_read/swap/write/set accessors use a family of macros that expand into a switch with 128 cases covering the EIP and EIE register windows. This mirrors the csr_read_num() idiom required where a CSR number must be encoded as a csrr instruction immediate (e.g. drivers/perf/riscv_pmu.c), but here the register number is merely written to vsiselect as data, so the expansion solves no real constraint. It costs ~90 lines of nested macros and trips ~30 checkpatch errors and warnings. Since the generated cases form two contiguous windows, [IMSIC_EIP0, IMSIC_EIP63] and [IMSIC_EIE0, IMSIC_EIE63], the same behavior is expressed with a simple range check. Introduce imsic_eix_ireg_is_valid() and use it in the four accessors, removing the macros along with the ~30 checkpatch errors and warnings they caused. No functional change intended. Compile tested with riscv64-linux-gnu-gcc and tested on QEMU (RISC-V virt, AIA). Signed-off-by: Xiaofeng Yuan Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260917121411.2445-1-yuanxiaofeng@eswincomputing.com Signed-off-by: Anup Patel --- arch/riscv/kvm/aia_imsic.c | 124 +++++-------------------------------- 1 file changed, 15 insertions(+), 109 deletions(-) diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index f17dc559a709..984b2212328a 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -67,36 +67,17 @@ struct imsic { __r; \ }) -#define imsic_read_switchcase(__ireg) \ - case __ireg: \ - return imsic_vs_csr_read(__ireg); -#define imsic_read_switchcase_2(__ireg) \ - imsic_read_switchcase(__ireg + 0) \ - imsic_read_switchcase(__ireg + 1) -#define imsic_read_switchcase_4(__ireg) \ - imsic_read_switchcase_2(__ireg + 0) \ - imsic_read_switchcase_2(__ireg + 2) -#define imsic_read_switchcase_8(__ireg) \ - imsic_read_switchcase_4(__ireg + 0) \ - imsic_read_switchcase_4(__ireg + 4) -#define imsic_read_switchcase_16(__ireg) \ - imsic_read_switchcase_8(__ireg + 0) \ - imsic_read_switchcase_8(__ireg + 8) -#define imsic_read_switchcase_32(__ireg) \ - imsic_read_switchcase_16(__ireg + 0) \ - imsic_read_switchcase_16(__ireg + 16) -#define imsic_read_switchcase_64(__ireg) \ - imsic_read_switchcase_32(__ireg + 0) \ - imsic_read_switchcase_32(__ireg + 32) +static inline bool imsic_eix_ireg_is_valid(int ireg) +{ + return (ireg >= IMSIC_EIP0 && ireg <= IMSIC_EIP63) || + (ireg >= IMSIC_EIE0 && ireg <= IMSIC_EIE63); +} static unsigned long imsic_eix_read(int ireg) { - switch (ireg) { - imsic_read_switchcase_64(IMSIC_EIP0) - imsic_read_switchcase_64(IMSIC_EIE0) - } - - return 0; + if (!imsic_eix_ireg_is_valid(ireg)) + return 0; + return imsic_vs_csr_read(ireg); } #define imsic_vs_csr_swap(__c, __v) \ @@ -107,36 +88,11 @@ static unsigned long imsic_eix_read(int ireg) __r; \ }) -#define imsic_swap_switchcase(__ireg, __v) \ - case __ireg: \ - return imsic_vs_csr_swap(__ireg, __v); -#define imsic_swap_switchcase_2(__ireg, __v) \ - imsic_swap_switchcase(__ireg + 0, __v) \ - imsic_swap_switchcase(__ireg + 1, __v) -#define imsic_swap_switchcase_4(__ireg, __v) \ - imsic_swap_switchcase_2(__ireg + 0, __v) \ - imsic_swap_switchcase_2(__ireg + 2, __v) -#define imsic_swap_switchcase_8(__ireg, __v) \ - imsic_swap_switchcase_4(__ireg + 0, __v) \ - imsic_swap_switchcase_4(__ireg + 4, __v) -#define imsic_swap_switchcase_16(__ireg, __v) \ - imsic_swap_switchcase_8(__ireg + 0, __v) \ - imsic_swap_switchcase_8(__ireg + 8, __v) -#define imsic_swap_switchcase_32(__ireg, __v) \ - imsic_swap_switchcase_16(__ireg + 0, __v) \ - imsic_swap_switchcase_16(__ireg + 16, __v) -#define imsic_swap_switchcase_64(__ireg, __v) \ - imsic_swap_switchcase_32(__ireg + 0, __v) \ - imsic_swap_switchcase_32(__ireg + 32, __v) - static unsigned long imsic_eix_swap(int ireg, unsigned long val) { - switch (ireg) { - imsic_swap_switchcase_64(IMSIC_EIP0, val) - imsic_swap_switchcase_64(IMSIC_EIE0, val) - } - - return 0; + if (!imsic_eix_ireg_is_valid(ireg)) + return 0; + return imsic_vs_csr_swap(ireg, val); } #define imsic_vs_csr_write(__c, __v) \ @@ -145,35 +101,10 @@ do { \ csr_write(CSR_VSIREG, __v); \ } while (0) -#define imsic_write_switchcase(__ireg, __v) \ - case __ireg: \ - imsic_vs_csr_write(__ireg, __v); \ - break; -#define imsic_write_switchcase_2(__ireg, __v) \ - imsic_write_switchcase(__ireg + 0, __v) \ - imsic_write_switchcase(__ireg + 1, __v) -#define imsic_write_switchcase_4(__ireg, __v) \ - imsic_write_switchcase_2(__ireg + 0, __v) \ - imsic_write_switchcase_2(__ireg + 2, __v) -#define imsic_write_switchcase_8(__ireg, __v) \ - imsic_write_switchcase_4(__ireg + 0, __v) \ - imsic_write_switchcase_4(__ireg + 4, __v) -#define imsic_write_switchcase_16(__ireg, __v) \ - imsic_write_switchcase_8(__ireg + 0, __v) \ - imsic_write_switchcase_8(__ireg + 8, __v) -#define imsic_write_switchcase_32(__ireg, __v) \ - imsic_write_switchcase_16(__ireg + 0, __v) \ - imsic_write_switchcase_16(__ireg + 16, __v) -#define imsic_write_switchcase_64(__ireg, __v) \ - imsic_write_switchcase_32(__ireg + 0, __v) \ - imsic_write_switchcase_32(__ireg + 32, __v) - static void imsic_eix_write(int ireg, unsigned long val) { - switch (ireg) { - imsic_write_switchcase_64(IMSIC_EIP0, val) - imsic_write_switchcase_64(IMSIC_EIE0, val) - } + if (imsic_eix_ireg_is_valid(ireg)) + imsic_vs_csr_write(ireg, val); } #define imsic_vs_csr_set(__c, __v) \ @@ -182,35 +113,10 @@ do { \ csr_set(CSR_VSIREG, __v); \ } while (0) -#define imsic_set_switchcase(__ireg, __v) \ - case __ireg: \ - imsic_vs_csr_set(__ireg, __v); \ - break; -#define imsic_set_switchcase_2(__ireg, __v) \ - imsic_set_switchcase(__ireg + 0, __v) \ - imsic_set_switchcase(__ireg + 1, __v) -#define imsic_set_switchcase_4(__ireg, __v) \ - imsic_set_switchcase_2(__ireg + 0, __v) \ - imsic_set_switchcase_2(__ireg + 2, __v) -#define imsic_set_switchcase_8(__ireg, __v) \ - imsic_set_switchcase_4(__ireg + 0, __v) \ - imsic_set_switchcase_4(__ireg + 4, __v) -#define imsic_set_switchcase_16(__ireg, __v) \ - imsic_set_switchcase_8(__ireg + 0, __v) \ - imsic_set_switchcase_8(__ireg + 8, __v) -#define imsic_set_switchcase_32(__ireg, __v) \ - imsic_set_switchcase_16(__ireg + 0, __v) \ - imsic_set_switchcase_16(__ireg + 16, __v) -#define imsic_set_switchcase_64(__ireg, __v) \ - imsic_set_switchcase_32(__ireg + 0, __v) \ - imsic_set_switchcase_32(__ireg + 32, __v) - static void imsic_eix_set(int ireg, unsigned long val) { - switch (ireg) { - imsic_set_switchcase_64(IMSIC_EIP0, val) - imsic_set_switchcase_64(IMSIC_EIE0, val) - } + if (imsic_eix_ireg_is_valid(ireg)) + imsic_vs_csr_set(ireg, val); } static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif, From 21d3b33234ebbb675c2a2a0fa7b12d2d83953d7a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Oct 2026 11:05:33 +0100 Subject: [PATCH 11/42] KVM: arm64: nv: Move L2->L1 switch on interrupt early NV request generation to switch from L2 to L1 on interrupt is buried into the vgic flush code, while we should be able to deliver requests as early as possible. Similarly, MI generation happens *after* the need for a request has been evaluated, which means we potentially delay the maintenance interrupt by a full exit/entry cycle. Move these two things to happen much earlier in the entry code, and make sure the MI generation and PMU interrupt serve as input to the request switching back from L2 to L1. Cc: stable@vger.kernel.org Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261015100539.1011760-2-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 4 ++-- arch/arm64/kvm/nested.c | 8 ++++++++ arch/arm64/kvm/vgic/vgic.c | 8 +------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 0576c2022ef5..73706464d265 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1314,11 +1314,11 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) */ preempt_disable(); - kvm_nested_flush_hwstate(vcpu); - if (kvm_vcpu_has_pmu(vcpu)) kvm_pmu_flush_hwstate(vcpu); + kvm_nested_flush_hwstate(vcpu); + local_irq_disable(); kvm_vgic_flush_hwstate(vcpu); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index b191365d97cc..2fac0f267812 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -2046,6 +2046,9 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) * * What follows is some slop to (1) react to exception routing / masking and (2) * preserve the pending SError state across translation regimes. + * + * Also generate the maintenance interrupt state, and finally check that if + * we need to take an interrupt targetting the guest hypervisor. */ void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu) { @@ -2054,6 +2057,11 @@ void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu) if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING))) kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu)); + + vgic_v3_nested_update_mi(vcpu); + + if (vgic_state_is_nested(vcpu) && kvm_vgic_vcpu_pending_irq(vcpu)) + kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu); } void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5cf5a1ef86cd..7061b4ab206c 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -1152,7 +1152,7 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) * * We do that by placing a request to ourselves which will * abort the entry procedure and inject the exception at the - * beginning of the run loop. + * beginning of the run loop (see kvm_nested_flush_hwstate()). * * - Otherwise, do exactly *NOTHING* apart from enabling the virtual * CPU interface. The guest state is already loaded, and we can @@ -1162,16 +1162,10 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) * maintenance interrupt state, as it may fire. */ if (vgic_state_is_nested(vcpu)) { - if (kvm_vgic_vcpu_pending_irq(vcpu)) - kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu); - vgic_v3_flush_nested(vcpu); return; } - if (vcpu_has_nv(vcpu)) - vgic_v3_nested_update_mi(vcpu); - DEBUG_SPINLOCK_BUG_ON(!irqs_disabled()); vgic_flush_state(vcpu); From 1d4ce2901f1e25b06d447c0b02d105d6dd3550d8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Oct 2026 11:05:34 +0100 Subject: [PATCH 12/42] KVM: arm64: Move OUTSIDE_GUEST_MODE publication past context being saved OUTSIDE_GUEST_MODE indicates to the rest of KVM that the vcpu has exited. So far, this is set as soon as we return from the notional hypervisor (which with VHE is just a function call away). However, code that uses OUTSIDE_GUEST_MODE as a synchronisation point relies on the vcpu state to have been written back to the in-memory structures. Clearly, this is not what is happening. Move the publication of OUTSIDE_GUEST_MODE to the point where the state is actually written, and give this write release semantics to ensure the correct ordering. Cc: stable@vger.kernel.org Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261015100539.1011760-3-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 73706464d265..7ce9f3bfa7ff 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1334,13 +1334,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) smp_store_mb(vcpu->mode, IN_GUEST_MODE); if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) { - vcpu->mode = OUTSIDE_GUEST_MODE; isb(); /* Ensure work in x_flush_hwstate is committed */ if (kvm_vcpu_has_pmu(vcpu)) kvm_pmu_sync_hwstate(vcpu); if (unlikely(!irqchip_in_kernel(vcpu->kvm))) kvm_timer_sync_user(vcpu); kvm_vgic_sync_hwstate(vcpu); + smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); local_irq_enable(); preempt_enable(); continue; @@ -1356,7 +1356,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) ret = kvm_arm_vcpu_enter_exit(vcpu); - vcpu->mode = OUTSIDE_GUEST_MODE; vcpu->stat.exits++; /* * Back from guest @@ -1390,6 +1389,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_arch_vcpu_ctxsync_fp(vcpu); + /* + * The state that was pushed into the CPU on entering + * the guest has been synchronised back, let's + * advertise we're outside of the guest. + */ + smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); + /* * We must ensure that any pending interrupts are taken before * we exit guest timing so that timer ticks are accounted as From 22b6557ad05bf05c73246f62c19af3e04017270e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Oct 2026 11:05:35 +0100 Subject: [PATCH 13/42] KVM: arm64: Move IN_GUEST_MODE ahead of state being loaded IN_GUEST_MODE is advertised too late, when state has already been moved onto CPU registers and cannot be observed by another CPU anymore. Move the IN_GUEST_MODE publication ahead of the state being loaded, and immediately check for pending request. This ensures that there is no window of opportunity for another CPU to observe OUTSIDE_GUEST_MODE and yet for state to be loaded. This also simplifies the "state rollback", which solely amounts to synchronising the timer when no irqchip is present. Cc: stable@vger.kernel.org Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261015100539.1011760-4-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/arm.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 7ce9f3bfa7ff..cadc82ba5a5a 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1321,31 +1321,33 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) local_irq_disable(); - kvm_vgic_flush_hwstate(vcpu); - - kvm_pmu_update_vcpu_events(vcpu); - /* * Ensure we set mode to IN_GUEST_MODE after we disable * interrupts and before the final VCPU requests check. * See the comment in kvm_vcpu_exiting_guest_mode() and * Documentation/virt/kvm/vcpu-requests.rst + * + * Requests to self must be posted before this point, or + * won't be taken into account until the next exit. No state + * loading other than what happens at vcpu_load() time can + * be done before we have checked for pending requests. */ smp_store_mb(vcpu->mode, IN_GUEST_MODE); if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) { - isb(); /* Ensure work in x_flush_hwstate is committed */ - if (kvm_vcpu_has_pmu(vcpu)) - kvm_pmu_sync_hwstate(vcpu); if (unlikely(!irqchip_in_kernel(vcpu->kvm))) kvm_timer_sync_user(vcpu); - kvm_vgic_sync_hwstate(vcpu); smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); local_irq_enable(); preempt_enable(); continue; } + /* From here onwards, we are commited to entering the guest */ + kvm_vgic_flush_hwstate(vcpu); + + kvm_pmu_update_vcpu_events(vcpu); + kvm_arch_vcpu_ctxflush_fp(vcpu); /************************************************************** From e64309198eda29415a95113def92aa40196cd422 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Oct 2026 11:05:36 +0100 Subject: [PATCH 14/42] KVM: arm64: Turn vcpu->arch.pause into a counter We have situations where we can call kvm_arm_halt_guest() without holding config_lock, which means that halt and resume can overlap in funny ways when called from separate contexts, and result in situations where a vcpu is resumed while other parts of KVM assume it is halted. Consequences are left to the imagination of the reader. Fix this sorry situation by turning kvm_vcpu_arch::pause into an atomic counter, which makes the races described above harmless. Fixes: 3b92830ad41b2 ("KVM: arm/arm64: implement kvm_arm_[halt,resume]_guest") Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20261015100539.1011760-5-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 7 +++---- arch/arm64/kvm/arm.c | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index cd9b9d2462f9..9c068f27f9b8 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -889,11 +889,10 @@ struct kvm_vcpu_arch { /* * Don't run the guest (internal implementation need). * - * Contrary to the flags above, this is set/cleared outside of - * a vcpu context, and thus cannot be mixed with the flags - * themselves (or the flag accesses need to be made atomic). + * Contrary to the flags above, this is updated outside of + * a vcpu context, and thus cannot be mixed with the flags. */ - bool pause; + atomic_t pause; /* * We maintain more than a single set of debug registers to support diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index cadc82ba5a5a..9b773ab0b51d 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -565,6 +565,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kvm_arm_pvtime_vcpu_init(&vcpu->arch); vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; + atomic_set(&vcpu->arch.pause, 0); /* * This vCPU may have been created after mpidr_data was initialized. @@ -839,6 +840,11 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, return ret; } +static bool vcpu_can_run(struct kvm_vcpu *vcpu) +{ + return !kvm_arm_vcpu_stopped(vcpu) && !atomic_read(&vcpu->arch.pause); +} + /** * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled * @v: The VCPU pointer @@ -854,8 +860,7 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) (kvm_timer_should_notify_user(v) || kvm_pmu_should_notify_user(v))); - return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) - && !kvm_arm_vcpu_stopped(v) && !v->arch.pause); + return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) && vcpu_can_run(v)); } bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) @@ -1018,7 +1023,7 @@ void kvm_arm_halt_guest(struct kvm *kvm) struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) - vcpu->arch.pause = true; + atomic_inc(&vcpu->arch.pause); kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP); } @@ -1028,8 +1033,8 @@ void kvm_arm_resume_guest(struct kvm *kvm) struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) { - vcpu->arch.pause = false; - __kvm_vcpu_wake_up(vcpu); + if (atomic_dec_and_test(&vcpu->arch.pause)) + __kvm_vcpu_wake_up(vcpu); } } @@ -1037,11 +1042,9 @@ static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu) { struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); - rcuwait_wait_event(wait, - (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause), - TASK_INTERRUPTIBLE); + rcuwait_wait_event(wait, vcpu_can_run(vcpu), TASK_INTERRUPTIBLE); - if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) { + if (!vcpu_can_run(vcpu)) { /* Awaken to handle a signal, request we sleep again later. */ kvm_make_request(KVM_REQ_SLEEP, vcpu); } From 134b6486dd3452d3ac2642b152e8149e7891d962 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Thu, 15 Oct 2026 11:05:37 +0100 Subject: [PATCH 15/42] KVM: arm64: Add helpers to halt a vCPU Currently we pause the entire VM when modifying VGIC state that requires state to be synchronized back from hardware. This is perfectly fine for heavyweight operations but not ideal for 'simple' things like migrating an active/pending IRQ. Add helpers for halting a single vCPU such that they can be used to manipulate the AP list remotely. [maz: fixed to use kvm_make_vcpus_request_mask(), guaranteeing that the caller waits for the target vcpu to handle the IPI] Signed-off-by: Oliver Upton Link: https://patch.msgid.link/20260929212925.31775-2-oupton@kernel.org Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261015100539.1011760-6-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/kvm/arm.c | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 9c068f27f9b8..7b4f7b6d94b8 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1254,6 +1254,8 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, void kvm_arm_halt_guest(struct kvm *kvm); void kvm_arm_resume_guest(struct kvm *kvm); +void kvm_pause_vcpu(struct kvm_vcpu *vcpu); +void kvm_resume_vcpu(struct kvm_vcpu *vcpu); #define vcpu_has_run_once(vcpu) (!!READ_ONCE((vcpu)->pid)) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 9b773ab0b51d..b1ed5e204a6f 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1017,6 +1017,21 @@ bool kvm_arch_intc_initialized(struct kvm *kvm) return vgic_initialized(kvm); } +void kvm_pause_vcpu(struct kvm_vcpu *vcpu) +{ + DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS) = { }; + + atomic_inc(&vcpu->arch.pause); + __set_bit(vcpu->vcpu_idx, vcpu_bitmap); + kvm_make_vcpus_request_mask(vcpu->kvm, KVM_REQ_SLEEP, vcpu_bitmap); +} + +void kvm_resume_vcpu(struct kvm_vcpu *vcpu) +{ + if (atomic_dec_and_test(&vcpu->arch.pause)) + __kvm_vcpu_wake_up(vcpu); +} + void kvm_arm_halt_guest(struct kvm *kvm) { unsigned long i; @@ -1032,10 +1047,8 @@ void kvm_arm_resume_guest(struct kvm *kvm) unsigned long i; struct kvm_vcpu *vcpu; - kvm_for_each_vcpu(i, vcpu, kvm) { - if (atomic_dec_and_test(&vcpu->arch.pause)) - __kvm_vcpu_wake_up(vcpu); - } + kvm_for_each_vcpu(i, vcpu, kvm) + kvm_resume_vcpu(vcpu); } static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu) From a1b4bd2c2c262b8094c86afb99364d16ac08cb66 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Oct 2026 11:05:38 +0100 Subject: [PATCH 16/42] KVM: arm64: vgic: Stop the target vcpu when disabling LPIs Disabling LPIs is pretty nasty, as it directly messes with the AP list of the affected CPU, which could be running... This has the potential to lead to really bad behaviours, and we should not be doing that. Take example on the way the Active state is handled and simply pause the the vcpu so that we are sure it is in a quiescent state, and the state be safely manipulated. Nobody has any expectation of performance for this anyway. Fixes: 96085b949672d ("KVM: arm/arm64: vgic-v3: Retire pending interrupts on disabling LPIs") Co-developed-by: Oliver Upton Signed-off-by: Oliver Upton Cc: stable@vger.kernel.org Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261015100539.1011760-7-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 5913a20d8301..f5bd29fbe3a9 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -303,9 +303,19 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, if (ctlr != GICR_CTLR_ENABLE_LPIS) return; + /* + * Yes, disabling LPIs is painful, since it can be done from + * a *remote* vcpu! So let's not take any chance, and make + * sure that the target vCPU has written their LRs back to the + * irq structures, and release any reference they would have. + * + * If it hurts, don't do it. + */ + kvm_pause_vcpu(vcpu); vgic_flush_pending_lpis(vcpu); vgic_its_invalidate_all_caches(vcpu->kvm); atomic_set_release(&vgic_cpu->ctlr, 0); + kvm_resume_vcpu(vcpu); } else { ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0, GICR_CTLR_ENABLE_LPIS); From b31bce255a009b17c26bcada1f23c15ebc3fbff1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Oct 2026 11:05:39 +0100 Subject: [PATCH 17/42] KVM: arm64: vgic-its: Stop the VM when handling MOVALL MOVALL is a royal pain, as it requires us to iterate over all the LPIs and move them around if they are on the correct source vcpu, and requiring locking for each of them. This locking can result in contention for vcpus that enter and exit the guest, and hinders forward progress. In order to make sure such process is not contended, just stop the VM altogether. Nobody in their right mind uses MOVALL anyway, as this is a rather braindead aspect of the GICv3 architecture. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261015100539.1011760-8-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-its.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index c10a11373e10..a9949ab4e413 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -1382,6 +1382,15 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, if (vcpu1 == vcpu2) return 0; + /* + * Bulk operations such as MOVALL are a pain, as they can clash + * badly locking-wise with other vcpus entering and exiting the + * guest, should they be affected by it. Stopping the guest is a + * safer bet to ensure uncontended access and ultimately forward + * progress. Yeah... + */ + kvm_arm_halt_guest(kvm); + xa_for_each(&dist->lpi_xa, intid, irq) { irq = vgic_get_irq(kvm, intid); if (!irq) @@ -1394,6 +1403,8 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, vgic_its_invalidate_cache(its); + kvm_arm_resume_guest(kvm); + return 0; } From de81b28f0b83b22163d7d9a9b708d7aa912e0ef7 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 13 Oct 2026 09:16:45 +0100 Subject: [PATCH 18/42] KVM: arm64: vgic-its: Don't map an LPI from more than one event KVM keeps one vgic_irq per LPI. A MAPTI of an LPI whose vgic_irq already exists reuses it for the new ITE and leaves its target vCPU as it was. Mapping one LPI from two events is UNPREDICTABLE (IHI 0069H.b, 5.2.10), but nothing stops a guest from doing it. If the first MAPTI names a collection the guest never maps and the second a mapped one, the LPI translates through the second event with a NULL target vCPU. When the VMM routes an assigned device's MSI to that event on a host with GICv4 enabled, kvm_vgic_v4_set_forwarding() takes the vPE from the NULL target vCPU and the host oopses: Unable to handle kernel paging request at virtual address 0000000000001980 [...] Call trace: its_build_vmapti_cmd (drivers/irqchip/irq-gic-v3-its.c:943) (P) its_irq_set_vcpu_affinity (drivers/irqchip/irq-gic-v3-its.c:1380) irq_set_vcpu_affinity (kernel/irq/manage.c:670) its_map_vlpi (drivers/irqchip/irq-gic-v4.c:326) kvm_vgic_v4_set_forwarding (arch/arm64/kvm/vgic/vgic-v4.c:485) kvm_arch_irq_bypass_add_producer (arch/arm64/kvm/arm.c:2977) irq_bypass_register_consumer (virt/lib/irqbypass.c:44) kvm_irqfd (arch/arm64/kvm/../../../virt/kvm/eventfd.c:496) kvm_vm_ioctl (arch/arm64/kvm/../../../virt/kvm/kvm_main.c:5270) Track in each vgic_irq whether an ITE maps it. Drop a MAPTI of an LPI that is already mapped, as a failed MAPTI already is. Restoring the ITS tables keeps the first ITE it reads for each LPI and drops any later one, so a guest saved with a double mapping still restores. Key the check on the mapping, not on the vgic_irq. A vgic_irq outlives its ITE while anything holds a reference to it, such as the vCPU it was last queued on. A later MAPTI of such an LPI succeeds as before, and now retargets it to its collection. Fixes: 196b136498b35 ("KVM: arm/arm64: GICv4: Wire mapping/unmapping of VLPIs in VFIO irq bypass") Suggested-by: Marc Zyngier Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20261013081646.933312-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-its.c | 26 ++++++++++++++++++++++++-- include/kvm/arm_vgic.h | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index a9949ab4e413..73d7aa700687 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -776,6 +776,7 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite) its_unmap_vlpi(ite->irq->host_irq); irq->hw = false; + irq->mapped = false; } vgic_put_irq(kvm, ite->irq); @@ -784,6 +785,23 @@ static void its_free_ite(struct kvm *kvm, struct its_ite *ite) kfree(ite); } +/* Mapping one LPI from two events is UNPREDICTABLE: the first ITE keeps it. */ +static bool vgic_its_map_lpi(struct kvm *kvm, struct its_ite *ite, + struct vgic_irq *irq, struct kvm_vcpu *vcpu) +{ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (!irq->mapped) { + irq->mapped = true; + irq->target_vcpu = vcpu; + ite->irq = irq; + return true; + } + } + + vgic_put_irq(kvm, irq); + return false; +} + static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size) { return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1); @@ -1093,13 +1111,14 @@ static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its, vcpu = collection_to_vcpu(kvm, collection); irq = vgic_add_lpi(kvm, lpi_nr, vcpu); + if (!IS_ERR(irq) && !vgic_its_map_lpi(kvm, ite, irq, vcpu)) + irq = ERR_PTR(-EEXIST); if (IS_ERR(irq)) { if (new_coll) vgic_its_free_collection(its, coll_id); its_free_ite(kvm, ite); return PTR_ERR(irq); } - ite->irq = irq; return 0; } @@ -2219,7 +2238,10 @@ static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id, its_free_ite(kvm, ite); return PTR_ERR(irq); } - ite->irq = irq; + + /* Drop an ITE whose LPI another ITE maps, as MAPTI does. */ + if (!vgic_its_map_lpi(kvm, ite, irq, vcpu)) + its_free_ite(kvm, ite); return offset; } diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 086b7578e5f3..5c3a0154becc 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -256,6 +256,7 @@ struct vgic_irq { bool active:1; bool hw:1; /* Tied to HW IRQ */ bool on_lr:1; /* Present in a CPU LR */ + bool mapped:1; /* LPIs: mapped by an ITE */ refcount_t refcount; /* Used for LPIs */ u32 hwintid; /* HW INTID number */ unsigned int host_irq; /* linux irq corresponding to hwintid */ From e99ff76f4a02be8542e2c3ecc8843d7339a0200d Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 13 Oct 2026 09:16:46 +0100 Subject: [PATCH 19/42] KVM: arm64: vgic-its: Remove the pending state of a discarded LPI DISCARD removes the mapping of an event and instructs the Redistributor to remove the pending state of its LPI (IHI 0069H.b, 5.3.4). KVM only frees the ITE, so an LPI that is pending when the guest discards it stays queued on its target vCPU, and the guest still takes it. Clear the pending state before freeing the ITE. A queued LPI can be pending in a running vCPU's list register, so halt the guest first when the LPI is queued, to fold that state back. A forwarded LPI already loses its pending state in its_free_ite(), whose unmap from the vPE sends a DISCARD to the host ITS. [maz: folded-in rework to only pause the involved vcpus] Fixes: df9f58fbea9bc ("KVM: arm64: vgic-its: Implement ITS command queue command handlers") Suggested-by: Marc Zyngier Signed-off-by: Fuad Tabba Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20261013081646.933312-3-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-its.c | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 73d7aa700687..37059d133840 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -833,13 +833,36 @@ static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its, struct its_device *device = find_its_device(its, device_id); int ite_esz = vgic_its_get_abi(its)->ite_esz; gpa_t gpa = device->itt_addr + ite->event_id * ite_esz; - /* - * Though the spec talks about removing the pending state, we - * don't bother here since we clear the ITTE anyway and the - * pending state is a property of the ITTE struct. - */ + struct vgic_irq *irq = ite->irq; + struct kvm_vcpu *vcpu, *target; + vgic_its_invalidate_cache(its); + /* + * Only an LPI on a vCPU's AP list can be in its list registers, + * which the exit path folds back, and that exit moves the LPI + * to its target vCPU if the target has changed: pause both. + */ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + vcpu = irq->vcpu; + target = irq->target_vcpu; + } + + if (vcpu) { + if (target != vcpu) + kvm_pause_vcpu(target); + kvm_pause_vcpu(vcpu); + } + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) + irq->pending_latch = false; + + if (vcpu) { + kvm_resume_vcpu(vcpu); + if (target != vcpu) + kvm_resume_vcpu(target); + } + its_free_ite(kvm, ite); return vgic_its_write_entry_lock(its, gpa, 0ULL, ite); From 09d5e940500c2e51b697a0522c54132c1ef078ef Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 13 Oct 2026 17:43:34 +0100 Subject: [PATCH 20/42] KVM: arm64: Don't WARN on SMCCC filter reserved range allocation failure The first time userspace sets an SMCCC filter, kvm_smccc_set_filter() inserts the reserved architecture ranges into the maple tree and WARNs if that fails. The insertion allocates with GFP_KERNEL_ACCOUNT, so a tight memory cgroup limit is enough for userspace to hit the WARN, even though the -ENOMEM is also returned to it. The WARN dates back to when the ranges were inserted at VM creation. Leave -ENOMEM out of the WARN and keep it for every other error. The two reserved ranges are fixed and disjoint and go into an empty tree under config_lock, so any other error is a KVM bug. Fixes: fb88707dd39bd ("KVM: arm64: Use a maple tree to represent the SMCCC filter") Reported-by: Sashiko Closes: https://lore.kernel.org/all/20260829055809.29AD31F000E9@smtp.kernel.org/ Suggested-by: Oliver Upton Signed-off-by: Fuad Tabba Reviewed-by: Suzuki K Poulose Reviewed-by: Oliver Upton Link: https://patch.msgid.link/20261013164334.1492070-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hypercalls.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index b11b8821c9fb..c9d488c586e5 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -197,8 +197,10 @@ static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user if (!kvm_smccc_filter_configured(kvm)) { r = kvm_smccc_filter_insert_reserved(kvm); - if (WARN_ON_ONCE(r)) + if (r) { + WARN_ON_ONCE(r != -ENOMEM); goto out_unlock; + } } r = mtree_insert_range(&kvm->arch.smccc_filter, start, end, From ecb6b43cd22245296d910cb392594c9e33691b87 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 12 Oct 2026 17:15:20 +0100 Subject: [PATCH 21/42] KVM: arm64: Reject the GICv5 CPU interface hypercalls under pKVM __vgic_v5_save_apr() and __vgic_v5_restore_vmcr_apr() remain callable once pKVM has finalised. pKVM never registers a GICv5 vgic, so neither has a valid caller in protected mode, and on a GICv3 machine both reach a register that is UNDEFINED at EL2 and panic the hypervisor. Move both into the band the dispatcher rejects once pKVM has finalised. [maz: Dropped hyp-main.c changes] Fixes: af325e87af5da ("KVM: arm64: gic-v5: Add vgic-v5 save/restore hyp interface") Reviewed-by: Suzuki K Poulose Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261012161524.4016464-2-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index e5b92ac09e69..ea319c496bb2 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -62,6 +62,8 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs, __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs, __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config, + __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, + __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM), @@ -87,8 +89,6 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___tracing_write_event, __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, - __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, - __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, MARKER(__KVM_HOST_SMCCC_FUNC_PKVM_ONLY), From 5b34f0b4adbf983174e53721f68b617113376f2f Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Sat, 17 Oct 2026 12:52:04 -0400 Subject: [PATCH 22/42] KVM: arm64: Take a protected vCPU's TWI and TWE only at vCPU load Commit 06499de8f998 ("KVM: arm64: Skip fixed-feature state flush for protected vCPUs") moved the copy of the host's TWI and TWE in flush_hyp_vcpu() into the non-protected branch: a protected vCPU takes them in __pkvm_vcpu_load() instead, and only VSE comes from the host on each entry. Commit 04447b95c62b ("KVM: arm64: Use the host's HCR_EL2 for non-protected VMs in pKVM") was written against the old code, where both VM types took TWI, TWE and VSE on each entry, and kept that for a protected VM as PKVM_HCR_EL2_HOST_PVM. The merge of the two applies PKVM_HCR_EL2_HOST_PVM in the common path, so a protected vCPU takes TWI and TWE from the host's vCPU on every entry again. That doesn't change what a protected guest runs with today: once the vCPU is initialised, the host only changes TWI and TWE in vcpu_set_wfx_traps(), which pkvm_vcpu_load() calls right before __pkvm_vcpu_load(). But it undoes part of 06499de8f998, and has EL2 read a protected vCPU's trap configuration from host memory on every entry, which nothing needs. Make PKVM_HCR_EL2_HOST_PVM VSE alone and add TWI and TWE to PKVM_HCR_EL2_HOST_NPVM, so that each mask again lists what EL2 takes from the host on each entry for its VM type. A non-protected VM takes the same bits as before. Say in the comments that a protected vCPU's TWI and TWE come at vCPU load. llminus, an LLM-assisted merge conflict resolution tool, found the difference while resolving the same conflict in an integration tree. Fixes: 439d8f9cb01c ("Merge tag 'kvmarm-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD") Assisted-by: LLM Signed-off-by: Sasha Levin Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261017165204.3343275-1-sashal@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 13 +++++++------ arch/arm64/kvm/hyp/nvhe/pkvm.c | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 4373e9b63e87..0261b2bc681f 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -14,13 +14,14 @@ #include /* - * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. The rest - * are EL2's own and nothing the host sets there reaches the guest. + * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. A protected + * VM also takes TWI and TWE, at vCPU load. The rest are EL2's own and nothing + * the host sets there reaches the guest. */ -#define PKVM_HCR_EL2_HOST_PVM (HCR_EL2_TWI | HCR_EL2_TWE | HCR_EL2_VSE) -#define PKVM_HCR_EL2_HOST_NPVM (PKVM_HCR_EL2_HOST_PVM | HCR_EL2_VI | HCR_EL2_VF | \ - HCR_EL2_TVM | HCR_EL2_TID2 | HCR_EL2_TID4 | \ - HCR_EL2_TID5 | HCR_EL2_TTLBOS) +#define PKVM_HCR_EL2_HOST_PVM HCR_EL2_VSE +#define PKVM_HCR_EL2_HOST_NPVM (PKVM_HCR_EL2_HOST_PVM | HCR_EL2_TWI | HCR_EL2_TWE | \ + HCR_EL2_VI | HCR_EL2_VF | HCR_EL2_TVM | HCR_EL2_TID2 | \ + HCR_EL2_TID4 | HCR_EL2_TID5 | HCR_EL2_TTLBOS) /* * Holds the relevant data for maintaining the vcpu state completely at hyp. diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 5168321713d9..01f531c45ff6 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -38,7 +38,10 @@ unsigned int kvm_host_sve_max_vl; */ static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); -/* The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on each entry. */ +/* + * The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on + * each entry, and a protected vCPU's TWI and TWE at vCPU load. + */ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; From 80f6e72b21aa773e71c748e1e5a6d3aa8d458bd7 Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Fri, 25 Sep 2026 17:51:16 +0200 Subject: [PATCH 23/42] RISC-V: KVM: Use a raw lock for IMSIC wakeup state kvm_riscv_vcpu_aia_imsic_put() can run from the scheduler preemption notifier while the raw runqueue lock is held. Taking vsfile_lock, an rwlock_t, there violates the lockdep wait-context rules. Protect the CPU/HGEI pair with a separate raw spinlock. All updates to that pair retain vsfile_lock and take the new lock, while scheduler-out uses only the new lock around the HGEIE update. This keeps the wakeup setup serialized with VS-file publication and release without bringing wider IMSIC operations into a raw critical section. Fixes: 4cec89db80ba ("RISC-V: KVM: Move HGEI[E|P] CSR access to IMSIC virtualization") Signed-off-by: Andrew Jones Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260925155116.465203-1-andrew.jones@oss.qualcomm.com Signed-off-by: Anup Patel --- arch/riscv/kvm/aia_imsic.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index 984b2212328a..302938972aa0 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -48,6 +48,8 @@ struct imsic { /* IMSIC VS-file */ rwlock_t vsfile_lock; + /* Writers hold both locks; scheduler-out only takes vsfile_hgei_lock. */ + raw_spinlock_t vsfile_hgei_lock; int vsfile_cpu; int vsfile_hgei; void __iomem *vsfile_va; @@ -488,6 +490,17 @@ static void imsic_vsfile_local_update(int vsfile_hgei, u32 nr_eix, csr_write(CSR_VSISELECT, old_vsiselect); } +static void imsic_vsfile_set_hgei(struct imsic *imsic, int cpu, int hgei) +{ + unsigned long flags; + + /* Serialize the CPU/HGEI pair with scheduler-out's wakeup setup. */ + raw_spin_lock_irqsave(&imsic->vsfile_hgei_lock, flags); + imsic->vsfile_cpu = cpu; + imsic->vsfile_hgei = hgei; + raw_spin_unlock_irqrestore(&imsic->vsfile_hgei_lock, flags); +} + static void imsic_vsfile_cleanup(struct imsic *imsic) { int old_vsfile_hgei, old_vsfile_cpu; @@ -502,7 +515,7 @@ static void imsic_vsfile_cleanup(struct imsic *imsic) write_lock_irqsave(&imsic->vsfile_lock, flags); old_vsfile_hgei = imsic->vsfile_hgei; old_vsfile_cpu = imsic->vsfile_cpu; - imsic->vsfile_cpu = imsic->vsfile_hgei = -1; + imsic_vsfile_set_hgei(imsic, -1, -1); imsic->vsfile_va = NULL; imsic->vsfile_pa = 0; write_unlock_irqrestore(&imsic->vsfile_lock, flags); @@ -633,10 +646,10 @@ void kvm_riscv_vcpu_aia_imsic_put(struct kvm_vcpu *vcpu) if (!kvm_vcpu_is_blocking(vcpu)) return; - read_lock_irqsave(&imsic->vsfile_lock, flags); + raw_spin_lock_irqsave(&imsic->vsfile_hgei_lock, flags); if (imsic->vsfile_cpu > -1) csr_set(CSR_HGEIE, BIT(imsic->vsfile_hgei)); - read_unlock_irqrestore(&imsic->vsfile_lock, flags); + raw_spin_unlock_irqrestore(&imsic->vsfile_hgei_lock, flags); } void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) @@ -653,7 +666,7 @@ void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu) write_lock_irqsave(&imsic->vsfile_lock, flags); old_vsfile_hgei = imsic->vsfile_hgei; old_vsfile_cpu = imsic->vsfile_cpu; - imsic->vsfile_cpu = imsic->vsfile_hgei = -1; + imsic_vsfile_set_hgei(imsic, -1, -1); imsic->vsfile_va = NULL; imsic->vsfile_pa = 0; write_unlock_irqrestore(&imsic->vsfile_lock, flags); @@ -767,8 +780,7 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) /* Update new IMSIC VS-file details in IMSIC context */ write_lock_irqsave(&imsic->vsfile_lock, flags); - imsic->vsfile_hgei = new_vsfile_hgei; - imsic->vsfile_cpu = vcpu->cpu; + imsic_vsfile_set_hgei(imsic, vcpu->cpu, new_vsfile_hgei); imsic->vsfile_va = new_vsfile_va; imsic->vsfile_pa = new_vsfile_pa; write_unlock_irqrestore(&imsic->vsfile_lock, flags); @@ -1025,6 +1037,7 @@ int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu) /* Setup IMSIC context */ imsic->nr_msis = kvm->arch.aia.nr_ids + 1; rwlock_init(&imsic->vsfile_lock); + raw_spin_lock_init(&imsic->vsfile_hgei_lock); imsic->nr_eix = BITS_TO_U64(imsic->nr_msis); imsic->nr_hw_eix = BITS_TO_U64(kvm_riscv_aia_max_ids); imsic->vsfile_hgei = imsic->vsfile_cpu = -1; From d8538f942fd1006cd6f9b687ace6d45a2c3c015c Mon Sep 17 00:00:00 2001 From: Stefan Butz Date: Thu, 1 Oct 2026 09:35:07 +0200 Subject: [PATCH 24/42] RISC-V: KVM: Don't write MMIO load result to x0 An MMIO load with rd=x0 writes the result into guest_context.zero, which is never cleared, so later emulated reads of x0 return garbage. Fixes: 9f7013265112 ("RISC-V: KVM: Handle MMIO exits for VCPU") Signed-off-by: Stefan Butz Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20261001-b4-mmio_load_protect_x0-v1-1-90515d86f201@butz.st Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_insn.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index adbbb59a0f36..2a8dc2b88ba8 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -647,7 +647,7 @@ int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) vcpu->arch.mmio_decode.return_handled = 1; insn = vcpu->arch.mmio_decode.insn; - if (run->mmio.is_write) + if (run->mmio.is_write || !((insn >> SH_RD) & MASK_RX)) goto done; len = vcpu->arch.mmio_decode.len; From 7e8fa1a887a044ded2971217196cb07e56902ded Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sat, 26 Sep 2026 13:52:27 +0800 Subject: [PATCH 25/42] MAINTAINERS: name the kvm-riscv/linux riscv_kvm_next branch The T: entry for KERNEL VIRTUAL MACHINE FOR RISC-V (KVM/riscv) names github.com/kvm-riscv/linux without a branch. The repository's HEAD points to branch master, which mainline already contains. linux-next merges the riscv_kvm_next branch of this repository (Next/Trees). Name that branch in the entry. Signed-off-by: Matthias Goergens Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20260926055227.3678332-1-matthias.goergens@gmail.com Signed-off-by: Anup Patel --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 72294ddfa5b7..fa481b2dd40f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14358,7 +14358,7 @@ L: kvm@vger.kernel.org L: kvm-riscv@lists.infradead.org L: linux-riscv@lists.infradead.org S: Maintained -T: git https://github.com/kvm-riscv/linux.git +T: git https://github.com/kvm-riscv/linux.git riscv_kvm_next F: arch/riscv/include/asm/kvm* F: arch/riscv/include/uapi/asm/kvm* F: arch/riscv/kvm/ From 65452f8b9c02d79d200731f627db9c6f5d8c387d Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Thu, 15 Oct 2026 00:42:38 +0200 Subject: [PATCH 26/42] RISC-V: KVM: Allow Zicclsm, Ziccamoa, Ziccif, Za64rs in guest Extend the KVM ISA extension array to allow KVM userspace to detect Zicclsm, Ziccamoa, Ziccif, Za64rs. This is just notifying of hardware provided capabilities. No KVM specific support is needed. Signed-off-by: Heinrich Schuchardt Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20261014224239.963635-2-heinrich.schuchardt@canonical.com Signed-off-by: Anup Patel --- arch/riscv/include/uapi/asm/kvm.h | 4 ++++ arch/riscv/kvm/isa.c | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index 20d9959ca44f..39b0f3f1d486 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -206,6 +206,10 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_ZALASR, KVM_RISCV_ISA_EXT_ZICFILP, KVM_RISCV_ISA_EXT_ZICFISS, + KVM_RISCV_ISA_EXT_ZICCLSM, + KVM_RISCV_ISA_EXT_ZICCAMOA, + KVM_RISCV_ISA_EXT_ZICCIF, + KVM_RISCV_ISA_EXT_ZA64RS, KVM_RISCV_ISA_EXT_MAX, }; diff --git a/arch/riscv/kvm/isa.c b/arch/riscv/kvm/isa.c index 971f44655100..72e3dc15d294 100644 --- a/arch/riscv/kvm/isa.c +++ b/arch/riscv/kvm/isa.c @@ -38,6 +38,7 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), KVM_ISA_EXT_ARR(SVVPTC), + KVM_ISA_EXT_ARR(ZA64RS), KVM_ISA_EXT_ARR(ZAAMO), KVM_ISA_EXT_ARR(ZABHA), KVM_ISA_EXT_ARR(ZACAS), @@ -64,6 +65,9 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(ZICBOM), KVM_ISA_EXT_ARR(ZICBOP), KVM_ISA_EXT_ARR(ZICBOZ), + KVM_ISA_EXT_ARR(ZICCAMOA), + KVM_ISA_EXT_ARR(ZICCIF), + KVM_ISA_EXT_ARR(ZICCLSM), KVM_ISA_EXT_ARR(ZICCRSE), KVM_ISA_EXT_ARR(ZICFILP), KVM_ISA_EXT_ARR(ZICFISS), @@ -183,6 +187,7 @@ bool kvm_riscv_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_SVINVAL: case KVM_RISCV_ISA_EXT_SVNAPOT: case KVM_RISCV_ISA_EXT_SVVPTC: + case KVM_RISCV_ISA_EXT_ZA64RS: case KVM_RISCV_ISA_EXT_ZAAMO: case KVM_RISCV_ISA_EXT_ZABHA: case KVM_RISCV_ISA_EXT_ZACAS: @@ -206,6 +211,9 @@ bool kvm_riscv_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_ZFH: case KVM_RISCV_ISA_EXT_ZFHMIN: case KVM_RISCV_ISA_EXT_ZICBOP: + case KVM_RISCV_ISA_EXT_ZICCAMOA: + case KVM_RISCV_ISA_EXT_ZICCIF: + case KVM_RISCV_ISA_EXT_ZICCLSM: case KVM_RISCV_ISA_EXT_ZICCRSE: case KVM_RISCV_ISA_EXT_ZICNTR: case KVM_RISCV_ISA_EXT_ZICOND: From 633b3931a87438282d6454646ff40c669a6896cb Mon Sep 17 00:00:00 2001 From: Heinrich Schuchardt Date: Thu, 15 Oct 2026 00:42:39 +0200 Subject: [PATCH 27/42] KVM: riscv: selftest: add Zicclsm, Ziccamoa, Ziccif, Za64rs KVM supports extensions Zicclsm, Ziccamoa, Ziccif, Za64rs for the guest. Add these extensions to the get-reg-list test. Signed-off-by: Heinrich Schuchardt Reviewed-by: Anup Patel Link: https://lore.kernel.org/r/20261014224239.963635-3-heinrich.schuchardt@canonical.com Signed-off-by: Anup Patel --- tools/testing/selftests/kvm/riscv/get-reg-list.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/testing/selftests/kvm/riscv/get-reg-list.c b/tools/testing/selftests/kvm/riscv/get-reg-list.c index bf3980e9db12..71e5969142b3 100644 --- a/tools/testing/selftests/kvm/riscv/get-reg-list.c +++ b/tools/testing/selftests/kvm/riscv/get-reg-list.c @@ -63,6 +63,7 @@ bool filter_reg(__u64 reg) case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVNAPOT: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVPBMT: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVVPTC: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZA64RS: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZAAMO: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZABHA: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZACAS: @@ -89,6 +90,9 @@ bool filter_reg(__u64 reg) case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOM: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOP: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICBOZ: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCAMOA: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCIF: + case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCLSM: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICCRSE: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICFILP: case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_ZICFISS: @@ -576,6 +580,7 @@ static const char *isa_ext_single_id_to_str(__u64 reg_off) KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), KVM_ISA_EXT_ARR(SVVPTC), + KVM_ISA_EXT_ARR(ZA64RS), KVM_ISA_EXT_ARR(ZAAMO), KVM_ISA_EXT_ARR(ZABHA), KVM_ISA_EXT_ARR(ZACAS), @@ -602,6 +607,9 @@ static const char *isa_ext_single_id_to_str(__u64 reg_off) KVM_ISA_EXT_ARR(ZICBOM), KVM_ISA_EXT_ARR(ZICBOP), KVM_ISA_EXT_ARR(ZICBOZ), + KVM_ISA_EXT_ARR(ZICCAMOA), + KVM_ISA_EXT_ARR(ZICCIF), + KVM_ISA_EXT_ARR(ZICCLSM), KVM_ISA_EXT_ARR(ZICCRSE), KVM_ISA_EXT_ARR(ZICFILP), KVM_ISA_EXT_ARR(ZICFISS), @@ -1238,6 +1246,7 @@ KVM_ISA_EXT_SIMPLE_CONFIG(svinval, SVINVAL); KVM_ISA_EXT_SIMPLE_CONFIG(svnapot, SVNAPOT); KVM_ISA_EXT_SIMPLE_CONFIG(svpbmt, SVPBMT); KVM_ISA_EXT_SIMPLE_CONFIG(svvptc, SVVPTC); +KVM_ISA_EXT_SIMPLE_CONFIG(za64rs, ZA64RS); KVM_ISA_EXT_SIMPLE_CONFIG(zaamo, ZAAMO); KVM_ISA_EXT_SIMPLE_CONFIG(zabha, ZABHA); KVM_ISA_EXT_SIMPLE_CONFIG(zacas, ZACAS); @@ -1264,6 +1273,9 @@ KVM_ISA_EXT_SIMPLE_CONFIG(zfhmin, ZFHMIN); KVM_ISA_EXT_SUBLIST_CONFIG(zicbom, ZICBOM); KVM_ISA_EXT_SUBLIST_CONFIG(zicbop, ZICBOP); KVM_ISA_EXT_SUBLIST_CONFIG(zicboz, ZICBOZ); +KVM_ISA_EXT_SIMPLE_CONFIG(ziccamoa, ZICCAMOA); +KVM_ISA_EXT_SIMPLE_CONFIG(ziccif, ZICCIF); +KVM_ISA_EXT_SIMPLE_CONFIG(zicclsm, ZICCLSM); KVM_ISA_EXT_SIMPLE_CONFIG(ziccrse, ZICCRSE); KVM_ISA_EXT_SIMPLE_CONFIG(zicfilp, ZICFILP); KVM_ISA_EXT_SUBLIST_CONFIG(zicfiss, ZICFISS); @@ -1372,6 +1384,7 @@ struct vcpu_reg_list *vcpu_configs[] = { &config_svnapot, &config_svpbmt, &config_svvptc, + &config_za64rs, &config_zaamo, &config_zabha, &config_zacas, @@ -1398,6 +1411,9 @@ struct vcpu_reg_list *vcpu_configs[] = { &config_zicbom, &config_zicbop, &config_zicboz, + &config_ziccamoa, + &config_ziccif, + &config_zicclsm, &config_ziccrse, &config_zicfilp, &config_zicfiss, From c4c362949dcaaaabaa5db34acaa6e91b9912094c Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 22 Oct 2026 13:30:54 +0100 Subject: [PATCH 28/42] KVM: arm64: Return on kvm_gmem_get_pfn() failure in gmem_abort() The kvm-arm64/mmu-fixes-7.4 merge dropped the return from the kvm_gmem_get_pfn() error path in gmem_abort(). Both parents of the merge keep that return. On a failed kvm_gmem_get_pfn(), gmem_abort() then carries on with a zero pfn, and on a nested stage-2 it uses the mapping it has just freed. Restore the return. Fixes: 65f95b2874a15 ("Merge branch kvm-arm64/mmu-fixes-7.4 into kvmarm-master/next") Signed-off-by: Fuad Tabba Reviewed-by: Suzuki K Poulose Link: https://patch.msgid.link/20261022123054.31729-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/mmu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 745cb86f9e04..b1cbeca2104d 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1781,6 +1781,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, s2vi.vma_pagesize, write_fault, exec_fault, false); kfree(mapping); + return ret; } if (!(s2fd->memslot->flags & KVM_MEM_READONLY)) From 121d74bee8cd81ce9212837db98ceaef2e530637 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Sat, 17 Oct 2026 17:13:50 +0100 Subject: [PATCH 29/42] KVM: arm64: Reject the stage-2 flush hypercalls under pKVM The stage-2 TLB and context flush hypercalls remain callable once pKVM has finalised, and five of the six dereference a host-supplied struct kvm_s2_mmu pointer. Under pKVM none has a valid caller: the host walks no guest stage-2, its TLB flushes go through the handle-based __pkvm_tlb_flush_vmid, and kvm_arch_vcpu_load() skips both the VMID update and the CPU context flush. Move them into the band the dispatcher rejects once pKVM has finalised. [maz: Dropped cosmetic hyp-main.c changes] Reported-by: Sashiko Closes: https://lore.kernel.org/all/20260914130152.F0D5F1F000FF@smtp.kernel.org/ Reviewed-by: Suzuki K Poulose Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261012161524.4016464-3-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 12 ++++++------ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index ea319c496bb2..b182c908c1a9 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -64,6 +64,12 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config, __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, + __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range, + __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context, MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM), @@ -72,12 +78,6 @@ enum __kvm_host_smccc_func { /* Hypercalls that are always available and common to [nh]VHE/pKVM. */ __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc, __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run, - __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context, - __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa, - __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh, - __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid, - __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range, - __KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context, __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff, __KVM_HOST_SMCCC_FUNC___tracing_load, __KVM_HOST_SMCCC_FUNC___tracing_unload, diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9a3b92e626ad..546e34e5fd6f 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -969,7 +969,8 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) /* * If pKVM has been initialised then reject any calls to the - * early "privileged" hypercalls. Note that we cannot reject + * early "privileged" hypercalls, and to the ones the host has no + * use for in protected mode. Note that we cannot reject * calls to __pkvm_prot_finalize for two reasons: (1) The static * key used to determine initialisation must be toggled prior to * finalisation and (2) finalisation is performed on a per-CPU From fb77bb2a0ff253e6c370ec7dc1afa45b2ecd8d2e Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 12 Oct 2026 17:15:22 +0100 Subject: [PATCH 30/42] KVM: arm64: Validate the host-provided vgic model in pKVM EL2 copies vgic_model from the host's struct kvm unchecked, and the nVHE world switch dispatches on it with no cpucap guard. A host writing KVM_DEV_TYPE_ARM_VGIC_V5 makes EL2 access GICv5 CPU interface registers that are UNDEFINED without FEAT_GCIE, and panics the hypervisor on any GICv3 machine. Accept only GICv3, the one model pKVM supports, and force anything else to 0, as for a VM with no in-kernel irqchip. EL2 reads the model only to pick the GICv5 world switch and to emulate GICv3 CPU interface accesses, and the host's own copy is left alone. Fixes: 9b8e3d4ca0e73 ("KVM: arm64: gic-v5: Implement GICv5 load/put and save/restore") Reviewed-by: Suzuki K Poulose Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261012161524.4016464-4-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 459bd9eb7e4b..a9d0e9470919 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -340,13 +340,19 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc { struct kvm *kvm = &hyp_vm->kvm; unsigned long host_arch_flags = READ_ONCE(host_kvm->arch.flags); + u32 vgic_model = READ_ONCE(host_kvm->arch.vgic.vgic_model); DECLARE_BITMAP(allowed_features, KVM_VCPU_MAX_FEATURES); /* CTR_EL0 is always under host control, even for protected VMs. */ hyp_vm->kvm.arch.ctr_el0 = host_kvm->arch.ctr_el0; - /* Preserve the vgic model so that GICv3 emulation works */ - hyp_vm->kvm.arch.vgic.vgic_model = host_kvm->arch.vgic.vgic_model; + /* + * pKVM runs only GICv3. Anything else becomes 0, the model of a VM + * with no in-kernel irqchip, which keeps EL2 off the GICv5 registers. + */ + if (vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) + vgic_model = 0; + hyp_vm->kvm.arch.vgic.vgic_model = vgic_model; /* No restrictions for non-protected VMs. */ if (!kvm_vm_is_protected(kvm)) { From cd38c10e191eb2c37b9904a6ffc3b6bf4d9c0c48 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 12 Oct 2026 17:15:23 +0100 Subject: [PATCH 31/42] KVM: arm64: vgic: Do not access the GICv5 CPU interface from EL1 can_access_vgic_from_kernel() requires VHE only for the GICv3 system register interface, so on a native GICv5 system without FEAT_GCIE_LEGACY it returns true under nVHE. The kernel then saves and restores the CPU interface from EL1, where ICH_VMCR_EL2 and the ICH_PPI_* registers are UNDEFINED, and running a vCPU oopses the host. The nVHE world switch already does that work at EL2. Anything that requires EL2 system registers or instructions is out of reach of a kernel running at EL1, which leaves GICv2, the one memory-mapped CPU interface. Allow GICv2 from EL1 and require VHE for everything else. Fixes: 9b8e3d4ca0e73 ("KVM: arm64: gic-v5: Implement GICv5 load/put and save/restore") Suggested-by: Marc Zyngier Reviewed-by: Sascha Bischoff Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261012161524.4016464-5-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index b25303d9919f..fe5b14a8b121 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -1045,11 +1045,10 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) static inline bool can_access_vgic_from_kernel(void) { /* - * GICv2 can always be accessed from the kernel because it is - * memory-mapped, and VHE systems can access GICv3 EL2 system - * registers. + * GICv2 is memory-mapped. Every other CPU interface uses EL2 system + * registers, which only VHE can access from the kernel. */ - return !static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif) || has_vhe(); + return has_vhe() || kvm_vgic_global_state.type == VGIC_V2; } static inline void vgic_save_state(struct kvm_vcpu *vcpu) From 7ad25ad110608320348669488297e06ea8c72f85 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 12 Oct 2026 17:15:24 +0100 Subject: [PATCH 32/42] KVM: arm64: Fix stale VGICv3 comments in the nVHE world switch __hyp_vgic_save_state() and __hyp_vgic_restore_state() handle GICv5 as well as GICv3, but their comments name VGICv3 only. Reviewed-by: Sascha Bischoff Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261012161524.4016464-6-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hyp/nvhe/switch.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 7318e3e6a5f3..6b9e20f6b31b 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -110,7 +110,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) write_sysreg(__kvm_hyp_host_vector, vbar_el2); } -/* Save VGICv3 state on non-VHE systems */ +/* Save vgic state on non-VHE systems */ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) { if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) { @@ -125,7 +125,7 @@ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) } } -/* Restore VGICv3 state on non-VHE systems */ +/* Restore vgic state on non-VHE systems */ static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) { if (vgic_is_v5(kern_hyp_va(vcpu->kvm))) { From 00e73757518020273ccf6308a51f214e58c21ad5 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 16 Oct 2026 09:15:58 +0100 Subject: [PATCH 33/42] KVM: arm64: Set HCR_EL2.BSU to Inner Shareable for guests KVM sets HCR_EL2.BSU to Inner Shareable for guests, so that a guest's barriers cover the TLB and I-cache invalidations that HCR_EL2.FB broadcasts. However, since HCR_BSU_IS became an alias of the generated HCR_EL2_BSU_IS, it is the unshifted field value 0b01, which sets bit 0 (VM, already set) and leaves BSU at 0b00. A guest's DSB NSH is then not required to wait for a broadcast TLBI to complete on other CPUs. Build the field with SYS_FIELD_PREP_ENUM(). Fixes: d0f39259eff4 ("arm64: sysreg: Add layout for HCR_EL2") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20261016081558.805411-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org --- arch/arm64/include/asm/kvm_arm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 678f0daed445..50f74cbf39fa 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -100,7 +100,7 @@ * TID1: Trap REVIDR_EL1, AIDR_EL1, and SMIDR_EL1 */ #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ - HCR_BSU_IS | HCR_FB | HCR_TACR | \ + SYS_FIELD_PREP_ENUM(HCR_EL2, BSU, IS) | HCR_EL2_FB | HCR_EL2_TACR | \ HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 | HCR_TID1) #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK) From 00869b4a26deb3e52b2ad21f4239f3d533eb16e0 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Oct 2026 18:36:04 +0100 Subject: [PATCH 34/42] KVM: arm64: Convert HCR_xyz to HCR_EL2_xyz Bulk convert all usages of HCR_xyz to HCR_EL2_xyz (the autogenerated version) using the following proof that I can't write shell scripts: awk 'match($3, /__HCR.*/) { print $2}' arch/arm64/include/asm/kvm_arm.h | while read i; do j=$(echo $i | sed -e "s/HCR_\(.*\)/\1/"); git grep -lw $i arch/arm64/ | xargs sed -i -e "s/\b$i\b/HCR_EL2_$j/g"; done ... and fixup the result. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Tested-by: Itaru Kitayama Link: https://patch.msgid.link/20261015173605.1072963-2-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/el2_setup.h | 4 +- arch/arm64/include/asm/hardirq.h | 6 +- arch/arm64/include/asm/kvm_arm.h | 12 +-- arch/arm64/include/asm/kvm_emulate.h | 18 ++-- arch/arm64/include/asm/kvm_mte.h | 4 +- arch/arm64/include/asm/kvm_ptrauth.h | 4 +- arch/arm64/kernel/head.S | 4 +- arch/arm64/kvm/arm.c | 28 +++--- arch/arm64/kvm/at.c | 18 ++-- arch/arm64/kvm/emulate-nested.c | 116 ++++++++++++------------ arch/arm64/kvm/guest.c | 2 +- arch/arm64/kvm/handle_exit.c | 2 +- arch/arm64/kvm/hyp/include/hyp/switch.h | 14 +-- arch/arm64/kvm/hyp/nvhe/host.S | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 20 ++-- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 6 +- arch/arm64/kvm/hyp/nvhe/pkvm.c | 32 +++---- arch/arm64/kvm/hyp/vgic-v3-sr.c | 4 +- arch/arm64/kvm/hyp/vhe/switch.c | 12 +-- arch/arm64/kvm/hyp/vhe/tlb.c | 2 +- arch/arm64/kvm/inject_fault.c | 6 +- arch/arm64/kvm/mmu.c | 8 +- arch/arm64/kvm/nested.c | 8 +- arch/arm64/kvm/sys_regs.c | 24 ++--- arch/arm64/kvm/vgic/vgic-v3-nested.c | 4 +- 26 files changed, 181 insertions(+), 181 deletions(-) diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index aa8ec9df8024..b15db1b1d561 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -61,7 +61,7 @@ b.ne .LnVHE_\@ // Otherwise, we know that nVHE works. .LnE2H0_\@: - orr x0, x0, #HCR_E2H + orr x0, x0, #HCR_EL2_E2H msr_hcr_el2 x0 .LnVHE_\@: .endm @@ -102,7 +102,7 @@ /* Check if running in host at EL2 mode, i.e., (h)VHE. Jump to fail if not. */ .macro __check_hvhe fail, tmp mrs \tmp, hcr_el2 - and \tmp, \tmp, #HCR_E2H + and \tmp, \tmp, #HCR_EL2_E2H cbz \tmp, \fail .endm diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h index 77d6b8c63d4e..024e5a24973e 100644 --- a/arch/arm64/include/asm/hardirq.h +++ b/arch/arm64/include/asm/hardirq.h @@ -40,8 +40,8 @@ do { \ } \ \ ___hcr = read_sysreg(hcr_el2); \ - if (!(___hcr & HCR_TGE)) { \ - write_sysreg_hcr(___hcr | HCR_TGE); \ + if (!(___hcr & HCR_EL2_TGE)) { \ + write_sysreg_hcr(___hcr | HCR_EL2_TGE); \ isb(); \ } \ /* \ @@ -81,7 +81,7 @@ do { \ * we secured things for it. \ */ \ barrier(); \ - if (!___ctx->cnt && !(___hcr & HCR_TGE)) \ + if (!___ctx->cnt && !(___hcr & HCR_EL2_TGE)) \ write_sysreg_hcr(___hcr); \ } while (0) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 50f74cbf39fa..99acd1525282 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -99,13 +99,13 @@ * TID3: Trap EL1 reads of group 3 ID registers * TID1: Trap REVIDR_EL1, AIDR_EL1, and SMIDR_EL1 */ -#define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ +#define HCR_GUEST_FLAGS (HCR_EL2_TSC | HCR_EL2_TSW | HCR_EL2_TWE | HCR_EL2_TWI | HCR_EL2_VM | \ SYS_FIELD_PREP_ENUM(HCR_EL2, BSU, IS) | HCR_EL2_FB | HCR_EL2_TACR | \ - HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ - HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 | HCR_TID1) -#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK) -#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) -#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H | HCR_AMO | HCR_IMO | HCR_FMO) + HCR_EL2_AMO | HCR_EL2_SWIO | HCR_EL2_TIDCP | HCR_EL2_RW | HCR_EL2_TLOR | \ + HCR_EL2_FMO | HCR_EL2_IMO | HCR_EL2_PTW | HCR_EL2_TID3 | HCR_EL2_TID1) +#define HCR_HOST_NVHE_FLAGS (HCR_EL2_RW | HCR_EL2_API | HCR_EL2_APK) +#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_EL2_TSC) +#define HCR_HOST_VHE_FLAGS (HCR_EL2_RW | HCR_EL2_TGE | HCR_EL2_E2H | HCR_EL2_AMO | HCR_EL2_IMO | HCR_EL2_FMO) #define MPAMHCR_HOST_FLAGS 0 diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 85289a4b8d52..334211b8b2ee 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -95,7 +95,7 @@ static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu) #if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__) static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) { - return !(vcpu->arch.hcr_el2 & HCR_RW); + return !(vcpu->arch.hcr_el2 & HCR_EL2_RW); } #else static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) @@ -116,7 +116,7 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) * then. */ if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - vcpu->arch.hcr_el2 |= HCR_TVM; + vcpu->arch.hcr_el2 |= HCR_EL2_TVM; } static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) @@ -227,12 +227,12 @@ static inline bool vcpu_is_el2(const struct kvm_vcpu *vcpu) static inline bool vcpu_el2_e2h_is_set(const struct kvm_vcpu *vcpu) { return (!cpus_have_final_cap(ARM64_HAS_HCR_NV1) || - (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_E2H)); + (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_E2H)); } static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu) { - return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE; + return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_EL2_TGE; } static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu) @@ -251,7 +251,7 @@ static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu) if (vcpu_is_el2(vcpu) && vcpu_el2_e2h_is_set(vcpu) && !vcpu_el2_tge_is_set(vcpu)) return true; - return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO; + return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_EL2_AMO; } static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu) @@ -264,8 +264,8 @@ static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu) hcr = __vcpu_sys_reg(vcpu, HCR_EL2); - e2h = (hcr & HCR_E2H); - tge = (hcr & HCR_TGE); + e2h = (hcr & HCR_EL2_E2H); + tge = (hcr & HCR_EL2_TGE); /* * We are in a hypervisor context if the vcpu mode is EL2 or @@ -376,8 +376,8 @@ static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu) if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu)) return false; - return ((is_wfe && (hcr_el2 & HCR_TWE)) || - (!is_wfe && (hcr_el2 & HCR_TWI))); + return ((is_wfe && (hcr_el2 & HCR_EL2_TWE)) || + (!is_wfe && (hcr_el2 & HCR_EL2_TWI))); } static __always_inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu) diff --git a/arch/arm64/include/asm/kvm_mte.h b/arch/arm64/include/asm/kvm_mte.h index 3171963ad25c..502d6df91691 100644 --- a/arch/arm64/include/asm/kvm_mte.h +++ b/arch/arm64/include/asm/kvm_mte.h @@ -16,7 +16,7 @@ alternative_if_not ARM64_MTE b .L__skip_switch\@ alternative_else_nop_endif mrs \reg1, hcr_el2 - tbz \reg1, #(HCR_ATA_SHIFT), .L__skip_switch\@ + tbz \reg1, #(HCR_EL2_ATA_SHIFT), .L__skip_switch\@ mrs_s \reg1, SYS_RGSR_EL1 str \reg1, [\h_ctxt, #CPU_RGSR_EL1] @@ -36,7 +36,7 @@ alternative_if_not ARM64_MTE b .L__skip_switch\@ alternative_else_nop_endif mrs \reg1, hcr_el2 - tbz \reg1, #(HCR_ATA_SHIFT), .L__skip_switch\@ + tbz \reg1, #(HCR_EL2_ATA_SHIFT), .L__skip_switch\@ mrs_s \reg1, SYS_RGSR_EL1 str \reg1, [\g_ctxt, #CPU_RGSR_EL1] diff --git a/arch/arm64/include/asm/kvm_ptrauth.h b/arch/arm64/include/asm/kvm_ptrauth.h index e50987b32483..24f75457a712 100644 --- a/arch/arm64/include/asm/kvm_ptrauth.h +++ b/arch/arm64/include/asm/kvm_ptrauth.h @@ -71,7 +71,7 @@ alternative_if_not ARM64_HAS_ADDRESS_AUTH b .L__skip_switch\@ alternative_else_nop_endif mrs \reg1, hcr_el2 - and \reg1, \reg1, #(HCR_API | HCR_APK) + and \reg1, \reg1, #(HCR_EL2_API | HCR_EL2_APK) cbz \reg1, .L__skip_switch\@ add \reg1, \g_ctxt, #CPU_APIAKEYLO_EL1 ptrauth_restore_state \reg1, \reg2, \reg3 @@ -83,7 +83,7 @@ alternative_if_not ARM64_HAS_ADDRESS_AUTH b .L__skip_switch\@ alternative_else_nop_endif mrs \reg1, hcr_el2 - and \reg1, \reg1, #(HCR_API | HCR_APK) + and \reg1, \reg1, #(HCR_EL2_API | HCR_EL2_APK) cbz \reg1, .L__skip_switch\@ add \reg1, \g_ctxt, #CPU_APIAKEYLO_EL1 ptrauth_save_state \reg1, \reg2, \reg3 diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 87a822e5c4ca..cb684f40beac 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -299,7 +299,7 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) isb 0: - init_el2_hcr HCR_HOST_NVHE_FLAGS | HCR_ATA + init_el2_hcr HCR_HOST_NVHE_FLAGS | HCR_EL2_ATA init_el2_state /* Hypervisor stub */ @@ -310,7 +310,7 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) mov_q x1, INIT_SCTLR_EL1_MMU_OFF mrs x0, hcr_el2 - and x0, x0, #HCR_E2H + and x0, x0, #HCR_EL2_E2H cbz x0, 2f /* Set a sane SCTLR_EL1, the VHE way */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index d593ba928093..456f51f2b89a 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -650,11 +650,11 @@ static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) u64 val; val = __vcpu_sys_reg(vcpu, HCR_EL2); - val &= (HCR_API | HCR_APK); - vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); + val &= (HCR_EL2_API | HCR_EL2_APK); + vcpu->arch.hcr_el2 &= ~(HCR_EL2_API | HCR_EL2_APK); vcpu->arch.hcr_el2 |= val; } else { - vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); + vcpu->arch.hcr_el2 |= (HCR_EL2_API | HCR_EL2_APK); } /* @@ -662,7 +662,7 @@ static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) * to use pauth, as the entry code will reload the guest * keys in that case. */ - if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { + if (vcpu->arch.hcr_el2 & (HCR_EL2_API | HCR_EL2_APK)) { struct kvm_cpu_context *ctxt; ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); @@ -736,14 +736,14 @@ static void vcpu_put_mmu(struct kvm_vcpu *vcpu) static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) { if (kvm_vcpu_should_clear_twe(vcpu)) - vcpu->arch.hcr_el2 &= ~HCR_TWE; + vcpu->arch.hcr_el2 &= ~HCR_EL2_TWE; else - vcpu->arch.hcr_el2 |= HCR_TWE; + vcpu->arch.hcr_el2 |= HCR_EL2_TWE; if (kvm_vcpu_should_clear_twi(vcpu)) - vcpu->arch.hcr_el2 &= ~HCR_TWI; + vcpu->arch.hcr_el2 &= ~HCR_EL2_TWI; else - vcpu->arch.hcr_el2 |= HCR_TWI; + vcpu->arch.hcr_el2 |= HCR_EL2_TWI; } static void vcpu_load_pvtime(struct kvm_vcpu *vcpu) @@ -963,7 +963,7 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, */ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) { - bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE); + bool irq_lines = *vcpu_hcr(v) & (HCR_EL2_VI | HCR_EL2_VF | HCR_EL2_VSE); irq_lines |= (!irqchip_in_kernel(v->kvm) && (kvm_timer_should_notify_user(v) || @@ -1592,9 +1592,9 @@ static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) unsigned long *hcr; if (number == KVM_ARM_IRQ_CPU_IRQ) - bit_index = __ffs(HCR_VI); + bit_index = __ffs(HCR_EL2_VI); else /* KVM_ARM_IRQ_CPU_FIQ */ - bit_index = __ffs(HCR_VF); + bit_index = __ffs(HCR_EL2_VF); hcr = vcpu_hcr(vcpu); if (level) @@ -2384,12 +2384,12 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) params->hcr_el2 = HCR_HOST_NVHE_FLAGS; if (system_supports_mte()) - params->hcr_el2 |= HCR_ATA; + params->hcr_el2 |= HCR_EL2_ATA; else - params->hcr_el2 |= HCR_TID5; + params->hcr_el2 |= HCR_EL2_TID5; if (cpus_have_final_cap(ARM64_KVM_HVHE)) - params->hcr_el2 |= HCR_E2H; + params->hcr_el2 |= HCR_EL2_E2H; params->vttbr = params->vtcr = 0; /* diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 0926426b8798..30d8f062b2c2 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -229,7 +229,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, if (vcpu_has_nv(vcpu)) { hcr = __vcpu_sys_reg(vcpu, HCR_EL2); - wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_VM | HCR_DC)); + wi->s2 = wi->regime == TR_EL10 && (hcr & (HCR_EL2_VM | HCR_EL2_DC)); } else { WARN_ON_ONCE(wi->regime != TR_EL10); wi->s2 = false; @@ -305,7 +305,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, * is EL1&0, then it means that either {E2H,TGE} == {1,0} or * {0,x}, and we only need to test for TGE == 1. */ - if (hcr & (HCR_DC | HCR_TGE)) { + if (hcr & (HCR_EL2_DC | HCR_EL2_TGE)) { wr->level = S1_MMU_DISABLED; break; } @@ -870,7 +870,7 @@ static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par, s1_parattr = FIELD_GET(SYS_PAR_EL1_ATTR, s1_par); s2_memattr = FIELD_GET(GENMASK(5, 2), tr->desc); - if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_FWB) { + if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_FWB) { if (!kvm_has_feat(vcpu->kvm, ID_AA64PFR2_EL1, MTEPERM, IMP)) s2_memattr &= ~BIT(3); @@ -924,7 +924,7 @@ static u64 compute_par_s12(struct kvm_vcpu *vcpu, u64 s1_par, } } - if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_CD) && + if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_CD) && !MEMATTR_IS_DEVICE(final_attr)) final_attr = MEMATTR(NC, NC); @@ -956,7 +956,7 @@ static u64 compute_par_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, par |= wr->pa & SYS_PAR_EL1_PA; if (wi->regime == TR_EL10 && vcpu_has_nv(vcpu) && - (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_DC)) { + (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_DC)) { par |= FIELD_PREP(SYS_PAR_EL1_ATTR, MEMATTR(WbRaWa, WbRaWa)); par |= FIELD_PREP(SYS_PAR_EL1_SH, ATTR_NSH); @@ -1548,11 +1548,11 @@ int __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) bool fail; val = hcr = read_sysreg(hcr_el2); - val &= ~HCR_TGE; - val |= HCR_VM; + val &= ~HCR_EL2_TGE; + val |= HCR_EL2_VM; if (!vcpu_el2_e2h_is_set(vcpu)) - val |= HCR_NV | HCR_NV1; + val |= HCR_EL2_NV | HCR_EL2_NV1; write_sysreg_hcr(val); isb(); @@ -1635,7 +1635,7 @@ int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) * early. Same thing if {VM,DC}=={0,0}. */ if (compute_translation_regime(vcpu, op) == TR_EL20 || - !(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_VM | HCR_DC))) + !(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_VM | HCR_EL2_DC))) return 0; /* Do the stage-2 translation */ diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 3806ff0920fe..65686ddb3b7d 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -145,176 +145,176 @@ enum cgt_group_id { static const struct trap_bits coarse_trap_bits[] = { [CGT_HCR_TID1] = { .index = HCR_EL2, - .value = HCR_TID1, - .mask = HCR_TID1, + .value = HCR_EL2_TID1, + .mask = HCR_EL2_TID1, .behaviour = BEHAVE_FORWARD_READ, }, [CGT_HCR_TID2] = { .index = HCR_EL2, - .value = HCR_TID2, - .mask = HCR_TID2, + .value = HCR_EL2_TID2, + .mask = HCR_EL2_TID2, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TID3] = { .index = HCR_EL2, - .value = HCR_TID3, - .mask = HCR_TID3, + .value = HCR_EL2_TID3, + .mask = HCR_EL2_TID3, .behaviour = BEHAVE_FORWARD_READ, }, [CGT_HCR_IMO] = { .index = HCR_EL2, - .value = HCR_IMO, - .mask = HCR_IMO, + .value = HCR_EL2_IMO, + .mask = HCR_EL2_IMO, .behaviour = BEHAVE_FORWARD_WRITE, }, [CGT_HCR_FMO] = { .index = HCR_EL2, - .value = HCR_FMO, - .mask = HCR_FMO, + .value = HCR_EL2_FMO, + .mask = HCR_EL2_FMO, .behaviour = BEHAVE_FORWARD_WRITE, }, [CGT_HCR_TIDCP] = { .index = HCR_EL2, - .value = HCR_TIDCP, - .mask = HCR_TIDCP, + .value = HCR_EL2_TIDCP, + .mask = HCR_EL2_TIDCP, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TACR] = { .index = HCR_EL2, - .value = HCR_TACR, - .mask = HCR_TACR, + .value = HCR_EL2_TACR, + .mask = HCR_EL2_TACR, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TSW] = { .index = HCR_EL2, - .value = HCR_TSW, - .mask = HCR_TSW, + .value = HCR_EL2_TSW, + .mask = HCR_EL2_TSW, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TPC] = { /* Also called TCPC when FEAT_DPB is implemented */ .index = HCR_EL2, - .value = HCR_TPC, - .mask = HCR_TPC, + .value = HCR_EL2_TPCP, + .mask = HCR_EL2_TPCP, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TPU] = { .index = HCR_EL2, - .value = HCR_TPU, - .mask = HCR_TPU, + .value = HCR_EL2_TPU, + .mask = HCR_EL2_TPU, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TTLB] = { .index = HCR_EL2, - .value = HCR_TTLB, - .mask = HCR_TTLB, + .value = HCR_EL2_TTLB, + .mask = HCR_EL2_TTLB, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TVM] = { .index = HCR_EL2, - .value = HCR_TVM, - .mask = HCR_TVM, + .value = HCR_EL2_TVM, + .mask = HCR_EL2_TVM, .behaviour = BEHAVE_FORWARD_WRITE, }, [CGT_HCR_TDZ] = { .index = HCR_EL2, - .value = HCR_TDZ, - .mask = HCR_TDZ, + .value = HCR_EL2_TDZ, + .mask = HCR_EL2_TDZ, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TRVM] = { .index = HCR_EL2, - .value = HCR_TRVM, - .mask = HCR_TRVM, + .value = HCR_EL2_TRVM, + .mask = HCR_EL2_TRVM, .behaviour = BEHAVE_FORWARD_READ, }, [CGT_HCR_TLOR] = { .index = HCR_EL2, - .value = HCR_TLOR, - .mask = HCR_TLOR, + .value = HCR_EL2_TLOR, + .mask = HCR_EL2_TLOR, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TERR] = { .index = HCR_EL2, - .value = HCR_TERR, - .mask = HCR_TERR, + .value = HCR_EL2_TERR, + .mask = HCR_EL2_TERR, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_APK] = { .index = HCR_EL2, .value = 0, - .mask = HCR_APK, + .mask = HCR_EL2_APK, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_NV] = { .index = HCR_EL2, - .value = HCR_NV, - .mask = HCR_NV, + .value = HCR_EL2_NV, + .mask = HCR_EL2_NV, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_NV_nNV2] = { .index = HCR_EL2, - .value = HCR_NV, - .mask = HCR_NV | HCR_NV2, + .value = HCR_EL2_NV, + .mask = HCR_EL2_NV | HCR_EL2_NV2, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_NV1_nNV2] = { .index = HCR_EL2, - .value = HCR_NV | HCR_NV1, - .mask = HCR_NV | HCR_NV1 | HCR_NV2, + .value = HCR_EL2_NV | HCR_EL2_NV1, + .mask = HCR_EL2_NV | HCR_EL2_NV1 | HCR_EL2_NV2, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_AT] = { .index = HCR_EL2, - .value = HCR_AT, - .mask = HCR_AT, + .value = HCR_EL2_AT, + .mask = HCR_EL2_AT, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_nFIEN] = { .index = HCR_EL2, .value = 0, - .mask = HCR_FIEN, + .mask = HCR_EL2_FIEN, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TID4] = { .index = HCR_EL2, - .value = HCR_TID4, - .mask = HCR_TID4, + .value = HCR_EL2_TID4, + .mask = HCR_EL2_TID4, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TICAB] = { .index = HCR_EL2, - .value = HCR_TICAB, - .mask = HCR_TICAB, + .value = HCR_EL2_TICAB, + .mask = HCR_EL2_TICAB, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TOCU] = { .index = HCR_EL2, - .value = HCR_TOCU, - .mask = HCR_TOCU, + .value = HCR_EL2_TOCU, + .mask = HCR_EL2_TOCU, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_ENSCXT] = { .index = HCR_EL2, .value = 0, - .mask = HCR_ENSCXT, + .mask = HCR_EL2_EnSCXT, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TTLBIS] = { .index = HCR_EL2, - .value = HCR_TTLBIS, - .mask = HCR_TTLBIS, + .value = HCR_EL2_TTLBIS, + .mask = HCR_EL2_TTLBIS, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TTLBOS] = { .index = HCR_EL2, - .value = HCR_TTLBOS, - .mask = HCR_TTLBOS, + .value = HCR_EL2_TTLBOS, + .mask = HCR_EL2_TTLBOS, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_HCR_TID5] = { .index = HCR_EL2, - .value = HCR_TID5, - .mask = HCR_TID5, + .value = HCR_EL2_TID5, + .mask = HCR_EL2_TID5, .behaviour = BEHAVE_FORWARD_RW, }, [CGT_MDCR_TPMCR] = { @@ -528,7 +528,7 @@ static bool is_nested_nv2_guest(struct kvm_vcpu *vcpu) u64 val; val = __vcpu_sys_reg(vcpu, HCR_EL2); - return ((val & (HCR_E2H | HCR_TGE | HCR_NV2 | HCR_NV1 | HCR_NV)) == (HCR_E2H | HCR_NV2 | HCR_NV)); + return ((val & (HCR_EL2_E2H | HCR_EL2_TGE | HCR_EL2_NV2 | HCR_EL2_NV1 | HCR_EL2_NV)) == (HCR_EL2_E2H | HCR_EL2_NV2 | HCR_EL2_NV)); } static enum trap_behaviour check_cnthctl_el1nvpct(struct kvm_vcpu *vcpu) @@ -2725,7 +2725,7 @@ static bool forward_hcr_traps(struct kvm_vcpu *vcpu, u64 control_bit) bool forward_smc_trap(struct kvm_vcpu *vcpu) { - return forward_hcr_traps(vcpu, HCR_TSC); + return forward_hcr_traps(vcpu, HCR_EL2_TSC); } static bool forward_mdcr_traps(struct kvm_vcpu *vcpu, u64 control_bit) @@ -2947,7 +2947,7 @@ int kvm_inject_nested_irq(struct kvm_vcpu *vcpu) */ if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) && - !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO)) + !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_IMO)) return 1; /* esr_el2 value doesn't matter for exits due to irqs. */ diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 1079136d5cd7..878ea4500c5c 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -755,7 +755,7 @@ int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events) { events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN); - events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) || + events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_EL2_VSE) || vcpu_get_flag(vcpu, NESTED_SERROR_PENDING); if (events->exception.serror_pending && events->exception.serror_has_esr) diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index db37678dcb05..9931192735d4 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -43,7 +43,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu) /* Forward hvc instructions to the virtual EL2 if the guest has EL2. */ if (vcpu_has_nv(vcpu)) { - if (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_HCD) + if (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_HCD) kvm_inject_undefined(vcpu); else kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu)); diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index a2a054fec2e2..e0c55644224c 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -403,11 +403,11 @@ static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr) { if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM)) - hcr |= HCR_TVM; + hcr |= HCR_EL2_TVM; write_sysreg_hcr(hcr); - if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) { + if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_EL2_VSE)) { u64 vsesr; /* @@ -454,9 +454,9 @@ static inline void ___deactivate_traps(struct kvm_vcpu *vcpu) * Additionally, when in a nested context we need to propagate the * updated state to the guest hypervisor's HCR_EL2. */ - if (*hcr & HCR_VSE) { - *hcr &= ~HCR_VSE; - *hcr |= read_sysreg(hcr_el2) & HCR_VSE; + if (*hcr & HCR_EL2_VSE) { + *hcr &= ~HCR_EL2_VSE; + *hcr |= read_sysreg(hcr_el2) & HCR_EL2_VSE; } } @@ -655,7 +655,7 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) write_sysreg_s(__vcpu_sys_reg(vcpu, FPMR), SYS_FPMR); /* Skip restoring fpexc32 for AArch64 guests */ - if (!(read_sysreg(hcr_el2) & HCR_RW)) + if (!(read_sysreg(hcr_el2) & HCR_EL2_RW)) write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2); *host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED; @@ -680,7 +680,7 @@ static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu) * The normal sysreg handling code expects to see the traps, * let's not do anything here. */ - if (vcpu->arch.hcr_el2 & HCR_TVM) + if (vcpu->arch.hcr_el2 & HCR_EL2_TVM) return false; switch (sysreg) { diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 9393fe3ea6a1..50581822ec74 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -123,7 +123,7 @@ SYM_FUNC_START(__hyp_do_panic) #ifdef CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC /* Ensure host stage-2 is disabled */ mrs x0, hcr_el2 - bic x0, x0, #HCR_VM + bic x0, x0, #HCR_EL2_VM msr_hcr_el2 x0 tlbi vmalls12e1 dsb nsh diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 0b3e0b28dfc7..49127e03cf93 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -102,7 +102,7 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init) ldr x1, [x0, #NVHE_INIT_HCR_EL2] msr_hcr_el2 x1 - mov x2, #HCR_E2H + mov x2, #HCR_EL2_E2H and x2, x1, x2 cbz x2, 1f diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index d715aa942577..f5f77e62d1ba 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -708,9 +708,9 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) flush_hyp_vcpu_state(hyp_vcpu); - hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE); + hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_EL2_TWI | HCR_EL2_TWE); hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & - (HCR_TWI | HCR_TWE); + (HCR_EL2_TWI | HCR_EL2_TWE); hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; @@ -723,8 +723,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) * A host-injected vSError is masked by the guest's own PSTATE.A, so it * applies to protected guests too. */ - hyp_vcpu->vcpu.arch.hcr_el2 &= ~HCR_VSE; - hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & HCR_VSE; + hyp_vcpu->vcpu.arch.hcr_el2 &= ~HCR_EL2_VSE; + hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & HCR_EL2_VSE; hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; flush_hyp_vgic_state(hyp_vcpu); @@ -811,8 +811,8 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) } /* Cleared by hardware once the guest takes the vSError. */ - host_vcpu->arch.hcr_el2 &= ~HCR_VSE; - host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE; + host_vcpu->arch.hcr_el2 &= ~HCR_EL2_VSE; + host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_EL2_VSE; sync_hyp_vgic_state(hyp_vcpu); sync_hyp_timer_state(hyp_vcpu); @@ -836,8 +836,8 @@ DEFINE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, u64 dfr0 = read_sysreg(id_aa64dfr0_el1); /* Propagate WFx trapping flags */ - hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI); - hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI); + hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_EL2_TWE | HCR_EL2_TWI); + hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_EL2_TWE | HCR_EL2_TWI); /* HPMN == 0 is reserved without FEAT_HPMN0. */ if (pmuv3_implemented(SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0))) @@ -1739,12 +1739,12 @@ static bool handle_host_mte(u64 esr) case SYS_TFSR_EL1: case SYS_TFSRE0_EL1: /* If we're here for any reason other than MTE, it's a bug. */ - if (read_sysreg(HCR_EL2) & HCR_ATA) + if (read_sysreg(HCR_EL2) & HCR_EL2_ATA) return false; break; case SYS_GMID_EL1: /* If we're here for any reason other than MTE, it's a bug. */ - if (!(read_sysreg(HCR_EL2) & HCR_TID5)) + if (!(read_sysreg(HCR_EL2) & HCR_EL2_TID5)) return false; break; default: diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index c3035aa5a470..d4c332a91803 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -344,14 +344,14 @@ int __pkvm_prot_finalize(void) struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu; struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params); - if (params->hcr_el2 & HCR_VM) + if (params->hcr_el2 & HCR_EL2_VM) return -EPERM; params->vttbr = kvm_get_vttbr(mmu); params->vtcr = mmu->vtcr; - params->hcr_el2 |= HCR_VM; + params->hcr_el2 |= HCR_EL2_VM; if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - params->hcr_el2 |= HCR_FWB; + params->hcr_el2 |= HCR_EL2_FWB; /* * The CMO below not only cleans the updated params to the diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index ac020d0501d2..01fe8d085ee2 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -43,30 +43,30 @@ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; if (has_hvhe()) - vcpu->arch.hcr_el2 |= HCR_E2H; + vcpu->arch.hcr_el2 |= HCR_EL2_E2H; if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) { /* route synchronous external abort exceptions to EL2 */ - vcpu->arch.hcr_el2 |= HCR_TEA; + vcpu->arch.hcr_el2 |= HCR_EL2_TEA; /* trap error record accesses */ - vcpu->arch.hcr_el2 |= HCR_TERR; + vcpu->arch.hcr_el2 |= HCR_EL2_TERR; } if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - vcpu->arch.hcr_el2 |= HCR_FWB; + vcpu->arch.hcr_el2 |= HCR_EL2_FWB; if (cpus_have_final_cap(ARM64_HAS_EVT) && !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && kvm_read_vm_id_reg(vcpu->kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0)) - vcpu->arch.hcr_el2 |= HCR_TID4; + vcpu->arch.hcr_el2 |= HCR_EL2_TID4; else - vcpu->arch.hcr_el2 |= HCR_TID2; + vcpu->arch.hcr_el2 |= HCR_EL2_TID2; if (vcpu_has_ptrauth(vcpu)) - vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); + vcpu->arch.hcr_el2 |= (HCR_EL2_API | HCR_EL2_APK); if (kvm_has_mte(vcpu->kvm)) - vcpu->arch.hcr_el2 |= HCR_ATA; + vcpu->arch.hcr_el2 |= HCR_EL2_ATA; } static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) @@ -75,30 +75,30 @@ static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) u64 val = vcpu->arch.hcr_el2; /* No support for AArch32. */ - val |= HCR_RW; + val |= HCR_EL2_RW; /* * Always trap: * - Feature id registers: to control features exposed to guests * - Implementation-defined features */ - val |= HCR_TACR | HCR_TIDCP | HCR_TID3 | HCR_TID1; + val |= HCR_EL2_TACR | HCR_EL2_TIDCP | HCR_EL2_TID3 | HCR_EL2_TID1; if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) { - val |= HCR_TERR | HCR_TEA; - val &= ~(HCR_FIEN); + val |= HCR_EL2_TERR | HCR_EL2_TEA; + val &= ~(HCR_EL2_FIEN); } if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, IMP)) - val &= ~(HCR_AMVOFFEN); + val &= ~(HCR_EL2_AMVOFFEN); if (!kvm_has_mte(kvm)) { - val |= HCR_TID5; - val &= ~(HCR_DCT | HCR_ATA); + val |= HCR_EL2_TID5; + val &= ~(HCR_EL2_DCT | HCR_EL2_ATA); } if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) - val |= HCR_TLOR; + val |= HCR_EL2_TLOR; vcpu->arch.hcr_el2 = val; } diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 74d4a509f1de..2dfb5ff9db57 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -464,7 +464,7 @@ bool __vgic_v3_get_gic_config(void) if (has_vhe()) { flags = local_daif_save(); } else { - sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO); + sysreg_clear_set_hcr(0, HCR_EL2_AMO | HCR_EL2_FMO | HCR_EL2_IMO); isb(); } @@ -479,7 +479,7 @@ bool __vgic_v3_get_gic_config(void) if (has_vhe()) { local_daif_restore(flags); } else { - sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0); + sysreg_clear_set_hcr(HCR_EL2_AMO | HCR_EL2_FMO | HCR_EL2_IMO, 0); isb(); } diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 4c965f9b1e1d..3f2390a13173 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -47,7 +47,7 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); * - FIEN: no way we let a guest have access to the RAS "Common Fault * Injection" thing, whatever that does */ -#define NV_HCR_GUEST_EXCLUDE (HCR_TGE | HCR_API | HCR_APK | HCR_FIEN) +#define NV_HCR_GUEST_EXCLUDE (HCR_EL2_TGE | HCR_EL2_API | HCR_EL2_APK | HCR_EL2_FIEN) static u64 __compute_hcr(struct kvm_vcpu *vcpu) { @@ -65,10 +65,10 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) if (is_hyp_ctxt(vcpu)) { host_data_set_flag(VCPU_IN_HYP_CONTEXT); - hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB; + hcr |= HCR_EL2_NV | HCR_EL2_NV2 | HCR_EL2_AT | HCR_EL2_TTLB; if (!vcpu_el2_e2h_is_set(vcpu)) - hcr |= HCR_NV1; + hcr |= HCR_EL2_NV1; /* Publish the guest's view of HCR_EL2 to the HW */ if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu)) @@ -91,7 +91,7 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) host_data_clear_flag(VCPU_IN_HYP_CONTEXT); guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2); - if (guest_hcr & HCR_NV) { + if (guest_hcr & HCR_EL2_NV) { u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id())); /* Inherit the low bits from the actual register */ @@ -99,14 +99,14 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) write_sysreg_s(va, SYS_VNCR_EL2); /* Force NV2 in case the guest is forgetful... */ - guest_hcr |= HCR_NV2; + guest_hcr |= HCR_EL2_NV2; } /* * Exclude the guest's TWED configuration if it hasn't set TWE * to avoid potentially delaying traps for the host. */ - if (!(guest_hcr & HCR_TWE)) + if (!(guest_hcr & HCR_EL2_TWE)) guest_hcr &= ~(HCR_EL2_TWEDEn | HCR_EL2_TWEDEL); } diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c index c386d9f1c101..33ac9a8c954e 100644 --- a/arch/arm64/kvm/hyp/vhe/tlb.c +++ b/arch/arm64/kvm/hyp/vhe/tlb.c @@ -62,7 +62,7 @@ static void enter_vmid_context(struct kvm_s2_mmu *mmu, */ __load_stage2(mmu); val = read_sysreg(hcr_el2); - val &= ~HCR_TGE; + val &= ~HCR_EL2_TGE; write_sysreg_hcr(val); isb(); } diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 96bf926ca520..aa2011cc0d5a 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -228,7 +228,7 @@ static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr) static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu) { - if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_TGE | HCR_TEA)) + if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_TGE | HCR_EL2_TEA)) return true; if (!vcpu_mode_priv(vcpu)) @@ -272,7 +272,7 @@ int kvm_inject_dabt_excl_atomic(struct kvm_vcpu *vcpu, u64 addr) { u64 esr; - if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM)) + if (is_nested_ctxt(vcpu) && (vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_VM)) return kvm_inject_nested_excl_atomic(vcpu, addr); __kvm_inject_sea(vcpu, false, addr); @@ -387,6 +387,6 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) } vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK); - *vcpu_hcr(vcpu) |= HCR_VSE; + *vcpu_hcr(vcpu) |= HCR_EL2_VSE; return 1; } diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index a48f3f531e82..a3eb084a2cf8 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2988,18 +2988,18 @@ void kvm_set_way_flush(struct kvm_vcpu *vcpu) /* * If this is the first time we do a S/W operation - * (i.e. HCR_TVM not set) flush the whole memory, and set the + * (i.e. HCR_EL2_TVM not set) flush the whole memory, and set the * VM trapping. * * Otherwise, rely on the VM trapping to wait for the MMU + * Caches to be turned off. At that point, we'll be able to * clean the caches again. */ - if (!(hcr & HCR_TVM)) { + if (!(hcr & HCR_EL2_TVM)) { trace_kvm_set_way_flush(*vcpu_pc(vcpu), vcpu_has_cache_enabled(vcpu)); stage2_flush_vm(vcpu->kvm); - *vcpu_hcr(vcpu) = hcr | HCR_TVM; + *vcpu_hcr(vcpu) = hcr | HCR_EL2_TVM; } } @@ -3017,7 +3017,7 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) /* Caches are now on, stop trapping VM ops (until a S/W op) */ if (now_enabled) - *vcpu_hcr(vcpu) &= ~HCR_TVM; + *vcpu_hcr(vcpu) &= ~HCR_EL2_TVM; trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); } diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index f0f473978d98..8dbde3428e80 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -777,7 +777,7 @@ struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); - nested_stage2_enabled = hcr & HCR_VM; + nested_stage2_enabled = hcr & HCR_EL2_VM; /* Don't consider the CnP bit for the vttbr match */ vttbr &= ~VTTBR_CNP_BIT; @@ -858,7 +858,7 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) */ s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); - s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; + s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_EL2_VM; out: atomic_inc(&s2_mmu->refcnt); @@ -979,7 +979,7 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); } - if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) + if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); } } @@ -2202,7 +2202,7 @@ void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu) * We previously decided that an SError was deliverable to the guest. * Reap the pending state from HCR_EL2 and... */ - if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr))) + if (unlikely(__test_and_clear_bit(__ffs(HCR_EL2_VSE), hcr))) vcpu_set_flag(vcpu, NESTED_SERROR_PENDING); /* diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index e9cf6cc34034..4f18ed2793c3 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -586,7 +586,7 @@ static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift) } /* - * Generic accessor for VM registers. Only called as long as HCR_TVM + * Generic accessor for VM registers. Only called as long as HCR_EL2_TVM * is set. If the guest enables the MMU, we stop trapping the VM * sys_regs and leave it in complete control of the caches. */ @@ -2891,7 +2891,7 @@ static u64 reset_hcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) u64 val = r->val; if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1)) - val |= HCR_E2H; + val |= HCR_EL2_E2H; __vcpu_assign_sys_reg(vcpu, r->reg, val); @@ -5796,31 +5796,31 @@ static void vcpu_set_hcr(struct kvm_vcpu *vcpu) struct kvm *kvm = vcpu->kvm; if (has_vhe() || has_hvhe()) - vcpu->arch.hcr_el2 |= HCR_E2H; + vcpu->arch.hcr_el2 |= HCR_EL2_E2H; if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN)) { /* route synchronous external abort exceptions to EL2 */ - vcpu->arch.hcr_el2 |= HCR_TEA; + vcpu->arch.hcr_el2 |= HCR_EL2_TEA; /* trap error record accesses */ - vcpu->arch.hcr_el2 |= HCR_TERR; + vcpu->arch.hcr_el2 |= HCR_EL2_TERR; } if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - vcpu->arch.hcr_el2 |= HCR_FWB; + vcpu->arch.hcr_el2 |= HCR_EL2_FWB; if (cpus_have_final_cap(ARM64_HAS_EVT) && !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_sanitised_ftr_reg(SYS_CTR_EL0)) - vcpu->arch.hcr_el2 |= HCR_TID4; + vcpu->arch.hcr_el2 |= HCR_EL2_TID4; else - vcpu->arch.hcr_el2 |= HCR_TID2; + vcpu->arch.hcr_el2 |= HCR_EL2_TID2; if (vcpu_el1_is_32bit(vcpu)) - vcpu->arch.hcr_el2 &= ~HCR_RW; + vcpu->arch.hcr_el2 &= ~HCR_EL2_RW; if (kvm_has_mte(vcpu->kvm)) - vcpu->arch.hcr_el2 |= HCR_ATA; + vcpu->arch.hcr_el2 |= HCR_EL2_ATA; else - vcpu->arch.hcr_el2 |= HCR_TID5; + vcpu->arch.hcr_el2 |= HCR_EL2_TID5; /* * In the absence of FGT, we cannot independently trap TLBI @@ -5828,7 +5828,7 @@ static void vcpu_set_hcr(struct kvm_vcpu *vcpu) * TLBIs would be far worse. Live with it... */ if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) - vcpu->arch.hcr_el2 |= HCR_TTLBOS; + vcpu->arch.hcr_el2 |= HCR_EL2_TTLBOS; } void kvm_calculate_traps(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/vgic/vgic-v3-nested.c b/arch/arm64/kvm/vgic/vgic-v3-nested.c index e3e84d7f5ad5..82bfc4d4f122 100644 --- a/arch/arm64/kvm/vgic/vgic-v3-nested.c +++ b/arch/arm64/kvm/vgic/vgic-v3-nested.c @@ -120,8 +120,8 @@ bool vgic_state_is_nested(struct kvm_vcpu *vcpu) u64 xmo; if (is_nested_ctxt(vcpu)) { - xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_IMO | HCR_FMO); - WARN_ONCE(xmo && xmo != (HCR_IMO | HCR_FMO), + xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_EL2_IMO | HCR_EL2_FMO); + WARN_ONCE(xmo && xmo != (HCR_EL2_IMO | HCR_EL2_FMO), "Separate virtual IRQ/FIQ settings not supported\n"); return !!xmo; From d32084d26c217b85e53c61d577178d5aaebc2c5b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Oct 2026 18:36:05 +0100 Subject: [PATCH 35/42] KVM: arm64: Get rid of compatibility HCR_EL2 aliases Now that we dropped out reliance on the old HCR_EL2 aliases, do the right thing and purge the kernel of the compatibility definitions. Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Tested-by: Itaru Kitayama Link: https://patch.msgid.link/20261015173605.1072963-3-maz@kernel.org Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_arm.h | 65 -------------------------------- 1 file changed, 65 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 99acd1525282..ba0062849be9 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -14,71 +14,6 @@ #include #include -/* - * Because I'm terribly lazy and that repainting the whole of the KVM - * code with the proper names is a pain, use a helper to map the names - * inherited from AArch32 with the new fancy nomenclature. One day... - */ -#define __HCR(x) HCR_EL2_##x - -#define HCR_TID5 __HCR(TID5) -#define HCR_DCT __HCR(DCT) -#define HCR_ATA_SHIFT __HCR(ATA_SHIFT) -#define HCR_ATA __HCR(ATA) -#define HCR_TTLBOS __HCR(TTLBOS) -#define HCR_TTLBIS __HCR(TTLBIS) -#define HCR_ENSCXT __HCR(EnSCXT) -#define HCR_TOCU __HCR(TOCU) -#define HCR_AMVOFFEN __HCR(AMVOFFEN) -#define HCR_TICAB __HCR(TICAB) -#define HCR_TID4 __HCR(TID4) -#define HCR_FIEN __HCR(FIEN) -#define HCR_FWB __HCR(FWB) -#define HCR_NV2 __HCR(NV2) -#define HCR_AT __HCR(AT) -#define HCR_NV1 __HCR(NV1) -#define HCR_NV __HCR(NV) -#define HCR_API __HCR(API) -#define HCR_APK __HCR(APK) -#define HCR_TEA __HCR(TEA) -#define HCR_TERR __HCR(TERR) -#define HCR_TLOR __HCR(TLOR) -#define HCR_E2H __HCR(E2H) -#define HCR_ID __HCR(ID) -#define HCR_CD __HCR(CD) -#define HCR_RW __HCR(RW) -#define HCR_TRVM __HCR(TRVM) -#define HCR_HCD __HCR(HCD) -#define HCR_TDZ __HCR(TDZ) -#define HCR_TGE __HCR(TGE) -#define HCR_TVM __HCR(TVM) -#define HCR_TTLB __HCR(TTLB) -#define HCR_TPU __HCR(TPU) -#define HCR_TPC __HCR(TPCP) -#define HCR_TSW __HCR(TSW) -#define HCR_TACR __HCR(TACR) -#define HCR_TIDCP __HCR(TIDCP) -#define HCR_TSC __HCR(TSC) -#define HCR_TID3 __HCR(TID3) -#define HCR_TID2 __HCR(TID2) -#define HCR_TID1 __HCR(TID1) -#define HCR_TID0 __HCR(TID0) -#define HCR_TWE __HCR(TWE) -#define HCR_TWI __HCR(TWI) -#define HCR_DC __HCR(DC) -#define HCR_BSU __HCR(BSU) -#define HCR_BSU_IS __HCR(BSU_IS) -#define HCR_FB __HCR(FB) -#define HCR_VSE __HCR(VSE) -#define HCR_VI __HCR(VI) -#define HCR_VF __HCR(VF) -#define HCR_AMO __HCR(AMO) -#define HCR_IMO __HCR(IMO) -#define HCR_FMO __HCR(FMO) -#define HCR_PTW __HCR(PTW) -#define HCR_SWIO __HCR(SWIO) -#define HCR_VM __HCR(VM) - /* * The bits we set in HCR: * TLOR: Trap LORegion register accesses From 0fa4e506a054702d03d46aac9272a37f820fc2e5 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Mon, 19 Oct 2026 10:42:58 +0100 Subject: [PATCH 36/42] KVM: arm64: Move pkvm_selftests() after all CPUs have initialized EL2 init_subsystems() installs the hyp vector on all CPUs. By running pkvm_selftests() in init_hyp_mode(), there's a risk for a migration to a CPU where the hyp stub is still in place which would fail to run one of the HVCs necessary for the selftest. While init_subsystems() is the earliest we can call pkvm_selftests(), move it to finalize_pkvm() to avoid having to export the selftest function out of pkvm.c. Fixes: bd93068340e9 ("KVM: arm64: Add selftests for the pKVM heap allocator") Reported-by: Fuad Tabba Signed-off-by: Vincent Donnefort Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261019094258.1582756-1-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_pkvm.h | 1 - arch/arm64/kvm/arm.c | 2 -- arch/arm64/kvm/pkvm.c | 4 +++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index e7562770e7cb..fd042d2b5462 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -17,7 +17,6 @@ #define HYP_MEMBLOCK_REGIONS 128 -void pkvm_selftests(void); int pkvm_init_host_vm(struct kvm *kvm); int pkvm_create_hyp_vm(struct kvm *kvm); bool pkvm_hyp_vm_is_created(struct kvm *kvm); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 775d11f96813..57332bbba9d5 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -3109,8 +3109,6 @@ static int __init init_hyp_mode(void) kvm_err("Failed to init hyp memory protection\n"); goto out_err; } - - pkvm_selftests(); } return 0; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 380dfb1db0be..016997288fd2 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -333,7 +333,7 @@ static int __init pkvm_drop_host_privileges(void) return ret; } -void __init pkvm_selftests(void) +static void __init pkvm_selftests(void) { #ifdef CONFIG_NVHE_EL2_DEBUG int ret = pkvm_call_hyp_req(__pkvm_hyp_alloc_selftest); @@ -387,6 +387,8 @@ static int __init finalize_pkvm(void) if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised()) return 0; + pkvm_selftests(); + /* * Exclude HYP sections from kmemleak so that they don't get peeked * at, which would end badly once inaccessible. From 3bb853ad84244f0b86ab12f029bc4a0c9c7100e8 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 15 Oct 2026 12:28:14 +0100 Subject: [PATCH 37/42] tracing/remotes: Fix TRACE_REMOTE build when TRACING=n TRACE_REMOTE relies on tracefs, eventfs and ring-buffer which are only built and declared when TRACING is enabled. It is however possible for a config to have FTRACE=y TRACING=n and TRACE_REMOTE=y. To fix this configuration, have TRACE_REMOTE select TRACING directly, just like other FTRACE options do and drop the TRACING dependency from NVHE_EL2_TRACING to avoid a circular Kconfig dependency. Cc: stable@vger.kernel.org Fixes: ea908a2b79c8 ("tracing: Add a trace remote module for testing") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202610142301.GBepPVmS-lkp@intel.com/ Signed-off-by: Vincent Donnefort Acked-by: Steven Rostedt Link: https://patch.msgid.link/20261015112814.332084-1-vdonnefort@google.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/Kconfig | 2 +- kernel/trace/Kconfig | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 449154f9a485..e179b8ad79e8 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -74,7 +74,7 @@ if NVHE_EL2_DEBUG config NVHE_EL2_TRACING bool - depends on TRACING && FTRACE + depends on FTRACE select TRACE_REMOTE default y diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 0ab5916575a9..6587450cfcd1 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -1329,6 +1329,7 @@ source "kernel/trace/rv/Kconfig" config TRACE_REMOTE bool + select TRACING config SIMPLE_RING_BUFFER bool From a4be8d69698b11d976c122be2d32bb72b8a52809 Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Sun, 25 Oct 2026 21:21:28 -0400 Subject: [PATCH 38/42] KVM: arm64: selftests: Skip unwritable ID fields in the failed-run test test_idreg_frozen_after_failed_run() checks that changing a feature ID register after a failed first KVM_RUN is rejected with -EBUSY, and does so for every field of every register in test_regs. On a host without AArch32 support at EL0, KVM treats the AArch32 ID registers as RAZ/WI: aa32_id_visibility() returns REG_RAZ | REG_USER_WI, and kvm_sys_reg_set_user() ignores the write and returns 0 before set_id_reg() checks whether the ID registers are final. The test then fails on the first such register: arm64/set_id_regs.c:870: r < 0 && errno == EBUSY ID_DFR0_EL1_PerfMon write after failed first run: r=0 errno=16 KernelCI hits this on every run on an AArch64 only GB300 host. Only a field that userspace can change can be frozen, and KVM reports these registers as not writable in KVM_ARM_GET_REG_WRITABLE_MASKS. Skip the fields that the writable mask does not cover. test_vm_ftr_id_regs() already requires the mask to cover every other field in test_regs, so this only drops the fields KVM never lets userspace change. Whether KVM accepts writes to fields it reports as not writable is a separate question from finalization. Fixes: 00e309933879 ("KVM: arm64: selftests: Check ID regs are immutable after a failed run") Link: https://d.kernelci.org/test/maestro:6adcad0c55fda50760800a00 Signed-off-by: Sasha Levin Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20261026012128.2270251-1-sashal@kernel.org Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/set_id_regs.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/arm64/set_id_regs.c b/tools/testing/selftests/kvm/arm64/set_id_regs.c index 10849d21c0dd..1d2cc25cb3f4 100644 --- a/tools/testing/selftests/kvm/arm64/set_id_regs.c +++ b/tools/testing/selftests/kvm/arm64/set_id_regs.c @@ -815,6 +815,10 @@ static void test_idreg_frozen_after_failed_run(void) SYS_REVIDR_EL1, SYS_AIDR_EL1, }; + u64 masks[KVM_ARM_FEATURE_ID_RANGE_SIZE]; + struct reg_mask_range range = { + .addr = (__u64)masks, + }; struct kvm_vcpu_init init; struct kvm_vcpu *vcpu; struct kvm_vm *vm; @@ -831,6 +835,7 @@ static void test_idreg_frozen_after_failed_run(void) vm = vm_create(1); vm_enable_cap(vm, KVM_CAP_ARM_WRITABLE_IMP_ID_REGS, 0); + vm_ioctl(vm, KVM_ARM_GET_REG_WRITABLE_MASKS, &range); kvm_get_default_vcpu_target(vm, &init); init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3); vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); @@ -852,7 +857,9 @@ static void test_idreg_frozen_after_failed_run(void) */ for (int i = 0; i < ARRAY_SIZE(test_regs); i++) { const struct reg_ftr_bits *ftr_bits = test_regs[i].ftr_bits; - u64 reg = KVM_ARM64_SYS_REG(test_regs[i].reg); + u32 reg_id = test_regs[i].reg; + u64 reg = KVM_ARM64_SYS_REG(reg_id); + u64 mask = masks[encoding_to_range_idx(reg_id)]; u64 val = vcpu_get_reg(vcpu, reg); for (int j = 0; ftr_bits[j].type != FTR_END; j++) { @@ -860,6 +867,10 @@ static void test_idreg_frozen_after_failed_run(void) u64 safe = get_safe_value(&ftr_bits[j], ftr); u64 new_val; + /* Only a field userspace could change can be frozen. */ + if ((mask & ftr_bits[j].mask) != ftr_bits[j].mask) + continue; + if (safe == ftr) continue; From 68c11eed4f8097ef1b71c30679e96eb654df29e0 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 20 Oct 2026 07:43:06 +0100 Subject: [PATCH 39/42] KVM: arm64: Sanitise ID_AA64MMFR3_EL1.S1P[IO]E KVM uses the VM's ID registers as the source of truth for what the guest has, but userspace can expose S1PIE or S1POE without the features they imply. As per the architecture, both imply TCR2 and ATS1A, and S1POE also implies HPDS. With TCR2 hidden, the world switch skips PIR_EL1, PIRE0_EL1 and POR_EL1 while KVM doesn't trap them, so the guest accesses registers that are neither trapped nor switched. Under VHE, that leaks values between guests. On nVHE and hVHE, pKVM included, the host kernel resumes with the guest's PIR_EL1 and PIRE0_EL1. On first run, clear S1PIE and S1POE from the VM's ID registers when the features they imply are missing, before anything derives state from them. The guest and userspace then see neither feature, and the guest's accesses to their registers UNDEF. Fixes: 70ed7238297fb ("KVM: arm64: Sanitise ID_AA64MMFR3_EL1") Fixes: d4a89e5aee23e ("KVM: arm64: Expose S1PIE to guests") Reported-by: Mark Brown Link: https://lore.kernel.org/r/20260930-arm64-gcs-v21-2-3556644cd927@kernel.org Link: https://lore.kernel.org/r/8633u229lq.wl-maz@kernel.org Cc: stable@vger.kernel.org Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Reviewed-by: Mark Brown Reviewed-by: Lorenzo Stoakes (ARM) Link: https://patch.msgid.link/20261020064306.1542865-1-fuad.tabba@linux.dev Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/config.c | 78 +++++++++++++++++++++++++++++++ arch/arm64/kvm/sys_regs.c | 2 + 3 files changed, 81 insertions(+) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..524c01d92e71 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1659,6 +1659,7 @@ static inline bool kvm_arch_has_irq_bypass(void) return true; } +void kvm_idregs_enforce_dependencies(struct kvm *kvm); void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt); struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg); void check_feature_map(void); diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index 1053676551af..cfdb6c713320 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -59,6 +59,7 @@ struct reg_feat_map_desc { const struct reg_bits_to_feat_map feat_map; const struct reg_bits_to_feat_map *bit_feat_map; const unsigned int bit_feat_map_sz; + const unsigned int idreg_idx; }; #define __NEEDS_FEAT_3(m, f, w, id, fld, lim) \ @@ -130,6 +131,21 @@ struct reg_feat_map_desc { .bit_feat_map_sz = ARRAY_SIZE(m), \ } +/* + * Declare the dependency between the features advertised in an id + * register, and the set of other features they depend on. This + * generates a struct reg_feat_map_desc. + */ +#define DECLARE_IDREG_FEAT_MAP(n, r, m, f) \ + struct reg_feat_map_desc n = { \ + .name = #r, \ + .feat_map = NEEDS_FEAT(~(r##_RES0 | \ + r##_RES1), f), \ + .bit_feat_map = m, \ + .bit_feat_map_sz = ARRAY_SIZE(m), \ + .idreg_idx = IDREG_IDX(SYS_##r), \ + } + #define FEAT_SPE ID_AA64DFR0_EL1, PMSVer, IMP #define FEAT_BRBE ID_AA64DFR0_EL1, BRBE, IMP #define FEAT_TRC_SR ID_AA64DFR0_EL1, TraceVer, IMP @@ -205,6 +221,7 @@ struct reg_feat_map_desc { #define FEAT_MEC ID_AA64MMFR3_EL1, MEC, IMP #define FEAT_HAFT ID_AA64MMFR1_EL1, HAFDBS, HAFT #define FEAT_HDBSS ID_AA64MMFR1_EL1, HAFDBS, HDBSS +#define FEAT_HPDS ID_AA64MMFR1_EL1, HPDS, IMP #define FEAT_HPDS2 ID_AA64MMFR1_EL1, HPDS, HPDS2 #define FEAT_BTI ID_AA64PFR1_EL1, BT, IMP #define FEAT_ExS ID_AA64MMFR0_EL1, EXS, IMP @@ -280,6 +297,25 @@ static bool feat_anerr(struct kvm *kvm) kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SNERR, FEAT_ANERR)); } +/* + * S1PIE and S1POE imply TCR2 and ATS1A, and S1POE also implies HPDS, but + * userspace can set them independently. + */ +static bool feat_s1pie(struct kvm *kvm) +{ + return (kvm_has_feat(kvm, FEAT_S1PIE) && + kvm_has_feat(kvm, FEAT_TCR2) && + kvm_has_feat(kvm, FEAT_ATS1A)); +} + +static bool feat_s1poe(struct kvm *kvm) +{ + return (kvm_has_feat(kvm, FEAT_S1POE) && + kvm_has_feat(kvm, FEAT_TCR2) && + kvm_has_feat(kvm, FEAT_ATS1A) && + kvm_has_feat(kvm, FEAT_HPDS)); +} + static bool feat_sme_smps(struct kvm *kvm) { /* @@ -1299,6 +1335,14 @@ static const struct reg_bits_to_feat_map vtcr_el2_feat_map[] = { static const DECLARE_FEAT_MAP(vtcr_el2_desc, VTCR_EL2, vtcr_el2_feat_map, FEAT_AA64EL2); +static const struct reg_bits_to_feat_map id_aa64mmfr3_el1_feat_map[] = { + NEEDS_FEAT(ID_AA64MMFR3_EL1_S1PIE, feat_s1pie), + NEEDS_FEAT(ID_AA64MMFR3_EL1_S1POE, feat_s1poe), +}; + +static const DECLARE_IDREG_FEAT_MAP(id_aa64mmfr3_el1_desc, ID_AA64MMFR3_EL1, + id_aa64mmfr3_el1_feat_map, FEAT_AA64EL1); + static const struct reg_bits_to_feat_map ich_hfgrtr_feat_map[] = { NEEDS_FEAT(ICH_HFGRTR_EL2_ICC_APR_EL1 | ICH_HFGRTR_EL2_ICC_IDRn_EL1 | @@ -1408,6 +1452,40 @@ void __init check_feature_map(void) check_reg_desc(&ich_hfgitr_desc); } +static const struct reg_feat_map_desc *idreg_feat_maps[] = { + &id_aa64mmfr3_el1_desc, +}; + +static bool idreg_apply_deps(struct kvm *kvm, const struct reg_feat_map_desc *r) +{ + struct resx resx = {}; + u64 val; + + for (int i = 0; i < r->bit_feat_map_sz; i++) { + if ((r->bit_feat_map[i].flags & CALL_FUNC) && + !r->bit_feat_map[i].match(kvm)) + resx.res0 |= reg_feat_map_bits(&r->bit_feat_map[i]); + } + + val = kvm->arch.id_regs[r->idreg_idx]; + kvm->arch.id_regs[r->idreg_idx] &= ~resx.res0; + kvm->arch.id_regs[r->idreg_idx] |= resx.res1; + + return val != kvm->arch.id_regs[r->idreg_idx]; +} + +void kvm_idregs_enforce_dependencies(struct kvm *kvm) +{ + bool again; + + do { + again = false; + + for (int i = 0; i < ARRAY_SIZE(idreg_feat_maps); i++) + again |= idreg_apply_deps(kvm, idreg_feat_maps[i]); + } while (again); +} + static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map) { u64 regval = kvm->arch.id_regs[map->regidx]; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 44aae52c473d..11f08e2980c4 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -5883,6 +5883,8 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) if (kvm_vm_has_ran_once(kvm)) return 0; + kvm_idregs_enforce_dependencies(kvm); + /* * This hacks into the ID registers, so only perform it when the * first vcpu runs, or the kvm_set_vm_id_reg() helper will scream. From 576a1b5ab4b8be03ab5cf7b41b475850795e3fe2 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Wed, 21 Oct 2026 09:22:04 +0100 Subject: [PATCH 40/42] KVM: arm64: gic-v5: Drop unused addr variable vgic_v5_has_attr_regs() assigns reg_attr.addr to addr, but never uses addr. Drop the unused local and assignment to avoid a W=1 warning. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202610210232.uzFGXnjw-lkp@intel.com/ Signed-off-by: Sascha Bischoff Link: https://patch.msgid.link/20261021082204.1494581-1-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-irs-v5.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c index 1874c3c289a6..8d0ad74ebc64 100644 --- a/arch/arm64/kvm/vgic/vgic-irs-v5.c +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -1134,7 +1134,7 @@ int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) const struct vgic_register_region *region; struct vgic_reg_attr reg_attr; struct kvm_vcpu *vcpu; - gpa_t addr, offset; + gpa_t offset; int ret, align; ret = vgic_v5_parse_attr(dev, attr, ®_attr); @@ -1142,7 +1142,6 @@ int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) return ret; vcpu = reg_attr.vcpu; - addr = reg_attr.addr; if (attr->group == KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS) return vgic_v5_has_cpu_sysregs_attr(vcpu, attr); From 05620a31421935f737d16a060d2dfe731f7b2f74 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 20 Oct 2026 17:39:22 +0100 Subject: [PATCH 41/42] KVM: arm64: vgic: Require matching VGIC and ITS models On a GICv5 host with legacy support, KVM registers both VGICv3 and VGICv5 devices. Without a model check, a VM can create a GICv3 ITS alongside an incompatible VGIC, regardless of creation order. The same problem exists for GICv2 on hosts that support a GICv3 ITS. KVM used to allow userspace to create both devices in either order, then ignored the ITS. Record the VGIC model required by an ITS. Reject an ITS if the VM already has an incompatible VGIC, and reject a VGIC if an existing ITS requires a different model. These checks apply on every host: a GICv2 VGIC and a GICv3 ITS now fail with -ENODEV in either order. Allow either creation order when the models match. Have vgic_has_its() require matching VGIC and ITS models, and drop the separate has_its flag. As part of this change, swap the type of vgic_model from u32 to enum kvm_device_type. This requires updating the switch cases in kvm_vgic_vcpu_init() and kvm_vgic_dist_destroy() to include default cases, which we mark as unreachable(). Signed-off-by: Sascha Bischoff Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261020163923.1407856-2-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- arch/arm64/kvm/vgic/vgic-init.c | 17 +++++++++++++++++ arch/arm64/kvm/vgic/vgic-its.c | 15 ++++++++++++++- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 5 +---- include/kvm/arm_vgic.h | 5 +++-- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 979301c3dbd8..c9991e24cb6b 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -121,6 +121,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) goto out_unlock; } + if (kvm->arch.vgic.its_model && kvm->arch.vgic.its_model != type) { + ret = -ENODEV; + goto out_unlock; + } + if (kvm_id_regs_final(kvm)) goto out_unlock; ret = 0; @@ -433,6 +438,8 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) return ret; switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V2: + break; case KVM_DEV_TYPE_ARM_VGIC_V3: /* * If we are creating a VCPU with a GICv3 we must also register @@ -451,6 +458,9 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) */ if (vcpu->vcpu_id >= kvm_vgic_global_state.max_gicv5_vcpus) return -EINVAL; + break; + default: + unreachable(); } return ret; @@ -550,6 +560,10 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) dist->nr_spis = 0; dist->vgic_dist_base = VGIC_ADDR_UNDEF; + /* A VM may be destroyed without ever creating a VGIC. */ + if (!dist->vgic_model) + goto out; + switch (dist->vgic_model) { case KVM_DEV_TYPE_ARM_VGIC_V2: dist->vgic_cpu_base = VGIC_ADDR_UNDEF; @@ -567,8 +581,11 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) kfree(dist->vgic_v5_irs_data); dist->vgic_v5_irs_data = NULL; break; + default: + unreachable(); } +out: xa_destroy(&dist->lpi_xa); } diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index e5657cb60356..7df5869cff99 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -1875,6 +1875,19 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) mutex_lock(&dev->kvm->arch.config_lock); + /* + * Reject creation of a GICv3 ITS if: + * - The VM already has a VGIC other than GICv3. + * - An existing ITS requires a VGIC model other than GICv3. + */ + if ((irqchip_in_kernel(dev->kvm) && !vgic_is_v3(dev->kvm)) || + (dev->kvm->arch.vgic.its_model && + dev->kvm->arch.vgic.its_model != KVM_DEV_TYPE_ARM_VGIC_V3)) { + mutex_unlock(&dev->kvm->arch.config_lock); + kfree(its); + return -ENODEV; + } + if (vgic_initialized(dev->kvm)) { ret = vgic_v4_init(dev->kvm); if (ret < 0) { @@ -1902,7 +1915,7 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) xa_init(&its->translation_cache); dev->kvm->arch.vgic.msis_require_devid = true; - dev->kvm->arch.vgic.has_its = true; + dev->kvm->arch.vgic.its_model = KVM_DEV_TYPE_ARM_VGIC_V3; its->enabled = false; its->dev = dev; diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 5ab789accc9d..08f867ebc0df 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -42,10 +42,7 @@ bool vgic_has_its(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; - if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) - return false; - - return dist->has_its; + return dist->its_model && dist->its_model == dist->vgic_model; } bool vgic_supports_direct_msis(struct kvm *kvm) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 7be81b936bf1..cad74eaff0a7 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -502,7 +502,9 @@ struct vgic_dist { bool initialized; /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */ - u32 vgic_model; + enum kvm_device_type vgic_model; + /* VGIC model required by the VM's ITS devices. */ + enum kvm_device_type its_model; /* Implementation revision as reported in the GICD_IIDR */ u32 implementation_rev; @@ -547,7 +549,6 @@ struct vgic_dist { struct vgic_io_device dist_iodev; struct vgic_io_device cpuif_iodev; - bool has_its; bool table_write_in_progress; /* From b17eea28dc815f63aa3279a66fd1e4de47a8a446 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 20 Oct 2026 17:39:23 +0100 Subject: [PATCH 42/42] KVM: selftests: Reject GICv3 ITS with a GICv5 VGIC Check that a GICv3 ITS and a GICv5 VGIC cannot be paired, regardless of which device is created first. Skip the check when the host does not expose a GICv3 ITS. Signed-off-by: Sascha Bischoff Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20261020163923.1407856-3-sascha.bischoff@arm.com Signed-off-by: Marc Zyngier --- tools/testing/selftests/kvm/arm64/vgic_v5.c | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index d5df742b2300..5bf7d99d2914 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -2003,8 +2003,40 @@ int test_kvm_device(u32 gic_dev_type) return 0; } +static void test_vgic_v3_its_with_vgic_v5(void) +{ + struct kvm_vm *vm = vm_create_barebones(); + int gic_fd, its_fd, ret; + + if (__kvm_test_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS)) { + pr_info("No GICv3 ITS support; skipping mixed-model test.\n"); + kvm_vm_free(vm); + return; + } + + gic_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V5); + ret = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS); + TEST_ASSERT(ret < 0 && errno == ENODEV, + "GICv3 ITS created after GICv5 VGIC: ret=%d errno=%d", + ret, errno); + close(gic_fd); + kvm_vm_free(vm); + + vm = vm_create_barebones(); + its_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_ITS); + ret = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V5); + TEST_ASSERT(ret < 0 && errno == ENODEV, + "GICv5 VGIC created after GICv3 ITS: ret=%d errno=%d", + ret, errno); + close(its_fd); + kvm_vm_free(vm); +} + void run_tests(u32 gic_dev_type) { + pr_info("Test VGICv5 and GICv3 ITS model mismatch\n"); + test_vgic_v3_its_with_vgic_v5(); + pr_info("Test VGICv5 address attrs\n"); test_vgic_v5_addr_attrs();