From 8cc9dc1ae4fb075e29e2d5cf2386761d3497049a Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sat, 5 Jul 2025 07:17:12 +0000 Subject: [PATCH 1/5] KVM: arm64: Rename the device variable to s2_force_noncacheable To perform cache maintenance on a region of memory, KVM/arm64 relies on that region having a cacheable alias in the kernel's address space which can be used with CMO instructions. The 'device' variable is somewhat of a misnomer, as it actually indicates whether or not the stage-2 alias is allowed to have cacheable memory attributes. The resulting stage-2 memory attributes are further modified by VM_ALLOW_ANY_UNCACHED, selecting between Normal-NC or Device-nGnRE depending on what the endpoint supports. Rename the to s2_force_noncacheable such that its purpose is a bit more obvious. CC: Catalin Marinas Suggested-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Reviewed-by: David Hildenbrand Tested-by: Donald Dutile Signed-off-by: Ankit Agrawal Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20250705071717.5062-2-ankita@nvidia.com [ Oliver: addressed typos, wound up rewriting changelog ] Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 2942ec92c5a4..1601ab9527d4 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1478,7 +1478,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, int ret = 0; bool write_fault, writable, force_pte = false; bool exec_fault, mte_allowed; - bool device = false, vfio_allow_any_uc = false; + bool s2_force_noncacheable = false, vfio_allow_any_uc = false; unsigned long mmu_seq; phys_addr_t ipa = fault_ipa; struct kvm *kvm = vcpu->kvm; @@ -1653,7 +1653,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * In both cases, we don't let transparent_hugepage_adjust() * change things at the last minute. */ - device = true; + s2_force_noncacheable = true; } else if (logging_active && !write_fault) { /* * Only actually map the page as writable if this was a write @@ -1662,7 +1662,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, writable = false; } - if (exec_fault && device) + if (exec_fault && s2_force_noncacheable) return -ENOEXEC; /* @@ -1695,7 +1695,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * If we are not forced to use page mapping, check if we are * backed by a THP and thus use block mapping if possible. */ - if (vma_pagesize == PAGE_SIZE && !(force_pte || device)) { + if (vma_pagesize == PAGE_SIZE && !(force_pte || s2_force_noncacheable)) { if (fault_is_perm && fault_granule > PAGE_SIZE) vma_pagesize = fault_granule; else @@ -1709,7 +1709,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, } } - if (!fault_is_perm && !device && kvm_has_mte(kvm)) { + if (!fault_is_perm && !s2_force_noncacheable && kvm_has_mte(kvm)) { /* Check the VMM hasn't introduced a new disallowed VMA */ if (mte_allowed) { sanitise_mte_tags(kvm, pfn, vma_pagesize); @@ -1725,7 +1725,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, if (exec_fault) prot |= KVM_PGTABLE_PROT_X; - if (device) { + if (s2_force_noncacheable) { if (vfio_allow_any_uc) prot |= KVM_PGTABLE_PROT_NORMAL_NC; else From 216887f79d9878d9cb169729fea3bb16c56db465 Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sat, 5 Jul 2025 07:17:13 +0000 Subject: [PATCH 2/5] KVM: arm64: Assume non-PFNMAP/MIXEDMAP VMAs can be mapped cacheable Despite its name, kvm_is_device_pfn() is actually used to determine if a given PFN has a kernel mapping that can be used to perform cache maintenance, as it calls pfn_is_map_memory() internally. Expand the helper into its single callsite and further condition the check on the VMA having either VM_PFNMAP or VM_MIXEDMAP set. VMAs that set neither of these flags must always contain Normal, struct page backed memory with valid aliases in the kernel address space. Suggested-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Reviewed-by: David Hildenbrand Tested-by: Donald Dutile Signed-off-by: Ankit Agrawal Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20250705071717.5062-3-ankita@nvidia.com [ Oliver: fixed typos, refined changelog ] Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 1601ab9527d4..5fe24f30999d 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -193,11 +193,6 @@ int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, return 0; } -static bool kvm_is_device_pfn(unsigned long pfn) -{ - return !pfn_is_map_memory(pfn); -} - static void *stage2_memcache_zalloc_page(void *arg) { struct kvm_mmu_memory_cache *mc = arg; @@ -1492,6 +1487,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; struct kvm_pgtable *pgt; struct page *page; + vm_flags_t vm_flags; enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED; if (fault_is_perm) @@ -1619,6 +1615,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, vfio_allow_any_uc = vma->vm_flags & VM_ALLOW_ANY_UNCACHED; + vm_flags = vma->vm_flags; + /* Don't use the VMA after the unlock -- it may have vanished */ vma = NULL; @@ -1642,7 +1640,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, if (is_error_noslot_pfn(pfn)) return -EFAULT; - if (kvm_is_device_pfn(pfn)) { + if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP) && !pfn_is_map_memory(pfn)) { /* * If the page was identified as device early by looking at * the VMA flags, vma_pagesize is already representing the From 2a8dfab26677ae37243a40d170704588f791cfd3 Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sat, 5 Jul 2025 07:17:14 +0000 Subject: [PATCH 3/5] KVM: arm64: Block cacheable PFNMAP mapping Fixes a security bug due to mismatched attributes between S1 and S2 mapping. Currently, it is possible for a region to be cacheable in the userspace VMA, but mapped non cached in S2. This creates a potential issue where the VMM may sanitize cacheable memory across VMs using cacheable stores, ensuring it is zeroed. However, if KVM subsequently assigns this memory to a VM as uncached, the VM could end up accessing stale, non-zeroed data from a previous VM, leading to unintended data exposure. This is a security risk. Block such mismatch attributes case by returning EINVAL when userspace try to map PFNMAP cacheable. Only allow NORMAL_NC and DEVICE_*. CC: Oliver Upton CC: Catalin Marinas CC: Sean Christopherson Suggested-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Reviewed-by: David Hildenbrand Tested-by: Donald Dutile Signed-off-by: Ankit Agrawal Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20250705071717.5062-4-ankita@nvidia.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 5fe24f30999d..708a635e38bc 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1465,6 +1465,18 @@ static bool kvm_vma_mte_allowed(struct vm_area_struct *vma) return vma->vm_flags & VM_MTE_ALLOWED; } +static bool kvm_vma_is_cacheable(struct vm_area_struct *vma) +{ + switch (FIELD_GET(PTE_ATTRINDX_MASK, pgprot_val(vma->vm_page_prot))) { + case MT_NORMAL_NC: + case MT_DEVICE_nGnRnE: + case MT_DEVICE_nGnRE: + return false; + default: + return true; + } +} + static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_s2_trans *nested, struct kvm_memory_slot *memslot, unsigned long hva, @@ -1472,7 +1484,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, { int ret = 0; bool write_fault, writable, force_pte = false; - bool exec_fault, mte_allowed; + bool exec_fault, mte_allowed, is_vma_cacheable; bool s2_force_noncacheable = false, vfio_allow_any_uc = false; unsigned long mmu_seq; phys_addr_t ipa = fault_ipa; @@ -1617,6 +1629,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, vm_flags = vma->vm_flags; + is_vma_cacheable = kvm_vma_is_cacheable(vma); + /* Don't use the VMA after the unlock -- it may have vanished */ vma = NULL; @@ -1660,6 +1674,15 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, writable = false; } + /* + * Prevent non-cacheable mappings in the stage-2 if a region of memory + * is cacheable in the primary MMU and the kernel lacks a cacheable + * alias. KVM cannot guarantee coherency between the guest/host aliases + * without the ability to perform CMOs. + */ + if (is_vma_cacheable && s2_force_noncacheable) + return -EINVAL; + if (exec_fault && s2_force_noncacheable) return -ENOEXEC; @@ -2219,6 +2242,12 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, ret = -EINVAL; break; } + + /* Cacheable PFNMAP is not allowed */ + if (kvm_vma_is_cacheable(vma)) { + ret = -EINVAL; + break; + } } hva = min(reg_end, vma->vm_end); } while (hva < reg_end); From 0c67288e0c8bc1e39d7057fbae65bf57ca943676 Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sat, 5 Jul 2025 07:17:15 +0000 Subject: [PATCH 4/5] KVM: arm64: Allow cacheable stage 2 mapping using VMA flags KVM currently forces non-cacheable memory attributes (either Normal-NC or Device-nGnRE) for a region based on pfn_is_map_memory(), i.e. whether or not the kernel has a cacheable alias for it. This is necessary in situations where KVM needs to perform CMOs on the region but is unnecessarily restrictive when hardware obviates the need for CMOs. KVM doesn't need to perform any CMOs on hardware with FEAT_S2FWB and CTR_EL0.DIC. As luck would have it, there are implementations in the wild that need to map regions of a device with cacheable attributes to function properly. An example of this is Nvidia's Grace Hopper/Blackwell systems where GPU memory is interchangeable with DDR and retains properties such as cacheability, unaligned accesses, atomics and handling of executable faults. Of course, for this to work in a VM the GPU memory needs to have a cacheable mapping at stage-2. Allow cacheable stage-2 mappings to be created on supporting hardware when the VMA has cacheable memory attributes. Check these preconditions during memslot creation (in addition to fault handling) to potentially 'fail-fast' as a courtesy to userspace. CC: Oliver Upton CC: Sean Christopherson Suggested-by: Jason Gunthorpe Suggested-by: Catalin Marinas Suggested-by: David Hildenbrand Tested-by: Donald Dutile Signed-off-by: Ankit Agrawal Reviewed-by: David Hildenbrand Reviewed-by: Jason Gunthorpe Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20250705071717.5062-6-ankita@nvidia.com [ Oliver: refine changelog, squash kvm_supports_cacheable_pfnmap() patch ] Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_mmu.h | 18 ++++++++++ arch/arm64/kvm/mmu.c | 59 ++++++++++++++++++++------------ 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index b98ac6aa631f..ae563ebd6aee 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -371,6 +371,24 @@ static inline void kvm_fault_unlock(struct kvm *kvm) read_unlock(&kvm->mmu_lock); } +/* + * ARM64 KVM relies on a simple conversion from physaddr to a kernel + * virtual address (KVA) when it does cache maintenance as the CMO + * instructions work on virtual addresses. This is incompatible with + * VM_PFNMAP VMAs which may not have a kernel direct mapping to a + * virtual address. + * + * With S2FWB and CACHE DIC features, KVM need not do cache flushing + * and CMOs are NOP'd. This has the effect of no longer requiring a + * KVA for addresses mapped into the S2. The presence of these features + * are thus necessary to support cacheable S2 mapping of VM_PFNMAP. + */ +static inline bool kvm_supports_cacheable_pfnmap(void) +{ + return cpus_have_final_cap(ARM64_HAS_STAGE2_FWB) && + cpus_have_final_cap(ARM64_HAS_CACHE_DIC); +} + #ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS void kvm_s2_ptdump_create_debugfs(struct kvm *kvm); #else diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 708a635e38bc..3a9e2248f82d 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1654,18 +1654,39 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, if (is_error_noslot_pfn(pfn)) return -EFAULT; + /* + * Check if this is non-struct page memory PFN, and cannot support + * CMOs. It could potentially be unsafe to access as cachable. + */ if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP) && !pfn_is_map_memory(pfn)) { - /* - * If the page was identified as device early by looking at - * the VMA flags, vma_pagesize is already representing the - * largest quantity we can map. If instead it was mapped - * via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE - * and must not be upgraded. - * - * In both cases, we don't let transparent_hugepage_adjust() - * change things at the last minute. - */ - s2_force_noncacheable = true; + if (is_vma_cacheable) { + /* + * Whilst the VMA owner expects cacheable mapping to this + * PFN, hardware also has to support the FWB and CACHE DIC + * features. + * + * ARM64 KVM relies on kernel VA mapping to the PFN to + * perform cache maintenance as the CMO instructions work on + * virtual addresses. VM_PFNMAP region are not necessarily + * mapped to a KVA and hence the presence of hardware features + * S2FWB and CACHE DIC are mandatory to avoid the need for + * cache maintenance. + */ + if (!kvm_supports_cacheable_pfnmap()) + return -EFAULT; + } else { + /* + * If the page was identified as device early by looking at + * the VMA flags, vma_pagesize is already representing the + * largest quantity we can map. If instead it was mapped + * via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE + * and must not be upgraded. + * + * In both cases, we don't let transparent_hugepage_adjust() + * change things at the last minute. + */ + s2_force_noncacheable = true; + } } else if (logging_active && !write_fault) { /* * Only actually map the page as writable if this was a write @@ -1674,15 +1695,6 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, writable = false; } - /* - * Prevent non-cacheable mappings in the stage-2 if a region of memory - * is cacheable in the primary MMU and the kernel lacks a cacheable - * alias. KVM cannot guarantee coherency between the guest/host aliases - * without the ability to perform CMOs. - */ - if (is_vma_cacheable && s2_force_noncacheable) - return -EINVAL; - if (exec_fault && s2_force_noncacheable) return -ENOEXEC; @@ -2243,8 +2255,11 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, break; } - /* Cacheable PFNMAP is not allowed */ - if (kvm_vma_is_cacheable(vma)) { + /* + * Cacheable PFNMAP is allowed only if the hardware + * supports it. + */ + if (kvm_vma_is_cacheable(vma) && !kvm_supports_cacheable_pfnmap()) { ret = -EINVAL; break; } From f55ce5a6cd33211c8cc5bce0554b6ac710a6a28b Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sat, 5 Jul 2025 07:17:17 +0000 Subject: [PATCH 5/5] KVM: arm64: Expose new KVM cap for cacheable PFNMAP Introduce a new KVM capability to expose to the userspace whether cacheable mapping of PFNMAP is supported. The ability to safely do the cacheable mapping of PFNMAP is contingent on S2FWB and ARM64_HAS_CACHE_DIC. S2FWB allows KVM to avoid flushing the D cache, ARM64_HAS_CACHE_DIC allows KVM to avoid flushing the icache and turns icache_inval_pou() into a NOP. The cap would be false if those requirements are missing and is checked by making use of kvm_arch_supports_cacheable_pfnmap. This capability would allow userspace to discover the support. It could for instance be used by userspace to prevent live-migration across FWB and non-FWB hosts. CC: Catalin Marinas CC: Jason Gunthorpe CC: Oliver Upton CC: David Hildenbrand Suggested-by: Marc Zyngier Reviewed-by: Jason Gunthorpe Tested-by: Donald Dutile Signed-off-by: Ankit Agrawal Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20250705071717.5062-7-ankita@nvidia.com Signed-off-by: Oliver Upton --- Documentation/virt/kvm/api.rst | 13 ++++++++++++- arch/arm64/kvm/arm.c | 7 +++++++ include/uapi/linux/kvm.h | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 9abf93ee5f65..53e0179d5294 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -8585,7 +8585,7 @@ ENOSYS for the others. When enabled, KVM will exit to userspace with KVM_EXIT_SYSTEM_EVENT of type KVM_SYSTEM_EVENT_SUSPEND to process the guest suspend request. -7.37 KVM_CAP_ARM_WRITABLE_IMP_ID_REGS +7.42 KVM_CAP_ARM_WRITABLE_IMP_ID_REGS ------------------------------------- :Architectures: arm64 @@ -8614,6 +8614,17 @@ given VM. When this capability is enabled, KVM resets the VCPU when setting MP_STATE_INIT_RECEIVED through IOCTL. The original MP_STATE is preserved. +7.43 KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED +------------------------------------------- + +:Architectures: arm64 +:Target: VM +:Parameters: None + +This capability indicate to the userspace whether a PFNMAP memory region +can be safely mapped as cacheable. This relies on the presence of +force write back (FWB) feature support on the hardware. + 8. Other capabilities. ====================== diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 38a91bb5d4c7..25609aab032c 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -408,6 +408,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES: r = BIT(0); break; + case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED: + if (!kvm) + r = -EINVAL; + else + r = kvm_supports_cacheable_pfnmap(); + break; + default: r = 0; } diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 37891580d05d..e4e566ff348b 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -956,6 +956,7 @@ struct kvm_enable_cap { #define KVM_CAP_ARM_EL2 240 #define KVM_CAP_ARM_EL2_E2H0 241 #define KVM_CAP_RISCV_MP_STATE_RESET 242 +#define KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED 243 struct kvm_irq_routing_irqchip { __u32 irqchip;