github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch kvm-arm64/mmu-fixes-7.4 into kvmarm-master/next

Browse Source 
* kvm-arm64/mmu-fixes-7.4:
  : \
  : Allow running protected VMs and guest_memfd-back VMs on systems with
  : page size larger than 4K, as both pKVM and guest-memfd misinterpret
  : the faulting addresses with 16k and 64k pages.
  :
  : Patches courtesy of Vincent Donnefort and Fuad Tabba.
  : /
  KVM: arm64: Use kvm_s2_fault_vma_info in pkvm_mem_abort()
  KVM: arm64: Use kvm_s2_fault_vma_info in gmem_abort()
  KVM: arm64: Pass kvm_s2_fault_desc to fault_supports_stage2_huge_mapping()
  KVM: arm64: Fix MMFR0 TGRAN advertisement for pVMs

Signed-off-by: Marc Zyngier <maz@kernel.org>
This commit is contained in:
Marc Zyngier 2026-10-11 10:50:30 +01:00
commit 65f95b2874
3 changed files with 87 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
arch/arm64/kvm/hyp/nvhe/sys_regs.c
View File

@ -121,6 +121,9 @@ static const struct pvm_ftr_bits pvmid_aa64mmfr0[] = {
MAX_FEAT(ID_AA64MMFR0_EL1, SNSMEM, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, SNSMEM, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, BIGENDEL0, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, BIGENDEL0, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, EXS, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, EXS, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN4, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN16, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN64, IMP),
FEAT_END FEAT_END
}; };

153
arch/arm64/kvm/mmu.c
View File

@ -1406,21 +1406,53 @@ static void kvm_send_hwpoison_signal(unsigned long address, short lsb)
send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current); send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current);
} }
static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, struct kvm_s2_fault_desc {
unsigned long hva, struct kvm_vcpu *vcpu;
phys_addr_t fault_ipa;
struct kvm_s2_trans *nested;
struct kvm_memory_slot *memslot;
unsigned long hva;
unsigned long esr;
struct kvm_s2_mmu *mmu;
};
struct kvm_s2_fault_vma_info {
unsigned long mmu_seq;
long vma_pagesize;
vm_flags_t vm_flags;
unsigned long max_map_size;
struct page *page;
kvm_pfn_t pfn;
gfn_t gfn;
bool device;
bool mte_allowed;
bool is_vma_cacheable;
bool map_writable;
bool map_non_cacheable;
};
struct kvm_s2_fault_result {
unsigned long mapping_size;
};
static bool fault_supports_stage2_huge_mapping(const struct kvm_s2_fault_desc *s2fd,
unsigned long map_size) unsigned long map_size)
{ {
gpa_t gpa_start; struct kvm_memory_slot *memslot = s2fd->memslot;
unsigned long hva = s2fd->hva;
hva_t uaddr_start, uaddr_end; hva_t uaddr_start, uaddr_end;
gpa_t gpa_start;
size_t size; size_t size;
/* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */
if (map_size == PAGE_SIZE) if (map_size == PAGE_SIZE)
return true; return true;
/* pKVM only supports PMD_SIZE huge-mappings */ /* pKVM only supports PMD_SIZE huge-mappings for non-protected VMs */
if (is_protected_kvm_enabled() && map_size != PMD_SIZE) if (is_protected_kvm_enabled()) {
return false; if (vcpu_is_protected(s2fd->vcpu) || map_size != PMD_SIZE)
return false;
}
size = memslot->npages * PAGE_SIZE; size = memslot->npages * PAGE_SIZE;
@ -1480,9 +1512,9 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot,
* Returns the size of the mapping. * Returns the size of the mapping.
*/ */
static long static long
transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, transparent_hugepage_adjust(const struct kvm_s2_fault_desc *s2fd, kvm_pfn_t *pfnp, gfn_t *gfnp)
unsigned long hva, kvm_pfn_t *pfnp, gfn_t *gfnp)
{ {
struct kvm *kvm = s2fd->vcpu->kvm;
kvm_pfn_t pfn = *pfnp; kvm_pfn_t pfn = *pfnp;
gfn_t gfn = *gfnp; gfn_t gfn = *gfnp;
@ -1491,8 +1523,8 @@ transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot,
* sure that the HVA and IPA are sufficiently aligned and that the * sure that the HVA and IPA are sufficiently aligned and that the
* block map is contained within the memslot. * block map is contained within the memslot.
*/ */
if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) {
int sz = get_user_mapping_size(kvm, hva); int sz = get_user_mapping_size(kvm, s2fd->hva);
if (sz < 0) if (sz < 0)
return sz; return sz;
@ -1650,20 +1682,6 @@ static enum kvm_pgtable_prot adjust_nested_exec_perms(struct kvm *kvm,
return prot; return prot;
} }
struct kvm_s2_fault_desc {
struct kvm_vcpu *vcpu;
phys_addr_t fault_ipa;
struct kvm_s2_trans *nested;
struct kvm_memory_slot *memslot;
unsigned long hva;
unsigned long esr;
struct kvm_s2_mmu *mmu;
};
struct kvm_s2_fault_result {
unsigned long mapping_size;
};
static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd)
{ {
return esr_fsc_is_permission_fault(s2fd->esr); return esr_fsc_is_permission_fault(s2fd->esr);
@ -1689,6 +1707,12 @@ static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd)
return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level));
} }
static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd,
struct kvm_s2_fault_vma_info *s2vi);
static gfn_t get_canonical_gfn(const struct kvm_s2_fault_desc *s2fd,
const struct kvm_s2_fault_vma_info *s2vi);
static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
struct kvm_s2_fault_result *result) struct kvm_s2_fault_result *result)
{ {
@ -1698,12 +1722,10 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
struct kvm_pgtable *pgt = s2fd->mmu->pgt; struct kvm_pgtable *pgt = s2fd->mmu->pgt;
struct kvm_guest_s2_mapping *mapping = NULL; struct kvm_guest_s2_mapping *mapping = NULL;
unsigned long mmu_seq; struct kvm_s2_fault_vma_info s2vi = {};
struct page *page;
struct kvm *kvm = s2fd->vcpu->kvm; struct kvm *kvm = s2fd->vcpu->kvm;
void *memcache = NULL; void *memcache = NULL;
kvm_pfn_t pfn; gfn_t canonical_gfn;
gfn_t gfn;
int ret; int ret;
if (!perm_fault) { if (!perm_fault) {
@ -1718,28 +1740,27 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
} }
} }
if (s2fd->nested) s2vi.vma_pagesize = PAGE_SIZE;
gfn = kvm_s2_trans_output(s2fd->nested) >> PAGE_SHIFT; s2vi.gfn = ALIGN_DOWN(s2fd->fault_ipa, s2vi.vma_pagesize) >> PAGE_SHIFT;
else canonical_gfn = get_canonical_gfn(s2fd, &s2vi);
gfn = s2fd->fault_ipa >> PAGE_SHIFT;
write_fault = kvm_s2_fault_is_write(s2fd); write_fault = kvm_s2_fault_is_write(s2fd);
exec_fault = kvm_s2_fault_is_exec(s2fd); exec_fault = kvm_s2_fault_is_exec(s2fd);
VM_WARN_ON_ONCE(write_fault && exec_fault); VM_WARN_ON_ONCE(write_fault && exec_fault);
mmu_seq = kvm->mmu_invalidate_seq; s2vi.mmu_seq = kvm->mmu_invalidate_seq;
/* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */
smp_rmb(); smp_rmb();
ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL); ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, &s2vi.page, NULL);
if (ret) { if (ret) {
/* If result is non-NULL this is a synthetic fault. */ /* If result is non-NULL this is a synthetic fault. */
if (!result) if (!result)
kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, kvm_prepare_memory_fault_exit(s2fd->vcpu, gfn_to_gpa(canonical_gfn),
write_fault, exec_fault, false); s2vi.vma_pagesize, write_fault, exec_fault,
false);
kfree(mapping); kfree(mapping);
return ret;
} }
if (!(s2fd->memslot->flags & KVM_MEM_READONLY)) if (!(s2fd->memslot->flags & KVM_MEM_READONLY))
@ -1755,7 +1776,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
prot = adjust_nested_exec_perms(kvm, s2fd->nested, prot); prot = adjust_nested_exec_perms(kvm, s2fd->nested, prot);
kvm_fault_lock(kvm); kvm_fault_lock(kvm);
if (mmu_invalidate_retry(kvm, mmu_seq)) { if (mmu_invalidate_retry(kvm, s2vi.mmu_seq)) {
ret = -EAGAIN; ret = -EAGAIN;
goto out_unlock; goto out_unlock;
} }
@ -1766,11 +1787,12 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
* PTE, which will be preserved. * PTE, which will be preserved.
*/ */
prot &= ~KVM_NV_GUEST_MAP_SZ; prot &= ~KVM_NV_GUEST_MAP_SZ;
ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, s2fd->fault_ipa, ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, gfn_to_gpa(s2vi.gfn),
prot, flags); prot, flags);
} else { } else {
ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE, ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(s2vi.gfn),
__pfn_to_phys(pfn), prot, s2vi.vma_pagesize,
__pfn_to_phys(s2vi.pfn), prot,
memcache, flags); memcache, flags);
/* /*
* -EAGAIN from kvm_pgtable_stage2_map() can install mappings. * -EAGAIN from kvm_pgtable_stage2_map() can install mappings.
@ -1778,53 +1800,38 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
* thing. * thing.
*/ */
if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) {
kvm_record_guest_s2_mapping(pgt->mmu, gfn << PAGE_SHIFT, kvm_record_guest_s2_mapping(pgt->mmu, canonical_gfn << PAGE_SHIFT,
s2fd->fault_ipa, PAGE_SIZE, mapping); s2fd->fault_ipa, s2vi.vma_pagesize, mapping);
mapping = NULL; mapping = NULL;
} }
} }
out_unlock: out_unlock:
kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W); kvm_release_faultin_page(kvm, s2vi.page, !!ret, prot & KVM_PGTABLE_PROT_W);
kvm_fault_unlock(kvm); kvm_fault_unlock(kvm);
kfree(mapping); kfree(mapping);
if ((prot & KVM_PGTABLE_PROT_W) && !ret) if ((prot & KVM_PGTABLE_PROT_W) && !ret)
mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn); mark_page_dirty_in_slot(kvm, s2fd->memslot, canonical_gfn);
if (ret == -EAGAIN) if (ret == -EAGAIN)
return result ? ret : 0; return result ? ret : 0;
if (result && !ret) if (result && !ret)
result->mapping_size = PAGE_SIZE; result->mapping_size = s2vi.vma_pagesize;
return ret; return ret;
} }
struct kvm_s2_fault_vma_info {
unsigned long mmu_seq;
long vma_pagesize;
vm_flags_t vm_flags;
unsigned long max_map_size;
struct page *page;
kvm_pfn_t pfn;
gfn_t gfn;
bool device;
bool mte_allowed;
bool is_vma_cacheable;
bool map_writable;
bool map_non_cacheable;
};
static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
{ {
unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE;
struct kvm_s2_fault_vma_info s2vi = {};
struct kvm_vcpu *vcpu = s2fd->vcpu; struct kvm_vcpu *vcpu = s2fd->vcpu;
struct kvm_pgtable *pgt = s2fd->mmu->pgt; struct kvm_pgtable *pgt = s2fd->mmu->pgt;
struct mm_struct *mm = current->mm; struct mm_struct *mm = current->mm;
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
void *hyp_memcache; void *hyp_memcache;
struct page *page;
int ret; int ret;
hyp_memcache = get_mmu_memcache(vcpu); hyp_memcache = get_mmu_memcache(vcpu);
@ -1832,12 +1839,16 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
if (ret) if (ret)
return -ENOMEM; return -ENOMEM;
ret = kvm_s2_fault_get_vma_info(s2fd, &s2vi);
if (ret)
return ret;
ret = account_locked_vm(mm, 1, true); ret = account_locked_vm(mm, 1, true);
if (ret) if (ret)
return ret; return ret;
mmap_read_lock(mm); mmap_read_lock(mm);
ret = pin_user_pages(s2fd->hva, 1, flags, &page); ret = pin_user_pages(s2fd->hva, 1, flags, &s2vi.page);
mmap_read_unlock(mm); mmap_read_unlock(mm);
if (ret == -EHWPOISON) { if (ret == -EHWPOISON) {
@ -1847,7 +1858,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
} else if (ret != 1) { } else if (ret != 1) {
ret = -EFAULT; ret = -EFAULT;
goto dec_account; goto dec_account;
} else if (!folio_test_swapbacked(page_folio(page))) { } else if (!folio_test_swapbacked(page_folio(s2vi.page))) {
/* /*
* We really can't deal with page-cache pages returned by GUP * We really can't deal with page-cache pages returned by GUP
* because (a) we may trigger writeback of a page for which we * because (a) we may trigger writeback of a page for which we
@ -1867,8 +1878,8 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
} }
write_lock(&kvm->mmu_lock); write_lock(&kvm->mmu_lock);
ret = pkvm_pgtable_stage2_map(pgt, s2fd->fault_ipa, PAGE_SIZE, ret = pkvm_pgtable_stage2_map(pgt, gfn_to_gpa(s2vi.gfn), PAGE_SIZE,
page_to_phys(page), KVM_PGTABLE_PROT_RWX, page_to_phys(s2vi.page), KVM_PGTABLE_PROT_RWX,
hyp_memcache, 0); hyp_memcache, 0);
write_unlock(&kvm->mmu_lock); write_unlock(&kvm->mmu_lock);
if (ret) { if (ret) {
@ -1879,7 +1890,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
return 0; return 0;
unpin: unpin:
unpin_user_pages(&page, 1); unpin_user_page(s2vi.page);
dec_account: dec_account:
account_locked_vm(mm, 1, false); account_locked_vm(mm, 1, false);
return ret; return ret;
@ -1902,7 +1913,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd,
switch (vma_shift) { switch (vma_shift) {
#ifndef __PAGETABLE_PMD_FOLDED #ifndef __PAGETABLE_PMD_FOLDED
case PUD_SHIFT: case PUD_SHIFT:
if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PUD_SIZE)) if (fault_supports_stage2_huge_mapping(s2fd, PUD_SIZE))
break; break;
fallthrough; fallthrough;
#endif #endif
@ -1910,7 +1921,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd,
vma_shift = PMD_SHIFT; vma_shift = PMD_SHIFT;
fallthrough; fallthrough;
case PMD_SHIFT: case PMD_SHIFT:
if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PMD_SIZE)) if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE))
break; break;
fallthrough; fallthrough;
case CONT_PTE_SHIFT: case CONT_PTE_SHIFT:
@ -2165,9 +2176,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
if (perm_fault_granule > PAGE_SIZE) { if (perm_fault_granule > PAGE_SIZE) {
mapping_size = perm_fault_granule; mapping_size = perm_fault_granule;
} else { } else {
mapping_size = transparent_hugepage_adjust(kvm, s2fd->memslot, mapping_size = transparent_hugepage_adjust(s2fd, &pfn, &gfn);
s2fd->hva, &pfn,
&gfn);
if (mapping_size < 0) { if (mapping_size < 0) {
ret = mapping_size; ret = mapping_size;
goto out_unlock; goto out_unlock;

3
arch/arm64/kvm/pkvm.c
View File

@ -414,6 +414,9 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
u64 end = addr + size; u64 end = addr + size;
int ret; int ret;
if (WARN_ON_ONCE(!PAGE_ALIGNED(addr | size)))
return -EINVAL;
lockdep_assert_held_write(&kvm->mmu_lock); lockdep_assert_held_write(&kvm->mmu_lock);
mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1); mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1);