diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8758c6801776..ec469d96516a 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -121,6 +121,9 @@ static const struct pvm_ftr_bits pvmid_aa64mmfr0[] = { MAX_FEAT(ID_AA64MMFR0_EL1, SNSMEM, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, BIGENDEL0, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, EXS, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN4, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN16, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN64, IMP), FEAT_END }; diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 2a281488792f..e10f71db62cf 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1406,21 +1406,53 @@ static void kvm_send_hwpoison_signal(unsigned long address, short lsb) send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current); } -static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, - unsigned long hva, +struct kvm_s2_fault_desc { + struct kvm_vcpu *vcpu; + phys_addr_t fault_ipa; + struct kvm_s2_trans *nested; + struct kvm_memory_slot *memslot; + unsigned long hva; + unsigned long esr; + struct kvm_s2_mmu *mmu; +}; + +struct kvm_s2_fault_vma_info { + unsigned long mmu_seq; + long vma_pagesize; + vm_flags_t vm_flags; + unsigned long max_map_size; + struct page *page; + kvm_pfn_t pfn; + gfn_t gfn; + bool device; + bool mte_allowed; + bool is_vma_cacheable; + bool map_writable; + bool map_non_cacheable; +}; + +struct kvm_s2_fault_result { + unsigned long mapping_size; +}; + +static bool fault_supports_stage2_huge_mapping(const struct kvm_s2_fault_desc *s2fd, unsigned long map_size) { - gpa_t gpa_start; + struct kvm_memory_slot *memslot = s2fd->memslot; + unsigned long hva = s2fd->hva; hva_t uaddr_start, uaddr_end; + gpa_t gpa_start; size_t size; /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ if (map_size == PAGE_SIZE) return true; - /* pKVM only supports PMD_SIZE huge-mappings */ - if (is_protected_kvm_enabled() && map_size != PMD_SIZE) - return false; + /* pKVM only supports PMD_SIZE huge-mappings for non-protected VMs */ + if (is_protected_kvm_enabled()) { + if (vcpu_is_protected(s2fd->vcpu) || map_size != PMD_SIZE) + return false; + } size = memslot->npages * PAGE_SIZE; @@ -1480,9 +1512,9 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, * Returns the size of the mapping. */ static long -transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, - unsigned long hva, kvm_pfn_t *pfnp, gfn_t *gfnp) +transparent_hugepage_adjust(const struct kvm_s2_fault_desc *s2fd, kvm_pfn_t *pfnp, gfn_t *gfnp) { + struct kvm *kvm = s2fd->vcpu->kvm; kvm_pfn_t pfn = *pfnp; gfn_t gfn = *gfnp; @@ -1491,8 +1523,8 @@ transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, * sure that the HVA and IPA are sufficiently aligned and that the * block map is contained within the memslot. */ - if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { - int sz = get_user_mapping_size(kvm, hva); + if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) { + int sz = get_user_mapping_size(kvm, s2fd->hva); if (sz < 0) return sz; @@ -1650,20 +1682,6 @@ static enum kvm_pgtable_prot adjust_nested_exec_perms(struct kvm *kvm, return prot; } -struct kvm_s2_fault_desc { - struct kvm_vcpu *vcpu; - phys_addr_t fault_ipa; - struct kvm_s2_trans *nested; - struct kvm_memory_slot *memslot; - unsigned long hva; - unsigned long esr; - struct kvm_s2_mmu *mmu; -}; - -struct kvm_s2_fault_result { - unsigned long mapping_size; -}; - static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) { return esr_fsc_is_permission_fault(s2fd->esr); @@ -1689,6 +1707,12 @@ static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd) return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); } +static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_vma_info *s2vi); + +static gfn_t get_canonical_gfn(const struct kvm_s2_fault_desc *s2fd, + const struct kvm_s2_fault_vma_info *s2vi); + static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, struct kvm_s2_fault_result *result) { @@ -1698,12 +1722,10 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; struct kvm_pgtable *pgt = s2fd->mmu->pgt; struct kvm_guest_s2_mapping *mapping = NULL; - unsigned long mmu_seq; - struct page *page; + struct kvm_s2_fault_vma_info s2vi = {}; struct kvm *kvm = s2fd->vcpu->kvm; void *memcache = NULL; - kvm_pfn_t pfn; - gfn_t gfn; + gfn_t canonical_gfn; int ret; if (!perm_fault) { @@ -1718,28 +1740,27 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, } } - if (s2fd->nested) - gfn = kvm_s2_trans_output(s2fd->nested) >> PAGE_SHIFT; - else - gfn = s2fd->fault_ipa >> PAGE_SHIFT; + s2vi.vma_pagesize = PAGE_SIZE; + s2vi.gfn = ALIGN_DOWN(s2fd->fault_ipa, s2vi.vma_pagesize) >> PAGE_SHIFT; + canonical_gfn = get_canonical_gfn(s2fd, &s2vi); write_fault = kvm_s2_fault_is_write(s2fd); exec_fault = kvm_s2_fault_is_exec(s2fd); VM_WARN_ON_ONCE(write_fault && exec_fault); - mmu_seq = kvm->mmu_invalidate_seq; + s2vi.mmu_seq = kvm->mmu_invalidate_seq; /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ smp_rmb(); - ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, &s2vi.page, NULL); if (ret) { /* If result is non-NULL this is a synthetic fault. */ if (!result) - kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, - write_fault, exec_fault, false); + kvm_prepare_memory_fault_exit(s2fd->vcpu, gfn_to_gpa(canonical_gfn), + s2vi.vma_pagesize, write_fault, exec_fault, + false); kfree(mapping); - return ret; } if (!(s2fd->memslot->flags & KVM_MEM_READONLY)) @@ -1755,7 +1776,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, prot = adjust_nested_exec_perms(kvm, s2fd->nested, prot); kvm_fault_lock(kvm); - if (mmu_invalidate_retry(kvm, mmu_seq)) { + if (mmu_invalidate_retry(kvm, s2vi.mmu_seq)) { ret = -EAGAIN; goto out_unlock; } @@ -1766,11 +1787,12 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, * PTE, which will be preserved. */ prot &= ~KVM_NV_GUEST_MAP_SZ; - ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, s2fd->fault_ipa, + ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, gfn_to_gpa(s2vi.gfn), prot, flags); } else { - ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE, - __pfn_to_phys(pfn), prot, + ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(s2vi.gfn), + s2vi.vma_pagesize, + __pfn_to_phys(s2vi.pfn), prot, memcache, flags); /* * -EAGAIN from kvm_pgtable_stage2_map() can install mappings. @@ -1778,53 +1800,38 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, * thing. */ if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { - kvm_record_guest_s2_mapping(pgt->mmu, gfn << PAGE_SHIFT, - s2fd->fault_ipa, PAGE_SIZE, mapping); + kvm_record_guest_s2_mapping(pgt->mmu, canonical_gfn << PAGE_SHIFT, + s2fd->fault_ipa, s2vi.vma_pagesize, mapping); mapping = NULL; } } out_unlock: - kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W); + kvm_release_faultin_page(kvm, s2vi.page, !!ret, prot & KVM_PGTABLE_PROT_W); kvm_fault_unlock(kvm); kfree(mapping); if ((prot & KVM_PGTABLE_PROT_W) && !ret) - mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn); + mark_page_dirty_in_slot(kvm, s2fd->memslot, canonical_gfn); if (ret == -EAGAIN) return result ? ret : 0; if (result && !ret) - result->mapping_size = PAGE_SIZE; + result->mapping_size = s2vi.vma_pagesize; return ret; } -struct kvm_s2_fault_vma_info { - unsigned long mmu_seq; - long vma_pagesize; - vm_flags_t vm_flags; - unsigned long max_map_size; - struct page *page; - kvm_pfn_t pfn; - gfn_t gfn; - bool device; - bool mte_allowed; - bool is_vma_cacheable; - bool map_writable; - bool map_non_cacheable; -}; - static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) { unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; + struct kvm_s2_fault_vma_info s2vi = {}; struct kvm_vcpu *vcpu = s2fd->vcpu; struct kvm_pgtable *pgt = s2fd->mmu->pgt; struct mm_struct *mm = current->mm; struct kvm *kvm = vcpu->kvm; void *hyp_memcache; - struct page *page; int ret; hyp_memcache = get_mmu_memcache(vcpu); @@ -1832,12 +1839,16 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) if (ret) return -ENOMEM; + ret = kvm_s2_fault_get_vma_info(s2fd, &s2vi); + if (ret) + return ret; + ret = account_locked_vm(mm, 1, true); if (ret) return ret; mmap_read_lock(mm); - ret = pin_user_pages(s2fd->hva, 1, flags, &page); + ret = pin_user_pages(s2fd->hva, 1, flags, &s2vi.page); mmap_read_unlock(mm); if (ret == -EHWPOISON) { @@ -1847,7 +1858,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) } else if (ret != 1) { ret = -EFAULT; goto dec_account; - } else if (!folio_test_swapbacked(page_folio(page))) { + } else if (!folio_test_swapbacked(page_folio(s2vi.page))) { /* * We really can't deal with page-cache pages returned by GUP * because (a) we may trigger writeback of a page for which we @@ -1867,8 +1878,8 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) } write_lock(&kvm->mmu_lock); - ret = pkvm_pgtable_stage2_map(pgt, s2fd->fault_ipa, PAGE_SIZE, - page_to_phys(page), KVM_PGTABLE_PROT_RWX, + ret = pkvm_pgtable_stage2_map(pgt, gfn_to_gpa(s2vi.gfn), PAGE_SIZE, + page_to_phys(s2vi.page), KVM_PGTABLE_PROT_RWX, hyp_memcache, 0); write_unlock(&kvm->mmu_lock); if (ret) { @@ -1879,7 +1890,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) return 0; unpin: - unpin_user_pages(&page, 1); + unpin_user_page(s2vi.page); dec_account: account_locked_vm(mm, 1, false); return ret; @@ -1902,7 +1913,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, switch (vma_shift) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SHIFT: - if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PUD_SIZE)) + if (fault_supports_stage2_huge_mapping(s2fd, PUD_SIZE)) break; fallthrough; #endif @@ -1910,7 +1921,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, vma_shift = PMD_SHIFT; fallthrough; case PMD_SHIFT: - if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PMD_SIZE)) + if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) break; fallthrough; case CONT_PTE_SHIFT: @@ -2165,9 +2176,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, if (perm_fault_granule > PAGE_SIZE) { mapping_size = perm_fault_granule; } else { - mapping_size = transparent_hugepage_adjust(kvm, s2fd->memslot, - s2fd->hva, &pfn, - &gfn); + mapping_size = transparent_hugepage_adjust(s2fd, &pfn, &gfn); if (mapping_size < 0) { ret = mapping_size; goto out_unlock; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4c6e4bec12..c298a5b9b12a 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -414,6 +414,9 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 end = addr + size; int ret; + if (WARN_ON_ONCE(!PAGE_ALIGNED(addr | size))) + return -EINVAL; + lockdep_assert_held_write(&kvm->mmu_lock); mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1);