diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 2b01bfbce3ea..9bb700d25ccb 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -1652,6 +1652,34 @@ static unsigned long vma_to_pfn(struct vm_area_struct *vma) return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff; } +vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, + struct vm_fault *vmf, + unsigned long pfn, + unsigned int order) +{ + lockdep_assert_held_read(&vdev->memory_lock); + + if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) + return VM_FAULT_SIGBUS; + + switch (order) { + case 0: + return vmf_insert_pfn(vmf->vma, vmf->address, pfn); +#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP + case PMD_ORDER: + return vmf_insert_pfn_pmd(vmf, pfn, false); +#endif +#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP + case PUD_ORDER: + return vmf_insert_pfn_pud(vmf, pfn, false); + break; +#endif + default: + return VM_FAULT_FALLBACK; + } +} +EXPORT_SYMBOL_GPL(vfio_pci_vmf_insert_pfn); + static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf, unsigned int order) { @@ -1660,41 +1688,13 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf, unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1); unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; unsigned long pfn = vma_to_pfn(vma) + pgoff; - vm_fault_t ret = VM_FAULT_SIGBUS; + vm_fault_t ret = VM_FAULT_FALLBACK; - if (order && (addr < vma->vm_start || - addr + (PAGE_SIZE << order) > vma->vm_end || - pfn & ((1 << order) - 1))) { - ret = VM_FAULT_FALLBACK; - goto out; + if (is_aligned_for_order(vma, addr, pfn, order)) { + scoped_guard(rwsem_read, &vdev->memory_lock) + ret = vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order); } - down_read(&vdev->memory_lock); - - if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) - goto out_unlock; - - switch (order) { - case 0: - ret = vmf_insert_pfn(vma, vmf->address, pfn); - break; -#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP - case PMD_ORDER: - ret = vmf_insert_pfn_pmd(vmf, pfn, false); - break; -#endif -#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP - case PUD_ORDER: - ret = vmf_insert_pfn_pud(vmf, pfn, false); - break; -#endif - default: - ret = VM_FAULT_FALLBACK; - } - -out_unlock: - up_read(&vdev->memory_lock); -out: dev_dbg_ratelimited(&vdev->pdev->dev, "%s(,order = %d) BAR %ld page offset 0x%lx: 0x%x\n", __func__, order, diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index a1eddd55dab8..5569488ec4dc 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -170,6 +170,9 @@ ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, size_t count, loff_t *ppos); ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, size_t count, loff_t *ppos); +vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, + struct vm_fault *vmf, unsigned long pfn, + unsigned int order); int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma); void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count); int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf); @@ -212,4 +215,14 @@ VFIO_IOREAD_DECLARATION(32) VFIO_IOREAD_DECLARATION(64) #endif +static inline bool is_aligned_for_order(struct vm_area_struct *vma, + unsigned long addr, + unsigned long pfn, + unsigned int order) +{ + return !(order && (addr < vma->vm_start || + addr + (PAGE_SIZE << order) > vma->vm_end || + !IS_ALIGNED(pfn, 1 << order))); +} + #endif /* VFIO_PCI_CORE_H */