diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index c745f7ad2298..fac05120ab77 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -12,9 +12,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); bool huge_pmd_set_accessed(struct vm_fault *vmf); -int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, - pud_t *dst_pud, pud_t *src_pud, unsigned long addr, - struct vm_area_struct *vma); +int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + pud_t *dst_pud, pud_t *src_pud, unsigned long addr); #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 4c0194af9016..283c4bc8b927 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2072,11 +2072,13 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr, update_mmu_cache_pud(vma, addr, pud); } -int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, - pud_t *dst_pud, pud_t *src_pud, unsigned long addr, - struct vm_area_struct *vma) +int copy_huge_pud(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + pud_t *dst_pud, pud_t *src_pud, unsigned long addr) { + struct mm_struct *dst_mm = dst_vma->vm_mm; + struct mm_struct *src_mm = src_vma->vm_mm; spinlock_t *dst_ptl, *src_ptl; + struct page *page; pud_t pud; int ret; @@ -2089,11 +2091,20 @@ int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (unlikely(!pud_trans_huge(pud))) goto out_unlock; - /* - * TODO: once we support anonymous pages, use - * folio_try_dup_anon_rmap_*() and split if duplicating fails. - */ - if (vma_is_cow_mapping(vma) && pud_write(pud)) { + page = vm_normal_page_pud(src_vma, addr, pud); + if (page) { + struct folio *folio = page_folio(page); + + folio_get(folio); + /* + * TODO: once we support anonymous pages, use + * folio_try_dup_anon_rmap_*() and split if duplicating fails. + */ + folio_add_file_rmap_pud(folio, page, dst_vma); + add_mm_counter(dst_mm, mm_counter_file(folio), HPAGE_PUD_NR); + } + + if (vma_is_cow_mapping(src_vma) && pud_write(pud)) { pudp_set_wrprotect(src_mm, addr, src_pud); pud = pud_wrprotect(pud); } diff --git a/mm/memory.c b/mm/memory.c index 0fb676dd2bd6..5392bcb1d24d 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1463,7 +1463,6 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, unsigned long end) { struct mm_struct *dst_mm = dst_vma->vm_mm; - struct mm_struct *src_mm = src_vma->vm_mm; pud_t *src_pud, *dst_pud; unsigned long next; @@ -1477,8 +1476,7 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, int err; VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma); - err = copy_huge_pud(dst_mm, src_mm, - dst_pud, src_pud, addr, src_vma); + err = copy_huge_pud(dst_vma, src_vma, dst_pud, src_pud, addr); if (err == -ENOMEM) return -ENOMEM; if (!err)