diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 8408aee7571b..e6b6170c2f4f 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -265,28 +265,28 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) softleaf_t entry; if (!(pvmw->flags & PVMW_MIGRATION)) - return not_found(pvmw); + goto next_pmd; entry = softleaf_from_pmd(pmde); if (!check_pmd(softleaf_to_pfn(entry), pvmw)) - return not_found(pvmw); + goto next_pmd; return true; } else if (pmd_is_device_private_entry(pmde)) { softleaf_t entry; if (pvmw->flags & PVMW_MIGRATION) - return not_found(pvmw); + goto next_pmd; entry = softleaf_from_pmd(pmde); if (!check_pmd(softleaf_to_pfn(entry), pvmw)) - return not_found(pvmw); + goto next_pmd; return true; } else if (!pmd_present(pmde)) { - return not_found(pvmw); + goto next_pmd; } if (likely(pmd_trans_huge(pmde))) { if (pvmw->flags & PVMW_MIGRATION) - return not_found(pvmw); + goto next_pmd; if (!check_pmd(pmd_pfn(pmde), pvmw)) - return not_found(pvmw); + goto next_pmd; return true; } /* THP/device-private pmd was split under us: handle on pte level */ @@ -299,8 +299,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) (pvmw->nr_pages >= HPAGE_PMD_NR)) sync_with_folio_pmd_zap(mm, pvmw->pmd); - step_forward(pvmw, PMD_SIZE); - continue; + goto next_pmd; } if (!map_pte(pvmw, &pmde, &ptl)) { if (!pvmw->pte) @@ -343,6 +342,17 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) pvmw->ptl = ptl; } goto this_pte; +next_pmd: + /* + * A large folio can be PTE-mapped across PMDs, so a + * nonmatching PMD does not end the walk. + */ + if (pvmw->ptl) { + spin_unlock(pvmw->ptl); + pvmw->ptl = NULL; + } + pvmw->pmd = NULL; + step_forward(pvmw, PMD_SIZE); } while (pvmw->address < end); return false;