diff --git a/mm/collapse.h b/mm/collapse.h index dcd117071955..759ec9cd4541 100644 --- a/mm/collapse.h +++ b/mm/collapse.h @@ -98,6 +98,24 @@ struct collapse_control { /* Each bit marks a PTE the scan accepted as a collapse source */ DECLARE_BITMAP(eligible_ptes, MAX_PTRS_PER_PTE); + + /* + * What a scan found and the collapse after it needs. Live only between + * the two, and read by nobody else. + */ + + /* Anonymous only: orders to try, referenced and swap PTE counts */ + unsigned long scan_orders; + int scan_referenced; + int scan_unmapped; + + /* + * File only. A file collapse works on the page cache and never sees a + * VMA, so the scan takes a reference to the file while it still has + * one, and the collapse gives it back. + */ + struct file *scan_file; + pgoff_t scan_pgoff; }; #endif /* __MM_COLLAPSE_H */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 7a6acb5ba863..17e0e0162e90 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -646,7 +646,7 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, goto out; } - /* See collapse_scan_pmd(). */ + /* See collapse_scan_anon(). */ if (folio_maybe_mapped_shared(folio)) { /* * TODO: Support shared pages without leading to further @@ -1099,7 +1099,7 @@ static enum scan_result check_pmd_still_valid(struct mm_struct *mm, /* * Bring missing pages in from swap, to complete THP collapse. - * Only done if collapse_scan_pmd() believes it is worthwhile. + * Only done if collapse_scan_anon() believes it is worthwhile. * * For mTHP orders the function bails on the first swap entry, because * faulting pages back in during collapse could re-populate PTEs that @@ -1252,8 +1252,8 @@ static pgtable_t alloc_deposit_pte_table(struct mm_struct *mm) * while allocating a THP, as that could trigger direct reclaim/compaction. * Note that the VMA must be rechecked after grabbing the mmap_lock again. */ -static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long start_addr, - int referenced, int unmapped, struct collapse_control *cc, +static enum scan_result collapse_huge_page(struct mm_struct *mm, + unsigned long start_addr, struct collapse_control *cc, unsigned int order) { const unsigned long pmd_addr = start_addr & HPAGE_PMD_MASK; @@ -1300,14 +1300,14 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s goto out_nolock; } - if (unmapped) { + if (cc->scan_unmapped) { /* * __collapse_huge_page_swapin() will return with mmap_lock * released when it fails. So we jump out_nolock directly in * that case. Continuing to collapse causes inconsistency. */ result = __collapse_huge_page_swapin(mm, vma, start_addr, pmd, - referenced, order); + cc->scan_referenced, order); if (result != SCAN_SUCCEED) goto out_nolock; } @@ -1460,8 +1460,8 @@ static unsigned int max_order_from_offset(unsigned int offset) } /* - * mthp_collapse() consumes the bitmap that is generated during - * collapse_scan_pmd() to determine what regions and mTHP orders fit best. + * collapse_anon() consumes the bitmap that is generated during + * collapse_scan_anon() to determine what regions and mTHP orders fit best. * * Each bit in cc->eligible_ptes marks a PTE the scan accepted as a collapse * source. We start at the PMD order and check if it is eligible for collapse; @@ -1476,9 +1476,8 @@ static unsigned int max_order_from_offset(unsigned int offset) * If a collapse is permitted, we attempt to collapse the PTE range into a * mTHP. */ -static enum scan_result mthp_collapse(struct mm_struct *mm, - unsigned long address, int referenced, int unmapped, - struct collapse_control *cc, unsigned long enabled_orders) +static enum scan_result collapse_anon(struct mm_struct *mm, + unsigned long address, struct collapse_control *cc) { unsigned int nr_eligible_ptes, nr_ptes, max_ptes_none; enum scan_result last_result = SCAN_FAIL; @@ -1491,7 +1490,7 @@ static enum scan_result mthp_collapse(struct mm_struct *mm, while (offset < HPAGE_PMD_NR) { nr_ptes = 1UL << order; - if (!test_bit(order, &enabled_orders)) + if (!test_bit(order, &cc->scan_orders)) goto next_order; max_ptes_none = collapse_max_ptes_none(cc, NULL, order); @@ -1499,19 +1498,18 @@ static enum scan_result mthp_collapse(struct mm_struct *mm, offset + nr_ptes); /* - * Swap PTEs accepted during the scan are counted in @unmapped, - * not in cc->eligible_ptes. Account them for the PMD-order - * candidate. + * Swap PTEs accepted during the scan are counted in + * cc->scan_unmapped, not in cc->eligible_ptes. Account them for + * the PMD-order candidate. */ if (is_pmd_order(order)) - nr_eligible_ptes += unmapped; + nr_eligible_ptes += cc->scan_unmapped; if (nr_eligible_ptes >= nr_ptes - max_ptes_none) { enum scan_result ret; collapse_address = address + offset * PAGE_SIZE; - ret = collapse_huge_page(mm, collapse_address, referenced, - unmapped, cc, order); + ret = collapse_huge_page(mm, collapse_address, cc, order); switch (ret) { /* Cases where we continue to next collapse candidate */ @@ -1553,7 +1551,7 @@ static enum scan_result mthp_collapse(struct mm_struct *mm, * we must always move to the next offset. */ if (order > COLLAPSE_MIN_MTHP_ORDER && - (enabled_orders & GENMASK(order - 1, 0))) { + (cc->scan_orders & GENMASK(order - 1, 0))) { order--; continue; } @@ -1578,14 +1576,14 @@ static enum scan_result mthp_collapse(struct mm_struct *mm, return last_result; } -static enum scan_result collapse_scan_pmd(struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long start_addr, - bool *lock_dropped, struct collapse_control *cc) +static enum scan_result collapse_scan_anon(struct vm_area_struct *vma, + unsigned long start_addr, struct collapse_control *cc) { const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, HPAGE_PMD_ORDER); const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER); unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER); enum tva_type tva_flags = cc->policy.tva_type; + struct mm_struct *mm = vma->vm_mm; pmd_t *pmd; pte_t *pte, *_pte, pteval; int i; @@ -1614,7 +1612,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, /* * If PMD is the only enabled order, enforce max_ptes_none, otherwise * scan all pages to populate the bitmap for mTHP collapse. The bitmap - * is then checked again in mthp_collapse() for each attempted order. + * is then checked again in collapse_anon() for each attempted order. */ if (enabled_orders != BIT(HPAGE_PMD_ORDER)) max_ptes_none = COLLAPSE_MAX_PTES_LIMIT; @@ -1765,12 +1763,9 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, out_unmap: pte_unmap_unlock(pte, ptl); if (result == SCAN_SUCCEED) { - /* collapse_huge_page() expects the lock to be dropped before calling */ - mmap_read_unlock(mm); - result = mthp_collapse(mm, start_addr, referenced, - unmapped, cc, enabled_orders); - /* mmap_lock was released above, set lock_dropped */ - *lock_dropped = true; + cc->scan_orders = enabled_orders; + cc->scan_referenced = referenced; + cc->scan_unmapped = unmapped; } out: trace_mm_khugepaged_scan_pmd(mm, failed_pfn, referenced, @@ -2768,41 +2763,67 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } - trace_mm_khugepaged_scan_file(mm, failed_pfn, file, present, swap, result); + trace_mm_khugepaged_scan_file(mm, failed_pfn, file, present, swap, + result); return result; } -/* - * Try to collapse a single PMD starting at a PMD aligned addr, and return - * the results. - */ -static enum scan_result collapse_single_pmd(unsigned long addr, - struct vm_area_struct *vma, bool *lock_dropped, - struct collapse_control *cc) +static void collapse_control_init(struct collapse_control *cc) +{ + cc->progress = 0; + cc->scan_file = NULL; +} + +static enum scan_result collapse_scan_pmd(struct vm_area_struct *vma, + unsigned long addr, struct collapse_control *cc) { - struct mm_struct *mm = vma->vm_mm; - bool triggered_wb = false; enum scan_result result; - struct file *file; pgoff_t pgoff; - mmap_assert_locked(mm); + mmap_assert_locked(vma->vm_mm); + /* A scan that found work has to have been followed by collapse_pmd() */ + if (WARN_ON_ONCE(cc->scan_file)) { + fput(cc->scan_file); + cc->scan_file = NULL; + } if (vma_is_anonymous(vma)) - return collapse_scan_pmd(mm, vma, addr, lock_dropped, cc); + return collapse_scan_anon(vma, addr, cc); - file = get_file(vma->vm_file); pgoff = linear_page_index(vma, addr); - - mmap_read_unlock(mm); - *lock_dropped = true; - + result = collapse_scan_file(vma->vm_mm, addr, vma->vm_file, pgoff, cc); /* * SCAN_PTE_MAPPED_HUGEPAGE is work too: the page cache already holds - * the PMD folio, and only the PTE table is left to retract. + * the PMD folio, and collapse_pmd() still has to retract the PTE table. */ - result = collapse_scan_file(mm, addr, file, pgoff, cc); - if (result != SCAN_SUCCEED) + if (result != SCAN_SUCCEED && result != SCAN_PTE_MAPPED_HUGEPAGE) + return result; + + /* + * A file collapse works on the page cache and never sees a VMA, so take + * what it needs from this one while it is still here. + */ + cc->scan_file = get_file(vma->vm_file); + cc->scan_pgoff = pgoff; + return result; +} + +static enum scan_result collapse_pmd(struct mm_struct *mm, + unsigned long addr, enum scan_result result, + struct collapse_control *cc) +{ + struct file *file = cc->scan_file; + bool triggered_wb = false; + pgoff_t pgoff; + + if (!file) + return collapse_anon(mm, addr, cc); + + cc->scan_file = NULL; + pgoff = cc->scan_pgoff; + + /* The scan found the PMD folio in place: nothing to collapse */ + if (result == SCAN_PTE_MAPPED_HUGEPAGE) goto put; retry: result = collapse_file(mm, addr, file, pgoff, cc); @@ -2820,6 +2841,10 @@ static enum scan_result collapse_single_pmd(unsigned long addr, put: fput(file); + /* + * A PMD folio is in the page cache, whether the collapse just put it + * there or found it: retract the PTE table, and map the PMD if asked. + */ if (result == SCAN_PTE_MAPPED_HUGEPAGE) { mmap_read_lock(mm); if (collapse_test_exit_or_disable(mm)) @@ -2834,6 +2859,28 @@ static enum scan_result collapse_single_pmd(unsigned long addr, return result; } +/* + * Try to collapse a single PMD starting at a PMD aligned addr, and return + * the results. + */ +static enum scan_result collapse_single_pmd(unsigned long addr, + struct vm_area_struct *vma, bool *lock_dropped, + struct collapse_control *cc) +{ + struct mm_struct *mm = vma->vm_mm; + enum scan_result result; + + result = collapse_scan_pmd(vma, addr, cc); + if (result != SCAN_SUCCEED && result != SCAN_PTE_MAPPED_HUGEPAGE) + return result; + + /* The collapse takes its own locks, so give this up */ + mmap_read_unlock(mm); + *lock_dropped = true; + + return collapse_pmd(mm, addr, result, cc); +} + static void collapse_scan_mm_slot(unsigned int progress_max, enum scan_result *result, struct collapse_control *cc) __releases(&khugepaged_mm_lock) @@ -2976,9 +3023,9 @@ static void khugepaged_do_scan(struct collapse_control *cc) lru_add_drain_all(); + collapse_control_init(cc); collapse_policy_khugepaged(&cc->policy); - cc->progress = 0; while (true) { cond_resched(); @@ -3205,8 +3252,8 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, cc = kmalloc_obj(*cc); if (!cc) return -ENOMEM; + collapse_control_init(cc); collapse_policy_madvise(&cc->policy); - cc->progress = 0; lru_add_drain_all();