diff --git a/include/linux/swap.h b/include/linux/swap.h index 330a420fd6de..b4b1c0a84c8b 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -298,6 +298,14 @@ void folio_add_lru(struct folio *folio); void folio_mark_accessed(struct folio *folio); void lru_add_drain_all(void); +enum lru_cache_drained { + LRU_CACHE_NOT_DRAINED, + LRU_CACHE_DRAINED, + LRU_CACHE_DRAINED_ALL, +}; +void lru_cache_drain_for_folio(const struct folio *folio, + unsigned int extra_refs, enum lru_cache_drained *drained); + /* linux/mm/folio-compat.c */ void mark_page_accessed(struct page *page); diff --git a/mm/folio.c b/mm/folio.c index a9e328c3f21b..59c477120b9a 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -881,6 +881,52 @@ void lru_add_drain_all(void) } #endif /* CONFIG_SMP */ +/** + * lru_cache_drain_for_folio() - drain LRU caches if the caches might hold + * folio references + * @folio: The folio. + * @extra_refs: Extra folio references held by the caller. + * @drained: Drain status for batch folio processing. + * + * Drain LRU caches if the caches might hold folio references. Start + * with a local LRU cache drain, to then drain LRU caches on all CPUs if + * local draining was insufficient. + * + * This function detects LRU cache references by comparing the folio refcount + * with the sum of the expected folio refcount + extra references held by the + * caller. Note that we cannot rely on PG_lru to reliably detect all LRU + * cache references, and there are rare scenarios (concurrent folio (un)mapping) + * where this function might miss detecting LRU cache references. + * + * If @drained is not NULL, the function will avoid re-draining LRU caches + * when processing multiple folios in a row. In that case, the variable + * @drained points at must be initialized to LRU_CACHE_NOT_DRAINED before + * the first invocation by the caller. + */ +void lru_cache_drain_for_folio(const struct folio *folio, + unsigned int extra_refs, enum lru_cache_drained *drained) +{ + if (!folio_may_be_lru_cached(folio)) + return; + + if (!drained || *drained == LRU_CACHE_NOT_DRAINED) { + if (folio_ref_count(folio) == + folio_expected_ref_count(folio) + extra_refs) + return; + lru_add_drain(); + if (drained) + *drained = LRU_CACHE_DRAINED; + } + if (!drained || *drained == LRU_CACHE_DRAINED) { + if (folio_ref_count(folio) == + folio_expected_ref_count(folio) + extra_refs) + return; + lru_add_drain_all(); + if (drained) + *drained = LRU_CACHE_DRAINED_ALL; + } +} + atomic_t lru_disable_count = ATOMIC_INIT(0); /* diff --git a/mm/gup.c b/mm/gup.c index bce275c7dbb6..eb898ea1ee22 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2266,9 +2266,9 @@ static unsigned long collect_longterm_unpinnable_folios( struct list_head *movable_folio_list, struct pages_or_folios *pofs) { + enum lru_cache_drained drained = LRU_CACHE_NOT_DRAINED; unsigned long collected = 0; struct folio *folio; - int drained = 0; long i = 0; for (folio = pofs_get_folio(pofs, i); folio; @@ -2293,18 +2293,7 @@ static unsigned long collect_longterm_unpinnable_folios( * but also to remove any other folio references from LRU * caches. */ - if (drained == 0 && folio_may_be_lru_cached(folio) && - folio_ref_count(folio) != - folio_expected_ref_count(folio) + pin_refs) { - lru_add_drain(); - drained = 1; - } - if (drained == 1 && folio_may_be_lru_cached(folio) && - folio_ref_count(folio) != - folio_expected_ref_count(folio) + pin_refs) { - lru_add_drain_all(); - drained = 2; - } + lru_cache_drain_for_folio(folio, pin_refs, &drained); if (!folio_isolate_lru(folio)) continue; diff --git a/mm/internal.h b/mm/internal.h index 07f60ca0b201..38b1165212c9 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -43,7 +43,7 @@ void workingset_activation(struct folio *folio); /* mm/folio.c */ void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); -static inline bool folio_may_be_lru_cached(struct folio *folio) +static inline bool folio_may_be_lru_cached(const struct folio *folio) { /* * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting.