diff --git a/mm/swapfile.c b/mm/swapfile.c index 4875b3d3e658..60d8f0df3f32 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1899,21 +1899,46 @@ void __swap_cluster_free_entries(struct swap_info_struct *si, unsigned int ci_start, unsigned int nr_pages) { unsigned long old_tb; + unsigned int type = si->type; + unsigned short batch_id = 0, id_cur; unsigned int ci_off = ci_start, ci_end = ci_start + nr_pages; - unsigned long offset = cluster_offset(si, ci) + ci_start; + unsigned long ci_head = cluster_offset(si, ci); + unsigned int batch_off = ci_off; + swp_entry_t entry; VM_WARN_ON(ci->count < nr_pages); ci->count -= nr_pages; do { old_tb = __swap_table_get(ci, ci_off); - /* Release the last ref, or after swap cache is dropped */ + /* + * Freeing is done after release of the last swap count + * ref, or after swap cache is dropped + */ VM_WARN_ON(!swp_tb_is_shadow(old_tb) || __swp_tb_get_count(old_tb) > 1); __swap_table_set(ci, ci_off, null_to_swp_tb()); + + /* + * Uncharge swap slots by memcg in batches. Consecutive + * slots with the same cgroup id are uncharged together. + */ + entry = swp_entry(type, ci_head + ci_off); + id_cur = lookup_swap_cgroup_id(entry); + if (batch_id != id_cur) { + if (batch_id) + mem_cgroup_uncharge_swap(swp_entry(type, ci_head + batch_off), + ci_off - batch_off); + batch_id = id_cur; + batch_off = ci_off; + } } while (++ci_off < ci_end); - mem_cgroup_uncharge_swap(swp_entry(si->type, offset), nr_pages); - swap_range_free(si, offset, nr_pages); + if (batch_id) { + mem_cgroup_uncharge_swap(swp_entry(type, ci_head + batch_off), + ci_off - batch_off); + } + + swap_range_free(si, ci_head + ci_start, nr_pages); swap_cluster_assert_empty(ci, ci_start, nr_pages, false); if (!ci->count)