diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index ecb6e0b7819a..7cf7c18119b4 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -179,6 +179,34 @@ static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css) hugetlb_cgroup_free(hugetlb_cgroup_from_css(css)); } +static void hugetlb_cgroup_move_usage(struct hugetlb_cgroup *from, + struct hugetlb_cgroup *to, + struct folio *from_folio, + struct folio *to_folio) +{ + int idx = hstate_index(folio_hstate(from_folio)); + unsigned long nr_pages = folio_nr_pages(from_folio); + int from_nid = folio_nid(from_folio); + int to_nid = folio_nid(to_folio); + unsigned long usage; + + lockdep_assert_held(&hugetlb_lock); + + if (!from || !to) + return; + + if (from == to && from_nid == to_nid) + return; + + usage = from->nodeinfo[from_nid]->usage[idx]; + if (WARN_ON_ONCE(usage < nr_pages)) + return; + WRITE_ONCE(from->nodeinfo[from_nid]->usage[idx], usage - nr_pages); + + usage = to->nodeinfo[to_nid]->usage[idx]; + WRITE_ONCE(to->nodeinfo[to_nid]->usage[idx], usage + nr_pages); +} + /* * Should be called with hugetlb_lock held. * Since we are holding hugetlb_lock, pages cannot get moved from @@ -906,6 +934,9 @@ void hugetlb_cgroup_migrate(struct folio *old_folio, struct folio *new_folio) /* move the h_cg details to new cgroup */ set_hugetlb_cgroup(new_folio, h_cg); set_hugetlb_cgroup_rsvd(new_folio, h_cg_rsvd); + + hugetlb_cgroup_move_usage(h_cg, h_cg, old_folio, new_folio); + list_move(&new_folio->lru, &h->hugepage_activelist); spin_unlock_irq(&hugetlb_lock); }