diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index c0c9805b6f03..4f720791a31c 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -23,6 +23,7 @@ #include #include #include +#include struct mem_cgroup; struct obj_cgroup; @@ -233,6 +234,7 @@ struct mem_cgroup { #endif /* Range enforcement for interrupt charges */ + struct irq_work high_irq_work; struct work_struct high_work; __cacheline_group_end_aligned(memcg_write_hot); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e539881f58f1..3e14e0ce9e7e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -62,6 +62,7 @@ #include #include #include +#include #include "internal.h" #include "swap.h" #include "swap_table.h" @@ -2424,6 +2425,11 @@ static void high_work_func(struct work_struct *work) reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL); } +static void high_irq_work_func(struct irq_work *work) +{ + schedule_work(&container_of(work, struct mem_cgroup, high_irq_work)->high_work); +} + /* * Clamp the maximum sleep time per allocation batch to 2 seconds. This is * enough to still cause a significant slowdown in most cases, while still @@ -2832,7 +2838,10 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, /* Don't bother a random interrupted task */ if (!in_task()) { if (mem_high) { - schedule_work(&memcg->high_work); + if (allow_spinning) + schedule_work(&memcg->high_work); + else + irq_work_queue(&memcg->high_irq_work); break; } continue; @@ -4207,6 +4216,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent) goto fail; INIT_WORK(&memcg->high_work, high_work_func); + init_irq_work(&memcg->high_irq_work, high_irq_work_func); vmpressure_init(&memcg->vmpressure); INIT_LIST_HEAD(&memcg->memory_peaks); INIT_LIST_HEAD(&memcg->swap_peaks); @@ -4415,6 +4425,7 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css) static_branch_dec(&memcg_bpf_enabled_key); vmpressure_cleanup(&memcg->vmpressure); + irq_work_sync(&memcg->high_irq_work); cancel_work_sync(&memcg->high_work); free_shrinker_info(memcg); mem_cgroup_free(memcg);