github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/memcontrol: skip non-hierarchical memcg-wide stats when v1 is unavailable

Browse Source 
memcg_vmstats keeps a non-hierarchical copy of every memcg-wide stat item
and event alongside the hierarchical one.  The only readers however are
the legacy memory.stat and memory.numa_stat, and reparenting on offline. 
All of them live under CONFIG_MEMCG_V1, and their accessors
(memcg_page_state_local() and memcg_events_local()) are already compiled
out with it.  This means on a CONFIG_MEMCG_V1=n kernel, memcg_vmstats's
non-hierarchical arrays are written to on every rstat flush, despite their
values never being read / accessed.

The same holds when the kernel does support v1 but the controller has been
blocked from v1 hierarchies with the boot param cgroup_no_v1={memory,all}.
A v1 mount is refused in that case, so the legacy memory.stat can never
exist and the arrays are just as unread / unaccessed.

Compile out the non-hierarchical memcg-wide arrays if CONFIG_MEMCG_V1 is
not set.  If it is set but cgroup_no_v1= has blocked the controller, skip
updates on the arrays.

This makes flushes cheaper.  mem_cgroup_stat_aggregate() can now skip the
read-modify-write of ac->local[i].  Nothing else accesses state_local or
events_local, so those cachelines were getting pulled in solely for the
writes and they are separate from the ones the loop is already walking.

On an 80-cpu x86_64 machine with 500 cgroups each running a workload that
dirties anon, file, dirty/writeback, slab, kmem, mlock, and reclaim
counters, timing mem_cgroup_css_rstat_flush() in-kernel in TSC ticks per
flush showed roughly

                         before   after    delta
memcg-wide aggregation   1231     1180     -4.1%
overall flush function   2452     2397     -2.2%

These numbers are from taking the median of 70 samples, one per 20s window
on each kernel.  The 95% intervals observed on the two deltas are [-5.41%,
-1.76%] and [-4.53%, -0.14%].  The values above include the timing
overhead itself, so only the delta is meaningful here.

Counting the items that actually changed, a median of 1.5 of the 77
memcg-wide items (57 state + 20 events) had a non-zero per-cpu delta at
each flush, which means the benchmarks above are with one or two fewer
cachelines pulled in per flush.  The count is low because the benchmark
reads memory.stat in a loop to keep the flush rate up.  For cases where
flushes are triggered only by the 2s periodic worker, more changes will
have accumulated between flushes, so more cachelines are skipped and the
per-flush saving should be larger.

Link: https://lore.kernel.org/20260903215616.1456239-1-joannelkoong@gmail.com
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reviewed-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: Yosry Ahmed <yosry@kernel.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
This commit is contained in:
Joanne Koong 2026-09-03 14:56:16 -07:00 committed by Andrew Morton
parent bffa47f1ca
commit ba99faaf67
3 changed files with 44 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/linux/cgroup.h
View File

@ -154,6 +154,9 @@ struct cgroup *cgroup_get_from_path(const char *path);
struct cgroup *cgroup_get_from_fd(int fd);
struct cgroup *cgroup_v1v2_get_from_fd(int fd);
/* Was this controller blocked from v1 hierarchies by cgroup_no_v1= ? */
bool cgroup1_ssid_disabled(int ssid);
int cgroup_attach_task_all(struct task_struct *from, struct task_struct *);
int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from);

1
kernel/cgroup/cgroup-internal.h
View File

@ -285,7 +285,6 @@ extern struct kernfs_syscall_ops cgroup1_kf_syscall_ops;
extern const struct fs_parameter_spec cgroup1_fs_parameters[];
int proc_cgroupstats_show(struct seq_file *m, void *v);
bool cgroup1_ssid_disabled(int ssid);
void cgroup1_pidlist_destroy_all(struct cgroup *cgrp);
void cgroup1_release_agent(struct work_struct *work);
void cgroup1_check_for_release(struct cgroup *cgrp);

48
mm/memcontrol.c
View File

@ -676,9 +676,11 @@ struct memcg_vmstats {
long state[MEMCG_VMSTAT_SIZE];
unsigned long events[NR_MEMCG_EVENTS];
#ifdef CONFIG_MEMCG_V1
/* Non-hierarchical (CPU aggregated) page state & events */
long state_local[MEMCG_VMSTAT_SIZE];
unsigned long events_local[NR_MEMCG_EVENTS];
#endif
/* Pending child counts during tree propagation */
long state_pending[MEMCG_VMSTAT_SIZE];
@ -688,6 +690,31 @@ struct memcg_vmstats {
atomic_long_t stats_updates;
};
/*
* The non-hierarchical memcg-wide counters are read back only by the legacy
* memory.stat and memory.numa_stat, and by reparenting on offline, all of which
* are v1-only. If the kernel is built without CONFIG_MEMCG_V1, or if the boot
* param cgroup_no_v1= has blocked the memory controller from v1 hierarchies,
* then nothing reads them and writers can skip the updates.
*/
static long *memcg_state_local_array(struct mem_cgroup *memcg)
{
#ifdef CONFIG_MEMCG_V1
if (!cgroup1_ssid_disabled(memory_cgrp_id))
return memcg->vmstats->state_local;
#endif
return NULL;
}
static unsigned long *memcg_events_local_array(struct mem_cgroup *memcg)
{
#ifdef CONFIG_MEMCG_V1
if (!cgroup1_ssid_disabled(memory_cgrp_id))
return memcg->vmstats->events_local;
#endif
return NULL;
}
/*
* memcg and lruvec stats flushing
*
@ -4469,7 +4496,10 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
struct aggregate_control {
/* pointer to the aggregated (CPU and subtree aggregated) counters */
long *aggregate;
/* pointer to the non-hierarchichal (CPU aggregated) counters */
/*
* pointer to the non-hierarchical (CPU aggregated) counters or NULL to
* skip updating them (see memcg_state_local_array())
*/
long *local;
/* pointer to the pending child counters during tree propagation */
long *pending;
@ -4508,7 +4538,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac)
}
/* Aggregate counts on this level and propagate upwards */
if (delta_cpu)
if (delta_cpu && ac->local)
ac->local[i] += delta_cpu;
if (delta) {
@ -4522,6 +4552,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac)
#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
{
long *state_local = memcg_state_local_array(memcg);
int nid;
if (atomic_read(&memcg->kmem_stat)) {
@ -4529,7 +4560,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
int index = memcg_stats_index(MEMCG_KMEM);
memcg->vmstats->state[index] += kmem;
memcg->vmstats->state_local[index] += kmem;
if (state_local)
state_local[index] += kmem;
if (parent)
parent->vmstats->state_pending[index] += kmem;
}
@ -4551,7 +4583,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
if (plstats)
plstats->state_pending[index] += slab;
memcg->vmstats->state[index] += slab;
memcg->vmstats->state_local[index] += slab;
if (state_local)
state_local[index] += slab;
if (parent)
parent->vmstats->state_pending[index] += slab;
}
@ -4564,7 +4597,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
if (plstats)
plstats->state_pending[index] += slab;
memcg->vmstats->state[index] += slab;
memcg->vmstats->state_local[index] += slab;
if (state_local)
state_local[index] += slab;
if (parent)
parent->vmstats->state_pending[index] += slab;
}
@ -4589,7 +4623,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
ac = (struct aggregate_control) {
.aggregate = memcg->vmstats->state,
.local = memcg->vmstats->state_local,
.local = memcg_state_local_array(memcg),
.pending = memcg->vmstats->state_pending,
.ppending = parent ? parent->vmstats->state_pending : NULL,
.cstat = statc->state,
@ -4600,7 +4634,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
ac = (struct aggregate_control) {
.aggregate = memcg->vmstats->events,
.local = memcg->vmstats->events_local,
.local = memcg_events_local_array(memcg),
.pending = memcg->vmstats->events_pending,
.ppending = parent ? parent->vmstats->events_pending : NULL,
.cstat = statc->events,