github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-07 14:04:54 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Revert "cgroup/cpuset: Convert cpuset_mutex to percpu_rwsem"

Browse Source 
This reverts commit 1243dc518c.

percpu_rwsem is an rcu based lock.  Under loaded conditions
this lock will require that each cpu perform a switch, causing
exessive delays with long-running tasks on those CPUs.

In the case of hotplug/pause, this can slow down the ability
to activate/deactivate a CPU.

Revert the change from cpuset_mutex to percpu_rwsem, to
eliminate that overhead, and revert to a global lock.

Bug: 161210528
Change-Id: Id00dcaa6d601b561d1321d3e944b6c52e9663f1a
Signed-off-by: Stephen Dickey <dickey@codeaurora.org>
This commit is contained in:
Stephen Dickey 2020-12-01 11:37:27 -08:00 committed by Todd Kjos
parent 782131fed0
commit 18695a1200
1 changed files with 33 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
kernel/cgroup/cpuset.c
View File

@ -336,7 +336,7 @@ static struct cpuset top_cpuset = {
* guidelines for accessing subsystem state in kernel/cgroup.c
*/
DEFINE_STATIC_PERCPU_RWSEM(cpuset_rwsem);
static DEFINE_MUTEX(cpuset_mutex);
static DEFINE_SPINLOCK(callback_lock);
static struct workqueue_struct *cpuset_migrate_mm_wq;
@ -928,7 +928,7 @@ static void rebuild_root_domains(void)
struct cpuset *cs = NULL;
struct cgroup_subsys_state *pos_css;
percpu_rwsem_assert_held(&cpuset_rwsem);
lockdep_assert_held(&cpuset_mutex);
lockdep_assert_cpus_held();
lockdep_assert_held(&sched_domains_mutex);
@ -986,8 +986,7 @@ static void rebuild_sched_domains_locked(void)
cpumask_var_t *doms;
int ndoms;
lockdep_assert_cpus_held();
percpu_rwsem_assert_held(&cpuset_rwsem);
lockdep_assert_held(&cpuset_mutex);
/*
* We have raced with CPU hotplug. Don't do anything to avoid
@ -1017,9 +1016,9 @@ static void rebuild_sched_domains_locked(void)
void rebuild_sched_domains(void)
{
get_online_cpus();
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
rebuild_sched_domains_locked();
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
put_online_cpus();
}
@ -1138,7 +1137,7 @@ static int update_parent_subparts_cpumask(struct cpuset *cpuset, int cmd,
int deleting; /* Moving cpus from subparts_cpus to effective_cpus */
bool part_error = false; /* Partition error? */
percpu_rwsem_assert_held(&cpuset_rwsem);
lockdep_assert_held(&cpuset_mutex);
/*
* The parent must be a partition root.
@ -2128,7 +2127,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
cs = css_cs(css);
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
/* allow moving tasks into an empty cpuset if on default hierarchy */
ret = -ENOSPC;
@ -2152,7 +2151,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
cs->attach_in_progress++;
ret = 0;
out_unlock:
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
return ret;
}
@ -2162,9 +2161,9 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset)
cgroup_taskset_first(tset, &css);
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
css_cs(css)->attach_in_progress--;
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
}
/*
@ -2187,7 +2186,7 @@ static void cpuset_attach(struct cgroup_taskset *tset)
cgroup_taskset_first(tset, &css);
cs = css_cs(css);
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
/* prepare for attach */
if (cs == &top_cpuset)
@ -2241,7 +2240,7 @@ static void cpuset_attach(struct cgroup_taskset *tset)
if (!cs->attach_in_progress)
wake_up(&cpuset_attach_wq);
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
}
/* The various types of files and directories in a cpuset file system */
@ -2273,7 +2272,7 @@ static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
int retval = 0;
get_online_cpus();
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
if (!is_cpuset_online(cs)) {
retval = -ENODEV;
goto out_unlock;
@ -2309,7 +2308,7 @@ static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
break;
}
out_unlock:
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
put_online_cpus();
return retval;
}
@ -2322,7 +2321,7 @@ static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
int retval = -ENODEV;
get_online_cpus();
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
if (!is_cpuset_online(cs))
goto out_unlock;
@ -2335,7 +2334,7 @@ static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
break;
}
out_unlock:
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
put_online_cpus();
return retval;
}
@ -2376,7 +2375,7 @@ static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
flush_work(&cpuset_hotplug_work);
get_online_cpus();
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
if (!is_cpuset_online(cs))
goto out_unlock;
@ -2400,7 +2399,7 @@ static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
free_cpuset(trialcs);
out_unlock:
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
put_online_cpus();
kernfs_unbreak_active_protection(of->kn);
css_put(&cs->css);
@ -2533,13 +2532,13 @@ static ssize_t sched_partition_write(struct kernfs_open_file *of, char *buf,
css_get(&cs->css);
get_online_cpus();
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
if (!is_cpuset_online(cs))
goto out_unlock;
retval = update_prstate(cs, val);
out_unlock:
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
put_online_cpus();
css_put(&cs->css);
return retval ?: nbytes;
@ -2747,7 +2746,7 @@ static int cpuset_css_online(struct cgroup_subsys_state *css)
return 0;
get_online_cpus();
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
set_bit(CS_ONLINE, &cs->flags);
if (is_spread_page(parent))
@ -2799,7 +2798,7 @@ static int cpuset_css_online(struct cgroup_subsys_state *css)
cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
spin_unlock_irq(&callback_lock);
out_unlock:
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
put_online_cpus();
return 0;
}
@ -2820,7 +2819,7 @@ static void cpuset_css_offline(struct cgroup_subsys_state *css)
struct cpuset *cs = css_cs(css);
get_online_cpus();
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
if (is_partition_root(cs))
update_prstate(cs, 0);
@ -2839,7 +2838,7 @@ static void cpuset_css_offline(struct cgroup_subsys_state *css)
cpuset_dec();
clear_bit(CS_ONLINE, &cs->flags);
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
put_online_cpus();
}
@ -2852,7 +2851,7 @@ static void cpuset_css_free(struct cgroup_subsys_state *css)
static void cpuset_bind(struct cgroup_subsys_state *root_css)
{
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
spin_lock_irq(&callback_lock);
if (is_in_v2_mode()) {
@ -2865,7 +2864,7 @@ static void cpuset_bind(struct cgroup_subsys_state *root_css)
}
spin_unlock_irq(&callback_lock);
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
}
/*
@ -2907,8 +2906,6 @@ struct cgroup_subsys cpuset_cgrp_subsys = {
int __init cpuset_init(void)
{
BUG_ON(percpu_init_rwsem(&cpuset_rwsem));
BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL));
BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL));
BUG_ON(!zalloc_cpumask_var(&top_cpuset.subparts_cpus, GFP_KERNEL));
@ -2982,7 +2979,7 @@ hotplug_update_tasks_legacy(struct cpuset *cs,
is_empty = cpumask_empty(cs->cpus_allowed) ||
nodes_empty(cs->mems_allowed);
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
/*
* Move tasks to the nearest ancestor with execution resources,
@ -2992,7 +2989,7 @@ hotplug_update_tasks_legacy(struct cpuset *cs,
if (is_empty)
remove_tasks_in_empty_cpuset(cs);
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
}
static void
@ -3042,14 +3039,14 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
retry:
wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
/*
* We have raced with task attaching. We wait until attaching
* is finished, so we won't attach a task to an empty cpuset.
*/
if (cs->attach_in_progress) {
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
goto retry;
}
@ -3117,7 +3114,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems,
cpus_updated, mems_updated);
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
}
/**
@ -3147,7 +3144,7 @@ void cpuset_hotplug_workfn(struct work_struct *work)
if (on_dfl && !alloc_cpumasks(NULL, &tmp))
ptmp = &tmp;
percpu_down_write(&cpuset_rwsem);
mutex_lock(&cpuset_mutex);
/* fetch the available cpus/mems and find out which changed how */
cpumask_copy(&new_cpus, cpu_active_mask);
@ -3197,7 +3194,7 @@ void cpuset_hotplug_workfn(struct work_struct *work)
update_tasks_nodemask(&top_cpuset);
}
percpu_up_write(&cpuset_rwsem);
mutex_unlock(&cpuset_mutex);
/* if cpus or mems changed, we need to propagate to descendants */
if (cpus_updated || mems_updated) {