mirror of
https://github.com/torvalds/linux.git
synced 2026-09-30 05:54:02 +02:00
Scheduler fixes:
- Fix LLC mis-scheduling bugs (Tim Chen, Lu Wang)
- Fix cache-grouping related scheduling statistics UAF bugs (Tim Chen)
- Skip kernel threads for cache aware scheduling to rubustify the code
(Chen Yu)
- Refresh LLC capacity across CPU hotplug, to fix capacity
underestimation bug (Davi Chaves Azevedo)
- Account PSI IRQ time to the execution context, not the scheduling
context, to fix proxy scheduling accounting bug (Zhan Xusheng)
Signed-off-by: Ingo Molnar <mingo@kernel.org>
-----BEGIN PGP SIGNATURE-----
iQJFBAABCgAvFiEEBpT5eoXrXCwVQwEKEnMQ0APhK1gFAmq4zcIRHG1pbmdvQGtl
cm5lbC5vcmcACgkQEnMQ0APhK1iN/Q/9HCvfdQ0tTjZHLVhL5f7ZnIjwxVNzmKTK
ysqfXacW3KIwuulXjEDXgAXRi85nKiAxV4sHVp8WaDfpu4NEVzekXZV7dHPf8fPZ
9LCfFJrrYhcAhKHwEIEyGKn7uSSYzFNVjUxHVfc7W7JtqNtrXOYmYKtJuaMdE8ya
fxV/wGHw3HHCmDv/v4BurqOgNyPFEzhN6vl875J0qEdsauxakwvlcIJsAWDstbxi
4kDF+47lLyDZoolrWsVCl8BoxERznK8JelmNVx0FkEpKi2DnLarWjkpAnjfGkIFg
lz2jxdQnTW5v60BTLw4yAUIam8SAzbDNnfKexzT3oG1UgdyqEOMZ4bpynOhoCu5b
3c/2E7Ah7zfMnyQrItnZ7JaXiCrHgPM+aHxjskmMV08Zr54S5hbTxws7uwfl/MA0
5r64GzrStcjswNZKKQycDFCwtkZIcsBmSyJvJqb3RPm4NVYHIBJWDd1YLlH7pM8R
BuNiW0zM9IdXl6EgD2gX41WoeESobaPPPb67b0DQ6WRSZkz7UAF0Xzfz/xu66WlO
AwMMFHF1qhAcBkUBYU9R+hGSQoaTClf7f2stuSr4zVebAZSAzGLRQob5s/ldmfel
2c1se8w3fPUC29eVB7AQMFsh1xFMzkR53r6zZEKJsG5MHu06fdKD2FhdQcfQDAob
K8Y4Yh4LhOo=
=2067
-----END PGP SIGNATURE-----
Merge tag 'sched-urgent-2026-09-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull scheduler fixes from Ingo Molnar:
- Fix LLC mis-scheduling bugs (Tim Chen, Lu Wang)
- Fix cache-grouping related scheduling statistics UAF bugs (Tim Chen)
- Skip kernel threads for cache aware scheduling to rubustify the code
(Chen Yu)
- Refresh LLC capacity across CPU hotplug, to fix capacity
underestimation bug (Davi Chaves Azevedo)
- Account PSI IRQ time to the execution context, not the scheduling
context, to fix proxy scheduling accounting bug (Zhan Xusheng)
* tag 'sched-urgent-2026-09-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
sched/core: Account PSI IRQ time to the execution context, not the scheduling context
sched/cache: Refresh LLC capacity across CPU hotplug, to fix capacity underestimation bug
sched/cache: Skip kernel threads for cache aware scheduling to rubustify the code
sched/cache: Introduce task_struct->sched_cache_grp to fix UAF
sched/cache: Decouple sched_cache_group from mm to fix UAF
sched/cache: Honor migrate_llc_task semantics in active load balance, to fix LLC mis-scheduling bug
sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug
This commit is contained in:
Linus Torvalds
commit
673dab7eac
|
|
@ -1040,9 +1040,10 @@ static int cacheinfo_cpu_online(unsigned int cpu)
|
|||
rc = cache_add_dev(cpu);
|
||||
if (rc)
|
||||
goto err;
|
||||
if (cpu_map_shared_cache(true, cpu, &cpu_map))
|
||||
if (cpu_map_shared_cache(true, cpu, &cpu_map)) {
|
||||
update_per_cpu_data_slice_size(true, cpu, cpu_map);
|
||||
sched_update_llc_bytes(cpu);
|
||||
sched_update_llc_bytes(cpu_map);
|
||||
}
|
||||
return 0;
|
||||
err:
|
||||
free_cache_attributes(cpu);
|
||||
|
|
@ -1059,10 +1060,10 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu)
|
|||
cpu_cache_sysfs_exit(cpu);
|
||||
|
||||
free_cache_attributes(cpu);
|
||||
if (nr_shared > 1)
|
||||
if (nr_shared > 1) {
|
||||
update_per_cpu_data_slice_size(false, cpu, cpu_map);
|
||||
|
||||
sched_update_llc_bytes(cpu);
|
||||
sched_update_llc_bytes(cpu_map);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -882,6 +882,7 @@ static int exec_mmap(struct linux_binprm *bprm)
|
|||
active_mm = tsk->active_mm;
|
||||
tsk->active_mm = mm;
|
||||
tsk->mm = mm;
|
||||
sched_cache_exec_mmap(tsk, mm);
|
||||
mm_init_cid(mm, tsk);
|
||||
exec_state = task_exec_state_replace(tsk, exec_state);
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -1226,7 +1226,7 @@ struct mm_struct {
|
|||
struct mm_mm_cid mm_cid;
|
||||
|
||||
/* sched_cache related statistics */
|
||||
struct sched_cache_stat sc_stat;
|
||||
struct sched_cache_group *sched_cache_grp;
|
||||
#ifdef CONFIG_MMU
|
||||
atomic_long_t pgtables_bytes; /* size of all page tables */
|
||||
#endif
|
||||
|
|
@ -1624,8 +1624,9 @@ static inline unsigned int mm_cid_size(void)
|
|||
#endif /* CONFIG_SCHED_MM_CID */
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
void mm_init_sched(struct mm_struct *mm,
|
||||
struct sched_cache_time __percpu *pcpu_sched);
|
||||
int mm_init_sched(struct mm_struct *mm,
|
||||
struct sched_cache_time __percpu *pcpu_sched);
|
||||
void mm_destroy_sched(struct mm_struct *mm);
|
||||
|
||||
static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
|
||||
{
|
||||
|
|
@ -1635,17 +1636,11 @@ static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
|
|||
if (!pcpu_sched)
|
||||
return -ENOMEM;
|
||||
|
||||
mm_init_sched(mm, pcpu_sched);
|
||||
return 0;
|
||||
return mm_init_sched(mm, pcpu_sched);
|
||||
}
|
||||
|
||||
#define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__))
|
||||
|
||||
static inline void mm_destroy_sched(struct mm_struct *mm)
|
||||
{
|
||||
free_percpu(mm->sc_stat.pcpu_sched);
|
||||
mm->sc_stat.pcpu_sched = NULL;
|
||||
}
|
||||
#else /* !CONFIG_SCHED_CACHE */
|
||||
|
||||
static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; }
|
||||
|
|
|
|||
|
|
@ -1433,6 +1433,7 @@ struct task_struct {
|
|||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
struct callback_head cache_work;
|
||||
struct sched_cache_group __rcu *sched_cache_grp;
|
||||
int preferred_llc;
|
||||
/* 1: task was enqueued to its preferred LLC, 0 otherwise */
|
||||
int pref_llc_queued;
|
||||
|
|
@ -2405,7 +2406,7 @@ struct sched_cache_time {
|
|||
unsigned long epoch;
|
||||
};
|
||||
|
||||
struct sched_cache_stat {
|
||||
struct sched_cache_group {
|
||||
struct sched_cache_time __percpu *pcpu_sched;
|
||||
raw_spinlock_t lock;
|
||||
unsigned long epoch;
|
||||
|
|
@ -2413,11 +2414,26 @@ struct sched_cache_stat {
|
|||
unsigned long next_scan;
|
||||
unsigned long footprint;
|
||||
int cpu;
|
||||
refcount_t refcnt;
|
||||
struct rcu_head rcu;
|
||||
} ____cacheline_aligned_in_smp;
|
||||
|
||||
struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp);
|
||||
struct sched_cache_group *task_cache_group_get(struct task_struct *p);
|
||||
|
||||
void sched_cache_fork(struct task_struct *p);
|
||||
void sched_cache_fork_cleanup(struct task_struct *p);
|
||||
void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm);
|
||||
void sched_cache_exit_mm(struct task_struct *p);
|
||||
|
||||
#else
|
||||
|
||||
struct sched_cache_stat { };
|
||||
struct sched_cache_group { };
|
||||
|
||||
static inline void sched_cache_fork(struct task_struct *p) { }
|
||||
static inline void sched_cache_fork_cleanup(struct task_struct *p) { }
|
||||
static inline void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) { }
|
||||
static inline void sched_cache_exit_mm(struct task_struct *p) { }
|
||||
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -281,9 +281,9 @@ static inline int task_node(const struct task_struct *p)
|
|||
}
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
extern void sched_update_llc_bytes(unsigned int cpu);
|
||||
extern void sched_update_llc_bytes(const struct cpumask *cpus);
|
||||
#else
|
||||
static inline void sched_update_llc_bytes(unsigned int cpu) { }
|
||||
static inline void sched_update_llc_bytes(const struct cpumask *cpus) { }
|
||||
#endif
|
||||
|
||||
#endif /* _LINUX_SCHED_TOPOLOGY_H */
|
||||
|
|
|
|||
|
|
@ -551,32 +551,6 @@ void mm_update_next_owner(struct mm_struct *mm)
|
|||
}
|
||||
#endif /* CONFIG_MEMCG */
|
||||
|
||||
#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING)
|
||||
/*
|
||||
* Subtract the memory footprint of the current task from
|
||||
* mm.
|
||||
*/
|
||||
static void exit_mm_sched_cache(struct mm_struct *mm)
|
||||
{
|
||||
unsigned long fp, sub;
|
||||
|
||||
if (!current->total_numa_faults)
|
||||
return;
|
||||
/*
|
||||
* No lock protection due to performance considerations.
|
||||
* Make sure mm->sc_stat.footprint does not become
|
||||
* negative.
|
||||
*/
|
||||
fp = READ_ONCE(mm->sc_stat.footprint);
|
||||
sub = min(fp, current->total_numa_faults);
|
||||
WRITE_ONCE(mm->sc_stat.footprint, fp - sub);
|
||||
}
|
||||
#else
|
||||
static inline void exit_mm_sched_cache(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */
|
||||
|
||||
/*
|
||||
* Turn us into a lazy TLB process if we
|
||||
* aren't already..
|
||||
|
|
@ -589,7 +563,7 @@ static void exit_mm(void)
|
|||
if (!mm)
|
||||
return;
|
||||
|
||||
exit_mm_sched_cache(mm);
|
||||
sched_cache_exit_mm(current);
|
||||
|
||||
mmap_read_lock(mm);
|
||||
mmgrab_lazy_tlb(mm);
|
||||
|
|
|
|||
|
|
@ -1599,6 +1599,7 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk)
|
|||
|
||||
tsk->mm = mm;
|
||||
tsk->active_mm = mm;
|
||||
sched_cache_fork(tsk);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -2602,6 +2603,7 @@ __latent_entropy struct task_struct *copy_process(
|
|||
bad_fork_cleanup_namespaces:
|
||||
exit_nsproxy_namespaces(p);
|
||||
bad_fork_cleanup_mm:
|
||||
sched_cache_fork_cleanup(p);
|
||||
if (p->mm) {
|
||||
mm_clear_owner(p->mm, p);
|
||||
mmput(p->mm);
|
||||
|
|
|
|||
|
|
@ -5798,7 +5798,7 @@ void sched_tick(void)
|
|||
curr = rq->curr;
|
||||
donor = rq->donor;
|
||||
|
||||
psi_account_irqtime(rq, donor, NULL);
|
||||
psi_account_irqtime(rq, curr, NULL);
|
||||
|
||||
update_rq_clock(rq);
|
||||
hw_pressure = arch_scale_hw_pressure(cpu_of(rq));
|
||||
|
|
|
|||
|
|
@ -1478,7 +1478,7 @@ static inline int get_sched_cache_scale(int mul)
|
|||
return (1 + (tol - 1) * mul);
|
||||
}
|
||||
|
||||
static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
|
||||
static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu)
|
||||
{
|
||||
#ifdef CONFIG_NUMA_BALANCING
|
||||
unsigned long llc, footprint;
|
||||
|
|
@ -1497,7 +1497,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
|
|||
* excluded.
|
||||
*/
|
||||
llc = sd->llc_bytes;
|
||||
footprint = READ_ONCE(mm->sc_stat.footprint);
|
||||
footprint = READ_ONCE(grp->footprint);
|
||||
|
||||
/*
|
||||
* Scale the LLC size by 256*llc_aggr_tolerance
|
||||
|
|
@ -1526,7 +1526,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
|
|||
return false;
|
||||
}
|
||||
|
||||
static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
|
||||
static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p,
|
||||
int cpu)
|
||||
{
|
||||
int scale;
|
||||
|
|
@ -1542,10 +1542,32 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
|
|||
if (scale == INT_MAX)
|
||||
return false;
|
||||
|
||||
return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads),
|
||||
return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads),
|
||||
(scale * per_cpu(sd_llc_size, cpu)));
|
||||
}
|
||||
|
||||
/*
|
||||
* A task counts in nr_pref_llc_running while it is queued on its preferred
|
||||
* LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in
|
||||
* the runnable domain so alb_break_llc() can compare it with h_nr_runnable.
|
||||
*/
|
||||
static bool task_pref_llc_runnable(struct task_struct *p)
|
||||
{
|
||||
return p->pref_llc_queued && !p->se.sched_delayed;
|
||||
}
|
||||
|
||||
static void pref_llc_running_inc(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
if (task_pref_llc_runnable(p))
|
||||
rq->nr_pref_llc_running++;
|
||||
}
|
||||
|
||||
static void pref_llc_running_dec(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
if (task_pref_llc_runnable(p))
|
||||
rq->nr_pref_llc_running--;
|
||||
}
|
||||
|
||||
static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
int pref_llc, pref_llc_queued;
|
||||
|
|
@ -1557,7 +1579,6 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
|
|||
|
||||
pref_llc_queued = (pref_llc == task_llc(p));
|
||||
rq->nr_llc_running++;
|
||||
rq->nr_pref_llc_running += pref_llc_queued;
|
||||
|
||||
/*
|
||||
* Record whether p is enqueued on its preferred
|
||||
|
|
@ -1575,6 +1596,9 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
|
|||
*/
|
||||
p->pref_llc_queued = pref_llc_queued;
|
||||
|
||||
/* Skipped while delayed; clear_delayed() adds it back on wake. */
|
||||
pref_llc_running_inc(rq, p);
|
||||
|
||||
sd = rcu_dereference_all(rq->sd);
|
||||
if (sd && (unsigned int)pref_llc < sd->llc_max)
|
||||
sd->llc_counts[pref_llc]++;
|
||||
|
|
@ -1591,7 +1615,12 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
|
|||
|
||||
rq->nr_llc_running--;
|
||||
if (p->pref_llc_queued) {
|
||||
rq->nr_pref_llc_running--;
|
||||
/*
|
||||
* Skipped if still delayed (set_delayed() already removed it);
|
||||
* clearing pref_llc_queued below also stops clear_delayed()
|
||||
* from re-adding it.
|
||||
*/
|
||||
pref_llc_running_dec(rq, p);
|
||||
/*
|
||||
* Update the status in case
|
||||
* other logic might query
|
||||
|
|
@ -1619,12 +1648,20 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
|
|||
}
|
||||
}
|
||||
|
||||
void mm_init_sched(struct mm_struct *mm,
|
||||
struct sched_cache_time __percpu *_pcpu_sched)
|
||||
int mm_init_sched(struct mm_struct *mm,
|
||||
struct sched_cache_time __percpu *_pcpu_sched)
|
||||
{
|
||||
struct sched_cache_group *grp;
|
||||
unsigned long epoch = 0;
|
||||
int i;
|
||||
|
||||
grp = kzalloc_obj(*grp);
|
||||
if (!grp) {
|
||||
free_percpu(_pcpu_sched);
|
||||
mm->sched_cache_grp = NULL;
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i);
|
||||
struct rq *rq = cpu_rq(i);
|
||||
|
|
@ -1635,18 +1672,141 @@ void mm_init_sched(struct mm_struct *mm,
|
|||
epoch = rq->cpu_epoch;
|
||||
}
|
||||
|
||||
raw_spin_lock_init(&mm->sc_stat.lock);
|
||||
mm->sc_stat.epoch = epoch;
|
||||
mm->sc_stat.cpu = -1;
|
||||
mm->sc_stat.next_scan = jiffies;
|
||||
mm->sc_stat.nr_running_avg = 0;
|
||||
mm->sc_stat.footprint = 0;
|
||||
raw_spin_lock_init(&grp->lock);
|
||||
grp->epoch = epoch;
|
||||
grp->cpu = -1;
|
||||
grp->next_scan = jiffies;
|
||||
grp->nr_running_avg = 0;
|
||||
grp->footprint = 0;
|
||||
refcount_set(&grp->refcnt, 1);
|
||||
/*
|
||||
* The update to mm->sc_stat should not be reordered
|
||||
* before initialization to mm's other fields, in case
|
||||
* The update to grp->pcpu_sched should not be reordered
|
||||
* before initialization to grp's other fields, in case
|
||||
* the readers may get invalid mm_sched_epoch, etc.
|
||||
*/
|
||||
smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched);
|
||||
smp_store_release(&grp->pcpu_sched, _pcpu_sched);
|
||||
/*
|
||||
* Publish the group last. Not every reader qualifies it by
|
||||
* grp->pcpu_sched - can_migrate_llc_task() only checks that the
|
||||
* pointer is non-NULL before reading grp->footprint and
|
||||
* grp->nr_running_avg - so a reachable group must already be
|
||||
* fully initialized.
|
||||
*/
|
||||
smp_store_release(&mm->sched_cache_grp, grp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void sched_cache_group_free_rcu(struct rcu_head *rcu)
|
||||
{
|
||||
struct sched_cache_group *grp =
|
||||
container_of(rcu, struct sched_cache_group, rcu);
|
||||
|
||||
free_percpu(grp->pcpu_sched);
|
||||
kfree(grp);
|
||||
}
|
||||
|
||||
static void sched_cache_group_put(struct sched_cache_group *grp)
|
||||
{
|
||||
if (!grp || !refcount_dec_and_test(&grp->refcnt))
|
||||
return;
|
||||
|
||||
call_rcu(&grp->rcu, sched_cache_group_free_rcu);
|
||||
}
|
||||
|
||||
DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *,
|
||||
sched_cache_group_put(_T));
|
||||
|
||||
#define rcu_deref_sched_cache_grp(tsk) \
|
||||
rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current)
|
||||
|
||||
static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p,
|
||||
struct sched_cache_group *new)
|
||||
{
|
||||
struct sched_cache_group *old;
|
||||
|
||||
old = rcu_deref_sched_cache_grp(p);
|
||||
rcu_assign_pointer(p->sched_cache_grp, new);
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp)
|
||||
{
|
||||
/*
|
||||
* refcount_inc_not_zero() is the acquire primitive for lockless
|
||||
* (RCU) lookups; plain refcount_inc() would scribble the count if
|
||||
* it already reached zero. Return NULL in that case.
|
||||
*/
|
||||
if (grp && !refcount_inc_not_zero(&grp->refcnt))
|
||||
grp = NULL;
|
||||
|
||||
return grp;
|
||||
}
|
||||
|
||||
struct sched_cache_group *task_cache_group_get(struct task_struct *p)
|
||||
{
|
||||
guard(rcu)();
|
||||
return sched_cache_group_get(rcu_dereference(p->sched_cache_grp));
|
||||
}
|
||||
|
||||
void sched_cache_fork(struct task_struct *p)
|
||||
{
|
||||
/*
|
||||
* The child takes its own reference on the mm's cache group, separate
|
||||
* from the reference held by the mm. @p is not yet visible to readers,
|
||||
* so a plain initializing store is enough.
|
||||
*/
|
||||
RCU_INIT_POINTER(p->sched_cache_grp,
|
||||
sched_cache_group_get(p->mm->sched_cache_grp));
|
||||
}
|
||||
|
||||
void sched_cache_fork_cleanup(struct task_struct *p)
|
||||
{
|
||||
/*
|
||||
* A fork that fails after sched_cache_fork() never reaches exit_mm(),
|
||||
* so drop the reference here. @p never became visible, so there are no
|
||||
* concurrent readers and the reference we hold keeps the group alive.
|
||||
*/
|
||||
sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp));
|
||||
RCU_INIT_POINTER(p->sched_cache_grp, NULL);
|
||||
}
|
||||
|
||||
void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm)
|
||||
{
|
||||
struct sched_cache_group *old;
|
||||
|
||||
/*
|
||||
* Acquire the new reference before publishing the pointer, then drop
|
||||
* the old one. @p is current and the only writer of its own pointer.
|
||||
*/
|
||||
old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp));
|
||||
sched_cache_group_put(old);
|
||||
}
|
||||
|
||||
void sched_cache_exit_mm(struct task_struct *p)
|
||||
{
|
||||
struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL);
|
||||
|
||||
#ifdef CONFIG_NUMA_BALANCING
|
||||
/*
|
||||
* Subtract this task's footprint from the group before dropping the
|
||||
* reference, so the group footprint converges as its threads exit.
|
||||
* Unlocked for performance; clamp to avoid underflow.
|
||||
*/
|
||||
if (grp && p->total_numa_faults) {
|
||||
unsigned long fp = READ_ONCE(grp->footprint);
|
||||
unsigned long sub = min(fp, p->total_numa_faults);
|
||||
|
||||
WRITE_ONCE(grp->footprint, fp - sub);
|
||||
}
|
||||
#endif
|
||||
sched_cache_group_put(grp);
|
||||
}
|
||||
|
||||
void mm_destroy_sched(struct mm_struct *mm)
|
||||
{
|
||||
sched_cache_group_put(mm->sched_cache_grp);
|
||||
mm->sched_cache_grp = NULL;
|
||||
}
|
||||
|
||||
/* because why would C be fully specified */
|
||||
|
|
@ -1697,14 +1857,14 @@ static unsigned long fraction_mm_sched(struct rq *rq,
|
|||
return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1);
|
||||
}
|
||||
|
||||
static int get_pref_llc(struct task_struct *p, struct mm_struct *mm)
|
||||
static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp)
|
||||
{
|
||||
int mm_sched_llc = -1, mm_sched_cpu;
|
||||
|
||||
if (!mm)
|
||||
if (!grp)
|
||||
return -1;
|
||||
|
||||
mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu);
|
||||
mm_sched_cpu = READ_ONCE(grp->cpu);
|
||||
if (mm_sched_cpu != -1) {
|
||||
mm_sched_llc = llc_id(mm_sched_cpu);
|
||||
|
||||
|
|
@ -1734,8 +1894,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
|
|||
static inline
|
||||
void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
|
||||
{
|
||||
struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
|
||||
struct sched_cache_time *pcpu_sched;
|
||||
struct mm_struct *mm = p->mm;
|
||||
int mm_sched_llc = -1;
|
||||
unsigned long epoch;
|
||||
|
||||
|
|
@ -1746,12 +1906,18 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
|
|||
return;
|
||||
/*
|
||||
* init_task, kthreads and user thread created
|
||||
* by user_mode_thread() don't have mm.
|
||||
* by user_mode_thread() don't have a cache group.
|
||||
* In theory a kernel thread does not have any valid
|
||||
* cache group, because sched_cache_fork() is not
|
||||
* invoked for a kernel thread - !grp should gate the
|
||||
* kernel thread. Use the PF_KTHREAD check explicitly
|
||||
* here for safety reasons, to guard against future
|
||||
* modifications and to pair with task_tick_cache().
|
||||
*/
|
||||
if (!mm || !mm->sc_stat.pcpu_sched)
|
||||
if (p->flags & PF_KTHREAD || !grp || !grp->pcpu_sched)
|
||||
return;
|
||||
|
||||
pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq));
|
||||
pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq));
|
||||
|
||||
scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) {
|
||||
__update_mm_sched(rq, pcpu_sched);
|
||||
|
|
@ -1764,14 +1930,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
|
|||
* If this process hasn't hit task_cache_work() for a while invalidate
|
||||
* its preferred state.
|
||||
*/
|
||||
if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout ||
|
||||
invalid_llc_nr(mm, p, cpu_of(rq)) ||
|
||||
exceed_llc_capacity(mm, cpu_of(rq))) {
|
||||
if (READ_ONCE(mm->sc_stat.cpu) != -1)
|
||||
WRITE_ONCE(mm->sc_stat.cpu, -1);
|
||||
if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout ||
|
||||
invalid_llc_nr(grp, p, cpu_of(rq)) ||
|
||||
exceed_llc_capacity(grp, cpu_of(rq))) {
|
||||
if (READ_ONCE(grp->cpu) != -1)
|
||||
WRITE_ONCE(grp->cpu, -1);
|
||||
}
|
||||
|
||||
mm_sched_llc = get_pref_llc(p, mm);
|
||||
mm_sched_llc = get_pref_llc(p, grp);
|
||||
|
||||
/* task not on rq accounted later in account_entity_enqueue() */
|
||||
if (task_running_on_cpu(rq->cpu, p) &&
|
||||
|
|
@ -1784,31 +1950,32 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
|
|||
|
||||
static void task_tick_cache(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
|
||||
struct callback_head *work = &p->cache_work;
|
||||
struct mm_struct *mm = p->mm;
|
||||
unsigned long epoch;
|
||||
|
||||
if (!sched_cache_enabled())
|
||||
return;
|
||||
|
||||
if (!mm || p->flags & PF_KTHREAD ||
|
||||
!mm->sc_stat.pcpu_sched)
|
||||
if (!grp || p->flags & PF_KTHREAD ||
|
||||
!grp->pcpu_sched)
|
||||
return;
|
||||
|
||||
epoch = rq->cpu_epoch;
|
||||
/* avoid moving backwards */
|
||||
if (time_after_eq(mm->sc_stat.epoch, epoch))
|
||||
if (time_after_eq(grp->epoch, epoch))
|
||||
return;
|
||||
|
||||
guard(raw_spinlock)(&mm->sc_stat.lock);
|
||||
guard(raw_spinlock)(&grp->lock);
|
||||
|
||||
if (work->next == work) {
|
||||
task_work_add(p, work, TWA_RESUME);
|
||||
WRITE_ONCE(mm->sc_stat.epoch, epoch);
|
||||
WRITE_ONCE(grp->epoch, epoch);
|
||||
}
|
||||
}
|
||||
|
||||
static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
|
||||
static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p,
|
||||
struct sched_cache_group *grp)
|
||||
{
|
||||
#ifdef CONFIG_NUMA_BALANCING
|
||||
int cpu, curr_cpu, nid, pref_nid;
|
||||
|
|
@ -1816,7 +1983,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
|
|||
if (!static_branch_likely(&sched_numa_balancing))
|
||||
goto out;
|
||||
|
||||
cpu = READ_ONCE(p->mm->sc_stat.cpu);
|
||||
cpu = READ_ONCE(grp->cpu);
|
||||
if (cpu != -1)
|
||||
nid = cpu_to_node(cpu);
|
||||
curr_cpu = task_cpu(p);
|
||||
|
|
@ -1873,13 +2040,13 @@ static inline void update_avg_scale(u64 *avg, u64 sample)
|
|||
|
||||
static void task_cache_work(struct callback_head *work)
|
||||
{
|
||||
struct sched_cache_group *grp __free(sched_cache_group_put) = NULL;
|
||||
cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL;
|
||||
int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu;
|
||||
unsigned long next_scan, now = jiffies;
|
||||
struct task_struct *p = current, *cur;
|
||||
unsigned long curr_m_a_occ = 0;
|
||||
struct mm_struct *mm = p->mm;
|
||||
unsigned long m_a_occ = 0;
|
||||
cpumask_var_t cpus;
|
||||
|
||||
WARN_ON_ONCE(work != &p->cache_work);
|
||||
|
||||
|
|
@ -1888,21 +2055,30 @@ static void task_cache_work(struct callback_head *work)
|
|||
if (p->flags & PF_EXITING)
|
||||
return;
|
||||
|
||||
next_scan = READ_ONCE(mm->sc_stat.next_scan);
|
||||
/*
|
||||
* A reference makes sure grp is not released by others. The rcu
|
||||
* lock can not be held till after zalloc_cpumask_var() below,
|
||||
* because the latter might sleep.
|
||||
*/
|
||||
grp = task_cache_group_get(p);
|
||||
if (!grp)
|
||||
return;
|
||||
|
||||
next_scan = READ_ONCE(grp->next_scan);
|
||||
if (time_before(now, next_scan))
|
||||
return;
|
||||
|
||||
/* only 1 thread is allowed to scan */
|
||||
if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan,
|
||||
if (!try_cmpxchg(&grp->next_scan, &next_scan,
|
||||
now + max_t(unsigned long,
|
||||
READ_ONCE(llc_epoch_period), 1)))
|
||||
return;
|
||||
|
||||
curr_cpu = task_cpu(p);
|
||||
if (invalid_llc_nr(mm, p, curr_cpu) ||
|
||||
exceed_llc_capacity(mm, curr_cpu)) {
|
||||
if (READ_ONCE(mm->sc_stat.cpu) != -1)
|
||||
WRITE_ONCE(mm->sc_stat.cpu, -1);
|
||||
if (invalid_llc_nr(grp, p, curr_cpu) ||
|
||||
exceed_llc_capacity(grp, curr_cpu)) {
|
||||
if (READ_ONCE(grp->cpu) != -1)
|
||||
WRITE_ONCE(grp->cpu, -1);
|
||||
|
||||
return;
|
||||
}
|
||||
|
|
@ -1913,7 +2089,7 @@ static void task_cache_work(struct callback_head *work)
|
|||
scoped_guard (cpus_read_lock) {
|
||||
guard(rcu)();
|
||||
|
||||
get_scan_cpumasks(cpus, p);
|
||||
get_scan_cpumasks(cpus, p, grp);
|
||||
|
||||
for_each_cpu(cpu, cpus) {
|
||||
/* XXX sched_cluster_active */
|
||||
|
|
@ -1926,16 +2102,20 @@ static void task_cache_work(struct callback_head *work)
|
|||
|
||||
for_each_cpu(i, sched_domain_span(sd)) {
|
||||
occ = fraction_mm_sched(cpu_rq(i),
|
||||
per_cpu_ptr(mm->sc_stat.pcpu_sched, i));
|
||||
per_cpu_ptr(grp->pcpu_sched, i));
|
||||
a_occ += occ;
|
||||
if (occ > m_occ) {
|
||||
m_occ = occ;
|
||||
m_cpu = i;
|
||||
}
|
||||
|
||||
/*
|
||||
* rcu_access_pointer() is used because the
|
||||
* pointer is only compared, never dereferenced.
|
||||
*/
|
||||
cur = rcu_dereference_all(cpu_rq(i)->curr);
|
||||
if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) &&
|
||||
cur->mm == mm)
|
||||
rcu_access_pointer(cur->sched_cache_grp) == grp)
|
||||
nr_running++;
|
||||
}
|
||||
|
||||
|
|
@ -1959,7 +2139,7 @@ static void task_cache_work(struct callback_head *work)
|
|||
m_a_cpu = m_cpu;
|
||||
}
|
||||
|
||||
if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu)))
|
||||
if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu)))
|
||||
curr_m_a_occ = a_occ;
|
||||
|
||||
cpumask_andnot(cpus, cpus, sched_domain_span(sd));
|
||||
|
|
@ -1968,7 +2148,7 @@ static void task_cache_work(struct callback_head *work)
|
|||
|
||||
if (m_a_occ > (2 * curr_m_a_occ)) {
|
||||
/*
|
||||
* Avoid switching sc_stat.cpu too fast.
|
||||
* Avoid switching sched_cache_grp->cpu too fast.
|
||||
* The reason to choose 2X is because:
|
||||
* 1. It is better to keep the preferred LLC stable,
|
||||
* rather than changing it frequently and cause migrations
|
||||
|
|
@ -1977,11 +2157,10 @@ static void task_cache_work(struct callback_head *work)
|
|||
* 3. 2X is chosen based on test results, as it delivers
|
||||
* the optimal performance gain so far.
|
||||
*/
|
||||
WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu);
|
||||
WRITE_ONCE(grp->cpu, m_a_cpu);
|
||||
}
|
||||
|
||||
update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running);
|
||||
free_cpumask_var(cpus);
|
||||
update_avg_scale(&grp->nr_running_avg, nr_running);
|
||||
}
|
||||
|
||||
void init_sched_mm(struct task_struct *p)
|
||||
|
|
@ -1990,11 +2169,19 @@ void init_sched_mm(struct task_struct *p)
|
|||
|
||||
init_task_work(work, task_cache_work);
|
||||
work->next = work;
|
||||
/*
|
||||
* dup_task_struct() copies the parent's task_struct, including its
|
||||
* sched_cache_grp, for which the child holds no reference. Clear it
|
||||
* here - before copy_mm() runs - so the child never carries a
|
||||
* borrowed pointer that the fork error path would put.
|
||||
*/
|
||||
RCU_INIT_POINTER(p->sched_cache_grp, NULL);
|
||||
/*
|
||||
* Reset new task's preference to avoid
|
||||
* polluting account_llc_enqueue().
|
||||
*/
|
||||
p->preferred_llc = -1;
|
||||
p->pref_llc_queued = 0;
|
||||
}
|
||||
|
||||
#else /* CONFIG_SCHED_CACHE */
|
||||
|
|
@ -2016,6 +2203,10 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {}
|
|||
|
||||
static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {}
|
||||
|
||||
static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) {}
|
||||
|
||||
static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) {}
|
||||
|
||||
#endif /* CONFIG_SCHED_CACHE */
|
||||
|
||||
/*
|
||||
|
|
@ -3692,6 +3883,7 @@ static int preferred_group_nid(struct task_struct *p, int nid)
|
|||
static void task_numa_placement(struct task_struct *p)
|
||||
__context_unsafe(/* conditional locking */)
|
||||
{
|
||||
struct sched_cache_group __maybe_unused *grp;
|
||||
int seq, nid, max_nid = NUMA_NO_NODE;
|
||||
unsigned long max_faults = 0;
|
||||
unsigned long fault_types[2] = { 0, 0 };
|
||||
|
|
@ -3784,19 +3976,24 @@ static void task_numa_placement(struct task_struct *p)
|
|||
* heuristic and occasional lost updates are tolerable.
|
||||
*
|
||||
* If a task exits, its corresponding footprint must
|
||||
* be subtracted from the mm->sc_stat.footprint, otherwise
|
||||
* the mm->sc_stat.footprint will not converge:
|
||||
* the exiting thread's footprint remains unchanged/undecayed
|
||||
* in mm->sc_stat.footprint. See exit_mm().
|
||||
* be subtracted from p->sched_cache_grp->footprint,
|
||||
* otherwise the footprint will not converge: the
|
||||
* exiting thread's footprint remains unchanged/undecayed.
|
||||
* See exit_mm().
|
||||
*
|
||||
* Lost updates and unsynchronized subtraction
|
||||
* in exit_mm() can cause footprint + diff to
|
||||
* go negative. Clamp to zero to prevent the
|
||||
* unsigned footprint from wrapping.
|
||||
*/
|
||||
new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff;
|
||||
WRITE_ONCE(p->mm->sc_stat.footprint,
|
||||
max(new_fp, 0L));
|
||||
scoped_guard(rcu) {
|
||||
grp = rcu_dereference(p->sched_cache_grp);
|
||||
|
||||
if (grp) {
|
||||
new_fp = (long)READ_ONCE(grp->footprint) + diff;
|
||||
WRITE_ONCE(grp->footprint, max(new_fp, 0L));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -6390,15 +6587,27 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
|
|||
|
||||
static void set_delayed(struct sched_entity *se)
|
||||
{
|
||||
se->sched_delayed = 1;
|
||||
|
||||
/*
|
||||
* Delayed se of cfs_rq have no tasks queued on them.
|
||||
* Do not adjust h_nr_runnable since __dequeue_task()
|
||||
* will account it for blocked tasks.
|
||||
*
|
||||
* This check can be removed because when flat pick
|
||||
* patches get merged as only task can get delayed,
|
||||
* same for clear_delayed().
|
||||
*/
|
||||
if (!entity_is_task(se))
|
||||
if (!entity_is_task(se)) {
|
||||
se->sched_delayed = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Drop a task leaving the runnable set.
|
||||
* Needs to be called before sched_delayed is set.
|
||||
* clear_delayed() mirrors this after clearing the flag.
|
||||
*/
|
||||
pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se));
|
||||
se->sched_delayed = 1;
|
||||
|
||||
for_each_sched_entity(se) {
|
||||
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
||||
|
|
@ -6420,6 +6629,13 @@ static void clear_delayed(struct sched_entity *se)
|
|||
if (!entity_is_task(se))
|
||||
return;
|
||||
|
||||
/*
|
||||
* Re-add on wake, after sched_delayed is cleared. On a final delayed
|
||||
* dequeue account_llc_dequeue() already cleared pref_llc_queued, so
|
||||
* this does nothing.
|
||||
*/
|
||||
pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se));
|
||||
|
||||
for_each_sched_entity(se) {
|
||||
struct cfs_rq *cfs_rq = cfs_rq_of(se);
|
||||
|
||||
|
|
@ -10395,6 +10611,7 @@ enum migration_type {
|
|||
#define LBF_SOME_PINNED 0x08
|
||||
#define LBF_ACTIVE_LB 0x10
|
||||
#define LBF_LLC_PINNED 0x20
|
||||
#define LBF_ACTIVE_LB_LLC 0x40
|
||||
|
||||
struct lb_env {
|
||||
struct sched_domain *sd;
|
||||
|
|
@ -10724,7 +10941,7 @@ static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct
|
|||
static enum llc_mig can_migrate_llc_task(struct lb_env *env,
|
||||
struct task_struct *p)
|
||||
{
|
||||
struct mm_struct *mm;
|
||||
struct sched_cache_group *grp;
|
||||
bool to_pref;
|
||||
int cpu, src_cpu, dst_cpu;
|
||||
|
||||
|
|
@ -10733,19 +10950,19 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env,
|
|||
|
||||
src_cpu = env->src_cpu;
|
||||
dst_cpu = env->dst_cpu;
|
||||
mm = p->mm;
|
||||
if (!mm)
|
||||
grp = rcu_dereference_all(p->sched_cache_grp);
|
||||
if (!grp)
|
||||
return mig_unrestricted;
|
||||
|
||||
cpu = READ_ONCE(mm->sc_stat.cpu);
|
||||
cpu = READ_ONCE(grp->cpu);
|
||||
if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu))
|
||||
return mig_unrestricted;
|
||||
|
||||
/* skip cache aware load balance for too many threads */
|
||||
if (invalid_llc_nr(mm, p, dst_cpu) ||
|
||||
exceed_llc_capacity(mm, dst_cpu)) {
|
||||
if (READ_ONCE(mm->sc_stat.cpu) != -1)
|
||||
WRITE_ONCE(mm->sc_stat.cpu, -1);
|
||||
if (invalid_llc_nr(grp, p, dst_cpu) ||
|
||||
exceed_llc_capacity(grp, dst_cpu)) {
|
||||
if (READ_ONCE(grp->cpu) != -1)
|
||||
WRITE_ONCE(grp->cpu, -1);
|
||||
return mig_unrestricted;
|
||||
}
|
||||
|
||||
|
|
@ -10813,6 +11030,21 @@ alb_break_llc(struct lb_env *env)
|
|||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns true if p's preferred LLC does not match the destination CPU
|
||||
* under migrate_llc_task semantics. Passive LB passes migrate_llc_task
|
||||
* in env->migration_type, while active LB carries LBF_ACTIVE_LB_LLC in
|
||||
* env->flags to avoid overwriting env->migration_type.
|
||||
*/
|
||||
static inline bool
|
||||
migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env)
|
||||
{
|
||||
return sched_cache_enabled() &&
|
||||
(env->migration_type == migrate_llc_task ||
|
||||
env->flags & LBF_ACTIVE_LB_LLC) &&
|
||||
READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu);
|
||||
}
|
||||
|
||||
/*
|
||||
* Check if migrating task p from env->src_cpu to
|
||||
* env->dst_cpu breaks LLC localiy.
|
||||
|
|
@ -10841,8 +11073,7 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
|
|||
* run on env->dst_cpu, skip the tasks do not prefer
|
||||
* env->dst_cpu, and find the one that prefers.
|
||||
*/
|
||||
if (env->migration_type == migrate_llc_task &&
|
||||
READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu))
|
||||
if (migrate_llc_task_wrong_dst(p, env))
|
||||
return true;
|
||||
|
||||
if (can_migrate_llc_task(env, p) != mig_forbid)
|
||||
|
|
@ -10864,6 +11095,12 @@ alb_break_llc(struct lb_env *env)
|
|||
return false;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
|
||||
{
|
||||
|
|
@ -10963,7 +11200,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
|
|||
* 4) too many balance attempts have failed.
|
||||
*/
|
||||
if (env->flags & LBF_ACTIVE_LB)
|
||||
return 1;
|
||||
return !migrate_llc_task_wrong_dst(p, env);
|
||||
|
||||
degrades = migrate_degrades_locality(p, env);
|
||||
if (!degrades) {
|
||||
|
|
@ -13362,6 +13599,20 @@ static int need_active_balance(struct lb_env *env)
|
|||
}
|
||||
|
||||
static int active_load_balance_cpu_stop(void *data);
|
||||
static int active_load_balance_llc_cpu_stop(void *data);
|
||||
|
||||
/*
|
||||
* migration_type is checked elsewhere to decide migration policy, so
|
||||
* it shouldn't be repurposed just to flag an LLC-directed active
|
||||
* balance across the stopper. Pick the callback here instead.
|
||||
*/
|
||||
static inline cpu_stop_fn_t alb_stop_fn(struct lb_env *env)
|
||||
{
|
||||
if (env->migration_type == migrate_llc_task)
|
||||
return active_load_balance_llc_cpu_stop;
|
||||
|
||||
return active_load_balance_cpu_stop;
|
||||
}
|
||||
|
||||
static int should_we_balance(struct lb_env *env)
|
||||
{
|
||||
|
|
@ -13707,7 +13958,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
|
|||
}
|
||||
if (active_balance) {
|
||||
stop_one_cpu_nowait(cpu_of(busiest),
|
||||
active_load_balance_cpu_stop, busiest,
|
||||
alb_stop_fn(&env), busiest,
|
||||
&busiest->active_balance_work);
|
||||
}
|
||||
preempt_enable();
|
||||
|
|
@ -13812,7 +14063,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
|
|||
* least 1 task to be running on each physical CPU where possible, and
|
||||
* avoids physical / logical imbalances.
|
||||
*/
|
||||
static int active_load_balance_cpu_stop(void *data)
|
||||
static int __active_load_balance_cpu_stop(void *data, unsigned int lb_flags)
|
||||
{
|
||||
struct rq *busiest_rq = data;
|
||||
int busiest_cpu = cpu_of(busiest_rq);
|
||||
|
|
@ -13862,7 +14113,7 @@ static int active_load_balance_cpu_stop(void *data)
|
|||
.src_cpu = busiest_rq->cpu,
|
||||
.src_rq = busiest_rq,
|
||||
.idle = CPU_IDLE,
|
||||
.flags = LBF_ACTIVE_LB,
|
||||
.flags = LBF_ACTIVE_LB | lb_flags,
|
||||
};
|
||||
|
||||
schedstat_inc(sd->alb_count);
|
||||
|
|
@ -13890,6 +14141,16 @@ static int active_load_balance_cpu_stop(void *data)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int active_load_balance_cpu_stop(void *data)
|
||||
{
|
||||
return __active_load_balance_cpu_stop(data, 0);
|
||||
}
|
||||
|
||||
static int active_load_balance_llc_cpu_stop(void *data)
|
||||
{
|
||||
return __active_load_balance_cpu_stop(data, LBF_ACTIVE_LB_LLC);
|
||||
}
|
||||
|
||||
/*
|
||||
* Scale the max sched_balance_rq interval with the number of CPUs in the system.
|
||||
* This trades load-balance latency on larger machines for less cross talk.
|
||||
|
|
|
|||
|
|
@ -985,8 +985,8 @@ void sched_cache_active_set(void)
|
|||
}
|
||||
|
||||
/*
|
||||
* Update the bottom sched_domain's llc_bytes for @cpu and all its
|
||||
* LLC siblings. Called from cacheinfo_cpu_online() or
|
||||
* Update the bottom sched_domain's llc_bytes for @cpus sharing a physical
|
||||
* LLC. Called from cacheinfo_cpu_online() or
|
||||
* cacheinfo_cpu_pre_down() with cpu hotplug lock held.
|
||||
*
|
||||
* Note: get_effective_llc_bytes() returns 0 on PowerPC.
|
||||
|
|
@ -996,17 +996,13 @@ void sched_cache_active_set(void)
|
|||
* and does not populates the per-CPU struct cpu_cacheinfo array
|
||||
* that get_cpu_cacheinfo_llc() reads.
|
||||
*/
|
||||
void sched_update_llc_bytes(unsigned int cpu)
|
||||
void sched_update_llc_bytes(const struct cpumask *cpus)
|
||||
{
|
||||
struct sched_domain *sd, *sdp;
|
||||
unsigned int i;
|
||||
|
||||
sched_domains_mutex_lock();
|
||||
|
||||
sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu));
|
||||
if (!sdp)
|
||||
goto unlock;
|
||||
|
||||
/*
|
||||
* ci->shared_cpu_map is built incrementally as CPUs come
|
||||
* online, so the first CPU in an LLC initially sees
|
||||
|
|
@ -1014,14 +1010,22 @@ void sched_update_llc_bytes(unsigned int cpu)
|
|||
* get_effective_llc_bytes(). Re-evaluating every LLC
|
||||
* sibling on each online event corrects this once the full
|
||||
* shared_cpu_map is known.
|
||||
*
|
||||
* The departing CPU's domains have already been detached when
|
||||
* cacheinfo removes it. Use the surviving cache siblings instead.
|
||||
* They may belong to different cpuset partitions, so use each CPU's
|
||||
* own LLC domain to scale its share of the physical cache.
|
||||
*/
|
||||
for_each_cpu(i, sched_domain_span(sdp)) {
|
||||
for_each_cpu(i, cpus) {
|
||||
sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, i));
|
||||
if (!sdp)
|
||||
continue;
|
||||
|
||||
sd = rcu_dereference_sched_domain(cpu_rq(i)->sd);
|
||||
if (sd)
|
||||
sd->llc_bytes = get_effective_llc_bytes(i, sdp);
|
||||
}
|
||||
|
||||
unlock:
|
||||
sched_domains_mutex_unlock();
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user