mirror of
https://github.com/torvalds/linux.git
synced 2026-07-27 09:36:22 +02:00
Merge branch 'sched/cache'
Merge the cache aware balancer topic branch. # Conflicts: # kernel/sched/topology.c
This commit is contained in:
Peter Zijlstra
commit
a26d9208c1
|
|
@ -7236,6 +7236,18 @@ Kernel parameters
|
|||
Not specifying this option is equivalent to
|
||||
spec_store_bypass_disable=auto.
|
||||
|
||||
split_llc=
|
||||
[X86,EARLY] Split the LLC N-ways
|
||||
|
||||
When set, the LLC is split this many ways by matching
|
||||
'core_id % n'. This is setup before SMP bringup and
|
||||
used during SMP bringup before it knows the full
|
||||
topology. If your core count doesn't nicely divide by
|
||||
the number given, you get to keep the pieces.
|
||||
|
||||
This is mostly a debug feature to emulate multiple LLCs
|
||||
on hardware that only have a single LLC.
|
||||
|
||||
split_lock_detect=
|
||||
[X86] Enable split lock detection or bus lock detection
|
||||
|
||||
|
|
|
|||
|
|
@ -704,6 +704,11 @@ static inline u32 per_cpu_l2c_id(unsigned int cpu)
|
|||
return per_cpu(cpu_info.topo.l2c_id, cpu);
|
||||
}
|
||||
|
||||
static inline u32 per_cpu_core_id(unsigned int cpu)
|
||||
{
|
||||
return per_cpu(cpu_info.topo.core_id, cpu);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CPU_SUP_AMD
|
||||
/*
|
||||
* Issue a DIV 0/1 insn to clear any division data from previous DIV
|
||||
|
|
|
|||
|
|
@ -424,6 +424,21 @@ static const struct x86_cpu_id intel_cod_cpu[] = {
|
|||
{}
|
||||
};
|
||||
|
||||
/*
|
||||
* Allows splitting the LLC by matching 'core_id % split_llc'.
|
||||
*
|
||||
* This is mostly a debug hack to emulate systems with multiple LLCs per node
|
||||
* on systems that do not naturally have this.
|
||||
*/
|
||||
static unsigned int split_llc = 0;
|
||||
|
||||
static int __init split_llc_setup(char *str)
|
||||
{
|
||||
get_option(&str, &split_llc);
|
||||
return 0;
|
||||
}
|
||||
early_param("split_llc", split_llc_setup);
|
||||
|
||||
static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
|
||||
{
|
||||
const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu);
|
||||
|
|
@ -438,6 +453,11 @@ static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
|
|||
if (per_cpu_llc_id(cpu1) != per_cpu_llc_id(cpu2))
|
||||
return false;
|
||||
|
||||
if (split_llc &&
|
||||
(per_cpu_core_id(cpu1) % split_llc) !=
|
||||
(per_cpu_core_id(cpu2) % split_llc))
|
||||
return false;
|
||||
|
||||
/*
|
||||
* Allow the SNC topology without warning. Return of false
|
||||
* means 'c' does not share the LLC of 'o'. This will be
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include <linux/init.h>
|
||||
#include <linux/of.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/sched/topology.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/smp.h>
|
||||
#include <linux/sysfs.h>
|
||||
|
|
@ -68,6 +69,24 @@ bool last_level_cache_is_valid(unsigned int cpu)
|
|||
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the cacheinfo of the LLC associated with @cpu.
|
||||
* Derived from update_per_cpu_data_slice_size_cpu().
|
||||
*/
|
||||
struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu)
|
||||
{
|
||||
struct cacheinfo *llc;
|
||||
|
||||
if (!last_level_cache_is_valid(cpu))
|
||||
return NULL;
|
||||
|
||||
llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
|
||||
if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
|
||||
return NULL;
|
||||
|
||||
return llc;
|
||||
}
|
||||
|
||||
bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y)
|
||||
{
|
||||
struct cacheinfo *llc_x, *llc_y;
|
||||
|
|
@ -1018,6 +1037,7 @@ static int cacheinfo_cpu_online(unsigned int cpu)
|
|||
goto err;
|
||||
if (cpu_map_shared_cache(true, cpu, &cpu_map))
|
||||
update_per_cpu_data_slice_size(true, cpu, cpu_map);
|
||||
sched_update_llc_bytes(cpu);
|
||||
return 0;
|
||||
err:
|
||||
free_cache_attributes(cpu);
|
||||
|
|
@ -1036,6 +1056,9 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu)
|
|||
free_cache_attributes(cpu);
|
||||
if (nr_shared > 1)
|
||||
update_per_cpu_data_slice_size(false, cpu, cpu_map);
|
||||
|
||||
sched_update_llc_bytes(cpu);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ int populate_cache_leaves(unsigned int cpu);
|
|||
int cache_setup_acpi(unsigned int cpu);
|
||||
bool last_level_cache_is_valid(unsigned int cpu);
|
||||
bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y);
|
||||
struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu);
|
||||
int fetch_cache_info(unsigned int cpu);
|
||||
int detect_cache_attributes(unsigned int cpu);
|
||||
#ifndef CONFIG_ACPI_PPTT
|
||||
|
|
|
|||
|
|
@ -1222,6 +1222,8 @@ struct mm_struct {
|
|||
/* MM CID related storage */
|
||||
struct mm_mm_cid mm_cid;
|
||||
|
||||
/* sched_cache related statistics */
|
||||
struct sched_cache_stat sc_stat;
|
||||
#ifdef CONFIG_MMU
|
||||
atomic_long_t pgtables_bytes; /* size of all page tables */
|
||||
#endif
|
||||
|
|
@ -1628,6 +1630,36 @@ static inline unsigned int mm_cid_size(void)
|
|||
# define MM_CID_STATIC_SIZE 0
|
||||
#endif /* CONFIG_SCHED_MM_CID */
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
void mm_init_sched(struct mm_struct *mm,
|
||||
struct sched_cache_time __percpu *pcpu_sched);
|
||||
|
||||
static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
|
||||
{
|
||||
struct sched_cache_time __percpu *pcpu_sched =
|
||||
alloc_percpu_noprof(struct sched_cache_time);
|
||||
|
||||
if (!pcpu_sched)
|
||||
return -ENOMEM;
|
||||
|
||||
mm_init_sched(mm, pcpu_sched);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__))
|
||||
|
||||
static inline void mm_destroy_sched(struct mm_struct *mm)
|
||||
{
|
||||
free_percpu(mm->sc_stat.pcpu_sched);
|
||||
mm->sc_stat.pcpu_sched = NULL;
|
||||
}
|
||||
#else /* !CONFIG_SCHED_CACHE */
|
||||
|
||||
static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; }
|
||||
static inline void mm_destroy_sched(struct mm_struct *mm) { }
|
||||
|
||||
#endif /* CONFIG_SCHED_CACHE */
|
||||
|
||||
struct mmu_gather;
|
||||
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
|
||||
extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);
|
||||
|
|
|
|||
|
|
@ -1408,6 +1408,13 @@ struct task_struct {
|
|||
unsigned long numa_pages_migrated;
|
||||
#endif /* CONFIG_NUMA_BALANCING */
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
struct callback_head cache_work;
|
||||
int preferred_llc;
|
||||
/* 1: task was enqueued to its preferred LLC, 0 otherwise */
|
||||
int pref_llc_queued;
|
||||
#endif
|
||||
|
||||
struct rseq_data rseq;
|
||||
struct sched_mm_cid mm_cid;
|
||||
|
||||
|
|
@ -2408,6 +2415,29 @@ static __always_inline int task_mm_cid(struct task_struct *t)
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
|
||||
struct sched_cache_time {
|
||||
u64 runtime;
|
||||
unsigned long epoch;
|
||||
};
|
||||
|
||||
struct sched_cache_stat {
|
||||
struct sched_cache_time __percpu *pcpu_sched;
|
||||
raw_spinlock_t lock;
|
||||
unsigned long epoch;
|
||||
u64 nr_running_avg;
|
||||
unsigned long next_scan;
|
||||
unsigned long footprint;
|
||||
int cpu;
|
||||
} ____cacheline_aligned_in_smp;
|
||||
|
||||
#else
|
||||
|
||||
struct sched_cache_stat { };
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef MODULE
|
||||
#ifndef COMPILE_OFFSETS
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,10 @@ struct sched_domain_shared {
|
|||
atomic_t nr_busy_cpus;
|
||||
int has_idle_cores;
|
||||
int nr_idle_scan;
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
unsigned long util_avg;
|
||||
unsigned long capacity;
|
||||
#endif
|
||||
};
|
||||
|
||||
struct sched_domain {
|
||||
|
|
@ -99,6 +103,12 @@ struct sched_domain {
|
|||
u64 max_newidle_lb_cost;
|
||||
unsigned long last_decay_max_lb_cost;
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
unsigned int llc_max;
|
||||
unsigned int *llc_counts __counted_by_ptr(llc_max);
|
||||
unsigned long llc_bytes;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SCHEDSTATS
|
||||
/* sched_balance_rq() stats */
|
||||
unsigned int lb_count[CPU_MAX_IDLE_TYPES];
|
||||
|
|
@ -256,4 +266,10 @@ static inline int task_node(const struct task_struct *p)
|
|||
return cpu_to_node(task_cpu(p));
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
extern void sched_update_llc_bytes(unsigned int cpu);
|
||||
#else
|
||||
static inline void sched_update_llc_bytes(unsigned int cpu) { }
|
||||
#endif
|
||||
|
||||
#endif /* _LINUX_SCHED_TOPOLOGY_H */
|
||||
|
|
|
|||
11
init/Kconfig
11
init/Kconfig
|
|
@ -1019,6 +1019,17 @@ config NUMA_BALANCING
|
|||
|
||||
This system will be inactive on UMA systems.
|
||||
|
||||
config SCHED_CACHE
|
||||
bool "Cache aware load balance"
|
||||
default y
|
||||
depends on SMP
|
||||
help
|
||||
When enabled, the scheduler will attempt to aggregate tasks from
|
||||
the same process onto a single Last Level Cache (LLC) domain when
|
||||
possible. This improves cache locality by keeping tasks that share
|
||||
resources within the same cache domain, reducing cache misses and
|
||||
lowering data access latency.
|
||||
|
||||
config NUMA_BALANCING_DEFAULT_ENABLED
|
||||
bool "Automatically enable NUMA aware memory/task placement"
|
||||
default y
|
||||
|
|
|
|||
|
|
@ -215,6 +215,10 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
|
|||
.numa_group = NULL,
|
||||
.numa_faults = NULL,
|
||||
#endif
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
.preferred_llc = -1,
|
||||
.pref_llc_queued = 0,
|
||||
#endif
|
||||
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
|
||||
.kasan_depth = 1,
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -543,6 +543,32 @@ void mm_update_next_owner(struct mm_struct *mm)
|
|||
}
|
||||
#endif /* CONFIG_MEMCG */
|
||||
|
||||
#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING)
|
||||
/*
|
||||
* Subtract the memory footprint of the current task from
|
||||
* mm.
|
||||
*/
|
||||
static void exit_mm_sched_cache(struct mm_struct *mm)
|
||||
{
|
||||
unsigned long fp, sub;
|
||||
|
||||
if (!current->total_numa_faults)
|
||||
return;
|
||||
/*
|
||||
* No lock protection due to performance considerations.
|
||||
* Make sure mm->sc_stat.footprint does not become
|
||||
* negative.
|
||||
*/
|
||||
fp = READ_ONCE(mm->sc_stat.footprint);
|
||||
sub = min(fp, current->total_numa_faults);
|
||||
WRITE_ONCE(mm->sc_stat.footprint, fp - sub);
|
||||
}
|
||||
#else
|
||||
static inline void exit_mm_sched_cache(struct mm_struct *mm)
|
||||
{
|
||||
}
|
||||
#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */
|
||||
|
||||
/*
|
||||
* Turn us into a lazy TLB process if we
|
||||
* aren't already..
|
||||
|
|
@ -554,6 +580,9 @@ static void exit_mm(void)
|
|||
exit_mm_release(current, mm);
|
||||
if (!mm)
|
||||
return;
|
||||
|
||||
exit_mm_sched_cache(mm);
|
||||
|
||||
mmap_read_lock(mm);
|
||||
mmgrab_lazy_tlb(mm);
|
||||
BUG_ON(mm != current->active_mm);
|
||||
|
|
|
|||
|
|
@ -726,6 +726,7 @@ void __mmdrop(struct mm_struct *mm)
|
|||
cleanup_lazy_tlbs(mm);
|
||||
|
||||
WARN_ON_ONCE(mm == current->active_mm);
|
||||
mm_destroy_sched(mm);
|
||||
mm_free_pgd(mm);
|
||||
mm_free_id(mm);
|
||||
destroy_context(mm);
|
||||
|
|
@ -1128,6 +1129,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
|
|||
if (mm_alloc_cid(mm, p))
|
||||
goto fail_cid;
|
||||
|
||||
if (mm_alloc_sched(mm))
|
||||
goto fail_sched;
|
||||
|
||||
if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
|
||||
NR_MM_COUNTERS))
|
||||
goto fail_pcpu;
|
||||
|
|
@ -1137,6 +1141,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
|
|||
return mm;
|
||||
|
||||
fail_pcpu:
|
||||
mm_destroy_sched(mm);
|
||||
fail_sched:
|
||||
mm_destroy_cid(mm);
|
||||
fail_cid:
|
||||
destroy_context(mm);
|
||||
|
|
|
|||
|
|
@ -548,6 +548,11 @@ void __trace_set_current_state(int state_value)
|
|||
}
|
||||
EXPORT_SYMBOL(__trace_set_current_state);
|
||||
|
||||
int task_llc(const struct task_struct *p)
|
||||
{
|
||||
return per_cpu(sd_llc_id, task_cpu(p));
|
||||
}
|
||||
|
||||
/*
|
||||
* Serialization rules:
|
||||
*
|
||||
|
|
@ -4506,6 +4511,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
|
|||
init_numa_balancing(clone_flags, p);
|
||||
p->wake_entry.u_flags = CSD_TYPE_TTWU;
|
||||
p->migration_pending = NULL;
|
||||
init_sched_mm(p);
|
||||
}
|
||||
|
||||
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
|
||||
|
|
@ -8699,6 +8705,8 @@ int sched_cpu_deactivate(unsigned int cpu)
|
|||
*/
|
||||
synchronize_rcu();
|
||||
|
||||
sched_domains_free_llc_id(cpu);
|
||||
|
||||
sched_set_rq_offline(rq, cpu);
|
||||
|
||||
scx_rq_deactivate(rq);
|
||||
|
|
@ -9030,6 +9038,11 @@ void __init sched_init(void)
|
|||
|
||||
rq->core_cookie = 0UL;
|
||||
#endif
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
raw_spin_lock_init(&rq->cpu_epoch_lock);
|
||||
rq->cpu_epoch_next = jiffies;
|
||||
#endif
|
||||
|
||||
zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -210,6 +210,48 @@ static const struct file_operations sched_scaling_fops = {
|
|||
.release = single_release,
|
||||
};
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
static ssize_t
|
||||
sched_cache_enable_write(struct file *filp, const char __user *ubuf,
|
||||
size_t cnt, loff_t *ppos)
|
||||
{
|
||||
bool val;
|
||||
int ret;
|
||||
|
||||
ret = kstrtobool_from_user(ubuf, cnt, &val);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
sysctl_sched_cache_user = val;
|
||||
|
||||
sched_cache_active_set();
|
||||
|
||||
*ppos += cnt;
|
||||
|
||||
return cnt;
|
||||
}
|
||||
|
||||
static int sched_cache_enable_show(struct seq_file *m, void *v)
|
||||
{
|
||||
seq_printf(m, "%d\n", sysctl_sched_cache_user);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sched_cache_enable_open(struct inode *inode,
|
||||
struct file *filp)
|
||||
{
|
||||
return single_open(filp, sched_cache_enable_show, NULL);
|
||||
}
|
||||
|
||||
static const struct file_operations sched_cache_enable_fops = {
|
||||
.open = sched_cache_enable_open,
|
||||
.write = sched_cache_enable_write,
|
||||
.read = seq_read,
|
||||
.llseek = seq_lseek,
|
||||
.release = single_release,
|
||||
};
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_PREEMPT_DYNAMIC
|
||||
|
||||
static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
|
||||
|
|
@ -593,7 +635,7 @@ static void debugfs_ext_server_init(void)
|
|||
|
||||
static __init int sched_init_debug(void)
|
||||
{
|
||||
struct dentry __maybe_unused *numa;
|
||||
struct dentry __maybe_unused *numa, *llc;
|
||||
|
||||
debugfs_sched = debugfs_create_dir("sched", NULL);
|
||||
|
||||
|
|
@ -626,6 +668,22 @@ static __init int sched_init_debug(void)
|
|||
debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
|
||||
#endif /* CONFIG_NUMA_BALANCING */
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
llc = debugfs_create_dir("llc_balancing", debugfs_sched);
|
||||
debugfs_create_file("enabled", 0644, llc, NULL,
|
||||
&sched_cache_enable_fops);
|
||||
debugfs_create_u32("aggr_tolerance", 0644, llc,
|
||||
&llc_aggr_tolerance);
|
||||
debugfs_create_u32("epoch_period", 0644, llc,
|
||||
&llc_epoch_period);
|
||||
debugfs_create_u32("epoch_affinity_timeout", 0644, llc,
|
||||
&llc_epoch_affinity_timeout);
|
||||
debugfs_create_u32("overaggr_pct", 0644, llc,
|
||||
&llc_overaggr_pct);
|
||||
debugfs_create_u32("imb_pct", 0644, llc,
|
||||
&llc_imb_pct);
|
||||
#endif
|
||||
|
||||
debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
|
||||
|
||||
debugfs_fair_server_init();
|
||||
|
|
|
|||
1145
kernel/sched/fair.c
1145
kernel/sched/fair.c
File diff suppressed because it is too large
Load Diff
|
|
@ -1187,6 +1187,12 @@ struct rq {
|
|||
struct scx_rq scx;
|
||||
struct sched_dl_entity ext_server;
|
||||
#endif
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
raw_spinlock_t cpu_epoch_lock ____cacheline_aligned;
|
||||
u64 cpu_runtime;
|
||||
unsigned long cpu_epoch;
|
||||
unsigned long cpu_epoch_next;
|
||||
#endif
|
||||
|
||||
struct sched_dl_entity fair_server;
|
||||
|
||||
|
|
@ -1199,6 +1205,12 @@ struct rq {
|
|||
#ifdef CONFIG_NUMA_BALANCING
|
||||
unsigned int numa_migrate_on;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
unsigned int nr_pref_llc_running;
|
||||
unsigned int nr_llc_running;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* This is part of a global counter where only the total sum
|
||||
* over all CPUs matters. A task can increase this counter on
|
||||
|
|
@ -1546,6 +1558,14 @@ extern void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags);
|
|||
extern void sched_core_get(void);
|
||||
extern void sched_core_put(void);
|
||||
|
||||
static inline bool task_has_sched_core(struct task_struct *p)
|
||||
{
|
||||
if (sched_core_disabled())
|
||||
return false;
|
||||
|
||||
return !!p->core_cookie;
|
||||
}
|
||||
|
||||
#else /* !CONFIG_SCHED_CORE: */
|
||||
|
||||
static inline bool sched_core_enabled(struct rq *rq)
|
||||
|
|
@ -1586,6 +1606,11 @@ static inline bool sched_group_cookie_match(struct rq *rq,
|
|||
return true;
|
||||
}
|
||||
|
||||
static inline bool task_has_sched_core(struct task_struct *p)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* !CONFIG_SCHED_CORE */
|
||||
|
||||
#ifdef CONFIG_RT_GROUP_SCHED
|
||||
|
|
@ -2076,6 +2101,8 @@ init_numa_balancing(u64 clone_flags, struct task_struct *p)
|
|||
|
||||
#endif /* !CONFIG_NUMA_BALANCING */
|
||||
|
||||
int task_llc(const struct task_struct *p);
|
||||
|
||||
static inline void
|
||||
queue_balance_callback(struct rq *rq,
|
||||
struct balance_callback *head,
|
||||
|
|
@ -2164,6 +2191,7 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);
|
|||
DECLARE_PER_CPU(int, sd_llc_size);
|
||||
DECLARE_PER_CPU(int, sd_llc_id);
|
||||
DECLARE_PER_CPU(int, sd_share_id);
|
||||
DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
|
||||
DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
|
||||
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
|
||||
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
|
||||
|
|
@ -4031,6 +4059,29 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
|
|||
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
|
||||
#endif /* !CONFIG_SCHED_MM_CID */
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
DECLARE_STATIC_KEY_FALSE(sched_cache_present);
|
||||
DECLARE_STATIC_KEY_FALSE(sched_cache_active);
|
||||
extern int sysctl_sched_cache_user;
|
||||
extern unsigned int llc_aggr_tolerance;
|
||||
extern unsigned int llc_epoch_period;
|
||||
extern unsigned int llc_epoch_affinity_timeout;
|
||||
extern unsigned int llc_imb_pct;
|
||||
extern unsigned int llc_overaggr_pct;
|
||||
|
||||
static inline bool sched_cache_enabled(void)
|
||||
{
|
||||
return static_branch_unlikely(&sched_cache_active);
|
||||
}
|
||||
|
||||
extern void sched_cache_active_set(void);
|
||||
|
||||
#endif
|
||||
|
||||
void sched_domains_free_llc_id(int cpu);
|
||||
|
||||
extern void init_sched_mm(struct task_struct *p);
|
||||
|
||||
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
|
||||
extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se);
|
||||
static inline
|
||||
|
|
|
|||
|
|
@ -19,8 +19,10 @@ void sched_domains_mutex_unlock(void)
|
|||
}
|
||||
|
||||
/* Protected by sched_domains_mutex: */
|
||||
static cpumask_var_t sched_domains_llc_id_allocmask;
|
||||
static cpumask_var_t sched_domains_tmpmask;
|
||||
static cpumask_var_t sched_domains_tmpmask2;
|
||||
int max_lid;
|
||||
|
||||
static int __init sched_debug_setup(char *str)
|
||||
{
|
||||
|
|
@ -632,6 +634,11 @@ static void destroy_sched_domain(struct sched_domain *sd)
|
|||
|
||||
if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
|
||||
kfree(sd->shared);
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
/* only the bottom sd has llc_counts array */
|
||||
kfree(sd->llc_counts);
|
||||
#endif
|
||||
kfree(sd);
|
||||
}
|
||||
|
||||
|
|
@ -663,8 +670,9 @@ static void destroy_sched_domains(struct sched_domain *sd)
|
|||
*/
|
||||
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
|
||||
DEFINE_PER_CPU(int, sd_llc_size);
|
||||
DEFINE_PER_CPU(int, sd_llc_id);
|
||||
DEFINE_PER_CPU(int, sd_llc_id) = -1;
|
||||
DEFINE_PER_CPU(int, sd_share_id);
|
||||
DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
|
||||
DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
|
||||
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
|
||||
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
|
||||
|
|
@ -680,38 +688,19 @@ static void update_top_cache_domain(int cpu)
|
|||
int id = cpu;
|
||||
int size = 1;
|
||||
|
||||
sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL);
|
||||
/*
|
||||
* The shared object is attached to sd_asym_cpucapacity only when the
|
||||
* asym domain is non-overlapping (i.e., not built from SD_NUMA).
|
||||
* On overlapping (NUMA) asym domains we fall back to letting the
|
||||
* SD_SHARE_LLC path own the shared object, so sd->shared may be NULL
|
||||
* here.
|
||||
*/
|
||||
if (sd && sd->shared)
|
||||
sds = sd->shared;
|
||||
|
||||
rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
|
||||
|
||||
sd = highest_flag_domain(cpu, SD_SHARE_LLC);
|
||||
if (sd) {
|
||||
id = cpumask_first(sched_domain_span(sd));
|
||||
size = cpumask_weight(sched_domain_span(sd));
|
||||
|
||||
/*
|
||||
* If sd_asym_cpucapacity didn't claim the shared object,
|
||||
* sd_llc must have one linked.
|
||||
*/
|
||||
if (!sds) {
|
||||
WARN_ON_ONCE(!sd->shared);
|
||||
sds = sd->shared;
|
||||
}
|
||||
/* If sd_llc exists, sd_llc_shared should exist too. */
|
||||
WARN_ON_ONCE(!sd->shared);
|
||||
sds = sd->shared;
|
||||
}
|
||||
|
||||
rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
|
||||
per_cpu(sd_llc_size, cpu) = size;
|
||||
per_cpu(sd_llc_id, cpu) = id;
|
||||
rcu_assign_pointer(per_cpu(sd_balance_shared, cpu), sds);
|
||||
rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
|
||||
|
||||
sd = lowest_flag_domain(cpu, SD_CLUSTER);
|
||||
if (sd)
|
||||
|
|
@ -729,6 +718,20 @@ static void update_top_cache_domain(int cpu)
|
|||
|
||||
sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
|
||||
rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd);
|
||||
|
||||
sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL);
|
||||
/*
|
||||
* The shared object is attached to sd_asym_cpucapacity only when the
|
||||
* asym domain is non-overlapping (i.e., not built from SD_NUMA).
|
||||
* On overlapping (NUMA) asym domains we fall back to letting the
|
||||
* SD_SHARE_LLC path own the shared object, so sd->shared may be NULL
|
||||
* here.
|
||||
*/
|
||||
if (sd && sd->shared)
|
||||
sds = sd->shared;
|
||||
|
||||
rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
|
||||
rcu_assign_pointer(per_cpu(sd_balance_shared, cpu), sds);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -777,10 +780,20 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
|
|||
if (sd && sd_degenerate(sd)) {
|
||||
tmp = sd;
|
||||
sd = sd->parent;
|
||||
destroy_sched_domain(tmp);
|
||||
|
||||
if (sd) {
|
||||
struct sched_group *sg = sd->groups;
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
/* move buffer to parent as child is being destroyed */
|
||||
sd->llc_counts = tmp->llc_counts;
|
||||
sd->llc_max = tmp->llc_max;
|
||||
sd->llc_bytes = tmp->llc_bytes;
|
||||
/* make sure destroy_sched_domain() does not free it */
|
||||
tmp->llc_counts = NULL;
|
||||
tmp->llc_max = 0;
|
||||
tmp->llc_bytes = 0;
|
||||
#endif
|
||||
/*
|
||||
* sched groups hold the flags of the child sched
|
||||
* domain for convenience. Clear such flags since
|
||||
|
|
@ -792,6 +805,8 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
|
|||
|
||||
sd->child = NULL;
|
||||
}
|
||||
|
||||
destroy_sched_domain(tmp);
|
||||
}
|
||||
|
||||
sched_domain_debug(sd, cpu);
|
||||
|
|
@ -819,6 +834,239 @@ enum s_alloc {
|
|||
sa_none,
|
||||
};
|
||||
|
||||
#ifdef CONFIG_SCHED_CACHE
|
||||
/* hardware support for cache aware scheduling */
|
||||
DEFINE_STATIC_KEY_FALSE(sched_cache_present);
|
||||
/*
|
||||
* Indicator of whether cache aware scheduling
|
||||
* is active, used by the scheduler.
|
||||
*/
|
||||
DEFINE_STATIC_KEY_FALSE(sched_cache_active);
|
||||
/* user wants cache aware scheduling [0 or 1] */
|
||||
int sysctl_sched_cache_user = 1;
|
||||
|
||||
/*
|
||||
* Get the effective LLC size in bytes that @cpu's bottom sched_domain
|
||||
* can use. A CPU within a cpuset partition can only use a proportion
|
||||
* of the physical LLC, scaled by the ratio of the partition's span
|
||||
* weight to the hardware LLC sharing weight. @sd should be the
|
||||
* topmost domain with SD_SHARE_LLC.
|
||||
*
|
||||
* Returns 0 if cacheinfo is not yet populated. This happens during
|
||||
* early boot when build_sched_domains() runs before the generic
|
||||
* cacheinfo framework has been initialized (cacheinfo_cpu_online()
|
||||
* is a device_initcall cpuhp callback). In that case,
|
||||
* cacheinfo_cpu_online() will later call sched_update_llc_bytes()
|
||||
* to fill in the bottom domain's llc_bytes once the cache attributes
|
||||
* are available.
|
||||
*/
|
||||
static unsigned long get_effective_llc_bytes(int cpu,
|
||||
struct sched_domain *sd)
|
||||
{
|
||||
struct cacheinfo *ci;
|
||||
unsigned int hw_weight;
|
||||
|
||||
ci = get_cpu_cacheinfo_llc(cpu);
|
||||
if (!ci)
|
||||
return 0;
|
||||
|
||||
hw_weight = cpumask_weight(&ci->shared_cpu_map);
|
||||
if (!hw_weight)
|
||||
return 0;
|
||||
|
||||
return div_u64((u64)ci->size * sd->span_weight, hw_weight);
|
||||
}
|
||||
|
||||
static bool alloc_sd_llc(const struct cpumask *cpu_map,
|
||||
struct s_data *d)
|
||||
{
|
||||
struct sched_domain *sd, *top_llc, *parent;
|
||||
unsigned int *p;
|
||||
int i;
|
||||
|
||||
for_each_cpu(i, cpu_map) {
|
||||
sd = *per_cpu_ptr(d->sd, i);
|
||||
if (!sd)
|
||||
goto err;
|
||||
|
||||
p = kcalloc_node(max_lid + 1, sizeof(unsigned int),
|
||||
GFP_KERNEL, cpu_to_node(i));
|
||||
if (!p)
|
||||
goto err;
|
||||
|
||||
top_llc = sd;
|
||||
/*
|
||||
* Find the topmost SD_SHARE_LLC domain.
|
||||
* Not yet attached to the CPU, so per_cpu(sd_llc, i)
|
||||
* can not be used.
|
||||
*/
|
||||
while ((parent = rcu_dereference_protected(top_llc->parent, true)) &&
|
||||
(parent->flags & SD_SHARE_LLC))
|
||||
top_llc = parent;
|
||||
|
||||
if (top_llc->flags & SD_SHARE_LLC) {
|
||||
sd->llc_max = max_lid + 1;
|
||||
sd->llc_counts = p;
|
||||
sd->llc_bytes = get_effective_llc_bytes(i, top_llc);
|
||||
} else {
|
||||
/* avoid memory leak */
|
||||
kfree(p);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
err:
|
||||
for_each_cpu(i, cpu_map) {
|
||||
sd = *per_cpu_ptr(d->sd, i);
|
||||
if (sd) {
|
||||
kfree(sd->llc_counts);
|
||||
sd->llc_counts = NULL;
|
||||
sd->llc_max = 0;
|
||||
sd->llc_bytes = 0;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Enable/disable cache aware scheduling according to
|
||||
* user input and the presence of hardware support.
|
||||
*/
|
||||
static void _sched_cache_active_set(void)
|
||||
{
|
||||
lockdep_assert_cpus_held();
|
||||
lockdep_assert_held(&sched_domains_mutex);
|
||||
|
||||
/* hardware does not support */
|
||||
if (!static_branch_likely(&sched_cache_present)) {
|
||||
static_branch_disable_cpuslocked(&sched_cache_active);
|
||||
if (sched_debug())
|
||||
pr_info("%s: cache aware scheduling not supported on this platform\n", __func__);
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* user wants it or not ?
|
||||
* TBD: read before writing the static key.
|
||||
* It is not in the critical path, leave as-is
|
||||
* for now.
|
||||
*/
|
||||
if (sysctl_sched_cache_user) {
|
||||
static_branch_enable_cpuslocked(&sched_cache_active);
|
||||
if (sched_debug())
|
||||
pr_info("%s: enabling cache aware scheduling\n", __func__);
|
||||
} else {
|
||||
static_branch_disable_cpuslocked(&sched_cache_active);
|
||||
if (sched_debug())
|
||||
pr_info("%s: disabling cache aware scheduling\n", __func__);
|
||||
}
|
||||
}
|
||||
|
||||
/* used by debugfs */
|
||||
void sched_cache_active_set(void)
|
||||
{
|
||||
cpus_read_lock();
|
||||
sched_domains_mutex_lock();
|
||||
_sched_cache_active_set();
|
||||
sched_domains_mutex_unlock();
|
||||
cpus_read_unlock();
|
||||
}
|
||||
|
||||
/*
|
||||
* Update the bottom sched_domain's llc_bytes for @cpu and all its
|
||||
* LLC siblings. Called from cacheinfo_cpu_online() or
|
||||
* cacheinfo_cpu_pre_down() with cpu hotplug lock held.
|
||||
*
|
||||
* Note: get_effective_llc_bytes() returns 0 on PowerPC.
|
||||
* thus cache aware scheduling is disabled on PowerPC for
|
||||
* now. PowerPC does not use the generic cacheinfo framework --
|
||||
* it has its own cacheinfo with a separate struct cache hierarchy
|
||||
* and does not populates the per-CPU struct cpu_cacheinfo array
|
||||
* that get_cpu_cacheinfo_llc() reads.
|
||||
*/
|
||||
void sched_update_llc_bytes(unsigned int cpu)
|
||||
{
|
||||
struct sched_domain *sd, *sdp;
|
||||
unsigned int i;
|
||||
|
||||
sched_domains_mutex_lock();
|
||||
|
||||
sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu));
|
||||
if (!sdp)
|
||||
goto unlock;
|
||||
|
||||
/*
|
||||
* ci->shared_cpu_map is built incrementally as CPUs come
|
||||
* online, so the first CPU in an LLC initially sees
|
||||
* hw_weight == 1 and computes an inflated llc_bytes in
|
||||
* get_effective_llc_bytes(). Re-evaluating every LLC
|
||||
* sibling on each online event corrects this once the full
|
||||
* shared_cpu_map is known.
|
||||
*/
|
||||
for_each_cpu(i, sched_domain_span(sdp)) {
|
||||
sd = rcu_dereference_sched_domain(cpu_rq(i)->sd);
|
||||
if (sd)
|
||||
sd->llc_bytes = get_effective_llc_bytes(i, sdp);
|
||||
}
|
||||
|
||||
unlock:
|
||||
sched_domains_mutex_unlock();
|
||||
}
|
||||
|
||||
static void sched_cache_set(bool has_multi_llcs)
|
||||
{
|
||||
/*
|
||||
* TBD: check before writing to it. sched domain rebuild
|
||||
* is not in the critical path, leave as-is for now.
|
||||
*/
|
||||
if (has_multi_llcs)
|
||||
static_branch_enable_cpuslocked(&sched_cache_present);
|
||||
else
|
||||
static_branch_disable_cpuslocked(&sched_cache_present);
|
||||
|
||||
_sched_cache_active_set();
|
||||
}
|
||||
#else
|
||||
static bool alloc_sd_llc(const struct cpumask *cpu_map,
|
||||
struct s_data *d)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
static inline void sched_cache_set(bool has_multi_llcs) { }
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Return true if @sd belongs to an LLC group whose enclosing
|
||||
* partition spans more than one LLC. @sd must be the topmost
|
||||
* SD_SHARE_LLC domain.
|
||||
*
|
||||
* Any duplicated parent domains with the same span as @sd are
|
||||
* skipped: before cpu_attach_domain() degeneration these still
|
||||
* exist, after degeneration the loop is a no-op. This makes the
|
||||
* helper usable both during sched domain build and against an
|
||||
* already-attached domain tree.
|
||||
*
|
||||
* Note: For systems with a single LLC per node, cache-aware
|
||||
* scheduling is still enabled when multiple nodes exist.
|
||||
* However, NUMA balancing decisions take precedence over
|
||||
* cache-aware scheduling. Conversely, if there is only one
|
||||
* LLC per partition, cache-aware scheduling should be disabled.
|
||||
*/
|
||||
static bool sd_in_multi_llcs(struct sched_domain *sd)
|
||||
{
|
||||
struct sched_domain *sdp = sd->parent;
|
||||
|
||||
/* it does not make sense to aggregate to 1 CPU */
|
||||
if (sd->span_weight == 1)
|
||||
return false;
|
||||
|
||||
while (sdp && sdp->span_weight == sd->span_weight)
|
||||
sdp = sdp->parent;
|
||||
|
||||
return !!sdp;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the canonical balance CPU for this group, this is the first CPU
|
||||
* of this group that's also in the balance mask.
|
||||
|
|
@ -1803,6 +2051,11 @@ const struct cpumask *tl_mc_mask(struct sched_domain_topology_level *tl, int cpu
|
|||
{
|
||||
return cpu_coregroup_mask(cpu);
|
||||
}
|
||||
|
||||
#define llc_mask(cpu) cpu_coregroup_mask(cpu)
|
||||
|
||||
#else
|
||||
#define llc_mask(cpu) cpumask_of(cpu)
|
||||
#endif
|
||||
|
||||
const struct cpumask *tl_pkg_mask(struct sched_domain_topology_level *tl, int cpu)
|
||||
|
|
@ -2711,14 +2964,71 @@ static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu)
|
|||
return true;
|
||||
}
|
||||
|
||||
static int __sched_domains_alloc_llc_id(void)
|
||||
{
|
||||
int lid, max;
|
||||
|
||||
lockdep_assert_held(&sched_domains_mutex);
|
||||
|
||||
lid = cpumask_first_zero(sched_domains_llc_id_allocmask);
|
||||
/*
|
||||
* llc_id space should never grow larger than the
|
||||
* possible number of CPUs in the system.
|
||||
*/
|
||||
if (lid >= nr_cpu_ids)
|
||||
return -1;
|
||||
|
||||
__cpumask_set_cpu(lid, sched_domains_llc_id_allocmask);
|
||||
max = cpumask_last(sched_domains_llc_id_allocmask);
|
||||
if (max > max_lid)
|
||||
max_lid = max;
|
||||
|
||||
return lid;
|
||||
}
|
||||
|
||||
static void __sched_domains_free_llc_id(int cpu)
|
||||
{
|
||||
int i, lid, max;
|
||||
|
||||
lockdep_assert_held(&sched_domains_mutex);
|
||||
|
||||
lid = per_cpu(sd_llc_id, cpu);
|
||||
if (lid == -1 || lid >= nr_cpu_ids)
|
||||
return;
|
||||
|
||||
per_cpu(sd_llc_id, cpu) = -1;
|
||||
|
||||
for_each_cpu(i, llc_mask(cpu)) {
|
||||
/* An online CPU owns the llc_id. */
|
||||
if (per_cpu(sd_llc_id, i) == lid)
|
||||
return;
|
||||
}
|
||||
|
||||
__cpumask_clear_cpu(lid, sched_domains_llc_id_allocmask);
|
||||
|
||||
max = cpumask_last(sched_domains_llc_id_allocmask);
|
||||
/* shrink max lid to save memory */
|
||||
if (max < max_lid)
|
||||
max_lid = max;
|
||||
}
|
||||
|
||||
void sched_domains_free_llc_id(int cpu)
|
||||
{
|
||||
sched_domains_mutex_lock();
|
||||
__sched_domains_free_llc_id(cpu);
|
||||
sched_domains_mutex_unlock();
|
||||
}
|
||||
|
||||
/*
|
||||
* Build sched domains for a given set of CPUs and attach the sched domains
|
||||
* to the individual CPUs
|
||||
*/
|
||||
static int
|
||||
build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr)
|
||||
build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr,
|
||||
bool *multi_llcs)
|
||||
{
|
||||
enum s_alloc alloc_state = sa_none;
|
||||
bool has_multi_llcs = false;
|
||||
struct sched_domain *sd;
|
||||
struct s_data d;
|
||||
struct rq *rq = NULL;
|
||||
|
|
@ -2736,6 +3046,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
/* Set up domains for CPUs specified by the cpu_map: */
|
||||
for_each_cpu(i, cpu_map) {
|
||||
struct sched_domain_topology_level *tl;
|
||||
int lid;
|
||||
|
||||
sd = NULL;
|
||||
for_each_sd_topology(tl) {
|
||||
|
|
@ -2749,6 +3060,29 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
if (cpumask_equal(cpu_map, sched_domain_span(sd)))
|
||||
break;
|
||||
}
|
||||
|
||||
lid = per_cpu(sd_llc_id, i);
|
||||
if (lid == -1) {
|
||||
/* try to reuse the llc_id of its siblings */
|
||||
for (int j = cpumask_first(llc_mask(i));
|
||||
j < nr_cpu_ids;
|
||||
j = cpumask_next(j, llc_mask(i))) {
|
||||
if (i == j)
|
||||
continue;
|
||||
|
||||
lid = per_cpu(sd_llc_id, j);
|
||||
|
||||
if (lid != -1) {
|
||||
per_cpu(sd_llc_id, i) = lid;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* a new LLC is detected */
|
||||
if (lid == -1)
|
||||
per_cpu(sd_llc_id, i) = __sched_domains_alloc_llc_id();
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!topology_span_sane(cpu_map)))
|
||||
|
|
@ -2769,33 +3103,31 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
}
|
||||
|
||||
for_each_cpu(i, cpu_map) {
|
||||
bool asym_claimed = false;
|
||||
|
||||
sd = *per_cpu_ptr(d.sd, i);
|
||||
if (!sd)
|
||||
continue;
|
||||
|
||||
if (has_asym)
|
||||
asym_claimed = claim_asym_sched_domain_shared(&d, i);
|
||||
claim_asym_sched_domain_shared(&d, i);
|
||||
|
||||
/* First, find the topmost SD_SHARE_LLC domain */
|
||||
while (sd->parent && (sd->parent->flags & SD_SHARE_LLC))
|
||||
sd = sd->parent;
|
||||
|
||||
if (sd->flags & SD_SHARE_LLC) {
|
||||
/*
|
||||
* Initialize the sd->shared for SD_SHARE_LLC unless
|
||||
* the asym path above already claimed it.
|
||||
*/
|
||||
if (!asym_claimed)
|
||||
init_sched_domain_shared(&d, sd);
|
||||
init_sched_domain_shared(&d, sd);
|
||||
|
||||
/*
|
||||
* In presence of higher domains, adjust the
|
||||
* NUMA imbalance stats for the hierarchy.
|
||||
*/
|
||||
if (IS_ENABLED(CONFIG_NUMA) && sd->parent)
|
||||
adjust_numa_imbalance(sd);
|
||||
if (sd->parent) {
|
||||
if (IS_ENABLED(CONFIG_NUMA))
|
||||
adjust_numa_imbalance(sd);
|
||||
|
||||
if (sd_in_multi_llcs(sd))
|
||||
has_multi_llcs = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2810,6 +3142,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
init_sched_groups_capacity(i, sd);
|
||||
}
|
||||
|
||||
alloc_sd_llc(cpu_map, &d);
|
||||
|
||||
/* Attach the domains */
|
||||
rcu_read_lock();
|
||||
for_each_cpu(i, cpu_map) {
|
||||
|
|
@ -2834,6 +3168,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
|
||||
ret = 0;
|
||||
error:
|
||||
*multi_llcs = has_multi_llcs;
|
||||
__free_domain_allocs(&d, alloc_state, cpu_map);
|
||||
|
||||
return ret;
|
||||
|
|
@ -2896,8 +3231,10 @@ void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
|
|||
*/
|
||||
int __init sched_init_domains(const struct cpumask *cpu_map)
|
||||
{
|
||||
bool multi_llcs;
|
||||
int err;
|
||||
|
||||
zalloc_cpumask_var(&sched_domains_llc_id_allocmask, GFP_KERNEL);
|
||||
zalloc_cpumask_var(&sched_domains_tmpmask, GFP_KERNEL);
|
||||
zalloc_cpumask_var(&sched_domains_tmpmask2, GFP_KERNEL);
|
||||
zalloc_cpumask_var(&fallback_doms, GFP_KERNEL);
|
||||
|
|
@ -2909,7 +3246,9 @@ int __init sched_init_domains(const struct cpumask *cpu_map)
|
|||
if (!doms_cur)
|
||||
doms_cur = &fallback_doms;
|
||||
cpumask_and(doms_cur[0], cpu_map, housekeeping_cpumask(HK_TYPE_DOMAIN));
|
||||
err = build_sched_domains(doms_cur[0], NULL);
|
||||
err = build_sched_domains(doms_cur[0], NULL, &multi_llcs);
|
||||
if (!err)
|
||||
sched_cache_set(multi_llcs);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
|
@ -2982,6 +3321,7 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new
|
|||
struct sched_domain_attr *dattr_new)
|
||||
{
|
||||
bool __maybe_unused has_eas = false;
|
||||
bool has_multi_llcs = false, multi_llcs;
|
||||
int i, j, n;
|
||||
int new_topology;
|
||||
|
||||
|
|
@ -3031,14 +3371,41 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new
|
|||
for (i = 0; i < ndoms_new; i++) {
|
||||
for (j = 0; j < n && !new_topology; j++) {
|
||||
if (cpumask_equal(doms_new[i], doms_cur[j]) &&
|
||||
dattrs_equal(dattr_new, i, dattr_cur, j))
|
||||
dattrs_equal(dattr_new, i, dattr_cur, j)) {
|
||||
/*
|
||||
* Reused partition has to be taken care
|
||||
* of here, because there could be a corner
|
||||
* case that if the reused partition is skipped
|
||||
* and only new partition is considered, an
|
||||
* incorrect has_multi_llcs would be set. For
|
||||
* example:
|
||||
* If the only multi-LLC partition is reused
|
||||
* and a new single-LLC partition is built,
|
||||
* sched_cache_set(false) disables cache-aware
|
||||
* scheduling globally despite the reused
|
||||
* multi-LLC partition still being active.
|
||||
*/
|
||||
struct sched_domain *sd;
|
||||
int cpu = cpumask_first(doms_cur[j]);
|
||||
|
||||
guard(rcu)();
|
||||
sd = rcu_dereference(cpu_rq(cpu)->sd);
|
||||
while (sd && sd->parent && (sd->parent->flags & SD_SHARE_LLC))
|
||||
sd = sd->parent;
|
||||
if (sd && (sd->flags & SD_SHARE_LLC) && sd->parent &&
|
||||
sd_in_multi_llcs(sd))
|
||||
has_multi_llcs = true;
|
||||
goto match2;
|
||||
}
|
||||
}
|
||||
/* No match - add a new doms_new */
|
||||
build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
|
||||
build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL,
|
||||
&multi_llcs);
|
||||
has_multi_llcs |= multi_llcs;
|
||||
match2:
|
||||
;
|
||||
}
|
||||
sched_cache_set(has_multi_llcs);
|
||||
|
||||
#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
|
||||
/* Build perf domains: */
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user