From 51d4aae73f85a4e1760521577c5afcea9e731c4f Mon Sep 17 00:00:00 2001 From: Liang Chen Date: Wed, 15 Jul 2020 17:35:36 +0800 Subject: [PATCH] sched/fair: optimize scheduler for performance 1. init util_avg of new task to half the util of big core. 2. treat task wait time as busy time. 3. do not put task to the cpu whose max util can not fit the task. 4. boost cpufreq when a task is wakeup. if prefer powersave, we can disable this optimization by command: echo 0 > /proc/sys/kernel/sched_performance_bias Change-Id: I937d1cc2295645a1d044e0735ef575b6507bb752 Signed-off-by: Liang Chen --- drivers/soc/rockchip/Kconfig | 6 ++++ include/linux/sched/sysctl.h | 4 +++ kernel/sched/fair.c | 59 ++++++++++++++++++++++++++++++++++++ kernel/sched/pelt.c | 4 +++ kernel/sysctl.c | 9 ++++++ 5 files changed, 82 insertions(+) diff --git a/drivers/soc/rockchip/Kconfig b/drivers/soc/rockchip/Kconfig index 0c7d2074f50b..6e0a205efda2 100644 --- a/drivers/soc/rockchip/Kconfig +++ b/drivers/soc/rockchip/Kconfig @@ -128,4 +128,10 @@ config ROCKCHIP_THUNDER_BOOT_MMC help Say y if boot from MMC. +config ROCKCHIP_SCHED_PERFORMANCE_BIAS + bool "Rockchip task scheduler optimization" + default y + help + Say y here to enable rockchip optimize task scheduler for performance bias, + this would cause a little more power consumption. endif diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index a6198cb74f56..67e969f17ee9 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -67,6 +67,10 @@ extern unsigned int sysctl_sched_uclamp_util_max; extern unsigned int sysctl_sched_cfs_bandwidth_slice; #endif +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS +extern unsigned int sysctl_sched_performance_bias; +#endif + #ifdef CONFIG_SCHED_AUTOGROUP extern unsigned int sysctl_sched_autogroup_enabled; #endif diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 602f928bee9d..10d72a806d6b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -20,6 +20,9 @@ * Adaptive scheduling granularity, math enhancements by Peter Zijlstra * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra */ +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS +#include +#endif #include "sched.h" #include @@ -120,6 +123,10 @@ int __weak arch_asym_cpu_priority(int cpu) unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS +unsigned int sysctl_sched_performance_bias = 1; +#endif + /* * The margin used when comparing utilization with CPU capacity: * util * margin < capacity * 1024 @@ -737,6 +744,12 @@ void init_entity_runnable_average(struct sched_entity *se) se->runnable_weight = se->load.weight; +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS + if (sysctl_sched_performance_bias) { + sa->util_avg = SCHED_CAPACITY_SCALE >> 1; + sa->util_sum = sa->util_avg * LOAD_AVG_MAX; + } +#endif /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ } @@ -776,7 +789,11 @@ void post_init_entity_util_avg(struct sched_entity *se) long cpu_scale = arch_scale_cpu_capacity(NULL, cpu_of(rq_of(cfs_rq))); long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS + if (!sysctl_sched_performance_bias && (cap > 0)) { +#else if (cap > 0) { +#endif if (cfs_rq->avg.util_avg != 0) { sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; sa->util_avg /= (cfs_rq->avg.load_avg + 1); @@ -3863,6 +3880,22 @@ static inline int task_fits_capacity(struct task_struct *p, long capacity) return capacity * 1024 > uclamp_task_util(p) * capacity_margin; } +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS +static inline bool task_fits_max(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_of(cpu); + unsigned long max_capacity = cpu_rq(cpu)->rd->max_cpu_capacity.val; + + if (capacity == max_capacity) + return true; + + if (capacity * capacity_margin > max_capacity * 1024) + return true; + + return task_fits_capacity(p, capacity); +} +#endif + static inline void update_misfit_status(struct task_struct *p, struct rq *rq) { if (!static_branch_unlikely(&sched_asym_cpucapacity)) @@ -5247,6 +5280,11 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) struct cfs_rq *cfs_rq; struct sched_entity *se = &p->se; +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS + if (sysctl_sched_performance_bias) + cpufreq_task_boost(rq->cpu, task_util_est(p)); +#endif + /* * The code below (indirectly) updates schedutil which looks at * the cfs_rq utilization to select a frequency. @@ -6016,6 +6054,13 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, &p->cpus_allowed)) continue; +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS + if (sysctl_sched_performance_bias) { + if (!task_fits_max(p, group_first_cpu(group))) + continue; + } +#endif + local_group = cpumask_test_cpu(this_cpu, sched_group_span(group)); @@ -6105,6 +6150,13 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, if (!idlest) return NULL; +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS + if (sysctl_sched_performance_bias) { + if ((this_runnable_load == ULONG_MAX) || (this_avg_load == ULONG_MAX)) + return idlest; + } +#endif + /* * When comparing groups across NUMA domains, it's possible for the * local domain to be very lightly loaded relative to the remote @@ -8219,6 +8271,13 @@ static int detach_tasks(struct lb_env *env) break; } +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS + if (sysctl_sched_performance_bias) { + if ((env->idle == CPU_NOT_IDLE) && (!task_fits_max(p, env->dst_cpu))) + goto next; + } +#endif + if (!can_migrate_task(p, env)) goto next; diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c index 19ba404dfc28..1b00c6929cd4 100644 --- a/kernel/sched/pelt.c +++ b/kernel/sched/pelt.c @@ -279,7 +279,11 @@ int __update_load_avg_blocked_se(u64 now, struct sched_entity *se) int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se) { if (___update_load_sum(now, &se->avg, !!se->on_rq, !!se->on_rq, +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS + (sysctl_sched_performance_bias && se->on_rq) || (cfs_rq->curr == se))) { +#else cfs_rq->curr == se)) { +#endif ___update_load_avg(&se->avg, se_weight(se), se_runnable(se)); cfs_se_util_change(&se->avg); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 521c38e9ac14..56a40df1e73c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -499,6 +499,15 @@ static struct ctl_table kern_table[] = { .extra1 = &one, }, #endif +#ifdef CONFIG_ROCKCHIP_SCHED_PERFORMANCE_BIAS + { + .procname = "sched_performance_bias", + .data = &sysctl_sched_performance_bias, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) { .procname = "sched_energy_aware",