From ec30ed6a7590f06d8e451ba0097cfb8837f3eb01 Mon Sep 17 00:00:00 2001 From: Blagovest Kolenichev Date: Tue, 28 Jun 2022 15:02:26 -0700 Subject: [PATCH 01/12] sched: walt: fix build issue With latest Android Kernel update, below errors are seen kernel_platform/msm-kernel/include/../drivers/android/binder_internal.h:111:1: error: declaration specifier missing, defaulting to 'int' [-Werror] kernel_platform/msm-kernel/include/../drivers/android/binder_internal.h:111:23: error: a parameter list without types is only allowed in a function definition kernel_platform/msm-kernel/include/../drivers/android/binder_internal.h:114:1: error: declaration specifier missing, defaulting to 'int' [-Werror] kernel_platform/msm-kernel/include/../drivers/android/binder_internal.h:114:23: error: a parameter list without types is only allowed in a function definition kernel_platform/msm-kernel/include/../drivers/android/binder_internal.h:117:1: error: declaration specifier missing, defaulting to 'int' [-Werror] kernel_platform/msm-kernel/include/../drivers/android/binder_internal.h:117:23: error: a parameter list without types is only allowed in a function definition kernel_platform/msm-kernel/include/../drivers/android/binder_internal.h:120:1: error: declaration specifier missing, defaulting to 'int' [-Werror] kernel_platform/msm-kernel/include/../drivers/android/binder_internal.h:120:23: error: a parameter list without types is only allowed in a function definition Fix these build issues by including right header file. Change-Id: I4ff2f484b05ebae9b9929b4d2c3b7b1e717769b4 Signed-off-by: Blagovest Kolenichev --- kernel/sched/walt/walt_cfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index da6489938e69..ec5e03e34c92 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -4,6 +4,7 @@ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ +#include #include #include From ff2d763574820c531385a945389d4db6de906a79 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 5 Jul 2022 14:40:47 -0700 Subject: [PATCH 02/12] sched/walt: Handle -ve cpu capacity There are instances where rt and irq load exceeds thermally limited capacity and results in a very high cpu_capacity. Fix it by comparing it against 0. Change-Id: Ice6e71cd8b5e0635ec1d79c7338a964cef70271d Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 18dd18f34135..f97c7fdf91c2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4110,7 +4110,7 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long if (old != rq->cpu_capacity_orig) trace_update_cpu_capacity(cpu, rt_pressure, *capacity); - *capacity = max(rq->cpu_capacity_orig - rt_pressure, 1UL); + *capacity = max((int)(rq->cpu_capacity_orig - rt_pressure), 0); } static void android_rvh_sched_cpu_starting(void *unused, int cpu) From 48c24c731637d5dc3bd5ce2ef1a521554f32ee02 Mon Sep 17 00:00:00 2001 From: Ashay Jaiswal Date: Tue, 14 Jun 2022 23:47:29 +0530 Subject: [PATCH 03/12] sched: walt: update newidle balancing There is a case where when big cluster(Gold/Prime) is entering idle it fails to pull task from lower cluster if lower cluster is only having a single big task in it's runqueue. The reason for failure is, walt skips newidle balance for above condition if "idle_cpu" check fails for destination core and for newidle balance destination core is entering idle (not yet idle) thus "idle_cpu" returns false. Fix this by skipping "idle_cpu" check for new idle balancing. Change-Id: Id973bd8a67fbfcd9d47bec9a0894b697412f2c90 Signed-off-by: Ashay Jaiswal Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_lb.c | 43 ++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index a3020210bbc3..ff89294cdc7e 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -426,7 +426,7 @@ static int find_first_idle_if_others_are_busy(const cpumask_t *src_mask) } static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *src_mask, - int *has_misfit) + int *has_misfit, bool is_newidle) { int i; int busiest_cpu = -1; @@ -452,7 +452,7 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr } static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t *src_mask, - int *has_misfit) + int *has_misfit, bool is_newidle) { int i; int busiest_cpu = -1; @@ -512,7 +512,7 @@ static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t } static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t *src_mask, - int *has_misfit) + int *has_misfit, bool is_newidle) { int i; int busiest_cpu = -1; @@ -521,6 +521,7 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t int total_cpus = 0; int busy_nr_big_tasks = 0; struct walt_rq *wrq; + bool treat_dst_idle = is_newidle || available_idle_cpu(dst_cpu); /* * A higher capacity CPU is looking at a lower capacity @@ -552,7 +553,7 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t /* active migration is allowed only to idle cpu */ if (cpu_rq(i)->cfs.h_nr_running < 2 && - (!wrq->walt_stats.nr_big_tasks || !available_idle_cpu(dst_cpu))) + (!wrq->walt_stats.nr_big_tasks || !treat_dst_idle)) continue; if (!walt_rotation_enabled && !cpu_overutilized(i) && @@ -579,7 +580,8 @@ static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t return busiest_cpu; } -static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int *has_misfit) +static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int *has_misfit, + bool is_newidle) { int fsrc_cpu = cpumask_first(src_mask); int busiest_cpu; @@ -588,13 +590,13 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int if (dst_wrq->cluster->id == fsrc_wrq->cluster->id) busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, - src_mask, has_misfit); + src_mask, has_misfit, is_newidle); else if (dst_wrq->cluster->id > fsrc_wrq->cluster->id) busiest_cpu = walt_lb_find_busiest_from_lower_cap_cpu(dst_cpu, - src_mask, has_misfit); + src_mask, has_misfit, is_newidle); else busiest_cpu = walt_lb_find_busiest_from_higher_cap_cpu(dst_cpu, - src_mask, has_misfit); + src_mask, has_misfit, is_newidle); return busiest_cpu; } @@ -830,7 +832,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, */ if (num_sched_clusters <= 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; @@ -839,7 +841,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, find_next_cluster = (order_index == 0) ? enough_idle : 1; if (find_next_cluster) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, - &cpu_array[order_index][1], &has_misfit); + &cpu_array[order_index][1], &has_misfit, true); if (busy_cpu != -1 && (enough_idle || has_misfit)) goto found_busy_cpu; } @@ -850,13 +852,13 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (order_index == 0) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; if (enough_idle) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; } @@ -866,7 +868,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, * cluster. In case no idle is found, pull it in. */ busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); @@ -877,20 +879,20 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, } } else if (order_index == 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; /* help gold only if prime has had enough idle or gold has a misfit */ has_misfit = false; busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1 && (enough_idle || has_misfit)) goto found_busy_cpu; /* help the farthest cluster indirectly if it needs help */ busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); @@ -898,20 +900,21 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, } } else { busy_cpu = - walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], &has_misfit); + walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; if (enough_idle) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1) goto found_busy_cpu; } has_misfit = false; busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], - &has_misfit); + &has_misfit, true); if (busy_cpu != -1 && (enough_idle || has_misfit)) goto found_busy_cpu; } @@ -1008,7 +1011,7 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, * remain same. */ cpumask_and(&src_mask, sched_group_span(group), env_cpus); - busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask, &has_misfit); + busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask, &has_misfit, false); done: if (busiest_cpu != -1) *busiest = cpu_rq(busiest_cpu); From a112baf5fb3f427b7074193283e3ff5137d3e9dd Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 22 Jun 2022 13:19:55 -0700 Subject: [PATCH 04/12] sched/walt: help farthest cluster under rotation When rotation is enabled, the system is under heavy load, allow farthest cluster to help each other. Change-Id: Idd402fa120baa47e8506d7dc2439765dd6d5bcc9 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_lb.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index ff89294cdc7e..b69f40fcd0f2 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -872,10 +872,11 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - if (first_idle != -1) + if (first_idle != -1) { walt_kick_cpu(first_idle); - else + } else if (walt_rotation_enabled) { goto found_busy_cpu; + } } } else if (order_index == 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], @@ -896,7 +897,11 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - walt_kick_cpu(first_idle); + if (first_idle != -1) { + walt_kick_cpu(first_idle); + } else if (walt_rotation_enabled) { + goto found_busy_cpu; + } } } else { busy_cpu = From fe61e5329a4be0ffed7f8569d03a20c7f911d128 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 22 Jun 2022 13:50:12 -0700 Subject: [PATCH 05/12] sched/walt_lb: pull task based on their utilizations Currently the code simply pulls the last task in the rq. Instead make it visit at least 5 tasks in the RQ and find a suitable util task to pull. Change-Id: Ied5fa689ec94296f8655232cef5a3301f0cbc813 Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_lb.c | 51 +++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index b69f40fcd0f2..95dd2e1b645e 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -296,17 +296,19 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; struct walt_task_struct *wts; + struct task_struct *pull_me; + int task_visited; BUG_ON(src_cpu == dst_cpu); to_lower = dst_wrq->cluster->id < src_wrq->cluster->id; - to_higher = dst_wrq->cluster->id > src_wrq->cluster->id; raw_spin_lock_irqsave(&src_rq->__lock, flags); + pull_me = NULL; + task_visited = 0; list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { - if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) continue; @@ -317,13 +319,30 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) false)) continue; - walt_detach_task(p, src_rq, dst_rq); - pulled_task = p; + if (pull_me == NULL) { + pull_me = p; + } else { + if (to_lower) { + if (task_util(p) < task_util(pull_me)) + pull_me = p; + } else if (task_util(p) > task_util(pull_me)) { + pull_me = p; + } + } + + task_visited++; + if (task_visited > 5) + break; + } + if (pull_me) { + walt_detach_task(pull_me, src_rq, dst_rq); + pulled_task = pull_me; goto unlock; } + pull_me = NULL; + task_visited = 0; list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { - if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) continue; @@ -334,8 +353,24 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) true)) continue; - walt_detach_task(p, src_rq, dst_rq); - pulled_task = p; + if (pull_me == NULL) { + pull_me = p; + } else { + if (to_lower) { + if (task_util(p) < task_util(pull_me)) + pull_me = p; + } else if (task_util(p) > task_util(pull_me)) { + pull_me = p; + } + } + + task_visited++; + if (task_visited > 5) + break; + } + if (pull_me) { + walt_detach_task(pull_me, src_rq, dst_rq); + pulled_task = pull_me; goto unlock; } @@ -380,7 +415,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) return 0; raw_spin_lock_irqsave(&dst_rq->__lock, flags); - walt_attach_task(p, dst_rq); + walt_attach_task(pulled_task, dst_rq); raw_spin_unlock_irqrestore(&dst_rq->__lock, flags); return 1; /* we pulled 1 task */ From 3400232478b79dddd3d74c949905c266d2efb2fa Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 8 Jul 2022 17:44:35 -0700 Subject: [PATCH 06/12] sched/walt: Update cpu capacity during window rollover Ensure that all thermal pressure gets accounted in capacity_orig within the window rollover period. Change-Id: I3858c21aff31ff517f5e521df765b119bf5d3244 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 88 ++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index f97c7fdf91c2..cd2dfa3ba6f2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3798,6 +3798,50 @@ static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus) } } +static void update_cpu_capacity_helper(int cpu) +{ + unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu); + unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); + unsigned long thermal_cap, old; + struct walt_sched_cluster *cluster; + struct rq *rq = cpu_rq(cpu); + + if (unlikely(walt_disabled)) + return; + + /* + * thermal_pressure = cpu_scale - curr_cap_as_per_thermal. + * so, + * curr_cap_as_per_thermal = cpu_scale - thermal_pressure. + */ + + thermal_cap = fmax_capacity - thermal_pressure; + + cluster = cpu_cluster(cpu); + /* reduce the fmax_capacity under cpufreq constraints */ + if (cluster->max_freq != cluster->max_possible_freq) + fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq, + cluster->max_possible_freq); + + old = rq->cpu_capacity_orig; + rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap); + + if (old != rq->cpu_capacity_orig) + trace_update_cpu_capacity(cpu, 0, 0); +} + +/* + * The intention of this hook is to update cpu_capacity_orig as well as + * (*capacity), otherwise we will end up capacity_of() > capacity_orig_of(). + */ +static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity) +{ + unsigned long rt_pressure = arch_scale_cpu_capacity(cpu) - *capacity; + + update_cpu_capacity_helper(cpu); + *capacity = max((int)(cpu_rq(cpu)->cpu_capacity_orig - rt_pressure), 0); +} + /** * walt_irq_work() - perform walt irq work for rollover and migration * @@ -3839,6 +3883,11 @@ static void walt_irq_work(struct irq_work *irq_work) level++; } + if (!is_migration) { + for_each_cpu(cpu, &lock_cpus) + update_cpu_capacity_helper(cpu); + } + __walt_irq_work_locked(is_migration, &lock_cpus); for_each_cpu(cpu, &lock_cpus) @@ -4074,45 +4123,6 @@ static void walt_cpu_frequency_limits(void *unused, struct cpufreq_policy *polic cpu_cluster(policy->cpu)->max_freq = policy->max; } -/* - * The intention of this hook is to update cpu_capacity_orig as well as - * (*capacity), otherwise we will end up capacity_of() > capacity_orig_of(). - */ -static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity) -{ - unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu); - unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); - unsigned long thermal_cap, old; - unsigned long rt_pressure = fmax_capacity - *capacity; - struct walt_sched_cluster *cluster; - struct rq *rq = cpu_rq(cpu); - - if (unlikely(walt_disabled)) - return; - - /* - * thermal_pressure = cpu_scale - curr_cap_as_per_thermal. - * so, - * curr_cap_as_per_thermal = cpu_scale - thermal_pressure. - */ - - thermal_cap = fmax_capacity - thermal_pressure; - - cluster = cpu_cluster(cpu); - /* reduce the fmax_capacity under cpufreq constraints */ - if (cluster->max_freq != cluster->max_possible_freq) - fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq, - cluster->max_possible_freq); - - old = rq->cpu_capacity_orig; - rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap); - - if (old != rq->cpu_capacity_orig) - trace_update_cpu_capacity(cpu, rt_pressure, *capacity); - - *capacity = max((int)(rq->cpu_capacity_orig - rt_pressure), 0); -} - static void android_rvh_sched_cpu_starting(void *unused, int cpu) { if (unlikely(walt_disabled)) From 9698b4fcb0b6ef0d928763fddb78b812b00949c7 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 4 Aug 2021 15:56:08 -0700 Subject: [PATCH 07/12] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ifa74c7c827ed5a5a8b23fcb9d6de2325dfeb788f Signed-off-by: Stephen Dickey Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 11 +++++++++++ kernel/sched/walt/walt.h | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 0eab97e74b21..8a213e937820 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -93,6 +93,7 @@ struct walt_task_struct { * 'prev_on_rq' tracks enqueue/dequeue of a task for error conditions * 0 = nothing, 1 = enqueued, 2 = dequeued */ + u32 flags; u64 mark_start; u64 window_start; u32 sum, demand; @@ -141,6 +142,16 @@ struct walt_task_struct { u8 hung_detect_status; }; +/* + * enumeration to set the flags variable + * each index below represents an offset into + * wts->flags + */ +enum walt_flags { + WALT_INIT, + MAX_WALT_FLAGS +}; + #define wts_to_ts(wts) ({ \ void *__mptr = (void *)(wts); \ ((struct task_struct *)(__mptr - \ diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index b8a0e5bf8096..e5bfef031dc5 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -903,6 +903,23 @@ extern int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int wr void __user *buffer, size_t *lenp, loff_t *ppos); +static inline void walt_flag_set(struct task_struct *p, enum walt_flags feature, bool set) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (set) + wts->flags |= 1 << feature; + else + wts->flags &= ~(1 << feature); +} + +static inline bool walt_flag_test(struct task_struct *p, enum walt_flags feature) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return !!(wts->flags & (1 << feature)); +} + #define WALT_MVP_SLICE 3000000U #define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE) From 37f5cf75efdc59f10b32288430efcf365d941459 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 27 Jul 2022 16:49:28 -0700 Subject: [PATCH 08/12] sched/walt: Ensure crash only on initialized task In the event where a task is being dequeued from a CPU other than the CPU WALT has recorded as the previous enqueue cpu, a crash is induced. However, it could be the case during WALT initialization that due to a race where task is in the middle of being forked, an attempt is made to dequeue a task where the prev_on_cpu_rq member is not properly initialized. Change-Id: I3a52da5d6cc5ae5b71800d0f8097e6debb3b7187 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index cd2dfa3ba6f2..af6c8fda9339 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2457,6 +2457,7 @@ static void init_new_task_load(struct task_struct *p) wts->mvp_prio = WALT_NOT_MVP; wts->cidx = 0; __sched_fork_init(p); + walt_flag_set(p, WALT_INIT, 1); } static void init_existing_task_load(struct task_struct *p) @@ -4284,7 +4285,8 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st * therefore the check to ensure that prev_on_rq_cpu is needed to prevent * an invalid failure. */ - if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq)) + if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq) && + walt_flag_test(p, WALT_INIT)) WALT_BUG(WALT_BUG_UPSTREAM, p, "dequeue cpu %d not same as enqueue %d\n", cpu_of(rq), wts->prev_on_rq_cpu); From f970f604f2d18eea66194b8d5f7465ff9cfbc326 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Mon, 8 Aug 2022 15:05:35 -0700 Subject: [PATCH 09/12] sched/walt: use sched_idle_enough to skip packing When sysctl_sched_idle_enough is not set, the current code still finds a pakcing cpu only to reject it later. Fix this i.e. don't even bother finding a packing cpu when idle_enough is zero. Change-Id: I5cdce5e06072eee75593aa1e36953a2946e20efa Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 73 ------------------------------------ kernel/sched/walt/walt.h | 68 +++++++++++++++++++++++++++++++++ kernel/sched/walt/walt_cfs.c | 4 +- kernel/sched/walt/walt_rt.c | 8 ++-- 4 files changed, 74 insertions(+), 79 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index af6c8fda9339..cfedcae2de1d 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -354,79 +354,6 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, static void rollover_cpu_window(struct rq *rq, bool full_window); static void rollover_top_tasks(struct rq *rq, bool full_window); -/* walt_find_cluster_packing_cpu - Return a packing_cpu choice common for this cluster. - * @start_cpu: The cpu from the cluster to choose from - * - * If the cluster has a 32bit capable cpu return it regardless - * of whether it is halted or not. - * - * If the cluster does not have a 32 bit capable cpu, find the - * first unhalted, active cpu in this cluster. - */ -int walt_find_cluster_packing_cpu(int start_cpu) -{ - struct rq *rq = cpu_rq(start_cpu); - struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; - struct walt_sched_cluster *cluster = wrq->cluster; - cpumask_t unhalted_cpus; - cpumask_t cluster_32bit_cpus; - - /* find all 32 bit capable cpus in this cluster */ - cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask()); - - /* pack 32 bit and 64 bit tasks on the same cpu, if possible */ - if (cpumask_weight(&cluster_32bit_cpus) > 0) - return cpumask_first(&cluster_32bit_cpus); - - /* find all unhalted active cpus */ - cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); - - /* find all unhalted active cpus in this cluster */ - cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); - - /* return the first found unhalted, active cpu, in this cluster */ - return cpumask_first(&unhalted_cpus); -} - -/* for cfs and rt, determine if packing_cpu should be used */ -bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p) -{ - struct rq *rq; - struct walt_rq *wrq; - struct walt_sched_cluster *cluster; - - /* packing cpu must be a valid cpu for runqueue lookup */ - if (packing_cpu >= nr_cpu_ids) - return false; - - rq = cpu_rq(packing_cpu); - wrq = (struct walt_rq *)rq->android_vendor_data1; - cluster = wrq->cluster; - - /* if idle_enough feature is not enabled */ - if (!sysctl_sched_idle_enough) - return false; - - /* if cpu is not allowed for this task */ - if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr)) - return false; - - /* if cluster util is high */ - if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct) - return false; - - /* if cpu utilization is high */ - if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough) - return false; - - /* don't pack big tasks */ - if (task_util(p) >= sysctl_sched_idle_enough) - return false; - - /* the packing cpu can be used, so pack! */ - return true; -} - /* * Demand aggregation for frequency purpose: * diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index e5bfef031dc5..203ca9798fa1 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -962,6 +962,74 @@ extern struct cpumask __cpu_halt_mask; #define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask) #define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask) +/* walt_find_and_choose_cluster_packing_cpu - Return a packing_cpu choice common for this cluster. + * @start_cpu: The cpu from the cluster to choose from + * + * If the cluster has a 32bit capable cpu return it regardless + * of whether it is halted or not. + * + * If the cluster does not have a 32 bit capable cpu, find the + * first unhalted, active cpu in this cluster. + * + * Returns -1 if packing_cpu if not found or is unsuitable to be packed on to + * Returns a valid cpu number if packing_cpu is found and is usable + */ +static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct task_struct *p) +{ + struct rq *rq = cpu_rq(start_cpu); + struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + cpumask_t unhalted_cpus; + cpumask_t cluster_32bit_cpus; + int packing_cpu; + + /* if idle_enough feature is not enabled */ + if (!sysctl_sched_idle_enough) + return -1; + if (!sysctl_sched_cluster_util_thres_pct) + return -1; + + /* find all 32 bit capable cpus in this cluster */ + cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask()); + + /* pack 32 bit and 64 bit tasks on the same cpu, if possible */ + if (cpumask_weight(&cluster_32bit_cpus) > 0) { + packing_cpu = cpumask_first(&cluster_32bit_cpus); + } else { + /* find all unhalted active cpus */ + cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); + + /* find all unhalted active cpus in this cluster */ + cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); + + /* return the first found unhalted, active cpu, in this cluster */ + packing_cpu = cpumask_first(&unhalted_cpus); + } + + /* packing cpu must be a valid cpu for runqueue lookup */ + if (packing_cpu >= nr_cpu_ids) + return -1; + + /* if cpu is not allowed for this task */ + if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr)) + return -1; + + /* if cluster util is high */ + if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct) + return -1; + + /* if cpu utilization is high */ + if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough) + return -1; + + /* don't pack big tasks */ + if (task_util(p) >= sysctl_sched_idle_enough) + return -1; + + /* the packing cpu can be used, so pack! */ + return packing_cpu; +} + extern void walt_task_dump(struct task_struct *p); extern void walt_rq_dump(int cpu); extern void walt_dump(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index ec5e03e34c92..18543e113aad 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -281,8 +281,8 @@ static void walt_find_best_target(struct sched_domain *sd, } /* fast path for packing_cpu */ - packing_cpu = walt_find_cluster_packing_cpu(start_cpu); - if (walt_choose_packing_cpu(packing_cpu, p)) { + packing_cpu = walt_find_and_choose_cluster_packing_cpu(start_cpu, p); + if (packing_cpu >= 0) { fbt_env->fastpath = CLUSTER_PACKING_FASTPATH; cpumask_set_cpu(packing_cpu, candidates); goto out; diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index e69fe2a9e9de..3265be5ba664 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -290,8 +290,8 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c lowest_mask, walt_rt_task_fits_capacity); /* create a fastpath for finding a packing cpu */ - packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); - if (walt_choose_packing_cpu(packing_cpu, task)) { + packing_cpu = walt_find_and_choose_cluster_packing_cpu(task_cpu(task), task); + if (packing_cpu >= 0) { *new_cpu = packing_cpu; goto unlock; } @@ -333,8 +333,8 @@ static void walt_rt_find_lowest_rq(void *unused, struct task_struct *task, return; /* create a fastpath for finding a packing cpu */ - packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); - if (walt_choose_packing_cpu(packing_cpu, task)) { + packing_cpu = walt_find_and_choose_cluster_packing_cpu(task_cpu(task), task); + if (packing_cpu >= 0) { *best_cpu = packing_cpu; return; } From 90ec01ea1226ad77155fb7f30fb57a8ab669b3b0 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 4 Aug 2022 12:37:33 -0700 Subject: [PATCH 10/12] walt_cfs: MVP: rearrange priorities Update to prefer low_latency and pipeline as having higher priority than others. Change-Id: Ib5ba973d483cf923f886a906a158907335ccab9b Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.h | 2 ++ kernel/sched/walt/walt_cfs.c | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 203ca9798fa1..bed7dd0e49fd 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -923,9 +923,11 @@ static inline bool walt_flag_test(struct task_struct *p, enum walt_flags feature #define WALT_MVP_SLICE 3000000U #define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE) +/* higher number, better priority */ #define WALT_RTG_MVP 0 #define WALT_BINDER_MVP 1 #define WALT_TASK_BOOST_MVP 2 +#define WALT_LL_PIPE_MVP 3 #define WALT_NOT_MVP -1 diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 18543e113aad..6a44c6c9400c 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1042,14 +1042,17 @@ static void binder_restore_priority_hook(void *data, */ int walt_get_mvp_task_prio(struct task_struct *p) { + if (walt_procfs_low_latency_task(p) || + walt_pipeline_low_latency_task(p)) + return WALT_LL_PIPE_MVP; + if (per_task_boost(p) == TASK_BOOST_STRICT_MAX) return WALT_TASK_BOOST_MVP; if (walt_binder_low_latency_task(p)) return WALT_BINDER_MVP; - if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p) || - walt_pipeline_low_latency_task(p)) + if (task_rtg_high_prio(p)) return WALT_RTG_MVP; return WALT_NOT_MVP; From f1b09a77f5eec5c028cf438d565eea0325143eab Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 10 Aug 2022 14:04:10 -0700 Subject: [PATCH 11/12] sched/walt: move predictive freq to wakeup enqueue This will update the freq as soon the rqlock is dropped and also avoids us from acquiring a rq lock as enqueue is called with rq lock held. Change-Id: I24b91309b753f61a38d1eb8e26af451544fa3f52 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index cfedcae2de1d..5665b7e32ffc 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4193,6 +4193,10 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (!double_enqueue) walt_inc_cumulative_runnable_avg(rq, p); + + if ((flags & ENQUEUE_WAKEUP) && do_pl_notif(rq)) + waltgov_run_callback(rq, WALT_CPUFREQ_PL); + trace_sched_enq_deq_task(p, 1, cpumask_bits(p->cpus_ptr)[0], is_mvp(wts)); } @@ -4310,20 +4314,6 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) rcu_read_unlock(); } -static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct *p) -{ - unsigned long flags; - int cpu = p->cpu; - - if (unlikely(walt_disabled)) - return; - - raw_spin_lock_irqsave(&cpu_rq(cpu)->__lock, flags); - if (do_pl_notif(cpu_rq(cpu))) - waltgov_run_callback(cpu_rq(cpu), WALT_CPUFREQ_PL); - raw_spin_unlock_irqrestore(&cpu_rq(cpu)->__lock, flags); -} - static u64 tick_sched_clock; static DECLARE_COMPLETION(tick_sched_clock_completion); @@ -4495,7 +4485,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_after_enqueue_task(android_rvh_enqueue_task, NULL); register_trace_android_rvh_after_dequeue_task(android_rvh_dequeue_task, NULL); register_trace_android_rvh_try_to_wake_up(android_rvh_try_to_wake_up, NULL); - register_trace_android_rvh_try_to_wake_up_success(android_rvh_try_to_wake_up_success, NULL); register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL); register_trace_android_vh_scheduler_tick(android_vh_scheduler_tick, NULL); register_trace_android_rvh_schedule(android_rvh_schedule, NULL); From 1d077fe1d49df14aae99c05c10ede0cbccfb2c38 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 24 Feb 2022 10:55:28 -0800 Subject: [PATCH 12/12] sched/walt: Early Detect Boost Optimizations Currently, if an ED task is detected on a cluster, the entire cluster's frequency is raised to fmax. However, since ED tasks are only classified under boosted scenarios anyways, this degree of boost may not be necessary. This patch introduces a tunable, sched_ed_boost, which can be used to boost the highest load on the cluster with an ED task by x%, x being the set tunable. Change-Id: I90ce6e7435b89ed27d307a164fdd7b8f6c8f5c9e Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 9 +++++++++ kernel/sched/walt/sysctl.c | 10 ++++++++++ kernel/sched/walt/walt.c | 11 ++++------- kernel/sched/walt/walt.h | 2 ++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 7de85593f288..f36038918132 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -296,8 +296,14 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut bool is_migration = wg_cpu->flags & WALT_CPUFREQ_IC_MIGRATION; bool is_rtg_boost = wg_cpu->walt_load.rtgb_active; bool is_hiload; + bool is_ed_boost = wg_cpu->walt_load.ed_active; unsigned long pl = wg_cpu->walt_load.pl; + if (is_ed_boost) { + cpu_util = mult_frac(cpu_util, 100 + sysctl_ed_boost_pct, 100); + max_and_reason(util, cpu_util, wg_cpu, CPUFREQ_REASON_EARLY_DET); + } + if (is_rtg_boost) max_and_reason(util, wg_policy->rtg_boost_util, wg_cpu, CPUFREQ_REASON_RTG_BOOST); @@ -316,6 +322,9 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut pl = mult_frac(pl, TARGET_LOAD, 100); max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL); } + + if (is_ed_boost) + wg_cpu->reasons |= CPUFREQ_REASON_EARLY_DET; } static inline unsigned long target_util(struct waltgov_policy *wg_policy, diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 310956da94e0..0e8439d33799 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -74,6 +74,7 @@ unsigned int sysctl_sched_asymcap_boost; unsigned int sysctl_sched_long_running_rt_task_ms; unsigned int sysctl_sched_idle_enough; unsigned int sysctl_sched_cluster_util_thres_pct; +unsigned int sysctl_ed_boost_pct; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -922,6 +923,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &two_thousand, }, + { + .procname = "sched_ed_boost", + .data = &sysctl_ed_boost_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred, + }, { } }; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 5665b7e32ffc..21f86b54a5df 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -584,12 +584,6 @@ static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason) u64 load, tt_load = 0, kload = 0; struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu_of(rq)); - if (wrq->ed_task != NULL) { - load = sched_ravg_window; - *reason = CPUFREQ_REASON_EARLY_DET; - goto done; - } - if (sched_freq_aggr_en) { load = wrq->prev_runnable_sum + aggr_grp_load; *reason = CPUFREQ_REASON_FREQ_AGR; @@ -621,7 +615,6 @@ static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason) *reason = CPUFREQ_REASON_SUH; } -done: trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en, load, 0, walt_rotation_enabled, sysctl_sched_user_hint, wrq, *reason); @@ -659,6 +652,10 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea walt_load->pl = pl; walt_load->ws = walt_load_reported_window; walt_load->rtgb_active = rtgb_active; + if (wrq->ed_task) + walt_load->ed_active = true; + else + walt_load->ed_active = false; } return (util >= capacity) ? capacity : util; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index bed7dd0e49fd..0e547b69a0c5 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -59,6 +59,7 @@ struct walt_cpu_load { unsigned long pl; bool rtgb_active; u64 ws; + bool ed_active; }; #define DECLARE_BITMAP_ARRAY(name, nr, bits) \ @@ -216,6 +217,7 @@ extern unsigned int sysctl_sched_user_hint; extern unsigned int sysctl_sched_conservative_pl; extern unsigned int sysctl_sched_hyst_min_coloc_ns; extern unsigned int sysctl_sched_long_running_rt_task_ms; +extern unsigned int sysctl_ed_boost_pct; #define WALT_MANY_WAKEUP_DEFAULT 1000 extern unsigned int sysctl_sched_many_wakeup_threshold;