From e96b1cfd3c671fd68d159ae44f658e87b316bc30 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:38:47 -0700 Subject: [PATCH 001/346] sched/walt: Introduce WALT as a module WALT improves the scheduler performance from a power/perf and thermal perspective. Bring it in as a module. Change-Id: Ibeb6c0480796e8d8fcd81e1bdda7a117ae02c980 Signed-off-by: Shaleen Agrawal Signed-off-by: Sai Harshini Nimmala Signed-off-by: Sai Harshini Nimmala --- Kconfig | 2 + kernel/sched/Makefile | 2 + kernel/sched/walt/Kconfig | 31 + kernel/sched/walt/Makefile | 10 + kernel/sched/walt/boost.c | 301 ++ kernel/sched/walt/core_ctl.c | 1307 +++++++++ kernel/sched/walt/cpufreq_walt.c | 886 ++++++ kernel/sched/walt/fixup.c | 91 + kernel/sched/walt/input-boost.c | 300 ++ kernel/sched/walt/preemptirq_long.c | 177 ++ kernel/sched/walt/preemptirq_long.h | 45 + kernel/sched/walt/qc_vas.c | 52 + kernel/sched/walt/sched_avg.c | 250 ++ kernel/sched/walt/sysctl.c | 900 ++++++ kernel/sched/walt/trace.c | 84 + kernel/sched/walt/trace.h | 1097 +++++++ kernel/sched/walt/walt.c | 4136 +++++++++++++++++++++++++++ kernel/sched/walt/walt.h | 1006 +++++++ kernel/sched/walt/walt_cfs.c | 785 +++++ kernel/sched/walt/walt_debug.c | 50 + kernel/sched/walt/walt_debug.h | 5 + kernel/sched/walt/walt_lb.c | 742 +++++ kernel/sched/walt/walt_rt.c | 90 + 23 files changed, 12349 insertions(+) create mode 100644 kernel/sched/walt/Kconfig create mode 100644 kernel/sched/walt/Makefile create mode 100644 kernel/sched/walt/boost.c create mode 100644 kernel/sched/walt/core_ctl.c create mode 100644 kernel/sched/walt/cpufreq_walt.c create mode 100644 kernel/sched/walt/fixup.c create mode 100644 kernel/sched/walt/input-boost.c create mode 100644 kernel/sched/walt/preemptirq_long.c create mode 100644 kernel/sched/walt/preemptirq_long.h create mode 100644 kernel/sched/walt/qc_vas.c create mode 100644 kernel/sched/walt/sched_avg.c create mode 100644 kernel/sched/walt/sysctl.c create mode 100644 kernel/sched/walt/trace.c create mode 100644 kernel/sched/walt/trace.h create mode 100644 kernel/sched/walt/walt.c create mode 100644 kernel/sched/walt/walt.h create mode 100644 kernel/sched/walt/walt_cfs.c create mode 100644 kernel/sched/walt/walt_debug.c create mode 100644 kernel/sched/walt/walt_debug.h create mode 100644 kernel/sched/walt/walt_lb.c create mode 100644 kernel/sched/walt/walt_rt.c diff --git a/Kconfig b/Kconfig index 57a142d8d8b4..2066ad68dd4f 100644 --- a/Kconfig +++ b/Kconfig @@ -33,3 +33,5 @@ source "Documentation/Kconfig" # ANDROID: Set KCONFIG_EXT_PREFIX to decend into an external project. source "$(KCONFIG_EXT_PREFIX)Kconfig.ext" + +source "kernel/sched/walt/Kconfig" diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 976092b7bd45..4b73828ecee6 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -32,3 +32,5 @@ obj-y += core.o obj-y += fair.o obj-y += build_policy.o obj-y += build_utility.o + +obj-$(CONFIG_SCHED_WALT) += walt/ diff --git a/kernel/sched/walt/Kconfig b/kernel/sched/walt/Kconfig new file mode 100644 index 000000000000..db60efdf4c8d --- /dev/null +++ b/kernel/sched/walt/Kconfig @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# QTI WALT based scheduler +# +menu "QTI WALT based scheduler features" + +config SCHED_WALT + tristate "Support window based load tracking" + depends on SMP + help + This feature will allow the scheduler to maintain a tunable window + based set of metrics for tasks and runqueues. These metrics can be + used to guide task placement as well as task frequency requirements + for cpufreq governors. + +config SCHED_WALT_DEBUG + tristate "WALT debug module" + select TRACE_PREEMPT_TOGGLE + select TRACE_IRQFLAGS + help + This module provides the means of debugging long preempt and + irq disable code. This helps in identifying the scheduling + latencies. The module rely on preemptirq trace hooks and + print the stacktrace to the ftrace upon long preempt and irq + events. Sysctl knobs are available for the user to configure + the thresholds. + + This module also used to crash the system to catch issues + in scenarios like RT throttling and sleeping while in atomic + context etc. +endmenu diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile new file mode 100644 index 000000000000..34987fd075d2 --- /dev/null +++ b/kernel/sched/walt/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only + +KCOV_INSTRUMENT := n +KCSAN_SANITIZE := n + +obj-$(CONFIG_SCHED_WALT) += sched-walt.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o qc_vas.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o + +obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o +sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c new file mode 100644 index 000000000000..17305aa10029 --- /dev/null +++ b/kernel/sched/walt/boost.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include + +#include "walt.h" +#include "trace.h" + +/* + * Scheduler boost is a mechanism to temporarily place tasks on CPUs + * with higher capacity than those where a task would have normally + * ended up with their load characteristics. Any entity enabling + * boost is responsible for disabling it as well. + */ + +static enum sched_boost_policy boost_policy_dt = SCHED_BOOST_NONE; +static DEFINE_MUTEX(boost_mutex); + +struct task_group *task_group_topapp; +struct task_group *task_group_foreground; + +void walt_init_tg(struct task_group *tg) +{ + struct walt_task_group *wtg; + + wtg = (struct walt_task_group *) tg->android_vendor_data1; + + wtg->colocate = false; + wtg->sched_boost_enable[NO_BOOST] = false; + wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true; + wtg->sched_boost_enable[CONSERVATIVE_BOOST] = false; + wtg->sched_boost_enable[RESTRAINED_BOOST] = false; +} + +void walt_init_topapp_tg(struct task_group *tg) +{ + struct walt_task_group *wtg; + + wtg = (struct walt_task_group *) tg->android_vendor_data1; + + wtg->colocate = true; + wtg->sched_boost_enable[NO_BOOST] = false; + wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true; + wtg->sched_boost_enable[CONSERVATIVE_BOOST] = true; + wtg->sched_boost_enable[RESTRAINED_BOOST] = false; +} + +void walt_init_foreground_tg(struct task_group *tg) +{ + struct walt_task_group *wtg; + + wtg = (struct walt_task_group *) tg->android_vendor_data1; + + wtg->colocate = false; + wtg->sched_boost_enable[NO_BOOST] = false; + wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true; + wtg->sched_boost_enable[CONSERVATIVE_BOOST] = true; + wtg->sched_boost_enable[RESTRAINED_BOOST] = false; +} + +/* + * Scheduler boost type and boost policy might at first seem unrelated, + * however, there exists a connection between them that will allow us + * to use them interchangeably during placement decisions. We'll explain + * the connection here in one possible way so that the implications are + * clear when looking at placement policies. + * + * When policy = SCHED_BOOST_NONE, type is either none or RESTRAINED + * When policy = SCHED_BOOST_ON_ALL or SCHED_BOOST_ON_BIG, type can + * neither be none nor RESTRAINED. + */ +static void set_boost_policy(int type) +{ + if (type == NO_BOOST || type == RESTRAINED_BOOST) { + boost_policy = SCHED_BOOST_NONE; + return; + } + + if (boost_policy_dt) { + boost_policy = boost_policy_dt; + return; + } + + if (hmp_capable()) { + boost_policy = SCHED_BOOST_ON_BIG; + return; + } + + boost_policy = SCHED_BOOST_ON_ALL; +} + +static bool verify_boost_params(int type) +{ + return type >= RESTRAINED_BOOST_DISABLE && type <= RESTRAINED_BOOST; +} + +static void sched_no_boost_nop(void) +{ +} + +static void sched_full_throttle_boost_enter(void) +{ + core_ctl_set_boost(true); + walt_enable_frequency_aggregation(true); +} + +static void sched_full_throttle_boost_exit(void) +{ + core_ctl_set_boost(false); + walt_enable_frequency_aggregation(false); +} + +static void sched_conservative_boost_enter(void) +{ +} + +static void sched_conservative_boost_exit(void) +{ +} + +static void sched_restrained_boost_enter(void) +{ + walt_enable_frequency_aggregation(true); +} + +static void sched_restrained_boost_exit(void) +{ + walt_enable_frequency_aggregation(false); +} + +struct sched_boost_data { + int refcount; + void (*enter)(void); + void (*exit)(void); +}; + +static struct sched_boost_data sched_boosts[] = { + [NO_BOOST] = { + .refcount = 0, + .enter = sched_no_boost_nop, + .exit = sched_no_boost_nop, + }, + [FULL_THROTTLE_BOOST] = { + .refcount = 0, + .enter = sched_full_throttle_boost_enter, + .exit = sched_full_throttle_boost_exit, + }, + [CONSERVATIVE_BOOST] = { + .refcount = 0, + .enter = sched_conservative_boost_enter, + .exit = sched_conservative_boost_exit, + }, + [RESTRAINED_BOOST] = { + .refcount = 0, + .enter = sched_restrained_boost_enter, + .exit = sched_restrained_boost_exit, + }, +}; + +#define SCHED_BOOST_START FULL_THROTTLE_BOOST +#define SCHED_BOOST_END (RESTRAINED_BOOST + 1) + +static int sched_effective_boost(void) +{ + int i; + + /* + * The boosts are sorted in descending order by + * priority. + */ + for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) { + if (sched_boosts[i].refcount >= 1) + return i; + } + + return NO_BOOST; +} + +static void sched_boost_disable(int type) +{ + struct sched_boost_data *sb = &sched_boosts[type]; + int next_boost; + + if (sb->refcount <= 0) + return; + + sb->refcount--; + + if (sb->refcount) + return; + + /* + * This boost's refcount becomes zero, so it must + * be disabled. Disable it first and then apply + * the next boost. + */ + sb->exit(); + + next_boost = sched_effective_boost(); + sched_boosts[next_boost].enter(); +} + +static void sched_boost_enable(int type) +{ + struct sched_boost_data *sb = &sched_boosts[type]; + int next_boost, prev_boost = sched_boost_type; + + sb->refcount++; + + if (sb->refcount != 1) + return; + + /* + * This boost enable request did not come before. + * Take this new request and find the next boost + * by aggregating all the enabled boosts. If there + * is a change, disable the previous boost and enable + * the next boost. + */ + + next_boost = sched_effective_boost(); + if (next_boost == prev_boost) + return; + + sched_boosts[prev_boost].exit(); + sched_boosts[next_boost].enter(); +} + +static void sched_boost_disable_all(void) +{ + int i; + + for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) { + if (sched_boosts[i].refcount > 0) { + sched_boosts[i].exit(); + sched_boosts[i].refcount = 0; + } + } +} + +static void _sched_set_boost(int type) +{ + if (type == 0) + sched_boost_disable_all(); + else if (type > 0) + sched_boost_enable(type); + else + sched_boost_disable(-type); + + /* + * sysctl_sched_boost holds the boost request from + * user space which could be different from the + * effectively enabled boost. Update the effective + * boost here. + */ + + sched_boost_type = sched_effective_boost(); + sysctl_sched_boost = sched_boost_type; + set_boost_policy(sysctl_sched_boost); + trace_sched_set_boost(sysctl_sched_boost); +} + +int sched_set_boost(int type) +{ + int ret = 0; + + mutex_lock(&boost_mutex); + if (verify_boost_params(type)) + _sched_set_boost(type); + else + ret = -EINVAL; + mutex_unlock(&boost_mutex); + return ret; +} + +int sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + unsigned int *data = (unsigned int *)table->data; + + mutex_lock(&boost_mutex); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (ret || !write) + goto done; + + if (verify_boost_params(*data)) + _sched_set_boost(*data); + else + ret = -EINVAL; + +done: + mutex_unlock(&boost_mutex); + return ret; +} diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c new file mode 100644 index 000000000000..1330e07b04f7 --- /dev/null +++ b/kernel/sched/walt/core_ctl.c @@ -0,0 +1,1307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014-2021, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) "core_ctl: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "walt.h" +#include "trace.h" + +struct cluster_data { + bool inited; + unsigned int min_cpus; + unsigned int max_cpus; + unsigned int offline_delay_ms; + unsigned int busy_up_thres[MAX_CPUS_PER_CLUSTER]; + unsigned int busy_down_thres[MAX_CPUS_PER_CLUSTER]; + unsigned int active_cpus; + unsigned int num_cpus; + unsigned int nr_paused_cpus; + unsigned int nr_not_preferred_cpus; + cpumask_t cpu_mask; + unsigned int need_cpus; + unsigned int task_thres; + unsigned int max_nr; + unsigned int nr_prev_assist; + unsigned int nr_prev_assist_thresh; + s64 need_ts; + struct list_head lru; + bool enable; + int nrrun; + unsigned int first_cpu; + unsigned int boost; + struct kobject kobj; + unsigned int strict_nrrun; +}; + +struct cpu_data { + bool is_busy; + unsigned int busy; + unsigned int cpu; + bool not_preferred; + struct cluster_data *cluster; + struct list_head sib; + bool paused_by_us; +}; + +static DEFINE_PER_CPU(struct cpu_data, cpu_state); +static struct cluster_data cluster_state[MAX_CLUSTERS]; +static unsigned int num_clusters; + +#define for_each_cluster(cluster, idx) \ + for (; (idx) < num_clusters && ((cluster) = &cluster_state[idx]);\ + idx++) + +/* single core_ctl thread for all pause/unpause core_ctl operations */ +struct task_struct *core_ctl_thread; + +/* single lock per single thread for core_ctl + * protects core_ctl_pending flag + */ +spinlock_t core_ctl_pending_lock; +bool core_ctl_pending; + +static DEFINE_SPINLOCK(state_lock); +static void apply_need(struct cluster_data *state); +static void wake_up_core_ctl_thread(void); +static bool initialized; + +ATOMIC_NOTIFIER_HEAD(core_ctl_notifier); +static unsigned int last_nr_big; + +static unsigned int get_active_cpu_count(const struct cluster_data *cluster); + +/* ========================= sysfs interface =========================== */ + +static ssize_t store_min_cpus(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + state->min_cpus = min(val, state->num_cpus); + apply_need(state); + + return count; +} + +static ssize_t show_min_cpus(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->min_cpus); +} + +static ssize_t store_max_cpus(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + state->max_cpus = min(val, state->num_cpus); + apply_need(state); + + return count; +} + +static ssize_t show_max_cpus(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->max_cpus); +} + +static ssize_t store_offline_delay_ms(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + state->offline_delay_ms = val; + apply_need(state); + + return count; +} + +static ssize_t show_task_thres(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->task_thres); +} + +static ssize_t store_task_thres(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + if (val < state->num_cpus) + return -EINVAL; + + state->task_thres = val; + apply_need(state); + + return count; +} + +static ssize_t show_nr_prev_assist_thresh(const struct cluster_data *state, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_prev_assist_thresh); +} + +static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + state->nr_prev_assist_thresh = val; + apply_need(state); + + return count; +} + +static ssize_t show_offline_delay_ms(const struct cluster_data *state, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->offline_delay_ms); +} + +static ssize_t store_busy_up_thres(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val[MAX_CPUS_PER_CLUSTER]; + int ret, i; + + ret = sscanf(buf, "%u %u %u %u %u %u\n", + &val[0], &val[1], &val[2], &val[3], + &val[4], &val[5]); + if (ret != 1 && ret != state->num_cpus) + return -EINVAL; + + if (ret == 1) { + for (i = 0; i < state->num_cpus; i++) + state->busy_up_thres[i] = val[0]; + } else { + for (i = 0; i < state->num_cpus; i++) + state->busy_up_thres[i] = val[i]; + } + apply_need(state); + return count; +} + +static ssize_t show_busy_up_thres(const struct cluster_data *state, char *buf) +{ + int i, count = 0; + + for (i = 0; i < state->num_cpus; i++) + count += scnprintf(buf + count, PAGE_SIZE - count, "%u ", + state->busy_up_thres[i]); + + count += scnprintf(buf + count, PAGE_SIZE - count, "\n"); + return count; +} + +static ssize_t store_busy_down_thres(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val[MAX_CPUS_PER_CLUSTER]; + int ret, i; + + ret = sscanf(buf, "%u %u %u %u %u %u\n", + &val[0], &val[1], &val[2], &val[3], + &val[4], &val[5]); + if (ret != 1 && ret != state->num_cpus) + return -EINVAL; + + if (ret == 1) { + for (i = 0; i < state->num_cpus; i++) + state->busy_down_thres[i] = val[0]; + } else { + for (i = 0; i < state->num_cpus; i++) + state->busy_down_thres[i] = val[i]; + } + apply_need(state); + return count; +} + +static ssize_t show_busy_down_thres(const struct cluster_data *state, char *buf) +{ + int i, count = 0; + + for (i = 0; i < state->num_cpus; i++) + count += scnprintf(buf + count, PAGE_SIZE - count, "%u ", + state->busy_down_thres[i]); + + count += scnprintf(buf + count, PAGE_SIZE - count, "\n"); + return count; +} + +static ssize_t store_enable(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + bool bval; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + bval = !!val; + if (bval != state->enable) { + state->enable = bval; + apply_need(state); + } + + return count; +} + +static ssize_t show_enable(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->enable); +} + +static ssize_t show_need_cpus(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->need_cpus); +} + +static ssize_t show_active_cpus(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->active_cpus); +} + +static ssize_t show_global_state(const struct cluster_data *state, char *buf) +{ + struct cpu_data *c; + struct cluster_data *cluster; + ssize_t count = 0; + unsigned int cpu; + + spin_lock_irq(&state_lock); + for_each_possible_cpu(cpu) { + c = &per_cpu(cpu_state, cpu); + cluster = c->cluster; + if (!cluster || !cluster->inited) + continue; + + count += scnprintf(buf + count, PAGE_SIZE - count, + "CPU%u\n", cpu); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tCPU: %u\n", c->cpu); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tOnline: %u\n", + cpu_online(c->cpu)); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tPaused: %u\n", + !cpu_active(c->cpu)); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tFirst CPU: %u\n", + cluster->first_cpu); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tBusy%%: %u\n", c->busy); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tIs busy: %u\n", c->is_busy); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tNot preferred: %u\n", + c->not_preferred); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tNr running: %u\n", cluster->nrrun); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tActive CPUs: %u\n", get_active_cpu_count(cluster)); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tNeed CPUs: %u\n", cluster->need_cpus); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tNr paused CPUs: %u\n", + cluster->nr_paused_cpus); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tBoost: %u\n", (unsigned int) cluster->boost); + } + spin_unlock_irq(&state_lock); + + return count; +} + +static ssize_t store_not_preferred(struct cluster_data *state, + const char *buf, size_t count) +{ + struct cpu_data *c; + unsigned int i; + unsigned int val[MAX_CPUS_PER_CLUSTER]; + unsigned long flags; + int ret; + int not_preferred_count = 0; + + ret = sscanf(buf, "%u %u %u %u %u %u\n", + &val[0], &val[1], &val[2], &val[3], + &val[4], &val[5]); + if (ret != state->num_cpus) + return -EINVAL; + + spin_lock_irqsave(&state_lock, flags); + for (i = 0; i < state->num_cpus; i++) { + c = &per_cpu(cpu_state, i + state->first_cpu); + c->not_preferred = val[i]; + not_preferred_count += !!val[i]; + } + state->nr_not_preferred_cpus = not_preferred_count; + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_not_preferred(const struct cluster_data *state, char *buf) +{ + struct cpu_data *c; + ssize_t count = 0; + unsigned long flags; + int i; + + spin_lock_irqsave(&state_lock, flags); + for (i = 0; i < state->num_cpus; i++) { + c = &per_cpu(cpu_state, i + state->first_cpu); + count += scnprintf(buf + count, PAGE_SIZE - count, + "CPU#%d: %u\n", c->cpu, c->not_preferred); + } + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +struct core_ctl_attr { + struct attribute attr; + ssize_t (*show)(const struct cluster_data *cd, char *c); + ssize_t (*store)(struct cluster_data *cd, const char *c, + size_t count); +}; + +#define core_ctl_attr_ro(_name) \ +static struct core_ctl_attr _name = \ +__ATTR(_name, 0444, show_##_name, NULL) + +#define core_ctl_attr_rw(_name) \ +static struct core_ctl_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) + +core_ctl_attr_rw(min_cpus); +core_ctl_attr_rw(max_cpus); +core_ctl_attr_rw(offline_delay_ms); +core_ctl_attr_rw(busy_up_thres); +core_ctl_attr_rw(busy_down_thres); +core_ctl_attr_rw(task_thres); +core_ctl_attr_rw(nr_prev_assist_thresh); +core_ctl_attr_ro(need_cpus); +core_ctl_attr_ro(active_cpus); +core_ctl_attr_ro(global_state); +core_ctl_attr_rw(not_preferred); +core_ctl_attr_rw(enable); + +static struct attribute *default_attrs[] = { + &min_cpus.attr, + &max_cpus.attr, + &offline_delay_ms.attr, + &busy_up_thres.attr, + &busy_down_thres.attr, + &task_thres.attr, + &nr_prev_assist_thresh.attr, + &enable.attr, + &need_cpus.attr, + &active_cpus.attr, + &global_state.attr, + ¬_preferred.attr, + NULL +}; + +#define to_cluster_data(k) container_of(k, struct cluster_data, kobj) +#define to_attr(a) container_of(a, struct core_ctl_attr, attr) +static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct cluster_data *data = to_cluster_data(kobj); + struct core_ctl_attr *cattr = to_attr(attr); + ssize_t ret = -EIO; + + if (cattr->show) + ret = cattr->show(data, buf); + + return ret; +} + +static ssize_t store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct cluster_data *data = to_cluster_data(kobj); + struct core_ctl_attr *cattr = to_attr(attr); + ssize_t ret = -EIO; + + if (cattr->store) + ret = cattr->store(data, buf, count); + + return ret; +} + +static const struct sysfs_ops sysfs_ops = { + .show = show, + .store = store, +}; + +static struct kobj_type ktype_core_ctl = { + .sysfs_ops = &sysfs_ops, + .default_attrs = default_attrs, +}; + +/* ==================== runqueue based core count =================== */ + +static struct sched_avg_stats nr_stats[WALT_NR_CPUS]; + +/* + * nr_need: + * Number of tasks running on this cluster plus + * tasks running on higher capacity clusters. + * To find out CPUs needed from this cluster. + * + * For example: + * On dual cluster system with 4 min capacity + * CPUs and 4 max capacity CPUs, if there are + * 4 small tasks running on min capacity CPUs + * and 2 big tasks running on 2 max capacity + * CPUs, nr_need has to be 6 for min capacity + * cluster and 2 for max capacity cluster. + * This is because, min capacity cluster has to + * account for tasks running on max capacity + * cluster, so that, the min capacity cluster + * can be ready to accommodate tasks running on max + * capacity CPUs if the demand of tasks goes down. + */ +static int compute_cluster_nr_need(int index) +{ + int cpu; + struct cluster_data *cluster; + int nr_need = 0; + + for_each_cluster(cluster, index) { + for_each_cpu(cpu, &cluster->cpu_mask) + nr_need += nr_stats[cpu].nr; + } + + return nr_need; +} + +/* + * prev_misfit_need: + * Tasks running on smaller capacity cluster which + * needs to be migrated to higher capacity cluster. + * To find out how many tasks need higher capacity CPUs. + * + * For example: + * On dual cluster system with 4 min capacity + * CPUs and 4 max capacity CPUs, if there are + * 2 small tasks and 2 big tasks running on + * min capacity CPUs and no tasks running on + * max cpacity, prev_misfit_need of min capacity + * cluster will be 0 and prev_misfit_need of + * max capacity cluster will be 2. + */ +static int compute_prev_cluster_misfit_need(int index) +{ + int cpu; + struct cluster_data *prev_cluster; + int prev_misfit_need = 0; + + /* + * Lowest capacity cluster does not have to + * accommodate any misfit tasks. + */ + if (index == 0) + return 0; + + prev_cluster = &cluster_state[index - 1]; + + for_each_cpu(cpu, &prev_cluster->cpu_mask) + prev_misfit_need += nr_stats[cpu].nr_misfit; + + return prev_misfit_need; +} + +static int compute_cluster_max_nr(int index) +{ + int cpu; + struct cluster_data *cluster = &cluster_state[index]; + int max_nr = 0; + + for_each_cpu(cpu, &cluster->cpu_mask) + max_nr = max(max_nr, nr_stats[cpu].nr_max); + + return max_nr; +} + +static int cluster_real_big_tasks(int index) +{ + int nr_big = 0; + int cpu; + struct cluster_data *cluster = &cluster_state[index]; + + if (index == 0) { + for_each_cpu(cpu, &cluster->cpu_mask) + nr_big += nr_stats[cpu].nr_misfit; + } else { + for_each_cpu(cpu, &cluster->cpu_mask) + nr_big += nr_stats[cpu].nr; + } + + return nr_big; +} + +/* + * prev_nr_need_assist: + * Tasks that are eligible to run on the previous + * cluster but cannot run because of insufficient + * CPUs there. prev_nr_need_assist is indicative + * of number of CPUs in this cluster that should + * assist its previous cluster to makeup for + * insufficient CPUs there. + * + * For example: + * On tri-cluster system with 4 min capacity + * CPUs, 3 intermediate capacity CPUs and 1 + * max capacity CPU, if there are 4 small + * tasks running on min capacity CPUs, 4 big + * tasks running on intermediate capacity CPUs + * and no tasks running on max capacity CPU, + * prev_nr_need_assist for min & max capacity + * clusters will be 0, but, for intermediate + * capacity cluster prev_nr_need_assist will + * be 1 as it has 3 CPUs, but, there are 4 big + * tasks to be served. + */ +static int prev_cluster_nr_need_assist(int index) +{ + int need = 0; + int cpu; + struct cluster_data *prev_cluster; + + if (index == 0) + return 0; + + index--; + prev_cluster = &cluster_state[index]; + + /* + * Next cluster should not assist, while there are paused cpus + * in this cluster. + */ + if (prev_cluster->nr_paused_cpus) + return 0; + + for_each_cpu(cpu, &prev_cluster->cpu_mask) + need += nr_stats[cpu].nr; + + need += compute_prev_cluster_misfit_need(index); + + if (need > prev_cluster->active_cpus) + need = need - prev_cluster->active_cpus; + else + need = 0; + + return need; +} + +/* + * This is only implemented for min capacity cluster. + * + * Bringing a little CPU out of pause and using it + * more does not hurt power as much as bringing big CPUs. + * + * little cluster provides help needed for the other clusters. + * we take nr_scaled (which gives better resolution) and find + * the total nr in the system. Then take out the active higher + * capacity CPUs from the nr and consider the remaining nr as + * strict and consider that many little CPUs are needed. + */ +static int compute_cluster_nr_strict_need(int index) +{ + int cpu; + struct cluster_data *cluster; + int nr_strict_need = 0; + + if (index != 0) + return 0; + + for_each_cluster(cluster, index) { + int nr_scaled = 0; + int active_cpus = cluster->active_cpus; + + for_each_cpu(cpu, &cluster->cpu_mask) + nr_scaled += nr_stats[cpu].nr_scaled; + + nr_scaled /= 100; + + /* + * For little cluster, nr_scaled becomes the nr_strict, + * for other cluster, overflow is counted towards + * the little cluster need. + */ + if (index == 0) + nr_strict_need += nr_scaled; + else + nr_strict_need += max(0, nr_scaled - active_cpus); + } + + return nr_strict_need; +} +static void update_running_avg(void) +{ + struct cluster_data *cluster; + unsigned int index = 0; + unsigned long flags; + int big_avg = 0; + + sched_get_nr_running_avg(nr_stats); + + spin_lock_irqsave(&state_lock, flags); + for_each_cluster(cluster, index) { + int nr_need, prev_misfit_need; + + if (!cluster->inited) + continue; + + nr_need = compute_cluster_nr_need(index); + prev_misfit_need = compute_prev_cluster_misfit_need(index); + + cluster->nrrun = nr_need + prev_misfit_need; + cluster->max_nr = compute_cluster_max_nr(index); + cluster->nr_prev_assist = prev_cluster_nr_need_assist(index); + + cluster->strict_nrrun = compute_cluster_nr_strict_need(index); + + trace_core_ctl_update_nr_need(cluster->first_cpu, nr_need, + prev_misfit_need, + cluster->nrrun, cluster->max_nr, + cluster->nr_prev_assist); + + big_avg += cluster_real_big_tasks(index); + } + spin_unlock_irqrestore(&state_lock, flags); + + last_nr_big = big_avg; + walt_rotation_checkpoint(big_avg); +} + +#define MAX_NR_THRESHOLD 4 +/* adjust needed CPUs based on current runqueue information */ +static unsigned int apply_task_need(const struct cluster_data *cluster, + unsigned int new_need) +{ + /* resume all cores if there are enough tasks */ + if (cluster->nrrun >= cluster->task_thres) + return cluster->num_cpus; + + /* + * resume as many cores as the previous cluster + * needs assistance with. + */ + if (cluster->nr_prev_assist >= cluster->nr_prev_assist_thresh) + new_need = new_need + cluster->nr_prev_assist; + + /* only resume more cores if there are tasks to run */ + if (cluster->nrrun > new_need) + new_need = new_need + 1; + + /* + * We don't want tasks to be overcrowded in a cluster. + * If any CPU has more than MAX_NR_THRESHOLD in the last + * window, bring another CPU to help out. + */ + if (cluster->max_nr > MAX_NR_THRESHOLD) + new_need = new_need + 1; + + /* + * For little cluster, we use a bit more relaxed approach + * and impose the strict nr condition. Because all tasks can + * spill onto little if big cluster is crowded. + */ + if (new_need < cluster->strict_nrrun) + new_need = cluster->strict_nrrun; + + return new_need; +} + +/* ======================= load based core count ====================== */ + +static unsigned int apply_limits(const struct cluster_data *cluster, + unsigned int need_cpus) +{ + return min(max(cluster->min_cpus, need_cpus), cluster->max_cpus); +} + +static unsigned int get_active_cpu_count(const struct cluster_data *cluster) +{ + return cluster->num_cpus - + sched_pause_count(&cluster->cpu_mask, true); +} + +static bool is_active(const struct cpu_data *state) +{ + return cpu_online(state->cpu) && cpu_active(state->cpu); +} + +static bool adjustment_possible(const struct cluster_data *cluster, + unsigned int need) +{ + return (need < cluster->active_cpus || (need > cluster->active_cpus && + cluster->nr_paused_cpus)); +} + +static bool need_all_cpus(const struct cluster_data *cluster) +{ + return (is_min_capacity_cpu(cluster->first_cpu) && + sched_ravg_window < DEFAULT_SCHED_RAVG_WINDOW); +} + +static bool eval_need(struct cluster_data *cluster) +{ + unsigned long flags; + struct cpu_data *c; + unsigned int need_cpus = 0, last_need, thres_idx; + int ret = 0; + bool need_flag = false; + unsigned int new_need; + s64 now, elapsed; + + if (unlikely(!cluster->inited)) + return false; + + spin_lock_irqsave(&state_lock, flags); + + if (cluster->boost || !cluster->enable || need_all_cpus(cluster)) { + need_cpus = cluster->max_cpus; + } else { + cluster->active_cpus = get_active_cpu_count(cluster); + thres_idx = cluster->active_cpus ? cluster->active_cpus - 1 : 0; + list_for_each_entry(c, &cluster->lru, sib) { + bool old_is_busy = c->is_busy; + + if (c->busy >= cluster->busy_up_thres[thres_idx] || + sched_cpu_high_irqload(c->cpu)) + c->is_busy = true; + else if (c->busy < cluster->busy_down_thres[thres_idx]) + c->is_busy = false; + + trace_core_ctl_set_busy(c->cpu, c->busy, old_is_busy, + c->is_busy); + need_cpus += c->is_busy; + } + need_cpus = apply_task_need(cluster, need_cpus); + } + new_need = apply_limits(cluster, need_cpus); + need_flag = adjustment_possible(cluster, new_need); + + last_need = cluster->need_cpus; + now = ktime_to_ms(ktime_get()); + + if (new_need > cluster->active_cpus) { + ret = 1; + } else { + /* + * When there is no change in need and there are no more + * active CPUs than currently needed, just update the + * need time stamp and return. + */ + if (new_need == last_need && new_need == cluster->active_cpus) { + cluster->need_ts = now; + spin_unlock_irqrestore(&state_lock, flags); + return false; + } + + elapsed = now - cluster->need_ts; + ret = elapsed >= cluster->offline_delay_ms; + } + + if (ret) { + cluster->need_ts = now; + cluster->need_cpus = new_need; + } + trace_core_ctl_eval_need(cluster->first_cpu, last_need, new_need, + ret && need_flag); + spin_unlock_irqrestore(&state_lock, flags); + + return ret && need_flag; +} + +static void apply_need(struct cluster_data *cluster) +{ + if (eval_need(cluster)) + wake_up_core_ctl_thread(); +} + +/* ========================= core count enforcement ==================== */ + +static void wake_up_core_ctl_thread(void) +{ + unsigned long flags; + + spin_lock_irqsave(&core_ctl_pending_lock, flags); + core_ctl_pending = true; + spin_unlock_irqrestore(&core_ctl_pending_lock, flags); + + wake_up_process(core_ctl_thread); +} + +static u64 core_ctl_check_timestamp; + +int core_ctl_set_boost(bool boost) +{ + unsigned int index = 0; + struct cluster_data *cluster = NULL; + unsigned long flags; + int ret = 0; + bool boost_state_changed = false; + + if (unlikely(!initialized)) + return 0; + + spin_lock_irqsave(&state_lock, flags); + for_each_cluster(cluster, index) { + if (boost) { + boost_state_changed = !cluster->boost; + ++cluster->boost; + } else { + if (!cluster->boost) { + ret = -EINVAL; + break; + } + --cluster->boost; + boost_state_changed = !cluster->boost; + } + } + spin_unlock_irqrestore(&state_lock, flags); + + if (boost_state_changed) { + index = 0; + for_each_cluster(cluster, index) + apply_need(cluster); + } + + if (cluster) + trace_core_ctl_set_boost(cluster->boost, ret); + + return ret; +} +EXPORT_SYMBOL(core_ctl_set_boost); + +void core_ctl_notifier_register(struct notifier_block *n) +{ + atomic_notifier_chain_register(&core_ctl_notifier, n); +} + +void core_ctl_notifier_unregister(struct notifier_block *n) +{ + atomic_notifier_chain_unregister(&core_ctl_notifier, n); +} + +static void core_ctl_call_notifier(void) +{ + struct core_ctl_notif_data ndata = {0}; + struct notifier_block *nb; + + /* + * Don't bother querying the stats when the notifier + * chain is empty. + */ + rcu_read_lock(); + nb = rcu_dereference_raw(core_ctl_notifier.head); + rcu_read_unlock(); + + if (!nb) + return; + + ndata.nr_big = last_nr_big; + walt_fill_ta_data(&ndata); + trace_core_ctl_notif_data(ndata.nr_big, ndata.coloc_load_pct, + ndata.ta_util_pct, ndata.cur_cap_pct); + + atomic_notifier_call_chain(&core_ctl_notifier, 0, &ndata); +} + +void core_ctl_check(u64 window_start) +{ + int cpu; + struct cpu_data *c; + struct cluster_data *cluster; + unsigned int index = 0; + unsigned long flags; + + if (unlikely(!initialized)) + return; + + if (window_start == core_ctl_check_timestamp) + return; + + core_ctl_check_timestamp = window_start; + + spin_lock_irqsave(&state_lock, flags); + for_each_possible_cpu(cpu) { + + c = &per_cpu(cpu_state, cpu); + cluster = c->cluster; + + if (!cluster || !cluster->inited) + continue; + + c->busy = sched_get_cpu_util(cpu); + } + spin_unlock_irqrestore(&state_lock, flags); + + update_running_avg(); + + for_each_cluster(cluster, index) { + if (eval_need(cluster)) + wake_up_core_ctl_thread(); + } + + core_ctl_call_notifier(); +} + +static void move_cpu_lru(struct cpu_data *cpu_data) +{ + unsigned long flags; + + spin_lock_irqsave(&state_lock, flags); + list_del(&cpu_data->sib); + list_add_tail(&cpu_data->sib, &cpu_data->cluster->lru); + spin_unlock_irqrestore(&state_lock, flags); +} + +static bool should_we_pause(int cpu, struct cluster_data *cluster) +{ + return true; +} + +static void try_to_pause(struct cluster_data *cluster, unsigned int need, + struct cpumask *pause_cpus) +{ + struct cpu_data *c, *tmp; + unsigned long flags; + unsigned int num_cpus = cluster->num_cpus; + unsigned int nr_paused = 0; + bool first_pass = cluster->nr_not_preferred_cpus; + + /* + * Protect against entry being removed (and added at tail) by other + * thread (hotplug). + */ + spin_lock_irqsave(&state_lock, flags); + list_for_each_entry_safe(c, tmp, &cluster->lru, sib) { + if (!num_cpus--) + break; + + if (!is_active(c)) + continue; + if (cluster->active_cpus == need) + break; + /* Don't pause busy CPUs. */ + if (c->is_busy) + continue; + /* + * We pause only the not_preferred CPUs. If none + * of the CPUs are selected as not_preferred, then + * all CPUs are eligible for pausing. + */ + if (cluster->nr_not_preferred_cpus && !c->not_preferred) + continue; + + if (!should_we_pause(c->cpu, cluster)) + continue; + + spin_unlock_irqrestore(&state_lock, flags); + + pr_debug("Trying to pause CPU%u\n", c->cpu); + + cpumask_set_cpu(c->cpu, pause_cpus); + sched_pause_pending(c->cpu); + + c->paused_by_us = true; + move_cpu_lru(c); + nr_paused++; + + cluster->active_cpus = get_active_cpu_count(cluster); + spin_lock_irqsave(&state_lock, flags); + } + cluster->nr_paused_cpus += nr_paused; + spin_unlock_irqrestore(&state_lock, flags); + +again: + /* + * If the number of active CPUs is within the limits, then + * don't force pause of any busy CPUs. + */ + if (cluster->active_cpus <= cluster->max_cpus) + return; + + nr_paused = 0; + num_cpus = cluster->num_cpus; + spin_lock_irqsave(&state_lock, flags); + list_for_each_entry_safe(c, tmp, &cluster->lru, sib) { + if (!num_cpus--) + break; + + if (!is_active(c)) + continue; + if (cluster->active_cpus <= cluster->max_cpus) + break; + + if (first_pass && !c->not_preferred) + continue; + + spin_unlock_irqrestore(&state_lock, flags); + + cpumask_set_cpu(c->cpu, pause_cpus); + sched_pause_pending(c->cpu); + + c->paused_by_us = true; + move_cpu_lru(c); + nr_paused++; + + cluster->active_cpus = get_active_cpu_count(cluster); + spin_lock_irqsave(&state_lock, flags); + } + + cluster->nr_paused_cpus += nr_paused; + spin_unlock_irqrestore(&state_lock, flags); + + if (first_pass && cluster->active_cpus > cluster->max_cpus) { + first_pass = false; + goto again; + } +} + +static void __try_to_resume(struct cluster_data *cluster, + unsigned int need, bool force, struct cpumask *unpause_cpus) +{ + struct cpu_data *c, *tmp; + unsigned long flags; + unsigned int num_cpus = cluster->num_cpus; + unsigned int nr_unpaused = 0; + + /* + * Protect against entry being removed (and added at tail) by other + * thread (hotplug). + */ + spin_lock_irqsave(&state_lock, flags); + list_for_each_entry_safe(c, tmp, &cluster->lru, sib) { + if (!num_cpus--) + break; + + if (!c->paused_by_us) + continue; + if ((cpu_online(c->cpu) && cpu_active(c->cpu)) || + (!force && c->not_preferred)) + continue; + if (cluster->active_cpus == need) + break; + + spin_unlock_irqrestore(&state_lock, flags); + + pr_debug("Trying to resume CPU%u\n", c->cpu); + + cpumask_set_cpu(c->cpu, unpause_cpus); + sched_unpause_pending(c->cpu); + + c->paused_by_us = false; + move_cpu_lru(c); + nr_unpaused++; + + cluster->active_cpus = get_active_cpu_count(cluster); + spin_lock_irqsave(&state_lock, flags); + } + cluster->nr_paused_cpus -= nr_unpaused; + spin_unlock_irqrestore(&state_lock, flags); +} + +static void try_to_resume(struct cluster_data *cluster, unsigned int need, + struct cpumask *unpause_cpus) +{ + bool force_use_non_preferred = false; + + __try_to_resume(cluster, need, force_use_non_preferred, unpause_cpus); + + if (cluster->active_cpus == need) + return; + + force_use_non_preferred = true; + __try_to_resume(cluster, need, force_use_non_preferred, unpause_cpus); +} + +static void __ref do_core_ctl(void) +{ + struct cluster_data *cluster; + unsigned int index = 0; + unsigned int need; + cpumask_t cpus_to_pause = { CPU_BITS_NONE }; + cpumask_t cpus_to_unpause = { CPU_BITS_NONE }; + + for_each_cluster(cluster, index) { + + eval_need(cluster); + + need = apply_limits(cluster, cluster->need_cpus); + + if (adjustment_possible(cluster, need)) { + pr_debug("Trying to adjust group %u from %u to %u\n", + cluster->first_cpu, cluster->active_cpus, need); + + if (cluster->active_cpus > need) + try_to_pause(cluster, need, &cpus_to_pause); + + else if (cluster->active_cpus < need) + try_to_resume(cluster, need, &cpus_to_unpause); + } + } + + if (cpumask_any(&cpus_to_pause) < nr_cpu_ids) + pause_cpus(&cpus_to_pause); + + if (cpumask_any(&cpus_to_unpause) < nr_cpu_ids) + resume_cpus(&cpus_to_unpause); +} + +static int __ref try_core_ctl(void *data) +{ + unsigned long flags; + + while (1) { + set_current_state(TASK_INTERRUPTIBLE); + spin_lock_irqsave(&core_ctl_pending_lock, flags); + if (!core_ctl_pending) { + spin_unlock_irqrestore(&core_ctl_pending_lock, flags); + schedule(); + if (kthread_should_stop()) + break; + spin_lock_irqsave(&core_ctl_pending_lock, flags); + } + set_current_state(TASK_RUNNING); + core_ctl_pending = false; + spin_unlock_irqrestore(&core_ctl_pending_lock, flags); + + do_core_ctl(); + } + + return 0; +} + +/* ============================ init code ============================== */ + +static struct cluster_data *find_cluster_by_first_cpu(unsigned int first_cpu) +{ + unsigned int i; + + for (i = 0; i < num_clusters; ++i) { + if (cluster_state[i].first_cpu == first_cpu) + return &cluster_state[i]; + } + + return NULL; +} + +static int cluster_init(const struct cpumask *mask) +{ + struct device *dev; + unsigned int first_cpu = cpumask_first(mask); + struct cluster_data *cluster; + struct cpu_data *state; + unsigned int cpu; + + if (find_cluster_by_first_cpu(first_cpu)) + return 0; + + dev = get_cpu_device(first_cpu); + if (!dev) + return -ENODEV; + + pr_info("Creating CPU group %d\n", first_cpu); + + if (num_clusters == MAX_CLUSTERS) { + pr_err("Unsupported number of clusters. Only %u supported\n", + MAX_CLUSTERS); + return -EINVAL; + } + cluster = &cluster_state[num_clusters]; + ++num_clusters; + + cpumask_copy(&cluster->cpu_mask, mask); + cluster->num_cpus = cpumask_weight(mask); + if (cluster->num_cpus > MAX_CPUS_PER_CLUSTER) { + pr_err("HW configuration not supported\n"); + return -EINVAL; + } + cluster->first_cpu = first_cpu; + cluster->min_cpus = 1; + cluster->max_cpus = cluster->num_cpus; + cluster->need_cpus = cluster->num_cpus; + cluster->offline_delay_ms = 100; + cluster->task_thres = UINT_MAX; + cluster->nr_prev_assist_thresh = UINT_MAX; + cluster->nrrun = cluster->num_cpus; + cluster->enable = true; + cluster->nr_not_preferred_cpus = 0; + cluster->strict_nrrun = 0; + INIT_LIST_HEAD(&cluster->lru); + + for_each_cpu(cpu, mask) { + pr_info("Init CPU%u state\n", cpu); + + state = &per_cpu(cpu_state, cpu); + state->cluster = cluster; + state->cpu = cpu; + list_add_tail(&state->sib, &cluster->lru); + } + cluster->active_cpus = get_active_cpu_count(cluster); + + cluster->inited = true; + + kobject_init(&cluster->kobj, &ktype_core_ctl); + return kobject_add(&cluster->kobj, &dev->kobj, "core_ctl"); +} + +int core_ctl_init(void) +{ + struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; + struct walt_sched_cluster *cluster; + int ret; + + /* initialize our single kthread */ + core_ctl_thread = kthread_run(try_core_ctl, NULL, "core_ctl"); + + if (IS_ERR(core_ctl_thread)) + return PTR_ERR(core_ctl_thread); + + spin_lock_init(&core_ctl_pending_lock); + + sched_setscheduler_nocheck(core_ctl_thread, SCHED_FIFO, ¶m); + + for_each_sched_cluster(cluster) { + ret = cluster_init(&cluster->cpus); + if (ret) + pr_warn("unable to create core ctl group: %d\n", ret); + } + + initialized = true; + + return 0; +} diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c new file mode 100644 index 000000000000..3d24cf3fd1e7 --- /dev/null +++ b/kernel/sched/walt/cpufreq_walt.c @@ -0,0 +1,886 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This is based on schedutil governor but modified to work with + * WALT. + * + * Copyright (C) 2016, Intel Corporation + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "walt.h" +#include "trace.h" + +struct waltgov_tunables { + struct gov_attr_set attr_set; + unsigned int up_rate_limit_us; + unsigned int down_rate_limit_us; + unsigned int hispeed_load; + unsigned int hispeed_freq; + unsigned int rtg_boost_freq; + bool pl; +}; + +struct waltgov_policy { + struct cpufreq_policy *policy; + u64 last_ws; + u64 curr_cycles; + u64 last_cyc_update_time; + unsigned long avg_cap; + struct waltgov_tunables *tunables; + struct list_head tunables_hook; + unsigned long hispeed_util; + unsigned long rtg_boost_util; + unsigned long max; + + raw_spinlock_t update_lock; + u64 last_freq_update_time; + s64 min_rate_limit_ns; + s64 up_rate_delay_ns; + s64 down_rate_delay_ns; + unsigned int next_freq; + unsigned int cached_raw_freq; + + /* The next fields are only needed if fast switch cannot be used: */ + struct irq_work irq_work; + struct kthread_work work; + struct mutex work_lock; + struct kthread_worker worker; + struct task_struct *thread; + + bool limits_changed; + bool need_freq_update; +}; + +struct waltgov_cpu { + struct waltgov_callback cb; + struct waltgov_policy *wg_policy; + unsigned int cpu; + struct walt_cpu_load walt_load; + unsigned long util; + unsigned long max; + unsigned int flags; +}; + +DEFINE_PER_CPU(struct waltgov_callback *, waltgov_cb_data); +static DEFINE_PER_CPU(struct waltgov_cpu, waltgov_cpu); +static DEFINE_PER_CPU(struct waltgov_tunables *, cached_tunables); + +/************************ Governor internals ***********************/ + +static bool waltgov_should_update_freq(struct waltgov_policy *wg_policy, u64 time) +{ + s64 delta_ns; + + if (unlikely(wg_policy->limits_changed)) { + wg_policy->limits_changed = false; + wg_policy->need_freq_update = true; + return true; + } + + /* + * No need to recalculate next freq for min_rate_limit_us + * at least. However we might still decide to further rate + * limit once frequency change direction is decided, according + * to the separate rate limits. + */ + + delta_ns = time - wg_policy->last_freq_update_time; + return delta_ns >= wg_policy->min_rate_limit_ns; +} + +static bool waltgov_up_down_rate_limit(struct waltgov_policy *wg_policy, u64 time, + unsigned int next_freq) +{ + s64 delta_ns; + + delta_ns = time - wg_policy->last_freq_update_time; + + if (next_freq > wg_policy->next_freq && + delta_ns < wg_policy->up_rate_delay_ns) + return true; + + if (next_freq < wg_policy->next_freq && + delta_ns < wg_policy->down_rate_delay_ns) + return true; + + return false; +} + +static bool waltgov_update_next_freq(struct waltgov_policy *wg_policy, u64 time, + unsigned int next_freq) +{ + if (wg_policy->next_freq == next_freq) + return false; + + if (waltgov_up_down_rate_limit(wg_policy, time, next_freq)) + return false; + + wg_policy->next_freq = next_freq; + wg_policy->last_freq_update_time = time; + + return true; +} + +static unsigned long freq_to_util(struct waltgov_policy *wg_policy, + unsigned int freq) +{ + return mult_frac(wg_policy->max, freq, + wg_policy->policy->cpuinfo.max_freq); +} + +#define KHZ 1000 +static void waltgov_track_cycles(struct waltgov_policy *wg_policy, + unsigned int prev_freq, + u64 upto) +{ + u64 delta_ns, cycles; + u64 next_ws = wg_policy->last_ws + sched_ravg_window; + + upto = min(upto, next_ws); + /* Track cycles in current window */ + delta_ns = upto - wg_policy->last_cyc_update_time; + delta_ns *= prev_freq; + do_div(delta_ns, (NSEC_PER_SEC / KHZ)); + cycles = delta_ns; + wg_policy->curr_cycles += cycles; + wg_policy->last_cyc_update_time = upto; +} + +static void waltgov_calc_avg_cap(struct waltgov_policy *wg_policy, u64 curr_ws, + unsigned int prev_freq) +{ + u64 last_ws = wg_policy->last_ws; + unsigned int avg_freq; + + BUG_ON(curr_ws < last_ws); + if (curr_ws <= last_ws) + return; + + /* If we skipped some windows */ + if (curr_ws > (last_ws + sched_ravg_window)) { + avg_freq = prev_freq; + /* Reset tracking history */ + wg_policy->last_cyc_update_time = curr_ws; + } else { + waltgov_track_cycles(wg_policy, prev_freq, curr_ws); + avg_freq = wg_policy->curr_cycles; + avg_freq /= sched_ravg_window / (NSEC_PER_SEC / KHZ); + } + wg_policy->avg_cap = freq_to_util(wg_policy, avg_freq); + wg_policy->curr_cycles = 0; + wg_policy->last_ws = curr_ws; +} + +static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time, + unsigned int next_freq) +{ + struct cpufreq_policy *policy = wg_policy->policy; + unsigned int cpu; + + if (!waltgov_update_next_freq(wg_policy, time, next_freq)) + return; + + waltgov_track_cycles(wg_policy, wg_policy->policy->cur, time); + next_freq = cpufreq_driver_fast_switch(policy, next_freq); + if (!next_freq) + return; + + policy->cur = next_freq; + + if (trace_cpu_frequency_enabled()) { + for_each_cpu(cpu, policy->cpus) + trace_cpu_frequency(next_freq, cpu); + } +} + +static void waltgov_deferred_update(struct waltgov_policy *wg_policy, u64 time, + unsigned int next_freq) +{ + if (!waltgov_update_next_freq(wg_policy, time, next_freq)) + return; + + walt_irq_work_queue(&wg_policy->irq_work); +} + +#define TARGET_LOAD 80 +static unsigned int get_next_freq(struct waltgov_policy *wg_policy, + unsigned long util, unsigned long max) +{ + struct cpufreq_policy *policy = wg_policy->policy; + /* + * TODO: + unsigned int freq = arch_scale_freq_invariant() ? + policy->cpuinfo.max_freq : policy->cur; + */ + unsigned int freq = policy->cpuinfo.max_freq; + + freq = map_util_freq(util, freq, max); + trace_waltgov_next_freq(policy->cpu, util, max, freq); + + if (freq == wg_policy->cached_raw_freq && !wg_policy->need_freq_update) + return wg_policy->next_freq; + + wg_policy->need_freq_update = false; + wg_policy->cached_raw_freq = freq; + return cpufreq_driver_resolve_freq(policy, freq); +} + +static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) +{ + struct rq *rq = cpu_rq(wg_cpu->cpu); + unsigned long max = arch_scale_cpu_capacity(wg_cpu->cpu); + unsigned long util; + + wg_cpu->max = max; + util = cpu_util_freq_walt(wg_cpu->cpu, &wg_cpu->walt_load); + return uclamp_rq_util_with(rq, util, NULL); +} + +#define NL_RATIO 75 +#define DEFAULT_HISPEED_LOAD 90 +#define DEFAULT_CPU0_RTG_BOOST_FREQ 1000000 +#define DEFAULT_CPU4_RTG_BOOST_FREQ 0 +#define DEFAULT_CPU7_RTG_BOOST_FREQ 0 +static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long *util, + unsigned long *max) +{ + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + bool is_migration = wg_cpu->flags & WALT_CPUFREQ_IC_MIGRATION; + bool is_rtg_boost = wg_cpu->walt_load.rtgb_active; + unsigned long nl = wg_cpu->walt_load.nl; + unsigned long cpu_util = wg_cpu->util; + bool is_hiload; + unsigned long pl = wg_cpu->walt_load.pl; + + if (is_rtg_boost) + *util = max(*util, wg_policy->rtg_boost_util); + + is_hiload = (cpu_util >= mult_frac(wg_policy->avg_cap, + wg_policy->tunables->hispeed_load, + 100)); + + if (is_hiload && !is_migration) + *util = max(*util, wg_policy->hispeed_util); + + if (is_hiload && nl >= mult_frac(cpu_util, NL_RATIO, 100)) + *util = *max; + + if (wg_policy->tunables->pl) { + if (sysctl_sched_conservative_pl) + pl = mult_frac(pl, TARGET_LOAD, 100); + *util = max(*util, pl); + } +} + +static inline unsigned long target_util(struct waltgov_policy *wg_policy, + unsigned int freq) +{ + unsigned long util; + + util = freq_to_util(wg_policy, freq); + util = mult_frac(util, TARGET_LOAD, 100); + return util; +} + +static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 time) +{ + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + struct cpufreq_policy *policy = wg_policy->policy; + unsigned long util = 0, max = 1; + unsigned int j; + + for_each_cpu(j, policy->cpus) { + struct waltgov_cpu *j_wg_cpu = &per_cpu(waltgov_cpu, j); + unsigned long j_util, j_max; + + /* + * If the util value for all CPUs in a policy is 0, just using > + * will result in a max value of 1. WALT stats can later update + * the aggregated util value, causing get_next_freq() to compute + * freq = max_freq * 1.25 * (util / max) for nonzero util, + * leading to spurious jumps to fmax. + */ + j_util = j_wg_cpu->util; + j_max = j_wg_cpu->max; + + if (j_util * max >= j_max * util) { + util = j_util; + max = j_max; + } + + waltgov_walt_adjust(j_wg_cpu, &util, &max); + } + + return get_next_freq(wg_policy, util, max); +} + +static void waltgov_update_freq(struct waltgov_callback *cb, u64 time, + unsigned int flags) +{ + struct waltgov_cpu *wg_cpu = container_of(cb, struct waltgov_cpu, cb); + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + unsigned long hs_util, boost_util; + unsigned int next_f; + + if (!wg_policy->tunables->pl && flags & WALT_CPUFREQ_PL) + return; + + wg_cpu->util = waltgov_get_util(wg_cpu); + wg_cpu->flags = flags; + raw_spin_lock(&wg_policy->update_lock); + + if (wg_policy->max != wg_cpu->max) { + wg_policy->max = wg_cpu->max; + hs_util = target_util(wg_policy, + wg_policy->tunables->hispeed_freq); + wg_policy->hispeed_util = hs_util; + + boost_util = target_util(wg_policy, + wg_policy->tunables->rtg_boost_freq); + wg_policy->rtg_boost_util = boost_util; + } + + waltgov_calc_avg_cap(wg_policy, wg_cpu->walt_load.ws, + wg_policy->policy->cur); + + trace_waltgov_util_update(wg_cpu->cpu, wg_cpu->util, wg_policy->avg_cap, + wg_cpu->max, wg_cpu->walt_load.nl, + wg_cpu->walt_load.pl, + wg_cpu->walt_load.rtgb_active, flags); + + if (waltgov_should_update_freq(wg_policy, time) && + !(flags & WALT_CPUFREQ_CONTINUE)) { + next_f = waltgov_next_freq_shared(wg_cpu, time); + + if (wg_policy->policy->fast_switch_enabled) + waltgov_fast_switch(wg_policy, time, next_f); + else + waltgov_deferred_update(wg_policy, time, next_f); + } + + raw_spin_unlock(&wg_policy->update_lock); +} + +static void waltgov_work(struct kthread_work *work) +{ + struct waltgov_policy *wg_policy = container_of(work, struct waltgov_policy, work); + unsigned int freq; + unsigned long flags; + + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + freq = wg_policy->next_freq; + waltgov_track_cycles(wg_policy, wg_policy->policy->cur, + ktime_get_ns()); + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + + mutex_lock(&wg_policy->work_lock); + __cpufreq_driver_target(wg_policy->policy, freq, CPUFREQ_RELATION_L); + mutex_unlock(&wg_policy->work_lock); +} + +static void waltgov_irq_work(struct irq_work *irq_work) +{ + struct waltgov_policy *wg_policy; + + wg_policy = container_of(irq_work, struct waltgov_policy, irq_work); + + kthread_queue_work(&wg_policy->worker, &wg_policy->work); +} + +/************************** sysfs interface ************************/ + +static inline struct waltgov_tunables *to_waltgov_tunables(struct gov_attr_set *attr_set) +{ + return container_of(attr_set, struct waltgov_tunables, attr_set); +} + +static DEFINE_MUTEX(min_rate_lock); + +static void update_min_rate_limit_ns(struct waltgov_policy *wg_policy) +{ + mutex_lock(&min_rate_lock); + wg_policy->min_rate_limit_ns = min(wg_policy->up_rate_delay_ns, + wg_policy->down_rate_delay_ns); + mutex_unlock(&min_rate_lock); +} + +static ssize_t up_rate_limit_us_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->up_rate_limit_us); +} + +static ssize_t down_rate_limit_us_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->down_rate_limit_us); +} + +static ssize_t up_rate_limit_us_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + struct waltgov_policy *wg_policy; + unsigned int rate_limit_us; + + if (kstrtouint(buf, 10, &rate_limit_us)) + return -EINVAL; + + tunables->up_rate_limit_us = rate_limit_us; + + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + wg_policy->up_rate_delay_ns = rate_limit_us * NSEC_PER_USEC; + update_min_rate_limit_ns(wg_policy); + } + + return count; +} + +static ssize_t down_rate_limit_us_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + struct waltgov_policy *wg_policy; + unsigned int rate_limit_us; + + if (kstrtouint(buf, 10, &rate_limit_us)) + return -EINVAL; + + tunables->down_rate_limit_us = rate_limit_us; + + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + wg_policy->down_rate_delay_ns = rate_limit_us * NSEC_PER_USEC; + update_min_rate_limit_ns(wg_policy); + } + + return count; +} + +static struct governor_attr up_rate_limit_us = __ATTR_RW(up_rate_limit_us); +static struct governor_attr down_rate_limit_us = __ATTR_RW(down_rate_limit_us); + +static ssize_t hispeed_load_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->hispeed_load); +} + +static ssize_t hispeed_load_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + if (kstrtouint(buf, 10, &tunables->hispeed_load)) + return -EINVAL; + + tunables->hispeed_load = min(100U, tunables->hispeed_load); + + return count; +} + +static ssize_t hispeed_freq_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->hispeed_freq); +} + +static ssize_t hispeed_freq_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + unsigned int val; + struct waltgov_policy *wg_policy; + unsigned long hs_util; + unsigned long flags; + + if (kstrtouint(buf, 10, &val)) + return -EINVAL; + + tunables->hispeed_freq = val; + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + hs_util = target_util(wg_policy, + wg_policy->tunables->hispeed_freq); + wg_policy->hispeed_util = hs_util; + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + } + + return count; +} + +static ssize_t rtg_boost_freq_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->rtg_boost_freq); +} + +static ssize_t rtg_boost_freq_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + unsigned int val; + struct waltgov_policy *wg_policy; + unsigned long boost_util; + unsigned long flags; + + if (kstrtouint(buf, 10, &val)) + return -EINVAL; + + tunables->rtg_boost_freq = val; + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + boost_util = target_util(wg_policy, + wg_policy->tunables->rtg_boost_freq); + wg_policy->rtg_boost_util = boost_util; + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + } + + return count; +} + +static ssize_t pl_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->pl); +} + +static ssize_t pl_store(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + if (kstrtobool(buf, &tunables->pl)) + return -EINVAL; + + return count; +} + +static struct governor_attr hispeed_load = __ATTR_RW(hispeed_load); +static struct governor_attr hispeed_freq = __ATTR_RW(hispeed_freq); +static struct governor_attr rtg_boost_freq = __ATTR_RW(rtg_boost_freq); +static struct governor_attr pl = __ATTR_RW(pl); + +static struct attribute *waltgov_attributes[] = { + &up_rate_limit_us.attr, + &down_rate_limit_us.attr, + &hispeed_load.attr, + &hispeed_freq.attr, + &rtg_boost_freq.attr, + &pl.attr, + NULL +}; + +static struct kobj_type waltgov_tunables_ktype = { + .default_attrs = waltgov_attributes, + .sysfs_ops = &governor_sysfs_ops, +}; + +/********************** cpufreq governor interface *********************/ + +static struct cpufreq_governor walt_gov; + +static struct waltgov_policy *waltgov_policy_alloc(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy; + + wg_policy = kzalloc(sizeof(*wg_policy), GFP_KERNEL); + if (!wg_policy) + return NULL; + + wg_policy->policy = policy; + raw_spin_lock_init(&wg_policy->update_lock); + return wg_policy; +} + +static void waltgov_policy_free(struct waltgov_policy *wg_policy) +{ + kfree(wg_policy); +} + +static int waltgov_kthread_create(struct waltgov_policy *wg_policy) +{ + struct task_struct *thread; + struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 }; + struct cpufreq_policy *policy = wg_policy->policy; + int ret; + + /* kthread only required for slow path */ + if (policy->fast_switch_enabled) + return 0; + + kthread_init_work(&wg_policy->work, waltgov_work); + kthread_init_worker(&wg_policy->worker); + thread = kthread_create(kthread_worker_fn, &wg_policy->worker, + "waltgov:%d", + cpumask_first(policy->related_cpus)); + if (IS_ERR(thread)) { + pr_err("failed to create waltgov thread: %ld\n", PTR_ERR(thread)); + return PTR_ERR(thread); + } + + ret = sched_setscheduler_nocheck(thread, SCHED_FIFO, ¶m); + if (ret) { + kthread_stop(thread); + pr_warn("%s: failed to set SCHED_FIFO\n", __func__); + return ret; + } + + wg_policy->thread = thread; + kthread_bind_mask(thread, policy->related_cpus); + init_irq_work(&wg_policy->irq_work, waltgov_irq_work); + mutex_init(&wg_policy->work_lock); + + wake_up_process(thread); + + return 0; +} + +static void waltgov_kthread_stop(struct waltgov_policy *wg_policy) +{ + /* kthread only required for slow path */ + if (wg_policy->policy->fast_switch_enabled) + return; + + kthread_flush_worker(&wg_policy->worker); + kthread_stop(wg_policy->thread); + mutex_destroy(&wg_policy->work_lock); +} + +static void waltgov_tunables_save(struct cpufreq_policy *policy, + struct waltgov_tunables *tunables) +{ + int cpu; + struct waltgov_tunables *cached = per_cpu(cached_tunables, policy->cpu); + + if (!cached) { + cached = kzalloc(sizeof(*tunables), GFP_KERNEL); + if (!cached) + return; + + for_each_cpu(cpu, policy->related_cpus) + per_cpu(cached_tunables, cpu) = cached; + } + + cached->pl = tunables->pl; + cached->hispeed_load = tunables->hispeed_load; + cached->rtg_boost_freq = tunables->rtg_boost_freq; + cached->hispeed_freq = tunables->hispeed_freq; + cached->up_rate_limit_us = tunables->up_rate_limit_us; + cached->down_rate_limit_us = tunables->down_rate_limit_us; +} + +static void waltgov_tunables_restore(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + struct waltgov_tunables *tunables = wg_policy->tunables; + struct waltgov_tunables *cached = per_cpu(cached_tunables, policy->cpu); + + if (!cached) + return; + + tunables->pl = cached->pl; + tunables->hispeed_load = cached->hispeed_load; + tunables->rtg_boost_freq = cached->rtg_boost_freq; + tunables->hispeed_freq = cached->hispeed_freq; + tunables->up_rate_limit_us = cached->up_rate_limit_us; + tunables->down_rate_limit_us = cached->down_rate_limit_us; +} + +static int waltgov_init(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy; + struct waltgov_tunables *tunables; + int ret = 0; + + /* State should be equivalent to EXIT */ + if (policy->governor_data) + return -EBUSY; + + cpufreq_enable_fast_switch(policy); + + if (policy->fast_switch_possible && !policy->fast_switch_enabled) + BUG_ON(1); + + wg_policy = waltgov_policy_alloc(policy); + if (!wg_policy) { + ret = -ENOMEM; + goto disable_fast_switch; + } + + ret = waltgov_kthread_create(wg_policy); + if (ret) + goto free_wg_policy; + + tunables = kzalloc(sizeof(*tunables), GFP_KERNEL); + if (!tunables) { + ret = -ENOMEM; + goto stop_kthread; + } + + gov_attr_set_init(&tunables->attr_set, &wg_policy->tunables_hook); + tunables->hispeed_load = DEFAULT_HISPEED_LOAD; + + switch (policy->cpu) { + default: + case 0: + tunables->rtg_boost_freq = DEFAULT_CPU0_RTG_BOOST_FREQ; + break; + case 4: + tunables->rtg_boost_freq = DEFAULT_CPU4_RTG_BOOST_FREQ; + break; + case 7: + tunables->rtg_boost_freq = DEFAULT_CPU7_RTG_BOOST_FREQ; + break; + } + + policy->governor_data = wg_policy; + wg_policy->tunables = tunables; + waltgov_tunables_restore(policy); + + ret = kobject_init_and_add(&tunables->attr_set.kobj, &waltgov_tunables_ktype, + get_governor_parent_kobj(policy), "%s", + walt_gov.name); + if (ret) + goto fail; + + return 0; + +fail: + kobject_put(&tunables->attr_set.kobj); + policy->governor_data = NULL; + kfree(tunables); +stop_kthread: + waltgov_kthread_stop(wg_policy); +free_wg_policy: + waltgov_policy_free(wg_policy); +disable_fast_switch: + cpufreq_disable_fast_switch(policy); + + pr_err("initialization failed (error %d)\n", ret); + return ret; +} + +static void waltgov_exit(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + struct waltgov_tunables *tunables = wg_policy->tunables; + unsigned int count; + + count = gov_attr_set_put(&tunables->attr_set, &wg_policy->tunables_hook); + policy->governor_data = NULL; + if (!count) { + waltgov_tunables_save(policy, tunables); + kfree(tunables); + } + + waltgov_kthread_stop(wg_policy); + waltgov_policy_free(wg_policy); + cpufreq_disable_fast_switch(policy); +} + +static int waltgov_start(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + unsigned int cpu; + + wg_policy->up_rate_delay_ns = + wg_policy->tunables->up_rate_limit_us * NSEC_PER_USEC; + wg_policy->down_rate_delay_ns = + wg_policy->tunables->down_rate_limit_us * NSEC_PER_USEC; + update_min_rate_limit_ns(wg_policy); + wg_policy->last_freq_update_time = 0; + wg_policy->next_freq = 0; + wg_policy->limits_changed = false; + wg_policy->need_freq_update = false; + wg_policy->cached_raw_freq = 0; + + for_each_cpu(cpu, policy->cpus) { + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + + memset(wg_cpu, 0, sizeof(*wg_cpu)); + wg_cpu->cpu = cpu; + wg_cpu->wg_policy = wg_policy; + } + + for_each_cpu(cpu, policy->cpus) { + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + + waltgov_add_callback(cpu, &wg_cpu->cb, waltgov_update_freq); + } + + return 0; +} + +static void waltgov_stop(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + unsigned int cpu; + + for_each_cpu(cpu, policy->cpus) + waltgov_remove_callback(cpu); + + synchronize_rcu(); + + if (!policy->fast_switch_enabled) { + irq_work_sync(&wg_policy->irq_work); + kthread_cancel_work_sync(&wg_policy->work); + } +} + +static void waltgov_limits(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + unsigned long flags, now; + unsigned int freq; + + if (!policy->fast_switch_enabled) { + mutex_lock(&wg_policy->work_lock); + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + waltgov_track_cycles(wg_policy, wg_policy->policy->cur, + ktime_get_ns()); + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + cpufreq_policy_apply_limits(policy); + mutex_unlock(&wg_policy->work_lock); + } else { + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + freq = policy->cur; + now = ktime_get_ns(); + + /* + * cpufreq_driver_resolve_freq() has a clamp, so we do not need + * to do any sort of additional validation here. + */ + freq = cpufreq_driver_resolve_freq(policy, freq); + wg_policy->cached_raw_freq = freq; + waltgov_fast_switch(wg_policy, now, freq); + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + } + + wg_policy->limits_changed = true; +} + +static struct cpufreq_governor walt_gov = { + .name = "walt", + .init = waltgov_init, + .exit = waltgov_exit, + .start = waltgov_start, + .stop = waltgov_stop, + .limits = waltgov_limits, + .owner = THIS_MODULE, +}; + +int waltgov_register(void) +{ + return cpufreq_register_governor(&walt_gov); +} diff --git a/kernel/sched/walt/fixup.c b/kernel/sched/walt/fixup.c new file mode 100644 index 000000000000..21c3abe003ec --- /dev/null +++ b/kernel/sched/walt/fixup.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + */ + +#include + +#include "walt.h" + +unsigned int cpuinfo_max_freq_cached; + +char sched_lib_name[LIB_PATH_LENGTH]; +unsigned int sched_lib_mask_force; + +bool is_sched_lib_based_app(pid_t pid) +{ + const char *name = NULL; + char *libname, *lib_list; + struct vm_area_struct *vma; + char path_buf[LIB_PATH_LENGTH]; + char *tmp_lib_name; + bool found = false; + struct task_struct *p; + struct mm_struct *mm; + + if (strnlen(sched_lib_name, LIB_PATH_LENGTH) == 0) + return false; + + tmp_lib_name = kmalloc(LIB_PATH_LENGTH, GFP_KERNEL); + if (!tmp_lib_name) + return false; + + rcu_read_lock(); + + p = pid ? get_pid_task(find_vpid(pid), PIDTYPE_PID) : current; + if (!p) { + rcu_read_unlock(); + kfree(tmp_lib_name); + return false; + } + + /* Prevent p going away */ + get_task_struct(p); + rcu_read_unlock(); + + mm = get_task_mm(p); + if (!mm) + goto put_task_struct; + + down_read(&mm->mmap_lock); + for (vma = mm->mmap; vma ; vma = vma->vm_next) { + if (vma->vm_file && vma->vm_flags & VM_EXEC) { + name = d_path(&vma->vm_file->f_path, + path_buf, LIB_PATH_LENGTH); + if (IS_ERR(name)) + goto release_sem; + + strlcpy(tmp_lib_name, sched_lib_name, LIB_PATH_LENGTH); + lib_list = tmp_lib_name; + while ((libname = strsep(&lib_list, ","))) { + libname = skip_spaces(libname); + if (strnstr(name, libname, + strnlen(name, LIB_PATH_LENGTH))) { + found = true; + goto release_sem; + } + } + } + } + +release_sem: + up_read(&mm->mmap_lock); + mmput(mm); +put_task_struct: + put_task_struct(p); + kfree(tmp_lib_name); + return found; +} + +void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy, + unsigned int *max_freq) +{ + if (!cpuinfo_max_freq_cached) + return; + + if (!(BIT(policy->cpu) & sched_lib_mask_force)) + return; + + if (is_sched_lib_based_app(current->pid)) + *max_freq = cpuinfo_max_freq_cached << 1; +} diff --git a/kernel/sched/walt/input-boost.c b/kernel/sched/walt/input-boost.c new file mode 100644 index 000000000000..c20454535dab --- /dev/null +++ b/kernel/sched/walt/input-boost.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013-2015,2017,2019-2021, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) "input-boost: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "walt.h" + +#define input_boost_attr_rw(_name) \ +static struct kobj_attribute _name##_attr = \ +__ATTR(_name, 0644, show_##_name, store_##_name) + +#define show_one(file_name) \ +static ssize_t show_##file_name \ +(struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ +{ \ + return scnprintf(buf, PAGE_SIZE, "%u\n", file_name); \ +} + +#define store_one(file_name) \ +static ssize_t store_##file_name \ +(struct kobject *kobj, struct kobj_attribute *attr, \ +const char *buf, size_t count) \ +{ \ + \ + sscanf(buf, "%u", &file_name); \ + return count; \ +} + +struct cpu_sync { + int cpu; + unsigned int input_boost_min; + unsigned int input_boost_freq; +}; + +static DEFINE_PER_CPU(struct cpu_sync, sync_info); +static struct workqueue_struct *input_boost_wq; + +static struct work_struct input_boost_work; + +static bool sched_boost_active; + +static struct delayed_work input_boost_rem; +static u64 last_input_time; +#define MIN_INPUT_INTERVAL (150 * USEC_PER_MSEC) + +static DEFINE_PER_CPU(struct freq_qos_request, qos_req); + +static void boost_adjust_notify(struct cpufreq_policy *policy) +{ + unsigned int cpu = policy->cpu; + struct cpu_sync *s = &per_cpu(sync_info, cpu); + unsigned int ib_min = s->input_boost_min; + struct freq_qos_request *req = &per_cpu(qos_req, cpu); + int ret; + + pr_debug("CPU%u policy min before boost: %u kHz\n", + cpu, policy->min); + pr_debug("CPU%u boost min: %u kHz\n", cpu, ib_min); + + ret = freq_qos_update_request(req, ib_min); + + if (ret < 0) + pr_err("Failed to update freq constraint in boost_adjust: %d\n", + ib_min); + + pr_debug("CPU%u policy min after boost: %u kHz\n", cpu, policy->min); +} + +static void update_policy_online(void) +{ + unsigned int i; + struct cpufreq_policy *policy; + struct cpumask online_cpus; + + /* Re-evaluate policy to trigger adjust notifier for online CPUs */ + get_online_cpus(); + online_cpus = *cpu_online_mask; + for_each_cpu(i, &online_cpus) { + policy = cpufreq_cpu_get(i); + if (!policy) { + pr_err("%s: cpufreq policy not found for cpu%d\n", + __func__, i); + return; + } + + cpumask_andnot(&online_cpus, &online_cpus, + policy->related_cpus); + boost_adjust_notify(policy); + } + put_online_cpus(); +} + +static void do_input_boost_rem(struct work_struct *work) +{ + unsigned int i, ret; + struct cpu_sync *i_sync_info; + + /* Reset the input_boost_min for all CPUs in the system */ + pr_debug("Resetting input boost min for all CPUs\n"); + for_each_possible_cpu(i) { + i_sync_info = &per_cpu(sync_info, i); + i_sync_info->input_boost_min = 0; + } + + /* Update policies for all online CPUs */ + update_policy_online(); + + if (sched_boost_active) { + ret = sched_set_boost(0); + if (!ret) + pr_err("input-boost: sched boost disable failed\n"); + sched_boost_active = false; + } +} + +static void do_input_boost(struct work_struct *work) +{ + unsigned int i, ret; + struct cpu_sync *i_sync_info; + + cancel_delayed_work_sync(&input_boost_rem); + if (sched_boost_active) { + sched_set_boost(0); + sched_boost_active = false; + } + + /* Set the input_boost_min for all CPUs in the system */ + pr_debug("Setting input boost min for all CPUs\n"); + for (i = 0; i < 8; i++) { + i_sync_info = &per_cpu(sync_info, i); + i_sync_info->input_boost_min = sysctl_input_boost_freq[i]; + } + + /* Update policies for all online CPUs */ + update_policy_online(); + + /* Enable scheduler boost to migrate tasks to big cluster */ + if (sysctl_sched_boost_on_input > 0) { + ret = sched_set_boost(sysctl_sched_boost_on_input); + if (ret) + pr_err("input-boost: sched boost enable failed\n"); + else + sched_boost_active = true; + } + + queue_delayed_work(input_boost_wq, &input_boost_rem, + msecs_to_jiffies(sysctl_input_boost_ms)); +} + +static void inputboost_input_event(struct input_handle *handle, + unsigned int type, unsigned int code, int value) +{ + u64 now; + int cpu; + int enabled = 0; + + for_each_possible_cpu(cpu) { + if (sysctl_input_boost_freq[cpu] > 0) { + enabled = 1; + break; + } + } + if (!enabled) + return; + + now = ktime_to_us(ktime_get()); + if (now - last_input_time < MIN_INPUT_INTERVAL) + return; + + if (work_pending(&input_boost_work)) + return; + + queue_work(input_boost_wq, &input_boost_work); + last_input_time = ktime_to_us(ktime_get()); +} + +static int inputboost_input_connect(struct input_handler *handler, + struct input_dev *dev, const struct input_device_id *id) +{ + struct input_handle *handle; + int error; + + handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = "cpufreq"; + + error = input_register_handle(handle); + if (error) + goto err2; + + error = input_open_device(handle); + if (error) + goto err1; + + return 0; +err1: + input_unregister_handle(handle); +err2: + kfree(handle); + return error; +} + +static void inputboost_input_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} + +static const struct input_device_id inputboost_ids[] = { + /* multi-touch touchscreen */ + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT | + INPUT_DEVICE_ID_MATCH_ABSBIT, + .evbit = { BIT_MASK(EV_ABS) }, + .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] = + BIT_MASK(ABS_MT_POSITION_X) | + BIT_MASK(ABS_MT_POSITION_Y) + }, + }, + /* touchpad */ + { + .flags = INPUT_DEVICE_ID_MATCH_KEYBIT | + INPUT_DEVICE_ID_MATCH_ABSBIT, + .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) }, + .absbit = { [BIT_WORD(ABS_X)] = + BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) + }, + }, + /* Keypad */ + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT, + .evbit = { BIT_MASK(EV_KEY) }, + }, + { }, +}; + +static struct input_handler inputboost_input_handler = { + .event = inputboost_input_event, + .connect = inputboost_input_connect, + .disconnect = inputboost_input_disconnect, + .name = "input-boost", + .id_table = inputboost_ids, +}; + +struct kobject *input_boost_kobj; +int input_boost_init(void) +{ + int cpu, ret; + struct cpu_sync *s; + struct cpufreq_policy *policy; + struct freq_qos_request *req; + + input_boost_wq = alloc_workqueue("inputboost_wq", WQ_HIGHPRI, 0); + if (!input_boost_wq) + return -EFAULT; + + INIT_WORK(&input_boost_work, do_input_boost); + INIT_DELAYED_WORK(&input_boost_rem, do_input_boost_rem); + + for_each_possible_cpu(cpu) { + s = &per_cpu(sync_info, cpu); + s->cpu = cpu; + req = &per_cpu(qos_req, cpu); + policy = cpufreq_cpu_get(cpu); + if (!policy) { + pr_err("%s: cpufreq policy not found for cpu%d\n", + __func__, cpu); + return -ESRCH; + } + + ret = freq_qos_add_request(&policy->constraints, req, + FREQ_QOS_MIN, policy->min); + if (ret < 0) { + pr_err("%s: Failed to add freq constraint (%d)\n", + __func__, ret); + return ret; + } + } + + ret = input_register_handler(&inputboost_input_handler); + return 0; +} diff --git a/kernel/sched/walt/preemptirq_long.c b/kernel/sched/walt/preemptirq_long.c new file mode 100644 index 000000000000..45b77150bf0a --- /dev/null +++ b/kernel/sched/walt/preemptirq_long.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#define CREATE_TRACE_POINTS +#include "preemptirq_long.h" + +#define IRQSOFF_SENTINEL 0x0fffDEAD + +static unsigned int sysctl_preemptoff_tracing_threshold_ns = 1000000; +static unsigned int sysctl_irqsoff_tracing_threshold_ns = 5000000; +static unsigned int sysctl_irqsoff_dmesg_output_enabled; +static unsigned int sysctl_irqsoff_crash_sentinel_value; +static unsigned int sysctl_irqsoff_crash_threshold_ns = 10000000; + +static unsigned int half_million = 500000; +static unsigned int one_hundred_million = 100000000; +static unsigned int one_million = 1000000; + +static DEFINE_PER_CPU(u64, irq_disabled_ts); + +/* + * preemption disable tracking require additional context + * to rule out false positives. see the comment in + * test_preempt_disable_long() for more details. + */ +struct preempt_store { + u64 ts; + int pid; + unsigned long ncsw; +}; +static DEFINE_PER_CPU(struct preempt_store, the_ps); + +static void note_irq_disable(void *u1, unsigned long u2, unsigned long u3) +{ + if (is_idle_task(current)) + return; + + /* + * We just have to note down the time stamp here. We + * use stacktrace trigger feature to print the stacktrace. + */ + this_cpu_write(irq_disabled_ts, sched_clock()); +} + +static void test_irq_disable_long(void *u1, unsigned long u2, unsigned long u3) +{ + u64 ts = this_cpu_read(irq_disabled_ts); + + if (!ts) + return; + + this_cpu_write(irq_disabled_ts, 0); + ts = sched_clock() - ts; + + if (ts > sysctl_irqsoff_tracing_threshold_ns) { + trace_irq_disable_long(ts); + + if (sysctl_irqsoff_dmesg_output_enabled == IRQSOFF_SENTINEL) + printk_deferred("D=%llu C:(%ps<-%ps<-%ps<-%ps)\n", + ts, (void *)CALLER_ADDR2, + (void *)CALLER_ADDR3, + (void *)CALLER_ADDR4, + (void *)CALLER_ADDR5); + } + + if (sysctl_irqsoff_crash_sentinel_value == IRQSOFF_SENTINEL && + ts > sysctl_irqsoff_crash_threshold_ns) { + printk_deferred("delta=%llu(ns) > crash_threshold=%u(ns) Task=%s\n", + ts, sysctl_irqsoff_crash_threshold_ns, + current->comm); + BUG_ON(1); + } +} + +static void note_preempt_disable(void *u1, unsigned long u2, unsigned long u3) +{ + struct preempt_store *ps = &per_cpu(the_ps, raw_smp_processor_id()); + + ps->ts = sched_clock(); + ps->pid = current->pid; + ps->ncsw = current->nvcsw + current->nivcsw; +} + +static void test_preempt_disable_long(void *u1, unsigned long u2, + unsigned long u3) +{ + struct preempt_store *ps = &per_cpu(the_ps, raw_smp_processor_id()); + u64 delta = 0; + + if (!ps->ts) + return; + + /* + * schedule() calls __schedule() with preemption disabled. + * if we had entered idle and exiting idle now, we think + * preemption is disabled the whole time. Detect this by + * checking if the preemption is disabled across the same + * task. There is a possiblity that the same task is scheduled + * after idle. To rule out this possibility, compare the + * context switch count also. + */ + if (ps->pid == current->pid && (ps->ncsw == current->nvcsw + + current->nivcsw)) + delta = sched_clock() - ps->ts; + + ps->ts = 0; + if (delta > sysctl_preemptoff_tracing_threshold_ns) + trace_preempt_disable_long(delta); +} + +static struct ctl_table preemptirq_long_table[] = { + { + .procname = "preemptoff_tracing_threshold_ns", + .data = &sysctl_preemptoff_tracing_threshold_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "irqsoff_tracing_threshold_ns", + .data = &sysctl_irqsoff_tracing_threshold_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = &half_million, + .extra2 = &one_hundred_million, + }, + { + .procname = "irqsoff_dmesg_output_enabled", + .data = &sysctl_irqsoff_dmesg_output_enabled, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "irqsoff_crash_sentinel_value", + .data = &sysctl_irqsoff_crash_sentinel_value, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "irqsoff_crash_threshold_ns", + .data = &sysctl_irqsoff_crash_threshold_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = &one_million, + .extra2 = &one_hundred_million, + }, + { } +}; + +int preemptirq_long_init(void) +{ + if (!register_sysctl("preemptirq", preemptirq_long_table)) { + pr_err("Fail to register sysctl table\n"); + return -EPERM; + } + + register_trace_android_rvh_irqs_disable(note_irq_disable, NULL); + register_trace_android_rvh_irqs_enable(test_irq_disable_long, NULL); + register_trace_android_rvh_preempt_disable(note_preempt_disable, NULL); + register_trace_android_rvh_preempt_enable(test_preempt_disable_long, + NULL); + + return 0; +} diff --git a/kernel/sched/walt/preemptirq_long.h b/kernel/sched/walt/preemptirq_long.h new file mode 100644 index 000000000000..48ede42a0655 --- /dev/null +++ b/kernel/sched/walt/preemptirq_long.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM preemptirq_long + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#if !defined(_TRACE_PREEMPTIRQ_LONG_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_PREEMPTIRQ_LONG_H + +#include + +DECLARE_EVENT_CLASS(preemptirq_long_template, + + TP_PROTO(u64 delta), + + TP_ARGS(delta), + + TP_STRUCT__entry( + __field(u64, delta) + ), + + TP_fast_assign( + __entry->delta = delta; + ), + + TP_printk("delta=%llu(ns)", __entry->delta) +); + +DEFINE_EVENT(preemptirq_long_template, irq_disable_long, + TP_PROTO(u64 delta), + TP_ARGS(delta)); + +DEFINE_EVENT(preemptirq_long_template, preempt_disable_long, + TP_PROTO(u64 delta), + TP_ARGS(delta)); + +#endif /* _TRACE_PREEMPTIRQ_LONG_H */ + +/* This part must be outside protection */ +#include diff --git a/kernel/sched/walt/qc_vas.c b/kernel/sched/walt/qc_vas.c new file mode 100644 index 000000000000..9db134f8363f --- /dev/null +++ b/kernel/sched/walt/qc_vas.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include + +#include "walt.h" + +#ifdef CONFIG_HOTPLUG_CPU + +cpumask_t pending_active_mask = CPU_MASK_NONE; +int sched_pause_count(const cpumask_t *mask, bool include_offline) +{ + cpumask_t count_mask = CPU_MASK_NONE; + cpumask_t pause_mask = CPU_MASK_NONE; + + if (cpumask_any(&pending_active_mask) >= nr_cpu_ids) { + /* initialize pending_active_state */ + cpumask_copy(&pending_active_mask, cpu_active_mask); + } + + if (include_offline) { + + /* get all offline or paused cpus */ + cpumask_complement(&pause_mask, &pending_active_mask); + cpumask_complement(&count_mask, cpu_online_mask); + cpumask_or(&count_mask, &count_mask, &pause_mask); + + /* get all offline or paused cpus in this cluster */ + cpumask_and(&count_mask, &count_mask, mask); + } else { + cpumask_andnot(&count_mask, mask, &pending_active_mask); + } + + return cpumask_weight(&count_mask); +} + +void sched_pause_pending(int cpu) +{ + cpumask_clear_cpu(cpu, &pending_active_mask); +} + +void sched_unpause_pending(int cpu) +{ + cpumask_set_cpu(cpu, &pending_active_mask); +} + +#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c new file mode 100644 index 000000000000..cec81ea98fba --- /dev/null +++ b/kernel/sched/walt/sched_avg.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012, 2015-2021, The Linux Foundation. All rights reserved. + */ + +/* + * Scheduler hook for average runqueue determination + */ +#include +#include +#include +#include +#include + +#include "walt.h" +#include "trace.h" + +static DEFINE_PER_CPU(u64, nr_prod_sum); +static DEFINE_PER_CPU(u64, last_time); +static DEFINE_PER_CPU(u64, nr_big_prod_sum); +static DEFINE_PER_CPU(u64, nr); +static DEFINE_PER_CPU(u64, nr_max); + +static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock); +static s64 last_get_time; + +static DEFINE_PER_CPU(atomic64_t, busy_hyst_end_time) = ATOMIC64_INIT(0); + +static DEFINE_PER_CPU(u64, hyst_time); +static DEFINE_PER_CPU(u64, coloc_hyst_busy); +static DEFINE_PER_CPU(u64, coloc_hyst_time); + +#define NR_THRESHOLD_PCT 15 +#define MAX_RTGB_TIME (sysctl_sched_coloc_busy_hyst_max_ms * NSEC_PER_MSEC) + +/** + * sched_get_nr_running_avg + * @return: Average nr_running, iowait and nr_big_tasks value since last poll. + * Returns the avg * 100 to return up to two decimal points + * of accuracy. + * + * Obtains the average nr_running value since the last poll. + * This function may not be called concurrently with itself + */ +void sched_get_nr_running_avg(struct sched_avg_stats *stats) +{ + int cpu; + u64 curr_time = sched_clock(); + u64 period = curr_time - last_get_time; + u64 tmp_nr, tmp_misfit; + bool any_hyst_time = false; + + if (!period) + return; + + /* read and reset nr_running counts */ + for_each_possible_cpu(cpu) { + unsigned long flags; + u64 diff; + + spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags); + curr_time = sched_clock(); + diff = curr_time - per_cpu(last_time, cpu); + BUG_ON((s64)diff < 0); + + tmp_nr = per_cpu(nr_prod_sum, cpu); + tmp_nr += per_cpu(nr, cpu) * diff; + tmp_nr = div64_u64((tmp_nr * 100), period); + + tmp_misfit = per_cpu(nr_big_prod_sum, cpu); + tmp_misfit += walt_big_tasks(cpu) * diff; + tmp_misfit = div64_u64((tmp_misfit * 100), period); + + /* + * NR_THRESHOLD_PCT is to make sure that the task ran + * at least 85% in the last window to compensate any + * over estimating being done. + */ + stats[cpu].nr = (int)div64_u64((tmp_nr + NR_THRESHOLD_PCT), + 100); + stats[cpu].nr_misfit = (int)div64_u64((tmp_misfit + + NR_THRESHOLD_PCT), 100); + stats[cpu].nr_max = per_cpu(nr_max, cpu); + stats[cpu].nr_scaled = tmp_nr; + + trace_sched_get_nr_running_avg(cpu, stats[cpu].nr, + stats[cpu].nr_misfit, stats[cpu].nr_max, + stats[cpu].nr_scaled); + + per_cpu(last_time, cpu) = curr_time; + per_cpu(nr_prod_sum, cpu) = 0; + per_cpu(nr_big_prod_sum, cpu) = 0; + per_cpu(nr_max, cpu) = per_cpu(nr, cpu); + + spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); + } + + for_each_possible_cpu(cpu) { + if (per_cpu(coloc_hyst_time, cpu)) { + any_hyst_time = true; + break; + } + } + if (any_hyst_time && get_rtgb_active_time() >= MAX_RTGB_TIME) + sched_update_hyst_times(); + + last_get_time = curr_time; + +} +EXPORT_SYMBOL(sched_get_nr_running_avg); + +void sched_update_hyst_times(void) +{ + bool rtgb_active; + int cpu; + unsigned long cpu_cap, coloc_busy_pct; + + rtgb_active = is_rtgb_active() && (sched_boost() != CONSERVATIVE_BOOST) + && (get_rtgb_active_time() < MAX_RTGB_TIME); + + for_each_possible_cpu(cpu) { + cpu_cap = arch_scale_cpu_capacity(cpu); + coloc_busy_pct = sysctl_sched_coloc_busy_hyst_cpu_busy_pct[cpu]; + per_cpu(hyst_time, cpu) = (BIT(cpu) + & sysctl_sched_busy_hyst_enable_cpus) ? + sysctl_sched_busy_hyst : 0; + per_cpu(coloc_hyst_time, cpu) = ((BIT(cpu) + & sysctl_sched_coloc_busy_hyst_enable_cpus) + && rtgb_active) ? + sysctl_sched_coloc_busy_hyst_cpu[cpu] : 0; + per_cpu(coloc_hyst_busy, cpu) = mult_frac(cpu_cap, + coloc_busy_pct, 100); + } +} + +#define BUSY_NR_RUN 3 +#define BUSY_LOAD_FACTOR 10 +static inline void update_busy_hyst_end_time(int cpu, bool dequeue, + unsigned long prev_nr_run, u64 curr_time) +{ + bool nr_run_trigger = false; + bool load_trigger = false, coloc_load_trigger = false; + u64 agg_hyst_time; + + if (!per_cpu(hyst_time, cpu) && !per_cpu(coloc_hyst_time, cpu)) + return; + + if (prev_nr_run >= BUSY_NR_RUN && per_cpu(nr, cpu) < BUSY_NR_RUN) + nr_run_trigger = true; + + if (dequeue && (cpu_util(cpu) * BUSY_LOAD_FACTOR) > + capacity_orig_of(cpu)) + load_trigger = true; + + if (dequeue && cpu_util(cpu) > per_cpu(coloc_hyst_busy, cpu)) + coloc_load_trigger = true; + + agg_hyst_time = max((nr_run_trigger || load_trigger) ? + per_cpu(hyst_time, cpu) : 0, + (nr_run_trigger || coloc_load_trigger) ? + per_cpu(coloc_hyst_time, cpu) : 0); + + if (agg_hyst_time) + atomic64_set(&per_cpu(busy_hyst_end_time, cpu), + curr_time + agg_hyst_time); +} + +int sched_busy_hyst_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret; + + if (table->maxlen > (sizeof(unsigned int) * num_possible_cpus())) + table->maxlen = sizeof(unsigned int) * num_possible_cpus(); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (!ret && write) + sched_update_hyst_times(); + + return ret; +} + +/** + * sched_update_nr_prod + * @cpu: The core id of the nr running driver. + * @enq: enqueue/dequeue happening on this CPU. + * @return: N/A + * + * Update average with latest nr_running value for CPU + */ +void sched_update_nr_prod(int cpu, bool enq) +{ + u64 diff; + u64 curr_time; + unsigned long flags, nr_running; + + spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags); + nr_running = per_cpu(nr, cpu); + curr_time = sched_clock(); + diff = curr_time - per_cpu(last_time, cpu); + BUG_ON((s64)diff < 0); + per_cpu(last_time, cpu) = curr_time; + per_cpu(nr, cpu) = cpu_rq(cpu)->nr_running; + + if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu)) + per_cpu(nr_max, cpu) = per_cpu(nr, cpu); + + update_busy_hyst_end_time(cpu, !enq, nr_running, curr_time); + + per_cpu(nr_prod_sum, cpu) += nr_running * diff; + per_cpu(nr_big_prod_sum, cpu) += walt_big_tasks(cpu) * diff; + spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); +} + +/* + * Returns the CPU utilization % in the last window. + */ +unsigned int sched_get_cpu_util(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + u64 util; + unsigned long capacity, flags; + unsigned int busy; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + raw_spin_lock_irqsave(&rq->lock, flags); + + capacity = capacity_orig_of(cpu); + + util = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; + util = div64_u64(util, sched_ravg_window >> SCHED_CAPACITY_SHIFT); + raw_spin_unlock_irqrestore(&rq->lock, flags); + + util = (util >= capacity) ? capacity : util; + busy = div64_ul((util * 100), capacity); + return busy; +} + +u64 sched_lpm_disallowed_time(int cpu) +{ + u64 now = sched_clock(); + u64 bias_end_time = atomic64_read(&per_cpu(busy_hyst_end_time, cpu)); + + if (now < bias_end_time) + return bias_end_time - now; + + return 0; +} +EXPORT_SYMBOL(sched_lpm_disallowed_time); diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c new file mode 100644 index 000000000000..6dd40e05cf13 --- /dev/null +++ b/kernel/sched/walt/sysctl.c @@ -0,0 +1,900 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include "walt.h" + +static int neg_three = -3; +static int three = 3; +static int two_hundred_fifty_five = 255; +static unsigned int ns_per_sec = NSEC_PER_SEC; +static unsigned int one_hundred_thousand = 100000; +static unsigned int two_hundred_million = 200000000; +static int __maybe_unused two = 2; +static int __maybe_unused four = 4; +static int one_hundred = 100; +static int one_thousand = 1000; + +/* + * CFS task prio range is [100 ... 139] + * 120 is the default prio. + * RTG boost range is [100 ... 119] because giving + * boost for [120 .. 139] does not make sense. + * 99 means disabled and it is the default value. + */ +static unsigned int min_cfs_boost_prio = 99; +static unsigned int max_cfs_boost_prio = 119; + +unsigned int sysctl_sched_capacity_margin_up_pct[MAX_MARGIN_LEVELS]; +unsigned int sysctl_sched_capacity_margin_dn_pct[MAX_MARGIN_LEVELS]; +unsigned int sysctl_sched_busy_hyst_enable_cpus; +unsigned int sysctl_sched_busy_hyst; +unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus; +unsigned int sysctl_sched_coloc_busy_hyst_cpu[WALT_NR_CPUS]; +unsigned int sysctl_sched_coloc_busy_hyst_max_ms; +unsigned int sysctl_sched_coloc_busy_hyst_cpu_busy_pct[WALT_NR_CPUS]; +unsigned int sysctl_sched_boost; +unsigned int sysctl_sched_wake_up_idle[2]; +unsigned int sysctl_input_boost_ms; +unsigned int sysctl_input_boost_freq[8]; +unsigned int sysctl_sched_boost_on_input; +unsigned int sysctl_sched_init_stage; +unsigned int sysctl_sched_load_boost[WALT_NR_CPUS]; + +/* sysctl nodes accesed by other files */ +unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; +unsigned int __read_mostly sysctl_sched_group_downmigrate_pct; +unsigned int __read_mostly sysctl_sched_group_upmigrate_pct; +unsigned int __read_mostly sysctl_sched_window_stats_policy; +unsigned int sysctl_sched_ravg_window_nr_ticks; +unsigned int sysctl_sched_dynamic_ravg_window_enable; +unsigned int sysctl_sched_walt_rotate_big_tasks; +unsigned int sysctl_sched_task_unfilter_period; +unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; +unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ +unsigned int sysctl_task_read_pid; +unsigned int sysctl_sched_conservative_pl; +unsigned int sysctl_sched_min_task_util_for_boost = 51; +unsigned int sysctl_sched_min_task_util_for_colocation = 35; +unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; +const int sched_user_hint_max = 1000; + +static void init_tg_pointers(void) +{ + struct cgroup_subsys_state *css = &root_task_group.css; + struct cgroup_subsys_state *top_css = css; + + /* ptrs are already initialized */ + if (task_group_topapp) + return; + + css_for_each_child(css, top_css) { + if (!strcmp(css->cgroup->kn->name, "top-app")) { + task_group_topapp = css_tg(css); + walt_init_topapp_tg(task_group_topapp); + } else if (!strcmp(css->cgroup->kn->name, "foreground")) { + task_group_foreground = css_tg(css); + walt_init_foreground_tg(task_group_foreground); + } else { + walt_init_tg(css_tg(css)); + } + } +} + +static int walt_init_stage_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + static DEFINE_MUTEX(mutex); + int old_value = sysctl_sched_init_stage; + + mutex_lock(&mutex); + + ret = proc_dointvec(table, write, buffer, lenp, ppos); + + if (ret || !write) + goto unlock; + + if (sysctl_sched_init_stage == 1 && + old_value != sysctl_sched_init_stage) { + init_tg_pointers(); + } + +unlock: + mutex_unlock(&mutex); + return ret; +} + +static int walt_proc_group_thresholds_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + static DEFINE_MUTEX(mutex); + struct rq *rq = cpu_rq(cpumask_first(cpu_possible_mask)); + unsigned long flags; + + if (unlikely(num_sched_clusters <= 0)) + return -EPERM; + + mutex_lock(&mutex); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write) { + mutex_unlock(&mutex); + return ret; + } + + /* + * The load scale factor update happens with all + * rqs locked. so acquiring 1 CPU rq lock and + * updating the thresholds is sufficient for + * an atomic update. + */ + raw_spin_lock_irqsave(&rq->lock, flags); + walt_update_group_thresholds(); + raw_spin_unlock_irqrestore(&rq->lock, flags); + + mutex_unlock(&mutex); + + return ret; +} + +static int walt_proc_user_hint_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + unsigned int old_value; + static DEFINE_MUTEX(mutex); + + mutex_lock(&mutex); + + sched_user_hint_reset_time = jiffies + HZ; + old_value = sysctl_sched_user_hint; + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write || (old_value == sysctl_sched_user_hint)) + goto unlock; + + walt_irq_work_queue(&walt_migration_irq_work); + +unlock: + mutex_unlock(&mutex); + return ret; +} + +static int sched_ravg_window_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret = -EPERM; + static DEFINE_MUTEX(mutex); + int val = sysctl_sched_ravg_window_nr_ticks; + + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .mode = table->mode, + }; + + mutex_lock(&mutex); + + if (write && (HZ != 250 || !sysctl_sched_dynamic_ravg_window_enable)) + goto unlock; + + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + if (ret || !write || (val == sysctl_sched_ravg_window_nr_ticks)) + goto unlock; + + if (val != 2 && val != 3 && val != 4 && val != 5 && val != 8) { + ret = -EINVAL; + goto unlock; + } + + sysctl_sched_ravg_window_nr_ticks = val; + sched_window_nr_ticks_change(); + +unlock: + mutex_unlock(&mutex); + return ret; +} + +enum { + TASK_BEGIN = 0, + WAKE_UP_IDLE, + INIT_TASK_LOAD, + GROUP_ID, + PER_TASK_BOOST, + PER_TASK_BOOST_PERIOD_MS, + LOW_LATENCY, +}; + +static int sched_task_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret, param; + struct task_struct *task; + int pid_and_val[2] = {-1, -1}; + int val; + struct walt_task_struct *wts; + + struct ctl_table tmp = { + .data = &pid_and_val, + .maxlen = sizeof(pid_and_val), + .mode = table->mode, + }; + static DEFINE_MUTEX(mutex); + + mutex_lock(&mutex); + + if (!write) { + if (sysctl_task_read_pid <= 0) { + ret = -ENOENT; + goto unlock_mutex; + } + task = get_pid_task(find_vpid(sysctl_task_read_pid), + PIDTYPE_PID); + if (!task) { + ret = -ENOENT; + goto put_task; + } + wts = (struct walt_task_struct *) task->android_vendor_data1; + pid_and_val[0] = sysctl_task_read_pid; + param = (unsigned long)table->data; + switch (param) { + case WAKE_UP_IDLE: + pid_and_val[1] = wts->wake_up_idle; + break; + case INIT_TASK_LOAD: + pid_and_val[1] = wts->init_load_pct; + break; + case GROUP_ID: + pid_and_val[1] = sched_get_group_id(task); + break; + case PER_TASK_BOOST: + pid_and_val[1] = wts->boost; + break; + case PER_TASK_BOOST_PERIOD_MS: + pid_and_val[1] = + div64_ul(wts->boost_period, + 1000000UL); + break; + case LOW_LATENCY: + pid_and_val[1] = wts->low_latency; + break; + default: + ret = -EINVAL; + goto put_task; + } + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + goto put_task; + } + + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + if (ret) + goto unlock_mutex; + + if (pid_and_val[0] <= 0 || pid_and_val[1] < 0) { + ret = -ENOENT; + goto unlock_mutex; + } + + /* parsed the values successfully in pid_and_val[] array */ + task = get_pid_task(find_vpid(pid_and_val[0]), PIDTYPE_PID); + if (!task) { + ret = -ENOENT; + goto unlock_mutex; + } + wts = (struct walt_task_struct *) task->android_vendor_data1; + param = (unsigned long)table->data; + val = pid_and_val[1]; + switch (param) { + case WAKE_UP_IDLE: + wts->wake_up_idle = val; + break; + case INIT_TASK_LOAD: + if (pid_and_val[1] < 0 || pid_and_val[1] > 100) { + ret = -EINVAL; + goto put_task; + } + wts->init_load_pct = val; + break; + case GROUP_ID: + ret = sched_set_group_id(task, val); + break; + case PER_TASK_BOOST: + if (val < TASK_BOOST_NONE || val >= TASK_BOOST_END) { + ret = -EINVAL; + goto put_task; + } + wts->boost = val; + if (val == 0) + wts->boost_period = 0; + break; + case PER_TASK_BOOST_PERIOD_MS: + if (wts->boost == 0 && val) { + /* setting boost period w/o boost is invalid */ + ret = -EINVAL; + goto put_task; + } + wts->boost_period = (u64)val * 1000 * 1000; + wts->boost_expires = sched_clock() + wts->boost_period; + break; + case LOW_LATENCY: + wts->low_latency = val; + break; + default: + ret = -EINVAL; + } + +put_task: + put_task_struct(task); +unlock_mutex: + mutex_unlock(&mutex); + + return ret; +} + +static int sched_load_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret, i; + unsigned int *data = (unsigned int *)table->data; + int val[WALT_NR_CPUS]; + + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .mode = table->mode, + }; + static DEFINE_MUTEX(mutex); + + mutex_lock(&mutex); + + if (!write) { + ret = proc_dointvec(table, write, buffer, lenp, ppos); + goto unlock_mutex; + } + + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + if (ret) + goto unlock_mutex; + + for (i = 0; i < WALT_NR_CPUS; i++) { + if (val[i] < -100 || val[i] > 1000) { + ret = -EINVAL; + goto unlock_mutex; + } + } + + /* all things checkout update the value */ + for (i = 0; i < WALT_NR_CPUS; i++) + data[i] = val[i]; + +unlock_mutex: + mutex_unlock(&mutex); + + return ret; +} + +#ifdef CONFIG_PROC_SYSCTL +static void sched_update_updown_migrate_values(bool up) +{ + int i = 0, cpu; + struct walt_sched_cluster *cluster; + int cap_margin_levels = num_sched_clusters - 1; + + if (cap_margin_levels > 1) { + /* + * No need to worry about CPUs in last cluster + * if there are more than 2 clusters in the system + */ + for_each_sched_cluster(cluster) { + for_each_cpu(cpu, &cluster->cpus) { + if (up) + sched_capacity_margin_up[cpu] = + SCHED_FIXEDPOINT_SCALE * 100 / + sysctl_sched_capacity_margin_up_pct[i]; + else + sched_capacity_margin_down[cpu] = + SCHED_FIXEDPOINT_SCALE * 100 / + sysctl_sched_capacity_margin_dn_pct[i]; + } + + if (++i >= cap_margin_levels) + break; + } + } else { + for_each_possible_cpu(cpu) { + if (up) + sched_capacity_margin_up[cpu] = + + SCHED_FIXEDPOINT_SCALE * 100 / + sysctl_sched_capacity_margin_up_pct[0]; + else + sched_capacity_margin_down[cpu] = + sysctl_sched_capacity_margin_dn_pct[0]; + } + } +} + +int sched_updown_migrate_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret, i; + unsigned int *data = (unsigned int *)table->data; + static DEFINE_MUTEX(mutex); + int cap_margin_levels = num_sched_clusters ? num_sched_clusters - 1 : 0; + int val[MAX_MARGIN_LEVELS]; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(int) * cap_margin_levels, + .mode = table->mode, + }; + + if (cap_margin_levels <= 0) + return -EINVAL; + + mutex_lock(&mutex); + + if (!write) { + ret = proc_dointvec(table, write, buffer, lenp, ppos); + goto unlock_mutex; + } + + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + if (ret) + goto unlock_mutex; + + /* check if valid pct values are passed in */ + for (i = 0; i < cap_margin_levels; i++) { + if (val[i] <= 0 || val[i] > 100) { + ret = -EINVAL; + goto unlock_mutex; + } + } + + /* check up pct is greater than dn pct */ + if (data == &sysctl_sched_capacity_margin_up_pct[0]) { + for (i = 0; i < cap_margin_levels; i++) { + if (val[i] < sysctl_sched_capacity_margin_dn_pct[i]) { + ret = -EINVAL; + goto unlock_mutex; + } + } + } else { + for (i = 0; i < cap_margin_levels; i++) { + if (sysctl_sched_capacity_margin_up_pct[i] < val[i]) { + ret = -EINVAL; + goto unlock_mutex; + } + } + } + + /* all things checkout update the value */ + for (i = 0; i < cap_margin_levels; i++) + data[i] = val[i]; + + /* update individual cpu thresholds */ + sched_update_updown_migrate_values(data == &sysctl_sched_capacity_margin_up_pct[0]); + +unlock_mutex: + mutex_unlock(&mutex); + + return ret; +} +#endif /* CONFIG_PROC_SYSCTL */ + +struct ctl_table input_boost_sysctls[] = { + { + .procname = "input_boost_ms", + .data = &sysctl_input_boost_ms, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred_thousand, + }, + { + .procname = "input_boost_freq", + .data = &sysctl_input_boost_freq, + .maxlen = sizeof(unsigned int) * 8, + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sched_boost_on_input", + .data = &sysctl_sched_boost_on_input, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { } +}; + +struct ctl_table walt_table[] = { + { + .procname = "sched_init_stage", + .data = &sysctl_sched_init_stage, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_init_stage_handler, + }, + { + .procname = "sched_user_hint", + .data = &sysctl_sched_user_hint, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_user_hint_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = (void *)&sched_user_hint_max, + }, + { + .procname = "sched_window_stats_policy", + .data = &sysctl_sched_window_stats_policy, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &four, + }, + { + .procname = "sched_group_upmigrate", + .data = &sysctl_sched_group_upmigrate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_group_thresholds_handler, + .extra1 = &sysctl_sched_group_downmigrate_pct, + }, + { + .procname = "sched_group_downmigrate", + .data = &sysctl_sched_group_downmigrate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_group_thresholds_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &sysctl_sched_group_upmigrate_pct, + }, + { + .procname = "sched_boost", + .data = &sysctl_sched_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_boost_handler, + .extra1 = &neg_three, + .extra2 = &three, + }, + { + .procname = "sched_conservative_pl", + .data = &sysctl_sched_conservative_pl, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_many_wakeup_threshold", + .data = &sysctl_sched_many_wakeup_threshold, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &two, + .extra2 = &one_thousand, + }, + { + .procname = "sched_walt_rotate_big_tasks", + .data = &sysctl_sched_walt_rotate_big_tasks, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_min_task_util_for_boost", + .data = &sysctl_sched_min_task_util_for_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_min_task_util_for_colocation", + .data = &sysctl_sched_min_task_util_for_colocation, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_asym_cap_sibling_freq_match_pct", + .data = &sysctl_sched_asym_cap_sibling_freq_match_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &one_hundred, + }, + { + .procname = "sched_coloc_downmigrate_ns", + .data = &sysctl_sched_coloc_downmigrate_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + }, + { + .procname = "sched_task_unfilter_period", + .data = &sysctl_sched_task_unfilter_period, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &two_hundred_million, + }, + { + .procname = "sched_busy_hysteresis_enable_cpus", + .data = &sysctl_sched_busy_hyst_enable_cpus, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_busy_hyst_ns", + .data = &sysctl_sched_busy_hyst, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &ns_per_sec, + }, + { + .procname = "sched_coloc_busy_hysteresis_enable_cpus", + .data = &sysctl_sched_coloc_busy_hyst_enable_cpus, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_coloc_busy_hyst_cpu_ns", + .data = &sysctl_sched_coloc_busy_hyst_cpu, + .maxlen = sizeof(unsigned int) * WALT_NR_CPUS, + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &ns_per_sec, + }, + { + .procname = "sched_coloc_busy_hyst_max_ms", + .data = &sysctl_sched_coloc_busy_hyst_max_ms, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred_thousand, + }, + { + .procname = "sched_coloc_busy_hyst_cpu_busy_pct", + .data = &sysctl_sched_coloc_busy_hyst_cpu_busy_pct, + .maxlen = sizeof(unsigned int) * WALT_NR_CPUS, + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred, + }, + { + .procname = "sched_ravg_window_nr_ticks", + .data = &sysctl_sched_ravg_window_nr_ticks, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_ravg_window_handler, + }, + { + .procname = "sched_dynamic_ravg_window_enable", + .data = &sysctl_sched_dynamic_ravg_window_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_upmigrate", + .data = &sysctl_sched_capacity_margin_up_pct, + .maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS, + .mode = 0644, + .proc_handler = sched_updown_migrate_handler, + }, + { + .procname = "sched_downmigrate", + .data = &sysctl_sched_capacity_margin_dn_pct, + .maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS, + .mode = 0644, + .proc_handler = sched_updown_migrate_handler, + }, + { + .procname = "sched_prefer_spread", + .data = &sysctl_sched_prefer_spread, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &four, + }, + { + .procname = "walt_rtg_cfs_boost_prio", + .data = &sysctl_walt_rtg_cfs_boost_prio, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &min_cfs_boost_prio, + .extra2 = &max_cfs_boost_prio, + }, + { + .procname = "walt_low_latency_task_threshold", + .data = &sysctl_walt_low_latency_task_threshold, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_force_lb_enable", + .data = &sysctl_sched_force_lb_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_lib_name", + .data = sched_lib_name, + .maxlen = LIB_PATH_LENGTH, + .mode = 0644, + .proc_handler = proc_dostring, + }, + { + .procname = "sched_lib_mask_force", + .data = &sched_lib_mask_force, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "input_boost", + .mode = 0555, + .child = input_boost_sysctls, + }, + { + .procname = "sched_wake_up_idle", + .data = (int *) WAKE_UP_IDLE, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_init_task_load", + .data = (int *) INIT_TASK_LOAD, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_group_id", + .data = (int *) GROUP_ID, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_per_task_boost", + .data = (int *) PER_TASK_BOOST, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_per_task_boost_period_ms", + .data = (int *) PER_TASK_BOOST_PERIOD_MS, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_low_latency", + .data = (int *) LOW_LATENCY, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_task_read_pid", + .data = &sysctl_task_read_pid, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_load_boost", + .data = &sysctl_sched_load_boost, + .maxlen = sizeof(unsigned int) * 8, + .mode = 0644, + .proc_handler = sched_load_boost_handler, + }, + { } +}; + +struct ctl_table walt_base_table[] = { + { + .procname = "walt", + .mode = 0555, + .child = walt_table, + }, + { }, +}; + +void walt_tunables(void) +{ + int i; + + for (i = 0; i < MAX_MARGIN_LEVELS; i++) { + sysctl_sched_capacity_margin_up_pct[i] = 95; /* ~5% margin */ + sysctl_sched_capacity_margin_dn_pct[i] = 85; /* ~15% margin */ + } + + sysctl_sched_group_upmigrate_pct = 100; + + sysctl_sched_group_downmigrate_pct = 95; + + sysctl_sched_asym_cap_sibling_freq_match_pct = 100; + + sysctl_sched_task_unfilter_period = 100000000; + + sysctl_sched_window_stats_policy = WINDOW_STATS_MAX_RECENT_AVG; + + sysctl_sched_ravg_window_nr_ticks = (HZ / NR_WINDOWS_PER_SEC); + + sysctl_sched_dynamic_ravg_window_enable = (HZ == 250); + + sched_load_granule = DEFAULT_SCHED_RAVG_WINDOW / NUM_LOAD_INDICES; + + sysctl_sched_min_task_util_for_boost = 51; + + sysctl_sched_min_task_util_for_colocation = 35; + + for (i = 0; i < WALT_NR_CPUS; i++) { + sysctl_sched_coloc_busy_hyst_cpu[i] = 39000000; + sysctl_sched_coloc_busy_hyst_cpu_busy_pct[i] = 10; + } + + sysctl_sched_coloc_busy_hyst_enable_cpus = 112; + + sysctl_sched_coloc_busy_hyst_max_ms = 5000; + + sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ + + sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; + + sysctl_input_boost_ms = 40; + + for (i = 0; i < 8; i++) + sysctl_input_boost_freq[i] = 0; +} diff --git a/kernel/sched/walt/trace.c b/kernel/sched/walt/trace.c new file mode 100644 index 000000000000..cf9c34077f2c --- /dev/null +++ b/kernel/sched/walt/trace.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + */ + +#include "walt.h" + +static inline void __window_data(u32 *dst, u32 *src) +{ + if (src) + memcpy(dst, src, nr_cpu_ids * sizeof(u32)); + else + memset(dst, 0, nr_cpu_ids * sizeof(u32)); +} + +struct trace_seq; +const char *__window_print(struct trace_seq *p, const u32 *buf, int buf_len) +{ + int i; + const char *ret = p->buffer + seq_buf_used(&p->seq); + + for (i = 0; i < buf_len; i++) + trace_seq_printf(p, "%u ", buf[i]); + + trace_seq_putc(p, 0); + + return ret; +} + +static inline s64 __rq_update_sum(struct rq *rq, bool curr, bool new) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (curr) + if (new) + return wrq->nt_curr_runnable_sum; + else + return wrq->curr_runnable_sum; + else + if (new) + return wrq->nt_prev_runnable_sum; + else + return wrq->prev_runnable_sum; +} + +static inline s64 __grp_update_sum(struct rq *rq, bool curr, bool new) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (curr) + if (new) + return wrq->grp_time.nt_curr_runnable_sum; + else + return wrq->grp_time.curr_runnable_sum; + else + if (new) + return wrq->grp_time.nt_prev_runnable_sum; + else + return wrq->grp_time.prev_runnable_sum; +} + +static inline s64 +__get_update_sum(struct rq *rq, enum migrate_types migrate_type, + bool src, bool new, bool curr) +{ + switch (migrate_type) { + case RQ_TO_GROUP: + if (src) + return __rq_update_sum(rq, curr, new); + else + return __grp_update_sum(rq, curr, new); + case GROUP_TO_RQ: + if (src) + return __grp_update_sum(rq, curr, new); + else + return __rq_update_sum(rq, curr, new); + default: + WARN_ON_ONCE(1); + return -EINVAL; + } +} + +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h new file mode 100644 index 000000000000..75be13d809bf --- /dev/null +++ b/kernel/sched/walt/trace.h @@ -0,0 +1,1097 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM schedwalt + +#if !defined(_TRACE_WALT_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_WALT_H + +#include + +#include "walt.h" + +struct rq; +struct group_cpu_time; +struct walt_task_struct; +struct walt_rq; +struct walt_related_thread_group; + +extern const char *task_event_names[]; + +TRACE_EVENT(sched_update_pred_demand, + + TP_PROTO(struct task_struct *p, u32 runtime, int pct, + unsigned int pred_demand, struct walt_task_struct *wts), + + TP_ARGS(p, runtime, pct, pred_demand, wts), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned int, runtime) + __field(int, pct) + __field(unsigned int, pred_demand) + __array(u8, bucket, NUM_BUSY_BUCKETS) + __field(int, cpu) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->pct = pct; + __entry->pred_demand = pred_demand; + memcpy(__entry->bucket, wts->busy_buckets, + NUM_BUSY_BUCKETS * sizeof(u8)); + __entry->cpu = task_cpu(p); + ), + + TP_printk("%d (%s): runtime %u pct %d cpu %d pred_demand %u (buckets: %u %u %u %u %u %u %u %u %u %u)", + __entry->pid, __entry->comm, + __entry->runtime, __entry->pct, __entry->cpu, + __entry->pred_demand, __entry->bucket[0], __entry->bucket[1], + __entry->bucket[2], __entry->bucket[3], __entry->bucket[4], + __entry->bucket[5], __entry->bucket[6], __entry->bucket[7], + __entry->bucket[8], __entry->bucket[9]) +); + +TRACE_EVENT(sched_update_history, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int samples, + enum task_event evt, struct walt_rq *wrq, struct walt_task_struct *wts), + + TP_ARGS(rq, p, runtime, samples, evt, wrq, wts), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned int, runtime) + __field(int, samples) + __field(enum task_event, evt) + __field(unsigned int, demand) + __field(unsigned int, coloc_demand) + __field(unsigned int, pred_demand) + __array(u32, hist, RAVG_HIST_SIZE_MAX) + __field(unsigned int, nr_big_tasks) + __field(int, cpu) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->samples = samples; + __entry->evt = evt; + __entry->demand = wts->demand; + __entry->coloc_demand = wts->coloc_demand; + __entry->pred_demand = wts->pred_demand; + memcpy(__entry->hist, wts->sum_history, + RAVG_HIST_SIZE_MAX * sizeof(u32)); + __entry->nr_big_tasks = wrq->walt_stats.nr_big_tasks; + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u samples %d event %s demand %u coloc_demand %u pred_demand %u (hist: %u %u %u %u %u) cpu %d nr_big %u", + __entry->pid, __entry->comm, + __entry->runtime, __entry->samples, + task_event_names[__entry->evt], + __entry->demand, __entry->coloc_demand, __entry->pred_demand, + __entry->hist[0], __entry->hist[1], + __entry->hist[2], __entry->hist[3], + __entry->hist[4], __entry->cpu, __entry->nr_big_tasks) +); + +TRACE_EVENT(sched_get_task_cpu_cycles, + + TP_PROTO(int cpu, int event, u64 cycles, + u64 exec_time, struct task_struct *p), + + TP_ARGS(cpu, event, cycles, exec_time, p), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, event) + __field(u64, cycles) + __field(u64, exec_time) + __field(u32, freq) + __field(u32, legacy_freq) + __field(u32, max_freq) + __field(pid_t, pid) + __array(char, comm, TASK_COMM_LEN) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->event = event; + __entry->cycles = cycles; + __entry->exec_time = exec_time; + __entry->freq = cpu_cycles_to_freq(cycles, exec_time); + __entry->legacy_freq = sched_cpu_legacy_freq(cpu); + __entry->max_freq = cpu_max_freq(cpu); + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + ), + + TP_printk("cpu=%d event=%d cycles=%llu exec_time=%llu freq=%u legacy_freq=%u max_freq=%u task=%d (%s)", + __entry->cpu, __entry->event, __entry->cycles, + __entry->exec_time, __entry->freq, __entry->legacy_freq, + __entry->max_freq, __entry->pid, __entry->comm) +); + +TRACE_EVENT(sched_update_task_ravg, + + TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, + u64 wallclock, u64 irqtime, + struct group_cpu_time *cpu_time, struct walt_rq *wrq, + struct walt_task_struct *wts), + + TP_ARGS(p, rq, evt, wallclock, irqtime, cpu_time, wrq, wts), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(pid_t, cur_pid) + __field(unsigned int, cur_freq) + __field(u64, wallclock) + __field(u64, mark_start) + __field(u64, delta_m) + __field(u64, win_start) + __field(u64, delta) + __field(u64, irqtime) + __field(enum task_event, evt) + __field(unsigned int, demand) + __field(unsigned int, coloc_demand) + __field(unsigned int, sum) + __field(int, cpu) + __field(unsigned int, pred_demand) + __field(u64, rq_cs) + __field(u64, rq_ps) + __field(u64, grp_cs) + __field(u64, grp_ps) + __field(u64, grp_nt_cs) + __field(u64, grp_nt_ps) + __field(u32, curr_window) + __field(u32, prev_window) + __dynamic_array(u32, curr_sum, nr_cpu_ids) + __dynamic_array(u32, prev_sum, nr_cpu_ids) + __field(u64, nt_cs) + __field(u64, nt_ps) + __field(u64, active_time) + __field(u32, curr_top) + __field(u32, prev_top) + ), + + TP_fast_assign( + __entry->wallclock = wallclock; + __entry->win_start = wrq->window_start; + __entry->delta = (wallclock - wrq->window_start); + __entry->evt = evt; + __entry->cpu = rq->cpu; + __entry->cur_pid = rq->curr->pid; + __entry->cur_freq = wrq->task_exec_scale; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->mark_start = wts->mark_start; + __entry->delta_m = (wallclock - wts->mark_start); + __entry->demand = wts->demand; + __entry->coloc_demand = wts->coloc_demand; + __entry->sum = wts->sum; + __entry->irqtime = irqtime; + __entry->pred_demand = wts->pred_demand; + __entry->rq_cs = wrq->curr_runnable_sum; + __entry->rq_ps = wrq->prev_runnable_sum; + __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; + __entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0; + __entry->grp_nt_cs = cpu_time ? + cpu_time->nt_curr_runnable_sum : 0; + __entry->grp_nt_ps = cpu_time ? + cpu_time->nt_prev_runnable_sum : 0; + __entry->curr_window = wts->curr_window; + __entry->prev_window = wts->prev_window; + __window_data(__get_dynamic_array(curr_sum), + wts->curr_window_cpu); + __window_data(__get_dynamic_array(prev_sum), + wts->prev_window_cpu); + __entry->nt_cs = wrq->nt_curr_runnable_sum; + __entry->nt_ps = wrq->nt_prev_runnable_sum; + __entry->active_time = wts->active_time; + __entry->curr_top = wrq->curr_top; + __entry->prev_top = wrq->prev_top; + ), + + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u", + __entry->wallclock, __entry->win_start, __entry->delta, + task_event_names[__entry->evt], __entry->cpu, + __entry->cur_freq, __entry->cur_pid, + __entry->pid, __entry->comm, __entry->mark_start, + __entry->delta_m, __entry->demand, __entry->coloc_demand, + __entry->sum, __entry->irqtime, __entry->pred_demand, + __entry->rq_cs, __entry->rq_ps, __entry->curr_window, + __window_print(p, __get_dynamic_array(curr_sum), nr_cpu_ids), + __entry->prev_window, + __window_print(p, __get_dynamic_array(prev_sum), nr_cpu_ids), + __entry->nt_cs, __entry->nt_ps, + __entry->active_time, __entry->grp_cs, + __entry->grp_ps, __entry->grp_nt_cs, __entry->grp_nt_ps, + __entry->curr_top, __entry->prev_top) +); + +TRACE_EVENT(sched_update_task_ravg_mini, + + TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, + u64 wallclock, u64 irqtime, + struct group_cpu_time *cpu_time, struct walt_rq *wrq, + struct walt_task_struct *wts), + + TP_ARGS(p, rq, evt, wallclock, irqtime, cpu_time, wrq, wts), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(u64, wallclock) + __field(u64, mark_start) + __field(u64, delta_m) + __field(u64, win_start) + __field(u64, delta) + __field(enum task_event, evt) + __field(unsigned int, demand) + __field(int, cpu) + __field(u64, rq_cs) + __field(u64, rq_ps) + __field(u64, grp_cs) + __field(u64, grp_ps) + __field(u32, curr_window) + __field(u32, prev_window) + ), + + TP_fast_assign( + __entry->wallclock = wallclock; + __entry->win_start = wrq->window_start; + __entry->delta = (wallclock - wrq->window_start); + __entry->evt = evt; + __entry->cpu = rq->cpu; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->mark_start = wts->mark_start; + __entry->delta_m = (wallclock - wts->mark_start); + __entry->demand = wts->demand; + __entry->rq_cs = wrq->curr_runnable_sum; + __entry->rq_ps = wrq->prev_runnable_sum; + __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; + __entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0; + __entry->curr_window = wts->curr_window; + __entry->prev_window = wts->prev_window; + ), + + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d task %d (%s) ms %llu delta %llu demand %u rq_cs %llu rq_ps %llu cur_window %u prev_window %u grp_cs %lld grp_ps %lld", + __entry->wallclock, __entry->win_start, __entry->delta, + task_event_names[__entry->evt], __entry->cpu, + __entry->pid, __entry->comm, __entry->mark_start, + __entry->delta_m, __entry->demand, + __entry->rq_cs, __entry->rq_ps, __entry->curr_window, + __entry->prev_window, __entry->grp_cs, __entry->grp_ps) +); + +struct migration_sum_data; +extern const char *migrate_type_names[]; + +TRACE_EVENT(sched_set_preferred_cluster, + + TP_PROTO(struct walt_related_thread_group *grp, u64 total_demand), + + TP_ARGS(grp, total_demand), + + TP_STRUCT__entry( + __field(int, id) + __field(u64, total_demand) + __field(bool, skip_min) + ), + + TP_fast_assign( + __entry->id = grp->id; + __entry->total_demand = total_demand; + __entry->skip_min = grp->skip_min; + ), + + TP_printk("group_id %d total_demand %llu skip_min %d", + __entry->id, __entry->total_demand, + __entry->skip_min) +); + +TRACE_EVENT(sched_migration_update_sum, + + TP_PROTO(struct task_struct *p, enum migrate_types migrate_type, + struct rq *rq), + + TP_ARGS(p, migrate_type, rq), + + TP_STRUCT__entry( + __field(int, tcpu) + __field(int, pid) + __field(enum migrate_types, migrate_type) + __field(s64, src_cs) + __field(s64, src_ps) + __field(s64, dst_cs) + __field(s64, dst_ps) + __field(s64, src_nt_cs) + __field(s64, src_nt_ps) + __field(s64, dst_nt_cs) + __field(s64, dst_nt_ps) + ), + + TP_fast_assign( + __entry->tcpu = task_cpu(p); + __entry->pid = p->pid; + __entry->migrate_type = migrate_type; + __entry->src_cs = __get_update_sum(rq, migrate_type, + true, false, true); + __entry->src_ps = __get_update_sum(rq, migrate_type, + true, false, false); + __entry->dst_cs = __get_update_sum(rq, migrate_type, + false, false, true); + __entry->dst_ps = __get_update_sum(rq, migrate_type, + false, false, false); + __entry->src_nt_cs = __get_update_sum(rq, migrate_type, + true, true, true); + __entry->src_nt_ps = __get_update_sum(rq, migrate_type, + true, true, false); + __entry->dst_nt_cs = __get_update_sum(rq, migrate_type, + false, true, true); + __entry->dst_nt_ps = __get_update_sum(rq, migrate_type, + false, true, false); + ), + + TP_printk("pid %d task_cpu %d migrate_type %s src_cs %llu src_ps %llu dst_cs %lld dst_ps %lld src_nt_cs %llu src_nt_ps %llu dst_nt_cs %lld dst_nt_ps %lld", + __entry->pid, __entry->tcpu, + migrate_type_names[__entry->migrate_type], + __entry->src_cs, __entry->src_ps, __entry->dst_cs, + __entry->dst_ps, __entry->src_nt_cs, __entry->src_nt_ps, + __entry->dst_nt_cs, __entry->dst_nt_ps) +); + +TRACE_EVENT(sched_set_boost, + + TP_PROTO(int type), + + TP_ARGS(type), + + TP_STRUCT__entry( + __field(int, type) + ), + + TP_fast_assign( + __entry->type = type; + ), + + TP_printk("type %d", __entry->type) +); + +TRACE_EVENT(sched_load_to_gov, + + TP_PROTO(struct rq *rq, u64 aggr_grp_load, u32 tt_load, + int freq_aggr, u64 load, int policy, + int big_task_rotation, + unsigned int user_hint, + struct walt_rq *wrq), + TP_ARGS(rq, aggr_grp_load, tt_load, freq_aggr, load, policy, + big_task_rotation, user_hint, wrq), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, policy) + __field(int, ed_task_pid) + __field(u64, aggr_grp_load) + __field(int, freq_aggr) + __field(u64, tt_load) + __field(u64, rq_ps) + __field(u64, grp_rq_ps) + __field(u64, nt_ps) + __field(u64, grp_nt_ps) + __field(u64, pl) + __field(u64, load) + __field(int, big_task_rotation) + __field(unsigned int, user_hint) + ), + + TP_fast_assign( + __entry->cpu = cpu_of(rq); + __entry->policy = policy; + __entry->ed_task_pid = + wrq->ed_task ? wrq->ed_task->pid : -1; + __entry->aggr_grp_load = aggr_grp_load; + __entry->freq_aggr = freq_aggr; + __entry->tt_load = tt_load; + __entry->rq_ps = wrq->prev_runnable_sum; + __entry->grp_rq_ps = wrq->grp_time.prev_runnable_sum; + __entry->nt_ps = wrq->nt_prev_runnable_sum; + __entry->grp_nt_ps = wrq->grp_time.nt_prev_runnable_sum; + __entry->pl = wrq->walt_stats.pred_demands_sum_scaled; + __entry->load = load; + __entry->big_task_rotation = big_task_rotation; + __entry->user_hint = user_hint; + ), + + TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr=%d tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d user_hint=%u", + __entry->cpu, __entry->policy, __entry->ed_task_pid, + __entry->aggr_grp_load, __entry->freq_aggr, + __entry->tt_load, __entry->rq_ps, __entry->grp_rq_ps, + __entry->nt_ps, __entry->grp_nt_ps, __entry->pl, __entry->load, + __entry->big_task_rotation, __entry->user_hint) +); + +TRACE_EVENT(core_ctl_eval_need, + + TP_PROTO(unsigned int cpu, unsigned int old_need, + unsigned int new_need, unsigned int updated), + TP_ARGS(cpu, old_need, new_need, updated), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, old_need) + __field(u32, new_need) + __field(u32, updated) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->old_need = old_need; + __entry->new_need = new_need; + __entry->updated = updated; + ), + TP_printk("cpu=%u, old_need=%u, new_need=%u, updated=%u", __entry->cpu, + __entry->old_need, __entry->new_need, __entry->updated) +); + +TRACE_EVENT(core_ctl_set_busy, + + TP_PROTO(unsigned int cpu, unsigned int busy, + unsigned int old_is_busy, unsigned int is_busy), + TP_ARGS(cpu, busy, old_is_busy, is_busy), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, busy) + __field(u32, old_is_busy) + __field(u32, is_busy) + __field(bool, high_irqload) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->busy = busy; + __entry->old_is_busy = old_is_busy; + __entry->is_busy = is_busy; + __entry->high_irqload = sched_cpu_high_irqload(cpu); + ), + TP_printk("cpu=%u, busy=%u, old_is_busy=%u, new_is_busy=%u high_irqload=%d", + __entry->cpu, __entry->busy, __entry->old_is_busy, + __entry->is_busy, __entry->high_irqload) +); + +TRACE_EVENT(core_ctl_set_boost, + + TP_PROTO(u32 refcount, s32 ret), + TP_ARGS(refcount, ret), + TP_STRUCT__entry( + __field(u32, refcount) + __field(s32, ret) + ), + TP_fast_assign( + __entry->refcount = refcount; + __entry->ret = ret; + ), + TP_printk("refcount=%u, ret=%d", __entry->refcount, __entry->ret) +); + +TRACE_EVENT(core_ctl_update_nr_need, + + TP_PROTO(int cpu, int nr_need, int prev_misfit_need, + int nrrun, int max_nr, int nr_prev_assist), + + TP_ARGS(cpu, nr_need, prev_misfit_need, nrrun, max_nr, nr_prev_assist), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, nr_need) + __field(int, prev_misfit_need) + __field(int, nrrun) + __field(int, max_nr) + __field(int, nr_prev_assist) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr_need = nr_need; + __entry->prev_misfit_need = prev_misfit_need; + __entry->nrrun = nrrun; + __entry->max_nr = max_nr; + __entry->nr_prev_assist = nr_prev_assist; + ), + + TP_printk("cpu=%d nr_need=%d prev_misfit_need=%d nrrun=%d max_nr=%d nr_prev_assist=%d", + __entry->cpu, __entry->nr_need, __entry->prev_misfit_need, + __entry->nrrun, __entry->max_nr, __entry->nr_prev_assist) +); + +TRACE_EVENT(core_ctl_notif_data, + + TP_PROTO(u32 nr_big, u32 ta_load, u32 *ta_util, u32 *cur_cap), + + TP_ARGS(nr_big, ta_load, ta_util, cur_cap), + + TP_STRUCT__entry( + __field(u32, nr_big) + __field(u32, ta_load) + __array(u32, ta_util, MAX_CLUSTERS) + __array(u32, cur_cap, MAX_CLUSTERS) + ), + + TP_fast_assign( + __entry->nr_big = nr_big; + __entry->ta_load = ta_load; + memcpy(__entry->ta_util, ta_util, MAX_CLUSTERS * sizeof(u32)); + memcpy(__entry->cur_cap, cur_cap, MAX_CLUSTERS * sizeof(u32)); + ), + + TP_printk("nr_big=%u ta_load=%u ta_util=(%u %u %u) cur_cap=(%u %u %u)", + __entry->nr_big, __entry->ta_load, + __entry->ta_util[0], __entry->ta_util[1], + __entry->ta_util[2], __entry->cur_cap[0], + __entry->cur_cap[1], __entry->cur_cap[2]) +); + +/* + * Tracepoint for sched_get_nr_running_avg + */ +TRACE_EVENT(sched_get_nr_running_avg, + + TP_PROTO(int cpu, int nr, int nr_misfit, int nr_max, int nr_scaled), + + TP_ARGS(cpu, nr, nr_misfit, nr_max, nr_scaled), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, nr) + __field(int, nr_misfit) + __field(int, nr_max) + __field(int, nr_scaled) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr = nr; + __entry->nr_misfit = nr_misfit; + __entry->nr_max = nr_max; + __entry->nr_scaled = nr_scaled; + ), + + TP_printk("cpu=%d nr=%d nr_misfit=%d nr_max=%d nr_scaled=%d", + __entry->cpu, __entry->nr, __entry->nr_misfit, __entry->nr_max, + __entry->nr_scaled) +); + +/* + * sched_pause - called when cores are paused/unpaused + * + * @start: 1 if start of pause/resume op, 0 otherwise + * @requested_cpus: mask of cpus requested in this op + * @active_cpus: mask of currently active cpus + * @start_time: time of the start of the operation + * @pause: 1 if pausing, 0 if resuming + */ +TRACE_EVENT(sched_pause, + + TP_PROTO(unsigned int start, unsigned int requested_cpus, unsigned int active_cpus, + u64 start_time, unsigned char pause), + + TP_ARGS(start, requested_cpus, active_cpus, start_time, pause), + + TP_STRUCT__entry( + __field(u32, start) + __field(u32, requested_cpus) + __field(u32, active_cpus) + __field(u32, time) + __field(unsigned char, pause) + ), + + TP_fast_assign( + __entry->start = start; + __entry->requested_cpus = requested_cpus; + __entry->active_cpus = active_cpus; + __entry->time = div64_u64(sched_clock() - start_time, 1000); + __entry->pause = pause; + ), + + TP_printk("start=%d req cpus=0x%x act cpus=0x%x time=%u us paused=%d", + __entry->start, __entry->requested_cpus, __entry->active_cpus, + __entry->time, __entry->pause) +); + +TRACE_EVENT(sched_ravg_window_change, + + TP_PROTO(unsigned int sched_ravg_window, unsigned int new_sched_ravg_window + , u64 change_time), + + TP_ARGS(sched_ravg_window, new_sched_ravg_window, change_time), + + TP_STRUCT__entry( + __field(unsigned int, sched_ravg_window) + __field(unsigned int, new_sched_ravg_window) + __field(u64, change_time) + ), + + TP_fast_assign( + __entry->sched_ravg_window = sched_ravg_window; + __entry->new_sched_ravg_window = new_sched_ravg_window; + __entry->change_time = change_time; + ), + + TP_printk("from=%u to=%u at=%lu", + __entry->sched_ravg_window, __entry->new_sched_ravg_window, + __entry->change_time) +); + +TRACE_EVENT(waltgov_util_update, + TP_PROTO(int cpu, + unsigned long util, unsigned long avg_cap, + unsigned long max_cap, unsigned long nl, unsigned long pl, + unsigned int rtgb, unsigned int flags), + TP_ARGS(cpu, util, avg_cap, max_cap, nl, pl, rtgb, flags), + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, util) + __field(unsigned long, avg_cap) + __field(unsigned long, max_cap) + __field(unsigned long, nl) + __field(unsigned long, pl) + __field(unsigned int, rtgb) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->util = util; + __entry->avg_cap = avg_cap; + __entry->max_cap = max_cap; + __entry->nl = nl; + __entry->pl = pl; + __entry->rtgb = rtgb; + __entry->flags = flags; + ), + TP_printk("cpu=%d util=%lu avg_cap=%lu max_cap=%lu nl=%lu pl=%lu rtgb=%u flags=0x%x", + __entry->cpu, __entry->util, __entry->avg_cap, + __entry->max_cap, __entry->nl, + __entry->pl, __entry->rtgb, __entry->flags) +); + +TRACE_EVENT(waltgov_next_freq, + TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, + unsigned int freq), + TP_ARGS(cpu, util, max, freq), + TP_STRUCT__entry( + __field(unsigned int, cpu) + __field(unsigned long, util) + __field(unsigned long, max) + __field(unsigned int, freq) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->util = util; + __entry->max = max; + __entry->freq = freq; + ), + TP_printk("cpu=%u util=%lu max=%lu freq=%u", + __entry->cpu, + __entry->util, + __entry->max, + __entry->freq) +); + +TRACE_EVENT(walt_active_load_balance, + + TP_PROTO(struct task_struct *p, int prev_cpu, int new_cpu, struct walt_task_struct *wts), + + TP_ARGS(p, prev_cpu, new_cpu, wts), + + TP_STRUCT__entry( + __field(pid_t, pid) + __field(bool, misfit) + __field(int, prev_cpu) + __field(int, new_cpu) + ), + + TP_fast_assign( + __entry->pid = p->pid; + __entry->misfit = wts->misfit; + __entry->prev_cpu = prev_cpu; + __entry->new_cpu = new_cpu; + ), + + TP_printk("pid=%d misfit=%d prev_cpu=%d new_cpu=%d\n", + __entry->pid, __entry->misfit, __entry->prev_cpu, + __entry->new_cpu) +); + +TRACE_EVENT(walt_find_busiest_queue, + + TP_PROTO(int dst_cpu, int busiest_cpu, unsigned long src_mask), + + TP_ARGS(dst_cpu, busiest_cpu, src_mask), + + TP_STRUCT__entry( + __field(int, dst_cpu) + __field(int, busiest_cpu) + __field(unsigned long, src_mask) + ), + + TP_fast_assign( + __entry->dst_cpu = dst_cpu; + __entry->busiest_cpu = busiest_cpu; + __entry->src_mask = src_mask; + ), + + TP_printk("dst_cpu=%d busiest_cpu=%d src_mask=%lx\n", + __entry->dst_cpu, __entry->busiest_cpu, + __entry->src_mask) +); + +TRACE_EVENT(walt_nohz_balance_kick, + + TP_PROTO(struct rq *rq), + + TP_ARGS(rq), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned int, nr_running) + __field(unsigned int, nr_cfs_running) + ), + + TP_fast_assign( + __entry->cpu = rq->cpu; + __entry->nr_running = rq->nr_running; + __entry->nr_cfs_running = rq->cfs.h_nr_running; + ), + + TP_printk("cpu=%d nr_running=%u nr_cfs_running=%u\n", + __entry->cpu, __entry->nr_running, + __entry->nr_cfs_running) +); + +TRACE_EVENT(walt_newidle_balance, + + TP_PROTO(int this_cpu, int busy_cpu, int pulled), + + TP_ARGS(this_cpu, busy_cpu, pulled), + + TP_STRUCT__entry( + __field(int, this_cpu) + __field(int, busy_cpu) + __field(int, pulled) + __field(unsigned int, this_nr_running) + ), + + TP_fast_assign( + __entry->this_cpu = this_cpu; + __entry->busy_cpu = busy_cpu; + __entry->pulled = pulled; + __entry->this_nr_running = cpu_rq(this_cpu)->nr_running; + ), + + TP_printk("this_cpu=%d busy_cpu=%d pulled=%d this_nr_running=%u\n", + __entry->this_cpu, __entry->busy_cpu, __entry->pulled, + __entry->this_nr_running) +); + +TRACE_EVENT(walt_lb_cpu_util, + + TP_PROTO(int cpu, struct walt_rq *wrq), + + TP_ARGS(cpu, wrq), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned int, nr_running) + __field(unsigned int, cfs_nr_running) + __field(unsigned int, nr_big) + __field(unsigned int, nr_rtg_high_prio_tasks) + __field(unsigned int, cpu_util) + __field(unsigned int, capacity_orig) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr_running = cpu_rq(cpu)->nr_running; + __entry->cfs_nr_running = cpu_rq(cpu)->cfs.h_nr_running; + __entry->nr_big = wrq->walt_stats.nr_big_tasks; + __entry->nr_rtg_high_prio_tasks = walt_nr_rtg_high_prio(cpu); + __entry->cpu_util = cpu_util(cpu); + __entry->capacity_orig = capacity_orig_of(cpu); + ), + + TP_printk("cpu=%d nr_running=%u cfs_nr_running=%u nr_big=%u nr_rtg_hp=%u cpu_util=%u capacity_orig=%u", + __entry->cpu, __entry->nr_running, __entry->cfs_nr_running, + __entry->nr_big, __entry->nr_rtg_high_prio_tasks, + __entry->cpu_util, __entry->capacity_orig) +); + +TRACE_EVENT(sched_cpu_util, + + TP_PROTO(int cpu), + + TP_ARGS(cpu), + + TP_STRUCT__entry( + __field(unsigned int, cpu) + __field(unsigned int, nr_running) + __field(long, cpu_util) + __field(long, cpu_util_cum) + __field(unsigned int, capacity_curr) + __field(unsigned int, capacity) + __field(unsigned int, capacity_orig) + __field(unsigned int, idle_exit_latency) + __field(u64, irqload) + __field(int, online) + __field(int, inactive) + __field(int, reserved) + __field(int, high_irq_load) + __field(unsigned int, nr_rtg_high_prio_tasks) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr_running = cpu_rq(cpu)->nr_running; + __entry->cpu_util = cpu_util(cpu); + __entry->cpu_util_cum = cpu_util_cum(cpu, 0); + __entry->capacity_curr = capacity_curr_of(cpu); + __entry->capacity = capacity_of(cpu); + __entry->capacity_orig = capacity_orig_of(cpu); + __entry->idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(cpu)); + __entry->irqload = sched_irqload(cpu); + __entry->online = cpu_online(cpu); + __entry->inactive = !cpu_active(cpu); + __entry->reserved = is_reserved(cpu); + __entry->high_irq_load = sched_cpu_high_irqload(cpu); + __entry->nr_rtg_high_prio_tasks = walt_nr_rtg_high_prio(cpu); + ), + + TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%u capacity=%u capacity_orig=%u idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u", + __entry->cpu, __entry->nr_running, __entry->cpu_util, + __entry->cpu_util_cum, __entry->capacity_curr, + __entry->capacity, __entry->capacity_orig, + __entry->idle_exit_latency, __entry->irqload, __entry->online, + __entry->inactive, __entry->reserved, __entry->high_irq_load, + __entry->nr_rtg_high_prio_tasks) +); + +TRACE_EVENT(sched_compute_energy, + + TP_PROTO(struct task_struct *p, int eval_cpu, + unsigned long eval_energy, + unsigned long prev_energy, + unsigned long best_energy, + unsigned long best_energy_cpu), + + TP_ARGS(p, eval_cpu, eval_energy, prev_energy, best_energy, + best_energy_cpu), + + TP_STRUCT__entry( + __field(int, pid) + __array(char, comm, TASK_COMM_LEN) + __field(unsigned long, util) + __field(int, prev_cpu) + __field(unsigned long, prev_energy) + __field(int, eval_cpu) + __field(unsigned long, eval_energy) + __field(int, best_energy_cpu) + __field(unsigned long, best_energy) + ), + + TP_fast_assign( + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->util = task_util(p); + __entry->prev_cpu = task_cpu(p); + __entry->prev_energy = prev_energy; + __entry->eval_cpu = eval_cpu; + __entry->eval_energy = eval_energy; + __entry->best_energy_cpu = best_energy_cpu; + __entry->best_energy = best_energy; + ), + + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d prev_energy=%lu eval_cpu=%d eval_energy=%lu best_energy_cpu=%d best_energy=%lu", + __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, + __entry->prev_energy, __entry->eval_cpu, __entry->eval_energy, + __entry->best_energy_cpu, __entry->best_energy) +) + +TRACE_EVENT(sched_task_util, + + TP_PROTO(struct task_struct *p, unsigned long candidates, + int best_energy_cpu, bool sync, int need_idle, int fastpath, + bool placement_boost, u64 start_t, + bool uclamp_boosted, bool is_rtg, bool rtg_skip_min, + int start_cpu), + + TP_ARGS(p, candidates, best_energy_cpu, sync, need_idle, fastpath, + placement_boost, start_t, uclamp_boosted, is_rtg, rtg_skip_min, + start_cpu), + + TP_STRUCT__entry( + __field(int, pid) + __array(char, comm, TASK_COMM_LEN) + __field(unsigned long, util) + __field(unsigned long, candidates) + __field(int, prev_cpu) + __field(int, best_energy_cpu) + __field(bool, sync) + __field(int, need_idle) + __field(int, fastpath) + __field(int, placement_boost) + __field(int, rtg_cpu) + __field(u64, latency) + __field(bool, uclamp_boosted) + __field(bool, is_rtg) + __field(bool, rtg_skip_min) + __field(int, start_cpu) + __field(u32, unfilter) + __field(unsigned long, cpus_allowed) + __field(int, task_boost) + __field(bool, low_latency) + ), + + TP_fast_assign( + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->util = task_util(p); + __entry->prev_cpu = task_cpu(p); + __entry->candidates = candidates; + __entry->best_energy_cpu = best_energy_cpu; + __entry->sync = sync; + __entry->need_idle = need_idle; + __entry->fastpath = fastpath; + __entry->placement_boost = placement_boost; + __entry->latency = (sched_clock() - start_t); + __entry->uclamp_boosted = uclamp_boosted; + __entry->is_rtg = is_rtg; + __entry->rtg_skip_min = rtg_skip_min; + __entry->start_cpu = start_cpu; + __entry->unfilter = + ((struct walt_task_struct *) p->android_vendor_data1)->unfilter; + __entry->cpus_allowed = cpumask_bits(&p->cpus_mask)[0]; + __entry->task_boost = per_task_boost(p); + __entry->low_latency = walt_low_latency_task(p); + ), + + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d", + __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, + __entry->candidates, __entry->best_energy_cpu, __entry->sync, + __entry->need_idle, __entry->fastpath, __entry->placement_boost, + __entry->latency, __entry->uclamp_boosted, + __entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu, + __entry->unfilter, __entry->cpus_allowed, __entry->task_boost, + __entry->low_latency) +); + +/* + * Tracepoint for find_best_target + */ +TRACE_EVENT(sched_find_best_target, + + TP_PROTO(struct task_struct *tsk, + unsigned long min_util, int start_cpu, + int best_idle, int most_spare_cap, int target, + int order_index, int end_index, + int skip, bool running), + + TP_ARGS(tsk, min_util, start_cpu, + best_idle, most_spare_cap, target, + order_index, end_index, skip, running), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned long, min_util) + __field(int, start_cpu) + __field(int, best_idle) + __field(int, most_spare_cap) + __field(int, target) + __field(int, order_index) + __field(int, end_index) + __field(int, skip) + __field(bool, running) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->min_util = min_util; + __entry->start_cpu = start_cpu; + __entry->best_idle = best_idle; + __entry->most_spare_cap = most_spare_cap; + __entry->target = target; + __entry->order_index = order_index; + __entry->end_index = end_index; + __entry->skip = skip; + __entry->running = running; + ), + + TP_printk("pid=%d comm=%s start_cpu=%d best_idle=%d most_spare_cap=%d target=%d order_index=%d end_index=%d skip=%d running=%d", + __entry->pid, __entry->comm, + __entry->start_cpu, + __entry->best_idle, + __entry->most_spare_cap, + __entry->target, + __entry->order_index, + __entry->end_index, + __entry->skip, + __entry->running) +); + +TRACE_EVENT(sched_enq_deq_task, + + TP_PROTO(struct task_struct *p, bool enqueue, + unsigned int cpus_allowed), + + TP_ARGS(p, enqueue, cpus_allowed), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, prio) + __field(int, cpu) + __field(bool, enqueue) + __field(unsigned int, nr_running) + __field(unsigned int, rt_nr_running) + __field(unsigned int, cpus_allowed) + __field(unsigned int, demand) + __field(unsigned int, pred_demand) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->cpu = task_cpu(p); + __entry->enqueue = enqueue; + __entry->nr_running = task_rq(p)->nr_running; + __entry->rt_nr_running = task_rq(p)->rt.rt_nr_running; + __entry->cpus_allowed = cpus_allowed; + __entry->demand = task_load(p); + __entry->pred_demand = task_pl(p); + ), + + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u", + __entry->cpu, + __entry->enqueue ? "enqueue" : "dequeue", + __entry->comm, __entry->pid, + __entry->prio, __entry->nr_running, + __entry->rt_nr_running, + __entry->cpus_allowed, __entry->demand, + __entry->pred_demand) +); +#endif /* _TRACE_WALT_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../kernel/sched/walt +#define TRACE_INCLUDE_FILE trace + +#include diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c new file mode 100644 index 000000000000..33822e50ec52 --- /dev/null +++ b/kernel/sched/walt/walt.c @@ -0,0 +1,4136 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "walt.h" +#include "trace.h" + +const char *task_event_names[] = { + "PUT_PREV_TASK", + "PICK_NEXT_TASK", + "TASK_WAKE", + "TASK_MIGRATE", + "TASK_UPDATE", + "IRQ_UPDATE" +}; + +const char *migrate_type_names[] = { + "GROUP_TO_RQ", + "RQ_TO_GROUP", + "RQ_TO_RQ", + "GROUP_TO_GROUP" +}; + +#define SCHED_FREQ_ACCOUNT_WAIT_TIME 0 +#define SCHED_ACCOUNT_WAIT_TIME 1 + +#define EARLY_DETECTION_DURATION 9500000 +#define MAX_NUM_CGROUP_COLOC_ID 20 + +#define MAX_NR_CLUSTERS 3 + +#define FREQ_REPORT_MAX_CPU_LOAD_TOP_TASK 0 +#define FREQ_REPORT_CPU_LOAD 1 +#define FREQ_REPORT_TOP_TASK 2 + +#define NEW_TASK_ACTIVE_TIME 100000000 + +unsigned int sysctl_sched_user_hint; + +static ktime_t ktime_last; +static bool sched_ktime_suspended; + +static bool use_cycle_counter; +static DEFINE_MUTEX(cluster_lock); +static u64 walt_load_reported_window; + +static struct irq_work walt_cpufreq_irq_work; +struct irq_work walt_migration_irq_work; +unsigned int walt_rotation_enabled; +cpumask_t asym_cap_sibling_cpus = CPU_MASK_NONE; +unsigned int sched_boost_type; +enum sched_boost_policy boost_policy; + +unsigned int __read_mostly sched_ravg_window = 20000000; +unsigned int min_max_possible_capacity = 1024; +unsigned int max_possible_capacity = 1024; /* max(rq->max_possible_capacity) */ +/* Initial task load. Newly created tasks are assigned this load. */ +unsigned int __read_mostly sched_init_task_load_windows; +/* + * Task load is categorized into buckets for the purpose of top task tracking. + * The entire range of load from 0 to sched_ravg_window needs to be covered + * in NUM_LOAD_INDICES number of buckets. Therefore the size of each bucket + * is given by sched_ravg_window / NUM_LOAD_INDICES. Since the default value + * of sched_ravg_window is DEFAULT_SCHED_RAVG_WINDOW, use that to compute + * sched_load_granule. + */ +unsigned int __read_mostly sched_load_granule; +__read_mostly bool sched_predl = true; + +/* + *@boost:should be 0,1,2. + *@period:boost time based on ms units. + */ +int set_task_boost(int boost, u64 period) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + + if (boost < TASK_BOOST_NONE || boost >= TASK_BOOST_END) + return -EINVAL; + if (boost) { + wts->boost = boost; + wts->boost_period = (u64)period * 1000 * 1000; + wts->boost_expires = sched_clock() + wts->boost_period; + } else { + wts->boost = 0; + wts->boost_expires = 0; + wts->boost_period = 0; + } + return 0; +} + +u64 sched_ktime_clock(void) +{ + if (unlikely(sched_ktime_suspended)) + return ktime_to_ns(ktime_last); + return ktime_get_ns(); +} + +static void sched_resume(void) +{ + sched_ktime_suspended = false; +} + +static int sched_suspend(void) +{ + ktime_last = ktime_get(); + sched_ktime_suspended = true; + return 0; +} + +static struct syscore_ops sched_syscore_ops = { + .resume = sched_resume, + .suspend = sched_suspend +}; + +int sched_init_ops(void) +{ + register_syscore_ops(&sched_syscore_ops); + return 0; +} + +void acquire_rq_locks_irqsave(const cpumask_t *cpus, + unsigned long *flags) +{ + int cpu; + int level = 0; + + local_irq_save(*flags); + + for_each_cpu(cpu, cpus) { + if (level == 0) + raw_spin_lock(&cpu_rq(cpu)->lock); + else + raw_spin_lock_nested(&cpu_rq(cpu)->lock, level); + level++; + } +} + +void release_rq_locks_irqrestore(const cpumask_t *cpus, + unsigned long *flags) +{ + int cpu; + + for_each_cpu(cpu, cpus) + raw_spin_unlock(&cpu_rq(cpu)->lock); + local_irq_restore(*flags); +} + +static unsigned int walt_cpu_high_irqload; + +__read_mostly unsigned int sched_ravg_hist_size = 5; + +static __read_mostly unsigned int sched_io_is_busy = 1; + +/* Window size (in ns) */ +__read_mostly unsigned int new_sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; + +static DEFINE_SPINLOCK(sched_ravg_window_lock); +u64 sched_ravg_window_change_time; + +/* + * A after-boot constant divisor for cpu_util_freq_walt() to apply the load + * boost. + */ +static __read_mostly unsigned int walt_cpu_util_freq_divisor; + +unsigned int __read_mostly sched_init_task_load_windows_scaled; +unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; + +/* Size of bitmaps maintained to track top tasks */ +static const unsigned int top_tasks_bitmap_size = + BITS_TO_LONGS(NUM_LOAD_INDICES + 1) * sizeof(unsigned long); + +/* + * This governs what load needs to be used when reporting CPU busy time + * to the cpufreq governor. + */ +__read_mostly unsigned int sysctl_sched_freq_reporting_policy; + +__read_mostly unsigned int walt_scale_demand_divisor; +#define scale_demand(d) ((d)/walt_scale_demand_divisor) + +#define SCHED_PRINT(arg) pr_emerg("%s=%llu", #arg, arg) +#define STRG(arg) #arg + +static inline void walt_task_dump(struct task_struct *p) +{ + char buff[WALT_NR_CPUS * 16]; + int i, j = 0; + int buffsz = WALT_NR_CPUS * 16; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + SCHED_PRINT(p->pid); + SCHED_PRINT(wts->mark_start); + SCHED_PRINT(wts->demand); + SCHED_PRINT(wts->coloc_demand); + SCHED_PRINT(sched_ravg_window); + SCHED_PRINT(new_sched_ravg_window); + + for (i = 0 ; i < nr_cpu_ids; i++) + j += scnprintf(buff + j, buffsz - j, "%u ", + wts->curr_window_cpu[i]); + printk_deferred("%s=%d (%s)\n", STRG(wts->curr_window), + wts->curr_window, buff); + + for (i = 0, j = 0 ; i < nr_cpu_ids; i++) + j += scnprintf(buff + j, buffsz - j, "%u ", + wts->prev_window_cpu[i]); + printk_deferred("%s=%d (%s)\n", STRG(wts->prev_window), + wts->prev_window, buff); + + SCHED_PRINT(wts->last_wake_ts); + SCHED_PRINT(wts->last_enqueued_ts); + SCHED_PRINT(wts->misfit); + SCHED_PRINT(wts->unfilter); +} + +static inline void walt_rq_dump(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct task_struct *tsk = cpu_curr(cpu); + int i; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + /* + * Increment the task reference so that it can't be + * freed on a remote CPU. Since we are going to + * enter panic, there is no need to decrement the + * task reference. Decrementing the task reference + * can't be done in atomic context, especially with + * rq locks held. + */ + get_task_struct(tsk); + pr_emerg("CPU:%d nr_running:%u current: %d (%s)\n", + cpu, rq->nr_running, tsk->pid, tsk->comm); + + printk_deferred("=========================================="); + SCHED_PRINT(wrq->window_start); + SCHED_PRINT(wrq->prev_window_size); + SCHED_PRINT(wrq->curr_runnable_sum); + SCHED_PRINT(wrq->prev_runnable_sum); + SCHED_PRINT(wrq->nt_curr_runnable_sum); + SCHED_PRINT(wrq->nt_prev_runnable_sum); + SCHED_PRINT(wrq->cum_window_demand_scaled); + SCHED_PRINT(wrq->task_exec_scale); + SCHED_PRINT(wrq->grp_time.curr_runnable_sum); + SCHED_PRINT(wrq->grp_time.prev_runnable_sum); + SCHED_PRINT(wrq->grp_time.nt_curr_runnable_sum); + SCHED_PRINT(wrq->grp_time.nt_prev_runnable_sum); + for (i = 0 ; i < NUM_TRACKED_WINDOWS; i++) { + printk_deferred("wrq->load_subs[%d].window_start=%llu)\n", i, + wrq->load_subs[i].window_start); + printk_deferred("wrq->load_subs[%d].subs=%llu)\n", i, + wrq->load_subs[i].subs); + printk_deferred("wrq->load_subs[%d].new_subs=%llu)\n", i, + wrq->load_subs[i].new_subs); + } + walt_task_dump(tsk); + SCHED_PRINT(sched_capacity_margin_up[cpu]); + SCHED_PRINT(sched_capacity_margin_down[cpu]); +} + +static inline void walt_dump(void) +{ + int cpu; + + pr_emerg("============ WALT RQ DUMP START ==============\n"); + pr_emerg("Sched ktime_get: %llu\n", sched_ktime_clock()); + pr_emerg("Time last window changed=%lu\n", + sched_ravg_window_change_time); + for_each_online_cpu(cpu) + walt_rq_dump(cpu); + SCHED_PRINT(max_possible_capacity); + SCHED_PRINT(min_max_possible_capacity); + pr_emerg("============ WALT RQ DUMP END ==============\n"); +} + +static int in_sched_bug; +#define SCHED_BUG_ON(condition) \ +({ \ + if (unlikely(!!(condition)) && !in_sched_bug) { \ + in_sched_bug = 1; \ + walt_dump(); \ + BUG_ON(condition); \ + } \ +}) + +static inline void +fixup_cumulative_runnable_avg(struct walt_sched_stats *stats, + s64 demand_scaled_delta, + s64 pred_demand_scaled_delta) +{ + stats->cumulative_runnable_avg_scaled += demand_scaled_delta; + BUG_ON((s64)stats->cumulative_runnable_avg_scaled < 0); + + stats->pred_demands_sum_scaled += pred_demand_scaled_delta; + BUG_ON((s64)stats->pred_demands_sum_scaled < 0); +} + +static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, + u16 updated_demand_scaled, + u16 updated_pred_demand_scaled) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + s64 task_load_delta = (s64)updated_demand_scaled - + wts->demand_scaled; + s64 pred_demand_delta = (s64)updated_pred_demand_scaled - + wts->pred_demand_scaled; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + fixup_cumulative_runnable_avg(&wrq->walt_stats, task_load_delta, + pred_demand_delta); +} + +/* + * Demand aggregation for frequency purpose: + * + * CPU demand of tasks from various related groups is aggregated per-cluster and + * added to the "max_busy_cpu" in that cluster, where max_busy_cpu is determined + * by just wrq->prev_runnable_sum. + * + * Some examples follow, which assume: + * Cluster0 = CPU0-3, Cluster1 = CPU4-7 + * One related thread group A that has tasks A0, A1, A2 + * + * A->cpu_time[X].curr/prev_sum = counters in which cpu execution stats of + * tasks belonging to group A are accumulated when they run on cpu X. + * + * CX->curr/prev_sum = counters in which cpu execution stats of all tasks + * not belonging to group A are accumulated when they run on cpu X + * + * Lets say the stats for window M was as below: + * + * C0->prev_sum = 1ms, A->cpu_time[0].prev_sum = 5ms + * Task A0 ran 5ms on CPU0 + * Task B0 ran 1ms on CPU0 + * + * C1->prev_sum = 5ms, A->cpu_time[1].prev_sum = 6ms + * Task A1 ran 4ms on CPU1 + * Task A2 ran 2ms on CPU1 + * Task B1 ran 5ms on CPU1 + * + * C2->prev_sum = 0ms, A->cpu_time[2].prev_sum = 0 + * CPU2 idle + * + * C3->prev_sum = 0ms, A->cpu_time[3].prev_sum = 0 + * CPU3 idle + * + * In this case, CPU1 was most busy going by just its prev_sum counter. Demand + * from all group A tasks are added to CPU1. IOW, at end of window M, cpu busy + * time reported to governor will be: + * + * + * C0 busy time = 1ms + * C1 busy time = 5 + 5 + 6 = 16ms + * + */ +__read_mostly bool sched_freq_aggr_en; + +static u64 +update_window_start(struct rq *rq, u64 wallclock, int event) +{ + s64 delta; + int nr_windows; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 old_window_start = wrq->window_start; + + delta = wallclock - wrq->window_start; + if (delta < 0) { + pr_emerg("WALT-BUG CPU%d; wallclock=%llu is lesser than window_start=%llu", + rq->cpu, wallclock, wrq->window_start); + SCHED_BUG_ON(1); + } + if (delta < sched_ravg_window) + return old_window_start; + + nr_windows = div64_u64(delta, sched_ravg_window); + wrq->window_start += (u64)nr_windows * (u64)sched_ravg_window; + + wrq->cum_window_demand_scaled = + wrq->walt_stats.cumulative_runnable_avg_scaled; + wrq->prev_window_size = sched_ravg_window; + + return old_window_start; +} + +/* + * Assumes rq_lock is held and wallclock was recorded in the same critical + * section as this function's invocation. + */ +static inline u64 read_cycle_counter(int cpu, u64 wallclock) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + if (wrq->last_cc_update != wallclock) { + wrq->cycles = qcom_cpufreq_get_cpu_cycle_counter(cpu); + wrq->last_cc_update = wallclock; + } + + return wrq->cycles; +} + +static void update_task_cpu_cycles(struct task_struct *p, int cpu, + u64 wallclock) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (use_cycle_counter) + wts->cpu_cycles = read_cycle_counter(cpu, wallclock); +} + +static inline bool is_ed_enabled(void) +{ + return (walt_rotation_enabled || (sched_boost_policy() != + SCHED_BOOST_NONE)); +} + +static inline bool is_ed_task(struct task_struct *p, u64 wallclock) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wallclock - wts->last_wake_ts >= EARLY_DETECTION_DURATION); +} + +static bool is_ed_task_present(struct rq *rq, u64 wallclock) +{ + struct task_struct *p; + int loop_max = 10; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + wrq->ed_task = NULL; + + if (!is_ed_enabled() || !rq->cfs.h_nr_running) + return false; + + list_for_each_entry(p, &rq->cfs_tasks, se.group_node) { + if (!loop_max) + break; + + if (is_ed_task(p, wallclock)) { + wrq->ed_task = p; + return true; + } + + loop_max--; + } + + return false; +} + +static void walt_sched_account_irqstart(int cpu, struct task_struct *curr) +{ + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (!wrq->window_start) + return; + + /* We're here without rq->lock held, IRQ disabled */ + raw_spin_lock(&rq->lock); + update_task_cpu_cycles(curr, cpu, sched_ktime_clock()); + raw_spin_unlock(&rq->lock); +} + +static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime); +static void walt_sched_account_irqend(int cpu, struct task_struct *curr, u64 delta) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + raw_spin_lock_irqsave(&rq->lock, flags); + walt_update_task_ravg(curr, rq, IRQ_UPDATE, sched_ktime_clock(), delta); + raw_spin_unlock_irqrestore(&rq->lock, flags); +} + +/* + * Return total number of tasks "eligible" to run on higher capacity cpus + */ +unsigned int walt_big_tasks(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->walt_stats.nr_big_tasks; +} + +void clear_walt_request(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + clear_reserved(cpu); + if (wrq->push_task) { + struct task_struct *push_task = NULL; + + raw_spin_lock_irqsave(&rq->lock, flags); + if (wrq->push_task) { + clear_reserved(rq->push_cpu); + push_task = wrq->push_task; + wrq->push_task = NULL; + } + rq->active_balance = 0; + raw_spin_unlock_irqrestore(&rq->lock, flags); + if (push_task) + put_task_struct(push_task); + } +} + +/* + * Special case the last index and provide a fast path for index = 0. + * Note that sched_load_granule can change underneath us if we are not + * holding any runqueue locks while calling the two functions below. + */ +static u32 top_task_load(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + int index = wrq->prev_top; + u8 prev = 1 - wrq->curr_table; + + if (!index) { + int msb = NUM_LOAD_INDICES - 1; + + if (!test_bit(msb, wrq->top_tasks_bitmap[prev])) + return 0; + else + return sched_load_granule; + } else if (index == NUM_LOAD_INDICES - 1) { + return sched_ravg_window; + } else { + return (index + 1) * sched_load_granule; + } +} + +unsigned long sched_user_hint_reset_time; +static bool is_cluster_hosting_top_app(struct walt_sched_cluster *cluster); + +static inline bool +should_apply_suh_freq_boost(struct walt_sched_cluster *cluster) +{ + if (sched_freq_aggr_en || !sysctl_sched_user_hint || + !cluster->aggr_grp_load) + return false; + + return is_cluster_hosting_top_app(cluster); +} + +static inline u64 freq_policy_load(struct rq *rq) +{ + unsigned int reporting_policy = sysctl_sched_freq_reporting_policy; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + u64 aggr_grp_load = cluster->aggr_grp_load; + u64 load, tt_load = 0; + struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu_of(rq)); + + if (wrq->ed_task != NULL) { + load = sched_ravg_window; + goto done; + } + + if (sched_freq_aggr_en) + load = wrq->prev_runnable_sum + aggr_grp_load; + else + load = wrq->prev_runnable_sum + + wrq->grp_time.prev_runnable_sum; + + if (cpu_ksoftirqd && cpu_ksoftirqd->state == TASK_RUNNING) + load = max_t(u64, load, task_load(cpu_ksoftirqd)); + + tt_load = top_task_load(rq); + switch (reporting_policy) { + case FREQ_REPORT_MAX_CPU_LOAD_TOP_TASK: + load = max_t(u64, load, tt_load); + break; + case FREQ_REPORT_TOP_TASK: + load = tt_load; + break; + case FREQ_REPORT_CPU_LOAD: + break; + default: + break; + } + + if (should_apply_suh_freq_boost(cluster)) { + if (is_suh_max()) + load = sched_ravg_window; + else + load = div64_u64(load * sysctl_sched_user_hint, + (u64)100); + } + +done: + trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en, + load, reporting_policy, walt_rotation_enabled, + sysctl_sched_user_hint, wrq); + return load; +} + +static bool rtgb_active; + +static inline unsigned long +__cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) +{ + u64 util, util_unboosted; + struct rq *rq = cpu_rq(cpu); + unsigned long capacity = capacity_orig_of(cpu); + int boost; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + boost = sysctl_sched_load_boost[cpu]; + util_unboosted = util = freq_policy_load(rq); + util = div64_u64(util * (100 + boost), + walt_cpu_util_freq_divisor); + + if (walt_load) { + u64 nl = wrq->nt_prev_runnable_sum + + wrq->grp_time.nt_prev_runnable_sum; + u64 pl = wrq->walt_stats.pred_demands_sum_scaled; + + /* do_pl_notif() needs unboosted signals */ + wrq->old_busy_time = div64_u64(util_unboosted, + sched_ravg_window >> + SCHED_CAPACITY_SHIFT); + wrq->old_estimated_time = pl; + + nl = div64_u64(nl * (100 + boost), walt_cpu_util_freq_divisor); + + walt_load->nl = nl; + walt_load->pl = pl; + walt_load->ws = walt_load_reported_window; + walt_load->rtgb_active = rtgb_active; + } + + return (util >= capacity) ? capacity : util; +} + +#define ADJUSTED_ASYM_CAP_CPU_UTIL(orig, other, x) \ + (max(orig, mult_frac(other, x, 100))) + +unsigned long +cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) +{ + struct walt_cpu_load wl_other = {0}; + unsigned long util = 0, util_other = 0; + unsigned long capacity = capacity_orig_of(cpu); + int i, mpct = sysctl_sched_asym_cap_sibling_freq_match_pct; + + if (!cpumask_test_cpu(cpu, &asym_cap_sibling_cpus)) + return __cpu_util_freq_walt(cpu, walt_load); + + for_each_cpu(i, &asym_cap_sibling_cpus) { + if (i == cpu) + util = __cpu_util_freq_walt(cpu, walt_load); + else + util_other = __cpu_util_freq_walt(i, &wl_other); + } + + if (cpu == cpumask_last(&asym_cap_sibling_cpus)) + mpct = 100; + + util = ADJUSTED_ASYM_CAP_CPU_UTIL(util, util_other, mpct); + + walt_load->nl = ADJUSTED_ASYM_CAP_CPU_UTIL(walt_load->nl, wl_other.nl, + mpct); + walt_load->pl = ADJUSTED_ASYM_CAP_CPU_UTIL(walt_load->pl, wl_other.pl, + mpct); + + return (util >= capacity) ? capacity : util; +} + +/* + * In this function we match the accumulated subtractions with the current + * and previous windows we are operating with. Ignore any entries where + * the window start in the load_subtraction struct does not match either + * the curent or the previous window. This could happen whenever CPUs + * become idle or busy with interrupts disabled for an extended period. + */ +static inline void account_load_subtractions(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 ws = wrq->window_start; + u64 prev_ws = ws - wrq->prev_window_size; + struct load_subtractions *ls = wrq->load_subs; + int i; + + for (i = 0; i < NUM_TRACKED_WINDOWS; i++) { + if (ls[i].window_start == ws) { + wrq->curr_runnable_sum -= ls[i].subs; + wrq->nt_curr_runnable_sum -= ls[i].new_subs; + } else if (ls[i].window_start == prev_ws) { + wrq->prev_runnable_sum -= ls[i].subs; + wrq->nt_prev_runnable_sum -= ls[i].new_subs; + } + + ls[i].subs = 0; + ls[i].new_subs = 0; + } + + SCHED_BUG_ON((s64)wrq->prev_runnable_sum < 0); + SCHED_BUG_ON((s64)wrq->curr_runnable_sum < 0); + SCHED_BUG_ON((s64)wrq->nt_prev_runnable_sum < 0); + SCHED_BUG_ON((s64)wrq->nt_curr_runnable_sum < 0); +} + +static inline void create_subtraction_entry(struct rq *rq, u64 ws, int index) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + wrq->load_subs[index].window_start = ws; + wrq->load_subs[index].subs = 0; + wrq->load_subs[index].new_subs = 0; +} + +static int get_top_index(unsigned long *bitmap, unsigned long old_top) +{ + int index = find_next_bit(bitmap, NUM_LOAD_INDICES, old_top); + + if (index == NUM_LOAD_INDICES) + return 0; + + return NUM_LOAD_INDICES - 1 - index; +} + +static bool get_subtraction_index(struct rq *rq, u64 ws) +{ + int i; + u64 oldest = ULLONG_MAX; + int oldest_index = 0; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + for (i = 0; i < NUM_TRACKED_WINDOWS; i++) { + u64 entry_ws = wrq->load_subs[i].window_start; + + if (ws == entry_ws) + return i; + + if (entry_ws < oldest) { + oldest = entry_ws; + oldest_index = i; + } + } + + create_subtraction_entry(rq, ws, oldest_index); + return oldest_index; +} + +static void update_rq_load_subtractions(int index, struct rq *rq, + u32 sub_load, bool new_task) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + wrq->load_subs[index].subs += sub_load; + if (new_task) + wrq->load_subs[index].new_subs += sub_load; +} + +static inline struct walt_sched_cluster *cpu_cluster(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster; +} + +static void update_cluster_load_subtractions(struct task_struct *p, + int cpu, u64 ws, bool new_task) +{ + struct walt_sched_cluster *cluster = cpu_cluster(cpu); + struct cpumask cluster_cpus = cluster->cpus; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + u64 prev_ws = ws - wrq->prev_window_size; + int i; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + cpumask_clear_cpu(cpu, &cluster_cpus); + raw_spin_lock(&cluster->load_lock); + + for_each_cpu(i, &cluster_cpus) { + struct rq *rq = cpu_rq(i); + int index; + + if (wts->curr_window_cpu[i]) { + index = get_subtraction_index(rq, ws); + update_rq_load_subtractions(index, rq, + wts->curr_window_cpu[i], new_task); + wts->curr_window_cpu[i] = 0; + } + + if (wts->prev_window_cpu[i]) { + index = get_subtraction_index(rq, prev_ws); + update_rq_load_subtractions(index, rq, + wts->prev_window_cpu[i], new_task); + wts->prev_window_cpu[i] = 0; + } + } + + raw_spin_unlock(&cluster->load_lock); +} + +static inline void inter_cluster_migration_fixup + (struct task_struct *p, int new_cpu, int task_cpu, bool new_task) +{ + struct rq *dest_rq = cpu_rq(new_cpu); + struct rq *src_rq = cpu_rq(task_cpu); + struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (same_freq_domain(new_cpu, task_cpu)) + return; + + wts->curr_window_cpu[new_cpu] = wts->curr_window; + wts->prev_window_cpu[new_cpu] = wts->prev_window; + + dest_wrq->curr_runnable_sum += wts->curr_window; + dest_wrq->prev_runnable_sum += wts->prev_window; + + if (src_wrq->curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { + printk_deferred("WALT-BUG pid=%u CPU%d -> CPU%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, dest_rq->cpu, + src_wrq->curr_runnable_sum, + wts->curr_window_cpu[task_cpu]); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + src_wrq->curr_runnable_sum -= wts->curr_window_cpu[task_cpu]; + + if (src_wrq->prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { + printk_deferred("WALT-BUG pid=%u CPU%d -> CPU%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, dest_rq->cpu, + src_wrq->prev_runnable_sum, + wts->prev_window_cpu[task_cpu]); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + src_wrq->prev_runnable_sum -= wts->prev_window_cpu[task_cpu]; + + if (new_task) { + dest_wrq->nt_curr_runnable_sum += wts->curr_window; + dest_wrq->nt_prev_runnable_sum += wts->prev_window; + + if (src_wrq->nt_curr_runnable_sum < + wts->curr_window_cpu[task_cpu]) { + printk_deferred("WALT-BUG pid=%u CPU%d -> CPU%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, dest_rq->cpu, + src_wrq->nt_curr_runnable_sum, + wts->curr_window_cpu[task_cpu]); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + src_wrq->nt_curr_runnable_sum -= + wts->curr_window_cpu[task_cpu]; + + if (src_wrq->nt_prev_runnable_sum < + wts->prev_window_cpu[task_cpu]) { + printk_deferred("WALT-BUG pid=%u CPU%d -> CPU%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, dest_rq->cpu, + src_wrq->nt_prev_runnable_sum, + wts->prev_window_cpu[task_cpu]); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + src_wrq->nt_prev_runnable_sum -= + wts->prev_window_cpu[task_cpu]; + } + + wts->curr_window_cpu[task_cpu] = 0; + wts->prev_window_cpu[task_cpu] = 0; + + update_cluster_load_subtractions(p, task_cpu, + src_wrq->window_start, new_task); +} + +static u32 load_to_index(u32 load) +{ + u32 index = load / sched_load_granule; + + return min(index, (u32)(NUM_LOAD_INDICES - 1)); +} + +static void +migrate_top_tasks(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) +{ + int index; + int top_index; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u32 curr_window = wts->curr_window; + u32 prev_window = wts->prev_window; + struct walt_rq *dst_wrq = (struct walt_rq *) dst_rq->android_vendor_data1; + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + u8 src = src_wrq->curr_table; + u8 dst = dst_wrq->curr_table; + u8 *src_table; + u8 *dst_table; + + if (curr_window) { + src_table = src_wrq->top_tasks[src]; + dst_table = dst_wrq->top_tasks[dst]; + index = load_to_index(curr_window); + src_table[index] -= 1; + dst_table[index] += 1; + + if (!src_table[index]) + __clear_bit(NUM_LOAD_INDICES - index - 1, + src_wrq->top_tasks_bitmap[src]); + + if (dst_table[index] == 1) + __set_bit(NUM_LOAD_INDICES - index - 1, + dst_wrq->top_tasks_bitmap[dst]); + + if (index > dst_wrq->curr_top) + dst_wrq->curr_top = index; + + top_index = src_wrq->curr_top; + if (index == top_index && !src_table[index]) + src_wrq->curr_top = get_top_index( + src_wrq->top_tasks_bitmap[src], top_index); + } + + if (prev_window) { + src = 1 - src; + dst = 1 - dst; + src_table = src_wrq->top_tasks[src]; + dst_table = dst_wrq->top_tasks[dst]; + index = load_to_index(prev_window); + src_table[index] -= 1; + dst_table[index] += 1; + + if (!src_table[index]) + __clear_bit(NUM_LOAD_INDICES - index - 1, + src_wrq->top_tasks_bitmap[src]); + + if (dst_table[index] == 1) + __set_bit(NUM_LOAD_INDICES - index - 1, + dst_wrq->top_tasks_bitmap[dst]); + + if (index > dst_wrq->prev_top) + dst_wrq->prev_top = index; + + top_index = src_wrq->prev_top; + if (index == top_index && !src_table[index]) + src_wrq->prev_top = get_top_index( + src_wrq->top_tasks_bitmap[src], top_index); + } +} + +static inline bool is_new_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->active_time < NEW_TASK_ACTIVE_TIME; +} + +static void fixup_busy_time(struct task_struct *p, int new_cpu) +{ + struct rq *src_rq = task_rq(p); + struct rq *dest_rq = cpu_rq(new_cpu); + u64 wallclock; + u64 *src_curr_runnable_sum, *dst_curr_runnable_sum; + u64 *src_prev_runnable_sum, *dst_prev_runnable_sum; + u64 *src_nt_curr_runnable_sum, *dst_nt_curr_runnable_sum; + u64 *src_nt_prev_runnable_sum, *dst_nt_prev_runnable_sum; + bool new_task; + struct walt_related_thread_group *grp; + long pstate; + struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!p->on_rq && p->state != TASK_WAKING) + return; + + pstate = p->state; + + if (pstate == TASK_WAKING) + double_rq_lock(src_rq, dest_rq); + + wallclock = sched_ktime_clock(); + + walt_update_task_ravg(task_rq(p)->curr, task_rq(p), + TASK_UPDATE, + wallclock, 0); + walt_update_task_ravg(dest_rq->curr, dest_rq, + TASK_UPDATE, wallclock, 0); + + walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, + wallclock, 0); + + update_task_cpu_cycles(p, new_cpu, wallclock); + + new_task = is_new_task(p); + /* Protected by rq_lock */ + grp = wts->grp; + + /* + * For frequency aggregation, we continue to do migration fixups + * even for intra cluster migrations. This is because, the aggregated + * load has to reported on a single CPU regardless. + */ + if (grp) { + struct group_cpu_time *cpu_time; + + cpu_time = &src_wrq->grp_time; + src_curr_runnable_sum = &cpu_time->curr_runnable_sum; + src_prev_runnable_sum = &cpu_time->prev_runnable_sum; + src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + cpu_time = &dest_wrq->grp_time; + dst_curr_runnable_sum = &cpu_time->curr_runnable_sum; + dst_prev_runnable_sum = &cpu_time->prev_runnable_sum; + dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + if (wts->curr_window) { + *src_curr_runnable_sum -= wts->curr_window; + *dst_curr_runnable_sum += wts->curr_window; + if (new_task) { + *src_nt_curr_runnable_sum -= wts->curr_window; + *dst_nt_curr_runnable_sum += wts->curr_window; + } + } + + if (wts->prev_window) { + *src_prev_runnable_sum -= wts->prev_window; + *dst_prev_runnable_sum += wts->prev_window; + if (new_task) { + *src_nt_prev_runnable_sum -= wts->prev_window; + *dst_nt_prev_runnable_sum += wts->prev_window; + } + } + } else { + inter_cluster_migration_fixup(p, new_cpu, + task_cpu(p), new_task); + } + + migrate_top_tasks(p, src_rq, dest_rq); + + if (!same_freq_domain(new_cpu, task_cpu(p))) { + src_wrq->notif_pending = true; + dest_wrq->notif_pending = true; + walt_irq_work_queue(&walt_migration_irq_work); + } + + if (is_ed_enabled()) { + if (p == src_wrq->ed_task) { + src_wrq->ed_task = NULL; + dest_wrq->ed_task = p; + } else if (is_ed_task(p, wallclock)) { + dest_wrq->ed_task = p; + } + } + + if (pstate == TASK_WAKING) + double_rq_unlock(src_rq, dest_rq); +} + +static void set_window_start(struct rq *rq) +{ + static int sync_cpu_available; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *sync_wrq; + struct walt_task_struct *wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; + + if (likely(wrq->window_start)) + return; + + if (!sync_cpu_available) { + wrq->window_start = 1; + sync_cpu_available = 1; + atomic64_set(&walt_irq_work_lastq_ws, wrq->window_start); + walt_load_reported_window = + atomic64_read(&walt_irq_work_lastq_ws); + + } else { + struct rq *sync_rq = cpu_rq(cpumask_any(cpu_online_mask)); + + sync_wrq = (struct walt_rq *) sync_rq->android_vendor_data1; + raw_spin_unlock(&rq->lock); + double_rq_lock(rq, sync_rq); + wrq->window_start = sync_wrq->window_start; + wrq->curr_runnable_sum = wrq->prev_runnable_sum = 0; + wrq->nt_curr_runnable_sum = wrq->nt_prev_runnable_sum = 0; + raw_spin_unlock(&sync_rq->lock); + } + + wts->mark_start = wrq->window_start; +} + +#define INC_STEP 8 +#define DEC_STEP 2 +#define CONSISTENT_THRES 16 +#define INC_STEP_BIG 16 +/* + * bucket_increase - update the count of all buckets + * + * @buckets: array of buckets tracking busy time of a task + * @idx: the index of bucket to be incremented + * + * Each time a complete window finishes, count of bucket that runtime + * falls in (@idx) is incremented. Counts of all other buckets are + * decayed. The rate of increase and decay could be different based + * on current count in the bucket. + */ +static inline void bucket_increase(u8 *buckets, int idx) +{ + int i, step; + + for (i = 0; i < NUM_BUSY_BUCKETS; i++) { + if (idx != i) { + if (buckets[i] > DEC_STEP) + buckets[i] -= DEC_STEP; + else + buckets[i] = 0; + } else { + step = buckets[i] >= CONSISTENT_THRES ? + INC_STEP_BIG : INC_STEP; + if (buckets[i] > U8_MAX - step) + buckets[i] = U8_MAX; + else + buckets[i] += step; + } + } +} + +static inline int busy_to_bucket(u32 normalized_rt) +{ + int bidx; + + bidx = mult_frac(normalized_rt, NUM_BUSY_BUCKETS, max_task_load()); + bidx = min(bidx, NUM_BUSY_BUCKETS - 1); + + /* + * Combine lowest two buckets. The lowest frequency falls into + * 2nd bucket and thus keep predicting lowest bucket is not + * useful. + */ + if (!bidx) + bidx++; + + return bidx; +} + +/* + * get_pred_busy - calculate predicted demand for a task on runqueue + * + * @p: task whose prediction is being updated + * @start: starting bucket. returned prediction should not be lower than + * this bucket. + * @runtime: runtime of the task. returned prediction should not be lower + * than this runtime. + * Note: @start can be derived from @runtime. It's passed in only to + * avoid duplicated calculation in some cases. + * + * A new predicted busy time is returned for task @p based on @runtime + * passed in. The function searches through buckets that represent busy + * time equal to or bigger than @runtime and attempts to find the bucket + * to use for prediction. Once found, it searches through historical busy + * time and returns the latest that falls into the bucket. If no such busy + * time exists, it returns the medium of that bucket. + */ +static u32 get_pred_busy(struct task_struct *p, + int start, u32 runtime) +{ + int i; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u8 *buckets = wts->busy_buckets; + u32 *hist = wts->sum_history; + u32 dmin, dmax; + u64 cur_freq_runtime = 0; + int first = NUM_BUSY_BUCKETS, final; + u32 ret = runtime; + + /* skip prediction for new tasks due to lack of history */ + if (unlikely(is_new_task(p))) + goto out; + + /* find minimal bucket index to pick */ + for (i = start; i < NUM_BUSY_BUCKETS; i++) { + if (buckets[i]) { + first = i; + break; + } + } + /* if no higher buckets are filled, predict runtime */ + if (first >= NUM_BUSY_BUCKETS) + goto out; + + /* compute the bucket for prediction */ + final = first; + + /* determine demand range for the predicted bucket */ + if (final < 2) { + /* lowest two buckets are combined */ + dmin = 0; + final = 1; + } else { + dmin = mult_frac(final, max_task_load(), NUM_BUSY_BUCKETS); + } + dmax = mult_frac(final + 1, max_task_load(), NUM_BUSY_BUCKETS); + + /* + * search through runtime history and return first runtime that falls + * into the range of predicted bucket. + */ + for (i = 0; i < sched_ravg_hist_size; i++) { + if (hist[i] >= dmin && hist[i] < dmax) { + ret = hist[i]; + break; + } + } + /* no historical runtime within bucket found, use average of the bin */ + if (ret < dmin) + ret = (dmin + dmax) / 2; + /* + * when updating in middle of a window, runtime could be higher + * than all recorded history. Always predict at least runtime. + */ + ret = max(runtime, ret); +out: + trace_sched_update_pred_demand(p, runtime, + mult_frac((unsigned int)cur_freq_runtime, 100, + sched_ravg_window), ret, wts); + return ret; +} + +static inline u32 calc_pred_demand(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (wts->pred_demand >= wts->curr_window) + return wts->pred_demand; + + return get_pred_busy(p, busy_to_bucket(wts->curr_window), + wts->curr_window); +} + +/* + * predictive demand of a task is calculated at the window roll-over. + * if the task current window busy time exceeds the predicted + * demand, update it here to reflect the task needs. + */ +static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int event) +{ + u32 new, old; + u16 new_scaled; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!sched_predl) + return; + + if (is_idle_task(p)) + return; + + if (event != PUT_PREV_TASK && event != TASK_UPDATE && + (!SCHED_FREQ_ACCOUNT_WAIT_TIME || + (event != TASK_MIGRATE && + event != PICK_NEXT_TASK))) + return; + + /* + * TASK_UPDATE can be called on sleeping task, when its moved between + * related groups + */ + if (event == TASK_UPDATE) { + if (!p->on_rq && !SCHED_FREQ_ACCOUNT_WAIT_TIME) + return; + } + + new = calc_pred_demand(p); + old = wts->pred_demand; + + if (old >= new) + return; + + new_scaled = scale_demand(new); + if (task_on_rq_queued(p) && (!task_has_dl_policy(p) || + !p->dl.dl_throttled)) + fixup_walt_sched_stats_common(rq, p, + wts->demand_scaled, + new_scaled); + + wts->pred_demand = new; + wts->pred_demand_scaled = new_scaled; +} + +static void clear_top_tasks_bitmap(unsigned long *bitmap) +{ + memset(bitmap, 0, top_tasks_bitmap_size); + __set_bit(NUM_LOAD_INDICES, bitmap); +} + +static inline void clear_top_tasks_table(u8 *table) +{ + memset(table, 0, NUM_LOAD_INDICES * sizeof(u8)); +} + +static void update_top_tasks(struct task_struct *p, struct rq *rq, + u32 old_curr_window, int new_window, bool full_window) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u8 curr = wrq->curr_table; + u8 prev = 1 - curr; + u8 *curr_table = wrq->top_tasks[curr]; + u8 *prev_table = wrq->top_tasks[prev]; + int old_index, new_index, update_index; + u32 curr_window = wts->curr_window; + u32 prev_window = wts->prev_window; + bool zero_index_update; + + if (old_curr_window == curr_window && !new_window) + return; + + old_index = load_to_index(old_curr_window); + new_index = load_to_index(curr_window); + + if (!new_window) { + zero_index_update = !old_curr_window && curr_window; + if (old_index != new_index || zero_index_update) { + if (old_curr_window) + curr_table[old_index] -= 1; + if (curr_window) + curr_table[new_index] += 1; + if (new_index > wrq->curr_top) + wrq->curr_top = new_index; + } + + if (!curr_table[old_index]) + __clear_bit(NUM_LOAD_INDICES - old_index - 1, + wrq->top_tasks_bitmap[curr]); + + if (curr_table[new_index] == 1) + __set_bit(NUM_LOAD_INDICES - new_index - 1, + wrq->top_tasks_bitmap[curr]); + + return; + } + + /* + * The window has rolled over for this task. By the time we get + * here, curr/prev swaps would has already occurred. So we need + * to use prev_window for the new index. + */ + update_index = load_to_index(prev_window); + + if (full_window) { + /* + * Two cases here. Either 'p' ran for the entire window or + * it didn't run at all. In either case there is no entry + * in the prev table. If 'p' ran the entire window, we just + * need to create a new entry in the prev table. In this case + * update_index will be correspond to sched_ravg_window + * so we can unconditionally update the top index. + */ + if (prev_window) { + prev_table[update_index] += 1; + wrq->prev_top = update_index; + } + + if (prev_table[update_index] == 1) + __set_bit(NUM_LOAD_INDICES - update_index - 1, + wrq->top_tasks_bitmap[prev]); + } else { + zero_index_update = !old_curr_window && prev_window; + if (old_index != update_index || zero_index_update) { + if (old_curr_window) + prev_table[old_index] -= 1; + + prev_table[update_index] += 1; + + if (update_index > wrq->prev_top) + wrq->prev_top = update_index; + + if (!prev_table[old_index]) + __clear_bit(NUM_LOAD_INDICES - old_index - 1, + wrq->top_tasks_bitmap[prev]); + + if (prev_table[update_index] == 1) + __set_bit(NUM_LOAD_INDICES - update_index - 1, + wrq->top_tasks_bitmap[prev]); + } + } + + if (curr_window) { + curr_table[new_index] += 1; + + if (new_index > wrq->curr_top) + wrq->curr_top = new_index; + + if (curr_table[new_index] == 1) + __set_bit(NUM_LOAD_INDICES - new_index - 1, + wrq->top_tasks_bitmap[curr]); + } +} + +static void rollover_top_tasks(struct rq *rq, bool full_window) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u8 curr_table = wrq->curr_table; + u8 prev_table = 1 - curr_table; + int curr_top = wrq->curr_top; + + clear_top_tasks_table(wrq->top_tasks[prev_table]); + clear_top_tasks_bitmap(wrq->top_tasks_bitmap[prev_table]); + + if (full_window) { + curr_top = 0; + clear_top_tasks_table(wrq->top_tasks[curr_table]); + clear_top_tasks_bitmap(wrq->top_tasks_bitmap[curr_table]); + } + + wrq->curr_table = prev_table; + wrq->prev_top = curr_top; + wrq->curr_top = 0; +} + +static u32 empty_windows[WALT_NR_CPUS]; + +static void rollover_task_window(struct task_struct *p, bool full_window) +{ + u32 *curr_cpu_windows = empty_windows; + u32 curr_window; + int i; + struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + /* Rollover the sum */ + curr_window = 0; + + if (!full_window) { + curr_window = wts->curr_window; + curr_cpu_windows = wts->curr_window_cpu; + } + + wts->prev_window = curr_window; + wts->curr_window = 0; + + /* Roll over individual CPU contributions */ + for (i = 0; i < nr_cpu_ids; i++) { + wts->prev_window_cpu[i] = curr_cpu_windows[i]; + wts->curr_window_cpu[i] = 0; + } + + if (is_new_task(p)) + wts->active_time += wrq->prev_window_size; +} + +static inline int cpu_is_waiting_on_io(struct rq *rq) +{ + if (!sched_io_is_busy) + return 0; + + return atomic_read(&rq->nr_iowait); +} + +static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, + u64 irqtime, int event) +{ + if (is_idle_task(p)) { + /* TASK_WAKE && TASK_MIGRATE is not possible on idle task! */ + if (event == PICK_NEXT_TASK) + return 0; + + /* PUT_PREV_TASK, TASK_UPDATE && IRQ_UPDATE are left */ + return irqtime || cpu_is_waiting_on_io(rq); + } + + if (event == TASK_WAKE) + return 0; + + if (event == PUT_PREV_TASK || event == IRQ_UPDATE) + return 1; + + /* + * TASK_UPDATE can be called on sleeping task, when its moved between + * related groups + */ + if (event == TASK_UPDATE) { + if (rq->curr == p) + return 1; + + return p->on_rq ? SCHED_FREQ_ACCOUNT_WAIT_TIME : 0; + } + + /* TASK_MIGRATE, PICK_NEXT_TASK left */ + return SCHED_FREQ_ACCOUNT_WAIT_TIME; +} + +#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) + +static inline u64 scale_exec_time(u64 delta, struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + return (delta * wrq->task_exec_scale) >> 10; +} + +/* Convert busy time to frequency equivalent + * Assumes load is scaled to 1024 + */ +static inline unsigned int load_to_freq(struct rq *rq, unsigned int load) +{ + return mult_frac(cpu_max_possible_freq(cpu_of(rq)), load, + (unsigned int)arch_scale_cpu_capacity(cpu_of(rq))); +} + +static bool do_pl_notif(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 prev = wrq->old_busy_time; + u64 pl = wrq->walt_stats.pred_demands_sum_scaled; + int cpu = cpu_of(rq); + + /* If already at max freq, bail out */ + if (capacity_orig_of(cpu) == capacity_curr_of(cpu)) + return false; + + prev = max(prev, wrq->old_estimated_time); + + /* 400 MHz filter. */ + return (pl > prev) && (load_to_freq(rq, pl - prev) > 400000); +} + +static void rollover_cpu_window(struct rq *rq, bool full_window) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 curr_sum = wrq->curr_runnable_sum; + u64 nt_curr_sum = wrq->nt_curr_runnable_sum; + u64 grp_curr_sum = wrq->grp_time.curr_runnable_sum; + u64 grp_nt_curr_sum = wrq->grp_time.nt_curr_runnable_sum; + + if (unlikely(full_window)) { + curr_sum = 0; + nt_curr_sum = 0; + grp_curr_sum = 0; + grp_nt_curr_sum = 0; + } + + wrq->prev_runnable_sum = curr_sum; + wrq->nt_prev_runnable_sum = nt_curr_sum; + wrq->grp_time.prev_runnable_sum = grp_curr_sum; + wrq->grp_time.nt_prev_runnable_sum = grp_nt_curr_sum; + + wrq->curr_runnable_sum = 0; + wrq->nt_curr_runnable_sum = 0; + wrq->grp_time.curr_runnable_sum = 0; + wrq->grp_time.nt_curr_runnable_sum = 0; +} + +/* + * Account cpu activity in its + * busy time counters(wrq->curr/prev_runnable_sum) + */ +static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) +{ + int new_window, full_window = 0; + int p_is_curr_task = (p == rq->curr); + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u64 mark_start = wts->mark_start; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 window_start = wrq->window_start; + u32 window_size = wrq->prev_window_size; + u64 delta; + u64 *curr_runnable_sum = &wrq->curr_runnable_sum; + u64 *prev_runnable_sum = &wrq->prev_runnable_sum; + u64 *nt_curr_runnable_sum = &wrq->nt_curr_runnable_sum; + u64 *nt_prev_runnable_sum = &wrq->nt_prev_runnable_sum; + bool new_task; + struct walt_related_thread_group *grp; + int cpu = rq->cpu; + u32 old_curr_window = wts->curr_window; + + new_window = mark_start < window_start; + if (new_window) + full_window = (window_start - mark_start) >= window_size; + + /* + * Handle per-task window rollover. We don't care about the + * idle task. + */ + if (!is_idle_task(p)) { + if (new_window) + rollover_task_window(p, full_window); + } + + new_task = is_new_task(p); + + if (p_is_curr_task && new_window) { + rollover_cpu_window(rq, full_window); + rollover_top_tasks(rq, full_window); + } + + if (!account_busy_for_cpu_time(rq, p, irqtime, event)) + goto done; + + grp = wts->grp; + if (grp) { + struct group_cpu_time *cpu_time = &wrq->grp_time; + + curr_runnable_sum = &cpu_time->curr_runnable_sum; + prev_runnable_sum = &cpu_time->prev_runnable_sum; + + nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + } + + if (!new_window) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. No rollover + * since we didn't start a new window. An example of this is + * when a task starts execution and then sleeps within the + * same window. + */ + + if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) + delta = wallclock - mark_start; + else + delta = irqtime; + delta = scale_exec_time(delta, rq); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + if (!is_idle_task(p)) { + wts->curr_window += delta; + wts->curr_window_cpu[cpu] += delta; + } + + goto done; + } + + if (!p_is_curr_task) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has also started, but p is not the current task, so the + * window is not rolled over - just split up and account + * as necessary into curr and prev. The window is only + * rolled over when a new window is processed for the current + * task. + * + * Irqtime can't be accounted by a task that isn't the + * currently running task. + */ + + if (!full_window) { + /* + * A full window hasn't elapsed, account partial + * contribution to previous completed window. + */ + delta = scale_exec_time(window_start - mark_start, rq); + wts->prev_window += delta; + wts->prev_window_cpu[cpu] += delta; + } else { + /* + * Since at least one full window has elapsed, + * the contribution to the previous window is the + * full window (window_size). + */ + delta = scale_exec_time(window_size, rq); + wts->prev_window = delta; + wts->prev_window_cpu[cpu] = delta; + } + + *prev_runnable_sum += delta; + if (new_task) + *nt_prev_runnable_sum += delta; + + /* Account piece of busy time in the current window. */ + delta = scale_exec_time(wallclock - window_start, rq); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + wts->curr_window = delta; + wts->curr_window_cpu[cpu] = delta; + + goto done; + } + + if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has started and p is the current task so rollover is + * needed. If any of these three above conditions are true + * then this busy time can't be accounted as irqtime. + * + * Busy time for the idle task need not be accounted. + * + * An example of this would be a task that starts execution + * and then sleeps once a new window has begun. + */ + + if (!full_window) { + /* + * A full window hasn't elapsed, account partial + * contribution to previous completed window. + */ + delta = scale_exec_time(window_start - mark_start, rq); + if (!is_idle_task(p)) { + wts->prev_window += delta; + wts->prev_window_cpu[cpu] += delta; + } + } else { + /* + * Since at least one full window has elapsed, + * the contribution to the previous window is the + * full window (window_size). + */ + delta = scale_exec_time(window_size, rq); + if (!is_idle_task(p)) { + wts->prev_window = delta; + wts->prev_window_cpu[cpu] = delta; + } + } + + /* + * Rollover is done here by overwriting the values in + * prev_runnable_sum and curr_runnable_sum. + */ + *prev_runnable_sum += delta; + if (new_task) + *nt_prev_runnable_sum += delta; + + /* Account piece of busy time in the current window. */ + delta = scale_exec_time(wallclock - window_start, rq); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + if (!is_idle_task(p)) { + wts->curr_window = delta; + wts->curr_window_cpu[cpu] = delta; + } + + goto done; + } + + if (irqtime) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has started and p is the current task so rollover is + * needed. The current task must be the idle task because + * irqtime is not accounted for any other task. + * + * Irqtime will be accounted each time we process IRQ activity + * after a period of idleness, so we know the IRQ busy time + * started at wallclock - irqtime. + */ + + SCHED_BUG_ON(!is_idle_task(p)); + mark_start = wallclock - irqtime; + + /* + * Roll window over. If IRQ busy time was just in the current + * window then that is all that need be accounted. + */ + if (mark_start > window_start) { + *curr_runnable_sum = scale_exec_time(irqtime, rq); + return; + } + + /* + * The IRQ busy time spanned multiple windows. Process the + * busy time preceding the current window start first. + */ + delta = window_start - mark_start; + if (delta > window_size) + delta = window_size; + delta = scale_exec_time(delta, rq); + *prev_runnable_sum += delta; + + /* Process the remaining IRQ busy time in the current window. */ + delta = wallclock - window_start; + wrq->curr_runnable_sum = scale_exec_time(delta, rq); + + return; + } + +done: + if (!is_idle_task(p)) + update_top_tasks(p, rq, old_curr_window, + new_window, full_window); +} + +static inline u32 predict_and_update_buckets( + struct task_struct *p, u32 runtime) { + int bidx; + u32 pred_demand; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!sched_predl) + return 0; + + bidx = busy_to_bucket(runtime); + pred_demand = get_pred_busy(p, bidx, runtime); + bucket_increase(wts->busy_buckets, bidx); + + return pred_demand; +} + +static int +account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) +{ + /* + * No need to bother updating task demand for the idle task. + */ + if (is_idle_task(p)) + return 0; + + /* + * When a task is waking up it is completing a segment of non-busy + * time. Likewise, if wait time is not treated as busy time, then + * when a task begins to run or is migrated, it is not running and + * is completing a segment of non-busy time. + */ + if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && + (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) + return 0; + + /* + * The idle exit time is not accounted for the first task _picked_ up to + * run on the idle CPU. + */ + if (event == PICK_NEXT_TASK && rq->curr == rq->idle) + return 0; + + /* + * TASK_UPDATE can be called on sleeping task, when its moved between + * related groups + */ + if (event == TASK_UPDATE) { + if (rq->curr == p) + return 1; + + return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; + } + + return 1; +} + +/* + * Called when new window is starting for a task, to record cpu usage over + * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 + * when, say, a real-time task runs without preemption for several windows at a + * stretch. + */ +static void update_history(struct rq *rq, struct task_struct *p, + u32 runtime, int samples, int event) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u32 *hist = &wts->sum_history[0]; + int ridx, widx; + u32 max = 0, avg, demand, pred_demand; + u64 sum = 0; + u16 demand_scaled, pred_demand_scaled; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + /* Ignore windows where task had no activity */ + if (!runtime || is_idle_task(p) || !samples) + goto done; + + /* Push new 'runtime' value onto stack */ + widx = sched_ravg_hist_size - 1; + ridx = widx - samples; + for (; ridx >= 0; --widx, --ridx) { + hist[widx] = hist[ridx]; + sum += hist[widx]; + if (hist[widx] > max) + max = hist[widx]; + } + + for (widx = 0; widx < samples && widx < sched_ravg_hist_size; widx++) { + hist[widx] = runtime; + sum += hist[widx]; + if (hist[widx] > max) + max = hist[widx]; + } + + wts->sum = 0; + + if (sysctl_sched_window_stats_policy == WINDOW_STATS_RECENT) { + demand = runtime; + } else if (sysctl_sched_window_stats_policy == WINDOW_STATS_MAX) { + demand = max; + } else { + avg = div64_u64(sum, sched_ravg_hist_size); + if (sysctl_sched_window_stats_policy == WINDOW_STATS_AVG) + demand = avg; + else + demand = max(avg, runtime); + } + pred_demand = predict_and_update_buckets(p, runtime); + demand_scaled = scale_demand(demand); + pred_demand_scaled = scale_demand(pred_demand); + + /* + * A throttled deadline sched class task gets dequeued without + * changing p->on_rq. Since the dequeue decrements walt stats + * avoid decrementing it here again. + * + * When window is rolled over, the cumulative window demand + * is reset to the cumulative runnable average (contribution from + * the tasks on the runqueue). If the current task is dequeued + * already, it's demand is not included in the cumulative runnable + * average. So add the task demand separately to cumulative window + * demand. + */ + if (!task_has_dl_policy(p) || !p->dl.dl_throttled) { + if (task_on_rq_queued(p)) + fixup_walt_sched_stats_common(rq, p, + demand_scaled, pred_demand_scaled); + } + + wts->demand = demand; + wts->demand_scaled = demand_scaled; + wts->coloc_demand = div64_u64(sum, sched_ravg_hist_size); + wts->pred_demand = pred_demand; + wts->pred_demand_scaled = pred_demand_scaled; + + if (demand_scaled > sysctl_sched_min_task_util_for_colocation) + wts->unfilter = sysctl_sched_task_unfilter_period; + else + if (wts->unfilter) + wts->unfilter = max_t(int, 0, + wts->unfilter - wrq->prev_window_size); + +done: + trace_sched_update_history(rq, p, runtime, samples, event, wrq, wts); +} + +static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + delta = scale_exec_time(delta, rq); + wts->sum += delta; + if (unlikely(wts->sum > sched_ravg_window)) + wts->sum = sched_ravg_window; + + return delta; +} + +/* + * Account cpu demand of task and/or update task's cpu demand history + * + * ms = wts->mark_start; + * wc = wallclock + * ws = wrq->window_start + * + * Three possibilities: + * + * a) Task event is contained within one window. + * window_start < mark_start < wallclock + * + * ws ms wc + * | | | + * V V V + * |---------------| + * + * In this case, wts->sum is updated *iff* event is appropriate + * (ex: event == PUT_PREV_TASK) + * + * b) Task event spans two windows. + * mark_start < window_start < wallclock + * + * ms ws wc + * | | | + * V V V + * -----|------------------- + * + * In this case, wts->sum is updated with (ws - ms) *iff* event + * is appropriate, then a new window sample is recorded followed + * by wts->sum being set to (wc - ws) *iff* event is appropriate. + * + * c) Task event spans more than two windows. + * + * ms ws_tmp ws wc + * | | | | + * V V V V + * ---|-------|-------|-------|-------|------ + * | | + * |<------ nr_full_windows ------>| + * + * In this case, wts->sum is updated with (ws_tmp - ms) first *iff* + * event is appropriate, window sample of wts->sum is recorded, + * 'nr_full_window' samples of window_size is also recorded *iff* + * event is appropriate and finally wts->sum is set to (wc - ws) + * *iff* event is appropriate. + * + * IMPORTANT : Leave wts->mark_start unchanged, as update_cpu_busy_time() + * depends on it! + */ +static u64 update_task_demand(struct task_struct *p, struct rq *rq, + int event, u64 wallclock) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u64 mark_start = wts->mark_start; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 delta, window_start = wrq->window_start; + int new_window, nr_full_windows; + u32 window_size = sched_ravg_window; + u64 runtime; + + new_window = mark_start < window_start; + if (!account_busy_for_task_demand(rq, p, event)) { + if (new_window) + /* + * If the time accounted isn't being accounted as + * busy time, and a new window started, only the + * previous window need be closed out with the + * pre-existing demand. Multiple windows may have + * elapsed, but since empty windows are dropped, + * it is not necessary to account those. + */ + update_history(rq, p, wts->sum, 1, event); + return 0; + } + + if (!new_window) { + /* + * The simple case - busy time contained within the existing + * window. + */ + return add_to_task_demand(rq, p, wallclock - mark_start); + } + + /* + * Busy time spans at least two windows. Temporarily rewind + * window_start to first window boundary after mark_start. + */ + delta = window_start - mark_start; + nr_full_windows = div64_u64(delta, window_size); + window_start -= (u64)nr_full_windows * (u64)window_size; + + /* Process (window_start - mark_start) first */ + runtime = add_to_task_demand(rq, p, window_start - mark_start); + + /* Push new sample(s) into task's demand history */ + update_history(rq, p, wts->sum, 1, event); + if (nr_full_windows) { + u64 scaled_window = scale_exec_time(window_size, rq); + + update_history(rq, p, scaled_window, nr_full_windows, event); + runtime += nr_full_windows * scaled_window; + } + + /* + * Roll window_start back to current to process any remainder + * in current window. + */ + window_start += (u64)nr_full_windows * (u64)window_size; + + /* Process (wallclock - window_start) next */ + mark_start = window_start; + runtime += add_to_task_demand(rq, p, wallclock - mark_start); + + return runtime; +} + +static inline unsigned int cpu_cur_freq(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster->cur_freq; +} + +static void +update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime) +{ + u64 cur_cycles; + u64 cycles_delta; + u64 time_delta; + int cpu = cpu_of(rq); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + lockdep_assert_held(&rq->lock); + + if (!use_cycle_counter) { + wrq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * + arch_scale_cpu_capacity(cpu), + wrq->cluster->max_possible_freq); + return; + } + + cur_cycles = read_cycle_counter(cpu, wallclock); + + /* + * If current task is idle task and irqtime == 0 CPU was + * indeed idle and probably its cycle counter was not + * increasing. We still need estimatied CPU frequency + * for IO wait time accounting. Use the previously + * calculated frequency in such a case. + */ + if (!is_idle_task(rq->curr) || irqtime) { + if (unlikely(cur_cycles < wts->cpu_cycles)) + cycles_delta = cur_cycles + (U64_MAX - + wts->cpu_cycles); + else + cycles_delta = cur_cycles - wts->cpu_cycles; + cycles_delta = cycles_delta * NSEC_PER_MSEC; + + if (event == IRQ_UPDATE && is_idle_task(p)) + /* + * Time between mark_start of idle task and IRQ handler + * entry time is CPU cycle counter stall period. + * Upon IRQ handler entry walt_sched_account_irqstart() + * replenishes idle task's cpu cycle counter so + * cycles_delta now represents increased cycles during + * IRQ handler rather than time between idle entry and + * IRQ exit. Thus use irqtime as time delta. + */ + time_delta = irqtime; + else + time_delta = wallclock - wts->mark_start; + SCHED_BUG_ON((s64)time_delta < 0); + + wrq->task_exec_scale = DIV64_U64_ROUNDUP(cycles_delta * + arch_scale_cpu_capacity(cpu), + time_delta * + wrq->cluster->max_possible_freq); + + trace_sched_get_task_cpu_cycles(cpu, event, + cycles_delta, time_delta, p); + } + + wts->cpu_cycles = cur_cycles; +} + +static inline void run_walt_irq_work(u64 old_window_start, struct rq *rq) +{ + u64 result; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (old_window_start == wrq->window_start) + return; + + result = atomic64_cmpxchg(&walt_irq_work_lastq_ws, old_window_start, + wrq->window_start); + if (result == old_window_start) + walt_irq_work_queue(&walt_cpufreq_irq_work); +} + +/* Reflect task activity on its demand and cpu's busy time statistics */ +static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime) +{ + u64 old_window_start; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!wrq->window_start || wts->mark_start == wallclock) + return; + + lockdep_assert_held(&rq->lock); + + old_window_start = update_window_start(rq, wallclock, event); + + if (!wts->mark_start) { + update_task_cpu_cycles(p, cpu_of(rq), wallclock); + goto done; + } + + update_task_rq_cpu_cycles(p, rq, event, wallclock, irqtime); + update_task_demand(p, rq, event, wallclock); + update_cpu_busy_time(p, rq, event, wallclock, irqtime); + update_task_pred_demand(rq, p, event); + + trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime, + &wrq->grp_time, wrq, wts); + trace_sched_update_task_ravg_mini(p, rq, event, wallclock, irqtime, + &wrq->grp_time, wrq, wts); + +done: + wts->mark_start = wallclock; + + run_walt_irq_work(old_window_start, rq); +} + +u32 sched_get_init_task_load(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->init_load_pct; +} + +int sched_set_init_task_load(struct task_struct *p, int init_load_pct) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (init_load_pct < 0 || init_load_pct > 100) + return -EINVAL; + + wts->init_load_pct = init_load_pct; + + return 0; +} + +static void init_new_task_load(struct task_struct *p) +{ + int i; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_task_struct *cur_wts = + (struct walt_task_struct *) current->android_vendor_data1; + u32 init_load_windows = sched_init_task_load_windows; + u32 init_load_windows_scaled = sched_init_task_load_windows_scaled; + u32 init_load_pct = cur_wts->init_load_pct; + + wts->init_load_pct = 0; + rcu_assign_pointer(wts->grp, NULL); + INIT_LIST_HEAD(&wts->grp_list); + + wts->mark_start = 0; + wts->sum = 0; + wts->curr_window = 0; + wts->prev_window = 0; + wts->active_time = 0; + for (i = 0; i < NUM_BUSY_BUCKETS; ++i) + wts->busy_buckets[i] = 0; + + wts->cpu_cycles = 0; + + memset(wts->curr_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); + memset(wts->prev_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); + + if (init_load_pct) { + init_load_windows = div64_u64((u64)init_load_pct * + (u64)sched_ravg_window, 100); + init_load_windows_scaled = scale_demand(init_load_windows); + } + + wts->demand = init_load_windows; + wts->demand_scaled = init_load_windows_scaled; + wts->coloc_demand = init_load_windows; + wts->pred_demand = 0; + wts->pred_demand_scaled = 0; + for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) + wts->sum_history[i] = init_load_windows; + wts->misfit = false; + wts->rtg_high_prio = false; + wts->unfilter = sysctl_sched_task_unfilter_period; +} + +static void init_existing_task_load(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + init_new_task_load(p); + cpumask_copy(&wts->cpus_requested, &p->cpus_mask); +} + +static void walt_task_dead(struct task_struct *p) +{ + sched_set_group_id(p, 0); +} + +static void reset_task_stats(struct task_struct *p) +{ + int i = 0; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + memset(wts->curr_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); + memset(wts->prev_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); + + wts->mark_start = 0; + wts->sum = 0; + wts->demand = 0; + wts->coloc_demand = 0; + for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) + wts->sum_history[i] = 0; + wts->curr_window = 0; + wts->prev_window = 0; + wts->pred_demand = 0; + for (i = 0; i < NUM_BUSY_BUCKETS; ++i) + wts->busy_buckets[i] = 0; + wts->demand_scaled = 0; + wts->pred_demand_scaled = 0; + wts->active_time = 0; +} + +static void mark_task_starting(struct task_struct *p) +{ + u64 wallclock; + struct rq *rq = task_rq(p); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!wrq->window_start) { + reset_task_stats(p); + return; + } + + wallclock = sched_ktime_clock(); + wts->mark_start = wts->last_wake_ts = wallclock; + wts->last_enqueued_ts = wallclock; + update_task_cpu_cycles(p, cpu_of(rq), wallclock); +} + +/* + * Task groups whose aggregate demand on a cpu is more than + * sched_group_upmigrate need to be up-migrated if possible. + */ +unsigned int __read_mostly sched_group_upmigrate = 20000000; + +/* + * Task groups, once up-migrated, will need to drop their aggregate + * demand to less than sched_group_downmigrate before they are "down" + * migrated. + */ +unsigned int __read_mostly sched_group_downmigrate = 19000000; + +void walt_update_group_thresholds(void) +{ + unsigned int min_scale = arch_scale_cpu_capacity( + cluster_first_cpu(sched_cluster[0])); + u64 min_ms = min_scale * (sched_ravg_window >> SCHED_CAPACITY_SHIFT); + + sched_group_upmigrate = div64_ul(min_ms * + sysctl_sched_group_upmigrate_pct, 100); + sched_group_downmigrate = div64_ul(min_ms * + sysctl_sched_group_downmigrate_pct, 100); +} + +struct walt_sched_cluster *sched_cluster[WALT_NR_CPUS]; +__read_mostly int num_sched_clusters; + +struct list_head cluster_head; + +static struct walt_sched_cluster init_cluster = { + .list = LIST_HEAD_INIT(init_cluster.list), + .id = 0, + .cur_freq = 1, + .max_possible_freq = 1, + .aggr_grp_load = 0, +}; + +static void init_clusters(void) +{ + init_cluster.cpus = *cpu_possible_mask; + raw_spin_lock_init(&init_cluster.load_lock); + INIT_LIST_HEAD(&cluster_head); + list_add(&init_cluster.list, &cluster_head); +} + +static void +insert_cluster(struct walt_sched_cluster *cluster, struct list_head *head) +{ + struct walt_sched_cluster *tmp; + struct list_head *iter = head; + + list_for_each_entry(tmp, head, list) { + if (arch_scale_cpu_capacity(cluster_first_cpu(cluster)) + < arch_scale_cpu_capacity(cluster_first_cpu(tmp))) + break; + iter = &tmp->list; + } + + list_add(&cluster->list, iter); +} + +static struct walt_sched_cluster *alloc_new_cluster(const struct cpumask *cpus) +{ + struct walt_sched_cluster *cluster = NULL; + + cluster = kzalloc(sizeof(struct walt_sched_cluster), GFP_ATOMIC); + BUG_ON(!cluster); + + INIT_LIST_HEAD(&cluster->list); + cluster->cur_freq = 1; + cluster->max_possible_freq = 1; + + raw_spin_lock_init(&cluster->load_lock); + cluster->cpus = *cpus; + + return cluster; +} + +static void add_cluster(const struct cpumask *cpus, struct list_head *head) +{ + struct walt_sched_cluster *cluster = alloc_new_cluster(cpus); + int i; + struct walt_rq *wrq; + + for_each_cpu(i, cpus) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq->cluster = cluster; + } + + insert_cluster(cluster, head); + num_sched_clusters++; +} + +static void cleanup_clusters(struct list_head *head) +{ + struct walt_sched_cluster *cluster, *tmp; + int i; + struct walt_rq *wrq; + + list_for_each_entry_safe(cluster, tmp, head, list) { + for_each_cpu(i, &cluster->cpus) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq->cluster = &init_cluster; + } + list_del(&cluster->list); + num_sched_clusters--; + kfree(cluster); + } +} + +static inline void assign_cluster_ids(struct list_head *head) +{ + struct walt_sched_cluster *cluster; + int pos = 0; + + list_for_each_entry(cluster, head, list) { + cluster->id = pos; + sched_cluster[pos++] = cluster; + } + + WARN_ON(pos > MAX_NR_CLUSTERS); +} + +static inline void +move_list(struct list_head *dst, struct list_head *src, bool sync_rcu) +{ + struct list_head *first, *last; + + first = src->next; + last = src->prev; + + if (sync_rcu) { + INIT_LIST_HEAD_RCU(src); + synchronize_rcu(); + } + + first->prev = dst; + dst->prev = last; + last->next = dst; + + /* Ensure list sanity before making the head visible to all CPUs. */ + smp_mb(); + dst->next = first; +} + +static void update_all_clusters_stats(void) +{ + struct walt_sched_cluster *cluster; + u64 highest_mpc = 0, lowest_mpc = U64_MAX; + + for_each_sched_cluster(cluster) { + u64 mpc = arch_scale_cpu_capacity( + cluster_first_cpu(cluster)); + + if (mpc > highest_mpc) + highest_mpc = mpc; + + if (mpc < lowest_mpc) + lowest_mpc = mpc; + } + + max_possible_capacity = highest_mpc; + min_max_possible_capacity = lowest_mpc; + walt_update_group_thresholds(); +} + +static bool walt_clusters_parsed; +cpumask_t __read_mostly **cpu_array; + +static void init_cpu_array(void) +{ + int i; + + cpu_array = kcalloc(num_sched_clusters, sizeof(cpumask_t *), + GFP_ATOMIC | __GFP_NOFAIL); + if (!cpu_array) + SCHED_BUG_ON(1); + + for (i = 0; i < num_sched_clusters; i++) { + cpu_array[i] = kcalloc(num_sched_clusters, sizeof(cpumask_t), + GFP_ATOMIC | __GFP_NOFAIL); + if (!cpu_array[i]) + SCHED_BUG_ON(1); + } +} + +static void build_cpu_array(void) +{ + int i; + + if (!cpu_array) + SCHED_BUG_ON(1); + /* Construct cpu_array row by row */ + for (i = 0; i < num_sched_clusters; i++) { + int j, k = 1; + + /* Fill out first column with appropriate cpu arrays */ + cpumask_copy(&cpu_array[i][0], &sched_cluster[i]->cpus); + /* + * k starts from column 1 because 0 is filled + * Fill clusters for the rest of the row, + * above i in ascending order + */ + for (j = i + 1; j < num_sched_clusters; j++) { + cpumask_copy(&cpu_array[i][k], + &sched_cluster[j]->cpus); + k++; + } + + /* + * k starts from where we left off above. + * Fill clusters below i in descending order. + */ + for (j = i - 1; j >= 0; j--) { + cpumask_copy(&cpu_array[i][k], + &sched_cluster[j]->cpus); + k++; + } + } +} + +static void walt_get_possible_siblings(int cpuid, struct cpumask *cluster_cpus) +{ + int cpu; + struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; + + if (cpuid_topo->package_id == -1) + return; + + for_each_possible_cpu(cpu) { + cpu_topo = &cpu_topology[cpu]; + + if (cpuid_topo->package_id != cpu_topo->package_id) + continue; + cpumask_set_cpu(cpu, cluster_cpus); + } +} + +static void walt_update_cluster_topology(void) +{ + struct cpumask cpus = *cpu_possible_mask; + struct cpumask cluster_cpus; + struct walt_sched_cluster *cluster; + struct list_head new_head; + int i; + struct walt_rq *wrq; + + INIT_LIST_HEAD(&new_head); + + for_each_cpu(i, &cpus) { + cpumask_clear(&cluster_cpus); + walt_get_possible_siblings(i, &cluster_cpus); + if (cpumask_empty(&cluster_cpus)) { + WARN(1, "WALT: Invalid cpu topology!!"); + cleanup_clusters(&new_head); + return; + } + cpumask_andnot(&cpus, &cpus, &cluster_cpus); + add_cluster(&cluster_cpus, &new_head); + } + + assign_cluster_ids(&new_head); + + list_for_each_entry(cluster, &new_head, list) { + struct cpufreq_policy *policy; + + policy = cpufreq_cpu_get_raw(cluster_first_cpu(cluster)); + /* + * walt_update_cluster_topology() must be called AFTER policies + * for all cpus are initialized. If not, simply BUG(). + */ + SCHED_BUG_ON(!policy); + + if (policy) { + cluster->max_possible_freq = policy->cpuinfo.max_freq; + for_each_cpu(i, &cluster->cpus) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + cpumask_copy(&wrq->freq_domain_cpumask, + policy->related_cpus); + } + cpuinfo_max_freq_cached = (cpuinfo_max_freq_cached > + policy->cpuinfo.max_freq) ?: policy->cpuinfo.max_freq; + } + } + + /* + * Ensure cluster ids are visible to all CPUs before making + * cluster_head visible. + */ + move_list(&cluster_head, &new_head, false); + update_all_clusters_stats(); + cluster = NULL; + + for_each_sched_cluster(cluster) { + if (cpumask_weight(&cluster->cpus) == 1) + cpumask_or(&asym_cap_sibling_cpus, + &asym_cap_sibling_cpus, &cluster->cpus); + } + + if (cpumask_weight(&asym_cap_sibling_cpus) == 1) + cpumask_clear(&asym_cap_sibling_cpus); + + init_cpu_array(); + build_cpu_array(); + + walt_clusters_parsed = true; +} + +static int cpufreq_notifier_trans(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data; + unsigned int cpu = freq->policy->cpu, new_freq = freq->new; + unsigned long flags; + struct walt_sched_cluster *cluster; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct cpumask policy_cpus = wrq->freq_domain_cpumask; + int i, j; + + if (use_cycle_counter) + return NOTIFY_DONE; + wrq = (struct walt_rq *) cpu_rq(cpumask_first(&policy_cpus))->android_vendor_data1; + if (wrq->cluster == &init_cluster) + return NOTIFY_DONE; + + if (val != CPUFREQ_POSTCHANGE) + return NOTIFY_DONE; + + if (cpu_cur_freq(cpu) == new_freq) + return NOTIFY_OK; + + for_each_cpu(i, &policy_cpus) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + cluster = wrq->cluster; + + for_each_cpu(j, &cluster->cpus) { + struct rq *rq = cpu_rq(j); + + raw_spin_lock_irqsave(&rq->lock, flags); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, + sched_ktime_clock(), 0); + raw_spin_unlock_irqrestore(&rq->lock, flags); + } + + cluster->cur_freq = new_freq; + cpumask_andnot(&policy_cpus, &policy_cpus, &cluster->cpus); + } + + return NOTIFY_OK; +} + +static struct notifier_block notifier_trans_block = { + .notifier_call = cpufreq_notifier_trans +}; + +static void walt_init_cycle_counter(void) +{ + if (qcom_cpufreq_get_cpu_cycle_counter(smp_processor_id()) != U64_MAX) { + use_cycle_counter = true; + return; + } + + cpufreq_register_notifier(¬ifier_trans_block, + CPUFREQ_TRANSITION_NOTIFIER); +} + +static void transfer_busy_time(struct rq *rq, + struct walt_related_thread_group *grp, + struct task_struct *p, int event); + +/* + * Enable colocation and frequency aggregation for all threads in a process. + * The children inherits the group id from the parent. + */ + +struct walt_related_thread_group + *related_thread_groups[MAX_NUM_CGROUP_COLOC_ID]; +static LIST_HEAD(active_related_thread_groups); +static DEFINE_RWLOCK(related_thread_group_lock); + +static inline +void update_best_cluster(struct walt_related_thread_group *grp, + u64 demand, bool boost) +{ + if (boost) { + /* + * since we are in boost, we can keep grp on min, the boosts + * will ensure tasks get to bigs + */ + grp->skip_min = false; + return; + } + + if (is_suh_max()) + demand = sched_group_upmigrate; + + if (!grp->skip_min) { + if (demand >= sched_group_upmigrate) + grp->skip_min = true; + return; + } + if (demand < sched_group_downmigrate) { + if (!sysctl_sched_coloc_downmigrate_ns) { + grp->skip_min = false; + return; + } + if (!grp->downmigrate_ts) { + grp->downmigrate_ts = grp->last_update; + return; + } + if (grp->last_update - grp->downmigrate_ts > + sysctl_sched_coloc_downmigrate_ns) { + grp->downmigrate_ts = 0; + grp->skip_min = false; + } + } else if (grp->downmigrate_ts) + grp->downmigrate_ts = 0; +} + +static void _set_preferred_cluster(struct walt_related_thread_group *grp) +{ + struct task_struct *p; + u64 combined_demand = 0; + bool group_boost = false; + u64 wallclock; + bool prev_skip_min = grp->skip_min; + struct walt_task_struct *wts; + struct list_head *task_list; + + if (list_empty(&grp->tasks)) { + grp->skip_min = false; + goto out; + } + + if (!hmp_capable()) { + grp->skip_min = false; + goto out; + } + + wallclock = sched_ktime_clock(); + + /* + * wakeup of two or more related tasks could race with each other and + * could result in multiple calls to _set_preferred_cluster being issued + * at same time. Avoid overhead in such cases of rechecking preferred + * cluster + */ + if (wallclock - grp->last_update < sched_ravg_window / 10) + return; + + list_for_each(task_list, &grp->tasks) { + p = (struct task_struct *) task_list; + wts = (struct walt_task_struct *) p->android_vendor_data1; + if (task_boost_policy(p) == SCHED_BOOST_ON_BIG) { + group_boost = true; + break; + } + + if (wts->mark_start < wallclock - + (sched_ravg_window * sched_ravg_hist_size)) + continue; + + combined_demand += wts->coloc_demand; + if (!trace_sched_set_preferred_cluster_enabled()) { + if (combined_demand > sched_group_upmigrate) + break; + } + } + + grp->last_update = wallclock; + update_best_cluster(grp, combined_demand, group_boost); + trace_sched_set_preferred_cluster(grp, combined_demand); + +out: + if (grp->id == DEFAULT_CGROUP_COLOC_ID + && grp->skip_min != prev_skip_min) { + if (grp->skip_min) + grp->start_ts = sched_clock(); + sched_update_hyst_times(); + } +} + +static void set_preferred_cluster(struct walt_related_thread_group *grp) +{ + raw_spin_lock(&grp->lock); + _set_preferred_cluster(grp); + raw_spin_unlock(&grp->lock); +} + +static int update_preferred_cluster(struct walt_related_thread_group *grp, + struct task_struct *p, u32 old_load, bool from_tick) +{ + u32 new_load = task_load(p); + + if (!grp) + return 0; + + if (unlikely(from_tick && is_suh_max())) + return 1; + + /* + * Update if task's load has changed significantly or a complete window + * has passed since we last updated preference + */ + if (abs(new_load - old_load) > sched_ravg_window / 4 || + sched_ktime_clock() - grp->last_update > sched_ravg_window) + return 1; + + return 0; +} + +#define ADD_TASK 0 +#define REM_TASK 1 + +static inline struct walt_related_thread_group* +lookup_related_thread_group(unsigned int group_id) +{ + return related_thread_groups[group_id]; +} + +static int alloc_related_thread_groups(void) +{ + int i; + struct walt_related_thread_group *grp; + + /* groupd_id = 0 is invalid as it's special id to remove group. */ + for (i = 1; i < MAX_NUM_CGROUP_COLOC_ID; i++) { + grp = kzalloc(sizeof(*grp), GFP_ATOMIC | GFP_NOWAIT); + BUG_ON(!grp); + + grp->id = i; + INIT_LIST_HEAD(&grp->tasks); + INIT_LIST_HEAD(&grp->list); + raw_spin_lock_init(&grp->lock); + + related_thread_groups[i] = grp; + } + + return 0; +} + +static void remove_task_from_group(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_related_thread_group *grp = wts->grp; + struct rq *rq; + int empty_group = 1; + struct rq_flags rf; + + raw_spin_lock(&grp->lock); + + rq = __task_rq_lock(p, &rf); + transfer_busy_time(rq, wts->grp, p, REM_TASK); + list_del_init(&wts->grp_list); + rcu_assign_pointer(wts->grp, NULL); + __task_rq_unlock(rq, &rf); + + if (!list_empty(&grp->tasks)) { + empty_group = 0; + _set_preferred_cluster(grp); + } + + raw_spin_unlock(&grp->lock); + + /* Reserved groups cannot be destroyed */ + if (empty_group && grp->id != DEFAULT_CGROUP_COLOC_ID) + /* + * We test whether grp->list is attached with list_empty() + * hence re-init the list after deletion. + */ + list_del_init(&grp->list); +} + +static int +add_task_to_group(struct task_struct *p, struct walt_related_thread_group *grp) +{ + struct rq *rq; + struct rq_flags rf; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + raw_spin_lock(&grp->lock); + + /* + * Change wts->grp under rq->lock. Will prevent races with read-side + * reference of wts->grp in various hot-paths + */ + rq = __task_rq_lock(p, &rf); + transfer_busy_time(rq, grp, p, ADD_TASK); + list_add(&wts->grp_list, &grp->tasks); + rcu_assign_pointer(wts->grp, grp); + __task_rq_unlock(rq, &rf); + + _set_preferred_cluster(grp); + + raw_spin_unlock(&grp->lock); + + return 0; +} + +#ifdef CONFIG_UCLAMP_TASK_GROUP +static inline bool uclamp_task_colocated(struct task_struct *p) +{ + struct cgroup_subsys_state *css; + struct task_group *tg; + bool colocate; + struct walt_task_group *wtg; + + rcu_read_lock(); + css = task_css(p, cpu_cgrp_id); + if (!css) { + rcu_read_unlock(); + return false; + } + tg = container_of(css, struct task_group, css); + wtg = (struct walt_task_group *) tg->android_vendor_data1; + colocate = wtg->colocate; + rcu_read_unlock(); + + return colocate; +} +#else +static inline bool uclamp_task_colocated(struct task_struct *p) +{ + return false; +} +#endif /* CONFIG_UCLAMP_TASK_GROUP */ + +static void add_new_task_to_grp(struct task_struct *new) +{ + unsigned long flags; + struct walt_related_thread_group *grp; + struct walt_task_struct *wts = (struct walt_task_struct *) new->android_vendor_data1; + + /* + * If the task does not belong to colocated schedtune + * cgroup, nothing to do. We are checking this without + * lock. Even if there is a race, it will be added + * to the co-located cgroup via cgroup attach. + */ + if (!uclamp_task_colocated(new)) + return; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + write_lock_irqsave(&related_thread_group_lock, flags); + + /* + * It's possible that someone already added the new task to the + * group. or it might have taken out from the colocated schedtune + * cgroup. check these conditions under lock. + */ + if (!uclamp_task_colocated(new) || wts->grp) { + write_unlock_irqrestore(&related_thread_group_lock, flags); + return; + } + + raw_spin_lock(&grp->lock); + + rcu_assign_pointer(wts->grp, grp); + list_add(&wts->grp_list, &grp->tasks); + + raw_spin_unlock(&grp->lock); + write_unlock_irqrestore(&related_thread_group_lock, flags); +} + +static int __sched_set_group_id(struct task_struct *p, unsigned int group_id) +{ + int rc = 0; + unsigned long flags; + struct walt_related_thread_group *grp = NULL; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (group_id >= MAX_NUM_CGROUP_COLOC_ID) + return -EINVAL; + + raw_spin_lock_irqsave(&p->pi_lock, flags); + write_lock(&related_thread_group_lock); + + /* Switching from one group to another directly is not permitted */ + if ((!wts->grp && !group_id) || (wts->grp && group_id)) + goto done; + + if (!group_id) { + remove_task_from_group(p); + goto done; + } + + grp = lookup_related_thread_group(group_id); + if (list_empty(&grp->list)) + list_add(&grp->list, &active_related_thread_groups); + + rc = add_task_to_group(p, grp); +done: + write_unlock(&related_thread_group_lock); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + + return rc; +} + +int sched_set_group_id(struct task_struct *p, unsigned int group_id) +{ + /* DEFAULT_CGROUP_COLOC_ID is a reserved id */ + if (group_id == DEFAULT_CGROUP_COLOC_ID) + return -EINVAL; + + return __sched_set_group_id(p, group_id); +} + +unsigned int sched_get_group_id(struct task_struct *p) +{ + unsigned int group_id; + struct walt_related_thread_group *grp; + + rcu_read_lock(); + grp = task_related_thread_group(p); + group_id = grp ? grp->id : 0; + rcu_read_unlock(); + + return group_id; +} + +/* + * We create a default colocation group at boot. There is no need to + * synchronize tasks between cgroups at creation time because the + * correct cgroup hierarchy is not available at boot. Therefore cgroup + * colocation is turned off by default even though the colocation group + * itself has been allocated. Furthermore this colocation group cannot + * be destroyted once it has been created. All of this has been as part + * of runtime optimizations. + * + * The job of synchronizing tasks to the colocation group is done when + * the colocation flag in the cgroup is turned on. + */ +static int create_default_coloc_group(void) +{ + struct walt_related_thread_group *grp = NULL; + unsigned long flags; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + write_lock_irqsave(&related_thread_group_lock, flags); + list_add(&grp->list, &active_related_thread_groups); + write_unlock_irqrestore(&related_thread_group_lock, flags); + return 0; +} + +static int sync_cgroup_colocation(struct task_struct *p, bool insert) +{ + unsigned int grp_id = insert ? DEFAULT_CGROUP_COLOC_ID : 0; + + return __sched_set_group_id(p, grp_id); +} + +static void android_rvh_cpu_cgroup_attach(void *unused, + struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + bool colocate; + struct task_group *tg; + struct walt_task_group *wtg; + + cgroup_taskset_first(tset, &css); + if (!css) + return; + + tg = container_of(css, struct task_group, css); + wtg = (struct walt_task_group *) tg->android_vendor_data1; + colocate = wtg->colocate; + + cgroup_taskset_for_each(task, css, tset) + sync_cgroup_colocation(task, colocate); +} + +static bool is_cluster_hosting_top_app(struct walt_sched_cluster *cluster) +{ + struct walt_related_thread_group *grp; + bool grp_on_min; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + + if (!grp) + return false; + + grp_on_min = !grp->skip_min && + (sched_boost_policy() != SCHED_BOOST_ON_BIG); + + return (is_min_capacity_cluster(cluster) == grp_on_min); +} + +static void note_task_waking(struct task_struct *p, u64 wallclock) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->last_wake_ts = wallclock; +} + +/* + * Task's cpu usage is accounted in: + * wrq->curr/prev_runnable_sum, when its ->grp is NULL + * grp->cpu_time[cpu]->curr/prev_runnable_sum, when its ->grp is !NULL + * + * Transfer task's cpu usage between those counters when transitioning between + * groups + */ +static void transfer_busy_time(struct rq *rq, + struct walt_related_thread_group *grp, + struct task_struct *p, int event) +{ + u64 wallclock; + struct group_cpu_time *cpu_time; + u64 *src_curr_runnable_sum, *dst_curr_runnable_sum; + u64 *src_prev_runnable_sum, *dst_prev_runnable_sum; + u64 *src_nt_curr_runnable_sum, *dst_nt_curr_runnable_sum; + u64 *src_nt_prev_runnable_sum, *dst_nt_prev_runnable_sum; + int migrate_type; + int cpu = cpu_of(rq); + bool new_task; + int i; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wallclock = sched_ktime_clock(); + + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + walt_update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); + new_task = is_new_task(p); + + cpu_time = &wrq->grp_time; + if (event == ADD_TASK) { + migrate_type = RQ_TO_GROUP; + + src_curr_runnable_sum = &wrq->curr_runnable_sum; + dst_curr_runnable_sum = &cpu_time->curr_runnable_sum; + src_prev_runnable_sum = &wrq->prev_runnable_sum; + dst_prev_runnable_sum = &cpu_time->prev_runnable_sum; + + src_nt_curr_runnable_sum = &wrq->nt_curr_runnable_sum; + dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &wrq->nt_prev_runnable_sum; + dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + if (*src_curr_runnable_sum < wts->curr_window_cpu[cpu]) { + printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_curr_runnable_sum, + wts->curr_window_cpu[cpu]); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + *src_curr_runnable_sum -= wts->curr_window_cpu[cpu]; + + if (*src_prev_runnable_sum < wts->prev_window_cpu[cpu]) { + printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_prev_runnable_sum, + wts->prev_window_cpu[cpu]); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + *src_prev_runnable_sum -= wts->prev_window_cpu[cpu]; + + if (new_task) { + if (*src_nt_curr_runnable_sum < + wts->curr_window_cpu[cpu]) { + printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_curr_runnable_sum, + wts->curr_window_cpu[cpu]); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + *src_nt_curr_runnable_sum -= + wts->curr_window_cpu[cpu]; + + if (*src_nt_prev_runnable_sum < + wts->prev_window_cpu[cpu]) { + printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_prev_runnable_sum, + wts->prev_window_cpu[cpu]); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + *src_nt_prev_runnable_sum -= + wts->prev_window_cpu[cpu]; + } + + update_cluster_load_subtractions(p, cpu, + wrq->window_start, new_task); + + } else { + migrate_type = GROUP_TO_RQ; + + src_curr_runnable_sum = &cpu_time->curr_runnable_sum; + dst_curr_runnable_sum = &wrq->curr_runnable_sum; + src_prev_runnable_sum = &cpu_time->prev_runnable_sum; + dst_prev_runnable_sum = &wrq->prev_runnable_sum; + + src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + dst_nt_curr_runnable_sum = &wrq->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + dst_nt_prev_runnable_sum = &wrq->nt_prev_runnable_sum; + + if (*src_curr_runnable_sum < wts->curr_window) { + printk_deferred("WALT-UG pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_curr_runnable_sum, + wts->curr_window); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + *src_curr_runnable_sum -= wts->curr_window; + + if (*src_prev_runnable_sum < wts->prev_window) { + printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_prev_runnable_sum, + wts->prev_window); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + *src_prev_runnable_sum -= wts->prev_window; + + if (new_task) { + if (*src_nt_curr_runnable_sum < wts->curr_window) { + printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_curr_runnable_sum, + wts->curr_window); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + *src_nt_curr_runnable_sum -= wts->curr_window; + + if (*src_nt_prev_runnable_sum < wts->prev_window) { + printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_prev_runnable_sum, + wts->prev_window); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + *src_nt_prev_runnable_sum -= wts->prev_window; + } + + /* + * Need to reset curr/prev windows for all CPUs, not just the + * ones in the same cluster. Since inter cluster migrations + * did not result in the appropriate book keeping, the values + * per CPU would be inaccurate. + */ + for_each_possible_cpu(i) { + wts->curr_window_cpu[i] = 0; + wts->prev_window_cpu[i] = 0; + } + } + + *dst_curr_runnable_sum += wts->curr_window; + *dst_prev_runnable_sum += wts->prev_window; + if (new_task) { + *dst_nt_curr_runnable_sum += wts->curr_window; + *dst_nt_prev_runnable_sum += wts->prev_window; + } + + /* + * When a task enter or exits a group, it's curr and prev windows are + * moved to a single CPU. This behavior might be sub-optimal in the + * exit case, however, it saves us the overhead of handling inter + * cluster migration fixups while the task is part of a related group. + */ + wts->curr_window_cpu[cpu] = wts->curr_window; + wts->prev_window_cpu[cpu] = wts->prev_window; + + trace_sched_migration_update_sum(p, migrate_type, rq); +} + +bool is_rtgb_active(void) +{ + struct walt_related_thread_group *grp; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + return grp && grp->skip_min; +} + +u64 get_rtgb_active_time(void) +{ + struct walt_related_thread_group *grp; + u64 now = sched_clock(); + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + + if (grp && grp->skip_min && grp->start_ts) + return now - grp->start_ts; + + return 0; +} + +static void walt_init_window_dep(void); +static void walt_tunables_fixup(void) +{ + if (likely(num_sched_clusters > 0)) + walt_update_group_thresholds(); + walt_init_window_dep(); +} + +static void walt_update_irqload(struct rq *rq) +{ + u64 irq_delta = 0; + unsigned int nr_windows = 0; + u64 cur_irq_time; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 last_irq_window = READ_ONCE(wrq->last_irq_window); + + if (wrq->window_start > last_irq_window) + nr_windows = div64_u64(wrq->window_start - last_irq_window, + sched_ravg_window); + + /* Decay CPU's irqload by 3/4 for each window. */ + if (nr_windows < 10) + wrq->avg_irqload = mult_frac(wrq->avg_irqload, 3, 4); + else + wrq->avg_irqload = 0; + + cur_irq_time = irq_time_read(cpu_of(rq)); + if (cur_irq_time > wrq->prev_irq_time) + irq_delta = cur_irq_time - wrq->prev_irq_time; + + wrq->avg_irqload += irq_delta; + wrq->prev_irq_time = cur_irq_time; + + if (nr_windows < SCHED_HIGH_IRQ_TIMEOUT) + wrq->high_irqload = (wrq->avg_irqload >= + walt_cpu_high_irqload); + else + wrq->high_irqload = false; +} + +/* + * Runs in hard-irq context. This should ideally run just after the latest + * window roll-over. + */ +static void walt_irq_work(struct irq_work *irq_work) +{ + struct walt_sched_cluster *cluster; + struct rq *rq; + int cpu; + u64 wc; + bool is_migration = false, is_asym_migration = false; + u64 total_grp_load = 0, min_cluster_grp_load = 0; + int level = 0; + unsigned long flags; + struct walt_rq *wrq; + + /* Am I the window rollover work or the migration work? */ + if (irq_work == &walt_migration_irq_work) + is_migration = true; + + for_each_cpu(cpu, cpu_possible_mask) { + if (level == 0) + raw_spin_lock(&cpu_rq(cpu)->lock); + else + raw_spin_lock_nested(&cpu_rq(cpu)->lock, level); + level++; + } + + wc = sched_ktime_clock(); + walt_load_reported_window = atomic64_read(&walt_irq_work_lastq_ws); + for_each_sched_cluster(cluster) { + u64 aggr_grp_load = 0; + + raw_spin_lock(&cluster->load_lock); + + for_each_cpu(cpu, &cluster->cpus) { + rq = cpu_rq(cpu); + wrq = (struct walt_rq *) rq->android_vendor_data1; + if (rq->curr) { + walt_update_task_ravg(rq->curr, rq, + TASK_UPDATE, wc, 0); + account_load_subtractions(rq); + aggr_grp_load += + wrq->grp_time.prev_runnable_sum; + } + if (is_migration && wrq->notif_pending && + cpumask_test_cpu(cpu, &asym_cap_sibling_cpus)) { + is_asym_migration = true; + wrq->notif_pending = false; + } + } + + cluster->aggr_grp_load = aggr_grp_load; + total_grp_load += aggr_grp_load; + + if (is_min_capacity_cluster(cluster)) + min_cluster_grp_load = aggr_grp_load; + raw_spin_unlock(&cluster->load_lock); + } + + if (total_grp_load) { + if (cpumask_weight(&asym_cap_sibling_cpus)) { + u64 big_grp_load = + total_grp_load - min_cluster_grp_load; + + for_each_cpu(cpu, &asym_cap_sibling_cpus) + cpu_cluster(cpu)->aggr_grp_load = big_grp_load; + } + rtgb_active = is_rtgb_active(); + } else { + rtgb_active = false; + } + + if (!is_migration && sysctl_sched_user_hint && time_after(jiffies, + sched_user_hint_reset_time)) + sysctl_sched_user_hint = 0; + + for_each_sched_cluster(cluster) { + cpumask_t cluster_online_cpus; + unsigned int num_cpus, i = 1; + + cpumask_and(&cluster_online_cpus, &cluster->cpus, + cpu_online_mask); + num_cpus = cpumask_weight(&cluster_online_cpus); + for_each_cpu(cpu, &cluster_online_cpus) { + int wflag = 0; + + /* + * FIXME: + * + * For now supporting both schedutil and waltgov. + * This is not by design but for convenience. + */ + rq = cpu_rq(cpu); + wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (is_migration) { + if (wrq->notif_pending) { + wrq->notif_pending = false; + + wflag |= WALT_CPUFREQ_IC_MIGRATION; + } + } else { + wflag |= WALT_CPUFREQ_ROLLOVER; + } + + if (is_asym_migration && cpumask_test_cpu(cpu, + &asym_cap_sibling_cpus)) { + wflag |= WALT_CPUFREQ_IC_MIGRATION; + } + + if (i == num_cpus) + waltgov_run_callback(cpu_rq(cpu), wflag); + else + waltgov_run_callback(cpu_rq(cpu), wflag | + WALT_CPUFREQ_CONTINUE); + i++; + + if (!is_migration) + walt_update_irqload(rq); + } + } + + /* + * If the window change request is in pending, good place to + * change sched_ravg_window since all rq locks are acquired. + * + * If the current window roll over is delayed such that the + * mark_start (current wallclock with which roll over is done) + * of the current task went past the window start with the + * updated new window size, delay the update to the next + * window roll over. Otherwise the CPU counters (prs and crs) are + * not rolled over properly as mark_start > window_start. + */ + if (!is_migration) { + spin_lock_irqsave(&sched_ravg_window_lock, flags); + wrq = (struct walt_rq *) this_rq()->android_vendor_data1; + if ((sched_ravg_window != new_sched_ravg_window) && + (wc < wrq->window_start + new_sched_ravg_window)) { + sched_ravg_window_change_time = sched_ktime_clock(); + trace_sched_ravg_window_change(sched_ravg_window, + new_sched_ravg_window, + sched_ravg_window_change_time); + sched_ravg_window = new_sched_ravg_window; + walt_tunables_fixup(); + } + spin_unlock_irqrestore(&sched_ravg_window_lock, flags); + } + + for_each_cpu(cpu, cpu_possible_mask) + raw_spin_unlock(&cpu_rq(cpu)->lock); + + if (!is_migration) { + wrq = (struct walt_rq *) this_rq()->android_vendor_data1; + core_ctl_check(wrq->window_start); + } +} + +void walt_rotation_checkpoint(int nr_big) +{ + if (!hmp_capable()) + return; + + if (!sysctl_sched_walt_rotate_big_tasks || sched_boost() != NO_BOOST) { + walt_rotation_enabled = 0; + return; + } + + walt_rotation_enabled = nr_big >= num_possible_cpus(); +} + +void walt_fill_ta_data(struct core_ctl_notif_data *data) +{ + struct walt_related_thread_group *grp; + unsigned long flags; + u64 total_demand = 0, wallclock; + struct task_struct *p; + int min_cap_cpu, scale = 1024; + struct walt_sched_cluster *cluster; + int i = 0; + struct walt_task_struct *wts; + struct list_head *task_list; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + + raw_spin_lock_irqsave(&grp->lock, flags); + if (list_empty(&grp->tasks)) { + raw_spin_unlock_irqrestore(&grp->lock, flags); + goto fill_util; + } + + wallclock = sched_ktime_clock(); + + list_for_each(task_list, &grp->tasks) { + p = (struct task_struct *) task_list; + wts = (struct walt_task_struct *) p->android_vendor_data1; + if (wts->mark_start < wallclock - + (sched_ravg_window * sched_ravg_hist_size)) + continue; + + total_demand += wts->coloc_demand; + } + + raw_spin_unlock_irqrestore(&grp->lock, flags); + + /* + * Scale the total demand to the lowest capacity CPU and + * convert into percentage. + * + * P = total_demand/sched_ravg_window * 1024/scale * 100 + */ + + min_cap_cpu = cpumask_first(&cpu_array[0][0]); + if (min_cap_cpu != -1) + scale = arch_scale_cpu_capacity(min_cap_cpu); + + data->coloc_load_pct = div64_u64(total_demand * 1024 * 100, + (u64)sched_ravg_window * scale); + +fill_util: + for_each_sched_cluster(cluster) { + int fcpu = cluster_first_cpu(cluster); + + if (i == MAX_CLUSTERS) + break; + + scale = arch_scale_cpu_capacity(fcpu); + data->ta_util_pct[i] = div64_u64(cluster->aggr_grp_load * 1024 * + 100, (u64)sched_ravg_window * scale); + + scale = arch_scale_freq_capacity(fcpu); + data->cur_cap_pct[i] = (scale * 100)/1024; + i++; + } +} + +static void walt_init_window_dep(void) +{ + walt_cpu_util_freq_divisor = + (sched_ravg_window >> SCHED_CAPACITY_SHIFT) * 100; + walt_scale_demand_divisor = sched_ravg_window >> SCHED_CAPACITY_SHIFT; + + sched_init_task_load_windows = + div64_u64((u64)sysctl_sched_init_task_load_pct * + (u64)sched_ravg_window, 100); + sched_init_task_load_windows_scaled = + scale_demand(sched_init_task_load_windows); + + walt_cpu_high_irqload = div64_u64((u64)sched_ravg_window * 95, (u64) 100); +} + +static void walt_init_once(void) +{ + init_irq_work(&walt_migration_irq_work, walt_irq_work); + init_irq_work(&walt_cpufreq_irq_work, walt_irq_work); + walt_init_window_dep(); +} + +static void walt_sched_init_rq(struct rq *rq) +{ + int j; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (cpu_of(rq) == 0) + walt_init_once(); + + cpumask_set_cpu(cpu_of(rq), &wrq->freq_domain_cpumask); + + wrq->walt_stats.cumulative_runnable_avg_scaled = 0; + wrq->prev_window_size = sched_ravg_window; + wrq->window_start = 0; + wrq->walt_stats.nr_big_tasks = 0; + wrq->walt_flags = 0; + wrq->avg_irqload = 0; + wrq->prev_irq_time = 0; + wrq->last_irq_window = 0; + wrq->high_irqload = false; + wrq->task_exec_scale = 1024; + wrq->push_task = NULL; + + /* + * All cpus part of same cluster by default. This avoids the + * need to check for wrq->cluster being non-NULL in hot-paths + * like select_best_cpu() + */ + wrq->cluster = &init_cluster; + wrq->curr_runnable_sum = wrq->prev_runnable_sum = 0; + wrq->nt_curr_runnable_sum = wrq->nt_prev_runnable_sum = 0; + memset(&wrq->grp_time, 0, sizeof(struct group_cpu_time)); + wrq->old_busy_time = 0; + wrq->old_estimated_time = 0; + wrq->walt_stats.pred_demands_sum_scaled = 0; + wrq->walt_stats.nr_rtg_high_prio_tasks = 0; + wrq->ed_task = NULL; + wrq->curr_table = 0; + wrq->prev_top = 0; + wrq->curr_top = 0; + wrq->last_cc_update = 0; + wrq->cycles = 0; + for (j = 0; j < NUM_TRACKED_WINDOWS; j++) { + memset(&wrq->load_subs[j], 0, + sizeof(struct load_subtractions)); + wrq->top_tasks[j] = kcalloc(NUM_LOAD_INDICES, + sizeof(u8), GFP_ATOMIC | GFP_NOWAIT); + /* No other choice */ + BUG_ON(!wrq->top_tasks[j]); + clear_top_tasks_bitmap(wrq->top_tasks_bitmap[j]); + } + wrq->cum_window_demand_scaled = 0; + wrq->notif_pending = false; +} + +void sched_window_nr_ticks_change(void) +{ + unsigned long flags; + + spin_lock_irqsave(&sched_ravg_window_lock, flags); + new_sched_ravg_window = mult_frac(sysctl_sched_ravg_window_nr_ticks, + NSEC_PER_SEC, HZ); + spin_unlock_irqrestore(&sched_ravg_window_lock, flags); +} + +static void +walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + fixup_cumulative_runnable_avg(&wrq->walt_stats, wts->demand_scaled, + wts->pred_demand_scaled); +} + +static void +walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + fixup_cumulative_runnable_avg(&wrq->walt_stats, + -(s64)wts->demand_scaled, + -(s64)wts->pred_demand_scaled); +} + +static void inc_rq_walt_stats(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (wts->misfit) + wrq->walt_stats.nr_big_tasks++; + + wts->rtg_high_prio = task_rtg_high_prio(p); + if (wts->rtg_high_prio) + wrq->walt_stats.nr_rtg_high_prio_tasks++; +} + +static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (wts->misfit) + wrq->walt_stats.nr_big_tasks--; + + if (wts->rtg_high_prio) + wrq->walt_stats.nr_rtg_high_prio_tasks--; + + BUG_ON(wrq->walt_stats.nr_big_tasks < 0); +} + +static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new) +{ + add_new_task_to_grp(new); +} + +/* + * The intention of this hook is to update cpu_capacity_orig as well as + * (*capacity), otherwise we will end up capacity_of() > capacity_orig_of(). + */ +static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity) +{ + unsigned long max_capacity = arch_scale_cpu_capacity(cpu); + unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); + unsigned long thermal_cap; + + /* + * thermal_pressure = max_capacity - curr_cap_as_per_thermal. + * so, + * curr_cap_as_per_thermal = max_capacity - thermal_pressure. + */ + + thermal_cap = max_capacity - thermal_pressure; + + /* + * TODO: + * Thermal is taken care now. but what about limits via + * cpufreq max. we don't have arch_scale_max_freq_capacity() + * in 5.10 now. + * + * Two options: + * #1 either port that max_frq_cap patch to AOSP + * #2 register for cpufreq policy updates.. + */ + cpu_rq(cpu)->cpu_capacity_orig = min(cpu_rq(cpu)->cpu_capacity_orig, + thermal_cap); + *capacity = cpu_rq(cpu)->cpu_capacity_orig; +} + +static void android_rvh_sched_cpu_starting(void *unused, int cpu) +{ + unsigned long flags; + struct rq *rq = cpu_rq(cpu); + + raw_spin_lock_irqsave(&rq->lock, flags); + set_window_start(rq); + raw_spin_unlock_irqrestore(&rq->lock, flags); + + clear_walt_request(cpu); +} + +static void android_rvh_sched_cpu_dying(void *unused, int cpu) +{ + clear_walt_request(cpu); +} + +static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsigned int new_cpu) +{ + if (new_cpu < 0) + return; + fixup_busy_time(p, (int) new_cpu); +} + +static void android_rvh_sched_fork(void *unused, struct task_struct *p) +{ + init_new_task_load(p); +} + +static void android_rvh_new_task_stats(void *unused, struct task_struct *p) +{ + mark_task_starting(p); +} + +static void android_rvh_account_irq(void *unused, struct task_struct *curr, int cpu, s64 delta) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + if (!!(curr->flags & PF_IDLE)) { + if (hardirq_count() || in_serving_softirq()) + walt_sched_account_irqend(cpu, curr, delta); + else + walt_sched_account_irqstart(cpu, curr); + } + wrq->last_irq_window = wrq->window_start; +} + +static void android_rvh_flush_task(void *unused, struct task_struct *p) +{ + walt_task_dead(p); +} + +static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) +{ + u64 wallclock = sched_ktime_clock(); + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->last_enqueued_ts = wallclock; + sched_update_nr_prod(rq->cpu, true); + + if (fair_policy(p->policy)) { + wts->misfit = !task_fits_max(p, rq->cpu); + inc_rq_walt_stats(rq, p); + } + + walt_inc_cumulative_runnable_avg(rq, p); + trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0]); +} + +static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) +{ + /* + * TODO: remove later. + * We don't have to check if p is ed task and clear it. the below + * code calls is_ed_task_present() which clears the rq's ed_task + * unconditionally. + */ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (p == wrq->ed_task) + is_ed_task_present(rq, sched_ktime_clock()); + + sched_update_nr_prod(rq->cpu, false); + + if (fair_policy(p->policy)) + dec_rq_walt_stats(rq, p); + + walt_dec_cumulative_runnable_avg(rq, p); + trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0]); +} + +static void android_rvh_update_misfit_status(void *unused, struct task_struct *p, + struct rq *rq, bool *need_update) +{ + struct walt_task_struct *wts; + struct walt_rq *wrq; + bool old_misfit, misfit; + int change; + + *need_update = false; + + if (!p) { + rq->misfit_task_load = 0; + return; + } + + wrq = (struct walt_rq *) rq->android_vendor_data1; + wts = (struct walt_task_struct *) p->android_vendor_data1; + old_misfit = wts->misfit; + + if (task_fits_capacity(p, capacity_orig_of(cpu_of(rq)), rq->cpu)) + rq->misfit_task_load = 0; + else + rq->misfit_task_load = task_load(p); + + misfit = rq->misfit_task_load; + + change = misfit - old_misfit; + if (change) { + sched_update_nr_prod(rq->cpu, true); + wts->misfit = misfit; + wrq->walt_stats.nr_big_tasks += change; + BUG_ON(wrq->walt_stats.nr_big_tasks < 0); + } +} + +/* utility function to update walt signals at wakeup */ +static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) +{ + struct rq *rq = cpu_rq(task_cpu(p)); + struct rq_flags rf; + u64 wallclock; + unsigned int old_load; + struct walt_related_thread_group *grp = NULL; + + rq_lock_irqsave(rq, &rf); + old_load = task_load(p); + wallclock = sched_ktime_clock(); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); + note_task_waking(p, wallclock); + rq_unlock_irqrestore(rq, &rf); + + rcu_read_lock(); + grp = task_related_thread_group(p); + if (update_preferred_cluster(grp, p, old_load, false)) + set_preferred_cluster(grp); + rcu_read_unlock(); +} + +static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct *p) +{ + unsigned long flags; + int cpu = p->cpu; + + if (!sched_predl) + return; + + raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags); + if (do_pl_notif(cpu_rq(cpu))) + waltgov_run_callback(cpu_rq(cpu), WALT_CPUFREQ_PL); + raw_spin_unlock_irqrestore(&cpu_rq(cpu)->lock, flags); +} + +static void android_rvh_tick_entry(void *unused, struct rq *rq) +{ + u64 wallclock; + u32 old_load; + struct walt_related_thread_group *grp; + + set_window_start(rq); + wallclock = sched_ktime_clock(); + + old_load = task_load(rq->curr); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + + rcu_read_lock(); + grp = task_related_thread_group(rq->curr); + if (update_preferred_cluster(grp, rq->curr, old_load, true)) + set_preferred_cluster(grp); + rcu_read_unlock(); + + if (is_ed_task_present(rq, wallclock)) + waltgov_run_callback(rq, WALT_CPUFREQ_EARLY_DET); + + /* TODO + * currently load balancer registered for a post-hook which + * takes care of rotation and migration for misfit tasks. + * + * See if that can also be done here. + */ +} + +static void android_rvh_schedule(void *unused, struct task_struct *prev, + struct task_struct *next, struct rq *rq) +{ + u64 wallclock = sched_ktime_clock(); + struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; + + if (likely(prev != next)) { + if (!prev->on_rq) + wts->last_sleep_ts = wallclock; + walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); + walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); + } else { + walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0); + } +} + +static void android_rvh_resume_cpus(void *unused, struct cpumask *resuming_cpus, int *err) +{ + int i; + struct rq *rq; + unsigned long flags; + + /* + * send a reschedule event on all resumed CPUs + * which trigger newly idle load balance. + */ + for_each_cpu(i, resuming_cpus) { + rq = cpu_rq(i); + raw_spin_lock_irqsave(&rq->lock, flags); + resched_curr(rq); + raw_spin_unlock_irqrestore(&rq->lock, flags); + } + + *err = 0; +} + +static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, + cpumask_var_t cpus_requested, + const struct cpumask *new_mask, int *ret) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (cpumask_subset(&wts->cpus_requested, cpus_requested)) + *ret = set_cpus_allowed_ptr(p, &wts->cpus_requested); +} + +static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->last_sleep_ts = 0; + wts->wake_up_idle = false; + wts->boost = 0; + wts->boost_expires = 0; + wts->boost_period = false; + wts->low_latency = false; +} + +static void android_rvh_ttwu_cond(void *unused, bool *cond) +{ + *cond = sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT; +} + +static void android_rvh_sched_exec(void *unused, bool *cond) +{ + *cond = true; +} + +static void android_rvh_build_perf_domains(void *unused, bool *eas_check) +{ + *eas_check = true; +} + +static void register_walt_hooks(void) +{ + register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); + register_trace_android_rvh_update_cpu_capacity(android_rvh_update_cpu_capacity, NULL); + register_trace_android_rvh_sched_cpu_starting(android_rvh_sched_cpu_starting, NULL); + register_trace_android_rvh_sched_cpu_dying(android_rvh_sched_cpu_dying, NULL); + register_trace_android_rvh_set_task_cpu(android_rvh_set_task_cpu, NULL); + register_trace_android_rvh_new_task_stats(android_rvh_new_task_stats, NULL); + register_trace_android_rvh_sched_fork(android_rvh_sched_fork, NULL); + register_trace_android_rvh_account_irq(android_rvh_account_irq, NULL); + register_trace_android_rvh_flush_task(android_rvh_flush_task, NULL); + register_trace_android_rvh_update_misfit_status(android_rvh_update_misfit_status, NULL); + register_trace_android_rvh_enqueue_task(android_rvh_enqueue_task, NULL); + register_trace_android_rvh_dequeue_task(android_rvh_dequeue_task, NULL); + register_trace_android_rvh_try_to_wake_up(android_rvh_try_to_wake_up, NULL); + register_trace_android_rvh_try_to_wake_up_success(android_rvh_try_to_wake_up_success, NULL); + register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL); + register_trace_android_rvh_schedule(android_rvh_schedule, NULL); + register_trace_android_rvh_resume_cpus(android_rvh_resume_cpus, NULL); + register_trace_android_vh_show_max_freq(android_vh_show_max_freq, NULL); + register_trace_android_rvh_cpu_cgroup_attach(android_rvh_cpu_cgroup_attach, NULL); + register_trace_android_rvh_update_cpus_allowed(android_rvh_update_cpus_allowed, NULL); + register_trace_android_rvh_sched_fork_init(android_rvh_sched_fork_init, NULL); + register_trace_android_rvh_ttwu_cond(android_rvh_ttwu_cond, NULL); + register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); + register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); +} + +atomic64_t walt_irq_work_lastq_ws; + +static int walt_init_stop_handler(void *data) +{ + int cpu; + struct task_struct *g, *p; + u64 window_start_ns, nr_windows; + struct walt_rq *wrq; + + read_lock(&tasklist_lock); + for_each_possible_cpu(cpu) { + raw_spin_lock(&cpu_rq(cpu)->lock); + } + + do_each_thread(g, p) { + init_existing_task_load(p); + } while_each_thread(g, p); + + window_start_ns = ktime_get_ns(); + nr_windows = div64_u64(window_start_ns, sched_ravg_window); + window_start_ns = (u64)nr_windows * (u64)sched_ravg_window; + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + /* Create task members for idle thread */ + init_new_task_load(rq->idle); + + walt_sched_init_rq(rq); + + wrq = (struct walt_rq *) rq->android_vendor_data1; + wrq->window_start = window_start_ns; + } + + atomic64_set(&walt_irq_work_lastq_ws, window_start_ns); + + register_walt_hooks(); + walt_lb_init(); + walt_rt_init(); + walt_cfs_init(); + create_default_coloc_group(); + + walt_update_cluster_topology(); + + for_each_possible_cpu(cpu) { + raw_spin_unlock(&cpu_rq(cpu)->lock); + } + read_unlock(&tasklist_lock); + + return 0; +} + +static int walt_module_init(void) +{ + struct ctl_table_header *hdr; + int i; + + walt_tunables(); + + sched_init_ops(); + BUG_ON(alloc_related_thread_groups()); + walt_init_cycle_counter(); + init_clusters(); + stop_machine(walt_init_stop_handler, NULL, NULL); + + hdr = register_sysctl_table(walt_base_table); + kmemleak_not_leak(hdr); + + input_boost_init(); + core_ctl_init(); + waltgov_register(); + + i = match_string(sched_feat_names, __SCHED_FEAT_NR, "TTWU_QUEUE"); + static_key_disable_cpuslocked(&sched_feat_keys[i]); + sysctl_sched_features &= ~(1UL << i); + + return 0; +} + +module_init(walt_module_init); +MODULE_LICENSE("GPL v2"); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h new file mode 100644 index 000000000000..0315be82852a --- /dev/null +++ b/kernel/sched/walt/walt.h @@ -0,0 +1,1006 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _WALT_H +#define _WALT_H + +#include "../../../kernel/sched/sched.h" +#include "../../../fs/proc/internal.h" +#include +#include + +#ifdef CONFIG_HZ_300 +/* + * Tick interval becomes to 3333333 due to + * rounding error when HZ=300. + */ +#define DEFAULT_SCHED_RAVG_WINDOW (3333333 * 5) +#else +/* Min window size (in ns) = 16ms */ +#define DEFAULT_SCHED_RAVG_WINDOW 16000000 +#endif + +/* Max window size (in ns) = 1s */ +#define MAX_SCHED_RAVG_WINDOW 1000000000 + +#define NR_WINDOWS_PER_SEC (NSEC_PER_SEC / DEFAULT_SCHED_RAVG_WINDOW) + +#define SCHED_CPUFREQ_MIGRATION (1U << 1) +#define SCHED_CPUFREQ_INTERCLUSTER_MIG (1U << 3) +#define SCHED_CPUFREQ_WALT (1U << 4) +#define SCHED_CPUFREQ_PL (1U << 5) +#define SCHED_CPUFREQ_EARLY_DET (1U << 6) +#define SCHED_CPUFREQ_CONTINUE (1U << 8) + +#define MAX_CLUSTERS 3 +/* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */ +#define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1) + +enum task_event { + PUT_PREV_TASK = 0, + PICK_NEXT_TASK = 1, + TASK_WAKE = 2, + TASK_MIGRATE = 3, + TASK_UPDATE = 4, + IRQ_UPDATE = 5, +}; + +/* Note: this need to be in sync with migrate_type_names array */ +enum migrate_types { + GROUP_TO_RQ, + RQ_TO_GROUP, +}; + +enum task_boost_type { + TASK_BOOST_NONE = 0, + TASK_BOOST_ON_MID, + TASK_BOOST_ON_MAX, + TASK_BOOST_STRICT_MAX, + TASK_BOOST_END, +}; + +#define WALT_NR_CPUS 8 +#define RAVG_HIST_SIZE_MAX 5 +#define NUM_BUSY_BUCKETS 10 + +struct walt_task_struct { + /* + * 'mark_start' marks the beginning of an event (task waking up, task + * starting to execute, task being preempted) within a window + * + * 'sum' represents how runnable a task has been within current + * window. It incorporates both running time and wait time and is + * frequency scaled. + * + * 'sum_history' keeps track of history of 'sum' seen over previous + * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are + * ignored. + * + * 'demand' represents maximum sum seen over previous + * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency + * demand for tasks. + * + * 'curr_window_cpu' represents task's contribution to cpu busy time on + * various CPUs in the current window + * + * 'prev_window_cpu' represents task's contribution to cpu busy time on + * various CPUs in the previous window + * + * 'curr_window' represents the sum of all entries in curr_window_cpu + * + * 'prev_window' represents the sum of all entries in prev_window_cpu + * + * 'pred_demand' represents task's current predicted cpu busy time + * + * 'busy_buckets' groups historical busy time into different buckets + * used for prediction + * + * 'demand_scaled' represents task's demand scaled to 1024 + */ + u64 mark_start; + u32 sum, demand; + u32 coloc_demand; + u32 sum_history[RAVG_HIST_SIZE_MAX]; + u32 curr_window_cpu[WALT_NR_CPUS]; + u32 prev_window_cpu[WALT_NR_CPUS]; + u32 curr_window, prev_window; + u32 pred_demand; + u8 busy_buckets[NUM_BUSY_BUCKETS]; + u16 demand_scaled; + u16 pred_demand_scaled; + u64 active_time; + u64 last_win_size; + int boost; + bool wake_up_idle; + bool misfit; + bool rtg_high_prio; + bool low_latency; + u64 boost_period; + u64 boost_expires; + u64 last_sleep_ts; + u32 init_load_pct; + u32 unfilter; + u64 last_wake_ts; + u64 last_enqueued_ts; + struct walt_related_thread_group __rcu *grp; + struct list_head grp_list; + u64 cpu_cycles; + cpumask_t cpus_requested; +}; + +/*End linux/sched.h port */ +/*SCHED.H PORT*/ +extern __read_mostly bool sched_predl; + +struct walt_cpu_load { + unsigned long nl; + unsigned long pl; + bool rtgb_active; + u64 ws; +}; + +#define DECLARE_BITMAP_ARRAY(name, nr, bits) \ + unsigned long name[nr][BITS_TO_LONGS(bits)] + +struct walt_sched_stats { + int nr_big_tasks; + u64 cumulative_runnable_avg_scaled; + u64 pred_demands_sum_scaled; + unsigned int nr_rtg_high_prio_tasks; +}; + +#define NUM_TRACKED_WINDOWS 2 +#define NUM_LOAD_INDICES 1000 + +struct group_cpu_time { + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; +}; + +struct load_subtractions { + u64 window_start; + u64 subs; + u64 new_subs; +}; + +struct walt_rq { + struct task_struct *push_task; + struct walt_sched_cluster *cluster; + struct cpumask freq_domain_cpumask; + struct walt_sched_stats walt_stats; + + u64 window_start; + u32 prev_window_size; + unsigned long walt_flags; + + u64 avg_irqload; + u64 last_irq_window; + u64 prev_irq_time; + struct task_struct *ed_task; + u64 task_exec_scale; + u64 old_busy_time; + u64 old_estimated_time; + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; + u64 cum_window_demand_scaled; + struct group_cpu_time grp_time; + struct load_subtractions load_subs[NUM_TRACKED_WINDOWS]; + DECLARE_BITMAP_ARRAY(top_tasks_bitmap, + NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES); + u8 *top_tasks[NUM_TRACKED_WINDOWS]; + u8 curr_table; + int prev_top; + int curr_top; + bool notif_pending; + bool high_irqload; + u64 last_cc_update; + u64 cycles; +}; + +struct walt_sched_cluster { + raw_spinlock_t load_lock; + struct list_head list; + struct cpumask cpus; + int id; + /* + * max_possible_freq = maximum supported by hardware + */ + unsigned int cur_freq; + unsigned int max_possible_freq; + u64 aggr_grp_load; +}; + +struct walt_related_thread_group { + int id; + raw_spinlock_t lock; + struct list_head tasks; + struct list_head list; + bool skip_min; + struct rcu_head rcu; + u64 last_update; + u64 downmigrate_ts; + u64 start_ts; +}; + +extern struct walt_sched_cluster *sched_cluster[WALT_NR_CPUS]; + +extern struct walt_sched_cluster *rq_cluster(struct rq *rq); + +/*END SCHED.H PORT*/ + +extern int num_sched_clusters; +extern unsigned int sched_capacity_margin_up[WALT_NR_CPUS]; +extern unsigned int sched_capacity_margin_down[WALT_NR_CPUS]; +extern cpumask_t asym_cap_sibling_cpus; +extern cpumask_t __read_mostly **cpu_array; + +extern void sched_update_nr_prod(int cpu, bool enq); +extern unsigned int walt_big_tasks(int cpu); +extern void walt_rotate_work_init(void); +extern void walt_rotation_checkpoint(int nr_big); +extern void walt_fill_ta_data(struct core_ctl_notif_data *data); +extern int sched_set_group_id(struct task_struct *p, unsigned int group_id); +extern unsigned int sched_get_group_id(struct task_struct *p); +extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct); +extern u32 sched_get_init_task_load(struct task_struct *p); +extern void core_ctl_check(u64 wallclock); +extern int sched_set_boost(int enable); +extern int sched_pause_count(const cpumask_t *mask, bool include_offline); +extern void sched_pause_pending(int cpu); +extern void sched_unpause_pending(int cpu); +extern int sched_wake_up_idle_show(struct seq_file *m, void *v); +extern ssize_t sched_wake_up_idle_write(struct file *file, + const char __user *buf, size_t count, loff_t *offset); +extern int sched_wake_up_idle_open(struct inode *inode, struct file *filp); +extern int sched_init_task_load_show(struct seq_file *m, void *v); +extern ssize_t sched_init_task_load_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset); +extern int sched_init_task_load_open(struct inode *inode, struct file *filp); +extern int sched_group_id_show(struct seq_file *m, void *v); +extern ssize_t sched_group_id_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset); +extern int sched_group_id_open(struct inode *inode, struct file *filp); +extern int sched_pause_cpus(struct cpumask *pause_cpus); +extern int sched_unpause_cpus(struct cpumask *unpause_cpus); + +extern int core_ctl_set_boost(bool boost); +extern void core_ctl_notifier_register(struct notifier_block *n); +extern void core_ctl_notifier_unregister(struct notifier_block *n); +extern unsigned int sched_get_cpu_util(int cpu); +extern void sched_update_hyst_times(void); +extern u64 sched_lpm_disallowed_time(int cpu); +extern int +sched_updown_migrate_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); +extern int sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); +extern int sched_busy_hyst_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); +extern u64 sched_ktime_clock(void); +extern void clear_walt_request(int cpu); +extern void walt_init_tg(struct task_group *tg); +extern void walt_init_topapp_tg(struct task_group *tg); +extern void walt_init_foreground_tg(struct task_group *tg); +extern int register_walt_callback(void); +extern void set_cpu_array(void); +extern int sched_init_ops(void); +extern int core_ctl_init(void); +extern void acquire_rq_locks_irqsave(const cpumask_t *cpus, + unsigned long *flags); +extern void release_rq_locks_irqrestore(const cpumask_t *cpus, + unsigned long *flags); +extern struct list_head cluster_head; +extern int set_sched_ravg_window(char *str); +extern int set_sched_predl(char *str); +extern int input_boost_init(void); +extern int core_ctl_init(void); + +extern atomic64_t walt_irq_work_lastq_ws; +extern unsigned int __read_mostly sched_ravg_window; +extern unsigned int min_max_possible_capacity; +extern unsigned int max_possible_capacity; +extern unsigned int __read_mostly sched_init_task_load_windows; +extern unsigned int __read_mostly sched_load_granule; + +/* 1ms default for 20ms window size scaled to 1024 */ +extern unsigned int sysctl_sched_min_task_util_for_boost; +/* 0.68ms default for 20ms window size scaled to 1024 */ +extern unsigned int sysctl_sched_min_task_util_for_colocation; +extern unsigned int sysctl_sched_busy_hyst_enable_cpus; +extern unsigned int sysctl_sched_busy_hyst; +extern unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus; +extern unsigned int sysctl_sched_coloc_busy_hyst_cpu[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_coloc_busy_hyst_max_ms; +extern unsigned int sysctl_sched_coloc_busy_hyst_cpu_busy_pct[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_boost; /* To/from userspace */ +extern unsigned int sysctl_sched_capacity_margin_up[MAX_MARGIN_LEVELS]; +extern unsigned int sysctl_sched_capacity_margin_down[MAX_MARGIN_LEVELS]; +extern unsigned int sched_boost_type; /* currently activated sched boost */ +extern enum sched_boost_policy boost_policy; +extern unsigned int sysctl_input_boost_ms; +extern unsigned int sysctl_input_boost_freq[8]; +extern unsigned int sysctl_sched_boost_on_input; +extern unsigned int sysctl_sched_load_boost[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_user_hint; +extern unsigned int sysctl_sched_conservative_pl; +#define WALT_MANY_WAKEUP_DEFAULT 1000 +extern unsigned int sysctl_sched_many_wakeup_threshold; +extern unsigned int sysctl_walt_rtg_cfs_boost_prio; +extern __read_mostly unsigned int sysctl_sched_force_lb_enable; +extern const int sched_user_hint_max; +extern unsigned int sysctl_sched_prefer_spread; + +#define for_each_sched_cluster(cluster) \ + list_for_each_entry_rcu(cluster, &cluster_head, list) + +static inline u32 cpu_cycles_to_freq(u64 cycles, u64 period) +{ + return div64_u64(cycles, period); +} + +static inline unsigned int sched_cpu_legacy_freq(int cpu) +{ + unsigned long curr_cap = arch_scale_freq_capacity(cpu); + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return (curr_cap * (u64) wrq->cluster->max_possible_freq) >> + SCHED_CAPACITY_SHIFT; +} + +extern __read_mostly bool sched_freq_aggr_en; +static inline void walt_enable_frequency_aggregation(bool enable) +{ + sched_freq_aggr_en = enable; +} + +#ifndef CONFIG_IRQ_TIME_ACCOUNTING +static inline u64 irq_time_read(int cpu) { return 0; } +#endif + +/*Sysctl related interface*/ +#define WINDOW_STATS_RECENT 0 +#define WINDOW_STATS_MAX 1 +#define WINDOW_STATS_MAX_RECENT_AVG 2 +#define WINDOW_STATS_AVG 3 +#define WINDOW_STATS_INVALID_POLICY 4 + +extern unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; +extern unsigned int __read_mostly sysctl_sched_group_downmigrate_pct; +extern unsigned int __read_mostly sysctl_sched_group_upmigrate_pct; +extern unsigned int __read_mostly sysctl_sched_window_stats_policy; +extern unsigned int sysctl_sched_ravg_window_nr_ticks; +extern unsigned int sysctl_sched_dynamic_ravg_window_enable; +extern unsigned int sysctl_sched_walt_rotate_big_tasks; +extern unsigned int sysctl_sched_task_unfilter_period; +extern unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; +extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ +extern unsigned int sysctl_task_read_pid; +extern struct ctl_table walt_table[]; +extern struct ctl_table walt_base_table[]; +extern void walt_tunables(void); +extern void walt_update_group_thresholds(void); +extern void sched_window_nr_ticks_change(void); +extern unsigned long sched_user_hint_reset_time; +extern struct irq_work walt_migration_irq_work; +extern __read_mostly unsigned int new_sched_ravg_window; +extern struct task_group *task_group_topapp; +extern struct task_group *task_group_foreground; + +#define LIB_PATH_LENGTH 512 +extern unsigned int cpuinfo_max_freq_cached; +extern char sched_lib_name[LIB_PATH_LENGTH]; +extern unsigned int sched_lib_mask_force; +extern bool is_sched_lib_based_app(pid_t pid); +void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy, + unsigned int *max_freq); + +/* WALT cpufreq interface */ +#define WALT_CPUFREQ_ROLLOVER (1U << 0) +#define WALT_CPUFREQ_CONTINUE (1U << 1) +#define WALT_CPUFREQ_IC_MIGRATION (1U << 2) +#define WALT_CPUFREQ_PL (1U << 3) +#define WALT_CPUFREQ_EARLY_DET (1U << 4) + +#define NO_BOOST 0 +#define FULL_THROTTLE_BOOST 1 +#define CONSERVATIVE_BOOST 2 +#define RESTRAINED_BOOST 3 +#define FULL_THROTTLE_BOOST_DISABLE -1 +#define CONSERVATIVE_BOOST_DISABLE -2 +#define RESTRAINED_BOOST_DISABLE -3 +#define MAX_NUM_BOOST_TYPE (RESTRAINED_BOOST+1) + +enum sched_boost_policy { + SCHED_BOOST_NONE, + SCHED_BOOST_ON_BIG, + SCHED_BOOST_ON_ALL, +}; + +struct walt_task_group { + /* + * Controls whether tasks of this cgroup should be colocated with each + * other and tasks of other cgroups that have the same flag turned on. + */ + bool colocate; + /* + * array indicating whether this task group participates in the + * particular boost type + */ + bool sched_boost_enable[MAX_NUM_BOOST_TYPE]; +}; + +struct sched_avg_stats { + int nr; + int nr_misfit; + int nr_max; + int nr_scaled; +}; + +struct waltgov_callback { + void (*func)(struct waltgov_callback *cb, u64 time, unsigned int flags); +}; + +DECLARE_PER_CPU(struct waltgov_callback *, waltgov_cb_data); + +static inline void waltgov_add_callback(int cpu, struct waltgov_callback *cb, + void (*func)(struct waltgov_callback *cb, u64 time, + unsigned int flags)) +{ + if (WARN_ON(!cb || !func)) + return; + + if (WARN_ON(per_cpu(waltgov_cb_data, cpu))) + return; + + cb->func = func; + rcu_assign_pointer(per_cpu(waltgov_cb_data, cpu), cb); +} + +static inline void waltgov_remove_callback(int cpu) +{ + rcu_assign_pointer(per_cpu(waltgov_cb_data, cpu), NULL); +} + +static inline void waltgov_run_callback(struct rq *rq, unsigned int flags) +{ + struct waltgov_callback *cb; + + cb = rcu_dereference_sched(*per_cpu_ptr(&waltgov_cb_data, cpu_of(rq))); + if (cb) + cb->func(cb, sched_ktime_clock(), flags); +} + +extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load); +int waltgov_register(void); + +extern void walt_lb_init(void); +extern unsigned int walt_rotation_enabled; + +/* + * Returns the current capacity of cpu after applying both + * cpu and freq scaling. + */ +static inline unsigned long capacity_curr_of(int cpu) +{ + unsigned long max_cap = cpu_rq(cpu)->cpu_capacity_orig; + unsigned long scale_freq = arch_scale_freq_capacity(cpu); + + return cap_scale(max_cap, scale_freq); +} + +static inline unsigned long task_util(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->demand_scaled; +} + +static inline unsigned long cpu_util(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + u64 walt_cpu_util = wrq->walt_stats.cumulative_runnable_avg_scaled; + + return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu)); +} + +static inline unsigned long cpu_util_cum(int cpu, int delta) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + u64 util = wrq->cum_window_demand_scaled; + unsigned long capacity = capacity_orig_of(cpu); + + delta += util; + if (delta < 0) + return 0; + + return (delta >= capacity) ? capacity : delta; +} + +extern unsigned int capacity_margin_freq; + +static inline unsigned long +add_capacity_margin(unsigned long cpu_capacity, int cpu) +{ + cpu_capacity = cpu_capacity * capacity_margin_freq * + (100 + sysctl_sched_load_boost[cpu]); + cpu_capacity /= 100; + cpu_capacity /= SCHED_CAPACITY_SCALE; + return cpu_capacity; +} + +static inline enum sched_boost_policy sched_boost_policy(void) +{ + return boost_policy; +} + +static inline int sched_boost(void) +{ + return sched_boost_type; +} + +static inline bool rt_boost_on_big(void) +{ + return sched_boost() == FULL_THROTTLE_BOOST ? + (sched_boost_policy() == SCHED_BOOST_ON_BIG) : false; +} + +static inline bool is_full_throttle_boost(void) +{ + return sched_boost() == FULL_THROTTLE_BOOST; +} + +static inline bool task_sched_boost(struct task_struct *p) +{ + struct cgroup_subsys_state *css; + struct task_group *tg; + bool sched_boost_enabled; + struct walt_task_group *wtg; + + /* optimization for FT boost, skip looking at tg */ + if (sched_boost() == FULL_THROTTLE_BOOST) + return true; + + rcu_read_lock(); + css = task_css(p, cpu_cgrp_id); + if (!css) { + rcu_read_unlock(); + return false; + } + tg = container_of(css, struct task_group, css); + wtg = (struct walt_task_group *) tg->android_vendor_data1; + sched_boost_enabled = wtg->sched_boost_enable[sched_boost()]; + rcu_read_unlock(); + + return sched_boost_enabled; +} + +static inline bool task_placement_boost_enabled(struct task_struct *p) +{ + if (likely(sched_boost_policy() == SCHED_BOOST_NONE)) + return false; + + return task_sched_boost(p); +} + +static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) +{ + enum sched_boost_policy policy; + + if (likely(sched_boost_policy() == SCHED_BOOST_NONE)) + return SCHED_BOOST_NONE; + + policy = task_sched_boost(p) ? sched_boost_policy() : SCHED_BOOST_NONE; + if (policy == SCHED_BOOST_ON_BIG) { + /* + * Filter out tasks less than min task util threshold + * under conservative boost. + */ + if (sched_boost() == CONSERVATIVE_BOOST && + task_util(p) <= sysctl_sched_min_task_util_for_boost) + policy = SCHED_BOOST_NONE; + } + + return policy; +} + +static inline unsigned long capacity_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity; +} + +static inline bool __cpu_overutilized(int cpu, int delta) +{ + return (capacity_orig_of(cpu) * 1024) < + ((cpu_util(cpu) + delta) * sched_capacity_margin_up[cpu]); +} + +static inline bool cpu_overutilized(int cpu) +{ + return __cpu_overutilized(cpu, 0); +} + +static inline int asym_cap_siblings(int cpu1, int cpu2) +{ + return (cpumask_test_cpu(cpu1, &asym_cap_sibling_cpus) && + cpumask_test_cpu(cpu2, &asym_cap_sibling_cpus)); +} + +static inline bool asym_cap_sibling_group_has_capacity(int dst_cpu, int margin) +{ + int sib1, sib2; + int nr_running; + unsigned long total_util, total_capacity; + + if (cpumask_empty(&asym_cap_sibling_cpus) || + cpumask_test_cpu(dst_cpu, &asym_cap_sibling_cpus)) + return false; + + sib1 = cpumask_first(&asym_cap_sibling_cpus); + sib2 = cpumask_last(&asym_cap_sibling_cpus); + + if (!cpu_active(sib1) || !cpu_active(sib2)) + return false; + + nr_running = cpu_rq(sib1)->cfs.h_nr_running + + cpu_rq(sib2)->cfs.h_nr_running; + + if (nr_running <= 2) + return true; + + total_capacity = capacity_of(sib1) + capacity_of(sib2); + total_util = cpu_util(sib1) + cpu_util(sib2); + + return ((total_capacity * 100) > (total_util * margin)); +} + +/* Is frequency of two cpus synchronized with each other? */ +static inline int same_freq_domain(int src_cpu, int dst_cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; + + if (src_cpu == dst_cpu) + return 1; + + if (asym_cap_siblings(src_cpu, dst_cpu)) + return 1; + + return cpumask_test_cpu(dst_cpu, &wrq->freq_domain_cpumask); +} + +static inline unsigned long task_util_est(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->demand_scaled; +} + +#ifdef CONFIG_UCLAMP_TASK +static inline unsigned long uclamp_task_util(struct task_struct *p) +{ + return clamp(task_util_est(p), + uclamp_eff_value(p, UCLAMP_MIN), + uclamp_eff_value(p, UCLAMP_MAX)); +} +#else +static inline unsigned long uclamp_task_util(struct task_struct *p) +{ + return task_util_est(p); +} +#endif + +static inline int per_task_boost(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (wts->boost_period) { + if (sched_clock() > wts->boost_expires) { + wts->boost_period = 0; + wts->boost_expires = 0; + wts->boost = 0; + } + } + return wts->boost; +} + +static inline int cluster_first_cpu(struct walt_sched_cluster *cluster) +{ + return cpumask_first(&cluster->cpus); +} + +static inline bool hmp_capable(void) +{ + return max_possible_capacity != min_max_possible_capacity; +} + +static inline bool is_max_capacity_cpu(int cpu) +{ + return arch_scale_cpu_capacity(cpu) == max_possible_capacity; +} + +static inline bool is_min_capacity_cpu(int cpu) +{ + return arch_scale_cpu_capacity(cpu) == min_max_possible_capacity; +} + +static inline bool is_min_capacity_cluster(struct walt_sched_cluster *cluster) +{ + return is_min_capacity_cpu(cluster_first_cpu(cluster)); +} + +/* + * This is only for tracepoints to print the avg irq load. For + * task placment considerations, use sched_cpu_high_irqload(). + */ +#define SCHED_HIGH_IRQ_TIMEOUT 3 +static inline u64 sched_irqload(int cpu) +{ + s64 delta; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + delta = wrq->window_start - wrq->last_irq_window; + if (delta < SCHED_HIGH_IRQ_TIMEOUT) + return wrq->avg_irqload; + else + return 0; +} + +static inline int sched_cpu_high_irqload(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->high_irqload; +} + +static inline u64 +scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq) +{ + return div64_u64(load * (u64)src_freq, (u64)dst_freq); +} + +static inline unsigned int max_task_load(void) +{ + return sched_ravg_window; +} + +static inline int same_cluster(int src_cpu, int dst_cpu) +{ + struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; + struct walt_rq *dest_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + + return src_wrq->cluster == dest_wrq->cluster; +} + +static inline bool is_suh_max(void) +{ + return sysctl_sched_user_hint == sched_user_hint_max; +} + +#define DEFAULT_CGROUP_COLOC_ID 1 +static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_related_thread_group *rtg = wts->grp; + + if (is_suh_max() && rtg && rtg->id == DEFAULT_CGROUP_COLOC_ID && + rtg->skip_min && wts->unfilter) + return is_min_capacity_cpu(cpu); + + return false; +} + +extern bool is_rtgb_active(void); +extern u64 get_rtgb_active_time(void); + +static inline unsigned int walt_nr_rtg_high_prio(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->walt_stats.nr_rtg_high_prio_tasks; +} + +static inline bool task_fits_capacity(struct task_struct *p, + long capacity, + int cpu) +{ + unsigned int margin; + + /* + * Derive upmigration/downmigrate margin wrt the src/dest CPU. + */ + if (capacity_orig_of(task_cpu(p)) > capacity_orig_of(cpu)) + margin = sched_capacity_margin_down[cpu]; + else + margin = sched_capacity_margin_up[task_cpu(p)]; + + return capacity * 1024 > uclamp_task_util(p) * margin; +} + +static inline bool task_fits_max(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_orig_of(cpu); + unsigned long max_capacity = max_possible_capacity; + unsigned long task_boost = per_task_boost(p); + + if (capacity == max_capacity) + return true; + + if (is_min_capacity_cpu(cpu)) { + if (task_boost_policy(p) == SCHED_BOOST_ON_BIG || + task_boost > 0 || + uclamp_boosted(p) || + walt_should_kick_upmigrate(p, cpu)) + return false; + } else { /* mid cap cpu */ + if (task_boost > TASK_BOOST_ON_MID) + return false; + } + + return task_fits_capacity(p, capacity, cpu); +} + +/* applying the task threshold for all types of low latency tasks. */ +static inline bool walt_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->low_latency && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) +{ + struct cpuidle_state *idle = idle_get_state(rq); + + if (idle) + return idle->exit_latency; + + return UINT_MAX; +} + +extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); +extern void sched_update_hyst_times(void); + +extern enum sched_boost_policy sched_boost_policy(void); +extern void walt_rt_init(void); +extern void walt_cfs_init(void); +extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, + int sync, int sibling_count_hint); + +static inline unsigned int cpu_max_possible_freq(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster->max_possible_freq; +} + +static inline unsigned int cpu_max_freq(int cpu) +{ + return mult_frac(cpu_max_possible_freq(cpu), capacity_orig_of(cpu), + arch_scale_cpu_capacity(cpu)); +} + +static inline unsigned int task_load(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->demand; +} + +static inline unsigned int task_pl(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->pred_demand; +} + +static inline bool task_in_related_thread_group(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (rcu_access_pointer(wts->grp) != NULL); +} + +static inline bool task_rtg_high_prio(struct task_struct *p) +{ + return task_in_related_thread_group(p) && + (p->prio <= sysctl_walt_rtg_cfs_boost_prio); +} + +static inline struct walt_related_thread_group +*task_related_thread_group(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return rcu_dereference(wts->grp); +} + +#define CPU_RESERVED 1 +static inline int is_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + return test_bit(CPU_RESERVED, &wrq->walt_flags); +} + +static inline int mark_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + return test_and_set_bit(CPU_RESERVED, &wrq->walt_flags); +} + +static inline void clear_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + clear_bit(CPU_RESERVED, &wrq->walt_flags); +} + +static inline bool +task_in_cum_window_demand(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return cpu_of(rq) == task_cpu(p) && (p->on_rq || + wts->last_sleep_ts >= wrq->window_start); +} + +static inline void walt_fixup_cum_window_demand(struct rq *rq, s64 scaled_delta) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + wrq->cum_window_demand_scaled += scaled_delta; + if (unlikely((s64)wrq->cum_window_demand_scaled < 0)) + wrq->cum_window_demand_scaled = 0; +} + +static inline void walt_irq_work_queue(struct irq_work *work) +{ + if (likely(cpu_online(raw_smp_processor_id()))) + irq_work_queue(work); + else + irq_work_queue_on(work, cpumask_any(cpu_online_mask)); +} + +#define PF_WAKE_UP_IDLE 1 +static inline u32 sched_get_wake_up_idle(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->wake_up_idle; +} + +static inline int sched_set_wake_up_idle(struct task_struct *p, + int wake_up_idle) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->wake_up_idle = !!wake_up_idle; + return 0; +} + +static inline void set_wake_up_idle(bool enabled) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + + wts->wake_up_idle = enabled; +} + +extern int set_task_boost(int boost, u64 period); + +static inline struct task_group *css_tg(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct task_group, css) : NULL; +} + +#endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c new file mode 100644 index 000000000000..c99337e54ea2 --- /dev/null +++ b/kernel/sched/walt/walt_cfs.c @@ -0,0 +1,785 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include + +#include "walt.h" +#include "trace.h" +#include "../../../drivers/android/binder_trace.h" + +/* Migration margins */ +unsigned int sched_capacity_margin_up[WALT_NR_CPUS] = { + [0 ... WALT_NR_CPUS-1] = 1078 /* ~5% margin */ +}; +unsigned int sched_capacity_margin_down[WALT_NR_CPUS] = { + [0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */ +}; + +__read_mostly unsigned int sysctl_sched_prefer_spread; +unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ +unsigned int sched_small_task_threshold = 102; +__read_mostly unsigned int sysctl_sched_force_lb_enable = 1; +unsigned int capacity_margin_freq = 1280; /* ~20% margin */ + +static inline bool prefer_spread_on_idle(int cpu, bool new_ilb) +{ + switch (sysctl_sched_prefer_spread) { + case 1: + return is_min_capacity_cpu(cpu); + case 2: + return true; + case 3: + return (new_ilb && is_min_capacity_cpu(cpu)); + case 4: + return new_ilb; + default: + return false; + } +} + +static inline bool +bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) +{ + bool base_test = cpumask_test_cpu(cpu, &p->cpus_mask) && + cpu_active(cpu); + bool start_cap_test = (capacity_orig_of(cpu) >= + capacity_orig_of(start_cpu)); + + return base_test && start_cap_test; +} + +static inline bool task_demand_fits(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_orig_of(cpu); + unsigned long max_capacity = max_possible_capacity; + + if (capacity == max_capacity) + return true; + + return task_fits_capacity(p, capacity, cpu); +} + +struct find_best_target_env { + bool is_rtg; + int need_idle; + bool boosted; + int fastpath; + int start_cpu; + int order_index; + int end_index; + bool strict_max; + int skip_cpu; +}; + +/* + * cpu_util_without: compute cpu utilization without any contributions from *p + * @cpu: the CPU which utilization is requested + * @p: the task which utilization should be discounted + * + * The utilization of a CPU is defined by the utilization of tasks currently + * enqueued on that CPU as well as tasks which are currently sleeping after an + * execution on that CPU. + * + * This method returns the utilization of the specified CPU by discounting the + * utilization of the specified task, whenever the task is currently + * contributing to the CPU utilization. + */ +static unsigned long cpu_util_without(int cpu, struct task_struct *p) +{ + unsigned int util; + + /* + * WALT does not decay idle tasks in the same manner + * as PELT, so it makes little sense to subtract task + * utilization from cpu utilization. Instead just use + * cpu_util for this case. + */ + if (likely(p->state == TASK_WAKING)) + return cpu_util(cpu); + + /* Task has no contribution or is new */ + if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) + return cpu_util(cpu); + + util = max_t(long, cpu_util(cpu) - task_util(p), 0); + + /* + * Utilization (estimated) can exceed the CPU capacity, thus let's + * clamp to the maximum CPU capacity to ensure consistency with + * the cpu_util call. + */ + return min_t(unsigned long, util, capacity_orig_of(cpu)); +} + +static inline bool walt_get_rtg_status(struct task_struct *p) +{ + struct walt_related_thread_group *grp; + bool ret = false; + + rcu_read_lock(); + + grp = task_related_thread_group(p); + if (grp) + ret = grp->skip_min; + + rcu_read_unlock(); + + return ret; +} + +static inline bool walt_task_skip_min_cpu(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return sched_boost() != CONSERVATIVE_BOOST && + walt_get_rtg_status(p) && wts->unfilter; +} + +static inline bool walt_is_many_wakeup(int sibling_count_hint) +{ + return sibling_count_hint >= sysctl_sched_many_wakeup_threshold; +} + +static inline bool walt_target_ok(int target_cpu, int order_index) +{ + return !((order_index != num_sched_clusters - 1) && + (cpumask_weight(&cpu_array[order_index][0]) == 1) && + (target_cpu == cpumask_first(&cpu_array[order_index][0]))); +} + +static void walt_get_indicies(struct task_struct *p, int *order_index, + int *end_index, int task_boost, bool boosted) +{ + int i = 0; + + *order_index = 0; + *end_index = 0; + + if (num_sched_clusters <= 1) + return; + + if (task_boost > TASK_BOOST_ON_MID) { + *order_index = num_sched_clusters - 1; + return; + } + + if (is_full_throttle_boost()) { + *order_index = num_sched_clusters - 1; + if ((*order_index > 1) && task_demand_fits(p, + cpumask_first(&cpu_array[*order_index][1]))) + *end_index = 1; + return; + } + + if (boosted || task_boost_policy(p) == SCHED_BOOST_ON_BIG || + walt_task_skip_min_cpu(p)) + *order_index = 1; + + for (i = *order_index ; i < num_sched_clusters - 1; i++) { + if (task_demand_fits(p, cpumask_first(&cpu_array[i][0]))) + break; + } + + *order_index = i; +} + +enum fastpaths { + NONE = 0, + SYNC_WAKEUP, + PREV_CPU_FASTPATH, +}; + +static void walt_find_best_target(struct sched_domain *sd, + cpumask_t *candidates, + struct task_struct *p, + struct find_best_target_env *fbt_env) +{ + unsigned long min_util = uclamp_task_util(p); + long target_max_spare_cap = 0; + unsigned long best_idle_cuml_util = ULONG_MAX; + unsigned int min_exit_latency = UINT_MAX; + int best_idle_cpu = -1; + int target_cpu = -1; + int i, start_cpu; + long spare_wake_cap, most_spare_wake_cap = 0; + int most_spare_cap_cpu = -1; + int prev_cpu = task_cpu(p); + int active_candidate = -1; + int order_index = fbt_env->order_index, end_index = fbt_env->end_index; + int cluster; + unsigned int target_nr_rtg_high_prio = UINT_MAX; + bool rtg_high_prio_task = task_rtg_high_prio(p); + cpumask_t visit_cpus; + bool io_task_pack = (order_index > 0 && p->in_iowait); + struct cfs_rq *cfs_rq; + + /* Find start CPU based on boost value */ + start_cpu = fbt_env->start_cpu; + + if (fbt_env->strict_max || io_task_pack) + target_max_spare_cap = LONG_MIN; + + if (p->state == TASK_RUNNING) + most_spare_wake_cap = ULONG_MAX; + + /* fast path for prev_cpu */ + if (((capacity_orig_of(prev_cpu) == capacity_orig_of(start_cpu)) || + asym_cap_siblings(prev_cpu, start_cpu)) && + cpu_active(prev_cpu) && cpu_online(prev_cpu) && + available_idle_cpu(prev_cpu)) { + target_cpu = prev_cpu; + fbt_env->fastpath = PREV_CPU_FASTPATH; + cpumask_set_cpu(target_cpu, candidates); + goto out; + } + + for (cluster = 0; cluster < num_sched_clusters; cluster++) { + cpumask_and(&visit_cpus, &p->cpus_mask, + &cpu_array[order_index][cluster]); + for_each_cpu(i, &visit_cpus) { + unsigned long capacity_orig = capacity_orig_of(i); + unsigned long wake_util, new_util, new_util_cuml; + long spare_cap; + unsigned int idle_exit_latency = UINT_MAX; + + trace_sched_cpu_util(i); + + if (!cpu_active(i)) + continue; + + if (active_candidate == -1) + active_candidate = i; + + /* + * This CPU is the target of an active migration that's + * yet to complete. Avoid placing another task on it. + */ + if (is_reserved(i)) + continue; + + if (sched_cpu_high_irqload(i)) + continue; + + if (fbt_env->skip_cpu == i) + continue; + + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due to the double + * accounting. However, the blocked utilization may be zero. + */ + wake_util = cpu_util_without(i, p); + new_util = wake_util + uclamp_task_util(p); + spare_wake_cap = capacity_orig - wake_util; + + if (spare_wake_cap > most_spare_wake_cap) { + most_spare_wake_cap = spare_wake_cap; + most_spare_cap_cpu = i; + } + + if ((per_task_boost(cpu_rq(i)->curr) == + TASK_BOOST_STRICT_MAX) && + !fbt_env->strict_max) + continue; + + /* get rq's utilization with this task included */ + cfs_rq = &cpu_rq(i)->cfs; + new_util_cuml = READ_ONCE(cfs_rq->avg.util_avg) + min_util; + + /* + * Ensure minimum capacity to grant the required boost. + * The target CPU can be already at a capacity level higher + * than the one required to boost the task. + */ + new_util = max(min_util, new_util); + if (!(fbt_env->strict_max || io_task_pack) && + new_util > capacity_orig) + continue; + + /* + * Pre-compute the maximum possible capacity we expect + * to have available on this CPU once the task is + * enqueued here. + */ + spare_cap = capacity_orig - new_util; + + /* + * Find an optimal backup IDLE CPU for non latency + * sensitive tasks. + * + * Looking for: + * - favoring shallowest idle states + * i.e. avoid to wakeup deep-idle CPUs + * + * The following code path is used by non latency + * sensitive tasks if IDLE CPUs are available. If at + * least one of such CPUs are available it sets the + * best_idle_cpu to the most suitable idle CPU to be + * selected. + * + * If idle CPUs are available, favour these CPUs to + * improve performances by spreading tasks. + * Indeed, the energy_diff() computed by the caller + * will take care to ensure the minimization of energy + * consumptions without affecting performance. + */ + if (available_idle_cpu(i)) { + idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(i)); + + /* + * Prefer shallowest over deeper idle state cpu, + * of same capacity cpus. + */ + if (idle_exit_latency > min_exit_latency) + continue; + if (min_exit_latency == idle_exit_latency && + (best_idle_cpu == prev_cpu || + (i != prev_cpu && + new_util_cuml > best_idle_cuml_util))) + continue; + + min_exit_latency = idle_exit_latency; + best_idle_cuml_util = new_util_cuml; + best_idle_cpu = i; + continue; + } + + /* + * Consider only idle CPUs for active migration. + */ + if (p->state == TASK_RUNNING) + continue; + + /* + * Try to spread the rtg high prio tasks so that they + * don't preempt each other. This is a optimisitc + * check assuming rtg high prio can actually preempt + * the current running task with the given vruntime + * boost. + */ + if (rtg_high_prio_task) { + if (walt_nr_rtg_high_prio(i) > target_nr_rtg_high_prio) + continue; + + /* Favor CPUs with maximum spare capacity */ + if (walt_nr_rtg_high_prio(i) == target_nr_rtg_high_prio && + spare_cap < target_max_spare_cap) + continue; + } else { + /* Favor CPUs with maximum spare capacity */ + if (spare_cap < target_max_spare_cap) + continue; + } + + target_max_spare_cap = spare_cap; + target_nr_rtg_high_prio = walt_nr_rtg_high_prio(i); + target_cpu = i; + } + + if (best_idle_cpu != -1) + break; + + if ((cluster >= end_index) && (target_cpu != -1) && + walt_target_ok(target_cpu, order_index)) + break; + } + + if (best_idle_cpu != -1) + target_cpu = -1; + /* + * We set both idle and target as long as they are valid CPUs. + * If we don't find either, then we fallback to most_spare_cap, + * If we don't find most spare cap, we fallback to prev_cpu, + * provided that the prev_cpu is active. + * If the prev_cpu is not active, we fallback to active_candidate. + */ + + if (unlikely(target_cpu == -1)) { + if (best_idle_cpu != -1) + target_cpu = best_idle_cpu; + else if (most_spare_cap_cpu != -1) + target_cpu = most_spare_cap_cpu; + else if (!cpu_active(prev_cpu)) + target_cpu = active_candidate; + } + + if (target_cpu != -1) + cpumask_set_cpu(target_cpu, candidates); + if (best_idle_cpu != -1 && target_cpu != best_idle_cpu) + cpumask_set_cpu(best_idle_cpu, candidates); +out: + trace_sched_find_best_target(p, min_util, start_cpu, + best_idle_cpu, most_spare_cap_cpu, + target_cpu, order_index, end_index, + fbt_env->skip_cpu, p->state == TASK_RUNNING); +} + +static inline unsigned long +cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + unsigned long util = wrq->walt_stats.cumulative_runnable_avg_scaled; + bool queued = task_on_rq_queued(p); + + /* + * When task is queued, + * (a) The evaluating CPU (cpu) is task's current CPU. If the + * task is migrating, discount the task contribution from the + * evaluation cpu. + * (b) The evaluating CPU (cpu) is task's current CPU. If the + * task is NOT migrating, nothing to do. The contribution is + * already present on the evaluation CPU. + * (c) The evaluating CPU (cpu) is not task's current CPU. But + * the task is migrating to the evaluating CPU. So add the + * task contribution to it. + * (d) The evaluating CPU (cpu) is neither the current CPU nor + * the destination CPU. don't care. + * + * When task is NOT queued i.e waking. Task contribution is not + * present on any CPU. + * + * (a) If the evaluating CPU is the destination CPU, add the task + * contribution. + * (b) The evaluation CPU is not the destination CPU, don't care. + */ + if (unlikely(queued)) { + if (task_cpu(p) == cpu) { + if (dst_cpu != cpu) + util = max_t(long, util - task_util(p), 0); + } else if (dst_cpu == cpu) { + util += task_util(p); + } + } else if (dst_cpu == cpu) { + util += task_util(p); + } + + return min_t(unsigned long, util, capacity_orig_of(cpu)); +} + +/* + * compute_energy(): Estimates the energy that @pd would consume if @p was + * migrated to @dst_cpu. compute_energy() predicts what will be the utilization + * landscape of @pd's CPUs after the task migration, and uses the Energy Model + * to compute what would be the energy if we decided to actually migrate that + * task. + */ +static long +compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) +{ + struct cpumask *pd_mask = perf_domain_span(pd); + unsigned long max_util = 0, sum_util = 0; + int cpu; + unsigned long cpu_util; + + /* + * The capacity state of CPUs of the current rd can be driven by CPUs + * of another rd if they belong to the same pd. So, account for the + * utilization of these CPUs too by masking pd with cpu_online_mask + * instead of the rd span. + * + * If an entire pd is outside of the current rd, it will not appear in + * its pd list and will not be accounted by compute_energy(). + */ + for_each_cpu_and(cpu, pd_mask, cpu_online_mask) { + cpu_util = cpu_util_next_walt(cpu, p, dst_cpu); + sum_util += cpu_util; + max_util = max(max_util, cpu_util); + } + + return em_cpu_energy(pd->em_pd, max_util, sum_util); +} + +static inline long +walt_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) +{ + long energy = 0; + + for (; pd; pd = pd->next) + energy += compute_energy(p, dst_cpu, pd); + + return energy; +} + +static inline int wake_to_idle(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_task_struct *cur_wts = + (struct walt_task_struct *) current->android_vendor_data1; + + return (cur_wts->wake_up_idle || wts->wake_up_idle); +} + +/* return true if cpu should be chosen over best_energy_cpu */ +static inline bool select_cpu_same_energy(int cpu, int best_cpu, int prev_cpu) +{ + if (capacity_orig_of(cpu) < capacity_orig_of(best_cpu)) + return true; + + if (best_cpu == prev_cpu) + return false; + + if (available_idle_cpu(best_cpu) && walt_get_idle_exit_latency(cpu_rq(best_cpu)) <= 1) + return false; /* best_cpu is idle wfi or shallower */ + + if (available_idle_cpu(cpu) && walt_get_idle_exit_latency(cpu_rq(cpu)) <= 1) + return true; /* new cpu is idle wfi or shallower */ + + /* + * If we are this far this must be a tie between a busy and deep idle, + * pick the busy. + */ + return available_idle_cpu(best_cpu); +} + +static DEFINE_PER_CPU(cpumask_t, energy_cpus); +int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, + int sync, int sibling_count_hint) +{ + unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; + struct root_domain *rd = cpu_rq(smp_processor_id())->rd; + int weight, cpu = smp_processor_id(), best_energy_cpu = prev_cpu; + struct perf_domain *pd; + unsigned long cur_energy; + cpumask_t *candidates; + bool is_rtg, curr_is_rtg; + struct find_best_target_env fbt_env; + bool need_idle = wake_to_idle(p) || uclamp_latency_sensitive(p); + u64 start_t = 0; + int delta = 0; + int task_boost = per_task_boost(p); + bool is_uclamp_boosted = uclamp_boosted(p); + bool boosted = is_uclamp_boosted || (task_boost > 0); + int start_cpu, order_index, end_index; + + if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && + cpumask_test_cpu(prev_cpu, &p->cpus_mask)) + return prev_cpu; + + if (unlikely(!cpu_array)) + return -EPERM; + + walt_get_indicies(p, &order_index, &end_index, task_boost, boosted); + start_cpu = cpumask_first(&cpu_array[order_index][0]); + + is_rtg = task_in_related_thread_group(p); + curr_is_rtg = task_in_related_thread_group(cpu_rq(cpu)->curr); + + fbt_env.fastpath = 0; + fbt_env.need_idle = need_idle; + + if (trace_sched_task_util_enabled()) + start_t = sched_clock(); + + /* Pre-select a set of candidate CPUs. */ + candidates = this_cpu_ptr(&energy_cpus); + cpumask_clear(candidates); + + if (sync && (need_idle || (is_rtg && curr_is_rtg))) + sync = 0; + + if (sync && bias_to_this_cpu(p, cpu, start_cpu)) { + best_energy_cpu = cpu; + fbt_env.fastpath = SYNC_WAKEUP; + goto done; + } + + rcu_read_lock(); + pd = rcu_dereference(rd->pd); + if (!pd) + goto fail; + + fbt_env.is_rtg = is_rtg; + fbt_env.start_cpu = start_cpu; + fbt_env.order_index = order_index; + fbt_env.end_index = end_index; + fbt_env.boosted = boosted; + fbt_env.strict_max = is_rtg && + (task_boost == TASK_BOOST_STRICT_MAX); + fbt_env.skip_cpu = walt_is_many_wakeup(sibling_count_hint) ? + cpu : -1; + + walt_find_best_target(NULL, candidates, p, &fbt_env); + + /* Bail out if no candidate was found. */ + weight = cpumask_weight(candidates); + if (!weight) + goto unlock; + + /* If there is only one sensible candidate, select it now. */ + cpu = cpumask_first(candidates); + if (weight == 1 && (available_idle_cpu(cpu) || cpu == prev_cpu)) { + best_energy_cpu = cpu; + goto unlock; + } + + if (p->state == TASK_WAKING) + delta = task_util(p); + + if (task_placement_boost_enabled(p) || fbt_env.need_idle || + boosted || is_rtg || __cpu_overutilized(prev_cpu, delta) || + !task_fits_max(p, prev_cpu) || !cpu_active(prev_cpu)) { + best_energy_cpu = cpu; + goto unlock; + } + + if (cpumask_test_cpu(prev_cpu, &p->cpus_mask)) + prev_delta = best_delta = + walt_compute_energy(p, prev_cpu, pd); + else + prev_delta = best_delta = ULONG_MAX; + + /* Select the best candidate energy-wise. */ + for_each_cpu(cpu, candidates) { + if (cpu == prev_cpu) + continue; + + cur_energy = walt_compute_energy(p, cpu, pd); + trace_sched_compute_energy(p, cpu, cur_energy, + prev_delta, best_delta, best_energy_cpu); + + if (cur_energy < best_delta) { + best_delta = cur_energy; + best_energy_cpu = cpu; + } else if (cur_energy == best_delta) { + if (select_cpu_same_energy(cpu, best_energy_cpu, + prev_cpu)) { + best_delta = cur_energy; + best_energy_cpu = cpu; + } + } + } + +unlock: + rcu_read_unlock(); + + /* + * Pick the prev CPU, if best energy CPU can't saves at least 6% of + * the energy used by prev_cpu. + */ + if (!(available_idle_cpu(best_energy_cpu) && + walt_get_idle_exit_latency(cpu_rq(best_energy_cpu)) <= 1) && + (prev_delta != ULONG_MAX) && (best_energy_cpu != prev_cpu) && + ((prev_delta - best_delta) <= prev_delta >> 4) && + (capacity_orig_of(prev_cpu) <= capacity_orig_of(start_cpu))) + best_energy_cpu = prev_cpu; + +done: + trace_sched_task_util(p, cpumask_bits(candidates)[0], best_energy_cpu, + sync, fbt_env.need_idle, fbt_env.fastpath, + task_boost_policy(p), start_t, boosted, is_rtg, + walt_get_rtg_status(p), start_cpu); + + return best_energy_cpu; + +fail: + rcu_read_unlock(); + return -EPERM; +} + +static void +walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, + int sd_flag, int wake_flags, int *target_cpu) +{ + int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); + int sibling_count_hint = p->wake_q_head ? p->wake_q_head->count : 1; + + *target_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, sync, sibling_count_hint); + if (unlikely(*target_cpu < 0)) + *target_cpu = prev_cpu; +} + +#ifdef CONFIG_FAIR_GROUP_SCHED +static unsigned long task_h_load(struct task_struct *p) +{ + struct cfs_rq *cfs_rq = task_cfs_rq(p); + + update_cfs_rq_h_load(cfs_rq); + return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, + cfs_rq_load_avg(cfs_rq) + 1); +} +#else +static unsigned long task_h_load(struct task_struct *p) +{ + return p->se.avg.load_avg; +} +#endif + +static void walt_update_misfit_status(void *unused, struct task_struct *p, + struct rq *rq, bool *need_update) +{ + *need_update = false; + + if (!p) { + rq->misfit_task_load = 0; + return; + } + + if (task_fits_max(p, cpu_of(rq))) { + rq->misfit_task_load = 0; + return; + } + + /* + * Make sure that misfit_task_load will not be null even if + * task_h_load() returns 0. + */ + rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); +} + +static inline struct task_struct *task_of(struct sched_entity *se) +{ + return container_of(se, struct task_struct, se); +} + +static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vruntime) +{ + if (entity_is_task(se)) { + unsigned long thresh = sysctl_sched_latency; + + /* + * Halve their sleep time's effect, to allow + * for a gentler effect of sleepers: + */ + if (sched_feat(GENTLE_FAIR_SLEEPERS)) + thresh >>= 1; + + if ((per_task_boost(task_of(se)) == TASK_BOOST_STRICT_MAX) || + walt_low_latency_task(task_of(se)) || + task_rtg_high_prio(task_of(se))) { + *vruntime -= sysctl_sched_latency; + *vruntime -= thresh; + se->vruntime = *vruntime; + } + } +} + +static void walt_binder_low_latency_set(void *unused, struct task_struct *task) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; + + if (task && current->signal && + (current->signal->oom_score_adj == 0) && + (current->prio < DEFAULT_PRIO)) + wts->low_latency = true; +} + +static void walt_binder_low_latency_clear(void *unused, struct binder_transaction *t) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + + if (wts->low_latency) + wts->low_latency = false; +} + +void walt_cfs_init(void) +{ + register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL); + register_trace_android_rvh_update_misfit_status(walt_update_misfit_status, NULL); + register_trace_android_rvh_place_entity(walt_place_entity, NULL); + + register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL); + register_trace_binder_transaction_received(walt_binder_low_latency_clear, NULL); +} diff --git a/kernel/sched/walt/walt_debug.c b/kernel/sched/walt/walt_debug.c new file mode 100644 index 000000000000..f8679c032f39 --- /dev/null +++ b/kernel/sched/walt/walt_debug.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include + +#include + +#include "walt.h" +#include "walt_debug.h" + +static void dump_throttled_rt_tasks(void *unused, int cpu, u64 clock, + ktime_t rt_period, u64 rt_runtime, s64 rt_period_timer_expires) +{ + printk_deferred("sched: RT throttling activated for cpu %d\n", cpu); + printk_deferred("rt_period_timer: expires=%lld now=%llu runtime=%llu period=%llu\n", + rt_period_timer_expires, ktime_get_ns(), rt_runtime, rt_period); + printk_deferred("potential CPU hogs:\n"); +#ifdef CONFIG_SCHED_INFO + if (sched_info_on()) + printk_deferred("current %s (%d) is running for %llu nsec\n", + current->comm, current->pid, + clock - current->sched_info.last_arrival); +#endif + BUG(); +} + +static void android_rvh_schedule_bug(void *unused, void *unused2) +{ + BUG(); +} + +static int __init walt_debug_init(void) +{ + int ret; + + ret = preemptirq_long_init(); + if (!ret) + return ret; + + register_trace_android_vh_dump_throttled_rt_tasks(dump_throttled_rt_tasks, NULL); + register_trace_android_rvh_schedule_bug(android_rvh_schedule_bug, NULL); + + return 0; +} +module_init(walt_debug_init); + +MODULE_DESCRIPTION("QTI WALT Debug Module"); +MODULE_LICENSE("GPL v2"); diff --git a/kernel/sched/walt/walt_debug.h b/kernel/sched/walt/walt_debug.h new file mode 100644 index 000000000000..282836c14743 --- /dev/null +++ b/kernel/sched/walt/walt_debug.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ +int preemptirq_long_init(void); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c new file mode 100644 index 000000000000..45c68da4f2c2 --- /dev/null +++ b/kernel/sched/walt/walt_lb.c @@ -0,0 +1,742 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include + +#include "walt.h" +#include "trace.h" + +extern u64 sched_ktime_clock(void); // TODO +static void walt_detach_task(struct task_struct *p, struct rq *src_rq, + struct rq *dst_rq) +{ + deactivate_task(src_rq, p, 0); + double_lock_balance(src_rq, dst_rq); + if (!(src_rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(src_rq); + set_task_cpu(p, dst_rq->cpu); + double_unlock_balance(src_rq, dst_rq); +} + +static void walt_attach_task(struct task_struct *p, struct rq *rq) +{ + activate_task(rq, p, 0); + check_preempt_curr(rq, p, 0); +} + +static int walt_lb_active_migration(void *data) +{ + struct rq *busiest_rq = data; + int busiest_cpu = cpu_of(busiest_rq); + int target_cpu = busiest_rq->push_cpu; + struct rq *target_rq = cpu_rq(target_cpu); + struct walt_rq *wrq = (struct walt_rq *) busiest_rq->android_vendor_data1; + struct task_struct *push_task = wrq->push_task; + int push_task_detached = 0; + + raw_spin_lock_irq(&busiest_rq->lock); + + /* sanity checks before initiating the pull */ + if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) + goto out_unlock; + + if (unlikely(busiest_cpu != raw_smp_processor_id() || + !busiest_rq->active_balance)) + goto out_unlock; + + if (busiest_rq->nr_running <= 1) + goto out_unlock; + + BUG_ON(busiest_rq == target_rq); + + if (task_on_rq_queued(push_task) && + push_task->state == TASK_RUNNING && + task_cpu(push_task) == busiest_cpu && + cpu_active(target_cpu)) { + walt_detach_task(push_task, busiest_rq, target_rq); + push_task_detached = 1; + } + +out_unlock: /* called with busiest_rq lock */ + busiest_rq->active_balance = 0; + target_cpu = busiest_rq->push_cpu; + clear_reserved(target_cpu); + wrq->push_task = NULL; + raw_spin_unlock(&busiest_rq->lock); + + if (push_task_detached) { + if (push_task_detached) { + raw_spin_lock(&target_rq->lock); + walt_attach_task(push_task, target_rq); + raw_spin_unlock(&target_rq->lock); + } + } + put_task_struct(push_task); + + local_irq_enable(); + return 0; +} + +struct walt_lb_rotate_work { + struct work_struct w; + struct task_struct *src_task; + struct task_struct *dst_task; + int src_cpu; + int dst_cpu; +}; + +DEFINE_PER_CPU(struct walt_lb_rotate_work, walt_lb_rotate_works); + +static void walt_lb_rotate_work_func(struct work_struct *work) +{ + struct walt_lb_rotate_work *wr = container_of(work, + struct walt_lb_rotate_work, w); + + migrate_swap(wr->src_task, wr->dst_task, wr->dst_cpu, wr->src_cpu); + + put_task_struct(wr->src_task); + put_task_struct(wr->dst_task); + + clear_reserved(wr->src_cpu); + clear_reserved(wr->dst_cpu); +} + +static void walt_lb_rotate_work_init(void) +{ + int i; + + for_each_possible_cpu(i) { + struct walt_lb_rotate_work *wr = &per_cpu(walt_lb_rotate_works, i); + + INIT_WORK(&wr->w, walt_lb_rotate_work_func); + } +} + +#define WALT_ROTATION_THRESHOLD_NS 16000000 +static void walt_lb_check_for_rotation(struct rq *src_rq) +{ + u64 wc, wait, max_wait = 0, run, max_run = 0; + int deserved_cpu = nr_cpu_ids, dst_cpu = nr_cpu_ids; + int i, src_cpu = cpu_of(src_rq); + struct rq *dst_rq; + struct walt_lb_rotate_work *wr = NULL; + struct walt_task_struct *wts; + + if (!is_min_capacity_cpu(src_cpu)) + return; + + wc = sched_ktime_clock(); + + for_each_possible_cpu(i) { + struct rq *rq = cpu_rq(i); + + if (!is_min_capacity_cpu(i)) + break; + + if (is_reserved(i)) + continue; + + if (!rq->misfit_task_load) + continue; + + wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; + wait = wc - wts->last_enqueued_ts; + if (wait > max_wait) { + max_wait = wait; + deserved_cpu = i; + } + } + + if (deserved_cpu != src_cpu) + return; + + for_each_possible_cpu(i) { + struct rq *rq = cpu_rq(i); + + if (is_min_capacity_cpu(i)) + continue; + + if (is_reserved(i)) + continue; + + if (rq->curr->prio < MAX_RT_PRIO) + continue; + + if (rq->nr_running > 1) + continue; + + wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; + run = wc - wts->last_enqueued_ts; + + if (run < WALT_ROTATION_THRESHOLD_NS) + continue; + + if (run > max_run) { + max_run = run; + dst_cpu = i; + } + } + + if (dst_cpu == nr_cpu_ids) + return; + + dst_rq = cpu_rq(dst_cpu); + + double_rq_lock(src_rq, dst_rq); + if (dst_rq->curr->prio >= MAX_RT_PRIO && dst_rq->curr != dst_rq->idle && + src_rq->curr->prio >= MAX_RT_PRIO && src_rq->curr != src_rq->idle) { + get_task_struct(src_rq->curr); + get_task_struct(dst_rq->curr); + + mark_reserved(src_cpu); + mark_reserved(dst_cpu); + wr = &per_cpu(walt_lb_rotate_works, src_cpu); + + wr->src_task = src_rq->curr; + wr->dst_task = dst_rq->curr; + + wr->src_cpu = src_cpu; + wr->dst_cpu = dst_cpu; + } + double_rq_unlock(src_rq, dst_rq); + + if (wr) + queue_work_on(src_cpu, system_highpri_wq, &wr->w); +} + +static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, + bool to_lower) +{ + struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; + + if (to_lower) { + if (p->in_iowait) + return false; + if (per_task_boost(p) == TASK_BOOST_STRICT_MAX && + task_in_related_thread_group(p)) + return false; + } + + /* Don't detach task if it is under active migration */ + if (wrq->push_task == p) + return false; + + return true; +} + +static inline bool need_active_lb(struct task_struct *p, int dst_cpu, + int src_cpu) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (cpu_rq(src_cpu)->active_balance) + return false; + + if (capacity_orig_of(dst_cpu) <= capacity_orig_of(src_cpu)) + return false; + + if (!wts->misfit) + return false; + + return true; +} + +static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) +{ + struct rq *dst_rq = cpu_rq(dst_cpu); + struct rq *src_rq = cpu_rq(src_cpu); + unsigned long flags; + struct task_struct *pulled_task = NULL, *p; + bool active_balance = false, to_lower; + struct walt_rq *wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_task_struct *wts; + + BUG_ON(src_cpu == dst_cpu); + + to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(src_cpu); + + raw_spin_lock_irqsave(&src_rq->lock, flags); + list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { + + if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) + continue; + + if (!_walt_can_migrate_task(p, dst_cpu, to_lower)) + continue; + + if (task_running(src_rq, p)) { + + if (need_active_lb(p, dst_cpu, src_cpu)) { + active_balance = true; + break; + } + continue; + } + + walt_detach_task(p, src_rq, dst_rq); + pulled_task = p; + break; + } + + if (active_balance) { + src_rq->active_balance = 1; + src_rq->push_cpu = dst_cpu; + get_task_struct(p); + wrq->push_task = p; + mark_reserved(dst_cpu); + } + /* lock must be dropped before waking the stopper */ + raw_spin_unlock_irqrestore(&src_rq->lock, flags); + + /* + * Using our custom active load balance callback so that + * the push_task is really pulled onto this CPU. + */ + if (active_balance) { + wts = (struct walt_task_struct *) p->android_vendor_data1; + trace_walt_active_load_balance(p, src_cpu, dst_cpu, wts); + stop_one_cpu_nowait(src_cpu, walt_lb_active_migration, + src_rq, &src_rq->active_balance_work); + return 0; /* we did not pull any task here */ + } + + if (!pulled_task) + return 0; + + raw_spin_lock_irqsave(&dst_rq->lock, flags); + walt_attach_task(p, dst_rq); + raw_spin_unlock_irqrestore(&dst_rq->lock, flags); + + return 1; /* we pulled 1 task */ +} + +static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *src_mask) +{ + int i; + int busiest_cpu = -1; + int busiest_nr = 1; /* we need atleast 2 */ + unsigned long util, busiest_util = 0; + struct walt_rq *wrq; + + for_each_cpu(i, src_mask) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + trace_walt_lb_cpu_util(i, wrq); + + if (cpu_rq(i)->cfs.h_nr_running < 2) + continue; + + util = cpu_util(i); + if (util < busiest_util) + continue; + + busiest_nr = cpu_rq(i)->cfs.h_nr_running; + busiest_util = util; + busiest_cpu = i; + } + + return busiest_cpu; +} + +#define SMALL_TASK_THRESHOLD 102 +static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src_mask) +{ + int i; + int busiest_cpu = -1; + int busiest_nr = 1; /* we need atleast 2 */ + unsigned long util, busiest_util = 0; + unsigned long total_capacity = 0, total_util = 0, total_nr = 0; + int total_cpus = 0; + struct walt_rq *wrq; + + for_each_cpu(i, src_mask) { + + if (!cpu_active(i)) + continue; + + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + trace_walt_lb_cpu_util(i, wrq); + + util = cpu_util(i); + total_cpus += 1; + total_util += util; + total_capacity += capacity_orig_of(i); + total_nr += cpu_rq(i)->cfs.h_nr_running; + + if (cpu_rq(i)->cfs.h_nr_running < 2) + continue; + + if (cpu_rq(i)->cfs.h_nr_running == 2 && + task_util(cpu_rq(i)->curr) < SMALL_TASK_THRESHOLD) + continue; + + /* + * During rotation, two silver fmax tasks gets + * placed on gold/prime and the CPU may not be + * overutilized but for rotation, we have to + * spread out. + */ + if (!walt_rotation_enabled && !cpu_overutilized(i)) + continue; + + if (util < busiest_util) + continue; + + busiest_nr = cpu_rq(i)->cfs.h_nr_running; + busiest_util = util; + busiest_cpu = i; + } + + /* + * Don't allow migrating to lower cluster unless this high + * capacity cluster is sufficiently loaded. + */ + if (!walt_rotation_enabled) { + if (total_nr <= total_cpus || total_util * 1280 < total_capacity * 1024) + busiest_cpu = -1; + } + + return busiest_cpu; +} + +static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_mask) +{ + int i; + int busiest_cpu = -1; + int busiest_nr = 1; /* we need atleast 2 */ + unsigned long util, busiest_util = 0; + unsigned long total_capacity = 0, total_util = 0, total_nr = 0; + int total_cpus = 0; + int busy_nr_big_tasks = 0; + struct walt_rq *wrq; + + /* + * A higher capacity CPU is looking at a lower capacity + * cluster. active balance and big tasks are in play. + * other than that, it is very much same as above. we + * really don't need this as a separate block. will + * refactor this after final testing is done. + */ + for_each_cpu(i, src_mask) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + + if (!cpu_active(i)) + continue; + + trace_walt_lb_cpu_util(i, wrq); + + util = cpu_util(i); + total_cpus += 1; + total_util += util; + total_capacity += capacity_orig_of(i); + total_nr += cpu_rq(i)->cfs.h_nr_running; + + /* + * no point in selecting this CPU as busy, as + * active balance is in progress. + */ + if (cpu_rq(i)->active_balance) + continue; + + if (cpu_rq(i)->cfs.h_nr_running < 2 && !wrq->walt_stats.nr_big_tasks) + continue; + + if (!walt_rotation_enabled && !cpu_overutilized(i)) + continue; + + if (util < busiest_util) + continue; + + busiest_nr = cpu_rq(i)->cfs.h_nr_running; + busiest_util = util; + busiest_cpu = i; + busy_nr_big_tasks = wrq->walt_stats.nr_big_tasks; + } + + if (!walt_rotation_enabled && !busy_nr_big_tasks) { + if (total_nr <= total_cpus || total_util * 1280 < total_capacity * 1024) + busiest_cpu = -1; + } + + return busiest_cpu; +} + +static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask) +{ + int fsrc_cpu = cpumask_first(src_mask); + int busiest_cpu; + + if (capacity_orig_of(dst_cpu) == capacity_orig_of(fsrc_cpu)) + busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, + src_mask); + else if (capacity_orig_of(dst_cpu) < capacity_orig_of(fsrc_cpu)) + busiest_cpu = walt_lb_find_busiest_lower_cap_cpu(dst_cpu, + src_mask); + else + busiest_cpu = walt_lb_find_busiest_higher_cap_cpu(dst_cpu, + src_mask); + + return busiest_cpu; +} + +static DEFINE_RAW_SPINLOCK(walt_lb_migration_lock); +static void walt_lb_tick(void *unused, struct rq *rq) +{ + int prev_cpu = rq->cpu, new_cpu, ret; + struct task_struct *p = rq->curr; + unsigned long flags; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!rq->misfit_task_load) + return; + + if (p->state != TASK_RUNNING || p->nr_cpus_allowed == 1) + return; + + raw_spin_lock_irqsave(&walt_lb_migration_lock, flags); + + if (walt_rotation_enabled) { + walt_lb_check_for_rotation(rq); + goto out_unlock; + } + + rcu_read_lock(); + new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1); + rcu_read_unlock(); + + if (new_cpu < 0 || same_cluster(new_cpu, prev_cpu)) + goto out_unlock; + + raw_spin_lock(&rq->lock); + if (rq->active_balance) { + raw_spin_unlock(&rq->lock); + goto out_unlock; + } + rq->active_balance = 1; + rq->push_cpu = new_cpu; + get_task_struct(p); + wrq->push_task = p; + raw_spin_unlock(&rq->lock); + + mark_reserved(new_cpu); + raw_spin_unlock_irqrestore(&walt_lb_migration_lock, flags); + + trace_walt_active_load_balance(p, prev_cpu, new_cpu, wts); + ret = stop_one_cpu_nowait(prev_cpu, + walt_lb_active_migration, rq, + &rq->active_balance_work); + if (!ret) + clear_reserved(new_cpu); + else + wake_up_if_idle(new_cpu); + + return; + +out_unlock: + raw_spin_unlock_irqrestore(&walt_lb_migration_lock, flags); +} + +static void walt_newidle_balance(void *unused, struct rq *this_rq, + struct rq_flags *rf, int *pulled_task, + int *done) +{ + int this_cpu = this_rq->cpu; + struct walt_rq *wrq = (struct walt_rq *) this_rq->android_vendor_data1; + int order_index = wrq->cluster->id; + int cluster = 0; + int busy_cpu; + + if (unlikely(!cpu_array)) + return; + + /* + * newly idle load balance is completely handled here, so + * set done to skip the load balance by the caller. + */ + *done = 1; + *pulled_task = 0; + + /* + * This CPU is about to enter idle, so clear the + * misfit_task_load and mark the idle stamp. + */ + this_rq->misfit_task_load = 0; + this_rq->idle_stamp = rq_clock(this_rq); + + if (!cpu_active(this_cpu)) + return; + + if (!READ_ONCE(this_rq->rd->overload)) + return; + + rq_unpin_lock(this_rq, rf); + raw_spin_unlock(&this_rq->lock); + + /* + * careful, we dropped the lock, and has to be acquired + * before returning. Since rq lock is dropped, tasks + * can be queued remotely, so keep a check on nr_running + * and bail out. + */ + do { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, + &cpu_array[order_index][cluster]); + + /* we got the busy/src cpu here. */ + if (busy_cpu != -1 || this_rq->nr_running > 0) + break; + + } while (++cluster < num_sched_clusters); + + /* sanity checks before attempting the pull */ + if (busy_cpu == -1 || this_rq->nr_running > 0 || (busy_cpu == this_cpu)) + goto out; + + *pulled_task = walt_lb_pull_tasks(this_cpu, busy_cpu); + +out: + raw_spin_lock(&this_rq->lock); + if (this_rq->cfs.h_nr_running && !*pulled_task) + *pulled_task = 1; + + /* Is there a task of a high priority class? */ + if (this_rq->nr_running != this_rq->cfs.h_nr_running) + *pulled_task = -1; + + /* reset the idle time stamp if we pulled any task */ + if (*pulled_task) + this_rq->idle_stamp = 0; + + rq_repin_lock(this_rq, rf); + + trace_walt_newidle_balance(this_cpu, busy_cpu, *pulled_task); +} + +static void walt_find_busiest_queue(void *unused, int dst_cpu, + struct sched_group *group, + struct cpumask *env_cpus, + struct rq **busiest, int *done) +{ + int fsrc_cpu = group_first_cpu(group); + int busiest_cpu = -1; + struct cpumask src_mask; + + *done = 1; + *busiest = NULL; + + /* + * same cluster means, there will only be 1 + * CPU in the busy group, so just select it. + */ + if (same_cluster(dst_cpu, fsrc_cpu)) { + busiest_cpu = fsrc_cpu; + goto done; + } + + /* + * We will allow inter cluster migrations + * only if the source group is sufficiently + * loaded. The upstream load balancer is a + * bit more generous. + * + * re-using the same code that we use it + * for newly idle load balance. The policies + * remain same. + */ + cpumask_and(&src_mask, sched_group_span(group), env_cpus); + busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask); +done: + if (busiest_cpu != -1) + *busiest = cpu_rq(busiest_cpu); + + trace_walt_find_busiest_queue(dst_cpu, busiest_cpu, src_mask.bits[0]); +} + +static void walt_migrate_queued_task(void *unused, struct rq *rq, + struct rq_flags *rf, + struct task_struct *p, + int new_cpu, int *detached) +{ + /* + * WALT expects both source and destination rqs to be + * held when set_task_cpu() is called on a queued task. + * so implementing this detach hook. unpin the lock + * before detaching and repin it later to make lockdep + * happy. + */ + BUG_ON(!rf); + + rq_unpin_lock(rq, rf); + walt_detach_task(p, rq, cpu_rq(new_cpu)); + rq_repin_lock(rq, rf); + + *detached = 1; +} + +/* + * we only decide if nohz balance kick is needed or not. the + * first CPU in the nohz.idle will come out of idle and do + * load balance on behalf of every CPU. adding another hook + * to decide which cpu to kick is useless. most of the time, + * it is impossible to decide which CPU has to come out because + * we get to kick only once. + */ +static void walt_nohz_balancer_kick(void *unused, struct rq *rq, + unsigned int *flags, int *done) +{ + *done = 1; + + /* + * tick path migration takes care of misfit task. + * so we have to check for nr_running >= 2 here. + */ + if (rq->nr_running >= 2 && cpu_overutilized(rq->cpu)) { + *flags = NOHZ_KICK_MASK; + trace_walt_nohz_balance_kick(rq); + } +} + +static void walt_can_migrate_task(void *unused, struct task_struct *p, + int dst_cpu, int *can_migrate) +{ + bool to_lower; + + to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(task_cpu(p)); + + if (_walt_can_migrate_task(p, dst_cpu, to_lower)) + return; + + *can_migrate = 0; +} + +/* + * when WALT becomes module, this init will be called from + * another file and we don't have to define module_init(). + */ +void walt_lb_init(void) +{ + /* + * Any task movement outside task placement is called + * load balance, so moving the tick path and rotation + * code to here. we also use our custom active load balance + * stopper function instad of adding hooks to + * active_load_balance_cpu_stop() in fair.c + */ + walt_lb_rotate_work_init(); + + register_trace_android_rvh_migrate_queued_task(walt_migrate_queued_task, NULL); + register_trace_android_rvh_sched_nohz_balancer_kick(walt_nohz_balancer_kick, NULL); + register_trace_android_rvh_can_migrate_task(walt_can_migrate_task, NULL); + register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); + register_trace_android_rvh_sched_newidle_balance(walt_newidle_balance, NULL); + + /* + * TODO: + * scheduler tick is not a restricted hook so multiple entities + * can register for it. but from WALT, we will have only 1 hook + * and it will call our load balancer function later. + */ + register_trace_android_vh_scheduler_tick(walt_lb_tick, NULL); +} diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c new file mode 100644 index 000000000000..87a783c9b974 --- /dev/null +++ b/kernel/sched/walt/walt_rt.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include + +#include "walt.h" +#include "trace.h" + +static void rt_energy_aware_wake_cpu(void *unused, struct task_struct *task, + struct cpumask *lowest_mask, int ret, int *best_cpu) +{ + int cpu; + unsigned long util, best_cpu_util = ULONG_MAX; + unsigned long best_cpu_util_cum = ULONG_MAX; + unsigned long util_cum; + unsigned long tutil = task_util(task); + unsigned int best_idle_exit_latency = UINT_MAX; + unsigned int cpu_idle_exit_latency = UINT_MAX; + bool boost_on_big = rt_boost_on_big(); + int cluster; + int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0; + + if (!ret) + return; /* No targets found */ + + rcu_read_lock(); + for (cluster = 0; cluster < num_sched_clusters; cluster++) { + for_each_cpu_and(cpu, lowest_mask, &cpu_array[order_index][cluster]) { + trace_sched_cpu_util(cpu); + + if (!cpu_active(cpu)) + continue; + + if (sched_cpu_high_irqload(cpu)) + continue; + + if (__cpu_overutilized(cpu, tutil)) + continue; + + util = cpu_util(cpu); + + /* Find the least loaded CPU */ + if (util > best_cpu_util) + continue; + + /* + * If the previous CPU has same load, keep it as + * best_cpu. + */ + if (best_cpu_util == util && *best_cpu == task_cpu(task)) + continue; + + /* + * If candidate CPU is the previous CPU, select it. + * Otherwise, if its load is same with best_cpu and in + * a shallower C-state, select it. If all above + * conditions are same, select the least cumulative + * window demand CPU. + */ + cpu_idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(cpu)); + + util_cum = cpu_util_cum(cpu, 0); + if (cpu != task_cpu(task) && best_cpu_util == util) { + if (best_idle_exit_latency < cpu_idle_exit_latency) + continue; + + if (best_idle_exit_latency == cpu_idle_exit_latency && + best_cpu_util_cum < util_cum) + continue; + } + + best_idle_exit_latency = cpu_idle_exit_latency; + best_cpu_util_cum = util_cum; + best_cpu_util = util; + *best_cpu = cpu; + } + + if (*best_cpu != -1) + break; + } + + rcu_read_unlock(); +} + +void walt_rt_init(void) +{ + register_trace_android_rvh_find_lowest_rq(rt_energy_aware_wake_cpu, NULL); +} From 2b4af30325153bd760fafb1a0d574e48fd970166 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 22 Jan 2021 07:10:12 +0530 Subject: [PATCH 002/346] sched/walt: Restrict new idle load balance based on avg_idle Optimize the new idle load balance entry conditions to improve the performance/power. Change-Id: Ia859bc499e320e36e3ff32be8d690c53152a26cc Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_lb.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 45c68da4f2c2..648adfdf066d 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -538,6 +538,25 @@ static void walt_lb_tick(void *unused, struct rq *rq) raw_spin_unlock_irqrestore(&walt_lb_migration_lock, flags); } +static bool should_help_min_cap(int this_cpu) +{ + int cpu; + + if (!sysctl_sched_force_lb_enable || is_min_capacity_cpu(this_cpu)) + return false; + + for_each_cpu(cpu, &cpu_array[0][0]) { + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + if (wrq->walt_stats.nr_big_tasks) + return true; + } + + return false; +} + +/* similar to sysctl_sched_migration_cost */ +#define NEWIDLE_BALANCE_THRESHOLD 500000 static void walt_newidle_balance(void *unused, struct rq *this_rq, struct rq_flags *rf, int *pulled_task, int *done) @@ -547,6 +566,8 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, int order_index = wrq->cluster->id; int cluster = 0; int busy_cpu; + bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); + bool help_min_cap; if (unlikely(!cpu_array)) return; @@ -571,6 +592,10 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (!READ_ONCE(this_rq->rd->overload)) return; + if (atomic_read(&this_rq->nr_iowait) && !enough_idle) + return; + + help_min_cap = should_help_min_cap(this_cpu); rq_unpin_lock(this_rq, rf); raw_spin_unlock(&this_rq->lock); @@ -588,6 +613,8 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1 || this_rq->nr_running > 0) break; + if (!enough_idle && !help_min_cap) + break; } while (++cluster < num_sched_clusters); /* sanity checks before attempting the pull */ From 915ef0bd8b0e3a2fcf5b4490970b85d18d6c4a28 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:39:37 -0700 Subject: [PATCH 003/346] sched: Improve the scheduler This change is for general scheduler improvement. Change-Id: I064002ae2c7d8ffd444975ad74524f5fa8de5465 Signed-off-by: Shaleen Agrawal Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 58 ++++++++++++++++++++++++++++++++---- kernel/sched/walt/walt.h | 4 +++ kernel/sched/walt/walt_cfs.c | 10 +++++++ kernel/sched/walt/walt_lb.c | 17 +++++++++-- kernel/sched/walt/walt_rt.c | 2 ++ 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 33822e50ec52..4b0dcd61e4fe 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3723,6 +3723,8 @@ static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new) { + if (static_branch_unlikely(&walt_disabled)) + return; add_new_task_to_grp(new); } @@ -3736,6 +3738,9 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); unsigned long thermal_cap; + if (static_branch_unlikely(&walt_disabled)) + return; + /* * thermal_pressure = max_capacity - curr_cap_as_per_thermal. * so, @@ -3764,6 +3769,8 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) unsigned long flags; struct rq *rq = cpu_rq(cpu); + if (static_branch_unlikely(&walt_disabled)) + return; raw_spin_lock_irqsave(&rq->lock, flags); set_window_start(rq); raw_spin_unlock_irqrestore(&rq->lock, flags); @@ -3773,11 +3780,15 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) static void android_rvh_sched_cpu_dying(void *unused, int cpu) { + if (static_branch_unlikely(&walt_disabled)) + return; clear_walt_request(cpu); } static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsigned int new_cpu) { + if (static_branch_unlikely(&walt_disabled)) + return; if (new_cpu < 0) return; fixup_busy_time(p, (int) new_cpu); @@ -3785,11 +3796,15 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign static void android_rvh_sched_fork(void *unused, struct task_struct *p) { + if (static_branch_unlikely(&walt_disabled)) + return; init_new_task_load(p); } static void android_rvh_new_task_stats(void *unused, struct task_struct *p) { + if (static_branch_unlikely(&walt_disabled)) + return; mark_task_starting(p); } @@ -3797,6 +3812,8 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int { struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (!!(curr->flags & PF_IDLE)) { if (hardirq_count() || in_serving_softirq()) walt_sched_account_irqend(cpu, curr, delta); @@ -3808,6 +3825,8 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int static void android_rvh_flush_task(void *unused, struct task_struct *p) { + if (static_branch_unlikely(&walt_disabled)) + return; walt_task_dead(p); } @@ -3816,6 +3835,8 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st u64 wallclock = sched_ktime_clock(); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; wts->last_enqueued_ts = wallclock; sched_update_nr_prod(rq->cpu, true); @@ -3838,6 +3859,8 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st */ struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (p == wrq->ed_task) is_ed_task_present(rq, sched_ktime_clock()); @@ -3858,6 +3881,8 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p bool old_misfit, misfit; int change; + if (static_branch_unlikely(&walt_disabled)) + return; *need_update = false; if (!p) { @@ -3894,6 +3919,8 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) unsigned int old_load; struct walt_related_thread_group *grp = NULL; + if (static_branch_unlikely(&walt_disabled)) + return; rq_lock_irqsave(rq, &rf); old_load = task_load(p); wallclock = sched_ktime_clock(); @@ -3914,6 +3941,8 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct unsigned long flags; int cpu = p->cpu; + if (static_branch_unlikely(&walt_disabled)) + return; if (!sched_predl) return; @@ -3929,6 +3958,8 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) u32 old_load; struct walt_related_thread_group *grp; + if (static_branch_unlikely(&walt_disabled)) + return; set_window_start(rq); wallclock = sched_ktime_clock(); @@ -3958,6 +3989,8 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, u64 wallclock = sched_ktime_clock(); struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (likely(prev != next)) { if (!prev->on_rq) wts->last_sleep_ts = wallclock; @@ -3974,6 +4007,8 @@ static void android_rvh_resume_cpus(void *unused, struct cpumask *resuming_cpus, struct rq *rq; unsigned long flags; + if (static_branch_unlikely(&walt_disabled)) + return; /* * send a reschedule event on all resumed CPUs * which trigger newly idle load balance. @@ -3994,6 +4029,8 @@ static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (cpumask_subset(&wts->cpus_requested, cpus_requested)) *ret = set_cpus_allowed_ptr(p, &wts->cpus_requested); } @@ -4002,6 +4039,8 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; wts->last_sleep_ts = 0; wts->wake_up_idle = false; wts->boost = 0; @@ -4012,16 +4051,22 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) static void android_rvh_ttwu_cond(void *unused, bool *cond) { + if (static_branch_unlikely(&walt_disabled)) + return; *cond = sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT; } static void android_rvh_sched_exec(void *unused, bool *cond) { + if (static_branch_unlikely(&walt_disabled)) + return; *cond = true; } static void android_rvh_build_perf_domains(void *unused, bool *eas_check) { + if (static_branch_unlikely(&walt_disabled)) + return; *eas_check = true; } @@ -4054,6 +4099,7 @@ static void register_walt_hooks(void) } atomic64_t walt_irq_work_lastq_ws; +DEFINE_STATIC_KEY_TRUE(walt_disabled); static int walt_init_stop_handler(void *data) { @@ -4089,10 +4135,6 @@ static int walt_init_stop_handler(void *data) atomic64_set(&walt_irq_work_lastq_ws, window_start_ns); - register_walt_hooks(); - walt_lb_init(); - walt_rt_init(); - walt_cfs_init(); create_default_coloc_group(); walt_update_cluster_topology(); @@ -4101,7 +4143,6 @@ static int walt_init_stop_handler(void *data) raw_spin_unlock(&cpu_rq(cpu)->lock); } read_unlock(&tasklist_lock); - return 0; } @@ -4116,7 +4157,14 @@ static int walt_module_init(void) BUG_ON(alloc_related_thread_groups()); walt_init_cycle_counter(); init_clusters(); + + register_walt_hooks(); + walt_lb_init(); + walt_rt_init(); + walt_cfs_init(); + stop_machine(walt_init_stop_handler, NULL, NULL); + static_branch_disable(&walt_disabled); hdr = register_sysctl_table(walt_base_table); kmemleak_not_leak(hdr); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 0315be82852a..c3e485844ba4 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -9,6 +9,8 @@ #include "../../../kernel/sched/sched.h" #include "../../../fs/proc/internal.h" #include +#include + #include #ifdef CONFIG_HZ_300 @@ -38,6 +40,8 @@ /* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */ #define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1) +extern struct static_key_true walt_disabled; + enum task_event { PUT_PREV_TASK = 0, PICK_NEXT_TASK = 1, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index c99337e54ea2..5de798da3d14 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -686,6 +686,8 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); int sibling_count_hint = p->wake_q_head ? p->wake_q_head->count : 1; + if (static_branch_unlikely(&walt_disabled)) + return; *target_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, sync, sibling_count_hint); if (unlikely(*target_cpu < 0)) *target_cpu = prev_cpu; @@ -710,6 +712,8 @@ static unsigned long task_h_load(struct task_struct *p) static void walt_update_misfit_status(void *unused, struct task_struct *p, struct rq *rq, bool *need_update) { + if (static_branch_unlikely(&walt_disabled)) + return; *need_update = false; if (!p) { @@ -736,6 +740,8 @@ static inline struct task_struct *task_of(struct sched_entity *se) static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vruntime) { + if (static_branch_unlikely(&walt_disabled)) + return; if (entity_is_task(se)) { unsigned long thresh = sysctl_sched_latency; @@ -760,6 +766,8 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task) { struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (task && current->signal && (current->signal->oom_score_adj == 0) && (current->prio < DEFAULT_PRIO)) @@ -770,6 +778,8 @@ static void walt_binder_low_latency_clear(void *unused, struct binder_transactio { struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (wts->low_latency) wts->low_latency = false; } diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 648adfdf066d..24a37c47be0a 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -489,6 +489,8 @@ static void walt_lb_tick(void *unused, struct rq *rq) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (static_branch_unlikely(&walt_disabled)) + return; if (!rq->misfit_task_load) return; @@ -563,15 +565,18 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, { int this_cpu = this_rq->cpu; struct walt_rq *wrq = (struct walt_rq *) this_rq->android_vendor_data1; - int order_index = wrq->cluster->id; + int order_index; int cluster = 0; int busy_cpu; bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); bool help_min_cap; - if (unlikely(!cpu_array)) + if (static_branch_unlikely(&walt_disabled)) return; + /*Cluster isn't initialized until after WALT is enabled*/ + order_index = wrq->cluster->id; + /* * newly idle load balance is completely handled here, so * set done to skip the load balance by the caller. @@ -650,6 +655,8 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, int busiest_cpu = -1; struct cpumask src_mask; + if (static_branch_unlikely(&walt_disabled)) + return; *done = 1; *busiest = NULL; @@ -686,6 +693,8 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, struct task_struct *p, int new_cpu, int *detached) { + if (static_branch_unlikely(&walt_disabled)) + return; /* * WALT expects both source and destination rqs to be * held when set_task_cpu() is called on a queued task. @@ -713,6 +722,8 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, static void walt_nohz_balancer_kick(void *unused, struct rq *rq, unsigned int *flags, int *done) { + if (static_branch_unlikely(&walt_disabled)) + return; *done = 1; /* @@ -730,6 +741,8 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, { bool to_lower; + if (static_branch_unlikely(&walt_disabled)) + return; to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(task_cpu(p)); if (_walt_can_migrate_task(p, dst_cpu, to_lower)) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 87a783c9b974..862df51479f9 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -22,6 +22,8 @@ static void rt_energy_aware_wake_cpu(void *unused, struct task_struct *task, int cluster; int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0; + if (static_branch_unlikely(&walt_disabled)) + return; if (!ret) return; /* No targets found */ From 07efb5ada940fe95cb720a079b7ef46bd23d99cb Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 22 Jan 2021 07:57:10 +0530 Subject: [PATCH 004/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I616acbe5bfdb0e1bf88e923d98b4a3d54d26d942 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 3 +++ kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_cfs.c | 3 ++- kernel/sched/walt/walt_lb.c | 3 ++- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4b0dcd61e4fe..4c57d6e9991f 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2187,6 +2187,8 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even update_task_demand(p, rq, event, wallclock); update_cpu_busy_time(p, rq, event, wallclock, irqtime); update_task_pred_demand(rq, p, event); + if (event == PUT_PREV_TASK && p->state) + wts->iowaited = p->in_iowait; trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime, &wrq->grp_time, wrq, wts); @@ -4047,6 +4049,7 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) wts->boost_expires = 0; wts->boost_period = false; wts->low_latency = false; + wts->iowaited = false; } static void android_rvh_ttwu_cond(void *unused, bool *cond) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index c3e485844ba4..be4ce71fbf0a 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -132,6 +132,7 @@ struct walt_task_struct { struct list_head grp_list; u64 cpu_cycles; cpumask_t cpus_requested; + bool iowaited; }; /*End linux/sched.h port */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 5de798da3d14..c5548f0e626e 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -213,7 +213,8 @@ static void walt_find_best_target(struct sched_domain *sd, unsigned int target_nr_rtg_high_prio = UINT_MAX; bool rtg_high_prio_task = task_rtg_high_prio(p); cpumask_t visit_cpus; - bool io_task_pack = (order_index > 0 && p->in_iowait); + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + bool io_task_pack = (order_index > 0 && wts->iowaited); struct cfs_rq *cfs_rq; /* Find start CPU based on boost value */ diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 24a37c47be0a..7f200acc5ed7 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -210,9 +210,10 @@ static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, bool to_lower) { struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (to_lower) { - if (p->in_iowait) + if (wts->iowaited) return false; if (per_task_boost(p) == TASK_BOOST_STRICT_MAX && task_in_related_thread_group(p)) From 4fb4acb57cea3ec3ab908b5145f3caa3d25073bd Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 22 Jan 2021 08:39:40 +0530 Subject: [PATCH 005/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I30b3899e191d880e741f922323d0826b5c292498 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_cfs.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index c5548f0e626e..a1a471a10663 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -209,18 +209,27 @@ static void walt_find_best_target(struct sched_domain *sd, int prev_cpu = task_cpu(p); int active_candidate = -1; int order_index = fbt_env->order_index, end_index = fbt_env->end_index; + int stop_index = INT_MAX; int cluster; unsigned int target_nr_rtg_high_prio = UINT_MAX; bool rtg_high_prio_task = task_rtg_high_prio(p); cpumask_t visit_cpus; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - bool io_task_pack = (order_index > 0 && wts->iowaited); struct cfs_rq *cfs_rq; /* Find start CPU based on boost value */ start_cpu = fbt_env->start_cpu; - if (fbt_env->strict_max || io_task_pack) + /* + * For higher capacity worth I/O tasks, stop the search + * at the end of higher capacity cluster(s). + */ + if (order_index > 0 && wts->iowaited) { + stop_index = num_sched_clusters - 2; + most_spare_wake_cap = LONG_MIN; + } + + if (fbt_env->strict_max) target_max_spare_cap = LONG_MIN; if (p->state == TASK_RUNNING) @@ -296,8 +305,7 @@ static void walt_find_best_target(struct sched_domain *sd, * than the one required to boost the task. */ new_util = max(min_util, new_util); - if (!(fbt_env->strict_max || io_task_pack) && - new_util > capacity_orig) + if (!fbt_env->strict_max && new_util > capacity_orig) continue; /* @@ -386,6 +394,9 @@ static void walt_find_best_target(struct sched_domain *sd, if ((cluster >= end_index) && (target_cpu != -1) && walt_target_ok(target_cpu, order_index)) break; + + if (most_spare_cap_cpu != -1 && cluster >= stop_index) + break; } if (best_idle_cpu != -1) From 0390d12e5d3d54e59b789a445a9bdc1008579254 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 22 Jan 2021 08:44:34 +0530 Subject: [PATCH 006/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Id19f38a2b9a67b130876cb8df1e88c6a56aa6aad Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_cfs.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index a1a471a10663..ce47fc59a017 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -229,8 +229,10 @@ static void walt_find_best_target(struct sched_domain *sd, most_spare_wake_cap = LONG_MIN; } - if (fbt_env->strict_max) - target_max_spare_cap = LONG_MIN; + if (fbt_env->strict_max) { + stop_index = 0; + most_spare_wake_cap = LONG_MIN; + } if (p->state == TASK_RUNNING) most_spare_wake_cap = ULONG_MAX; @@ -290,9 +292,8 @@ static void walt_find_best_target(struct sched_domain *sd, most_spare_cap_cpu = i; } - if ((per_task_boost(cpu_rq(i)->curr) == - TASK_BOOST_STRICT_MAX) && - !fbt_env->strict_max) + if (per_task_boost(cpu_rq(i)->curr) == + TASK_BOOST_STRICT_MAX) continue; /* get rq's utilization with this task included */ @@ -305,7 +306,7 @@ static void walt_find_best_target(struct sched_domain *sd, * than the one required to boost the task. */ new_util = max(min_util, new_util); - if (!fbt_env->strict_max && new_util > capacity_orig) + if (new_util > capacity_orig) continue; /* From 43cd57a2f23b4a7256e05b4fb38899f5de4a6c37 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 19 Jan 2021 15:55:31 -0800 Subject: [PATCH 007/346] sched/walt: wait for cpufreq policies walt_update_cluster_topology requires the availability of the cpufreq policies. These are not available immediately after insmoding walt, and this causes a crash through SCHED_BUG_ON. Register for the cpufreq notifier for POLICY creation, and init the second half of walt init after the policy creation for all cpus. Change-Id: If1af62a4991f3226bd0b661f5a5e419ceea3817a Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 50 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4c57d6e9991f..2d0623272dea 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4149,11 +4149,15 @@ static int walt_init_stop_handler(void *data) return 0; } -static int walt_module_init(void) +static void walt_init(void) { struct ctl_table_header *hdr; + static atomic_t already_inited = ATOMIC_INIT(0); int i; + if (atomic_cmpxchg(&already_inited, 0, 1)) + return; + walt_tunables(); sched_init_ops(); @@ -4179,6 +4183,50 @@ static int walt_module_init(void) i = match_string(sched_feat_names, __SCHED_FEAT_NR, "TTWU_QUEUE"); static_key_disable_cpuslocked(&sched_feat_keys[i]); sysctl_sched_features &= ~(1UL << i); +} + +static bool are_cpufreq_policies_available(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + if (!cpufreq_cpu_get_raw(cpu)) + return false; + } + + return true; +} + +struct work_struct walt_cpufreq_policy_work; + +static int walt_cpufreq_notifier_cb(struct notifier_block *nb, + unsigned long action, + void *data) +{ + if (are_cpufreq_policies_available()) + schedule_work(&walt_cpufreq_policy_work); + + return 0; +} + +static struct notifier_block walt_cpufreq_notifier_block = { + .notifier_call = walt_cpufreq_notifier_cb +}; + +static void walt_wq_cpufreq_policy_update(struct work_struct *work) +{ + walt_init(); + cpufreq_unregister_notifier(&walt_cpufreq_notifier_block, CPUFREQ_POLICY_NOTIFIER); +} + +static int walt_module_init(void) +{ + INIT_WORK(&walt_cpufreq_policy_work, walt_wq_cpufreq_policy_update); + cpufreq_register_notifier(&walt_cpufreq_notifier_block, + CPUFREQ_POLICY_NOTIFIER); + + if (are_cpufreq_policies_available()) + walt_init(); return 0; } From 11cab5f8abc373f3f68b5053e8b0c89981ad0a23 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Wed, 27 Jan 2021 11:49:39 +0530 Subject: [PATCH 008/346] sched/walt: Remove obsolete TODO comments The TODO comments in walt_lb.c are not applicable anymore. So remove them. Change-Id: I3cd780ad3f05e974944d6aa60f0a3da155c7b1be Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_lb.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 7f200acc5ed7..cff8b236c9a7 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -8,7 +8,6 @@ #include "walt.h" #include "trace.h" -extern u64 sched_ktime_clock(void); // TODO static void walt_detach_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) { @@ -772,12 +771,5 @@ void walt_lb_init(void) register_trace_android_rvh_can_migrate_task(walt_can_migrate_task, NULL); register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); register_trace_android_rvh_sched_newidle_balance(walt_newidle_balance, NULL); - - /* - * TODO: - * scheduler tick is not a restricted hook so multiple entities - * can register for it. but from WALT, we will have only 1 hook - * and it will call our load balancer function later. - */ register_trace_android_vh_scheduler_tick(walt_lb_tick, NULL); } From d5d3d33ff385eab2f628175cb35fa11e013d89ec Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 27 Jan 2021 09:22:25 -0800 Subject: [PATCH 009/346] sched: walt: Fixup root domain selection In WALT task placement algorithm, we select the root domain to be the root domain corresponding to the current cpu, however, this could be null if the cpu is inactive. This, in fact, resulted in undesired behaviour when evaluating a single old threaded usecase, since desired cpu was inactive, thread was being placed on a fallback cpu. Change-Id: I68eef10dc84db15796a0d43af7215f4dbbdc059e Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index ce47fc59a017..6c2c6415e444 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -552,7 +552,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync, int sibling_count_hint) { unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; - struct root_domain *rd = cpu_rq(smp_processor_id())->rd; + struct root_domain *rd = cpu_rq(cpumask_first(cpu_active_mask))->rd; int weight, cpu = smp_processor_id(), best_energy_cpu = prev_cpu; struct perf_domain *pd; unsigned long cur_energy; From 79b57c0e1c60a818d14b20f85415164e678fbd63 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 28 Jan 2021 08:15:00 +0530 Subject: [PATCH 010/346] sched/walt: Add a build time test for vendor data size check WALT is using padding avaialble in task_struct, rq and task_group structures. Add a build time test to make sure that we are not using more than the avaialble padding. Change-Id: Icd7e23b4536075bc53a975eeff838eedd58b29cc Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2d0623272dea..06f938ce6cc4 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4219,8 +4219,17 @@ static void walt_wq_cpufreq_policy_update(struct work_struct *work) cpufreq_unregister_notifier(&walt_cpufreq_notifier_block, CPUFREQ_POLICY_NOTIFIER); } +#define WALT_VENDOR_DATA_SIZE_TEST(wstruct, kstruct) \ + BUILD_BUG_ON(sizeof(wstruct) > (sizeof(u64) * \ + ARRAY_SIZE(((kstruct *)0)->android_vendor_data1))) + static int walt_module_init(void) { + /* compile time checks for vendor data size */ + WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_struct, struct task_struct); + WALT_VENDOR_DATA_SIZE_TEST(struct walt_rq, struct rq); + WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_group, struct task_group); + INIT_WORK(&walt_cpufreq_policy_work, walt_wq_cpufreq_policy_update); cpufreq_register_notifier(&walt_cpufreq_notifier_block, CPUFREQ_POLICY_NOTIFIER); From a699cd93698bca8a94cde07a69571b741a730593 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 28 Jan 2021 09:22:13 +0530 Subject: [PATCH 011/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I8f521dfd31ff1c8cc1f16805e4e0ac78228d9383 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/sysctl.c | 8 ++++++-- kernel/sched/walt/walt.h | 5 ++++- kernel/sched/walt/walt_cfs.c | 6 +++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 6dd40e05cf13..ded371ed3702 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -262,7 +262,8 @@ static int sched_task_handler(struct ctl_table *table, int write, 1000000UL); break; case LOW_LATENCY: - pid_and_val[1] = wts->low_latency; + pid_and_val[1] = wts->low_latency & + WALT_LOW_LATENCY_PROCFS; break; default: ret = -EINVAL; @@ -323,7 +324,10 @@ static int sched_task_handler(struct ctl_table *table, int write, wts->boost_expires = sched_clock() + wts->boost_period; break; case LOW_LATENCY: - wts->low_latency = val; + if (val) + wts->low_latency |= WALT_LOW_LATENCY_PROCFS; + else + wts->low_latency &= ~WALT_LOW_LATENCY_PROCFS; break; default: ret = -EINVAL; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index be4ce71fbf0a..f551ef30083b 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -69,6 +69,9 @@ enum task_boost_type { #define RAVG_HIST_SIZE_MAX 5 #define NUM_BUSY_BUCKETS 10 +#define WALT_LOW_LATENCY_PROCFS BIT(0) +#define WALT_LOW_LATENCY_BINDER BIT(1) + struct walt_task_struct { /* * 'mark_start' marks the beginning of an event (task waking up, task @@ -120,7 +123,7 @@ struct walt_task_struct { bool wake_up_idle; bool misfit; bool rtg_high_prio; - bool low_latency; + u8 low_latency; u64 boost_period; u64 boost_expires; u64 last_sleep_ts; diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 6c2c6415e444..9dbc9249ecfc 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -784,7 +784,7 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task) if (task && current->signal && (current->signal->oom_score_adj == 0) && (current->prio < DEFAULT_PRIO)) - wts->low_latency = true; + wts->low_latency |= WALT_LOW_LATENCY_BINDER; } static void walt_binder_low_latency_clear(void *unused, struct binder_transaction *t) @@ -793,8 +793,8 @@ static void walt_binder_low_latency_clear(void *unused, struct binder_transactio if (static_branch_unlikely(&walt_disabled)) return; - if (wts->low_latency) - wts->low_latency = false; + if (wts->low_latency & WALT_LOW_LATENCY_BINDER) + wts->low_latency &= ~WALT_LOW_LATENCY_BINDER; } void walt_cfs_init(void) From 89a180556f3c74572a27a8bca9392db99011de6d Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 28 Jan 2021 12:44:52 +0530 Subject: [PATCH 012/346] sched/walt: Synchronize per-task sysctl and sysctl_task_read_pid sysctl_task_read_pid sysctl needs to be set to the desired task's pid before reading any of the per-task sysctl. So use the same mutex for both of these sysctl handlers to prevent data tearing. While at it, fix an incorrect goto in sched_task_handler() function and restrict the range of sysctl_task_read_pid. Change-Id: I544010f51a28ff6e5c4f117d68816de147ef3d0f Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/sysctl.c | 35 ++++++++++++++++++++++++----------- kernel/sched/walt/walt.h | 1 - 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index ded371ed3702..83ad94ca4536 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -53,13 +53,15 @@ unsigned int sysctl_sched_walt_rotate_big_tasks; unsigned int sysctl_sched_task_unfilter_period; unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ -unsigned int sysctl_task_read_pid; unsigned int sysctl_sched_conservative_pl; unsigned int sysctl_sched_min_task_util_for_boost = 51; unsigned int sysctl_sched_min_task_util_for_colocation = 35; unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; const int sched_user_hint_max = 1000; +/* range is [1 .. INT_MAX] */ +static int sysctl_task_read_pid = 1; + static void init_tg_pointers(void) { struct cgroup_subsys_state *css = &root_task_group.css; @@ -200,6 +202,20 @@ static int sched_ravg_window_handler(struct ctl_table *table, return ret; } +static DEFINE_MUTEX(sysctl_pid_mutex); +static int sched_task_read_pid_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + + mutex_lock(&sysctl_pid_mutex); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + mutex_unlock(&sysctl_pid_mutex); + + return ret; +} + enum { TASK_BEGIN = 0, WAKE_UP_IDLE, @@ -225,20 +241,15 @@ static int sched_task_handler(struct ctl_table *table, int write, .maxlen = sizeof(pid_and_val), .mode = table->mode, }; - static DEFINE_MUTEX(mutex); - mutex_lock(&mutex); + mutex_lock(&sysctl_pid_mutex); if (!write) { - if (sysctl_task_read_pid <= 0) { - ret = -ENOENT; - goto unlock_mutex; - } task = get_pid_task(find_vpid(sysctl_task_read_pid), PIDTYPE_PID); if (!task) { ret = -ENOENT; - goto put_task; + goto unlock_mutex; } wts = (struct walt_task_struct *) task->android_vendor_data1; pid_and_val[0] = sysctl_task_read_pid; @@ -336,7 +347,7 @@ static int sched_task_handler(struct ctl_table *table, int write, put_task: put_task_struct(task); unlock_mutex: - mutex_unlock(&mutex); + mutex_unlock(&sysctl_pid_mutex); return ret; } @@ -832,9 +843,11 @@ struct ctl_table walt_table[] = { { .procname = "sched_task_read_pid", .data = &sysctl_task_read_pid, - .maxlen = sizeof(unsigned int), + .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = sched_task_read_pid_handler, + .extra1 = SYSCTL_ONE, + .extra2 = SYSCTL_INT_MAX, }, { .procname = "sched_load_boost", diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index f551ef30083b..d85ede69a67f 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -388,7 +388,6 @@ extern unsigned int sysctl_sched_walt_rotate_big_tasks; extern unsigned int sysctl_sched_task_unfilter_period; extern unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ -extern unsigned int sysctl_task_read_pid; extern struct ctl_table walt_table[]; extern struct ctl_table walt_base_table[]; extern void walt_tunables(void); From ceda5e2445ac99a2b29bb4511da4b646ccf726df Mon Sep 17 00:00:00 2001 From: Lingutla Chandrasekhar Date: Thu, 19 Nov 2020 20:52:54 +0530 Subject: [PATCH 013/346] sched/walt: Add window rollover trace event Add window rollover trace event to track cpu's WALT window rollover. Change-Id: I4a2e7291f31a1ac50144086eaf6ae920a07ca545 Signed-off-by: Lingutla Chandrasekhar Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/trace.h | 17 +++++++++++++++++ kernel/sched/walt/walt.c | 4 +++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 75be13d809bf..6dd8f72feb61 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1088,6 +1088,23 @@ TRACE_EVENT(sched_enq_deq_task, __entry->cpus_allowed, __entry->demand, __entry->pred_demand) ); + +TRACE_EVENT(walt_window_rollover, + + TP_PROTO(u64 window_start), + + TP_ARGS(window_start), + + TP_STRUCT__entry( + __field(u64, window_start) + ), + + TP_fast_assign( + __entry->window_start = window_start; + ), + + TP_printk("window_start=%llu", __entry->window_start) +); #endif /* _TRACE_WALT_H */ #undef TRACE_INCLUDE_PATH diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 06f938ce6cc4..7704c26fa488 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2159,8 +2159,10 @@ static inline void run_walt_irq_work(u64 old_window_start, struct rq *rq) result = atomic64_cmpxchg(&walt_irq_work_lastq_ws, old_window_start, wrq->window_start); - if (result == old_window_start) + if (result == old_window_start) { walt_irq_work_queue(&walt_cpufreq_irq_work); + trace_walt_window_rollover(wrq->window_start); + } } /* Reflect task activity on its demand and cpu's busy time statistics */ From d98f896a37b8c4e17adec6cb396fe2e4f3cc6f93 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 29 Jan 2021 08:47:44 +0530 Subject: [PATCH 014/346] sched/walt: Remove unused sched_pause trace point cpuhp_pause trace point is in place. so remove the sched_pause trace point from WALT module. Change-Id: Ife250254207321a9033142a34319f04cdf3a2dbe Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/trace.h | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 6dd8f72feb61..74db0fd41a22 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -589,43 +589,6 @@ TRACE_EVENT(sched_get_nr_running_avg, __entry->nr_scaled) ); -/* - * sched_pause - called when cores are paused/unpaused - * - * @start: 1 if start of pause/resume op, 0 otherwise - * @requested_cpus: mask of cpus requested in this op - * @active_cpus: mask of currently active cpus - * @start_time: time of the start of the operation - * @pause: 1 if pausing, 0 if resuming - */ -TRACE_EVENT(sched_pause, - - TP_PROTO(unsigned int start, unsigned int requested_cpus, unsigned int active_cpus, - u64 start_time, unsigned char pause), - - TP_ARGS(start, requested_cpus, active_cpus, start_time, pause), - - TP_STRUCT__entry( - __field(u32, start) - __field(u32, requested_cpus) - __field(u32, active_cpus) - __field(u32, time) - __field(unsigned char, pause) - ), - - TP_fast_assign( - __entry->start = start; - __entry->requested_cpus = requested_cpus; - __entry->active_cpus = active_cpus; - __entry->time = div64_u64(sched_clock() - start_time, 1000); - __entry->pause = pause; - ), - - TP_printk("start=%d req cpus=0x%x act cpus=0x%x time=%u us paused=%d", - __entry->start, __entry->requested_cpus, __entry->active_cpus, - __entry->time, __entry->pause) -); - TRACE_EVENT(sched_ravg_window_change, TP_PROTO(unsigned int sched_ravg_window, unsigned int new_sched_ravg_window From 6e8d29392c46c35fa9ddb6798e8fe4135372bb59 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 29 Jan 2021 09:47:42 +0530 Subject: [PATCH 015/346] sched/wakeup: Optimize core_ctl thread wakeup We have 1 core_ctl thread for all clusters. Evaluate all clusters and then wakeup the core_ctl if required. Change-Id: I8131ae97280d8c478fd061e0aac06f1e56c5b6ea Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/core_ctl.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 1330e07b04f7..1379ea789334 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -946,6 +946,7 @@ void core_ctl_check(u64 window_start) struct cluster_data *cluster; unsigned int index = 0; unsigned long flags; + unsigned int wakeup = 0; if (unlikely(!initialized)) return; @@ -970,11 +971,11 @@ void core_ctl_check(u64 window_start) update_running_avg(); - for_each_cluster(cluster, index) { - if (eval_need(cluster)) - wake_up_core_ctl_thread(); - } + for_each_cluster(cluster, index) + wakeup |= eval_need(cluster); + if (wakeup) + wake_up_core_ctl_thread(); core_ctl_call_notifier(); } From 9f4c2201c27c747267d1691562011ada56600769 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 29 Jan 2021 10:11:29 +0530 Subject: [PATCH 016/346] sched/walt: Remove evaluation from the core_ctl thread context The core_ctl thread is woken up only after evaluating the need. so another evaluation in core_ctl thread context is not needed. Change-Id: I28162b758c18cda4ff05f1b1a4b1a5553c895e7e Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/core_ctl.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 1379ea789334..193dd23777c2 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -1159,8 +1159,6 @@ static void __ref do_core_ctl(void) for_each_cluster(cluster, index) { - eval_need(cluster); - need = apply_limits(cluster, cluster->need_cpus); if (adjustment_possible(cluster, need)) { From 618298cf3699a973de22bc854336d0109971beba Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 29 Jan 2021 11:25:35 +0530 Subject: [PATCH 017/346] sched/walt: Remove should_we_pause() function should_we_pause() always return true. Update the conditional check at the caller and remove this function. Change-Id: I085bc7737888fcafc74350a7d22176ae27bdece2 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/core_ctl.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 193dd23777c2..e4fff356a4c4 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -989,11 +989,6 @@ static void move_cpu_lru(struct cpu_data *cpu_data) spin_unlock_irqrestore(&state_lock, flags); } -static bool should_we_pause(int cpu, struct cluster_data *cluster) -{ - return true; -} - static void try_to_pause(struct cluster_data *cluster, unsigned int need, struct cpumask *pause_cpus) { @@ -1027,9 +1022,6 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, if (cluster->nr_not_preferred_cpus && !c->not_preferred) continue; - if (!should_we_pause(c->cpu, cluster)) - continue; - spin_unlock_irqrestore(&state_lock, flags); pr_debug("Trying to pause CPU%u\n", c->cpu); From 899a1819426c0ab88d777627de299821c0308e20 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:40:24 -0700 Subject: [PATCH 018/346] sched: walt: Cleanup core_ctl We export necessary symbols as msm_perf driver needs to use core_ctl and expand the core_ctl header file to add non WALT stubs. Change-Id: I2fa65d6e5b00cfc88b97579df7074e8f82cb23ac Signed-off-by: Shaleen Agrawal Signed-off-by: Sai Harshini Nimmala --- include/linux/sched/core_ctl.h | 36 ++++++++++++++++++++++++++++++++++ kernel/sched/walt/core_ctl.c | 2 ++ kernel/sched/walt/walt.h | 4 ---- 3 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 include/linux/sched/core_ctl.h diff --git a/include/linux/sched/core_ctl.h b/include/linux/sched/core_ctl.h new file mode 100644 index 000000000000..c7ff34fbda5c --- /dev/null +++ b/include/linux/sched/core_ctl.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2016, 2019-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef __CORE_CTL_H +#define __CORE_CTL_H + +#include + +#define MAX_CPUS_PER_CLUSTER 6 +#define MAX_CLUSTERS 3 + +struct core_ctl_notif_data { + unsigned int nr_big; + unsigned int coloc_load_pct; + unsigned int ta_util_pct[MAX_CLUSTERS]; + unsigned int cur_cap_pct[MAX_CLUSTERS]; +}; + +struct notifier_block; + +#if IS_ENABLED(CONFIG_SCHED_WALT) +extern int core_ctl_set_boost(bool boost); +extern void core_ctl_notifier_register(struct notifier_block *n); +extern void core_ctl_notifier_unregister(struct notifier_block *n); +#else +static inline int core_ctl_set_boost(bool boost) +{ + return 0; +} +static inline void core_ctl_notifier_register(struct notifier_block *n) {} +static inline void core_ctl_notifier_unregister(struct notifier_block *n) {} +#endif + +#endif diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index e4fff356a4c4..f02888e95d3f 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -909,11 +909,13 @@ void core_ctl_notifier_register(struct notifier_block *n) { atomic_notifier_chain_register(&core_ctl_notifier, n); } +EXPORT_SYMBOL(core_ctl_notifier_register); void core_ctl_notifier_unregister(struct notifier_block *n) { atomic_notifier_chain_unregister(&core_ctl_notifier, n); } +EXPORT_SYMBOL(core_ctl_notifier_unregister); static void core_ctl_call_notifier(void) { diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index d85ede69a67f..50afd59cb54a 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -36,7 +36,6 @@ #define SCHED_CPUFREQ_EARLY_DET (1U << 6) #define SCHED_CPUFREQ_CONTINUE (1U << 8) -#define MAX_CLUSTERS 3 /* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */ #define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1) @@ -277,9 +276,6 @@ extern int sched_group_id_open(struct inode *inode, struct file *filp); extern int sched_pause_cpus(struct cpumask *pause_cpus); extern int sched_unpause_cpus(struct cpumask *unpause_cpus); -extern int core_ctl_set_boost(bool boost); -extern void core_ctl_notifier_register(struct notifier_block *n); -extern void core_ctl_notifier_unregister(struct notifier_block *n); extern unsigned int sched_get_cpu_util(int cpu); extern void sched_update_hyst_times(void); extern u64 sched_lpm_disallowed_time(int cpu); From 09445f02e81c678cf4518e2cddf9b28a2692772d Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 1 Feb 2021 10:16:25 +0530 Subject: [PATCH 019/346] sched/walt: Implement sched_lpm_disallowed_time() API LPM governor can query the scheduler preference about the CPU's c-state via sched_lpm_disallowed_time() API. It returns 0 when shallowest c-state is preferred and associated timeout is set. Change-Id: Ie8246aa9ec4638730d3442468aa9dc1969a90369 Signed-off-by: Pavankumar Kondeti --- include/linux/sched/walt.h | 20 ++++++++++++++++++++ kernel/sched/walt/sched_avg.c | 15 +++++++++++---- kernel/sched/walt/walt.h | 1 - 3 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 include/linux/sched/walt.h diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h new file mode 100644 index 000000000000..f3fdd4da27e8 --- /dev/null +++ b/include/linux/sched/walt.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _LINUX_SCHED_WALT_H +#define _LINUX_SCHED_WALT_H + +#include + +#if IS_ENABLED(CONFIG_SCHED_WALT) +extern int sched_lpm_disallowed_time(int cpu, u64 *timeout); +#else +static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout) +{ + return INT_MAX; +} +#endif + +#endif /* _LINUX_SCHED_WALT_H */ diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index cec81ea98fba..ededab2476e6 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -237,14 +237,21 @@ unsigned int sched_get_cpu_util(int cpu) return busy; } -u64 sched_lpm_disallowed_time(int cpu) +int sched_lpm_disallowed_time(int cpu, u64 *timeout) { u64 now = sched_clock(); u64 bias_end_time = atomic64_read(&per_cpu(busy_hyst_end_time, cpu)); - if (now < bias_end_time) - return bias_end_time - now; + if (unlikely(is_reserved(cpu))) { + *timeout = 10 * NSEC_PER_MSEC; + return 0; /* shallowest c-state */ + } - return 0; + if (now < bias_end_time) { + *timeout = bias_end_time - now; + return 0; /* shallowest c-state */ + } + + return INT_MAX; /* don't care */ } EXPORT_SYMBOL(sched_lpm_disallowed_time); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 50afd59cb54a..37c03d98182f 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -278,7 +278,6 @@ extern int sched_unpause_cpus(struct cpumask *unpause_cpus); extern unsigned int sched_get_cpu_util(int cpu); extern void sched_update_hyst_times(void); -extern u64 sched_lpm_disallowed_time(int cpu); extern int sched_updown_migrate_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); From 6f013c2fae6ce25a660fa2f98cbd90ef8c99d3e2 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 1 Feb 2021 15:34:27 +0530 Subject: [PATCH 020/346] sched/walt: Remove left over code related to kernel command line param There are references to kernel command line parameters like sched_predl and sched_ravg_window. These are not supported any more. So remove the left over code. Change-Id: I865f44fafee3a1d8f3d0d6f8f7e2341aff9ca964 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 9 --------- kernel/sched/walt/walt.h | 6 ------ 2 files changed, 15 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 7704c26fa488..15361ba46a96 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -80,7 +80,6 @@ unsigned int __read_mostly sched_init_task_load_windows; * sched_load_granule. */ unsigned int __read_mostly sched_load_granule; -__read_mostly bool sched_predl = true; /* *@boost:should be 0,1,2. @@ -1261,9 +1260,6 @@ static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int ev u16 new_scaled; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (!sched_predl) - return; - if (is_idle_task(p)) return; @@ -1809,9 +1805,6 @@ static inline u32 predict_and_update_buckets( u32 pred_demand; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (!sched_predl) - return 0; - bidx = busy_to_bucket(runtime); pred_demand = get_pred_busy(p, bidx, runtime); bucket_increase(wts->busy_buckets, bidx); @@ -3947,8 +3940,6 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct if (static_branch_unlikely(&walt_disabled)) return; - if (!sched_predl) - return; raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags); if (do_pl_notif(cpu_rq(cpu))) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 37c03d98182f..a423fbf649ca 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -137,10 +137,6 @@ struct walt_task_struct { bool iowaited; }; -/*End linux/sched.h port */ -/*SCHED.H PORT*/ -extern __read_mostly bool sched_predl; - struct walt_cpu_load { unsigned long nl; unsigned long pl; @@ -299,8 +295,6 @@ extern void acquire_rq_locks_irqsave(const cpumask_t *cpus, extern void release_rq_locks_irqrestore(const cpumask_t *cpus, unsigned long *flags); extern struct list_head cluster_head; -extern int set_sched_ravg_window(char *str); -extern int set_sched_predl(char *str); extern int input_boost_init(void); extern int core_ctl_init(void); From cadae13c7a47a6889e95ca6debc04b20a054e722 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 1 Feb 2021 15:39:11 +0530 Subject: [PATCH 021/346] sched/walt: Fix lockdep warning static_key_disable_cpuslocked() API is called to disable TTWU_QUEUE feature. This API expects read side of cpu_hotplug_lock lock is taken. Since we are not acquiring that lock, use static_key_disable() API which internally takes care of the locking. Change-Id: Ic8c7023235dd2e03bfe93d3f87a6a68e4f5b9b0d Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 15361ba46a96..baa9b8427c0a 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4174,8 +4174,10 @@ static void walt_init(void) waltgov_register(); i = match_string(sched_feat_names, __SCHED_FEAT_NR, "TTWU_QUEUE"); - static_key_disable_cpuslocked(&sched_feat_keys[i]); - sysctl_sched_features &= ~(1UL << i); + if (i >= 0) { + static_key_disable(&sched_feat_keys[i]); + sysctl_sched_features &= ~(1UL << i); + } } static bool are_cpufreq_policies_available(void) From 04c68623ec61cfb831513455f08f1eb7a5ad47a5 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 29 Jan 2021 07:13:20 +0530 Subject: [PATCH 022/346] sched/walt: Simplify core_ctl active cpu count management The core_ctl has to know how many CPUs are active in each cluster. If the current need is less than the number of active CPUs, it tries to pause the CPUs provided tunables are in agreement. On the other hand, if the current need is more than the number of active CPUs, it tries to resume the CPUs, provided those CPUs are paused by us in the first place. To determine the number of active CPUs, we should consider only active CPUs. The others could be in pause or hotplugged out. Since we only use cpu_active_mask for both, we don't need wrappers to tell this to us. All we need is to maintain how many CPUs that are going to be paused/resumed in the current iteration and check against the need/limits. Change-Id: I0a29da869b52eff35ff1dfa3094c51d6ece4e49f Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/Makefile | 2 +- kernel/sched/walt/core_ctl.c | 87 +++++++++++++++--------------------- kernel/sched/walt/qc_vas.c | 52 --------------------- kernel/sched/walt/walt.h | 3 -- 4 files changed, 37 insertions(+), 107 deletions(-) delete mode 100644 kernel/sched/walt/qc_vas.c diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile index 34987fd075d2..18e5264cf702 100644 --- a/kernel/sched/walt/Makefile +++ b/kernel/sched/walt/Makefile @@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n KCSAN_SANITIZE := n obj-$(CONFIG_SCHED_WALT) += sched-walt.o -sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o qc_vas.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index f02888e95d3f..c142a9ad8ad3 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -752,8 +752,10 @@ static unsigned int apply_limits(const struct cluster_data *cluster, static unsigned int get_active_cpu_count(const struct cluster_data *cluster) { - return cluster->num_cpus - - sched_pause_count(&cluster->cpu_mask, true); + cpumask_t cpus; + + cpumask_and(&cpus, &cluster->cpu_mask, cpu_active_mask); + return cpumask_weight(&cpus); } static bool is_active(const struct cpu_data *state) @@ -981,14 +983,11 @@ void core_ctl_check(u64 window_start) core_ctl_call_notifier(); } +/* must be called with state_lock held */ static void move_cpu_lru(struct cpu_data *cpu_data) { - unsigned long flags; - - spin_lock_irqsave(&state_lock, flags); list_del(&cpu_data->sib); list_add_tail(&cpu_data->sib, &cpu_data->cluster->lru); - spin_unlock_irqrestore(&state_lock, flags); } static void try_to_pause(struct cluster_data *cluster, unsigned int need, @@ -997,7 +996,7 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, struct cpu_data *c, *tmp; unsigned long flags; unsigned int num_cpus = cluster->num_cpus; - unsigned int nr_paused = 0; + unsigned int nr_pending = 0, active_cpus = cluster->active_cpus; bool first_pass = cluster->nr_not_preferred_cpus; /* @@ -1011,7 +1010,7 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, if (!is_active(c)) continue; - if (cluster->active_cpus == need) + if (active_cpus - nr_pending == need) break; /* Don't pause busy CPUs. */ if (c->is_busy) @@ -1024,75 +1023,57 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, if (cluster->nr_not_preferred_cpus && !c->not_preferred) continue; - spin_unlock_irqrestore(&state_lock, flags); - pr_debug("Trying to pause CPU%u\n", c->cpu); - cpumask_set_cpu(c->cpu, pause_cpus); - sched_pause_pending(c->cpu); - + nr_pending++; c->paused_by_us = true; + cluster->nr_paused_cpus++; move_cpu_lru(c); - nr_paused++; - - cluster->active_cpus = get_active_cpu_count(cluster); - spin_lock_irqsave(&state_lock, flags); } - cluster->nr_paused_cpus += nr_paused; - spin_unlock_irqrestore(&state_lock, flags); again: /* * If the number of active CPUs is within the limits, then * don't force pause of any busy CPUs. */ - if (cluster->active_cpus <= cluster->max_cpus) - return; + if (active_cpus - nr_pending <= cluster->max_cpus) + goto unlock; - nr_paused = 0; num_cpus = cluster->num_cpus; - spin_lock_irqsave(&state_lock, flags); list_for_each_entry_safe(c, tmp, &cluster->lru, sib) { if (!num_cpus--) break; if (!is_active(c)) continue; - if (cluster->active_cpus <= cluster->max_cpus) + if (active_cpus - nr_pending <= cluster->max_cpus) break; if (first_pass && !c->not_preferred) continue; - spin_unlock_irqrestore(&state_lock, flags); - cpumask_set_cpu(c->cpu, pause_cpus); - sched_pause_pending(c->cpu); - + nr_pending++; c->paused_by_us = true; + cluster->nr_paused_cpus++; move_cpu_lru(c); - nr_paused++; - - cluster->active_cpus = get_active_cpu_count(cluster); - spin_lock_irqsave(&state_lock, flags); } - cluster->nr_paused_cpus += nr_paused; - spin_unlock_irqrestore(&state_lock, flags); - - if (first_pass && cluster->active_cpus > cluster->max_cpus) { + if (first_pass && active_cpus - nr_pending > cluster->max_cpus) { first_pass = false; goto again; } +unlock: + spin_unlock_irqrestore(&state_lock, flags); } -static void __try_to_resume(struct cluster_data *cluster, - unsigned int need, bool force, struct cpumask *unpause_cpus) +static int __try_to_resume(struct cluster_data *cluster, unsigned int need, + bool force, struct cpumask *unpause_cpus) { struct cpu_data *c, *tmp; unsigned long flags; unsigned int num_cpus = cluster->num_cpus; - unsigned int nr_unpaused = 0; + unsigned int nr_pending = 0, active_cpus = cluster->active_cpus; /* * Protect against entry being removed (and added at tail) by other @@ -1108,35 +1089,38 @@ static void __try_to_resume(struct cluster_data *cluster, if ((cpu_online(c->cpu) && cpu_active(c->cpu)) || (!force && c->not_preferred)) continue; - if (cluster->active_cpus == need) + if (active_cpus + nr_pending == need) break; - spin_unlock_irqrestore(&state_lock, flags); - pr_debug("Trying to resume CPU%u\n", c->cpu); cpumask_set_cpu(c->cpu, unpause_cpus); - sched_unpause_pending(c->cpu); - + nr_pending++; c->paused_by_us = false; + cluster->nr_paused_cpus--; move_cpu_lru(c); - nr_unpaused++; - - cluster->active_cpus = get_active_cpu_count(cluster); - spin_lock_irqsave(&state_lock, flags); } - cluster->nr_paused_cpus -= nr_unpaused; + spin_unlock_irqrestore(&state_lock, flags); + + return nr_pending; } static void try_to_resume(struct cluster_data *cluster, unsigned int need, struct cpumask *unpause_cpus) { bool force_use_non_preferred = false; + unsigned int nr_pending; - __try_to_resume(cluster, need, force_use_non_preferred, unpause_cpus); + /* + * __try_to_resume() marks the CPUs to be resumed but active_cpus + * won't be reflected yet. So use the nr_pending to adjust active + * count. + */ + nr_pending = __try_to_resume(cluster, need, force_use_non_preferred, + unpause_cpus); - if (cluster->active_cpus == need) + if (cluster->active_cpus + nr_pending == need) return; force_use_non_preferred = true; @@ -1153,6 +1137,7 @@ static void __ref do_core_ctl(void) for_each_cluster(cluster, index) { + cluster->active_cpus = get_active_cpu_count(cluster); need = apply_limits(cluster, cluster->need_cpus); if (adjustment_possible(cluster, need)) { diff --git a/kernel/sched/walt/qc_vas.c b/kernel/sched/walt/qc_vas.c deleted file mode 100644 index 9db134f8363f..000000000000 --- a/kernel/sched/walt/qc_vas.c +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. - */ - -#include -#include -#include -#include - -#include "walt.h" - -#ifdef CONFIG_HOTPLUG_CPU - -cpumask_t pending_active_mask = CPU_MASK_NONE; -int sched_pause_count(const cpumask_t *mask, bool include_offline) -{ - cpumask_t count_mask = CPU_MASK_NONE; - cpumask_t pause_mask = CPU_MASK_NONE; - - if (cpumask_any(&pending_active_mask) >= nr_cpu_ids) { - /* initialize pending_active_state */ - cpumask_copy(&pending_active_mask, cpu_active_mask); - } - - if (include_offline) { - - /* get all offline or paused cpus */ - cpumask_complement(&pause_mask, &pending_active_mask); - cpumask_complement(&count_mask, cpu_online_mask); - cpumask_or(&count_mask, &count_mask, &pause_mask); - - /* get all offline or paused cpus in this cluster */ - cpumask_and(&count_mask, &count_mask, mask); - } else { - cpumask_andnot(&count_mask, mask, &pending_active_mask); - } - - return cpumask_weight(&count_mask); -} - -void sched_pause_pending(int cpu) -{ - cpumask_clear_cpu(cpu, &pending_active_mask); -} - -void sched_unpause_pending(int cpu) -{ - cpumask_set_cpu(cpu, &pending_active_mask); -} - -#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index a423fbf649ca..f1c11dded20c 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -254,9 +254,6 @@ extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct); extern u32 sched_get_init_task_load(struct task_struct *p); extern void core_ctl_check(u64 wallclock); extern int sched_set_boost(int enable); -extern int sched_pause_count(const cpumask_t *mask, bool include_offline); -extern void sched_pause_pending(int cpu); -extern void sched_unpause_pending(int cpu); extern int sched_wake_up_idle_show(struct seq_file *m, void *v); extern ssize_t sched_wake_up_idle_write(struct file *file, const char __user *buf, size_t count, loff_t *offset); From f3af9072ef3997bd00eebf8644a7bfa6a623245c Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Tue, 2 Feb 2021 12:36:44 +0530 Subject: [PATCH 023/346] sched/walt: Remove references to schedutil flags Schedutil flags are no longer used in WALT as we moved to waltgov cpufreq governor. So remove them. Change-Id: Iedc68215ed0e994b8e051b282a575ac699ebf7e0 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index f1c11dded20c..92531e06000b 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -29,13 +29,6 @@ #define NR_WINDOWS_PER_SEC (NSEC_PER_SEC / DEFAULT_SCHED_RAVG_WINDOW) -#define SCHED_CPUFREQ_MIGRATION (1U << 1) -#define SCHED_CPUFREQ_INTERCLUSTER_MIG (1U << 3) -#define SCHED_CPUFREQ_WALT (1U << 4) -#define SCHED_CPUFREQ_PL (1U << 5) -#define SCHED_CPUFREQ_EARLY_DET (1U << 6) -#define SCHED_CPUFREQ_CONTINUE (1U << 8) - /* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */ #define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1) From 6cab0b20dd5524a2ffdcaf1f5d08b8ca72c2f0a4 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Tue, 2 Feb 2021 12:02:59 +0530 Subject: [PATCH 024/346] sched/walt: Update cpufreq_driver_fast_switch() invocation cpu_frequency trace events are coming twice when a frequency switch happens. This is due to commit 08d8c65e849d ("cpufreq: Move traces and update to policy->cur to cpufreq core") in the upstream which takes care of cpu_frequency trace event and updates to policy->cur when cpufreq_driver_fast_switch() is called. So remove the redundant code from waltgov cpufreq governor to avoid duplicate cpu_frequency trave events. Change-Id: I490e766abcc09ed2a580e5f09930a54d76367999 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/cpufreq_walt.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 3d24cf3fd1e7..68de7690ac66 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -180,22 +180,12 @@ static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time, unsigned int next_freq) { struct cpufreq_policy *policy = wg_policy->policy; - unsigned int cpu; if (!waltgov_update_next_freq(wg_policy, time, next_freq)) return; waltgov_track_cycles(wg_policy, wg_policy->policy->cur, time); - next_freq = cpufreq_driver_fast_switch(policy, next_freq); - if (!next_freq) - return; - - policy->cur = next_freq; - - if (trace_cpu_frequency_enabled()) { - for_each_cpu(cpu, policy->cpus) - trace_cpu_frequency(next_freq, cpu); - } + cpufreq_driver_fast_switch(policy, next_freq); } static void waltgov_deferred_update(struct waltgov_policy *wg_policy, u64 time, From 22b44e45c745fcfa721e79186dff3a386ae63cfb Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Tue, 2 Feb 2021 12:13:39 +0530 Subject: [PATCH 025/346] sched/walt: Remove sched_load_boost related code The sched_load_boost feature will be introduced in a different form in the subsequent patch. Change-Id: Ib528ccd0fcb78f85685399264cb58212752ca26b Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/sysctl.c | 51 ------------------------------------ kernel/sched/walt/walt.c | 25 ++++-------------- kernel/sched/walt/walt.h | 13 --------- kernel/sched/walt/walt_cfs.c | 1 - 4 files changed, 5 insertions(+), 85 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 83ad94ca4536..d55ea70184fc 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -40,7 +40,6 @@ unsigned int sysctl_input_boost_ms; unsigned int sysctl_input_boost_freq[8]; unsigned int sysctl_sched_boost_on_input; unsigned int sysctl_sched_init_stage; -unsigned int sysctl_sched_load_boost[WALT_NR_CPUS]; /* sysctl nodes accesed by other files */ unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; @@ -352,49 +351,6 @@ static int sched_task_handler(struct ctl_table *table, int write, return ret; } -static int sched_load_boost_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, - loff_t *ppos) -{ - int ret, i; - unsigned int *data = (unsigned int *)table->data; - int val[WALT_NR_CPUS]; - - struct ctl_table tmp = { - .data = &val, - .maxlen = sizeof(val), - .mode = table->mode, - }; - static DEFINE_MUTEX(mutex); - - mutex_lock(&mutex); - - if (!write) { - ret = proc_dointvec(table, write, buffer, lenp, ppos); - goto unlock_mutex; - } - - ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); - if (ret) - goto unlock_mutex; - - for (i = 0; i < WALT_NR_CPUS; i++) { - if (val[i] < -100 || val[i] > 1000) { - ret = -EINVAL; - goto unlock_mutex; - } - } - - /* all things checkout update the value */ - for (i = 0; i < WALT_NR_CPUS; i++) - data[i] = val[i]; - -unlock_mutex: - mutex_unlock(&mutex); - - return ret; -} - #ifdef CONFIG_PROC_SYSCTL static void sched_update_updown_migrate_values(bool up) { @@ -849,13 +805,6 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_INT_MAX, }, - { - .procname = "sched_load_boost", - .data = &sysctl_sched_load_boost, - .maxlen = sizeof(unsigned int) * 8, - .mode = 0644, - .proc_handler = sched_load_boost_handler, - }, { } }; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index baa9b8427c0a..e6a9f6ddea37 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -172,12 +172,6 @@ __read_mostly unsigned int new_sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; static DEFINE_SPINLOCK(sched_ravg_window_lock); u64 sched_ravg_window_change_time; -/* - * A after-boot constant divisor for cpu_util_freq_walt() to apply the load - * boost. - */ -static __read_mostly unsigned int walt_cpu_util_freq_divisor; - unsigned int __read_mostly sched_init_task_load_windows_scaled; unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; @@ -616,30 +610,23 @@ static bool rtgb_active; static inline unsigned long __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) { - u64 util, util_unboosted; + u64 util; struct rq *rq = cpu_rq(cpu); unsigned long capacity = capacity_orig_of(cpu); - int boost; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - boost = sysctl_sched_load_boost[cpu]; - util_unboosted = util = freq_policy_load(rq); - util = div64_u64(util * (100 + boost), - walt_cpu_util_freq_divisor); + util = div64_u64(freq_policy_load(rq), + sched_ravg_window >> SCHED_CAPACITY_SHIFT); if (walt_load) { u64 nl = wrq->nt_prev_runnable_sum + wrq->grp_time.nt_prev_runnable_sum; u64 pl = wrq->walt_stats.pred_demands_sum_scaled; - /* do_pl_notif() needs unboosted signals */ - wrq->old_busy_time = div64_u64(util_unboosted, - sched_ravg_window >> - SCHED_CAPACITY_SHIFT); + wrq->old_busy_time = util; wrq->old_estimated_time = pl; - nl = div64_u64(nl * (100 + boost), walt_cpu_util_freq_divisor); - + nl = div64_u64(nl, sched_ravg_window >> SCHED_CAPACITY_SHIFT); walt_load->nl = nl; walt_load->pl = pl; walt_load->ws = walt_load_reported_window; @@ -3586,8 +3573,6 @@ void walt_fill_ta_data(struct core_ctl_notif_data *data) static void walt_init_window_dep(void) { - walt_cpu_util_freq_divisor = - (sched_ravg_window >> SCHED_CAPACITY_SHIFT) * 100; walt_scale_demand_divisor = sched_ravg_window >> SCHED_CAPACITY_SHIFT; sched_init_task_load_windows = diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 92531e06000b..81413e9d26cb 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -313,7 +313,6 @@ extern enum sched_boost_policy boost_policy; extern unsigned int sysctl_input_boost_ms; extern unsigned int sysctl_input_boost_freq[8]; extern unsigned int sysctl_sched_boost_on_input; -extern unsigned int sysctl_sched_load_boost[WALT_NR_CPUS]; extern unsigned int sysctl_sched_user_hint; extern unsigned int sysctl_sched_conservative_pl; #define WALT_MANY_WAKEUP_DEFAULT 1000 @@ -508,18 +507,6 @@ static inline unsigned long cpu_util_cum(int cpu, int delta) return (delta >= capacity) ? capacity : delta; } -extern unsigned int capacity_margin_freq; - -static inline unsigned long -add_capacity_margin(unsigned long cpu_capacity, int cpu) -{ - cpu_capacity = cpu_capacity * capacity_margin_freq * - (100 + sysctl_sched_load_boost[cpu]); - cpu_capacity /= 100; - cpu_capacity /= SCHED_CAPACITY_SCALE; - return cpu_capacity; -} - static inline enum sched_boost_policy sched_boost_policy(void) { return boost_policy; diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 9dbc9249ecfc..940c3f99e784 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -22,7 +22,6 @@ __read_mostly unsigned int sysctl_sched_prefer_spread; unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ unsigned int sched_small_task_threshold = 102; __read_mostly unsigned int sysctl_sched_force_lb_enable = 1; -unsigned int capacity_margin_freq = 1280; /* ~20% margin */ static inline bool prefer_spread_on_idle(int cpu, bool new_ilb) { From 3123055177f643eab2e1fbcdb7972daf25e13844 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Tue, 2 Feb 2021 12:43:57 +0530 Subject: [PATCH 026/346] sched/walt: Introduce waltgov boost feature Re-introduce the earlier sched_load_boost feature as waltgov's boost feature. A tunable called "boost" is exposed per cpufreq policy which can be used to boost (either positive or negative) the utilization, thus the selected frequency. Change-Id: I3a41a985d449e5c753b5afcbc375749e67036089 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/cpufreq_walt.c | 66 ++++++++++++++++++++++++++------ kernel/sched/walt/walt.h | 1 + 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 68de7690ac66..e729503e3d0c 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -23,6 +23,7 @@ struct waltgov_tunables { unsigned int hispeed_freq; unsigned int rtg_boost_freq; bool pl; + int boost; }; struct waltgov_policy { @@ -236,14 +237,13 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) #define DEFAULT_CPU0_RTG_BOOST_FREQ 1000000 #define DEFAULT_CPU4_RTG_BOOST_FREQ 0 #define DEFAULT_CPU7_RTG_BOOST_FREQ 0 -static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long *util, - unsigned long *max) +static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_util, + unsigned long nl, unsigned long *util, + unsigned long *max) { struct waltgov_policy *wg_policy = wg_cpu->wg_policy; bool is_migration = wg_cpu->flags & WALT_CPUFREQ_IC_MIGRATION; bool is_rtg_boost = wg_cpu->walt_load.rtgb_active; - unsigned long nl = wg_cpu->walt_load.nl; - unsigned long cpu_util = wg_cpu->util; bool is_hiload; unsigned long pl = wg_cpu->walt_load.pl; @@ -283,10 +283,11 @@ static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 tim struct cpufreq_policy *policy = wg_policy->policy; unsigned long util = 0, max = 1; unsigned int j; + int boost = wg_policy->tunables->boost; for_each_cpu(j, policy->cpus) { struct waltgov_cpu *j_wg_cpu = &per_cpu(waltgov_cpu, j); - unsigned long j_util, j_max; + unsigned long j_util, j_max, j_nl; /* * If the util value for all CPUs in a policy is 0, just using > @@ -296,14 +297,19 @@ static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 tim * leading to spurious jumps to fmax. */ j_util = j_wg_cpu->util; + j_nl = j_wg_cpu->walt_load.nl; j_max = j_wg_cpu->max; + if (boost) { + j_util = mult_frac(j_util, boost + 100, 100); + j_nl = mult_frac(j_nl, boost + 100, 100); + } if (j_util * max >= j_max * util) { util = j_util; max = j_max; } - waltgov_walt_adjust(j_wg_cpu, &util, &max); + waltgov_walt_adjust(j_wg_cpu, j_util, j_nl, &util, &max); } return get_next_freq(wg_policy, util, max); @@ -314,7 +320,7 @@ static void waltgov_update_freq(struct waltgov_callback *cb, u64 time, { struct waltgov_cpu *wg_cpu = container_of(cb, struct waltgov_cpu, cb); struct waltgov_policy *wg_policy = wg_cpu->wg_policy; - unsigned long hs_util, boost_util; + unsigned long hs_util, rtg_boost_util; unsigned int next_f; if (!wg_policy->tunables->pl && flags & WALT_CPUFREQ_PL) @@ -330,9 +336,9 @@ static void waltgov_update_freq(struct waltgov_callback *cb, u64 time, wg_policy->tunables->hispeed_freq); wg_policy->hispeed_util = hs_util; - boost_util = target_util(wg_policy, + rtg_boost_util = target_util(wg_policy, wg_policy->tunables->rtg_boost_freq); - wg_policy->rtg_boost_util = boost_util; + wg_policy->rtg_boost_util = rtg_boost_util; } waltgov_calc_avg_cap(wg_policy, wg_cpu->walt_load.ws, @@ -520,7 +526,7 @@ static ssize_t rtg_boost_freq_store(struct gov_attr_set *attr_set, struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); unsigned int val; struct waltgov_policy *wg_policy; - unsigned long boost_util; + unsigned long rtg_boost_util; unsigned long flags; if (kstrtouint(buf, 10, &val)) @@ -529,9 +535,9 @@ static ssize_t rtg_boost_freq_store(struct gov_attr_set *attr_set, tunables->rtg_boost_freq = val; list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { raw_spin_lock_irqsave(&wg_policy->update_lock, flags); - boost_util = target_util(wg_policy, + rtg_boost_util = target_util(wg_policy, wg_policy->tunables->rtg_boost_freq); - wg_policy->rtg_boost_util = boost_util; + wg_policy->rtg_boost_util = rtg_boost_util; raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); } @@ -556,10 +562,43 @@ static ssize_t pl_store(struct gov_attr_set *attr_set, const char *buf, return count; } +static ssize_t boost_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%d\n", tunables->boost); +} + +static ssize_t boost_store(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + struct waltgov_policy *wg_policy; + int val; + + if (kstrtoint(buf, 10, &val)) + return -EINVAL; + + if (val < -100 || val > 1000) + return -EINVAL; + + tunables->boost = val; + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + struct rq *rq = cpu_rq(wg_policy->policy->cpu); + unsigned long flags; + + raw_spin_lock_irqsave(&rq->lock, flags); + waltgov_run_callback(rq, WALT_CPUFREQ_BOOST_UPDATE); + raw_spin_unlock_irqrestore(&rq->lock, flags); + } + return count; +} + static struct governor_attr hispeed_load = __ATTR_RW(hispeed_load); static struct governor_attr hispeed_freq = __ATTR_RW(hispeed_freq); static struct governor_attr rtg_boost_freq = __ATTR_RW(rtg_boost_freq); static struct governor_attr pl = __ATTR_RW(pl); +static struct governor_attr boost = __ATTR_RW(boost); static struct attribute *waltgov_attributes[] = { &up_rate_limit_us.attr, @@ -568,6 +607,7 @@ static struct attribute *waltgov_attributes[] = { &hispeed_freq.attr, &rtg_boost_freq.attr, &pl.attr, + &boost.attr, NULL }; @@ -668,6 +708,7 @@ static void waltgov_tunables_save(struct cpufreq_policy *policy, cached->hispeed_freq = tunables->hispeed_freq; cached->up_rate_limit_us = tunables->up_rate_limit_us; cached->down_rate_limit_us = tunables->down_rate_limit_us; + cached->boost = tunables->boost; } static void waltgov_tunables_restore(struct cpufreq_policy *policy) @@ -685,6 +726,7 @@ static void waltgov_tunables_restore(struct cpufreq_policy *policy) tunables->hispeed_freq = cached->hispeed_freq; tunables->up_rate_limit_us = cached->up_rate_limit_us; tunables->down_rate_limit_us = cached->down_rate_limit_us; + tunables->boost = cached->boost; } static int waltgov_init(struct cpufreq_policy *policy) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 81413e9d26cb..14cf680953ee 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -391,6 +391,7 @@ void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy, #define WALT_CPUFREQ_IC_MIGRATION (1U << 2) #define WALT_CPUFREQ_PL (1U << 3) #define WALT_CPUFREQ_EARLY_DET (1U << 4) +#define WALT_CPUFREQ_BOOST_UPDATE (1U << 5) #define NO_BOOST 0 #define FULL_THROTTLE_BOOST 1 From 058346c758256f6cf9d392e09d26335530d7c4a4 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Wed, 3 Feb 2021 19:23:15 +0530 Subject: [PATCH 027/346] sched/walt: Export set_task_boost() API set_task_boost() API is meant for other modules, so export this function. Also move the function declaration to appropriate header. Change-Id: I7213b137a0763140c631bea30653321674fef272 Signed-off-by: Pavankumar Kondeti --- include/linux/sched/walt.h | 5 +++++ kernel/sched/walt/walt.c | 1 + kernel/sched/walt/walt.h | 2 -- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index f3fdd4da27e8..1684c50ef544 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -10,11 +10,16 @@ #if IS_ENABLED(CONFIG_SCHED_WALT) extern int sched_lpm_disallowed_time(int cpu, u64 *timeout); +extern int set_task_boost(int boost, u64 period); #else static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout) { return INT_MAX; } +static inline int set_task_boost(int boost, u64 period) +{ + return 0; +} #endif #endif /* _LINUX_SCHED_WALT_H */ diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index e6a9f6ddea37..706feba887f6 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -102,6 +102,7 @@ int set_task_boost(int boost, u64 period) } return 0; } +EXPORT_SYMBOL(set_task_boost); u64 sched_ktime_clock(void) { diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 14cf680953ee..9f2c0f4c9c01 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -970,8 +970,6 @@ static inline void set_wake_up_idle(bool enabled) wts->wake_up_idle = enabled; } -extern int set_task_boost(int boost, u64 period); - static inline struct task_group *css_tg(struct cgroup_subsys_state *css) { return css ? container_of(css, struct task_group, css) : NULL; From bcb58635218cadd4336e239dccb84daae8449bfd Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Wed, 3 Feb 2021 19:33:58 +0530 Subject: [PATCH 028/346] sched/walt: Make walt_task_struct struct public The walt_task_struct structure is used by WALT module. WALT type cast the task_struct's android vendor data to walt_task_struct and use it for the scheduler needs. However there can be other users who wants to cache information per task. Currently walt_task_struct is defined in WALT's private header and can't be included by other modules. Move the walt_task_struct and dependent structures to appropriate header file so that is becomes public. This movement also allows wake_up_idle related funcitons to define in the public header and keep the inline functionality instead of exporting them for the other modules usage. Change-Id: If8c94c7d6cd709bbc56aa08eb964d135a492a530 Signed-off-by: Pavankumar Kondeti --- include/linux/sched/walt.h | 85 ++++++++++++++++++++++++++++++++++++++ kernel/sched/walt/walt.h | 83 +------------------------------------ 2 files changed, 86 insertions(+), 82 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 1684c50ef544..9f8d2d437c3e 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -7,8 +7,93 @@ #define _LINUX_SCHED_WALT_H #include +#include +#include #if IS_ENABLED(CONFIG_SCHED_WALT) + +#define WALT_NR_CPUS 8 +#define RAVG_HIST_SIZE_MAX 5 +#define NUM_BUSY_BUCKETS 10 + +struct walt_related_thread_group { + int id; + raw_spinlock_t lock; + struct list_head tasks; + struct list_head list; + bool skip_min; + struct rcu_head rcu; + u64 last_update; + u64 downmigrate_ts; + u64 start_ts; +}; + +struct walt_task_struct { + /* + * 'mark_start' marks the beginning of an event (task waking up, task + * starting to execute, task being preempted) within a window + * + * 'sum' represents how runnable a task has been within current + * window. It incorporates both running time and wait time and is + * frequency scaled. + * + * 'sum_history' keeps track of history of 'sum' seen over previous + * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are + * ignored. + * + * 'demand' represents maximum sum seen over previous + * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency + * demand for tasks. + * + * 'curr_window_cpu' represents task's contribution to cpu busy time on + * various CPUs in the current window + * + * 'prev_window_cpu' represents task's contribution to cpu busy time on + * various CPUs in the previous window + * + * 'curr_window' represents the sum of all entries in curr_window_cpu + * + * 'prev_window' represents the sum of all entries in prev_window_cpu + * + * 'pred_demand' represents task's current predicted cpu busy time + * + * 'busy_buckets' groups historical busy time into different buckets + * used for prediction + * + * 'demand_scaled' represents task's demand scaled to 1024 + */ + u64 mark_start; + u32 sum, demand; + u32 coloc_demand; + u32 sum_history[RAVG_HIST_SIZE_MAX]; + u32 curr_window_cpu[WALT_NR_CPUS]; + u32 prev_window_cpu[WALT_NR_CPUS]; + u32 curr_window, prev_window; + u32 pred_demand; + u8 busy_buckets[NUM_BUSY_BUCKETS]; + u16 demand_scaled; + u16 pred_demand_scaled; + u64 active_time; + u64 last_win_size; + int boost; + bool wake_up_idle; + bool misfit; + bool rtg_high_prio; + u8 low_latency; + u64 boost_period; + u64 boost_expires; + u64 last_sleep_ts; + u32 init_load_pct; + u32 unfilter; + u64 last_wake_ts; + u64 last_enqueued_ts; + struct walt_related_thread_group __rcu *grp; + struct list_head grp_list; + u64 cpu_cycles; + cpumask_t cpus_requested; + bool iowaited; +}; + extern int sched_lpm_disallowed_time(int cpu, u64 *timeout); extern int set_task_boost(int boost, u64 period); #else diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 9f2c0f4c9c01..169095fb802c 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -8,6 +8,7 @@ #include "../../../kernel/sched/sched.h" #include "../../../fs/proc/internal.h" +#include #include #include @@ -57,79 +58,9 @@ enum task_boost_type { TASK_BOOST_END, }; -#define WALT_NR_CPUS 8 -#define RAVG_HIST_SIZE_MAX 5 -#define NUM_BUSY_BUCKETS 10 - #define WALT_LOW_LATENCY_PROCFS BIT(0) #define WALT_LOW_LATENCY_BINDER BIT(1) -struct walt_task_struct { - /* - * 'mark_start' marks the beginning of an event (task waking up, task - * starting to execute, task being preempted) within a window - * - * 'sum' represents how runnable a task has been within current - * window. It incorporates both running time and wait time and is - * frequency scaled. - * - * 'sum_history' keeps track of history of 'sum' seen over previous - * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are - * ignored. - * - * 'demand' represents maximum sum seen over previous - * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency - * demand for tasks. - * - * 'curr_window_cpu' represents task's contribution to cpu busy time on - * various CPUs in the current window - * - * 'prev_window_cpu' represents task's contribution to cpu busy time on - * various CPUs in the previous window - * - * 'curr_window' represents the sum of all entries in curr_window_cpu - * - * 'prev_window' represents the sum of all entries in prev_window_cpu - * - * 'pred_demand' represents task's current predicted cpu busy time - * - * 'busy_buckets' groups historical busy time into different buckets - * used for prediction - * - * 'demand_scaled' represents task's demand scaled to 1024 - */ - u64 mark_start; - u32 sum, demand; - u32 coloc_demand; - u32 sum_history[RAVG_HIST_SIZE_MAX]; - u32 curr_window_cpu[WALT_NR_CPUS]; - u32 prev_window_cpu[WALT_NR_CPUS]; - u32 curr_window, prev_window; - u32 pred_demand; - u8 busy_buckets[NUM_BUSY_BUCKETS]; - u16 demand_scaled; - u16 pred_demand_scaled; - u64 active_time; - u64 last_win_size; - int boost; - bool wake_up_idle; - bool misfit; - bool rtg_high_prio; - u8 low_latency; - u64 boost_period; - u64 boost_expires; - u64 last_sleep_ts; - u32 init_load_pct; - u32 unfilter; - u64 last_wake_ts; - u64 last_enqueued_ts; - struct walt_related_thread_group __rcu *grp; - struct list_head grp_list; - u64 cpu_cycles; - cpumask_t cpus_requested; - bool iowaited; -}; - struct walt_cpu_load { unsigned long nl; unsigned long pl; @@ -212,18 +143,6 @@ struct walt_sched_cluster { u64 aggr_grp_load; }; -struct walt_related_thread_group { - int id; - raw_spinlock_t lock; - struct list_head tasks; - struct list_head list; - bool skip_min; - struct rcu_head rcu; - u64 last_update; - u64 downmigrate_ts; - u64 start_ts; -}; - extern struct walt_sched_cluster *sched_cluster[WALT_NR_CPUS]; extern struct walt_sched_cluster *rq_cluster(struct rq *rq); From 6599bdd0dc68760ef6dfcb02ee0a75843165bdbd Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Wed, 3 Feb 2021 19:48:03 +0530 Subject: [PATCH 029/346] sched/walt: Make wake_up_idle related functions public wake_up_idle related functions are meant for other modules usage. Make them public by moving to appropriate header. Change-Id: I888b511e656d6c160c98b5da44bc873573a9beed Signed-off-by: Pavankumar Kondeti --- include/linux/sched/walt.h | 36 ++++++++++++++++++++++++++++++++++++ kernel/sched/walt/walt.h | 24 ------------------------ 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 9f8d2d437c3e..f2f69acb206e 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -94,6 +94,28 @@ struct walt_task_struct { bool iowaited; }; +static inline bool sched_get_wake_up_idle(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->wake_up_idle; +} + +static inline int sched_set_wake_up_idle(struct task_struct *p, bool wake_up_idle) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->wake_up_idle = wake_up_idle; + return 0; +} + +static inline void set_wake_up_idle(bool wake_up_idle) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + + wts->wake_up_idle = wake_up_idle; +} + extern int sched_lpm_disallowed_time(int cpu, u64 *timeout); extern int set_task_boost(int boost, u64 period); #else @@ -105,6 +127,20 @@ static inline int set_task_boost(int boost, u64 period) { return 0; } + +static inline bool sched_get_wake_up_idle(struct task_struct *p) +{ + return false; +} + +static inline int sched_set_wake_up_idle(struct task_struct *p, bool wake_up_idle) +{ + return 0; +} + +static inline void set_wake_up_idle(bool wake_up_idle) +{ +} #endif #endif /* _LINUX_SCHED_WALT_H */ diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 169095fb802c..087abb6d400a 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -865,30 +865,6 @@ static inline void walt_irq_work_queue(struct irq_work *work) irq_work_queue_on(work, cpumask_any(cpu_online_mask)); } -#define PF_WAKE_UP_IDLE 1 -static inline u32 sched_get_wake_up_idle(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return wts->wake_up_idle; -} - -static inline int sched_set_wake_up_idle(struct task_struct *p, - int wake_up_idle) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - wts->wake_up_idle = !!wake_up_idle; - return 0; -} - -static inline void set_wake_up_idle(bool enabled) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; - - wts->wake_up_idle = enabled; -} - static inline struct task_group *css_tg(struct cgroup_subsys_state *css) { return css ? container_of(css, struct task_group, css) : NULL; From 42840f1cc51142be777e0d888879313a99b6c3f6 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Wed, 3 Feb 2021 03:49:23 -0800 Subject: [PATCH 030/346] sched/walt: remove unused functions and variables remove unused functions and variables. Change-Id: I785674f9dba44ec02779a00651fb1eac0ec1cff1 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/sysctl.c | 16 +--------------- kernel/sched/walt/walt.h | 1 - kernel/sched/walt/walt_cfs.c | 19 ------------------- 3 files changed, 1 insertion(+), 35 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index d55ea70184fc..0f27133c7a6b 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -57,6 +57,7 @@ unsigned int sysctl_sched_min_task_util_for_boost = 51; unsigned int sysctl_sched_min_task_util_for_colocation = 35; unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; const int sched_user_hint_max = 1000; +unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -697,15 +698,6 @@ struct ctl_table walt_table[] = { .mode = 0644, .proc_handler = sched_updown_migrate_handler, }, - { - .procname = "sched_prefer_spread", - .data = &sysctl_sched_prefer_spread, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = &four, - }, { .procname = "walt_rtg_cfs_boost_prio", .data = &sysctl_walt_rtg_cfs_boost_prio, @@ -842,10 +834,6 @@ void walt_tunables(void) sched_load_granule = DEFAULT_SCHED_RAVG_WINDOW / NUM_LOAD_INDICES; - sysctl_sched_min_task_util_for_boost = 51; - - sysctl_sched_min_task_util_for_colocation = 35; - for (i = 0; i < WALT_NR_CPUS; i++) { sysctl_sched_coloc_busy_hyst_cpu[i] = 39000000; sysctl_sched_coloc_busy_hyst_cpu_busy_pct[i] = 10; @@ -855,8 +843,6 @@ void walt_tunables(void) sysctl_sched_coloc_busy_hyst_max_ms = 5000; - sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ - sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; sysctl_input_boost_ms = 40; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 087abb6d400a..dfb2e8f5826d 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -239,7 +239,6 @@ extern unsigned int sysctl_sched_many_wakeup_threshold; extern unsigned int sysctl_walt_rtg_cfs_boost_prio; extern __read_mostly unsigned int sysctl_sched_force_lb_enable; extern const int sched_user_hint_max; -extern unsigned int sysctl_sched_prefer_spread; #define for_each_sched_cluster(cluster) \ list_for_each_entry_rcu(cluster, &cluster_head, list) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 940c3f99e784..78c45a3aa058 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -18,27 +18,8 @@ unsigned int sched_capacity_margin_down[WALT_NR_CPUS] = { [0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */ }; -__read_mostly unsigned int sysctl_sched_prefer_spread; -unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ -unsigned int sched_small_task_threshold = 102; __read_mostly unsigned int sysctl_sched_force_lb_enable = 1; -static inline bool prefer_spread_on_idle(int cpu, bool new_ilb) -{ - switch (sysctl_sched_prefer_spread) { - case 1: - return is_min_capacity_cpu(cpu); - case 2: - return true; - case 3: - return (new_ilb && is_min_capacity_cpu(cpu)); - case 4: - return new_ilb; - default: - return false; - } -} - static inline bool bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) { From d90415f3745b474d207537c901b67ab6fd7bfa96 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:41:04 -0700 Subject: [PATCH 031/346] sched/walt: Move public declarations of core_ctl.h to walt.h core_ctl is part of WALT. There is no separate config for core_ctl. So it makes sense to declare all the WALT related public interfaces in a single header file. Change-Id: I0ae5585c0037295eee4268705f0e553d14c60fd9 Signed-off-by: Pavankumar Kondeti Signed-off-by: Sai Harshini Nimmala --- drivers/soc/qcom/msm_performance.c | 1231 ++++++++++++++++++++++++++++ include/linux/sched/core_ctl.h | 36 - include/linux/sched/walt.h | 29 + kernel/sched/walt/boost.c | 1 - kernel/sched/walt/core_ctl.c | 1 - kernel/sched/walt/walt.h | 1 - 6 files changed, 1260 insertions(+), 39 deletions(-) create mode 100644 drivers/soc/qcom/msm_performance.c delete mode 100644 include/linux/sched/core_ctl.h diff --git a/drivers/soc/qcom/msm_performance.c b/drivers/soc/qcom/msm_performance.c new file mode 100644 index 000000000000..9acd80761e53 --- /dev/null +++ b/drivers/soc/qcom/msm_performance.c @@ -0,0 +1,1231 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define POLL_INT 25 +#define NODE_NAME_MAX_CHARS 16 + +#define QUEUE_POOL_SIZE 512 /*2^8 always keep in 2^x */ +#define INST_EV 0x08 /* 0th event*/ +#define CYC_EV 0x11 /* 1st event*/ +#define INIT "Init" +#define CPU_CYCLE_THRESHOLD 650000 + + +static DEFINE_PER_CPU(bool, cpu_is_idle); +static DEFINE_PER_CPU(bool, cpu_is_hp); +static DEFINE_MUTEX(perfevent_lock); + +enum event_idx { + INST_EVENT, + CYC_EVENT, + NO_OF_EVENT +}; + +enum cpu_clusters { + MIN = 0, + MID = 1, + MAX = 2, + CLUSTER_MAX +}; + +static struct kset *msm_perf_kset; +static struct kobject *param_kobj; + +static ssize_t get_cpu_min_freq(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_cpu_min_freq(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +static ssize_t get_cpu_max_freq(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_cpu_max_freq(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +static ssize_t get_cpu_total_instruction(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t get_core_ctl_register(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_core_ctl_register(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +static ssize_t get_game_start_pid(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_game_start_pid(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +#ifdef CONFIG_QTI_PLH +static ssize_t get_plh_log_level(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_plh_log_level(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +static ssize_t get_splh_notif(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_splh_notif(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +#endif +static struct kobj_attribute cpu_min_freq_attr = + __ATTR(cpu_min_freq, 0644, get_cpu_min_freq, set_cpu_min_freq); +static struct kobj_attribute cpu_max_freq_attr = + __ATTR(cpu_max_freq, 0644, get_cpu_max_freq, set_cpu_max_freq); +static struct kobj_attribute inst_attr = + __ATTR(inst, 0444, get_cpu_total_instruction, NULL); +static struct kobj_attribute core_ctl_register_attr = + __ATTR(core_ctl_register, 0644, get_core_ctl_register, + set_core_ctl_register); +static struct kobj_attribute evnt_gplaf_pid_attr = + __ATTR(evnt_gplaf_pid, 0644, get_game_start_pid, set_game_start_pid); +#ifdef CONFIG_QTI_PLH +static struct kobj_attribute plh_log_level_attr = + __ATTR(plh_log_level, 0644, get_plh_log_level, set_plh_log_level); +static struct kobj_attribute splh_notify_attr = + __ATTR(splh_notify, 0644, get_splh_notif, set_splh_notif); +#endif + +static struct attribute *param_attrs[] = { + &cpu_min_freq_attr.attr, + &cpu_max_freq_attr.attr, + &inst_attr.attr, + &core_ctl_register_attr.attr, + &evnt_gplaf_pid_attr.attr, +#ifdef CONFIG_QTI_PLH + &plh_log_level_attr.attr, + &splh_notify_attr.attr, +#endif + NULL, +}; + +static struct attribute_group param_attr_group = { + .attrs = param_attrs, +}; + +static int add_module_params(void) +{ + int ret; + struct kobject *module_kobj; + + module_kobj = &msm_perf_kset->kobj; + + param_kobj = kobject_create_and_add("parameters", module_kobj); + if (!param_kobj) { + pr_err("msm_perf: Failed to add param_kobj\n"); + return -ENOMEM; + } + + ret = sysfs_create_group(param_kobj, ¶m_attr_group); + if (ret) { + pr_err("msm_perf: Failed to create sysfs\n"); + return ret; + } + return 0; +} + +/* To handle cpufreq min/max request */ +struct cpu_status { + unsigned int min; + unsigned int max; +}; +static DEFINE_PER_CPU(struct cpu_status, msm_perf_cpu_stats); +static DEFINE_PER_CPU(struct freq_qos_request, qos_req_min); +static DEFINE_PER_CPU(struct freq_qos_request, qos_req_max); + +static cpumask_var_t limit_mask_min; +static cpumask_var_t limit_mask_max; + +static DECLARE_COMPLETION(gfx_evt_arrival); + +struct gpu_data { + pid_t pid; + int ctx_id; + unsigned int timestamp; + ktime_t arrive_ts; + int evt_typ; +}; + +static struct gpu_data gpu_circ_buff[QUEUE_POOL_SIZE]; + +struct queue_indicies { + int head; + int tail; +}; +static struct queue_indicies curr_pos; + +static DEFINE_SPINLOCK(gfx_circ_buff_lock); + +struct event_data { + struct perf_event *pevent; + u64 prev_count; + u64 cur_delta; + u64 cached_total_count; +}; +static struct event_data **pmu_events; +static unsigned long min_cpu_capacity = ULONG_MAX; + +struct events { + spinlock_t cpu_hotplug_lock; + bool cpu_hotplug; + bool init_success; +}; +static struct events events_group; +static struct task_struct *events_notify_thread; + +static unsigned int aggr_big_nr; +static unsigned int aggr_top_load; +static unsigned int top_load[CLUSTER_MAX]; +static unsigned int curr_cap[CLUSTER_MAX]; +static atomic_t game_status_pid; +static bool ready_for_freq_updates; + +static int freq_qos_request_init(void) +{ + unsigned int cpu; + int ret; + + struct cpufreq_policy *policy; + struct freq_qos_request *req; + + for_each_present_cpu(cpu) { + policy = cpufreq_cpu_get(cpu); + if (!policy) { + pr_err("%s: Failed to get cpufreq policy for cpu%d\n", + __func__, cpu); + ret = -EAGAIN; + goto cleanup; + } + per_cpu(msm_perf_cpu_stats, cpu).min = 0; + req = &per_cpu(qos_req_min, cpu); + ret = freq_qos_add_request(&policy->constraints, req, + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) { + pr_err("%s: Failed to add min freq constraint (%d)\n", + __func__, ret); + cpufreq_cpu_put(policy); + goto cleanup; + } + + per_cpu(msm_perf_cpu_stats, cpu).max = UINT_MAX; + req = &per_cpu(qos_req_max, cpu); + ret = freq_qos_add_request(&policy->constraints, req, + FREQ_QOS_MAX, FREQ_QOS_MAX_DEFAULT_VALUE); + if (ret < 0) { + pr_err("%s: Failed to add max freq constraint (%d)\n", + __func__, ret); + cpufreq_cpu_put(policy); + goto cleanup; + } + + cpufreq_cpu_put(policy); + } + return 0; + +cleanup: + for_each_present_cpu(cpu) { + req = &per_cpu(qos_req_min, cpu); + if (req && freq_qos_request_active(req)) + freq_qos_remove_request(req); + + + req = &per_cpu(qos_req_max, cpu); + if (req && freq_qos_request_active(req)) + freq_qos_remove_request(req); + + per_cpu(msm_perf_cpu_stats, cpu).min = 0; + per_cpu(msm_perf_cpu_stats, cpu).max = UINT_MAX; + } + return ret; +} + +/*******************************sysfs start************************************/ +static ssize_t set_cpu_min_freq(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + int i, j, ntokens = 0; + unsigned int val, cpu; + const char *cp = buf; + struct cpu_status *i_cpu_stats; + struct cpufreq_policy policy; + struct freq_qos_request *req; + int ret = 0; + + if (!ready_for_freq_updates) { + ret = freq_qos_request_init(); + if (ret) { + pr_err("%s: Failed to init qos requests policy for ret=%d\n", + __func__, ret); + return ret; + } + ready_for_freq_updates = true; + } + + while ((cp = strpbrk(cp + 1, " :"))) + ntokens++; + + /* CPU:value pair */ + if (!(ntokens % 2)) + return -EINVAL; + + cp = buf; + cpumask_clear(limit_mask_min); + for (i = 0; i < ntokens; i += 2) { + if (sscanf(cp, "%u:%u", &cpu, &val) != 2) + return -EINVAL; + if (cpu > (num_present_cpus() - 1)) + return -EINVAL; + + i_cpu_stats = &per_cpu(msm_perf_cpu_stats, cpu); + + i_cpu_stats->min = val; + cpumask_set_cpu(cpu, limit_mask_min); + + cp = strnchr(cp, strlen(cp), ' '); + cp++; + } + + /* + * Since on synchronous systems policy is shared amongst multiple + * CPUs only one CPU needs to be updated for the limit to be + * reflected for the entire cluster. We can avoid updating the policy + * of other CPUs in the cluster once it is done for at least one CPU + * in the cluster + */ + get_online_cpus(); + for_each_cpu(i, limit_mask_min) { + i_cpu_stats = &per_cpu(msm_perf_cpu_stats, i); + + if (cpufreq_get_policy(&policy, i)) + continue; + + if (cpu_online(i)) { + req = &per_cpu(qos_req_min, i); + if (freq_qos_update_request(req, i_cpu_stats->min) < 0) + break; + } + + for_each_cpu(j, policy.related_cpus) + cpumask_clear_cpu(j, limit_mask_min); + } + put_online_cpus(); + + return count; +} + +static ssize_t get_cpu_min_freq(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int cnt = 0, cpu; + + for_each_present_cpu(cpu) { + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, + "%d:%u ", cpu, + per_cpu(msm_perf_cpu_stats, cpu).min); + } + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); + return cnt; +} + +static ssize_t set_cpu_max_freq(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + int i, j, ntokens = 0; + unsigned int val, cpu; + const char *cp = buf; + struct cpu_status *i_cpu_stats; + struct cpufreq_policy policy; + struct freq_qos_request *req; + int ret = 0; + + if (!ready_for_freq_updates) { + ret = freq_qos_request_init(); + if (ret) { + pr_err("%s: Failed to init qos requests policy for ret=%d\n", + __func__, ret); + return ret; + } + ready_for_freq_updates = true; + } + + while ((cp = strpbrk(cp + 1, " :"))) + ntokens++; + + /* CPU:value pair */ + if (!(ntokens % 2)) + return -EINVAL; + + cp = buf; + cpumask_clear(limit_mask_max); + for (i = 0; i < ntokens; i += 2) { + if (sscanf(cp, "%u:%u", &cpu, &val) != 2) + return -EINVAL; + if (cpu > (num_present_cpus() - 1)) + return -EINVAL; + + i_cpu_stats = &per_cpu(msm_perf_cpu_stats, cpu); + + i_cpu_stats->max = val; + cpumask_set_cpu(cpu, limit_mask_max); + + cp = strnchr(cp, strlen(cp), ' '); + cp++; + } + + get_online_cpus(); + for_each_cpu(i, limit_mask_max) { + i_cpu_stats = &per_cpu(msm_perf_cpu_stats, i); + if (cpufreq_get_policy(&policy, i)) + continue; + + if (cpu_online(i)) { + req = &per_cpu(qos_req_max, i); + if (freq_qos_update_request(req, i_cpu_stats->max) < 0) + break; + } + + for_each_cpu(j, policy.related_cpus) + cpumask_clear_cpu(j, limit_mask_max); + } + put_online_cpus(); + + return count; +} + +static ssize_t get_cpu_max_freq(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int cnt = 0, cpu; + + for_each_present_cpu(cpu) { + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, + "%d:%u ", cpu, + per_cpu(msm_perf_cpu_stats, cpu).max); + } + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); + return cnt; +} + +static struct kobject *events_kobj; + +static ssize_t show_cpu_hotplug(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "\n"); +} +static struct kobj_attribute cpu_hotplug_attr = +__ATTR(cpu_hotplug, 0444, show_cpu_hotplug, NULL); + +static struct attribute *events_attrs[] = { + &cpu_hotplug_attr.attr, + NULL, +}; + +static struct attribute_group events_attr_group = { + .attrs = events_attrs, +}; + +static ssize_t show_perf_gfx_evts(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct queue_indicies updated_pos; + unsigned long flags; + ssize_t retval = 0; + int idx = 0, size, act_idx, ret = -1; + + ret = wait_for_completion_interruptible(&gfx_evt_arrival); + if (ret) + return 0; + spin_lock_irqsave(&gfx_circ_buff_lock, flags); + updated_pos.head = curr_pos.head; + updated_pos.tail = curr_pos.tail; + size = CIRC_CNT(updated_pos.head, updated_pos.tail, QUEUE_POOL_SIZE); + curr_pos.tail = (curr_pos.tail + size) % QUEUE_POOL_SIZE; + spin_unlock_irqrestore(&gfx_circ_buff_lock, flags); + + for (idx = 0; idx < size; idx++) { + act_idx = (updated_pos.tail + idx) % QUEUE_POOL_SIZE; + retval += scnprintf(buf + retval, PAGE_SIZE - retval, + "%d %d %u %d %lu :", + gpu_circ_buff[act_idx].pid, + gpu_circ_buff[act_idx].ctx_id, + gpu_circ_buff[act_idx].timestamp, + gpu_circ_buff[act_idx].evt_typ, + ktime_to_us(gpu_circ_buff[act_idx].arrive_ts)); + if (retval >= PAGE_SIZE) { + pr_err("msm_perf:data limit exceed\n"); + break; + } + } + return retval; +} + +static struct kobj_attribute gfx_event_info_attr = +__ATTR(gfx_evt, 0444, show_perf_gfx_evts, NULL); + +static ssize_t show_big_nr(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", aggr_big_nr); +} + +static struct kobj_attribute big_nr_attr = +__ATTR(aggr_big_nr, 0444, show_big_nr, NULL); + +static ssize_t show_top_load(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", aggr_top_load); +} + +static struct kobj_attribute top_load_attr = +__ATTR(aggr_top_load, 0444, show_top_load, NULL); + + +static ssize_t show_top_load_cluster(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u %u %u\n", + top_load[MIN], top_load[MID], + top_load[MAX]); +} + +static struct kobj_attribute cluster_top_load_attr = +__ATTR(top_load_cluster, 0444, show_top_load_cluster, NULL); + +static ssize_t show_curr_cap_cluster(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u %u %u\n", + curr_cap[MIN], curr_cap[MID], + curr_cap[MAX]); +} + +static struct kobj_attribute cluster_curr_cap_attr = +__ATTR(curr_cap_cluster, 0444, show_curr_cap_cluster, NULL); + +static struct attribute *notify_attrs[] = { + &big_nr_attr.attr, + &top_load_attr.attr, + &cluster_top_load_attr.attr, + &cluster_curr_cap_attr.attr, + &gfx_event_info_attr.attr, + NULL, +}; + +static struct attribute_group notify_attr_group = { + .attrs = notify_attrs, +}; +static struct kobject *notify_kobj; + +/*******************************sysfs ends************************************/ + +/*****************PMU Data Collection*****************/ +static struct perf_event_attr attr; +static void msm_perf_init_attr(void) +{ + memset(&attr, 0, sizeof(struct perf_event_attr)); + + attr.type = PERF_TYPE_RAW; + attr.size = sizeof(struct perf_event_attr); + attr.pinned = 1; +} + +static int set_event(struct event_data *ev, int cpu) +{ + struct perf_event *pevent; + + pevent = perf_event_create_kernel_counter(&attr, + cpu, NULL, NULL, NULL); + if (IS_ERR(pevent)) { + pr_err("msm_perf: %s failed, eventId:0x%x, cpu:%d, error code:%ld\n", + __func__, attr.config, cpu, PTR_ERR(pevent)); + return PTR_ERR(pevent); + } + ev->pevent = pevent; + perf_event_enable(pevent); + + return 0; +} + +static void free_pmu_counters(unsigned int cpu) +{ + int i = 0; + + for (i = 0; i < NO_OF_EVENT; i++) { + pmu_events[i][cpu].prev_count = 0; + pmu_events[i][cpu].cur_delta = 0; + pmu_events[i][cpu].cached_total_count = 0; + if (pmu_events[i][cpu].pevent) { + perf_event_disable(pmu_events[i][cpu].pevent); + perf_event_release_kernel(pmu_events[i][cpu].pevent); + pmu_events[i][cpu].pevent = NULL; + } + } +} + +static int init_pmu_counter(void) +{ + int cpu; + unsigned long cpu_capacity; + int ret = 0; + + int i = 0, j = 0; + int no_of_cpus = 0; + + msm_perf_init_attr(); + for_each_possible_cpu(cpu) + no_of_cpus++; + + pmu_events = kcalloc(NO_OF_EVENT, sizeof(struct event_data *), GFP_KERNEL); + if (!pmu_events) + return -ENOMEM; + for (i = 0; i < NO_OF_EVENT; i++) { + pmu_events[i] = kcalloc(no_of_cpus, sizeof(struct event_data), GFP_KERNEL); + if (!pmu_events[i]) { + for (j = i; j >= 0; j--) { + kfree(pmu_events[j]); + pmu_events[j] = NULL; + } + kfree(pmu_events); + pmu_events = NULL; + return -ENOMEM; + } + } + + /* Create events per CPU */ + for_each_possible_cpu(cpu) { + /* create Instruction event */ + attr.config = INST_EV; + ret = set_event(&pmu_events[INST_EVENT][cpu], cpu); + if (ret < 0) + return ret; + /* create cycle event */ + attr.config = CYC_EV; + ret = set_event(&pmu_events[CYC_EVENT][cpu], cpu); + if (ret < 0) { + free_pmu_counters(cpu); + return ret; + } + /* find capacity per cpu */ + cpu_capacity = arch_scale_cpu_capacity(cpu); + if (cpu_capacity < min_cpu_capacity) + min_cpu_capacity = cpu_capacity; + } + return 0; +} + +static inline void msm_perf_read_event(struct event_data *event) +{ + u64 ev_count = 0; + u64 total, enabled, running; + + mutex_lock(&perfevent_lock); + if (!event->pevent) { + mutex_unlock(&perfevent_lock); + return; + } + + if (!per_cpu(cpu_is_idle, event->pevent->cpu) && + !per_cpu(cpu_is_hp, event->pevent->cpu)) + total = perf_event_read_value(event->pevent, &enabled, &running); + else + total = event->cached_total_count; + + ev_count = total - event->prev_count; + event->prev_count = total; + event->cur_delta = ev_count; + mutex_unlock(&perfevent_lock); +} + +static ssize_t get_cpu_total_instruction(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + u64 instruction = 0; + u64 cycles = 0; + u64 total_inst_big = 0; + u64 total_inst_little = 0; + u64 ipc_big = 0; + u64 ipc_little = 0; + int cnt = 0, cpu; + + for_each_possible_cpu(cpu) { + /* Read Instruction event */ + msm_perf_read_event(&pmu_events[INST_EVENT][cpu]); + /* Read Cycle event */ + msm_perf_read_event(&pmu_events[CYC_EVENT][cpu]); + instruction = pmu_events[INST_EVENT][cpu].cur_delta; + cycles = pmu_events[CYC_EVENT][cpu].cur_delta; + /* collecting max inst and ipc for max cap and min cap cpus */ + if (arch_scale_cpu_capacity(cpu) > min_cpu_capacity) { + if (cycles && cycles >= CPU_CYCLE_THRESHOLD) + ipc_big = max(ipc_big, + ((instruction*100)/cycles)); + total_inst_big += instruction; + } else { + if (cycles) + ipc_little = max(ipc_little, + ((instruction*100)/cycles)); + total_inst_little += instruction; + } + } + + cnt += scnprintf(buf, PAGE_SIZE, "%llu:%llu:%llu:%llu\n", + total_inst_big, ipc_big, + total_inst_little, ipc_little); + + return cnt; +} + +static int restart_events(unsigned int cpu, bool cpu_up) +{ + int ret = 0; + + msm_perf_init_attr(); + + if (cpu_up) { + /* create Instruction event */ + attr.config = INST_EV; + ret = set_event(&pmu_events[INST_EVENT][cpu], cpu); + if (ret < 0) + return ret; + /* create cycle event */ + attr.config = CYC_EV; + ret = set_event(&pmu_events[CYC_EVENT][cpu], cpu); + if (ret < 0) { + free_pmu_counters(cpu); + return ret; + } + } else { + free_pmu_counters(cpu); + } + + return 0; +} + +static int hotplug_notify_down(unsigned int cpu) +{ + mutex_lock(&perfevent_lock); + per_cpu(cpu_is_hp, cpu) = true; + restart_events(cpu, false); + mutex_unlock(&perfevent_lock); + + return 0; +} + +static int hotplug_notify_up(unsigned int cpu) +{ + unsigned long flags; + + mutex_lock(&perfevent_lock); + restart_events(cpu, true); + per_cpu(cpu_is_hp, cpu) = false; + mutex_unlock(&perfevent_lock); + + if (events_group.init_success) { + spin_lock_irqsave(&(events_group.cpu_hotplug_lock), flags); + events_group.cpu_hotplug = true; + spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), flags); + wake_up_process(events_notify_thread); + } + + return 0; +} + +static int msm_perf_idle_read_events(unsigned int cpu) +{ + int ret = 0, i; + + for (i = 0; i < NO_OF_EVENT; i++) { + if (pmu_events[i][cpu].pevent) + ret = perf_event_read_local(pmu_events[i][cpu].pevent, + &pmu_events[i][cpu].cached_total_count, NULL, NULL); + } + + return ret; +} + +static void msm_perf_idle_notif(void *unused, unsigned int state, + unsigned int cpu) +{ + if (state == PWR_EVENT_EXIT) { + __this_cpu_write(cpu_is_idle, false); + } else { + __this_cpu_write(cpu_is_idle, true); + if (!per_cpu(cpu_is_hp, cpu)) + msm_perf_idle_read_events(cpu); + } +} + +static int events_notify_userspace(void *data) +{ + unsigned long flags; + bool notify_change; + + while (1) { + + set_current_state(TASK_INTERRUPTIBLE); + spin_lock_irqsave(&(events_group.cpu_hotplug_lock), flags); + + if (!events_group.cpu_hotplug) { + spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), + flags); + + schedule(); + if (kthread_should_stop()) + break; + spin_lock_irqsave(&(events_group.cpu_hotplug_lock), + flags); + } + + set_current_state(TASK_RUNNING); + notify_change = events_group.cpu_hotplug; + events_group.cpu_hotplug = false; + spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), flags); + + if (notify_change) + sysfs_notify(events_kobj, NULL, "cpu_hotplug"); + } + + return 0; +} + +static int init_notify_group(void) +{ + int ret; + struct kobject *module_kobj = &msm_perf_kset->kobj; + + notify_kobj = kobject_create_and_add("notify", module_kobj); + if (!notify_kobj) { + pr_err("msm_perf: Failed to add notify_kobj\n"); + return -ENOMEM; + } + + ret = sysfs_create_group(notify_kobj, ¬ify_attr_group); + if (ret) { + kobject_put(notify_kobj); + pr_err("msm_perf: Failed to create sysfs\n"); + return ret; + } + return 0; +} + +static int init_events_group(void) +{ + int ret; + struct kobject *module_kobj = &msm_perf_kset->kobj; + + events_kobj = kobject_create_and_add("events", module_kobj); + if (!events_kobj) { + pr_err("msm_perf: Failed to add events_kobj\n"); + return -ENOMEM; + } + + ret = sysfs_create_group(events_kobj, &events_attr_group); + if (ret) { + pr_err("msm_perf: Failed to create sysfs\n"); + return ret; + } + + spin_lock_init(&(events_group.cpu_hotplug_lock)); + events_notify_thread = kthread_run(events_notify_userspace, + NULL, "msm_perf:events_notify"); + if (IS_ERR(events_notify_thread)) + return PTR_ERR(events_notify_thread); + + events_group.init_success = true; + + return 0; +} + +static void nr_notify_userspace(struct work_struct *work) +{ + sysfs_notify(notify_kobj, NULL, "aggr_top_load"); + sysfs_notify(notify_kobj, NULL, "aggr_big_nr"); + sysfs_notify(notify_kobj, NULL, "top_load_cluster"); + sysfs_notify(notify_kobj, NULL, "curr_cap_cluster"); +} + +static int msm_perf_core_ctl_notify(struct notifier_block *nb, + unsigned long unused, + void *data) +{ + static unsigned int tld, nrb, i; + static unsigned int top_ld[CLUSTER_MAX], curr_cp[CLUSTER_MAX]; + static DECLARE_WORK(sysfs_notify_work, nr_notify_userspace); + struct core_ctl_notif_data *d = data; + int cluster = 0; + + nrb += d->nr_big; + tld += d->coloc_load_pct; + for (cluster = 0; cluster < CLUSTER_MAX; cluster++) { + top_ld[cluster] += d->ta_util_pct[cluster]; + curr_cp[cluster] += d->cur_cap_pct[cluster]; + } + i++; + if (i == POLL_INT) { + aggr_big_nr = ((nrb%POLL_INT) ? 1 : 0) + nrb/POLL_INT; + aggr_top_load = tld/POLL_INT; + for (cluster = 0; cluster < CLUSTER_MAX; cluster++) { + top_load[cluster] = top_ld[cluster]/POLL_INT; + curr_cap[cluster] = curr_cp[cluster]/POLL_INT; + top_ld[cluster] = 0; + curr_cp[cluster] = 0; + } + tld = 0; + nrb = 0; + i = 0; + schedule_work(&sysfs_notify_work); + } + return NOTIFY_OK; +} + +static struct notifier_block msm_perf_nb = { + .notifier_call = msm_perf_core_ctl_notify +}; + +static bool core_ctl_register; +static ssize_t get_core_ctl_register(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%c\n", core_ctl_register ? 'Y' : 'N'); +} + +static ssize_t set_core_ctl_register(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + bool old_val = core_ctl_register; + int ret; + + ret = sscanf(buf, "%du", &core_ctl_register); + if (ret < 0) { + pr_err("msm_perf: getting new core_ctl_register failed, ret=%d\n", ret); + return ret; + } + + if (core_ctl_register == old_val) + return count; + + if (core_ctl_register) + core_ctl_notifier_register(&msm_perf_nb); + else + core_ctl_notifier_unregister(&msm_perf_nb); + + return count; +} + +void msm_perf_events_update(enum evt_update_t update_typ, + enum gfx_evt_t evt_typ, pid_t pid, + uint32_t ctx_id, uint32_t timestamp, bool end_of_frame) +{ + unsigned long flags; + int idx = 0; + + if (update_typ != MSM_PERF_GFX) + return; + + if (pid != atomic_read(&game_status_pid) || (timestamp == 0) + || !(end_of_frame)) + return; + + spin_lock_irqsave(&gfx_circ_buff_lock, flags); + idx = curr_pos.head; + curr_pos.head = ((curr_pos.head + 1) % QUEUE_POOL_SIZE); + spin_unlock_irqrestore(&gfx_circ_buff_lock, flags); + gpu_circ_buff[idx].pid = pid; + gpu_circ_buff[idx].ctx_id = ctx_id; + gpu_circ_buff[idx].timestamp = timestamp; + gpu_circ_buff[idx].evt_typ = evt_typ; + gpu_circ_buff[idx].arrive_ts = ktime_get(); + + if (evt_typ == MSM_PERF_QUEUE || evt_typ == MSM_PERF_RETIRED) + complete(&gfx_evt_arrival); +} +EXPORT_SYMBOL(msm_perf_events_update); + +static ssize_t set_game_start_pid(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + long usr_val = 0; + kstrtol(buf, 0, &usr_val); + atomic_set(&game_status_pid, usr_val); + return count; +} +static ssize_t get_game_start_pid(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + long usr_val = atomic_read(&game_status_pid); + + return scnprintf(buf, PAGE_SIZE, "%ld\n", usr_val); +} + +/*******************************GFX Call************************************/ +#ifdef CONFIG_QTI_PLH +static struct scmi_handle *plh_handle; +void rimps_plh_init(struct scmi_handle *handle) +{ + if (handle) + plh_handle = handle; +} +EXPORT_SYMBOL(rimps_plh_init); + +static int splh_notif, splh_init_done, plh_log_level; + +#define PLH_MIN_LOG_LEVEL 0 +#define PLH_MAX_LOG_LEVEL 0xF +#define SPLH_FPS_MAX_CNT 8 +#define SPLH_IPC_FREQ_VTBL_MAX_CNT 5 /* ipc freq pair */ +#define SPLH_INIT_IPC_FREQ_TBL_PARAMS \ + (2 + SPLH_FPS_MAX_CNT * (1 + (2 * SPLH_IPC_FREQ_VTBL_MAX_CNT))) + +static ssize_t get_plh_log_level(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%d\n", plh_log_level); +} + +static ssize_t set_plh_log_level(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret, log_val_backup; + struct scmi_plh_vendor_ops *ops; + + if (!plh_handle || !plh_handle->plh_ops) { + pr_err("msm_perf: plh scmi handle or vendor ops null\n"); + return -EINVAL; + } + + ops = plh_handle->plh_ops; + + log_val_backup = plh_log_level; + + ret = sscanf(buf, "%du", &plh_log_level); + + if (ret < 0) { + pr_err("msm_perf: getting new plh_log_level failed, ret=%d\n", ret); + return ret; + } + + plh_log_level = clamp(plh_log_level, PLH_MIN_LOG_LEVEL, PLH_MAX_LOG_LEVEL); + ret = ops->set_plh_log_level(plh_handle, plh_log_level); + if (ret < 0) { + plh_log_level = log_val_backup; + pr_err("msm_perf: setting new plh_log_level failed, ret=%d\n", ret); + return ret; + } + return count; +} + +static int init_splh_notif(const char *buf) +{ + int i, j, ret; + u16 tmp[SPLH_INIT_IPC_FREQ_TBL_PARAMS]; + u16 *ptmp = tmp, ntokens, nfps, n_ipc_freq_pair, tmp_valid_len = 0; + const char *cp, *cp1; + struct scmi_plh_vendor_ops *ops; + + /* buf contains the init info from user */ + if (buf == NULL || !plh_handle || !plh_handle->plh_ops) + return -EINVAL; + + cp = buf; + ntokens = 0; + while ((cp = strpbrk(cp + 1, ":"))) + ntokens++; + + /* format of cmd nfps, n_ipc_freq_pair, ,...>,... */ + cp = buf; + if (sscanf(cp, INIT ":%hu", &nfps)) { + if ((nfps != ntokens-1) || (nfps == 0) || (nfps > SPLH_FPS_MAX_CNT)) + return -EINVAL; + + cp = strnchr(cp, strlen(cp), ':'); /* skip INIT */ + cp++; + cp = strnchr(cp, strlen(cp), ':'); /* skip nfps */ + + *ptmp++ = nfps; /* nfps is first cmd param */ + tmp_valid_len++; + cp1 = cp; + ntokens = 0; + /* get count of nfps * n_ipc_freq_pair * */ + while ((cp1 = strpbrk(cp1 + 1, ","))) + ntokens++; + + if (ntokens % (2 * nfps)) /* ipc freq pair values should be multiple of nfps */ + return -EINVAL; + + n_ipc_freq_pair = ntokens / (2 * nfps); /* ipc_freq pair values for each FPS */ + if ((n_ipc_freq_pair == 0) || (n_ipc_freq_pair > SPLH_IPC_FREQ_VTBL_MAX_CNT)) + return -EINVAL; + + *ptmp++ = n_ipc_freq_pair; /* n_ipc_freq_pair is second cmd param */ + tmp_valid_len++; + cp1 = cp; + for (i = 0; i < nfps; i++) { + if (sscanf(cp1, ":%hu", ptmp) != 1) + return -EINVAL; + + ptmp++; /* increment after storing FPS val */ + tmp_valid_len++; + cp1 = strnchr(cp1, strlen(cp1), ','); /* move to ,ipc */ + for (j = 0; j < 2 * n_ipc_freq_pair; j++) { + if (sscanf(cp1, ",%hu", ptmp) != 1) + return -EINVAL; + + ptmp++; /* increment after storing ipc or freq */ + tmp_valid_len++; + cp1++; + if (j != (2 * n_ipc_freq_pair - 1)) + cp1 = strnchr(cp1, strlen(cp1), ','); /* move to next */ + } + + if (i != (nfps - 1)) + cp1 = strnchr(cp1, strlen(cp1), ':'); /* move to next FPS val */ + + } + } else { + return -EINVAL; + } + + ops = plh_handle->plh_ops; + ret = ops->init_splh_ipc_freq_tbl(plh_handle, tmp, tmp_valid_len); + if (ret < 0) + return -EINVAL; + + pr_info("msm_perf: nfps=%hu n_ipc_freq_pair=%hu last_freq_val=%hu len=%hu\n", + nfps, n_ipc_freq_pair, *--ptmp, tmp_valid_len); + splh_init_done = 1; + return 0; +} + +static void activate_splh_notif(void) +{ + int ret; + struct scmi_plh_vendor_ops *ops; + /* received event notification here */ + if (!plh_handle || !plh_handle->plh_ops) { + pr_err("msm_perf: splh not supported\n"); + return; + } + ops = plh_handle->plh_ops; + + if (splh_notif) + ret = ops->start_splh(plh_handle, splh_notif); /* splh_notif is fps */ + else + ret = ops->stop_splh(plh_handle); + + if (ret < 0) { + pr_err("msm_perf: splh start or stop failed, ret=%d\n", ret); + return; + } +} + +static ssize_t get_splh_notif(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%d\n", splh_notif); +} + +static ssize_t set_splh_notif(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret; + + if (strnstr(buf, INIT, sizeof(INIT)) != NULL) { + splh_init_done = 0; + ret = init_splh_notif(buf); + if (ret < 0) + pr_err("msm_perf: splh ipc freq tbl init failed, ret=%d\n", ret); + + return ret; + } + + if (!splh_init_done) { + pr_err("msm_perf: splh ipc freq tbl not initialized\n"); + return -EINVAL; + } + + ret = sscanf(buf, "%du", &splh_notif); + if (ret < 0) + return ret; + + activate_splh_notif(); + + return count; +} +#endif /* CONFIG_QTI_PLH */ + +static int __init msm_performance_init(void) +{ + unsigned int cpu; + int ret; + if (!alloc_cpumask_var(&limit_mask_min, GFP_KERNEL)) + return -ENOMEM; + + if (!alloc_cpumask_var(&limit_mask_max, GFP_KERNEL)) { + free_cpumask_var(limit_mask_min); + return -ENOMEM; + } + get_online_cpus(); + for_each_possible_cpu(cpu) { + if (!cpumask_test_cpu(cpu, cpu_online_mask)) + per_cpu(cpu_is_hp, cpu) = true; + } + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "msm_performance_cpu_hotplug", + hotplug_notify_up, + hotplug_notify_down); + + put_online_cpus(); + + msm_perf_kset = kset_create_and_add("msm_performance", NULL, kernel_kobj); + if (!msm_perf_kset) { + free_cpumask_var(limit_mask_min); + free_cpumask_var(limit_mask_max); + return -ENOMEM; + } + + add_module_params(); + + init_events_group(); + init_notify_group(); + init_pmu_counter(); + + register_trace_cpu_idle(msm_perf_idle_notif, NULL); + return 0; +} +MODULE_LICENSE("GPL v2"); +late_initcall(msm_performance_init); diff --git a/include/linux/sched/core_ctl.h b/include/linux/sched/core_ctl.h deleted file mode 100644 index c7ff34fbda5c..000000000000 --- a/include/linux/sched/core_ctl.h +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2016, 2019-2021, The Linux Foundation. All rights reserved. - */ - -#ifndef __CORE_CTL_H -#define __CORE_CTL_H - -#include - -#define MAX_CPUS_PER_CLUSTER 6 -#define MAX_CLUSTERS 3 - -struct core_ctl_notif_data { - unsigned int nr_big; - unsigned int coloc_load_pct; - unsigned int ta_util_pct[MAX_CLUSTERS]; - unsigned int cur_cap_pct[MAX_CLUSTERS]; -}; - -struct notifier_block; - -#if IS_ENABLED(CONFIG_SCHED_WALT) -extern int core_ctl_set_boost(bool boost); -extern void core_ctl_notifier_register(struct notifier_block *n); -extern void core_ctl_notifier_unregister(struct notifier_block *n); -#else -static inline int core_ctl_set_boost(bool boost) -{ - return 0; -} -static inline void core_ctl_notifier_register(struct notifier_block *n) {} -static inline void core_ctl_notifier_unregister(struct notifier_block *n) {} -#endif - -#endif diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index f2f69acb206e..aa503f24009a 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -12,6 +12,16 @@ #if IS_ENABLED(CONFIG_SCHED_WALT) +#define MAX_CPUS_PER_CLUSTER 6 +#define MAX_CLUSTERS 3 + +struct core_ctl_notif_data { + unsigned int nr_big; + unsigned int coloc_load_pct; + unsigned int ta_util_pct[MAX_CLUSTERS]; + unsigned int cur_cap_pct[MAX_CLUSTERS]; +}; + #define WALT_NR_CPUS 8 #define RAVG_HIST_SIZE_MAX 5 #define NUM_BUSY_BUCKETS 10 @@ -118,6 +128,11 @@ static inline void set_wake_up_idle(bool wake_up_idle) extern int sched_lpm_disallowed_time(int cpu, u64 *timeout); extern int set_task_boost(int boost, u64 period); + +struct notifier_block; +extern void core_ctl_notifier_register(struct notifier_block *n); +extern void core_ctl_notifier_unregister(struct notifier_block *n); +extern int core_ctl_set_boost(bool boost); #else static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout) { @@ -141,6 +156,20 @@ static inline int sched_set_wake_up_idle(struct task_struct *p, bool wake_up_idl static inline void set_wake_up_idle(bool wake_up_idle) { } + +static inline int core_ctl_set_boost(bool boost) +{ + return 0; +} + +static inline void core_ctl_notifier_register(struct notifier_block *n) +{ +} + +static inline void core_ctl_notifier_unregister(struct notifier_block *n) +{ +} + #endif #endif /* _LINUX_SCHED_WALT_H */ diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c index 17305aa10029..5ac7dce8160c 100644 --- a/kernel/sched/walt/boost.c +++ b/kernel/sched/walt/boost.c @@ -4,7 +4,6 @@ */ #include -#include #include "walt.h" #include "trace.h" diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index c142a9ad8ad3..6d0bed0bfbda 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -14,7 +14,6 @@ #include #include #include -#include #include "walt.h" #include "trace.h" diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index dfb2e8f5826d..a4c419829b3d 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -9,7 +9,6 @@ #include "../../../kernel/sched/sched.h" #include "../../../fs/proc/internal.h" #include -#include #include #include From f3d3bd3f15feee0e1d080dec93a6ad58878613a2 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 4 Feb 2021 14:10:11 +0530 Subject: [PATCH 032/346] sched/walt: Fix waking new tasks on higher capacity CPUs walt_find_best_target() which selects the best CPU has a special handling for running tasks. The idle CPUs are only considered as migrating a running task to a busy CPU will result in wait time. A running task is detected via task->state == TASK_RUNNING check, which is true for tick path active migration tasks. It is also true for newly created tasks. These tasks are getting placed on idle CPUs of the higher capacity clusters though the lower capacity CPUs are lightly loaded. Detect the running task via task->on_rq to fix this issue. Change-Id: I154cd6c6f934061526bce30fe596d9ad36c9e07f Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_cfs.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 78c45a3aa058..63e43de2f310 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -196,6 +196,7 @@ static void walt_find_best_target(struct sched_domain *sd, cpumask_t visit_cpus; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; struct cfs_rq *cfs_rq; + bool active_task = task_on_rq_queued(p); /* Find start CPU based on boost value */ start_cpu = fbt_env->start_cpu; @@ -214,7 +215,7 @@ static void walt_find_best_target(struct sched_domain *sd, most_spare_wake_cap = LONG_MIN; } - if (p->state == TASK_RUNNING) + if (active_task) most_spare_wake_cap = ULONG_MAX; /* fast path for prev_cpu */ @@ -340,7 +341,7 @@ static void walt_find_best_target(struct sched_domain *sd, /* * Consider only idle CPUs for active migration. */ - if (p->state == TASK_RUNNING) + if (active_task) continue; /* @@ -407,7 +408,7 @@ static void walt_find_best_target(struct sched_domain *sd, trace_sched_find_best_target(p, min_util, start_cpu, best_idle_cpu, most_spare_cap_cpu, target_cpu, order_index, end_index, - fbt_env->skip_cpu, p->state == TASK_RUNNING); + fbt_env->skip_cpu, active_task); } static inline unsigned long From 5d231eb1a853f8d6438d9a07bf57516241a5b1be Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 4 Feb 2021 17:00:14 +0530 Subject: [PATCH 033/346] sched/walt: Remove the active task special case from wfbt() The walt_find_best_target() has a special case for the active task migration attempt from tick path. It tries hard to find an idle CPU by disabling most_spare_cap_cpu selection. However this is all unnecessary if we can check for the idle CPU condition at the caller site. This helps the overall execution time of wfbt(). Change-Id: I89de44824dca8729b8c62898f9e75dc8e0e1c544 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_cfs.c | 12 +----------- kernel/sched/walt/walt_lb.c | 4 +++- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 63e43de2f310..49509dd5e5e2 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -196,7 +196,6 @@ static void walt_find_best_target(struct sched_domain *sd, cpumask_t visit_cpus; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; struct cfs_rq *cfs_rq; - bool active_task = task_on_rq_queued(p); /* Find start CPU based on boost value */ start_cpu = fbt_env->start_cpu; @@ -215,9 +214,6 @@ static void walt_find_best_target(struct sched_domain *sd, most_spare_wake_cap = LONG_MIN; } - if (active_task) - most_spare_wake_cap = ULONG_MAX; - /* fast path for prev_cpu */ if (((capacity_orig_of(prev_cpu) == capacity_orig_of(start_cpu)) || asym_cap_siblings(prev_cpu, start_cpu)) && @@ -338,12 +334,6 @@ static void walt_find_best_target(struct sched_domain *sd, continue; } - /* - * Consider only idle CPUs for active migration. - */ - if (active_task) - continue; - /* * Try to spread the rtg high prio tasks so that they * don't preempt each other. This is a optimisitc @@ -408,7 +398,7 @@ static void walt_find_best_target(struct sched_domain *sd, trace_sched_find_best_target(p, min_util, start_cpu, best_idle_cpu, most_spare_cap_cpu, target_cpu, order_index, end_index, - fbt_env->skip_cpu, active_task); + fbt_env->skip_cpu, task_on_rq_queued(p)); } static inline unsigned long diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index cff8b236c9a7..7caecafbd172 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -508,7 +508,9 @@ static void walt_lb_tick(void *unused, struct rq *rq) new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1); rcu_read_unlock(); - if (new_cpu < 0 || same_cluster(new_cpu, prev_cpu)) + /* prevent active task migration to busy or same/lower capacity CPU */ + if (new_cpu < 0 || !available_idle_cpu(new_cpu) || + capacity_orig_of(new_cpu) <= capacity_orig_of(prev_cpu)) goto out_unlock; raw_spin_lock(&rq->lock); From aa5f4744ea64bed588c01e50c3b8794e696cc313 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 5 Feb 2021 09:35:52 +0530 Subject: [PATCH 034/346] sched/walt: Make debug module depend on SCHED_WALT sched-walt-debug module does not get compiled unless SCHED_WALT config is turned on. However the config corresponding to the debug module shows in menuconfig even without enabling SCHED_WALT. Fix this by specifying the dependency in the Kconfig. Change-Id: Ie466c0ca11bbcdec43af2624c3631ddff5c77024 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sched/walt/Kconfig b/kernel/sched/walt/Kconfig index db60efdf4c8d..b9b2422f26f0 100644 --- a/kernel/sched/walt/Kconfig +++ b/kernel/sched/walt/Kconfig @@ -15,6 +15,7 @@ config SCHED_WALT config SCHED_WALT_DEBUG tristate "WALT debug module" + depends on SCHED_WALT select TRACE_PREEMPT_TOGGLE select TRACE_IRQFLAGS help From 28fa7231acf4261df60d1d988ffa2b438b77a547 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 5 Feb 2021 15:10:53 +0530 Subject: [PATCH 035/346] sched/walt: Clean up the code The clean up including removing unused variables and functions, converting global variables to static. Change-Id: Ib5126ac9df701fdde354e0e5d759f0177db5a28b Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/fixup.c | 11 +++- kernel/sched/walt/walt.c | 114 +++++------------------------------ kernel/sched/walt/walt.h | 14 +---- kernel/sched/walt/walt_cfs.c | 2 - kernel/sched/walt/walt_lb.c | 12 +--- 5 files changed, 26 insertions(+), 127 deletions(-) diff --git a/kernel/sched/walt/fixup.c b/kernel/sched/walt/fixup.c index 21c3abe003ec..6156b5827293 100644 --- a/kernel/sched/walt/fixup.c +++ b/kernel/sched/walt/fixup.c @@ -12,7 +12,7 @@ unsigned int cpuinfo_max_freq_cached; char sched_lib_name[LIB_PATH_LENGTH]; unsigned int sched_lib_mask_force; -bool is_sched_lib_based_app(pid_t pid) +static bool is_sched_lib_based_app(pid_t pid) { const char *name = NULL; char *libname, *lib_list; @@ -77,8 +77,8 @@ bool is_sched_lib_based_app(pid_t pid) return found; } -void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy, - unsigned int *max_freq) +static void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy, + unsigned int *max_freq) { if (!cpuinfo_max_freq_cached) return; @@ -89,3 +89,8 @@ void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy, if (is_sched_lib_based_app(current->pid)) *max_freq = cpuinfo_max_freq_cached << 1; } + +void walt_fixup_init(void) +{ + register_trace_android_vh_show_max_freq(android_vh_show_max_freq, NULL); +} diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 706feba887f6..c7a4984a4659 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -44,10 +44,6 @@ const char *migrate_type_names[] = { #define MAX_NR_CLUSTERS 3 -#define FREQ_REPORT_MAX_CPU_LOAD_TOP_TASK 0 -#define FREQ_REPORT_CPU_LOAD 1 -#define FREQ_REPORT_TOP_TASK 2 - #define NEW_TASK_ACTIVE_TIME 100000000 unsigned int sysctl_sched_user_hint; @@ -128,13 +124,13 @@ static struct syscore_ops sched_syscore_ops = { .suspend = sched_suspend }; -int sched_init_ops(void) +static int sched_init_ops(void) { register_syscore_ops(&sched_syscore_ops); return 0; } -void acquire_rq_locks_irqsave(const cpumask_t *cpus, +static inline void acquire_rq_locks_irqsave(const cpumask_t *cpus, unsigned long *flags) { int cpu; @@ -151,7 +147,7 @@ void acquire_rq_locks_irqsave(const cpumask_t *cpus, } } -void release_rq_locks_irqrestore(const cpumask_t *cpus, +static inline void release_rq_locks_irqrestore(const cpumask_t *cpus, unsigned long *flags) { int cpu; @@ -163,30 +159,24 @@ void release_rq_locks_irqrestore(const cpumask_t *cpus, static unsigned int walt_cpu_high_irqload; -__read_mostly unsigned int sched_ravg_hist_size = 5; +static __read_mostly unsigned int sched_ravg_hist_size = 5; static __read_mostly unsigned int sched_io_is_busy = 1; /* Window size (in ns) */ -__read_mostly unsigned int new_sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; +static __read_mostly unsigned int new_sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; static DEFINE_SPINLOCK(sched_ravg_window_lock); -u64 sched_ravg_window_change_time; +static u64 sched_ravg_window_change_time; -unsigned int __read_mostly sched_init_task_load_windows_scaled; -unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; +static unsigned int __read_mostly sched_init_task_load_windows_scaled; +static unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; /* Size of bitmaps maintained to track top tasks */ static const unsigned int top_tasks_bitmap_size = BITS_TO_LONGS(NUM_LOAD_INDICES + 1) * sizeof(unsigned long); -/* - * This governs what load needs to be used when reporting CPU busy time - * to the cpufreq governor. - */ -__read_mostly unsigned int sysctl_sched_freq_reporting_policy; - -__read_mostly unsigned int walt_scale_demand_divisor; +static __read_mostly unsigned int walt_scale_demand_divisor; #define scale_demand(d) ((d)/walt_scale_demand_divisor) #define SCHED_PRINT(arg) pr_emerg("%s=%llu", #arg, arg) @@ -556,7 +546,6 @@ should_apply_suh_freq_boost(struct walt_sched_cluster *cluster) static inline u64 freq_policy_load(struct rq *rq) { - unsigned int reporting_policy = sysctl_sched_freq_reporting_policy; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_sched_cluster *cluster = wrq->cluster; u64 aggr_grp_load = cluster->aggr_grp_load; @@ -578,18 +567,7 @@ static inline u64 freq_policy_load(struct rq *rq) load = max_t(u64, load, task_load(cpu_ksoftirqd)); tt_load = top_task_load(rq); - switch (reporting_policy) { - case FREQ_REPORT_MAX_CPU_LOAD_TOP_TASK: - load = max_t(u64, load, tt_load); - break; - case FREQ_REPORT_TOP_TASK: - load = tt_load; - break; - case FREQ_REPORT_CPU_LOAD: - break; - default: - break; - } + load = max_t(u64, load, tt_load); if (should_apply_suh_freq_boost(cluster)) { if (is_suh_max()) @@ -601,7 +579,7 @@ static inline u64 freq_policy_load(struct rq *rq) done: trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en, - load, reporting_policy, walt_rotation_enabled, + load, 0, walt_rotation_enabled, sysctl_sched_user_hint, wrq); return load; } @@ -2184,25 +2162,6 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even run_walt_irq_work(old_window_start, rq); } -u32 sched_get_init_task_load(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return wts->init_load_pct; -} - -int sched_set_init_task_load(struct task_struct *p, int init_load_pct) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - if (init_load_pct < 0 || init_load_pct > 100) - return -EINVAL; - - wts->init_load_pct = init_load_pct; - - return 0; -} - static void init_new_task_load(struct task_struct *p) { int i; @@ -2261,42 +2220,12 @@ static void walt_task_dead(struct task_struct *p) sched_set_group_id(p, 0); } -static void reset_task_stats(struct task_struct *p) -{ - int i = 0; - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - memset(wts->curr_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); - memset(wts->prev_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); - - wts->mark_start = 0; - wts->sum = 0; - wts->demand = 0; - wts->coloc_demand = 0; - for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) - wts->sum_history[i] = 0; - wts->curr_window = 0; - wts->prev_window = 0; - wts->pred_demand = 0; - for (i = 0; i < NUM_BUSY_BUCKETS; ++i) - wts->busy_buckets[i] = 0; - wts->demand_scaled = 0; - wts->pred_demand_scaled = 0; - wts->active_time = 0; -} - static void mark_task_starting(struct task_struct *p) { u64 wallclock; struct rq *rq = task_rq(p); - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (!wrq->window_start) { - reset_task_stats(p); - return; - } - wallclock = sched_ktime_clock(); wts->mark_start = wts->last_wake_ts = wallclock; wts->last_enqueued_ts = wallclock; @@ -2307,14 +2236,14 @@ static void mark_task_starting(struct task_struct *p) * Task groups whose aggregate demand on a cpu is more than * sched_group_upmigrate need to be up-migrated if possible. */ -unsigned int __read_mostly sched_group_upmigrate = 20000000; +static unsigned int __read_mostly sched_group_upmigrate = 20000000; /* * Task groups, once up-migrated, will need to drop their aggregate * demand to less than sched_group_downmigrate before they are "down" * migrated. */ -unsigned int __read_mostly sched_group_downmigrate = 19000000; +static unsigned int __read_mostly sched_group_downmigrate = 19000000; void walt_update_group_thresholds(void) { @@ -2679,7 +2608,7 @@ static void transfer_busy_time(struct rq *rq, * The children inherits the group id from the parent. */ -struct walt_related_thread_group +static struct walt_related_thread_group *related_thread_groups[MAX_NUM_CGROUP_COLOC_ID]; static LIST_HEAD(active_related_thread_groups); static DEFINE_RWLOCK(related_thread_group_lock); @@ -3834,12 +3763,6 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) { - /* - * TODO: remove later. - * We don't have to check if p is ed task and clear it. the below - * code calls is_ed_task_present() which clears the rq's ed_task - * unconditionally. - */ struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; if (static_branch_unlikely(&walt_disabled)) @@ -3955,13 +3878,6 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) if (is_ed_task_present(rq, wallclock)) waltgov_run_callback(rq, WALT_CPUFREQ_EARLY_DET); - - /* TODO - * currently load balancer registered for a post-hook which - * takes care of rotation and migration for misfit tasks. - * - * See if that can also be done here. - */ } static void android_rvh_schedule(void *unused, struct task_struct *prev, @@ -4071,7 +3987,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL); register_trace_android_rvh_schedule(android_rvh_schedule, NULL); register_trace_android_rvh_resume_cpus(android_rvh_resume_cpus, NULL); - register_trace_android_vh_show_max_freq(android_vh_show_max_freq, NULL); register_trace_android_rvh_cpu_cgroup_attach(android_rvh_cpu_cgroup_attach, NULL); register_trace_android_rvh_update_cpus_allowed(android_rvh_update_cpus_allowed, NULL); register_trace_android_rvh_sched_fork_init(android_rvh_sched_fork_init, NULL); @@ -4145,6 +4060,7 @@ static void walt_init(void) init_clusters(); register_walt_hooks(); + walt_fixup_init(); walt_lb_init(); walt_rt_init(); walt_cfs_init(); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index a4c419829b3d..fc186cf98c6b 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -161,8 +161,6 @@ extern void walt_rotation_checkpoint(int nr_big); extern void walt_fill_ta_data(struct core_ctl_notif_data *data); extern int sched_set_group_id(struct task_struct *p, unsigned int group_id); extern unsigned int sched_get_group_id(struct task_struct *p); -extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct); -extern u32 sched_get_init_task_load(struct task_struct *p); extern void core_ctl_check(u64 wallclock); extern int sched_set_boost(int enable); extern int sched_wake_up_idle_show(struct seq_file *m, void *v); @@ -196,13 +194,7 @@ extern void walt_init_topapp_tg(struct task_group *tg); extern void walt_init_foreground_tg(struct task_group *tg); extern int register_walt_callback(void); extern void set_cpu_array(void); -extern int sched_init_ops(void); extern int core_ctl_init(void); -extern void acquire_rq_locks_irqsave(const cpumask_t *cpus, - unsigned long *flags); -extern void release_rq_locks_irqrestore(const cpumask_t *cpus, - unsigned long *flags); -extern struct list_head cluster_head; extern int input_boost_init(void); extern int core_ctl_init(void); @@ -239,6 +231,7 @@ extern unsigned int sysctl_walt_rtg_cfs_boost_prio; extern __read_mostly unsigned int sysctl_sched_force_lb_enable; extern const int sched_user_hint_max; +extern struct list_head cluster_head; #define for_each_sched_cluster(cluster) \ list_for_each_entry_rcu(cluster, &cluster_head, list) @@ -290,7 +283,6 @@ extern void walt_update_group_thresholds(void); extern void sched_window_nr_ticks_change(void); extern unsigned long sched_user_hint_reset_time; extern struct irq_work walt_migration_irq_work; -extern __read_mostly unsigned int new_sched_ravg_window; extern struct task_group *task_group_topapp; extern struct task_group *task_group_foreground; @@ -298,9 +290,6 @@ extern struct task_group *task_group_foreground; extern unsigned int cpuinfo_max_freq_cached; extern char sched_lib_name[LIB_PATH_LENGTH]; extern unsigned int sched_lib_mask_force; -extern bool is_sched_lib_based_app(pid_t pid); -void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy, - unsigned int *max_freq); /* WALT cpufreq interface */ #define WALT_CPUFREQ_ROLLOVER (1U << 0) @@ -760,6 +749,7 @@ extern void sched_update_hyst_times(void); extern enum sched_boost_policy sched_boost_policy(void); extern void walt_rt_init(void); extern void walt_cfs_init(void); +extern void walt_fixup_init(void); extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync, int sibling_count_hint); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 49509dd5e5e2..74fd9c221354 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -18,8 +18,6 @@ unsigned int sched_capacity_margin_down[WALT_NR_CPUS] = { [0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */ }; -__read_mostly unsigned int sysctl_sched_force_lb_enable = 1; - static inline bool bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) { diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 7caecafbd172..774f145cbc6e 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -542,6 +542,7 @@ static void walt_lb_tick(void *unused, struct rq *rq) raw_spin_unlock_irqrestore(&walt_lb_migration_lock, flags); } +__read_mostly unsigned int sysctl_sched_force_lb_enable = 1; static bool should_help_min_cap(int this_cpu) { int cpu; @@ -753,19 +754,8 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, *can_migrate = 0; } -/* - * when WALT becomes module, this init will be called from - * another file and we don't have to define module_init(). - */ void walt_lb_init(void) { - /* - * Any task movement outside task placement is called - * load balance, so moving the tick path and rotation - * code to here. we also use our custom active load balance - * stopper function instad of adding hooks to - * active_load_balance_cpu_stop() in fair.c - */ walt_lb_rotate_work_init(); register_trace_android_rvh_migrate_queued_task(walt_migrate_queued_task, NULL); From 7fc920beca6e201c971f60f80478affcac48c77b Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 8 Feb 2021 22:05:41 -0800 Subject: [PATCH 036/346] sched/walt: Temporarily disable sibling_count_hint The core kernel side has race conditions which lead to memory faults. Disable this feature temporarily. Change-Id: I0013753d571f1e7265ce385a1130f2671c574ab0 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 74fd9c221354..9812cf7c1d1e 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -666,7 +666,7 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *target_cpu) { int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); - int sibling_count_hint = p->wake_q_head ? p->wake_q_head->count : 1; + int sibling_count_hint = 1; if (static_branch_unlikely(&walt_disabled)) return; From cc2a0e6b28418502db4ac05b6a42a8ec593a1614 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 26 Jan 2021 15:36:05 -0800 Subject: [PATCH 037/346] sched/trace: support compat in enq/deq trace Currently there is no trace support for indicating whether an application is 32 or 64 bit compatible. Since SOCs going forward will have a mix of support for 32 bit vs 64 bit applications, this is needed information. Update the trace_sched_enq_deq output such that the task's compatibility (32 or 64) is printed. Change-Id: I780b6b290a77a8405dd268b45a1e7affd035036b Signed-off-by: Stephen Dickey --- kernel/sched/walt/trace.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 74db0fd41a22..cae159a18b11 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1027,6 +1027,7 @@ TRACE_EVENT(sched_enq_deq_task, __field(unsigned int, cpus_allowed) __field(unsigned int, demand) __field(unsigned int, pred_demand) + __field(bool, compat_thread) ), TP_fast_assign( @@ -1040,16 +1041,18 @@ TRACE_EVENT(sched_enq_deq_task, __entry->cpus_allowed = cpus_allowed; __entry->demand = task_load(p); __entry->pred_demand = task_pl(p); + __entry->compat_thread = is_compat_thread(task_thread_info(p)); ), - TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u", + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d", __entry->cpu, __entry->enqueue ? "enqueue" : "dequeue", __entry->comm, __entry->pid, __entry->prio, __entry->nr_running, __entry->rt_nr_running, __entry->cpus_allowed, __entry->demand, - __entry->pred_demand) + __entry->pred_demand, + __entry->compat_thread) ); TRACE_EVENT(walt_window_rollover, From b9df052aa283dc2b6c32a5a54b12c764e48669b6 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:41:44 -0700 Subject: [PATCH 038/346] sched/walt: implement select_task_rq_rt trace hook Implement select_task_rq_rt trace hook to be able to find suitable CPU for the RT tasks from wake-up path. Change-Id: Id1e8a4529d7391bf39a2a069ae37c146fb8a55c9 Signed-off-by: Satya Durga Srinivasu Prabhala Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_rt.c | 103 +++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 862df51479f9..332e254b13c9 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -8,7 +8,9 @@ #include "walt.h" #include "trace.h" -static void rt_energy_aware_wake_cpu(void *unused, struct task_struct *task, +static DEFINE_PER_CPU(cpumask_var_t, walt_local_cpu_mask); + +static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task, struct cpumask *lowest_mask, int ret, int *best_cpu) { int cpu; @@ -24,6 +26,7 @@ static void rt_energy_aware_wake_cpu(void *unused, struct task_struct *task, if (static_branch_unlikely(&walt_disabled)) return; + if (!ret) return; /* No targets found */ @@ -86,7 +89,103 @@ static void rt_energy_aware_wake_cpu(void *unused, struct task_struct *task, rcu_read_unlock(); } +#ifdef CONFIG_UCLAMP_TASK +static inline bool walt_rt_task_fits_capacity(struct task_struct *p, int cpu) +{ + unsigned int min_cap; + unsigned int max_cap; + unsigned int cpu_cap; + + min_cap = uclamp_eff_value(p, UCLAMP_MIN); + max_cap = uclamp_eff_value(p, UCLAMP_MAX); + + cpu_cap = capacity_orig_of(cpu); + + return cpu_cap >= min(min_cap, max_cap); +} +#else +static inline bool walt_rt_task_fits_capacity(struct task_struct *p, int cpu) +{ + return true; +} +#endif + +static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int cpu, + int sd_flag, int wake_flags, int *new_cpu) +{ + struct task_struct *curr; + struct rq *rq; + bool may_not_preempt; + int ret, target = -1; + struct cpumask *lowest_mask; + + if (static_branch_unlikely(&walt_disabled)) + return; + + /* For anything but wake ups, just return the task_cpu */ + if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) + return; + + *new_cpu = cpu; /* previous CPU as back up */ + rq = cpu_rq(cpu); + + rcu_read_lock(); + curr = READ_ONCE(rq->curr); /* unlocked access */ + + /* + * If the current task on @p's runqueue is a softirq task, + * it may run without preemption for a time that is + * ill-suited for a waiting RT task. Therefore, try to + * wake this RT task on another runqueue. + * + * Otherwise, just let it ride on the affined RQ and the + * post-schedule router will push the preempted task away + * + * This test is optimistic, if we get it wrong the load-balancer + * will have to sort it out. + * + * We take into account the capacity of the CPU to ensure it fits the + * requirement of the task - which is only important on heterogeneous + * systems like big.LITTLE. + */ + may_not_preempt = task_may_not_preempt(curr, cpu); + + lowest_mask = this_cpu_cpumask_var_ptr(walt_local_cpu_mask); + + /* + * If we're on asym system ensure we consider the different capacities + * of the CPUs when searching for the lowest_mask. + */ + ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, task, + lowest_mask, walt_rt_task_fits_capacity); + + walt_rt_energy_aware_wake_cpu(NULL, task, lowest_mask, ret, &target); + + /* + * If cpu is non-preemptible, prefer remote cpu + * even if it's running a higher-prio task. + * Otherwise: Don't bother moving it if the destination CPU is + * not running a lower priority task. + */ + if (target != -1 && + (may_not_preempt || task->prio < cpu_rq(target)->rt.highest_prio.curr)) + *new_cpu = target; + + rcu_read_unlock(); +} + void walt_rt_init(void) { - register_trace_android_rvh_find_lowest_rq(rt_energy_aware_wake_cpu, NULL); + unsigned int i; + + for_each_possible_cpu(i) { + if(!(zalloc_cpumask_var_node(&per_cpu(walt_local_cpu_mask, i), + GFP_KERNEL, cpu_to_node(i)))) { + pr_err("walt_local_cpu_mask alloc failed for cpu%d\n", i); + return; + } + } + + register_trace_android_rvh_select_task_rq_rt(walt_select_task_rq_rt, NULL); + register_trace_android_rvh_find_lowest_rq(walt_rt_energy_aware_wake_cpu, NULL); } From 5b8490ab113f005811b96a169186831ac8850ca6 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 11 Feb 2021 13:25:39 +0530 Subject: [PATCH 039/346] sched/walt: Fix walt_lb_find_busiest_lower_cap_cpu() condition walt_lb_find_busiest_lower_cap_cpu() is meant to be called when the destination CPU has more capacity than the source CPU group. However, the condition for calling this function is inverted. Change-Id: I03dc62f42795479d21f676f8e27459955229dc82 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_lb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 774f145cbc6e..a04f98ac50a1 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -470,7 +470,7 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask) if (capacity_orig_of(dst_cpu) == capacity_orig_of(fsrc_cpu)) busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, src_mask); - else if (capacity_orig_of(dst_cpu) < capacity_orig_of(fsrc_cpu)) + else if (capacity_orig_of(dst_cpu) > capacity_orig_of(fsrc_cpu)) busiest_cpu = walt_lb_find_busiest_lower_cap_cpu(dst_cpu, src_mask); else From 6fff83754b379c5c271248b1dd67beb0209efef2 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 12 Feb 2021 13:21:46 +0530 Subject: [PATCH 040/346] sched/walt: Fix misfit accounting issues Currently we are using task_fits_capacity() for misfit task accounting. This does not consider boost scenarios. We also end up accounting 100% tasks running on the highest capacity CPU as misfit. A task on the highest capacity CPU should never be treated as misfit. rq->misfit_task_load should use task_util() not task_load(). The later is task demand scaled to sched_ravg_window in nano seconds. Change-Id: I1f923e33530d2d04d7bfca9dd42920f799794864 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c7a4984a4659..fc9d05e02fbe 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3800,10 +3800,10 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p wts = (struct walt_task_struct *) p->android_vendor_data1; old_misfit = wts->misfit; - if (task_fits_capacity(p, capacity_orig_of(cpu_of(rq)), rq->cpu)) + if (task_fits_max(p, rq->cpu)) rq->misfit_task_load = 0; else - rq->misfit_task_load = task_load(p); + rq->misfit_task_load = task_util(p); misfit = rq->misfit_task_load; From c7afff3259fdcd66ba9ae477e205617774fb8dc9 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 12 Feb 2021 14:24:03 +0530 Subject: [PATCH 041/346] sched/walt: Don't migrate running task to a busy CPU Active migration of running tasks is allowed only onto idle CPUs. Add a check for it in walt_lb_find_busiest_lower_cap_cpu(). Change-Id: I195aadc8b5822d4f231edd2f57d9df5ef1a6e350 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_lb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index a04f98ac50a1..0e60824e0381 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -439,7 +439,9 @@ static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_ if (cpu_rq(i)->active_balance) continue; - if (cpu_rq(i)->cfs.h_nr_running < 2 && !wrq->walt_stats.nr_big_tasks) + /* active migration is allowed only to idle cpu */ + if (cpu_rq(i)->cfs.h_nr_running < 2 && + (!wrq->walt_stats.nr_big_tasks || !available_idle_cpu(dst_cpu))) continue; if (!walt_rotation_enabled && !cpu_overutilized(i)) From 0505931c2b97055305f9e102cc483ee7f2e78026 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 12 Feb 2021 15:39:09 +0530 Subject: [PATCH 042/346] sched/walt: Use unbounded cpu util in load balancer The utilization return by cpu_util() is bounded by the original capacity of the CPU. Load balancer uses the utilization to break the ties. Use unbounded cpu utilization so that the most loaded CPU gets help. Change-Id: Id16cea205e6eb07a42a1e86fda533cd825df6908 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_lb.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 0e60824e0381..61972bf366fa 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -8,6 +8,13 @@ #include "walt.h" #include "trace.h" +static inline unsigned long walt_lb_cpu_util(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->walt_stats.cumulative_runnable_avg_scaled; +} + static void walt_detach_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) { @@ -327,7 +334,7 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr if (cpu_rq(i)->cfs.h_nr_running < 2) continue; - util = cpu_util(i); + util = walt_lb_cpu_util(i); if (util < busiest_util) continue; @@ -358,7 +365,7 @@ static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; trace_walt_lb_cpu_util(i, wrq); - util = cpu_util(i); + util = walt_lb_cpu_util(i); total_cpus += 1; total_util += util; total_capacity += capacity_orig_of(i); @@ -426,7 +433,7 @@ static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_ trace_walt_lb_cpu_util(i, wrq); - util = cpu_util(i); + util = walt_lb_cpu_util(i); total_cpus += 1; total_util += util; total_capacity += capacity_orig_of(i); From 0f4dd4ed4fb51c355f27422924707deaafd2889a Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 12 Feb 2021 15:52:17 +0530 Subject: [PATCH 043/346] sched/walt: Use nr_running > 1 condition for same cluster lb FIFO/RT tasks are supposed to run for short amount of time. So when a CPU is busy with 1 RT and 1 CFS task, the other idle CPU won't offer help thinking the CFS task will run immediately. However, results indicate that realaxing this policy and pulling the CFS task helps in reducing the runnables. This policy is limited to the same capacity cluster load balance as we want strict prefer idle behavior for it. Change-Id: I4df6e00233503e096b1bb87fd17a784ea53fd9be Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_lb.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 61972bf366fa..f19ca601cbec 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -323,7 +323,6 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr { int i; int busiest_cpu = -1; - int busiest_nr = 1; /* we need atleast 2 */ unsigned long util, busiest_util = 0; struct walt_rq *wrq; @@ -331,14 +330,13 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; trace_walt_lb_cpu_util(i, wrq); - if (cpu_rq(i)->cfs.h_nr_running < 2) + if (cpu_rq(i)->nr_running < 2 || !cpu_rq(i)->cfs.h_nr_running) continue; util = walt_lb_cpu_util(i); if (util < busiest_util) continue; - busiest_nr = cpu_rq(i)->cfs.h_nr_running; busiest_util = util; busiest_cpu = i; } @@ -351,7 +349,6 @@ static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src { int i; int busiest_cpu = -1; - int busiest_nr = 1; /* we need atleast 2 */ unsigned long util, busiest_util = 0; unsigned long total_capacity = 0, total_util = 0, total_nr = 0; int total_cpus = 0; @@ -390,7 +387,6 @@ static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src if (util < busiest_util) continue; - busiest_nr = cpu_rq(i)->cfs.h_nr_running; busiest_util = util; busiest_cpu = i; } @@ -411,7 +407,6 @@ static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_ { int i; int busiest_cpu = -1; - int busiest_nr = 1; /* we need atleast 2 */ unsigned long util, busiest_util = 0; unsigned long total_capacity = 0, total_util = 0, total_nr = 0; int total_cpus = 0; @@ -457,7 +452,6 @@ static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_ if (util < busiest_util) continue; - busiest_nr = cpu_rq(i)->cfs.h_nr_running; busiest_util = util; busiest_cpu = i; busy_nr_big_tasks = wrq->walt_stats.nr_big_tasks; From 2da77febb22a1a6bb3f5b2e19c9b58dc068dbfef Mon Sep 17 00:00:00 2001 From: Kishore Sri venkata Ganesh Bolisetty Date: Wed, 20 Jan 2021 00:06:53 -0800 Subject: [PATCH 044/346] binder: update low_latency selection for binder transactions The changes enables low latency for 120 priority FG binder transactions in the rendering path. Change-Id: I5736611c7f139362c54d0f0e73f07dfe078c8848 Signed-off-by: Kishore Sri venkata Ganesh Bolisetty Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_cfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 9812cf7c1d1e..f0f90c546107 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -752,7 +752,8 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task) return; if (task && current->signal && (current->signal->oom_score_adj == 0) && - (current->prio < DEFAULT_PRIO)) + ((current->prio < DEFAULT_PRIO) || + (task->group_leader->prio < MAX_RT_PRIO))) wts->low_latency |= WALT_LOW_LATENCY_BINDER; } From 581b1bb7eed133ca9eb1dfc4eb0123ae9b464cd6 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:42:16 -0700 Subject: [PATCH 045/346] sched/walt: Implement balance_rt vendor hook The balance_rt vendor hook is implemented to pull a runnable RT task from the other CPU. Change-Id: I288fc9a8858cb29ae9c6993b398a349b4a8c7475 Signed-off-by: Pavankumar Kondeti Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_lb.c | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index f19ca601cbec..fb290403daae 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -757,6 +757,70 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, *can_migrate = 0; } +static inline int rt_overloaded(struct rq *rq) +{ + return atomic_read(&rq->rd->rto_count); +} + +static inline int has_pushable_tasks(struct rq *rq) +{ + return !plist_head_empty(&rq->rt.pushable_tasks); +} + +#define WALT_RT_PULL_THRESHOLD_NS 250000 +static void walt_balance_rt(void *unused, struct rq *this_rq, + struct task_struct *prev, int *done) +{ + int i, this_cpu = this_rq->cpu, src_cpu = this_cpu; + struct rq *src_rq; + struct task_struct *p; + struct walt_task_struct *wts; + + /* Let RT push/pull handle the overloaded scenario */ + if (rt_overloaded(this_rq)) + return; + + /* can't help if this has a runnable RT */ + if (sched_rt_runnable(this_rq)) + return; + + /* check if any CPU has a pushable RT task */ + for_each_possible_cpu(i) { + struct rq *rq = cpu_rq(i); + + if (!has_pushable_tasks(rq)) + continue; + + src_cpu = i; + break; + } + + if (src_cpu == this_cpu) + return; + + src_rq = cpu_rq(src_cpu); + double_lock_balance(this_rq, src_rq); + + /* lock is dropped, so check again */ + if (sched_rt_runnable(this_rq)) + goto unlock; + + p = pick_highest_pushable_task(src_rq, this_cpu); + + if (!p) + goto unlock; + + wts = (struct walt_task_struct *) p->android_vendor_data1; + if (sched_ktime_clock() - wts->last_wake_ts < WALT_RT_PULL_THRESHOLD_NS) + goto unlock; + + deactivate_task(src_rq, p, 0); + set_task_cpu(p, this_cpu); + activate_task(this_rq, p, 0); +unlock: + double_unlock_balance(this_rq, src_rq); +} + void walt_lb_init(void) { walt_lb_rotate_work_init(); @@ -767,4 +831,5 @@ void walt_lb_init(void) register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); register_trace_android_rvh_sched_newidle_balance(walt_newidle_balance, NULL); register_trace_android_vh_scheduler_tick(walt_lb_tick, NULL); + register_trace_android_rvh_sched_balance_rt(walt_balance_rt, NULL); } From 9159c2eef902b963924adac9bdf4cf1ea09609ae Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 15 Feb 2021 14:28:21 +0530 Subject: [PATCH 046/346] sched/walt: Remove duplicate restricted vendor hook registration WALT core module register for android_rvh_update_misfit_status restricted vendor hook. Remove the duplicate registration from WALT CFS module. The later registered function has a compilation issue when FAIR_GROUP_SCHED is enabled. Change-Id: I2d89efa92ca704941f12263ee4994dc389202d5f Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_cfs.c | 41 ------------------------------------ 1 file changed, 41 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index f0f90c546107..78340f781fe3 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -675,46 +675,6 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, *target_cpu = prev_cpu; } -#ifdef CONFIG_FAIR_GROUP_SCHED -static unsigned long task_h_load(struct task_struct *p) -{ - struct cfs_rq *cfs_rq = task_cfs_rq(p); - - update_cfs_rq_h_load(cfs_rq); - return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, - cfs_rq_load_avg(cfs_rq) + 1); -} -#else -static unsigned long task_h_load(struct task_struct *p) -{ - return p->se.avg.load_avg; -} -#endif - -static void walt_update_misfit_status(void *unused, struct task_struct *p, - struct rq *rq, bool *need_update) -{ - if (static_branch_unlikely(&walt_disabled)) - return; - *need_update = false; - - if (!p) { - rq->misfit_task_load = 0; - return; - } - - if (task_fits_max(p, cpu_of(rq))) { - rq->misfit_task_load = 0; - return; - } - - /* - * Make sure that misfit_task_load will not be null even if - * task_h_load() returns 0. - */ - rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); -} - static inline struct task_struct *task_of(struct sched_entity *se) { return container_of(se, struct task_struct, se); @@ -770,7 +730,6 @@ static void walt_binder_low_latency_clear(void *unused, struct binder_transactio void walt_cfs_init(void) { register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL); - register_trace_android_rvh_update_misfit_status(walt_update_misfit_status, NULL); register_trace_android_rvh_place_entity(walt_place_entity, NULL); register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL); From eed86c5eb234ca9f5c07fb34bb2598fc25b46f5c Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Wed, 17 Feb 2021 13:43:19 +0530 Subject: [PATCH 047/346] sched/walt: Add fair task check in tick path load balance The tick path load balance is meant for fair tasks. Add a check for the same before considering the task for migration. The fair_policy() check alone does not work as it returns true for idle task and also for RT boosted tasks. Add a wrapper based on priority check. Change-Id: I05cc996cb15051e4f6bb1a73005234e4d5188d19 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 4 ++-- kernel/sched/walt/walt.h | 9 +++++++++ kernel/sched/walt/walt_lb.c | 9 ++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index fc9d05e02fbe..cdab278259e1 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3752,7 +3752,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st wts->last_enqueued_ts = wallclock; sched_update_nr_prod(rq->cpu, true); - if (fair_policy(p->policy)) { + if (walt_fair_task(p)) { wts->misfit = !task_fits_max(p, rq->cpu); inc_rq_walt_stats(rq, p); } @@ -3772,7 +3772,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st sched_update_nr_prod(rq->cpu, false); - if (fair_policy(p->policy)) + if (walt_fair_task(p)) dec_rq_walt_stats(rq, p); walt_dec_cumulative_runnable_avg(rq, p); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index fc186cf98c6b..f16314775ce6 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -858,4 +858,13 @@ static inline struct task_group *css_tg(struct cgroup_subsys_state *css) return css ? container_of(css, struct task_group, css) : NULL; } +/* + * The policy of a RT boosted task (via PI mutex) still indicates it is + * a fair task, so use prio check as well. The prio check alone is not + * sufficient since idle task also has 120 priority. + */ +static inline bool walt_fair_task(struct task_struct *p) +{ + return p->prio >= MAX_RT_PRIO && !is_idle_task(p); +} #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index fb290403daae..c5eb0e5a31e8 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -144,7 +144,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) if (is_reserved(i)) continue; - if (!rq->misfit_task_load) + if (!rq->misfit_task_load || !walt_fair_task(rq->curr)) continue; wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; @@ -167,7 +167,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) if (is_reserved(i)) continue; - if (rq->curr->prio < MAX_RT_PRIO) + if (!walt_fair_task(rq->curr)) continue; if (rq->nr_running > 1) @@ -191,8 +191,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) dst_rq = cpu_rq(dst_cpu); double_rq_lock(src_rq, dst_rq); - if (dst_rq->curr->prio >= MAX_RT_PRIO && dst_rq->curr != dst_rq->idle && - src_rq->curr->prio >= MAX_RT_PRIO && src_rq->curr != src_rq->idle) { + if (walt_fair_task(dst_rq->curr)) { get_task_struct(src_rq->curr); get_task_struct(dst_rq->curr); @@ -494,7 +493,7 @@ static void walt_lb_tick(void *unused, struct rq *rq) if (static_branch_unlikely(&walt_disabled)) return; - if (!rq->misfit_task_load) + if (!rq->misfit_task_load || !walt_fair_task(p)) return; if (p->state != TASK_RUNNING || p->nr_cpus_allowed == 1) From 81647d6a8478955ce8c41c89ed82502cc4024da3 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Wed, 17 Feb 2021 14:49:17 +0530 Subject: [PATCH 048/346] sched/walt: Remove left over references to cum_window_demand The cum_window_demand tracking is not supported in WALT. Remove the left over code and use the PELT signal based util_avg as replacement in RT task placement. The CFS task placement is already using the same. Change-Id: I078353ed33729b7bf4eab324c7ec861982a3bc57 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/trace.h | 2 +- kernel/sched/walt/walt.c | 4 ---- kernel/sched/walt/walt.h | 32 ++------------------------------ kernel/sched/walt/walt_cfs.c | 7 ++----- kernel/sched/walt/walt_rt.c | 2 +- 5 files changed, 6 insertions(+), 41 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index cae159a18b11..7ef3ecd88ee7 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -823,7 +823,7 @@ TRACE_EVENT(sched_cpu_util, __entry->cpu = cpu; __entry->nr_running = cpu_rq(cpu)->nr_running; __entry->cpu_util = cpu_util(cpu); - __entry->cpu_util_cum = cpu_util_cum(cpu, 0); + __entry->cpu_util_cum = cpu_util_cum(cpu); __entry->capacity_curr = capacity_curr_of(cpu); __entry->capacity = capacity_of(cpu); __entry->capacity_orig = capacity_orig_of(cpu); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index cdab278259e1..44aedc2579c6 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -240,7 +240,6 @@ static inline void walt_rq_dump(int cpu) SCHED_PRINT(wrq->prev_runnable_sum); SCHED_PRINT(wrq->nt_curr_runnable_sum); SCHED_PRINT(wrq->nt_prev_runnable_sum); - SCHED_PRINT(wrq->cum_window_demand_scaled); SCHED_PRINT(wrq->task_exec_scale); SCHED_PRINT(wrq->grp_time.curr_runnable_sum); SCHED_PRINT(wrq->grp_time.prev_runnable_sum); @@ -376,8 +375,6 @@ update_window_start(struct rq *rq, u64 wallclock, int event) nr_windows = div64_u64(delta, sched_ravg_window); wrq->window_start += (u64)nr_windows * (u64)sched_ravg_window; - wrq->cum_window_demand_scaled = - wrq->walt_stats.cumulative_runnable_avg_scaled; wrq->prev_window_size = sched_ravg_window; return old_window_start; @@ -3571,7 +3568,6 @@ static void walt_sched_init_rq(struct rq *rq) BUG_ON(!wrq->top_tasks[j]); clear_top_tasks_bitmap(wrq->top_tasks_bitmap[j]); } - wrq->cum_window_demand_scaled = 0; wrq->notif_pending = false; } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index f16314775ce6..8360b39ba3b9 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -114,7 +114,6 @@ struct walt_rq { u64 prev_runnable_sum; u64 nt_curr_runnable_sum; u64 nt_prev_runnable_sum; - u64 cum_window_demand_scaled; struct group_cpu_time grp_time; struct load_subtractions load_subs[NUM_TRACKED_WINDOWS]; DECLARE_BITMAP_ARRAY(top_tasks_bitmap, @@ -401,17 +400,9 @@ static inline unsigned long cpu_util(int cpu) return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu)); } -static inline unsigned long cpu_util_cum(int cpu, int delta) +static inline unsigned long cpu_util_cum(int cpu) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; - u64 util = wrq->cum_window_demand_scaled; - unsigned long capacity = capacity_orig_of(cpu); - - delta += util; - if (delta < 0) - return 0; - - return (delta >= capacity) ? capacity : delta; + return READ_ONCE(cpu_rq(cpu)->cfs.avg.util_avg); } static inline enum sched_boost_policy sched_boost_policy(void) @@ -826,25 +817,6 @@ static inline void clear_reserved(int cpu) clear_bit(CPU_RESERVED, &wrq->walt_flags); } -static inline bool -task_in_cum_window_demand(struct rq *rq, struct task_struct *p) -{ - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return cpu_of(rq) == task_cpu(p) && (p->on_rq || - wts->last_sleep_ts >= wrq->window_start); -} - -static inline void walt_fixup_cum_window_demand(struct rq *rq, s64 scaled_delta) -{ - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - - wrq->cum_window_demand_scaled += scaled_delta; - if (unlikely((s64)wrq->cum_window_demand_scaled < 0)) - wrq->cum_window_demand_scaled = 0; -} - static inline void walt_irq_work_queue(struct irq_work *work) { if (likely(cpu_online(raw_smp_processor_id()))) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 78340f781fe3..98a094de65b5 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -193,7 +193,6 @@ static void walt_find_best_target(struct sched_domain *sd, bool rtg_high_prio_task = task_rtg_high_prio(p); cpumask_t visit_cpus; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - struct cfs_rq *cfs_rq; /* Find start CPU based on boost value */ start_cpu = fbt_env->start_cpu; @@ -271,10 +270,6 @@ static void walt_find_best_target(struct sched_domain *sd, TASK_BOOST_STRICT_MAX) continue; - /* get rq's utilization with this task included */ - cfs_rq = &cpu_rq(i)->cfs; - new_util_cuml = READ_ONCE(cfs_rq->avg.util_avg) + min_util; - /* * Ensure minimum capacity to grant the required boost. * The target CPU can be already at a capacity level higher @@ -320,6 +315,8 @@ static void walt_find_best_target(struct sched_domain *sd, */ if (idle_exit_latency > min_exit_latency) continue; + + new_util_cuml = cpu_util_cum(i); if (min_exit_latency == idle_exit_latency && (best_idle_cpu == prev_cpu || (i != prev_cpu && diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 332e254b13c9..d08e730fcbce 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -66,7 +66,7 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task */ cpu_idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(cpu)); - util_cum = cpu_util_cum(cpu, 0); + util_cum = cpu_util_cum(cpu); if (cpu != task_cpu(task) && best_cpu_util == util) { if (best_idle_exit_latency < cpu_idle_exit_latency) continue; From 3789d05a044ad03f00aecd17bc20d5b1c8db97dd Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Wed, 17 Feb 2021 15:00:40 +0530 Subject: [PATCH 049/346] sched/walt: Fix !idle case in walt_get_idle_exit_latency When a CPU is not in idle state, the idle exit latency should be 0. However walt_get_idle_exit_latency() returns UINT_MAX. Fix this. Without this, the idle CPU that is serving an interrupt will not win the battle against a CPU that is in a deep c-state. Change-Id: Ic9ca36add5cd97cc7174c8e25b269ae40aadd451 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 8360b39ba3b9..934e3fadbba2 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -731,7 +731,7 @@ static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) if (idle) return idle->exit_latency; - return UINT_MAX; + return 0; /* CPU is not idle */ } extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); From 9d253eadc1400125101e2be809f4e56484f93885 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Sun, 21 Feb 2021 06:34:10 +0530 Subject: [PATCH 050/346] sched/walt: Track CPUs that can't be paused The first CPU that supports AArch32 in EL0 on asymmetric AArch32 systems can't be paused. Track such CPUs in core_ctl to not to request pause_cpus(). Change-Id: I4a9132884c3e322e9a6bd7d1f06a82bd9113eec8 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/core_ctl.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 6d0bed0bfbda..307d0788e913 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -53,6 +53,7 @@ struct cpu_data { struct cluster_data *cluster; struct list_head sib; bool paused_by_us; + bool disabled; }; static DEFINE_PER_CPU(struct cpu_data, cpu_state); @@ -1007,6 +1008,8 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, if (!num_cpus--) break; + if (c->disabled) + continue; if (!is_active(c)) continue; if (active_cpus - nr_pending == need) @@ -1043,6 +1046,8 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, if (!num_cpus--) break; + if (c->disabled) + continue; if (!is_active(c)) continue; if (active_cpus - nr_pending <= cluster->max_cpus) @@ -1246,6 +1251,8 @@ static int cluster_init(const struct cpumask *mask) state = &per_cpu(cpu_state, cpu); state->cluster = cluster; state->cpu = cpu; + state->disabled = get_cpu_device(cpu) && + get_cpu_device(cpu)->offline_disabled; list_add_tail(&state->sib, &cluster->lru); } cluster->active_cpus = get_active_cpu_count(cluster); From cf02ed946bcbba1ad2929ae9e618663eaa826948 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:43:31 -0700 Subject: [PATCH 051/346] sched/pause: support refcounting on pause cpus operation It is possible that a cpu paused with thermal might be unpaused by core-ctl, as well as the reverse. This means that a cpu that has been paused for thermal reasons can be unpaused when there is need. Ensure that cpus paused by more than one entity are sticky, such that only the last unpause request will actually perform the operation. Change-Id: Ieb2f836af7348d48a43ae1b22e11d97046ef9dea Signed-off-by: Stephen Dickey Signed-off-by: Sai Harshini Nimmala --- drivers/soc/qcom/hyp_core_ctl.c | 1099 ++++++++++++++++++++++++++ drivers/thermal/qcom/thermal_pause.c | 441 +++++++++++ include/linux/sched/walt.h | 10 + kernel/sched/walt/Makefile | 2 +- kernel/sched/walt/core_ctl.c | 5 +- kernel/sched/walt/walt_pause.c | 96 +++ 6 files changed, 1650 insertions(+), 3 deletions(-) create mode 100644 drivers/soc/qcom/hyp_core_ctl.c create mode 100644 drivers/thermal/qcom/thermal_pause.c create mode 100644 kernel/sched/walt/walt_pause.c diff --git a/drivers/soc/qcom/hyp_core_ctl.c b/drivers/soc/qcom/hyp_core_ctl.c new file mode 100644 index 000000000000..ae9eac7a548c --- /dev/null +++ b/drivers/soc/qcom/hyp_core_ctl.c @@ -0,0 +1,1099 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) "hyp_core_ctl: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#define MAX_RESERVE_CPUS (num_possible_cpus()/2) + +static DEFINE_PER_CPU(struct freq_qos_request, qos_min_req); +static DEFINE_PER_CPU(unsigned int, qos_min_freq); + +/** + * struct hyp_core_ctl_cpumap - vcpu to pcpu mapping for the other guest + * @cap_id: System call id to be used while referring to this vcpu + * @pcpu: The physical CPU number corresponding to this vcpu + * @curr_pcpu: The current physical CPU number corresponding to this vcpu. + * The curr_pcu is set to another CPU when the original assigned + * CPU i.e pcpu can't be used due to thermal condition. + * + */ +struct hyp_core_ctl_cpu_map { + hh_capid_t cap_id; + hh_label_t pcpu; + hh_label_t curr_pcpu; +}; + +/** + * struct hyp_core_ctl_data - The private data structure of this driver + * @lock: spinlock to serialize task wakeup and enable/reserve_cpus + * @task: task_struct pointer to the thread running the state machine + * @pending: state machine work pending status + * @reservation_enabled: status of the reservation + * @reservation_mutex: synchronization between thermal handling and + * reservation. The physical CPUs are re-assigned + * during thermal conditions while reservation is + * not enabled. So this synchronization is needed. + * @reserve_cpus: The CPUs to be reserved. input. + * @our_paused_cpus: The CPUs paused by hyp_core_ctl driver. output. + * @final_reserved_cpus: The CPUs reserved for the Hypervisor. output. + * @cpumap: The vcpu to pcpu mapping table + */ +struct hyp_core_ctl_data { + spinlock_t lock; + struct task_struct *task; + bool pending; + bool reservation_enabled; + struct mutex reservation_mutex; + cpumask_t reserve_cpus; + cpumask_t our_paused_cpus; + cpumask_t final_reserved_cpus; + struct hyp_core_ctl_cpu_map cpumap[NR_CPUS]; +}; + +#define CREATE_TRACE_POINTS +#include "hyp_core_ctl_trace.h" + +static struct hyp_core_ctl_data *the_hcd; +static struct hyp_core_ctl_cpu_map hh_cpumap[NR_CPUS]; +static bool is_vcpu_info_populated; +static bool init_done; +static int nr_vcpus; +static bool freq_qos_init_done; + +static inline void hyp_core_ctl_print_status(char *msg) +{ + trace_hyp_core_ctl_status(the_hcd, msg); + + pr_debug("%s: reserve=%*pbl reserved=%*pbl our_paused=%*pbl online=%*pbl active=%*pbl thermal=%*pbl\n", + msg, cpumask_pr_args(&the_hcd->reserve_cpus), + cpumask_pr_args(&the_hcd->final_reserved_cpus), + cpumask_pr_args(&the_hcd->our_paused_cpus), + cpumask_pr_args(cpu_online_mask), + cpumask_pr_args(cpu_active_mask), + cpumask_pr_args(cpu_cooling_multi_get_max_level_cpumask())); +} + +static inline int pause_cpu(int cpu) +{ + cpumask_t cpus_to_pause; + int ret; + + cpumask_clear(&cpus_to_pause); + cpumask_set_cpu(cpu, &cpus_to_pause); + + ret = walt_pause_cpus(&cpus_to_pause); + + return ret; +} + +static inline int resume_cpu(int cpu) +{ + cpumask_t cpus_to_resume; + int ret; + + cpumask_clear(&cpus_to_resume); + cpumask_set_cpu(cpu, &cpus_to_resume); + + ret = walt_resume_cpus(&cpus_to_resume); + + return ret; +} + +static void hyp_core_ctl_undo_reservation(struct hyp_core_ctl_data *hcd) +{ + int cpu, ret; + struct freq_qos_request *qos_req; + + hyp_core_ctl_print_status("undo_reservation_start"); + + for_each_cpu(cpu, &hcd->our_paused_cpus) { + ret = resume_cpu(cpu); + if (ret < 0) { + pr_err("fail to un-pause CPU%d. ret=%d\n", cpu, ret); + continue; + } + + cpumask_clear_cpu(cpu, &hcd->our_paused_cpus); + + if (freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, cpu); + ret = freq_qos_update_request(qos_req, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + cpu, ret); + } + } + + hyp_core_ctl_print_status("undo_reservation_end"); +} + +static void finalize_reservation(struct hyp_core_ctl_data *hcd, cpumask_t *temp) +{ + cpumask_t vcpu_adjust_mask; + int i, orig_cpu, curr_cpu, replacement_cpu; + int err; + + /* + * When thermal conditions are not present, we return + * from here. + */ + if (cpumask_equal(temp, &hcd->final_reserved_cpus)) + return; + + /* + * When we can't match with the original reserve CPUs request, + * don't change the existing scheme. We can't assign the + * same physical CPU to multiple virtual CPUs. + * + * This may only happen when thermal pause more CPUs. + */ + if (cpumask_weight(temp) < cpumask_weight(&hcd->reserve_cpus)) { + pr_debug("Fail to reserve some CPUs\n"); + return; + } + + cpumask_copy(&hcd->final_reserved_cpus, temp); + cpumask_clear(&vcpu_adjust_mask); + + /* + * In the first pass, we traverse all virtual CPUs and try + * to assign their original physical CPUs if they are + * reserved. if the original physical CPU is not reserved, + * then check the current physical CPU is reserved or not. + * so that we continue to use the current physical CPU. + * + * If both original CPU and the current CPU are not reserved, + * we have to find a replacement. These virtual CPUs are + * maintained in vcpu_adjust_mask and processed in the 2nd pass. + */ + for (i = 0; i < MAX_RESERVE_CPUS; i++) { + if (hcd->cpumap[i].cap_id == 0) + break; + + orig_cpu = hcd->cpumap[i].pcpu; + curr_cpu = hcd->cpumap[i].curr_pcpu; + + if (cpumask_test_cpu(orig_cpu, &hcd->final_reserved_cpus)) { + cpumask_clear_cpu(orig_cpu, temp); + + if (orig_cpu == curr_cpu) + continue; + + /* + * The original pcpu corresponding to this vcpu i.e i + * is available in final_reserved_cpus. so restore + * the assignment. + */ + err = hh_hcall_vcpu_affinity_set(hcd->cpumap[i].cap_id, + orig_cpu); + if (err != HH_ERROR_OK) { + pr_err("restore: fail to assign pcpu for vcpu#%d err=%d cap_id=%d cpu=%d\n", + i, err, hcd->cpumap[i].cap_id, orig_cpu); + continue; + } + + hcd->cpumap[i].curr_pcpu = orig_cpu; + pr_debug("err=%u vcpu=%d pcpu=%u curr_cpu=%u\n", + err, i, hcd->cpumap[i].pcpu, + hcd->cpumap[i].curr_pcpu); + continue; + } + + /* + * The original CPU is not available but the previously + * assigned CPU i.e curr_cpu is still available. so keep + * using it. + */ + if (cpumask_test_cpu(curr_cpu, &hcd->final_reserved_cpus)) { + cpumask_clear_cpu(curr_cpu, temp); + continue; + } + + /* + * A replacement CPU is found in the 2nd pass below. Make + * a note of this virtual CPU for which both original and + * current physical CPUs are not available in the + * final_reserved_cpus. + */ + cpumask_set_cpu(i, &vcpu_adjust_mask); + } + + /* + * The vcpu_adjust_mask contain the virtual CPUs that needs + * re-assignment. The temp CPU mask contains the remaining + * reserved CPUs. so we pick one by one from the remaining + * reserved CPUs and assign them to the pending virtual + * CPUs. + */ + for_each_cpu(i, &vcpu_adjust_mask) { + replacement_cpu = cpumask_any(temp); + cpumask_clear_cpu(replacement_cpu, temp); + + err = hh_hcall_vcpu_affinity_set(hcd->cpumap[i].cap_id, + replacement_cpu); + if (err != HH_ERROR_OK) { + pr_err("adjust: fail to assign pcpu for vcpu#%d err=%d cap_id=%d cpu=%d\n", + i, err, hcd->cpumap[i].cap_id, replacement_cpu); + continue; + } + + hcd->cpumap[i].curr_pcpu = replacement_cpu; + pr_debug("adjust err=%u vcpu=%d pcpu=%u curr_cpu=%u\n", + err, i, hcd->cpumap[i].pcpu, + hcd->cpumap[i].curr_pcpu); + + } + + /* Did we reserve more CPUs than needed? */ + WARN_ON(!cpumask_empty(temp)); +} + +static void hyp_core_ctl_do_reservation(struct hyp_core_ctl_data *hcd) +{ + cpumask_t offline_cpus, iter_cpus, temp_reserved_cpus; + int i, ret, pause_req, pause_done; + const cpumask_t *thermal_cpus = cpu_cooling_multi_get_max_level_cpumask(); + struct freq_qos_request *qos_req; + unsigned int min_freq; + + cpumask_clear(&offline_cpus); + cpumask_clear(&temp_reserved_cpus); + + hyp_core_ctl_print_status("reservation_start"); + + /* + * Iterate all reserve CPUs and pause them if not done already. + * The offline CPUs can't be paused but they are considered + * reserved. When an offline and reserved CPU comes online, it + * will be paused to honor the reservation. + */ + cpumask_andnot(&iter_cpus, &hcd->reserve_cpus, &hcd->our_paused_cpus); + cpumask_andnot(&iter_cpus, &iter_cpus, thermal_cpus); + + for_each_cpu(i, &iter_cpus) { + if (!cpu_online(i)) { + cpumask_set_cpu(i, &offline_cpus); + continue; + } + + ret = pause_cpu(i); + if (ret < 0) { + pr_debug("fail to pause CPU%d. ret=%d\n", i, ret); + continue; + } + + cpumask_set_cpu(i, &hcd->our_paused_cpus); + + min_freq = per_cpu(qos_min_freq, i); + if (min_freq && freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, i); + ret = freq_qos_update_request(qos_req, min_freq); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + i, ret); + } + } + + cpumask_andnot(&iter_cpus, &hcd->reserve_cpus, &offline_cpus); + pause_req = cpumask_weight(&iter_cpus); + pause_done = cpumask_weight(&hcd->our_paused_cpus); + + if (pause_done < pause_req) { + int pause_need; + + /* + * We have paused fewer CPUs than required. This happens + * when some of the CPUs from the reserved_cpus mask + * are managed by thermal. Find the replacement CPUs and + * pause them. + */ + pause_need = pause_req - pause_done; + + /* + * Create a cpumask from which replacement CPUs can be + * picked. Exclude our paused CPUs, thermal managed + * CPUs and offline CPUs, which are already considered + * as reserved. + */ + cpumask_andnot(&iter_cpus, cpu_possible_mask, + &hcd->our_paused_cpus); + cpumask_andnot(&iter_cpus, &iter_cpus, thermal_cpus); + cpumask_andnot(&iter_cpus, &iter_cpus, &offline_cpus); + + /* + * Keep the replacement policy simple. The offline CPUs + * comes for free. so pick them first. + */ + for_each_cpu(i, &iter_cpus) { + if (!cpu_online(i)) { + cpumask_set_cpu(i, &offline_cpus); + if (--pause_need == 0) + goto done; + } + } + + cpumask_andnot(&iter_cpus, &iter_cpus, &offline_cpus); + + for_each_cpu(i, &iter_cpus) { + ret = pause_cpu(i); + if (ret < 0) { + pr_debug("fail to pause CPU%d. ret=%d\n", + i, ret); + continue; + } + + cpumask_set_cpu(i, &hcd->our_paused_cpus); + + min_freq = per_cpu(qos_min_freq, i); + if (min_freq && freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, i); + ret = freq_qos_update_request(qos_req, + min_freq); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + i, ret); + } + + if (--pause_need == 0) + break; + } + } else if (pause_done > pause_req) { + int unpause_need; + + /* + * We have paused more CPUs than required. Un-pause + * the additional CPUs which are not part of the + * reserve_cpus mask. + * + * This happens in the following scenario. + * + * - Lets say reserve CPUs are CPU4 and CPU5. They are + * paused. + * - CPU4 is paused by thermal. We found CPU0 as the + * replacement CPU. Now CPU0 and CPU5 are paused by + * us. + * - CPU4 is un-paused by thermal. We first pause CPU4 + * since it is part of our reserve CPUs. Now CPU0, CPU4 + * and CPU5 are paused by us. + * - Since pause_done (3) > pause_req (2), un-pause + * a CPU which is not part of the reserve CPU. i.e CPU0. + */ + unpause_need = pause_done - pause_req; + cpumask_andnot(&iter_cpus, &hcd->our_paused_cpus, + &hcd->reserve_cpus); + for_each_cpu(i, &iter_cpus) { + ret = resume_cpu(i); + if (ret < 0) { + pr_err("fail to unpause CPU%d. ret=%d\n", + i, ret); + continue; + } + + cpumask_clear_cpu(i, &hcd->our_paused_cpus); + + if (freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, i); + ret = freq_qos_update_request(qos_req, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + i, ret); + } + + if (--unpause_need == 0) + break; + } + } + +done: + cpumask_or(&temp_reserved_cpus, &hcd->our_paused_cpus, &offline_cpus); + finalize_reservation(hcd, &temp_reserved_cpus); + + hyp_core_ctl_print_status("reservation_end"); +} + +static int hyp_core_ctl_thread(void *data) +{ + struct hyp_core_ctl_data *hcd = data; + + while (1) { + spin_lock(&hcd->lock); + if (!hcd->pending) { + set_current_state(TASK_INTERRUPTIBLE); + spin_unlock(&hcd->lock); + + schedule(); + + spin_lock(&hcd->lock); + set_current_state(TASK_RUNNING); + } + hcd->pending = false; + spin_unlock(&hcd->lock); + + if (kthread_should_stop()) + break; + + /* + * The reservation mutex synchronize the reservation + * happens in this thread against the thermal handling. + * The CPU re-assignment happens directly from the + * thermal callback context when the reservation is + * not enabled, since there is no need for isolating. + */ + mutex_lock(&hcd->reservation_mutex); + if (hcd->reservation_enabled) + hyp_core_ctl_do_reservation(hcd); + else + hyp_core_ctl_undo_reservation(hcd); + mutex_unlock(&hcd->reservation_mutex); + } + + return 0; +} + +static void hyp_core_ctl_handle_thermal(struct hyp_core_ctl_data *hcd, + int cpu, bool throttled) +{ + cpumask_t temp_mask, iter_cpus; + const cpumask_t *thermal_cpus = cpu_cooling_multi_get_max_level_cpumask(); + bool notify = false; + int replacement_cpu; + + hyp_core_ctl_print_status("handle_thermal_start"); + + /* + * Take a copy of the final_reserved_cpus and adjust the mask + * based on the notified CPU's thermal state. + */ + cpumask_copy(&temp_mask, &hcd->final_reserved_cpus); + + if (throttled) { + /* + * Find a replacement CPU for this throttled CPU. Select + * any CPU that is not managed by thermal and not already + * part of the assigned CPUs. + */ + cpumask_andnot(&iter_cpus, cpu_possible_mask, thermal_cpus); + cpumask_andnot(&iter_cpus, &iter_cpus, + &hcd->final_reserved_cpus); + replacement_cpu = cpumask_any(&iter_cpus); + + if (replacement_cpu < nr_cpu_ids) { + cpumask_clear_cpu(cpu, &temp_mask); + cpumask_set_cpu(replacement_cpu, &temp_mask); + notify = true; + } + } else { + /* + * One of the original assigned CPU is unthrottled by thermal. + * Swap this CPU with any one of the replacement CPUs. + */ + cpumask_andnot(&iter_cpus, &hcd->final_reserved_cpus, + &hcd->reserve_cpus); + replacement_cpu = cpumask_any(&iter_cpus); + + if (replacement_cpu < nr_cpu_ids) { + cpumask_clear_cpu(replacement_cpu, &temp_mask); + cpumask_set_cpu(cpu, &temp_mask); + notify = true; + } + } + + if (notify) + finalize_reservation(hcd, &temp_mask); + + hyp_core_ctl_print_status("handle_thermal_end"); +} + +static int hyp_core_ctl_cpu_cooling_cb(struct notifier_block *nb, + unsigned long val, void *data) +{ + int cpu = (long) data; + const cpumask_t *thermal_cpus = cpu_cooling_multi_get_max_level_cpumask(); + struct freq_qos_request *qos_req; + int ret; + + if (!the_hcd) + return NOTIFY_DONE; + + mutex_lock(&the_hcd->reservation_mutex); + + pr_debug("CPU%d is %s by thermal\n", cpu, + val ? "throttled" : "unthrottled"); + + if (val) { + /* + * The thermal mitigated CPU is not part of our reserved + * CPUs. So nothing to do. + */ + if (!cpumask_test_cpu(cpu, &the_hcd->final_reserved_cpus)) + goto out; + + /* + * The thermal mitigated CPU is part of our reserved CPUs. + * + * If it is paused by us, unpause it. If it is not + * paused, probably it is offline. In both cases, kick + * the state machine to find a replacement CPU. + */ + if (cpumask_test_cpu(cpu, &the_hcd->our_paused_cpus)) { + resume_cpu(cpu); + cpumask_clear_cpu(cpu, &the_hcd->our_paused_cpus); + if (freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, cpu); + ret = freq_qos_update_request(qos_req, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + cpu, ret); + } + } + } else { + /* + * A CPU is unblocked by thermal. We are interested if + * + * (1) This CPU is part of the original reservation request + * In this case, this CPU should be swapped with one of + * the replacement CPU that is currently reserved. + * (2) When some of the thermal mitigated CPUs are currently + * reserved due to unavailability of CPUs. Now that + * thermal unblocked a CPU, swap this with one of the + * thermal mitigated CPU that is currently reserved. + */ + if (!cpumask_test_cpu(cpu, &the_hcd->reserve_cpus) && + !cpumask_intersects(&the_hcd->final_reserved_cpus, + thermal_cpus)) + goto out; + } + + if (the_hcd->reservation_enabled) { + spin_lock(&the_hcd->lock); + the_hcd->pending = true; + wake_up_process(the_hcd->task); + spin_unlock(&the_hcd->lock); + } else { + /* + * When the reservation is enabled, the state machine + * takes care of finding the new replacement CPU or + * isolating the unthrottled CPU. However when the + * reservation is not enabled, we still want to + * re-assign another CPU for a throttled CPU. + */ + hyp_core_ctl_handle_thermal(the_hcd, cpu, val); + } +out: + mutex_unlock(&the_hcd->reservation_mutex); + return NOTIFY_OK; +} + +static struct notifier_block hyp_core_ctl_nb = { + .notifier_call = hyp_core_ctl_cpu_cooling_cb, +}; + +static int hyp_core_ctl_hp_offline(unsigned int cpu) +{ + struct freq_qos_request *qos_req; + int ret; + + if (!the_hcd || !the_hcd->reservation_enabled) + return 0; + + if (cpumask_test_and_clear_cpu(cpu, &the_hcd->our_paused_cpus)) { + if (freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, cpu); + ret = freq_qos_update_request(qos_req, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + cpu, ret); + } + } + + return 0; +} + +static int hyp_core_ctl_hp_online(unsigned int cpu) +{ + if (!the_hcd || !the_hcd->reservation_enabled) + return 0; + + /* + * A reserved CPU is coming online. It should be paused + * to honor the reservation. So kick the state machine. + */ + spin_lock(&the_hcd->lock); + if (cpumask_test_cpu(cpu, &the_hcd->final_reserved_cpus)) { + the_hcd->pending = true; + wake_up_process(the_hcd->task); + } + spin_unlock(&the_hcd->lock); + + return 0; +} + +static void hyp_core_ctl_init_reserve_cpus(struct hyp_core_ctl_data *hcd) +{ + int i; + + spin_lock(&hcd->lock); + cpumask_clear(&hcd->reserve_cpus); + + for (i = 0; i < MAX_RESERVE_CPUS; i++) { + if (hh_cpumap[i].cap_id == 0) + break; + + hcd->cpumap[i].cap_id = hh_cpumap[i].cap_id; + hcd->cpumap[i].pcpu = hh_cpumap[i].pcpu; + hcd->cpumap[i].curr_pcpu = hh_cpumap[i].curr_pcpu; + cpumask_set_cpu(hcd->cpumap[i].pcpu, &hcd->reserve_cpus); + pr_debug("vcpu%u map to pcpu%u\n", i, hcd->cpumap[i].pcpu); + } + + cpumask_copy(&hcd->final_reserved_cpus, &hcd->reserve_cpus); + spin_unlock(&hcd->lock); + pr_info("reserve_cpus=%*pbl\n", cpumask_pr_args(&hcd->reserve_cpus)); +} + +/* + * Called when vm_status is STATUS_READY, multiple times before status + * moves to STATUS_RUNNING + */ +int hh_vcpu_populate_affinity_info(u32 cpu_idx, u64 cap_id) +{ + if (!init_done) { + pr_err("Driver probe failed\n"); + return -ENXIO; + } + + if (!is_vcpu_info_populated) { + hh_cpumap[nr_vcpus].cap_id = cap_id; + hh_cpumap[nr_vcpus].pcpu = cpu_idx; + hh_cpumap[nr_vcpus].curr_pcpu = cpu_idx; + + nr_vcpus++; + pr_debug("cpu_index:%u vcpu_cap_id:%llu nr_vcpus:%d\n", + cpu_idx, cap_id, nr_vcpus); + } + + return 0; +} + +static int hh_vcpu_done_populate_affinity_info(struct notifier_block *nb, + unsigned long cmd, void *data) +{ + struct hh_rm_notif_vm_status_payload *vm_status_payload = data; + u8 vm_status = vm_status_payload->vm_status; + + if (cmd == HH_RM_NOTIF_VM_STATUS && + vm_status == HH_RM_VM_STATUS_RUNNING && + !is_vcpu_info_populated) { + mutex_lock(&the_hcd->reservation_mutex); + hyp_core_ctl_init_reserve_cpus(the_hcd); + is_vcpu_info_populated = true; + mutex_unlock(&the_hcd->reservation_mutex); + } + + return NOTIFY_DONE; +} + +static struct notifier_block hh_vcpu_nb = { + .notifier_call = hh_vcpu_done_populate_affinity_info, +}; + +static void hyp_core_ctl_enable(bool enable) +{ + mutex_lock(&the_hcd->reservation_mutex); + if (!is_vcpu_info_populated) { + pr_err("VCPU info isn't populated\n"); + goto err_out; + } + + spin_lock(&the_hcd->lock); + if (enable == the_hcd->reservation_enabled) + goto out; + + trace_hyp_core_ctl_enable(enable); + pr_debug("reservation %s\n", enable ? "enabled" : "disabled"); + + the_hcd->reservation_enabled = enable; + the_hcd->pending = true; + wake_up_process(the_hcd->task); +out: + spin_unlock(&the_hcd->lock); +err_out: + mutex_unlock(&the_hcd->reservation_mutex); +} + +static ssize_t enable_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + bool enable; + int ret; + + ret = kstrtobool(buf, &enable); + if (ret < 0) + return -EINVAL; + + hyp_core_ctl_enable(enable); + + return count; +} + +static ssize_t enable_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", the_hcd->reservation_enabled); +} + +static DEVICE_ATTR_RW(enable); + +static ssize_t status_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct hyp_core_ctl_data *hcd = the_hcd; + ssize_t count; + int i; + + mutex_lock(&hcd->reservation_mutex); + + count = scnprintf(buf, PAGE_SIZE, "enabled=%d\n", + hcd->reservation_enabled); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "reserve_cpus=%*pbl\n", + cpumask_pr_args(&hcd->reserve_cpus)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "reserved_cpus=%*pbl\n", + cpumask_pr_args(&hcd->final_reserved_cpus)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "our_paused_cpus=%*pbl\n", + cpumask_pr_args(&hcd->our_paused_cpus)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "online_cpus=%*pbl\n", + cpumask_pr_args(cpu_online_mask)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "active_cpus=%*pbl\n", + cpumask_pr_args(cpu_active_mask)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "thermal_cpus=%*pbl\n", + cpumask_pr_args(cpu_cooling_multi_get_max_level_cpumask())); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "Vcpu to Pcpu mappings:\n"); + + for (i = 0; i < MAX_RESERVE_CPUS; i++) { + if (hcd->cpumap[i].cap_id == 0) + break; + + count += scnprintf(buf + count, PAGE_SIZE - count, + "vcpu=%d pcpu=%u curr_pcpu=%u\n", + i, hcd->cpumap[i].pcpu, hcd->cpumap[i].curr_pcpu); + + } + + mutex_unlock(&hcd->reservation_mutex); + + return count; +} + +static DEVICE_ATTR_RO(status); + +static int init_freq_qos_req(void) +{ + int cpu, ret; + struct cpufreq_policy *policy; + struct freq_qos_request *qos_req; + + for_each_possible_cpu(cpu) { + policy = cpufreq_cpu_get(cpu); + if (!policy) { + pr_err("cpufreq policy not found for cpu%d\n", cpu); + ret = -ESRCH; + goto remove_qos_req; + } + + qos_req = &per_cpu(qos_min_req, cpu); + ret = freq_qos_add_request(&policy->constraints, qos_req, + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) { + pr_err("Failed to add min freq constraint (%d)\n", ret); + cpufreq_cpu_put(policy); + goto remove_qos_req; + } + cpufreq_cpu_put(policy); + } + + return 0; + +remove_qos_req: + for_each_possible_cpu(cpu) { + qos_req = &per_cpu(qos_min_req, cpu); + if (freq_qos_request_active(qos_req)) + freq_qos_remove_request(qos_req); + } + + return ret; +} + +static ssize_t hcc_min_freq_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int i, ret, ntokens = 0; + unsigned int val, cpu; + const char *cp = buf; + + mutex_lock(&the_hcd->reservation_mutex); + if (!is_vcpu_info_populated) { + pr_err("VCPU info isn't populated\n"); + goto err_out; + } + + if (!freq_qos_init_done) { + if (init_freq_qos_req()) + goto err_out; + freq_qos_init_done = true; + } + + while ((cp = strpbrk(cp + 1, " :"))) + ntokens++; + + /* CPU:value pair */ + if (!(ntokens % 2)) + goto err_out; + + cp = buf; + for (i = 0; i < ntokens; i += 2) { + if (sscanf(cp, "%u:%u", &cpu, &val) != 2) + goto err_out; + if (cpu >= num_possible_cpus()) + goto err_out; + + per_cpu(qos_min_freq, cpu) = val; + cp = strnchr(cp, strlen(cp), ' '); + cp++; + } + + mutex_unlock(&the_hcd->reservation_mutex); + return count; + +err_out: + ret = -EINVAL; + mutex_unlock(&the_hcd->reservation_mutex); + return ret; +} + +static ssize_t hcc_min_freq_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int cnt = 0, cpu; + + for_each_possible_cpu(cpu) { + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, + "%d:%u ", cpu, + per_cpu(qos_min_freq, cpu)); + } + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); + return cnt; +} + +static DEVICE_ATTR_RW(hcc_min_freq); + +static struct attribute *hyp_core_ctl_attrs[] = { + &dev_attr_enable.attr, + &dev_attr_status.attr, + &dev_attr_hcc_min_freq.attr, + NULL +}; + +static struct attribute_group hyp_core_ctl_attr_group = { + .attrs = hyp_core_ctl_attrs, + .name = "hyp_core_ctl", +}; + +#define CPULIST_SZ 32 +static ssize_t read_reserve_cpus(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + char kbuf[CPULIST_SZ]; + int ret; + + ret = scnprintf(kbuf, CPULIST_SZ, "%*pbl\n", + cpumask_pr_args(&the_hcd->reserve_cpus)); + + return simple_read_from_buffer(ubuf, count, ppos, kbuf, ret); +} + +static ssize_t write_reserve_cpus(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + char kbuf[CPULIST_SZ]; + int ret; + cpumask_t temp_mask; + + mutex_lock(&the_hcd->reservation_mutex); + if (!is_vcpu_info_populated) { + pr_err("VCPU info isn't populated\n"); + ret = -EPERM; + goto err_out; + } + + ret = simple_write_to_buffer(kbuf, CPULIST_SZ - 1, ppos, ubuf, count); + if (ret < 0) + goto err_out; + + kbuf[ret] = '\0'; + ret = cpulist_parse(kbuf, &temp_mask); + if (ret < 0) + goto err_out; + + if (cpumask_weight(&temp_mask) != + cpumask_weight(&the_hcd->reserve_cpus)) { + pr_err("incorrect reserve CPU count. expected=%u\n", + cpumask_weight(&the_hcd->reserve_cpus)); + ret = -EINVAL; + goto err_out; + } + + spin_lock(&the_hcd->lock); + if (the_hcd->reservation_enabled) { + count = -EPERM; + pr_err("reservation is enabled, can't change reserve_cpus\n"); + } else { + cpumask_copy(&the_hcd->reserve_cpus, &temp_mask); + } + spin_unlock(&the_hcd->lock); + mutex_unlock(&the_hcd->reservation_mutex); + + return count; +err_out: + mutex_unlock(&the_hcd->reservation_mutex); + return ret; +} + +static const struct file_operations debugfs_reserve_cpus_ops = { + .read = read_reserve_cpus, + .write = write_reserve_cpus, +}; + +static void hyp_core_ctl_debugfs_init(void) +{ + struct dentry *dir, *file; + + dir = debugfs_create_dir("hyp_core_ctl", NULL); + if (IS_ERR_OR_NULL(dir)) + return; + + file = debugfs_create_file("reserve_cpus", 0644, dir, NULL, + &debugfs_reserve_cpus_ops); + if (!file) + debugfs_remove(dir); +} + +static int hyp_core_ctl_probe(struct platform_device *pdev) +{ + int ret; + struct hyp_core_ctl_data *hcd; + struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; + + ret = hh_rm_register_notifier(&hh_vcpu_nb); + if (ret) + return ret; + + hcd = kzalloc(sizeof(*hcd), GFP_KERNEL); + if (!hcd) { + ret = -ENOMEM; + goto unregister_rm_notifier; + } + + spin_lock_init(&hcd->lock); + mutex_init(&hcd->reservation_mutex); + hcd->task = kthread_run(hyp_core_ctl_thread, (void *) hcd, + "hyp_core_ctl"); + + if (IS_ERR(hcd->task)) { + ret = PTR_ERR(hcd->task); + goto free_hcd; + } + + sched_setscheduler_nocheck(hcd->task, SCHED_FIFO, ¶m); + + ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, + &hyp_core_ctl_attr_group); + if (ret < 0) { + pr_err("Fail to create sysfs files. ret=%d\n", ret); + goto stop_task; + } + + cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN_END, + "qcom/hyp_core_ctl:online", + hyp_core_ctl_hp_online, + hyp_core_ctl_hp_offline); + + cpu_cooling_multi_max_level_notifier_register(&hyp_core_ctl_nb); + hyp_core_ctl_debugfs_init(); + + the_hcd = hcd; + init_done = true; + return 0; + +stop_task: + kthread_stop(hcd->task); +free_hcd: + kfree(hcd); +unregister_rm_notifier: + hh_rm_unregister_notifier(&hh_vcpu_nb); + + return ret; +} + +static const struct of_device_id hyp_core_ctl_match_table[] = { + { .compatible = "qcom,hyp-core-ctl" }, + {}, +}; + +static struct platform_driver hyp_core_ctl_driver = { + .probe = hyp_core_ctl_probe, + .driver = { + .name = "hyp_core_ctl", + .owner = THIS_MODULE, + .of_match_table = hyp_core_ctl_match_table, + }, +}; + +builtin_platform_driver(hyp_core_ctl_driver); +MODULE_DESCRIPTION("Core Control for Hypervisor"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/thermal/qcom/thermal_pause.c b/drivers/thermal/qcom/thermal_pause.c new file mode 100644 index 000000000000..b39817d4e500 --- /dev/null +++ b/drivers/thermal/qcom/thermal_pause.c @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) "%s:%s " fmt, KBUILD_MODNAME, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum thermal_pause_levels { + THERMAL_NO_CPU_PAUSE, + THERMAL_GROUP_CPU_PAUSE, + + /* define new pause levels above this line */ + MAX_THERMAL_PAUSE_LEVEL +}; + +struct thermal_pause_cdev { + struct list_head node; + cpumask_t cpu_mask; + bool thermal_pause_level; + struct thermal_cooling_device *cdev; + struct device_node *np; + char cdev_name[THERMAL_NAME_LENGTH]; + struct work_struct reg_work; +}; + +static DEFINE_MUTEX(cpus_pause_lock); +static LIST_HEAD(thermal_pause_cdev_list); +static struct cpumask cpus_paused_by_thermal; +static struct cpumask cpus_in_max_cooling_level; +static struct cpumask cpus_pending_online; +static enum cpuhp_state cpu_hp_online; + +static BLOCKING_NOTIFIER_HEAD(multi_max_cooling_level_notifer); + +void cpu_cooling_multi_max_level_notifier_register(struct notifier_block *n) +{ + blocking_notifier_chain_register(&multi_max_cooling_level_notifer, n); +} + +void cpu_cooling_multi_max_level_notifier_unregister(struct notifier_block *n) +{ + blocking_notifier_chain_unregister(&multi_max_cooling_level_notifer, n); +} + +const struct cpumask *cpu_cooling_multi_get_max_level_cpumask(void) +{ + return &cpus_in_max_cooling_level; +} + +static int thermal_pause_hp_online(unsigned int online_cpu) +{ + struct thermal_pause_cdev *thermal_pause_cdev; + int ret = 0; + + pr_debug("online entry CPU:%d. mask:%*pbl pend:%*pbl\n", online_cpu, + cpumask_pr_args(&cpus_paused_by_thermal), + cpumask_pr_args(&cpus_pending_online)); + + mutex_lock(&cpus_pause_lock); + + if (cpumask_test_cpu(online_cpu, &cpus_paused_by_thermal)) { + ret = NOTIFY_BAD; + cpumask_set_cpu(online_cpu, &cpus_pending_online); + } + + list_for_each_entry(thermal_pause_cdev, &thermal_pause_cdev_list, node) { + if (cpumask_test_cpu(online_cpu, &thermal_pause_cdev->cpu_mask) + && !thermal_pause_cdev->cdev) + queue_work(system_highpri_wq, + &thermal_pause_cdev->reg_work); + } + + mutex_unlock(&cpus_pause_lock); + pr_debug("online exit CPU:%d. mask:%*pbl pend:%*pbl\n", online_cpu, + cpumask_pr_args(&cpus_paused_by_thermal), + cpumask_pr_args(&cpus_pending_online)); + + return ret; +} + +/** + * thermal_pause_cpus_pause - function to pause a group of cpus at + * the specified level. + * + * @thermal_pause_cdev: the pause device + * + * function to handle setting the current cpus paused by + * this driver for the mask specified in the device. + * it assumes the mutex is locked. + * + * Returns 0 if CPUs were paused, error otherwise + */ +static int thermal_pause_cpus_pause(struct thermal_pause_cdev *thermal_pause_cdev) +{ + int cpu = 0; + int ret = -ENODEV; + cpumask_t cpus_to_pause; + + cpumask_clear(&cpus_to_pause); + cpumask_andnot(&cpus_to_pause, &thermal_pause_cdev->cpu_mask, + &cpus_paused_by_thermal); + + if (!cpumask_empty(&cpus_to_pause)) { + + cpumask_or(&cpus_paused_by_thermal, + &cpus_paused_by_thermal, &cpus_to_pause); + pr_debug("Active req:%*pbl pause:%*pbl mask:%*pbl\n", + cpumask_pr_args(&cpus_paused_by_thermal), + cpumask_pr_args(&cpus_to_pause), + cpumask_pr_args(&thermal_pause_cdev->cpu_mask)); + mutex_unlock(&cpus_pause_lock); + + ret = walt_pause_cpus(&cpus_to_pause); + + mutex_lock(&cpus_pause_lock); + + for_each_cpu(cpu, &cpus_to_pause) + blocking_notifier_call_chain( + &multi_max_cooling_level_notifer, + 1, (void *)(long)cpu); + } + + cpumask_copy(&cpus_in_max_cooling_level, &cpus_paused_by_thermal); + + return ret; +} + +/** + * thermal_pause_cpus_unpause - function to unpause a + * group of cpus in the mask for this cdev + * + * @thermal_pause_cdev: the pause device + * + * function to handle enabling the group of cpus in the cdev + * + * Returns 0 if CPUs were unpaused, + */ +static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_cdev) +{ + int cpu = 0; + int ret = -ENODEV; + struct thermal_pause_cdev *cdev; + cpumask_t cpus_offlined, new_cpu_pause, cpus_to_unpause; + + cpumask_clear(&new_cpu_pause); + list_for_each_entry(cdev, &thermal_pause_cdev_list, node) { + if (!cdev->thermal_pause_level) + continue; + cpumask_or(&new_cpu_pause, &new_cpu_pause, + &cdev->cpu_mask); + } + cpumask_andnot(&cpus_to_unpause, &cpus_paused_by_thermal, + &new_cpu_pause); + + cpumask_and(&cpus_offlined, &cpus_pending_online, + &cpus_to_unpause); + + pr_debug("Old req:%*pbl New req:%*pbl Unpause:%*pbl Online:%*pbl\n", + cpumask_pr_args(&cpus_paused_by_thermal), + cpumask_pr_args(&new_cpu_pause), + cpumask_pr_args(&cpus_to_unpause), + cpumask_pr_args(&cpus_offlined)); + + cpumask_copy(&cpus_in_max_cooling_level, &new_cpu_pause); + cpumask_copy(&cpus_paused_by_thermal, &new_cpu_pause); + + if (!cpumask_empty(&cpus_to_unpause)) { + mutex_unlock(&cpus_pause_lock); + ret = walt_resume_cpus(&cpus_to_unpause); + if (ret) + pr_err("Error resuming CPU:%*pbl. err:%d\n", + cpumask_pr_args(&cpus_to_unpause), ret); + mutex_lock(&cpus_pause_lock); + } + + for_each_cpu(cpu, &cpus_offlined) { + cpumask_clear_cpu(cpu, &cpus_pending_online); + mutex_unlock(&cpus_pause_lock); + ret = add_cpu(cpu); + if (ret) + pr_err("Error Adding CPU:%d. err:%d\n", cpu, ret); + mutex_lock(&cpus_pause_lock); + } + + for_each_cpu(cpu, &cpus_to_unpause) + blocking_notifier_call_chain( + &multi_max_cooling_level_notifer, + 0, (void *)(long)cpu); + + return ret; +} + +/** + * thermal_pause_set_cur_state - callback function to set the current cooling + * level. + * @cdev: thermal cooling device pointer. + * @level: set this variable to the current cooling level. + * + * Callback for the thermal cooling device to change the cpu pause + * current cooling level. + * + * Return: 0 on success, an error code otherwise. + */ +static int thermal_pause_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long level) +{ + struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata; + int ret = 0; + + if (level >= MAX_THERMAL_PAUSE_LEVEL) + return -EINVAL; + + if (thermal_pause_cdev->thermal_pause_level == level) + return 0; + + mutex_lock(&cpus_pause_lock); + + thermal_pause_cdev->thermal_pause_level = level; + + if (level == THERMAL_GROUP_CPU_PAUSE) + ret = thermal_pause_cpus_pause(thermal_pause_cdev); + else + ret = thermal_pause_cpus_unpause(thermal_pause_cdev); + + mutex_unlock(&cpus_pause_lock); + + return ret; +} + +/** + * thermal_pause_get_cur_state - callback function to get the current cooling + * state. + * @cdev: thermal cooling device pointer. + * @state: fill this variable with the current cooling state. + * + * Callback for the thermal cooling device to return the cpu pause + * current cooling level + * + * Return: 0 on success, an error code otherwise. + */ +static int thermal_pause_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *level) +{ + struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata; + + *level = thermal_pause_cdev->thermal_pause_level; + + return 0; +} + +/** + * thermal_pause_get_max_state - callback function to get the max cooling state. + * @cdev: thermal cooling device pointer. + * @level: fill this variable with the max cooling level + * + * Callback for the thermal cooling device to return the cpu + * pause max cooling state. + * + * Return: 0 on success, an error code otherwise. + */ +static int thermal_pause_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *level) +{ + *level = MAX_THERMAL_PAUSE_LEVEL - 1; + return 0; +} + +static struct thermal_cooling_device_ops thermal_pause_cooling_ops = { + .get_max_state = thermal_pause_get_max_state, + .get_cur_state = thermal_pause_get_cur_state, + .set_cur_state = thermal_pause_set_cur_state, +}; + +static void thermal_pause_register_cdev(struct work_struct *work) +{ + struct thermal_pause_cdev *thermal_pause_cdev = + container_of(work, struct thermal_pause_cdev, reg_work); + int ret = 0; + cpumask_t cpus_online; + + cpumask_and(&cpus_online, + &thermal_pause_cdev->cpu_mask, + cpu_online_mask); + if (!cpumask_equal(&thermal_pause_cdev->cpu_mask, &cpus_online)) + return; + + thermal_pause_cdev->cdev = thermal_of_cooling_device_register( + thermal_pause_cdev->np, + thermal_pause_cdev->cdev_name, + thermal_pause_cdev, + &thermal_pause_cooling_ops); + + if (IS_ERR(thermal_pause_cdev->cdev)) { + ret = PTR_ERR(thermal_pause_cdev->cdev); + pr_err("Cooling register failed for %s, ret:%d\n", + thermal_pause_cdev->cdev_name, ret); + thermal_pause_cdev->cdev = NULL; + return; + } + + pr_debug("Cooling device [%s] registered.\n", + thermal_pause_cdev->cdev_name); +} + +static int thermal_pause_probe(struct platform_device *pdev) +{ + int ret = 0, cpu = 0; + struct device_node *subsys_np = NULL, *cpu_phandle = NULL; + struct device *cpu_dev; + struct thermal_pause_cdev *thermal_pause_cdev = NULL; + struct device_node *np = pdev->dev.of_node; + struct of_phandle_iterator it; + cpumask_t cpu_mask; + unsigned long mask = 0; + + INIT_LIST_HEAD(&thermal_pause_cdev_list); + cpumask_clear(&cpus_in_max_cooling_level); + cpumask_clear(&cpus_paused_by_thermal); + cpumask_clear(&cpus_pending_online); + + for_each_available_child_of_node(np, subsys_np) { + + cpumask_clear(&cpu_mask); + mask = 0; + of_phandle_iterator_init(&it, subsys_np, "qcom,cpus", NULL, 0); + while (of_phandle_iterator_next(&it) == 0) { + cpu_phandle = it.node; + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (cpu_dev && cpu_dev->of_node + == cpu_phandle) { + cpumask_set_cpu(cpu, &cpu_mask); + mask = mask | BIT(cpu); + break; + } + } + } + + if (cpumask_empty(&cpu_mask)) + continue; + + thermal_pause_cdev = devm_kzalloc(&pdev->dev, + sizeof(*thermal_pause_cdev), GFP_KERNEL); + + if (!thermal_pause_cdev) { + of_node_put(subsys_np); + return -ENOMEM; + } + + snprintf(thermal_pause_cdev->cdev_name, THERMAL_NAME_LENGTH, + "thermal-pause-%X", mask); + + thermal_pause_cdev->thermal_pause_level = false; + thermal_pause_cdev->cdev = NULL; + thermal_pause_cdev->np = subsys_np; + cpumask_copy(&thermal_pause_cdev->cpu_mask, &cpu_mask); + + INIT_WORK(&thermal_pause_cdev->reg_work, + thermal_pause_register_cdev); + list_add(&thermal_pause_cdev->node, &thermal_pause_cdev_list); + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal-pause/cdev:online", + thermal_pause_hp_online, NULL); + if (ret < 0) + return ret; + cpu_hp_online = ret; + + return 0; +} + +static int thermal_pause_remove(struct platform_device *pdev) +{ + struct thermal_pause_cdev *thermal_pause_cdev = NULL, *next = NULL; + int ret = 0, cpu = 0; + + if (cpu_hp_online) { + cpuhp_remove_state_nocalls(cpu_hp_online); + cpu_hp_online = 0; + } + + mutex_lock(&cpus_pause_lock); + list_for_each_entry_safe(thermal_pause_cdev, next, + &thermal_pause_cdev_list, node) { + if (thermal_pause_cdev->cdev) + thermal_cooling_device_unregister( + thermal_pause_cdev->cdev); + list_del(&thermal_pause_cdev->node); + } + + cpumask_andnot(&cpus_paused_by_thermal, &cpus_paused_by_thermal, + &cpus_pending_online); + + if (!cpumask_empty(&cpus_paused_by_thermal)) { + mutex_unlock(&cpus_pause_lock); + ret = walt_resume_cpus(&cpus_paused_by_thermal); + if (ret) + pr_err("Error resuming CPU:%*pbl. err:%d\n", + cpumask_pr_args(&cpus_paused_by_thermal), ret); + mutex_lock(&cpus_pause_lock); + } + + for_each_cpu(cpu, &cpus_pending_online) { + mutex_unlock(&cpus_pause_lock); + ret = add_cpu(cpu); + if (ret) + pr_err("Error Adding CPU:%d. err:%d\n", cpu, ret); + mutex_lock(&cpus_pause_lock); + } + + mutex_unlock(&cpus_pause_lock); + + return 0; +} + +static const struct of_device_id thermal_pause_match[] = { + { .compatible = "qcom,thermal-pause", }, + {}, +}; + +static struct platform_driver thermal_pause_driver = { + .probe = thermal_pause_probe, + .remove = thermal_pause_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = thermal_pause_match, + }, +}; + +module_platform_driver(thermal_pause_driver); +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index aa503f24009a..4858cfaa4b7b 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -133,6 +133,8 @@ struct notifier_block; extern void core_ctl_notifier_register(struct notifier_block *n); extern void core_ctl_notifier_unregister(struct notifier_block *n); extern int core_ctl_set_boost(bool boost); +extern int walt_pause_cpus(struct cpumask *cpus); +extern int walt_resume_cpus(struct cpumask *cpus); #else static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout) { @@ -170,6 +172,14 @@ static inline void core_ctl_notifier_unregister(struct notifier_block *n) { } +inline int walt_pause_cpus(struct cpumask *cpus) +{ + return 0; +} +inline int walt_resume_cpus(struct cpumask *cpus) +{ + return 0; +} #endif #endif /* _LINUX_SCHED_WALT_H */ diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile index 18e5264cf702..870d9b2fcfe3 100644 --- a/kernel/sched/walt/Makefile +++ b/kernel/sched/walt/Makefile @@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n KCSAN_SANITIZE := n obj-$(CONFIG_SCHED_WALT) += sched-walt.o -sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 307d0788e913..309d60b7de38 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "walt.h" #include "trace.h" @@ -1157,10 +1158,10 @@ static void __ref do_core_ctl(void) } if (cpumask_any(&cpus_to_pause) < nr_cpu_ids) - pause_cpus(&cpus_to_pause); + walt_pause_cpus(&cpus_to_pause); if (cpumask_any(&cpus_to_unpause) < nr_cpu_ids) - resume_cpus(&cpus_to_unpause); + walt_resume_cpus(&cpus_to_unpause); } static int __ref try_core_ctl(void *data) diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c new file mode 100644 index 000000000000..7535d27561e3 --- /dev/null +++ b/kernel/sched/walt/walt_pause.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include + +#ifdef CONFIG_HOTPLUG_CPU + +DEFINE_SPINLOCK(pause_lock); + +struct pause_cpu_state { + int ref_count; +}; + +static DEFINE_PER_CPU(struct pause_cpu_state, pause_state); + +/* increment ref count for cpus in passed mask */ +static void inc_ref_counts(struct cpumask *cpus) +{ + int cpu; + struct pause_cpu_state *pause_cpu_state; + + spin_lock(&pause_lock); + for_each_cpu(cpu, cpus) { + pause_cpu_state = per_cpu_ptr(&pause_state, cpu); + if (pause_cpu_state->ref_count) + cpumask_clear_cpu(cpu, cpus); + pause_cpu_state->ref_count++; + } + spin_unlock(&pause_lock); +} + +/* + * decrement ref count for cpus in passed mask + * updates the cpus to include only cpus ready to be unpaused + */ +static void dec_test_ref_counts(struct cpumask *cpus) +{ + int cpu; + struct pause_cpu_state *pause_cpu_state; + + spin_lock(&pause_lock); + for_each_cpu(cpu, cpus) { + pause_cpu_state = per_cpu_ptr(&pause_state, cpu); + WARN_ON_ONCE(pause_cpu_state->ref_count == 0); + pause_cpu_state->ref_count--; + if (pause_cpu_state->ref_count) + cpumask_clear_cpu(cpu, cpus); + } + spin_unlock(&pause_lock); +} + +/* cpus will be modified */ +int walt_pause_cpus(struct cpumask *cpus) +{ + int ret; + cpumask_t saved_cpus; + + cpumask_copy(&saved_cpus, cpus); + + /* prior to operation, track cpus requested to be paused */ + inc_ref_counts(cpus); + ret = pause_cpus(cpus); + if (ret < 0) { + dec_test_ref_counts(&saved_cpus); + pr_err("pause_cpus failure ret=%d cpus=%*pbl\n", ret, + cpumask_pr_args(&saved_cpus)); + } + + return ret; +} +EXPORT_SYMBOL(walt_pause_cpus); + +/* cpus will be modified */ +int walt_resume_cpus(struct cpumask *cpus) +{ + int ret; + cpumask_t saved_cpus; + + cpumask_copy(&saved_cpus, cpus); + + dec_test_ref_counts(cpus); + ret = resume_cpus(cpus); + if (ret < 0) { + inc_ref_counts(&saved_cpus); + pr_err("resume_cpus failure ret=%d cpus=%*pbl\n", ret, + cpumask_pr_args(&saved_cpus)); + } + + return ret; +} +EXPORT_SYMBOL(walt_resume_cpus); + +#endif /* CONFIG_HOTPLUG_CPU */ From c2ed149716e8bf13a8b75c4b9c43af6f1975caba Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 17 Feb 2021 15:38:31 -0800 Subject: [PATCH 052/346] sched/walt: improve debug for time_delta < 0 SCHED_BUG_ON is called from update_task_rq_cpu_cycles() when current time is less than the previous recorded time for the task. SCHED_BUG_ON isn't providing the precise details of the time delta. Improve the debugging such that the previous time, current, time pid, and CPU are all printed before the scheduler dump. Change-Id: I2156310be7db792e5579f8f56a0b653ed1ace225 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 44aedc2579c6..bb0ef544a416 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2091,7 +2091,13 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, time_delta = irqtime; else time_delta = wallclock - wts->mark_start; - SCHED_BUG_ON((s64)time_delta < 0); + + if ((s64)time_delta < 0) { + printk_deferred("WALT-BUG pid=%u CPU%d wallclock=%llu < mark_start=%llu event=%d irqtime=%llu", + p->pid, rq->cpu, wallclock, + wts->mark_start, event, irqtime); + SCHED_BUG_ON((s64)time_delta < 0); + } wrq->task_exec_scale = DIV64_U64_ROUNDUP(cycles_delta * arch_scale_cpu_capacity(cpu), From 5f11af1fd6e41099a5f47b2ab1c8f342058c8303 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 25 Feb 2021 09:04:59 +0530 Subject: [PATCH 053/346] sched/walt: Improve walt_newidle_balance trace point Add more debugging information to walt_newidle_balance trace point to better understand newidle balance behavior. Change-Id: I946d1d89644cc4fffb026e00686eedf805b06c01 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/trace.h | 37 ++++++++++++++++++++++++++----------- kernel/sched/walt/walt_lb.c | 16 +++++++++------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 7ef3ecd88ee7..cdb08fb12771 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -741,27 +741,42 @@ TRACE_EVENT(walt_nohz_balance_kick, TRACE_EVENT(walt_newidle_balance, - TP_PROTO(int this_cpu, int busy_cpu, int pulled), + TP_PROTO(int this_cpu, int busy_cpu, int pulled, bool help_min_cap, bool enough_idle), - TP_ARGS(this_cpu, busy_cpu, pulled), + TP_ARGS(this_cpu, busy_cpu, pulled, help_min_cap, enough_idle), TP_STRUCT__entry( - __field(int, this_cpu) + __field(int, cpu) __field(int, busy_cpu) __field(int, pulled) - __field(unsigned int, this_nr_running) + __field(unsigned int, nr_running) + __field(unsigned int, rt_nr_running) + __field(int, nr_iowait) + __field(bool, help_min_cap) + __field(u64, avg_idle) + __field(bool, enough_idle) + __field(int, overload) ), TP_fast_assign( - __entry->this_cpu = this_cpu; - __entry->busy_cpu = busy_cpu; - __entry->pulled = pulled; - __entry->this_nr_running = cpu_rq(this_cpu)->nr_running; + __entry->cpu = this_cpu; + __entry->busy_cpu = busy_cpu; + __entry->pulled = pulled; + __entry->nr_running = cpu_rq(this_cpu)->nr_running; + __entry->rt_nr_running = cpu_rq(this_cpu)->rt.rt_nr_running; + __entry->nr_iowait = atomic_read(&(cpu_rq(this_cpu)->nr_iowait)); + __entry->help_min_cap = help_min_cap; + __entry->avg_idle = cpu_rq(this_cpu)->avg_idle; + __entry->enough_idle = enough_idle; + __entry->overload = cpu_rq(this_cpu)->rd->overload; ), - TP_printk("this_cpu=%d busy_cpu=%d pulled=%d this_nr_running=%u\n", - __entry->this_cpu, __entry->busy_cpu, __entry->pulled, - __entry->this_nr_running) + TP_printk("cpu=%d busy_cpu=%d pulled=%d nr_running=%u rt_nr_running=%u nr_iowait=%d help_min_cap=%d avg_idle=%llu enough_idle=%d overload=%d\n", + __entry->cpu, __entry->busy_cpu, __entry->pulled, + __entry->nr_running, __entry->rt_nr_running, + __entry->nr_iowait, __entry->help_min_cap, + __entry->avg_idle, __entry->enough_idle, + __entry->overload) ); TRACE_EVENT(walt_lb_cpu_util, diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index c5eb0e5a31e8..5a246c94564c 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -572,9 +572,9 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, struct walt_rq *wrq = (struct walt_rq *) this_rq->android_vendor_data1; int order_index; int cluster = 0; - int busy_cpu; + int busy_cpu = -1; bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); - bool help_min_cap; + bool help_min_cap = false; if (static_branch_unlikely(&walt_disabled)) return; @@ -600,10 +600,10 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, return; if (!READ_ONCE(this_rq->rd->overload)) - return; + goto out; if (atomic_read(&this_rq->nr_iowait) && !enough_idle) - return; + goto out; help_min_cap = should_help_min_cap(this_cpu); rq_unpin_lock(this_rq, rf); @@ -629,11 +629,11 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, /* sanity checks before attempting the pull */ if (busy_cpu == -1 || this_rq->nr_running > 0 || (busy_cpu == this_cpu)) - goto out; + goto unlock; *pulled_task = walt_lb_pull_tasks(this_cpu, busy_cpu); -out: +unlock: raw_spin_lock(&this_rq->lock); if (this_rq->cfs.h_nr_running && !*pulled_task) *pulled_task = 1; @@ -648,7 +648,9 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, rq_repin_lock(this_rq, rf); - trace_walt_newidle_balance(this_cpu, busy_cpu, *pulled_task); +out: + trace_walt_newidle_balance(this_cpu, busy_cpu, *pulled_task, + help_min_cap, enough_idle); } static void walt_find_busiest_queue(void *unused, int dst_cpu, From 4a20cb72b6d2c5a4fc9889217fbc43cb6bac6ffa Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 25 Feb 2021 08:44:45 +0530 Subject: [PATCH 054/346] sched/walt: Improve the chance of RT task pull Currently the code tries to pull RT tasks from balance_rt() i.e when previous task is a RT task and it is de-activated. To cover more cases, attempt this from newly idle load balance which gets called before a CPU goes to idle. Change-Id: Id3c9a68c19d2b5709f9953ac2383c14f5134b7de Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_lb.c | 140 ++++++++++++++++++------------------ 1 file changed, 71 insertions(+), 69 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 5a246c94564c..f6109e466487 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -544,6 +544,63 @@ static void walt_lb_tick(void *unused, struct rq *rq) raw_spin_unlock_irqrestore(&walt_lb_migration_lock, flags); } +static inline int has_pushable_tasks(struct rq *rq) +{ + return !plist_head_empty(&rq->rt.pushable_tasks); +} + +#define WALT_RT_PULL_THRESHOLD_NS 250000 +static bool walt_balance_rt(struct rq *this_rq) +{ + int i, this_cpu = this_rq->cpu, src_cpu = this_cpu; + struct rq *src_rq; + struct task_struct *p; + struct walt_task_struct *wts; + bool pulled = false; + + /* can't help if this has a runnable RT */ + if (sched_rt_runnable(this_rq)) + return false; + + /* check if any CPU has a pushable RT task */ + for_each_possible_cpu(i) { + struct rq *rq = cpu_rq(i); + + if (!has_pushable_tasks(rq)) + continue; + + src_cpu = i; + break; + } + + if (src_cpu == this_cpu) + return false; + + src_rq = cpu_rq(src_cpu); + double_lock_balance(this_rq, src_rq); + + /* lock is dropped, so check again */ + if (sched_rt_runnable(this_rq)) + goto unlock; + + p = pick_highest_pushable_task(src_rq, this_cpu); + + if (!p) + goto unlock; + + wts = (struct walt_task_struct *) p->android_vendor_data1; + if (sched_ktime_clock() - wts->last_wake_ts < WALT_RT_PULL_THRESHOLD_NS) + goto unlock; + + pulled = true; + deactivate_task(src_rq, p, 0); + set_task_cpu(p, this_cpu); + activate_task(this_rq, p, 0); +unlock: + double_unlock_balance(this_rq, src_rq); + return pulled; +} + __read_mostly unsigned int sysctl_sched_force_lb_enable = 1; static bool should_help_min_cap(int this_cpu) { @@ -599,14 +656,23 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (!cpu_active(this_cpu)) return; + rq_unpin_lock(this_rq, rf); + + /* + * Since we drop rq lock while doing RT balance, + * check if any tasks are queued on this and bail out + * early. + */ + if (walt_balance_rt(this_rq) || this_rq->nr_running) + goto rt_pulled; + if (!READ_ONCE(this_rq->rd->overload)) - goto out; + goto repin; if (atomic_read(&this_rq->nr_iowait) && !enough_idle) - goto out; + goto repin; help_min_cap = should_help_min_cap(this_cpu); - rq_unpin_lock(this_rq, rf); raw_spin_unlock(&this_rq->lock); /* @@ -635,6 +701,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, unlock: raw_spin_lock(&this_rq->lock); +rt_pulled: if (this_rq->cfs.h_nr_running && !*pulled_task) *pulled_task = 1; @@ -646,9 +713,9 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (*pulled_task) this_rq->idle_stamp = 0; +repin: rq_repin_lock(this_rq, rf); -out: trace_walt_newidle_balance(this_cpu, busy_cpu, *pulled_task, help_min_cap, enough_idle); } @@ -758,70 +825,6 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, *can_migrate = 0; } -static inline int rt_overloaded(struct rq *rq) -{ - return atomic_read(&rq->rd->rto_count); -} - -static inline int has_pushable_tasks(struct rq *rq) -{ - return !plist_head_empty(&rq->rt.pushable_tasks); -} - -#define WALT_RT_PULL_THRESHOLD_NS 250000 -static void walt_balance_rt(void *unused, struct rq *this_rq, - struct task_struct *prev, int *done) -{ - int i, this_cpu = this_rq->cpu, src_cpu = this_cpu; - struct rq *src_rq; - struct task_struct *p; - struct walt_task_struct *wts; - - /* Let RT push/pull handle the overloaded scenario */ - if (rt_overloaded(this_rq)) - return; - - /* can't help if this has a runnable RT */ - if (sched_rt_runnable(this_rq)) - return; - - /* check if any CPU has a pushable RT task */ - for_each_possible_cpu(i) { - struct rq *rq = cpu_rq(i); - - if (!has_pushable_tasks(rq)) - continue; - - src_cpu = i; - break; - } - - if (src_cpu == this_cpu) - return; - - src_rq = cpu_rq(src_cpu); - double_lock_balance(this_rq, src_rq); - - /* lock is dropped, so check again */ - if (sched_rt_runnable(this_rq)) - goto unlock; - - p = pick_highest_pushable_task(src_rq, this_cpu); - - if (!p) - goto unlock; - - wts = (struct walt_task_struct *) p->android_vendor_data1; - if (sched_ktime_clock() - wts->last_wake_ts < WALT_RT_PULL_THRESHOLD_NS) - goto unlock; - - deactivate_task(src_rq, p, 0); - set_task_cpu(p, this_cpu); - activate_task(this_rq, p, 0); -unlock: - double_unlock_balance(this_rq, src_rq); -} - void walt_lb_init(void) { walt_lb_rotate_work_init(); @@ -832,5 +835,4 @@ void walt_lb_init(void) register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); register_trace_android_rvh_sched_newidle_balance(walt_newidle_balance, NULL); register_trace_android_vh_scheduler_tick(walt_lb_tick, NULL); - register_trace_android_rvh_sched_balance_rt(walt_balance_rt, NULL); } From ab32bc3be607aa28dee21afbf37b766c3b69d9fe Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 26 Feb 2021 10:19:42 -0800 Subject: [PATCH 055/346] sched: walt: Move task_boost_type declarations Make task_boost_type declarations public, as it is being used by other modules, by moving to appropriate header. Change-Id: I24baca67b9fdba883d7393247ca774ce9a664039 Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 8 ++++++++ kernel/sched/walt/walt.h | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 4858cfaa4b7b..42e4d906bf6e 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -22,6 +22,14 @@ struct core_ctl_notif_data { unsigned int cur_cap_pct[MAX_CLUSTERS]; }; +enum task_boost_type { + TASK_BOOST_NONE = 0, + TASK_BOOST_ON_MID, + TASK_BOOST_ON_MAX, + TASK_BOOST_STRICT_MAX, + TASK_BOOST_END, +}; + #define WALT_NR_CPUS 8 #define RAVG_HIST_SIZE_MAX 5 #define NUM_BUSY_BUCKETS 10 diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 934e3fadbba2..aafedd563cfc 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -49,14 +49,6 @@ enum migrate_types { RQ_TO_GROUP, }; -enum task_boost_type { - TASK_BOOST_NONE = 0, - TASK_BOOST_ON_MID, - TASK_BOOST_ON_MAX, - TASK_BOOST_STRICT_MAX, - TASK_BOOST_END, -}; - #define WALT_LOW_LATENCY_PROCFS BIT(0) #define WALT_LOW_LATENCY_BINDER BIT(1) From 0a5adea3408470a3aa5b3dc407edf643779545fa Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 1 Mar 2021 14:53:16 -0800 Subject: [PATCH 056/346] sched: walt: Improve the Scheduler This change is for general scheduler improvement. Change-Id: If7c9fd9d76d54da94acec15018c82e0d66d4162c Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 5 +++++ kernel/sched/walt/walt.c | 12 +++--------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 42e4d906bf6e..4c3a6ccdb834 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -112,6 +112,11 @@ struct walt_task_struct { bool iowaited; }; +#define wts_to_ts(wts) ({ \ + void *__mptr = (void *)(wts); \ + ((struct task_struct *)(__mptr - \ + offsetof(struct task_struct, android_vendor_data1))); }) + static inline bool sched_get_wake_up_idle(struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index bb0ef544a416..634883768fdd 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2663,7 +2663,6 @@ static void _set_preferred_cluster(struct walt_related_thread_group *grp) u64 wallclock; bool prev_skip_min = grp->skip_min; struct walt_task_struct *wts; - struct list_head *task_list; if (list_empty(&grp->tasks)) { grp->skip_min = false; @@ -2686,9 +2685,8 @@ static void _set_preferred_cluster(struct walt_related_thread_group *grp) if (wallclock - grp->last_update < sched_ravg_window / 10) return; - list_for_each(task_list, &grp->tasks) { - p = (struct task_struct *) task_list; - wts = (struct walt_task_struct *) p->android_vendor_data1; + list_for_each_entry(wts, &grp->tasks, grp_list) { + p = wts_to_ts(wts); if (task_boost_policy(p) == SCHED_BOOST_ON_BIG) { group_boost = true; break; @@ -3444,12 +3442,10 @@ void walt_fill_ta_data(struct core_ctl_notif_data *data) struct walt_related_thread_group *grp; unsigned long flags; u64 total_demand = 0, wallclock; - struct task_struct *p; int min_cap_cpu, scale = 1024; struct walt_sched_cluster *cluster; int i = 0; struct walt_task_struct *wts; - struct list_head *task_list; grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); @@ -3461,9 +3457,7 @@ void walt_fill_ta_data(struct core_ctl_notif_data *data) wallclock = sched_ktime_clock(); - list_for_each(task_list, &grp->tasks) { - p = (struct task_struct *) task_list; - wts = (struct walt_task_struct *) p->android_vendor_data1; + list_for_each_entry(wts, &grp->tasks, grp_list) { if (wts->mark_start < wallclock - (sched_ravg_window * sched_ravg_hist_size)) continue; From 9b64c2a55e6775659d539a141dede3c3973b7091 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 8 Feb 2021 22:34:55 -0800 Subject: [PATCH 057/346] sched/walt: Track wake_q length using newer parameter There is a revised member to track the wake_q length upstream. Change-Id: Ie1b9619be3616d4749f5fa2da8584f5fd4b45a58 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 98a094de65b5..d7b9cac3fdab 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -662,11 +662,16 @@ static void walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *target_cpu) { - int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); - int sibling_count_hint = 1; + int sync; + int sibling_count_hint; if (static_branch_unlikely(&walt_disabled)) return; + + sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); + sibling_count_hint = p->wake_q_count; + p->wake_q_count = 0; + *target_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, sync, sibling_count_hint); if (unlikely(*target_cpu < 0)) *target_cpu = prev_cpu; From 741be7b858451a0b19a7297cbaa565eb77973cfa Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 8 Mar 2021 11:15:51 +0530 Subject: [PATCH 058/346] sched/walt: Remove incorrect TODO/FIXME comments Remove the TODO comment about frequency in-variance in walt governor as the code is completed. Also remove a FIXME comment from WALT IRQ work as the code looks correct there. Change-Id: Ib5447009ab009d7cf2991c0f22d04a766de59f3a Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/cpufreq_walt.c | 5 ----- kernel/sched/walt/walt.c | 6 ------ 2 files changed, 11 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index e729503e3d0c..4722a2a1dc96 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -203,11 +203,6 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, unsigned long util, unsigned long max) { struct cpufreq_policy *policy = wg_policy->policy; - /* - * TODO: - unsigned int freq = arch_scale_freq_invariant() ? - policy->cpuinfo.max_freq : policy->cur; - */ unsigned int freq = policy->cpuinfo.max_freq; freq = map_util_freq(util, freq, max); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 634883768fdd..b586bc70053e 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3353,12 +3353,6 @@ static void walt_irq_work(struct irq_work *irq_work) for_each_cpu(cpu, &cluster_online_cpus) { int wflag = 0; - /* - * FIXME: - * - * For now supporting both schedutil and waltgov. - * This is not by design but for convenience. - */ rq = cpu_rq(cpu); wrq = (struct walt_rq *) rq->android_vendor_data1; From e483442bb7acc78ec8579ddced28dc70ca3e22af Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 9 Mar 2021 12:06:52 -0800 Subject: [PATCH 059/346] sched: walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ib1ec2f655193a71d401d7e51f7890537d4f75aef Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 49 ++++++++++++++++++------------------ kernel/sched/walt/walt.h | 2 +- kernel/sched/walt/walt_cfs.c | 8 +++--- kernel/sched/walt/walt_lb.c | 12 ++++----- kernel/sched/walt/walt_rt.c | 4 +-- 5 files changed, 38 insertions(+), 37 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index b586bc70053e..c7c4ff7c2234 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3625,7 +3625,7 @@ static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; add_new_task_to_grp(new); } @@ -3640,7 +3640,7 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); unsigned long thermal_cap; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /* @@ -3671,7 +3671,7 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) unsigned long flags; struct rq *rq = cpu_rq(cpu); - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; raw_spin_lock_irqsave(&rq->lock, flags); set_window_start(rq); @@ -3682,14 +3682,14 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) static void android_rvh_sched_cpu_dying(void *unused, int cpu) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; clear_walt_request(cpu); } static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsigned int new_cpu) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (new_cpu < 0) return; @@ -3698,14 +3698,14 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign static void android_rvh_sched_fork(void *unused, struct task_struct *p) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; init_new_task_load(p); } static void android_rvh_new_task_stats(void *unused, struct task_struct *p) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; mark_task_starting(p); } @@ -3714,7 +3714,7 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int { struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (!!(curr->flags & PF_IDLE)) { if (hardirq_count() || in_serving_softirq()) @@ -3727,7 +3727,7 @@ static void android_rvh_account_irq(void *unused, struct task_struct *curr, int static void android_rvh_flush_task(void *unused, struct task_struct *p) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; walt_task_dead(p); } @@ -3737,7 +3737,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st u64 wallclock = sched_ktime_clock(); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; wts->last_enqueued_ts = wallclock; sched_update_nr_prod(rq->cpu, true); @@ -3755,7 +3755,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (p == wrq->ed_task) is_ed_task_present(rq, sched_ktime_clock()); @@ -3777,7 +3777,7 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p bool old_misfit, misfit; int change; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *need_update = false; @@ -3815,7 +3815,7 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) unsigned int old_load; struct walt_related_thread_group *grp = NULL; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; rq_lock_irqsave(rq, &rf); old_load = task_load(p); @@ -3837,7 +3837,7 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct unsigned long flags; int cpu = p->cpu; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags); @@ -3852,7 +3852,7 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) u32 old_load; struct walt_related_thread_group *grp; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; set_window_start(rq); wallclock = sched_ktime_clock(); @@ -3876,7 +3876,7 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, u64 wallclock = sched_ktime_clock(); struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (likely(prev != next)) { if (!prev->on_rq) @@ -3894,7 +3894,7 @@ static void android_rvh_resume_cpus(void *unused, struct cpumask *resuming_cpus, struct rq *rq; unsigned long flags; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /* * send a reschedule event on all resumed CPUs @@ -3916,7 +3916,7 @@ static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (cpumask_subset(&wts->cpus_requested, cpus_requested)) *ret = set_cpus_allowed_ptr(p, &wts->cpus_requested); @@ -3926,7 +3926,7 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; wts->last_sleep_ts = 0; wts->wake_up_idle = false; @@ -3939,21 +3939,21 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) static void android_rvh_ttwu_cond(void *unused, bool *cond) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *cond = sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT; } static void android_rvh_sched_exec(void *unused, bool *cond) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *cond = true; } static void android_rvh_build_perf_domains(void *unused, bool *eas_check) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *eas_check = true; } @@ -3986,7 +3986,7 @@ static void register_walt_hooks(void) } atomic64_t walt_irq_work_lastq_ws; -DEFINE_STATIC_KEY_TRUE(walt_disabled); +bool walt_disabled = true; static int walt_init_stop_handler(void *data) { @@ -4026,6 +4026,8 @@ static int walt_init_stop_handler(void *data) walt_update_cluster_topology(); + walt_disabled = false; + for_each_possible_cpu(cpu) { raw_spin_unlock(&cpu_rq(cpu)->lock); } @@ -4056,7 +4058,6 @@ static void walt_init(void) walt_cfs_init(); stop_machine(walt_init_stop_handler, NULL, NULL); - static_branch_disable(&walt_disabled); hdr = register_sysctl_table(walt_base_table); kmemleak_not_leak(hdr); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index aafedd563cfc..1c76ba19e24b 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -32,7 +32,7 @@ /* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */ #define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1) -extern struct static_key_true walt_disabled; +extern bool walt_disabled; enum task_event { PUT_PREV_TASK = 0, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index d7b9cac3fdab..cfd63eb6aa19 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -665,7 +665,7 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, int sync; int sibling_count_hint; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); @@ -684,7 +684,7 @@ static inline struct task_struct *task_of(struct sched_entity *se) static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vruntime) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (entity_is_task(se)) { unsigned long thresh = sysctl_sched_latency; @@ -710,7 +710,7 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task) { struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (task && current->signal && (current->signal->oom_score_adj == 0) && @@ -723,7 +723,7 @@ static void walt_binder_low_latency_clear(void *unused, struct binder_transactio { struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (wts->low_latency & WALT_LOW_LATENCY_BINDER) wts->low_latency &= ~WALT_LOW_LATENCY_BINDER; diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index f6109e466487..30202177a806 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -491,7 +491,7 @@ static void walt_lb_tick(void *unused, struct rq *rq) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (!rq->misfit_task_load || !walt_fair_task(p)) return; @@ -633,7 +633,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); bool help_min_cap = false; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /*Cluster isn't initialized until after WALT is enabled*/ @@ -729,7 +729,7 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, int busiest_cpu = -1; struct cpumask src_mask; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *done = 1; *busiest = NULL; @@ -767,7 +767,7 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, struct task_struct *p, int new_cpu, int *detached) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /* * WALT expects both source and destination rqs to be @@ -796,7 +796,7 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, static void walt_nohz_balancer_kick(void *unused, struct rq *rq, unsigned int *flags, int *done) { - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; *done = 1; @@ -815,7 +815,7 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, { bool to_lower; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(task_cpu(p)); diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index d08e730fcbce..5ae1936151ad 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -24,7 +24,7 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task int cluster; int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; if (!ret) @@ -119,7 +119,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c int ret, target = -1; struct cpumask *lowest_mask; - if (static_branch_unlikely(&walt_disabled)) + if (unlikely(walt_disabled)) return; /* For anything but wake ups, just return the task_cpu */ From 17444a20e3948507cd1bd60dab21884742538eee Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Feb 2021 10:01:33 -0800 Subject: [PATCH 060/346] sched/pause: handle cpu online processing for pause With refcounting, the walt wrapper for pause/resume can keep track of whether a CPU should be paused or not, where this information used to be distributed to the clients using the api. Simplify the design, such that a CPU is automatically paused as it is coming online, affecting the active/inactive state of the CPU, but not adjusting the ref-counts. Going forward, client software using walt_pause/walt_resume will not be required to keep track of the online state of a cpu. Change-Id: I83e3cc8743942c3a482eb588546b6ea9bb7bf2e3 Signed-off-by: Stephen Dickey --- drivers/thermal/qcom/thermal_pause.c | 55 +++++---------------- kernel/sched/walt/walt.c | 1 + kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_pause.c | 71 +++++++++++++++++++++++++++- 4 files changed, 85 insertions(+), 43 deletions(-) diff --git a/drivers/thermal/qcom/thermal_pause.c b/drivers/thermal/qcom/thermal_pause.c index b39817d4e500..41d511d82d59 100644 --- a/drivers/thermal/qcom/thermal_pause.c +++ b/drivers/thermal/qcom/thermal_pause.c @@ -35,9 +35,12 @@ struct thermal_pause_cdev { static DEFINE_MUTEX(cpus_pause_lock); static LIST_HEAD(thermal_pause_cdev_list); + +/* track cpus that thermal intends to pause, regardless of + * whether thermal was requesting an already paused cpu + */ static struct cpumask cpus_paused_by_thermal; static struct cpumask cpus_in_max_cooling_level; -static struct cpumask cpus_pending_online; static enum cpuhp_state cpu_hp_online; static BLOCKING_NOTIFIER_HEAD(multi_max_cooling_level_notifer); @@ -62,17 +65,11 @@ static int thermal_pause_hp_online(unsigned int online_cpu) struct thermal_pause_cdev *thermal_pause_cdev; int ret = 0; - pr_debug("online entry CPU:%d. mask:%*pbl pend:%*pbl\n", online_cpu, - cpumask_pr_args(&cpus_paused_by_thermal), - cpumask_pr_args(&cpus_pending_online)); + pr_debug("online entry CPU:%d. mask:%*pbl\n", online_cpu, + cpumask_pr_args(&cpus_paused_by_thermal)); mutex_lock(&cpus_pause_lock); - if (cpumask_test_cpu(online_cpu, &cpus_paused_by_thermal)) { - ret = NOTIFY_BAD; - cpumask_set_cpu(online_cpu, &cpus_pending_online); - } - list_for_each_entry(thermal_pause_cdev, &thermal_pause_cdev_list, node) { if (cpumask_test_cpu(online_cpu, &thermal_pause_cdev->cpu_mask) && !thermal_pause_cdev->cdev) @@ -81,9 +78,8 @@ static int thermal_pause_hp_online(unsigned int online_cpu) } mutex_unlock(&cpus_pause_lock); - pr_debug("online exit CPU:%d. mask:%*pbl pend:%*pbl\n", online_cpu, - cpumask_pr_args(&cpus_paused_by_thermal), - cpumask_pr_args(&cpus_pending_online)); + pr_debug("online exit CPU:%d. mask:%*pbl\n", online_cpu, + cpumask_pr_args(&cpus_paused_by_thermal)); return ret; } @@ -150,7 +146,7 @@ static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_c int cpu = 0; int ret = -ENODEV; struct thermal_pause_cdev *cdev; - cpumask_t cpus_offlined, new_cpu_pause, cpus_to_unpause; + cpumask_t new_cpu_pause, cpus_to_unpause; cpumask_clear(&new_cpu_pause); list_for_each_entry(cdev, &thermal_pause_cdev_list, node) { @@ -162,14 +158,10 @@ static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_c cpumask_andnot(&cpus_to_unpause, &cpus_paused_by_thermal, &new_cpu_pause); - cpumask_and(&cpus_offlined, &cpus_pending_online, - &cpus_to_unpause); - - pr_debug("Old req:%*pbl New req:%*pbl Unpause:%*pbl Online:%*pbl\n", + pr_debug("Old req:%*pbl New req:%*pbl Unpause:%*pbl\n", cpumask_pr_args(&cpus_paused_by_thermal), cpumask_pr_args(&new_cpu_pause), - cpumask_pr_args(&cpus_to_unpause), - cpumask_pr_args(&cpus_offlined)); + cpumask_pr_args(&cpus_to_unpause)); cpumask_copy(&cpus_in_max_cooling_level, &new_cpu_pause); cpumask_copy(&cpus_paused_by_thermal, &new_cpu_pause); @@ -183,15 +175,6 @@ static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_c mutex_lock(&cpus_pause_lock); } - for_each_cpu(cpu, &cpus_offlined) { - cpumask_clear_cpu(cpu, &cpus_pending_online); - mutex_unlock(&cpus_pause_lock); - ret = add_cpu(cpu); - if (ret) - pr_err("Error Adding CPU:%d. err:%d\n", cpu, ret); - mutex_lock(&cpus_pause_lock); - } - for_each_cpu(cpu, &cpus_to_unpause) blocking_notifier_call_chain( &multi_max_cooling_level_notifer, @@ -326,7 +309,6 @@ static int thermal_pause_probe(struct platform_device *pdev) INIT_LIST_HEAD(&thermal_pause_cdev_list); cpumask_clear(&cpus_in_max_cooling_level); cpumask_clear(&cpus_paused_by_thermal); - cpumask_clear(&cpus_pending_online); for_each_available_child_of_node(np, subsys_np) { @@ -382,7 +364,7 @@ static int thermal_pause_probe(struct platform_device *pdev) static int thermal_pause_remove(struct platform_device *pdev) { struct thermal_pause_cdev *thermal_pause_cdev = NULL, *next = NULL; - int ret = 0, cpu = 0; + int ret = 0; if (cpu_hp_online) { cpuhp_remove_state_nocalls(cpu_hp_online); @@ -398,26 +380,15 @@ static int thermal_pause_remove(struct platform_device *pdev) list_del(&thermal_pause_cdev->node); } - cpumask_andnot(&cpus_paused_by_thermal, &cpus_paused_by_thermal, - &cpus_pending_online); - if (!cpumask_empty(&cpus_paused_by_thermal)) { mutex_unlock(&cpus_pause_lock); ret = walt_resume_cpus(&cpus_paused_by_thermal); - if (ret) + if (ret < 0) pr_err("Error resuming CPU:%*pbl. err:%d\n", cpumask_pr_args(&cpus_paused_by_thermal), ret); mutex_lock(&cpus_pause_lock); } - for_each_cpu(cpu, &cpus_pending_online) { - mutex_unlock(&cpus_pause_lock); - ret = add_cpu(cpu); - if (ret) - pr_err("Error Adding CPU:%d. err:%d\n", cpu, ret); - mutex_lock(&cpus_pause_lock); - } - mutex_unlock(&cpus_pause_lock); return 0; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c7c4ff7c2234..2f81584124cd 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4056,6 +4056,7 @@ static void walt_init(void) walt_lb_init(); walt_rt_init(); walt_cfs_init(); + walt_pause_init(); stop_machine(walt_init_stop_handler, NULL, NULL); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 1c76ba19e24b..26036f96bb26 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -732,6 +732,7 @@ extern void sched_update_hyst_times(void); extern enum sched_boost_policy sched_boost_policy(void); extern void walt_rt_init(void); extern void walt_cfs_init(void); +extern void walt_pause_init(void); extern void walt_fixup_init(void); extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync, int sibling_count_hint); diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c index 7535d27561e3..6f28cc7ff53b 100644 --- a/kernel/sched/walt/walt_pause.c +++ b/kernel/sched/walt/walt_pause.c @@ -60,8 +60,12 @@ int walt_pause_cpus(struct cpumask *cpus) cpumask_copy(&saved_cpus, cpus); - /* prior to operation, track cpus requested to be paused */ + /* add ref counts for all cpus in mask */ inc_ref_counts(cpus); + + /* only actually pause online CPUs */ + cpumask_and(cpus, cpus, cpu_online_mask); + ret = pause_cpus(cpus); if (ret < 0) { dec_test_ref_counts(&saved_cpus); @@ -81,7 +85,12 @@ int walt_resume_cpus(struct cpumask *cpus) cpumask_copy(&saved_cpus, cpus); + /* remove ref counts for all cpus in mask */ dec_test_ref_counts(cpus); + + /* only actually resume online CPUs */ + cpumask_and(cpus, cpus, cpu_online_mask); + ret = resume_cpus(cpus); if (ret < 0) { inc_ref_counts(&saved_cpus); @@ -93,4 +102,64 @@ int walt_resume_cpus(struct cpumask *cpus) } EXPORT_SYMBOL(walt_resume_cpus); +struct work_struct walt_pause_online_work; + +/* workfn to perform re-pause operation by detecting + * ref-counts and attempting to restore the state. + * It must not adjust the ref-counts for each cpu, or + * the actual paused state will no longer reflect client's + * expectations. + */ +static void walt_pause_online_workfn(struct work_struct *work) +{ + struct pause_cpu_state *pause_cpu_state; + cpumask_t re_pause_cpus; + int cpu, ret = 0; + + cpumask_clear(&re_pause_cpus); + + /* search and test all online cpus */ + spin_lock(&pause_lock); + for_each_online_cpu(cpu) { + pause_cpu_state = per_cpu_ptr(&pause_state, cpu); + if (pause_cpu_state->ref_count) + cpumask_set_cpu(cpu, &re_pause_cpus); + } + spin_unlock(&pause_lock); + + if (cpumask_empty(&re_pause_cpus)) + return; + + /* will wait for existing hp operations to complete */ + ret = pause_cpus(&re_pause_cpus); + if (ret < 0) { + pr_err("pause_cpus during online failure ret=%d cpus=%*pb1\n", ret, + cpumask_pr_args(&re_pause_cpus)); + } +} + +/* do not perform online work in hotplug context */ +static int walt_pause_hp_online(unsigned int online_cpu) +{ + struct pause_cpu_state *pause_cpu_state; + + pause_cpu_state = per_cpu_ptr(&pause_state, online_cpu); + if (pause_cpu_state->ref_count) + schedule_work(&walt_pause_online_work); + return 0; +} + +void walt_pause_init(void) +{ + int ret; + + INIT_WORK(&walt_pause_online_work, walt_pause_online_workfn); + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "walt-pause/online", + walt_pause_hp_online, NULL); + + if (ret < 0) + pr_err("failure to register cpuhp online state ret=%d\n", ret); +} + #endif /* CONFIG_HOTPLUG_CPU */ From 5475717e77fe9dddbe3e1f4676d2226c29cf93e6 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 15 Mar 2021 11:17:33 -0700 Subject: [PATCH 061/346] walt/pause: switch over to mutex and protect call to pause/resume Client code can miss the actual paused state of a cpu, if ref-counting is performed when the pause/resume operation is incomplete, or if an additional pause/resume operation executes prior to updating ref-counts. This becomes a problem with the online work handler, which can run at the same time as any walt_pause/walt_resume call. Protect the calls to pause/resume CPUs, such that the actual state of the CPUS is known and stored, prior to beginning a new operation. Ensure that the work function has accurate information before performing its resume operation. Change-Id: Iac279ac731f7c97b289547d36cce7de6e2b91c3a Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_pause.c | 86 +++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c index 6f28cc7ff53b..132e01d3d86e 100644 --- a/kernel/sched/walt/walt_pause.c +++ b/kernel/sched/walt/walt_pause.c @@ -8,7 +8,7 @@ #ifdef CONFIG_HOTPLUG_CPU -DEFINE_SPINLOCK(pause_lock); +static DEFINE_MUTEX(pause_lock); struct pause_cpu_state { int ref_count; @@ -22,14 +22,12 @@ static void inc_ref_counts(struct cpumask *cpus) int cpu; struct pause_cpu_state *pause_cpu_state; - spin_lock(&pause_lock); for_each_cpu(cpu, cpus) { pause_cpu_state = per_cpu_ptr(&pause_state, cpu); if (pause_cpu_state->ref_count) cpumask_clear_cpu(cpu, cpus); pause_cpu_state->ref_count++; } - spin_unlock(&pause_lock); } /* @@ -41,7 +39,6 @@ static void dec_test_ref_counts(struct cpumask *cpus) int cpu; struct pause_cpu_state *pause_cpu_state; - spin_lock(&pause_lock); for_each_cpu(cpu, cpus) { pause_cpu_state = per_cpu_ptr(&pause_state, cpu); WARN_ON_ONCE(pause_cpu_state->ref_count == 0); @@ -49,30 +46,37 @@ static void dec_test_ref_counts(struct cpumask *cpus) if (pause_cpu_state->ref_count) cpumask_clear_cpu(cpu, cpus); } - spin_unlock(&pause_lock); } /* cpus will be modified */ int walt_pause_cpus(struct cpumask *cpus) { - int ret; - cpumask_t saved_cpus; + int ret = 0; + cpumask_t requested_cpus; - cpumask_copy(&saved_cpus, cpus); - - /* add ref counts for all cpus in mask */ + mutex_lock(&pause_lock); inc_ref_counts(cpus); - /* only actually pause online CPUs */ + /* + * Add ref counts for all cpus in mask, but + * only actually pause online CPUs + */ cpumask_and(cpus, cpus, cpu_online_mask); + if (cpumask_empty(cpus)) + goto unlock; + + cpumask_copy(&requested_cpus, cpus); ret = pause_cpus(cpus); if (ret < 0) { - dec_test_ref_counts(&saved_cpus); + dec_test_ref_counts(&requested_cpus); pr_err("pause_cpus failure ret=%d cpus=%*pbl\n", ret, - cpumask_pr_args(&saved_cpus)); + cpumask_pr_args(&requested_cpus)); } +unlock: + mutex_unlock(&pause_lock); + return ret; } EXPORT_SYMBOL(walt_pause_cpus); @@ -80,35 +84,60 @@ EXPORT_SYMBOL(walt_pause_cpus); /* cpus will be modified */ int walt_resume_cpus(struct cpumask *cpus) { - int ret; - cpumask_t saved_cpus; + int ret = 0; + cpumask_t requested_cpus; - cpumask_copy(&saved_cpus, cpus); + mutex_lock(&pause_lock); - /* remove ref counts for all cpus in mask */ dec_test_ref_counts(cpus); /* only actually resume online CPUs */ cpumask_and(cpus, cpus, cpu_online_mask); + if (cpumask_empty(cpus)) + goto unlock; + + cpumask_copy(&requested_cpus, cpus); ret = resume_cpus(cpus); if (ret < 0) { - inc_ref_counts(&saved_cpus); + inc_ref_counts(&requested_cpus); pr_err("resume_cpus failure ret=%d cpus=%*pbl\n", ret, - cpumask_pr_args(&saved_cpus)); + cpumask_pr_args(&requested_cpus)); } +unlock: + mutex_unlock(&pause_lock); + return ret; } EXPORT_SYMBOL(walt_resume_cpus); struct work_struct walt_pause_online_work; -/* workfn to perform re-pause operation by detecting - * ref-counts and attempting to restore the state. - * It must not adjust the ref-counts for each cpu, or - * the actual paused state will no longer reflect client's - * expectations. +/* + * With refcounting and online/offline operations of the CPU + * a recent and accurate value for the requested CPUs versus + * ref-counted CPUs, must be made. + * + * When a CPU is onlined, this chain of events gets out of order. + * The online workfn can be entered at the same time as the + * walt_resume. If both are resuming the same set of CPUs + * the call to walt_pause will decrement ref-counts and think that + * the CPU is unpaused. If the workfn has already found all the + * ref-counts (and they were still set) it will re-pause + * the CPUs thinking that is what the client intended. This + * leads to a conflict, because the client software is no longer + * tracking these CPUs, and the state doesn't match what the client + * intended. + * + * This case needs protection to maintain a valid state + * of the device (where ref-counts == # of pause requests) + * Use a mutex such that the values read at the start of walt_pause, + * walt_resume, or walt_pause_online_workfn remain valid until the + * operation is complete. A mutex must be used because pause_cpus + * (and resume_cpus) cannot be called with a spinlock held, and + * the operation is not complete + * until those routines return. */ static void walt_pause_online_workfn(struct work_struct *work) { @@ -116,22 +145,25 @@ static void walt_pause_online_workfn(struct work_struct *work) cpumask_t re_pause_cpus; int cpu, ret = 0; + mutex_lock(&pause_lock); + cpumask_clear(&re_pause_cpus); /* search and test all online cpus */ - spin_lock(&pause_lock); for_each_online_cpu(cpu) { pause_cpu_state = per_cpu_ptr(&pause_state, cpu); if (pause_cpu_state->ref_count) cpumask_set_cpu(cpu, &re_pause_cpus); } - spin_unlock(&pause_lock); if (cpumask_empty(&re_pause_cpus)) - return; + goto unlock; /* will wait for existing hp operations to complete */ ret = pause_cpus(&re_pause_cpus); + +unlock: + mutex_unlock(&pause_lock); if (ret < 0) { pr_err("pause_cpus during online failure ret=%d cpus=%*pb1\n", ret, cpumask_pr_args(&re_pause_cpus)); From 0485ca1db5bc17de009ab86b84324a358976c5fd Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Mar 2021 13:20:57 -0800 Subject: [PATCH 062/346] walt/core_ctl: handle error conditions During test cases where multiple entities (thermal, core_ctl, hotplug) attempting to pause or offline CPUs, error conditions caused core_ctl to get stuck, unable to pause or resume CPUs. Improve the core_ctl implementation such that an error to pause or resume is carefully tracked, such that core_ctl will retry those operations at a later time. Simplify paused_by_us to eliminate redundant information. Change-Id: Id156e6644da5264f5552daea7d3092916c44c434 Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 95 ++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 19 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 309d60b7de38..f1d2e09b95ae 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -19,6 +19,9 @@ #include "walt.h" #include "trace.h" +/* mask of CPUs on which there is an outstanding pause claim */ +static cpumask_t cpus_paused_by_us = { CPU_BITS_NONE }; + struct cluster_data { bool inited; unsigned int min_cpus; @@ -28,7 +31,6 @@ struct cluster_data { unsigned int busy_down_thres[MAX_CPUS_PER_CLUSTER]; unsigned int active_cpus; unsigned int num_cpus; - unsigned int nr_paused_cpus; unsigned int nr_not_preferred_cpus; cpumask_t cpu_mask; unsigned int need_cpus; @@ -53,7 +55,6 @@ struct cpu_data { bool not_preferred; struct cluster_data *cluster; struct list_head sib; - bool paused_by_us; bool disabled; }; @@ -289,6 +290,14 @@ static ssize_t show_active_cpus(const struct cluster_data *state, char *buf) return scnprintf(buf, PAGE_SIZE, "%u\n", state->active_cpus); } +static unsigned int nr_paused_cpus(const struct cluster_data *cluster) +{ + cpumask_t cluster_paused_cpus; + + cpumask_and(&cluster_paused_cpus, &cluster->cpu_mask, &cpus_paused_by_us); + return cpumask_weight(&cluster_paused_cpus); +} + static ssize_t show_global_state(const struct cluster_data *state, char *buf) { struct cpu_data *c; @@ -331,7 +340,7 @@ static ssize_t show_global_state(const struct cluster_data *state, char *buf) "\tNeed CPUs: %u\n", cluster->need_cpus); count += scnprintf(buf + count, PAGE_SIZE - count, "\tNr paused CPUs: %u\n", - cluster->nr_paused_cpus); + nr_paused_cpus(cluster)); count += scnprintf(buf + count, PAGE_SIZE - count, "\tBoost: %u\n", (unsigned int) cluster->boost); } @@ -607,7 +616,7 @@ static int prev_cluster_nr_need_assist(int index) * Next cluster should not assist, while there are paused cpus * in this cluster. */ - if (prev_cluster->nr_paused_cpus) + if (nr_paused_cpus(prev_cluster)) return 0; for_each_cpu(cpu, &prev_cluster->cpu_mask) @@ -768,7 +777,7 @@ static bool adjustment_possible(const struct cluster_data *cluster, unsigned int need) { return (need < cluster->active_cpus || (need > cluster->active_cpus && - cluster->nr_paused_cpus)); + nr_paused_cpus(cluster))); } static bool need_all_cpus(const struct cluster_data *cluster) @@ -1029,8 +1038,6 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, pr_debug("Trying to pause CPU%u\n", c->cpu); cpumask_set_cpu(c->cpu, pause_cpus); nr_pending++; - c->paused_by_us = true; - cluster->nr_paused_cpus++; move_cpu_lru(c); } @@ -1059,8 +1066,6 @@ static void try_to_pause(struct cluster_data *cluster, unsigned int need, cpumask_set_cpu(c->cpu, pause_cpus); nr_pending++; - c->paused_by_us = true; - cluster->nr_paused_cpus++; move_cpu_lru(c); } @@ -1089,7 +1094,7 @@ static int __try_to_resume(struct cluster_data *cluster, unsigned int need, if (!num_cpus--) break; - if (!c->paused_by_us) + if (!cpumask_test_cpu(c->cpu, &cpus_paused_by_us)) continue; if ((cpu_online(c->cpu) && cpu_active(c->cpu)) || (!force && c->not_preferred)) @@ -1101,8 +1106,6 @@ static int __try_to_resume(struct cluster_data *cluster, unsigned int need, cpumask_set_cpu(c->cpu, unpause_cpus); nr_pending++; - c->paused_by_us = false; - cluster->nr_paused_cpus--; move_cpu_lru(c); } @@ -1122,8 +1125,7 @@ static void try_to_resume(struct cluster_data *cluster, unsigned int need, * won't be reflected yet. So use the nr_pending to adjust active * count. */ - nr_pending = __try_to_resume(cluster, need, force_use_non_preferred, - unpause_cpus); + nr_pending = __try_to_resume(cluster, need, force_use_non_preferred, unpause_cpus); if (cluster->active_cpus + nr_pending == need) return; @@ -1132,6 +1134,64 @@ static void try_to_resume(struct cluster_data *cluster, unsigned int need, __try_to_resume(cluster, need, force_use_non_preferred, unpause_cpus); } +/* + * core_ctl_pause_cpus: pause a set of CPUs as requested by core_ctl, handling errors. + * + * In order to handle errors properly, and properly track success, the cpus being + * passed to walt_pause_cpus needs to be saved off. It needs to be saved because + * walt_pause_cpus will modify the value (through pause_cpus()). Pause_cpus modifies + * the value because it updates the variable to eliminate CPUs that are already paused. + * THIS code, however, must be very careful to track what cpus were requested, rather + * than what cpus actually were paused in this action. Otherwise, the ref-counts in + * walt_pause.c will get out of sync with this code. + */ +static void core_ctl_pause_cpus(struct cpumask *cpus_to_pause) +{ + cpumask_t saved_cpus; + + /* be careful to only pause CPUs not paused before to ensure ref-count sync */ + cpumask_andnot(cpus_to_pause, cpus_to_pause, &cpus_paused_by_us); + cpumask_copy(&saved_cpus, cpus_to_pause); + + if (cpumask_any(cpus_to_pause) < nr_cpu_ids) { + if (walt_pause_cpus(cpus_to_pause) < 0) + pr_warn("core_ctl pause operation failed cpus=%*pbl paused_by_us=%*pbl\n", + cpumask_pr_args(cpus_to_pause), + cpumask_pr_args(&cpus_paused_by_us)); + else + cpumask_or(&cpus_paused_by_us, &cpus_paused_by_us, &saved_cpus); + } +} + +/* + * core_ctl_resume_cpus: resume a set of CPUs as requested by core_ctl, handling errors. + * + * In order to handle errors properly, and properly track success, the cpus being + * passed to walt_resume_cpus needs to be saved off. It needs to be saved because + * walt_resume_cpus will modify the value (through resume_cpus()). Resume_cpus modifies + * the value because it updates the variable to eliminate CPUs that are already resumed. + * THIS code, however, must be very careful to track what cpus were requested, rather + * than what cpus actually were resumed in this action. Otherwise, the ref-counts in + * walt_pause.c will get out of sync with this code. + */ +static void core_ctl_resume_cpus(struct cpumask *cpus_to_unpause) +{ + cpumask_t saved_cpus; + + /* be careful to only unpause CPUs paused before to ensure ref-count sync */ + cpumask_and(cpus_to_unpause, cpus_to_unpause, &cpus_paused_by_us); + cpumask_copy(&saved_cpus, cpus_to_unpause); + + if (cpumask_any(cpus_to_unpause) < nr_cpu_ids) { + if (walt_resume_cpus(cpus_to_unpause) < 0) + pr_warn("core_ctl resume operation failed cpus=%*pbl paused_by_us=%*pbl\n", + cpumask_pr_args(cpus_to_unpause), + cpumask_pr_args(&cpus_paused_by_us)); + else + cpumask_andnot(&cpus_paused_by_us, &cpus_paused_by_us, &saved_cpus); + } +} + static void __ref do_core_ctl(void) { struct cluster_data *cluster; @@ -1157,11 +1217,8 @@ static void __ref do_core_ctl(void) } } - if (cpumask_any(&cpus_to_pause) < nr_cpu_ids) - walt_pause_cpus(&cpus_to_pause); - - if (cpumask_any(&cpus_to_unpause) < nr_cpu_ids) - walt_resume_cpus(&cpus_to_unpause); + core_ctl_pause_cpus(&cpus_to_pause); + core_ctl_resume_cpus(&cpus_to_unpause); } static int __ref try_core_ctl(void *data) From 54adb8abcc073e062f79576e33f5f2c27de60b05 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:47:54 -0700 Subject: [PATCH 063/346] thermal/pause: thermal driver not tracking pause state With refcounting, a cpu will get stuck in a paused state if client software doesn't account properly for paused CPUs. To properly track which CPUs it has made a request to be paused, the driver needs to record the success or failure of an operation and only record a change in CPU state, in the case of success. Change-Id: I7bab17026d67ac1733c77083eae97af51c30dd37 Signed-off-by: Stephen Dickey Signed-off-by: Sai Harshini Nimmala --- drivers/thermal/qcom/thermal_pause.c | 165 ++++++++++++++------------- kernel/sched/walt/walt_pause.c | 24 ++-- 2 files changed, 99 insertions(+), 90 deletions(-) diff --git a/drivers/thermal/qcom/thermal_pause.c b/drivers/thermal/qcom/thermal_pause.c index 41d511d82d59..f76db4033441 100644 --- a/drivers/thermal/qcom/thermal_pause.c +++ b/drivers/thermal/qcom/thermal_pause.c @@ -36,10 +36,6 @@ struct thermal_pause_cdev { static DEFINE_MUTEX(cpus_pause_lock); static LIST_HEAD(thermal_pause_cdev_list); -/* track cpus that thermal intends to pause, regardless of - * whether thermal was requesting an already paused cpu - */ -static struct cpumask cpus_paused_by_thermal; static struct cpumask cpus_in_max_cooling_level; static enum cpuhp_state cpu_hp_online; @@ -65,22 +61,17 @@ static int thermal_pause_hp_online(unsigned int online_cpu) struct thermal_pause_cdev *thermal_pause_cdev; int ret = 0; - pr_debug("online entry CPU:%d. mask:%*pbl\n", online_cpu, - cpumask_pr_args(&cpus_paused_by_thermal)); + pr_debug("online entry CPU:%d\n", online_cpu); mutex_lock(&cpus_pause_lock); - list_for_each_entry(thermal_pause_cdev, &thermal_pause_cdev_list, node) { if (cpumask_test_cpu(online_cpu, &thermal_pause_cdev->cpu_mask) && !thermal_pause_cdev->cdev) queue_work(system_highpri_wq, &thermal_pause_cdev->reg_work); } - mutex_unlock(&cpus_pause_lock); - pr_debug("online exit CPU:%d. mask:%*pbl\n", online_cpu, - cpumask_pr_args(&cpus_paused_by_thermal)); - + pr_debug("online exit CPU:%d\n", online_cpu); return ret; } @@ -99,35 +90,39 @@ static int thermal_pause_hp_online(unsigned int online_cpu) static int thermal_pause_cpus_pause(struct thermal_pause_cdev *thermal_pause_cdev) { int cpu = 0; - int ret = -ENODEV; - cpumask_t cpus_to_pause; + int ret = -EBUSY; + cpumask_t cpus_to_pause, cpus_to_notify; - cpumask_clear(&cpus_to_pause); - cpumask_andnot(&cpus_to_pause, &thermal_pause_cdev->cpu_mask, - &cpus_paused_by_thermal); + if (thermal_pause_cdev->thermal_pause_level) + return ret; - if (!cpumask_empty(&cpus_to_pause)) { + cpumask_copy(&cpus_to_pause, &thermal_pause_cdev->cpu_mask); + pr_debug("Pause:%*pbl\n", cpumask_pr_args(&thermal_pause_cdev->cpu_mask)); - cpumask_or(&cpus_paused_by_thermal, - &cpus_paused_by_thermal, &cpus_to_pause); - pr_debug("Active req:%*pbl pause:%*pbl mask:%*pbl\n", - cpumask_pr_args(&cpus_paused_by_thermal), - cpumask_pr_args(&cpus_to_pause), - cpumask_pr_args(&thermal_pause_cdev->cpu_mask)); - mutex_unlock(&cpus_pause_lock); + mutex_unlock(&cpus_pause_lock); + ret = walt_pause_cpus(&cpus_to_pause); + mutex_lock(&cpus_pause_lock); - ret = walt_pause_cpus(&cpus_to_pause); + if (ret == 0) { + /* remove CPUs that have already been notified */ + cpumask_andnot(&cpus_to_notify, &thermal_pause_cdev->cpu_mask, + &cpus_in_max_cooling_level); - mutex_lock(&cpus_pause_lock); + for_each_cpu(cpu, &cpus_to_notify) + blocking_notifier_call_chain(&thermal_pause_notifier, 1, + (void *)(long)cpu); - for_each_cpu(cpu, &cpus_to_pause) - blocking_notifier_call_chain( - &multi_max_cooling_level_notifer, - 1, (void *)(long)cpu); + /* track CPUs currently in cooling level */ + cpumask_or(&cpus_in_max_cooling_level, + &cpus_in_max_cooling_level, + &thermal_pause_cdev->cpu_mask); + } else { + /* Failure. These cpus not paused by thermal */ + pr_err("Error pausing CPU:%*pbl. err:%d\n", + cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret); + return ret; } - cpumask_copy(&cpus_in_max_cooling_level, &cpus_paused_by_thermal); - return ret; } @@ -145,40 +140,45 @@ static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_c { int cpu = 0; int ret = -ENODEV; + cpumask_t cpus_to_unpause, new_paused_cpus, cpus_to_notify; struct thermal_pause_cdev *cdev; - cpumask_t new_cpu_pause, cpus_to_unpause; - cpumask_clear(&new_cpu_pause); - list_for_each_entry(cdev, &thermal_pause_cdev_list, node) { - if (!cdev->thermal_pause_level) - continue; - cpumask_or(&new_cpu_pause, &new_cpu_pause, - &cdev->cpu_mask); + /* do not unpause a cooling device not paused */ + if (!thermal_pause_cdev->thermal_pause_level) + return ret; + + cpumask_copy(&cpus_to_unpause, &thermal_pause_cdev->cpu_mask); + pr_debug("Unpause:%*pbl\n", cpumask_pr_args(&cpus_to_unpause)); + + mutex_unlock(&cpus_pause_lock); + ret = walt_resume_cpus(&cpus_to_unpause); + mutex_lock(&cpus_pause_lock); + + if (ret == 0) { + /* gather up the cpus paused state from all the cdevs */ + cpumask_clear(&new_paused_cpus); + list_for_each_entry(cdev, &thermal_pause_cdev_list, node) { + if (!cdev->thermal_pause_level || cdev == thermal_pause_cdev) + continue; + cpumask_or(&new_paused_cpus, &new_paused_cpus, &cdev->cpu_mask); + } + + /* remove CPUs that will remain paused */ + cpumask_andnot(&cpus_to_notify, &cpus_in_max_cooling_level, &new_paused_cpus); + + /* Notify for each CPU that we intended to resume */ + for_each_cpu(cpu, &cpus_to_notify) + blocking_notifier_call_chain(&thermal_pause_notifier, 0, + (void *)(long)cpu); + + /* update the cpus cooling mask */ + cpumask_copy(&cpus_in_max_cooling_level, &new_paused_cpus); + } else { + /* Failure. Ref-count for cpus controlled by thermal still set */ + pr_err("Error resuming CPU:%*pbl. err:%d\n", + cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret); + return ret; } - cpumask_andnot(&cpus_to_unpause, &cpus_paused_by_thermal, - &new_cpu_pause); - - pr_debug("Old req:%*pbl New req:%*pbl Unpause:%*pbl\n", - cpumask_pr_args(&cpus_paused_by_thermal), - cpumask_pr_args(&new_cpu_pause), - cpumask_pr_args(&cpus_to_unpause)); - - cpumask_copy(&cpus_in_max_cooling_level, &new_cpu_pause); - cpumask_copy(&cpus_paused_by_thermal, &new_cpu_pause); - - if (!cpumask_empty(&cpus_to_unpause)) { - mutex_unlock(&cpus_pause_lock); - ret = walt_resume_cpus(&cpus_to_unpause); - if (ret) - pr_err("Error resuming CPU:%*pbl. err:%d\n", - cpumask_pr_args(&cpus_to_unpause), ret); - mutex_lock(&cpus_pause_lock); - } - - for_each_cpu(cpu, &cpus_to_unpause) - blocking_notifier_call_chain( - &multi_max_cooling_level_notifer, - 0, (void *)(long)cpu); return ret; } @@ -207,16 +207,19 @@ static int thermal_pause_set_cur_state(struct thermal_cooling_device *cdev, return 0; mutex_lock(&cpus_pause_lock); - - thermal_pause_cdev->thermal_pause_level = level; - if (level == THERMAL_GROUP_CPU_PAUSE) ret = thermal_pause_cpus_pause(thermal_pause_cdev); else ret = thermal_pause_cpus_unpause(thermal_pause_cdev); + /* + * only change the pause level if things were successful. Otherwise + * an unsuccessful pause operation can be followed by a resume + * operation, resuming cpus not paused by this cooling device. + */ + if (ret == 0) + thermal_pause_cdev->thermal_pause_level = level; mutex_unlock(&cpus_pause_lock); - return ret; } @@ -290,7 +293,6 @@ static void thermal_pause_register_cdev(struct work_struct *work) thermal_pause_cdev->cdev = NULL; return; } - pr_debug("Cooling device [%s] registered.\n", thermal_pause_cdev->cdev_name); } @@ -308,7 +310,6 @@ static int thermal_pause_probe(struct platform_device *pdev) INIT_LIST_HEAD(&thermal_pause_cdev_list); cpumask_clear(&cpus_in_max_cooling_level); - cpumask_clear(&cpus_paused_by_thermal); for_each_available_child_of_node(np, subsys_np) { @@ -374,24 +375,30 @@ static int thermal_pause_remove(struct platform_device *pdev) mutex_lock(&cpus_pause_lock); list_for_each_entry_safe(thermal_pause_cdev, next, &thermal_pause_cdev_list, node) { + + /* for each asserted cooling device, resume the CPUs */ + if (thermal_pause_cdev->thermal_pause_level) { + mutex_unlock(&cpus_pause_lock); + ret = walt_resume_cpus(&thermal_pause_cdev->cpu_mask); + + if (ret < 0) + pr_err("Error resuming CPU:%*pbl. err:%d\n", + cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret); + mutex_lock(&cpus_pause_lock); + } + if (thermal_pause_cdev->cdev) thermal_cooling_device_unregister( thermal_pause_cdev->cdev); list_del(&thermal_pause_cdev->node); } - if (!cpumask_empty(&cpus_paused_by_thermal)) { - mutex_unlock(&cpus_pause_lock); - ret = walt_resume_cpus(&cpus_paused_by_thermal); - if (ret < 0) - pr_err("Error resuming CPU:%*pbl. err:%d\n", - cpumask_pr_args(&cpus_paused_by_thermal), ret); - mutex_lock(&cpus_pause_lock); - } - mutex_unlock(&cpus_pause_lock); - return 0; + /* if the resume failed, thermal still controls the CPUs. + * ensure that the error is passed to the caller. + */ + return ret; } static const struct of_device_id thermal_pause_match[] = { diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c index 132e01d3d86e..a80b6d72f9a0 100644 --- a/kernel/sched/walt/walt_pause.c +++ b/kernel/sched/walt/walt_pause.c @@ -55,6 +55,8 @@ int walt_pause_cpus(struct cpumask *cpus) cpumask_t requested_cpus; mutex_lock(&pause_lock); + + cpumask_copy(&requested_cpus, cpus); inc_ref_counts(cpus); /* @@ -66,14 +68,15 @@ int walt_pause_cpus(struct cpumask *cpus) if (cpumask_empty(cpus)) goto unlock; - cpumask_copy(&requested_cpus, cpus); ret = pause_cpus(cpus); + if (ret < 0) { - dec_test_ref_counts(&requested_cpus); pr_err("pause_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); - } + /* ref counts recorded, suppress failure */ + ret = 0; + } unlock: mutex_unlock(&pause_lock); @@ -88,7 +91,7 @@ int walt_resume_cpus(struct cpumask *cpus) cpumask_t requested_cpus; mutex_lock(&pause_lock); - + cpumask_copy(&requested_cpus, cpus); dec_test_ref_counts(cpus); /* only actually resume online CPUs */ @@ -97,14 +100,15 @@ int walt_resume_cpus(struct cpumask *cpus) if (cpumask_empty(cpus)) goto unlock; - cpumask_copy(&requested_cpus, cpus); ret = resume_cpus(cpus); + if (ret < 0) { - inc_ref_counts(&requested_cpus); pr_err("resume_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); - } + /* ref counts recorded, suppress failure */ + ret = 0; + } unlock: mutex_unlock(&pause_lock); @@ -159,15 +163,14 @@ static void walt_pause_online_workfn(struct work_struct *work) if (cpumask_empty(&re_pause_cpus)) goto unlock; - /* will wait for existing hp operations to complete */ ret = pause_cpus(&re_pause_cpus); unlock: mutex_unlock(&pause_lock); - if (ret < 0) { + + if (ret < 0) pr_err("pause_cpus during online failure ret=%d cpus=%*pb1\n", ret, cpumask_pr_args(&re_pause_cpus)); - } } /* do not perform online work in hotplug context */ @@ -193,5 +196,4 @@ void walt_pause_init(void) if (ret < 0) pr_err("failure to register cpuhp online state ret=%d\n", ret); } - #endif /* CONFIG_HOTPLUG_CPU */ From a1ccf98f6fecaa75b0dc94a36b83ee21c2e9312e Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 8 Mar 2021 11:13:53 +0530 Subject: [PATCH 064/346] sched/walt: Update CPU capacity based on cpufreq constraints The cpufreq constraints must be reflected in the capacity of a CPU. Add a hook for cpufreq limits to implement this. Change-Id: I779cb16bd66ea99c16e55faea7383e034d22440a Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 39 ++++++++++++++++++++------------------- kernel/sched/walt/walt.h | 9 +++++++++ 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2f81584124cd..790a2167aded 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -16,6 +16,7 @@ #include #include +#include #include "walt.h" #include "trace.h" @@ -732,13 +733,6 @@ static void update_rq_load_subtractions(int index, struct rq *rq, wrq->load_subs[index].new_subs += sub_load; } -static inline struct walt_sched_cluster *cpu_cluster(int cpu) -{ - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; - - return wrq->cluster; -} - static void update_cluster_load_subtractions(struct task_struct *p, int cpu, u64 ws, bool new_task) { @@ -2306,6 +2300,7 @@ static struct walt_sched_cluster *alloc_new_cluster(const struct cpumask *cpus) INIT_LIST_HEAD(&cluster->list); cluster->cur_freq = 1; + cluster->max_freq = 1; cluster->max_possible_freq = 1; raw_spin_lock_init(&cluster->load_lock); @@ -2511,6 +2506,7 @@ static void walt_update_cluster_topology(void) if (policy) { cluster->max_possible_freq = policy->cpuinfo.max_freq; + cluster->max_freq = policy->max; for_each_cpu(i, &cluster->cpus) { wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; cpumask_copy(&wrq->freq_domain_cpumask, @@ -3630,6 +3626,14 @@ static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new) add_new_task_to_grp(new); } +static void walt_cpu_frequency_limits(void *unused, struct cpufreq_policy *policy) +{ + if (unlikely(walt_disabled)) + return; + + cpu_cluster(policy->cpu)->max_freq = policy->max; +} + /* * The intention of this hook is to update cpu_capacity_orig as well as * (*capacity), otherwise we will end up capacity_of() > capacity_orig_of(). @@ -3639,6 +3643,7 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long unsigned long max_capacity = arch_scale_cpu_capacity(cpu); unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); unsigned long thermal_cap; + struct walt_sched_cluster *cluster; if (unlikely(walt_disabled)) return; @@ -3651,18 +3656,13 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long thermal_cap = max_capacity - thermal_pressure; - /* - * TODO: - * Thermal is taken care now. but what about limits via - * cpufreq max. we don't have arch_scale_max_freq_capacity() - * in 5.10 now. - * - * Two options: - * #1 either port that max_frq_cap patch to AOSP - * #2 register for cpufreq policy updates.. - */ - cpu_rq(cpu)->cpu_capacity_orig = min(cpu_rq(cpu)->cpu_capacity_orig, - thermal_cap); + cluster = cpu_cluster(cpu); + /* reduce the max_capacity under cpufreq constraints */ + if (cluster->max_freq != cluster->max_possible_freq) + max_capacity = mult_frac(max_capacity, cluster->max_freq, + cluster->max_possible_freq); + + cpu_rq(cpu)->cpu_capacity_orig = min(max_capacity, thermal_cap); *capacity = cpu_rq(cpu)->cpu_capacity_orig; } @@ -3983,6 +3983,7 @@ static void register_walt_hooks(void) register_trace_android_rvh_ttwu_cond(android_rvh_ttwu_cond, NULL); register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); + register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); } atomic64_t walt_irq_work_lastq_ws; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 26036f96bb26..666fb458b79f 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -127,9 +127,11 @@ struct walt_sched_cluster { int id; /* * max_possible_freq = maximum supported by hardware + * max_freq = max freq as per cpufreq limits */ unsigned int cur_freq; unsigned int max_possible_freq; + unsigned int max_freq; u64 aggr_grp_load; }; @@ -226,6 +228,13 @@ extern struct list_head cluster_head; #define for_each_sched_cluster(cluster) \ list_for_each_entry_rcu(cluster, &cluster_head, list) +static inline struct walt_sched_cluster *cpu_cluster(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster; +} + static inline u32 cpu_cycles_to_freq(u64 cycles, u64 period) { return div64_u64(cycles, period); From c12499d9380d1930a64e7b25cdb3b3513b56c037 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 25 Mar 2021 09:49:45 +0530 Subject: [PATCH 065/346] sched/walt: Adjust code as per new dequeue_task() hook invocation dequeue_task() hook invocation is being changed from post dequeue to pre dequeue. Consider the task being dequeued is out of the scheduling class managed list. Change-Id: Ib95ec7f8a873587493487a3c31c3f53ea78b0d95 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 790a2167aded..ab3f4a6d0b62 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -419,7 +419,7 @@ static inline bool is_ed_task(struct task_struct *p, u64 wallclock) return (wallclock - wts->last_wake_ts >= EARLY_DETECTION_DURATION); } -static bool is_ed_task_present(struct rq *rq, u64 wallclock) +static bool is_ed_task_present(struct rq *rq, u64 wallclock, struct task_struct *deq_task) { struct task_struct *p; int loop_max = 10; @@ -434,6 +434,9 @@ static bool is_ed_task_present(struct rq *rq, u64 wallclock) if (!loop_max) break; + if (p == deq_task) + continue; + if (is_ed_task(p, wallclock)) { wrq->ed_task = p; return true; @@ -3758,7 +3761,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st if (unlikely(walt_disabled)) return; if (p == wrq->ed_task) - is_ed_task_present(rq, sched_ktime_clock()); + is_ed_task_present(rq, sched_ktime_clock(), p); sched_update_nr_prod(rq->cpu, false); @@ -3866,7 +3869,7 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) set_preferred_cluster(grp); rcu_read_unlock(); - if (is_ed_task_present(rq, wallclock)) + if (is_ed_task_present(rq, wallclock, NULL)) waltgov_run_callback(rq, WALT_CPUFREQ_EARLY_DET); } From a0d9ded275776fee0376c72850a25f74f6195a7d Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Mon, 29 Mar 2021 20:51:48 -0700 Subject: [PATCH 066/346] sched/walt: update check during debug module init preemptirq_long_init() return value is incorrectly being checked which is causing few of the trace hooks to be not even registered. Update the check to make sure return value is checked properly. Change-Id: If68a23d585c9e460ec1e0f4902a4ed9c9cae4c68 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_debug.c b/kernel/sched/walt/walt_debug.c index f8679c032f39..0165b354d3a4 100644 --- a/kernel/sched/walt/walt_debug.c +++ b/kernel/sched/walt/walt_debug.c @@ -36,7 +36,7 @@ static int __init walt_debug_init(void) int ret; ret = preemptirq_long_init(); - if (!ret) + if (ret) return ret; register_trace_android_vh_dump_throttled_rt_tasks(dump_throttled_rt_tasks, NULL); From 5e62bc10d05850efe8ee16004cecc883f0db74dd Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 6 Apr 2021 15:55:03 -0700 Subject: [PATCH 067/346] sched/walt: Modify sched_task_util tracepoint Reduce redundant parameters passed to sched_task_util tracepoint by invoking relevant functions that return these values during runtime. Change-Id: Ie146a5a7a2ac6e4879115f49d4ccc6f7dcadcb52 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/trace.h | 9 ++++----- kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_cfs.c | 5 ++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index cdb08fb12771..fb21263b15b5 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -905,12 +905,11 @@ TRACE_EVENT(sched_task_util, TP_PROTO(struct task_struct *p, unsigned long candidates, int best_energy_cpu, bool sync, int need_idle, int fastpath, - bool placement_boost, u64 start_t, - bool uclamp_boosted, bool is_rtg, bool rtg_skip_min, + u64 start_t, bool uclamp_boosted, bool is_rtg, int start_cpu), TP_ARGS(p, candidates, best_energy_cpu, sync, need_idle, fastpath, - placement_boost, start_t, uclamp_boosted, is_rtg, rtg_skip_min, + start_t, uclamp_boosted, is_rtg, start_cpu), TP_STRUCT__entry( @@ -946,11 +945,11 @@ TRACE_EVENT(sched_task_util, __entry->sync = sync; __entry->need_idle = need_idle; __entry->fastpath = fastpath; - __entry->placement_boost = placement_boost; + __entry->placement_boost = task_boost_policy(p); __entry->latency = (sched_clock() - start_t); __entry->uclamp_boosted = uclamp_boosted; __entry->is_rtg = is_rtg; - __entry->rtg_skip_min = rtg_skip_min; + __entry->rtg_skip_min = walt_get_rtg_status(p); __entry->start_cpu = start_cpu; __entry->unfilter = ((struct walt_task_struct *) p->android_vendor_data1)->unfilter; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 666fb458b79f..741d0c6e1717 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -738,6 +738,7 @@ static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); extern void sched_update_hyst_times(void); +extern bool walt_get_rtg_status(struct task_struct *p); extern enum sched_boost_policy sched_boost_policy(void); extern void walt_rt_init(void); extern void walt_cfs_init(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index cfd63eb6aa19..46960f30f70b 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -92,7 +92,7 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) return min_t(unsigned long, util, capacity_orig_of(cpu)); } -static inline bool walt_get_rtg_status(struct task_struct *p) +bool walt_get_rtg_status(struct task_struct *p) { struct walt_related_thread_group *grp; bool ret = false; @@ -648,8 +648,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, done: trace_sched_task_util(p, cpumask_bits(candidates)[0], best_energy_cpu, sync, fbt_env.need_idle, fbt_env.fastpath, - task_boost_policy(p), start_t, boosted, is_rtg, - walt_get_rtg_status(p), start_cpu); + start_t, boosted, is_rtg, start_cpu); return best_energy_cpu; From 60dadb83821a6c6646e1796f56be2328f1de5334 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 8 Apr 2021 16:40:57 +0530 Subject: [PATCH 068/346] sched/walt: sched_task_util trace point updates As part of reducing the arguments to sched_task_util trace point, remove is_rtg argument as well. It can be retrieved from task_struct. The recent update to this trace point made walt_get_rtg_status() an extern function. Move it to a header file and restore the inline functionality. Change-Id: If447600fd9835e3b78724eb7a94b87c79090fe7f Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/trace.h | 8 +++----- kernel/sched/walt/walt.h | 17 ++++++++++++++++- kernel/sched/walt/walt_cfs.c | 18 +----------------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index fb21263b15b5..4effae0bbaad 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -905,12 +905,10 @@ TRACE_EVENT(sched_task_util, TP_PROTO(struct task_struct *p, unsigned long candidates, int best_energy_cpu, bool sync, int need_idle, int fastpath, - u64 start_t, bool uclamp_boosted, bool is_rtg, - int start_cpu), + u64 start_t, bool uclamp_boosted, int start_cpu), TP_ARGS(p, candidates, best_energy_cpu, sync, need_idle, fastpath, - start_t, uclamp_boosted, is_rtg, - start_cpu), + start_t, uclamp_boosted, start_cpu), TP_STRUCT__entry( __field(int, pid) @@ -948,7 +946,7 @@ TRACE_EVENT(sched_task_util, __entry->placement_boost = task_boost_policy(p); __entry->latency = (sched_clock() - start_t); __entry->uclamp_boosted = uclamp_boosted; - __entry->is_rtg = is_rtg; + __entry->is_rtg = task_in_related_thread_group(p); __entry->rtg_skip_min = walt_get_rtg_status(p); __entry->start_cpu = start_cpu; __entry->unfilter = diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 741d0c6e1717..0f1a71064769 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -738,7 +738,6 @@ static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); extern void sched_update_hyst_times(void); -extern bool walt_get_rtg_status(struct task_struct *p); extern enum sched_boost_policy sched_boost_policy(void); extern void walt_rt_init(void); extern void walt_cfs_init(void); @@ -795,6 +794,22 @@ static inline struct walt_related_thread_group return rcu_dereference(wts->grp); } +static inline bool walt_get_rtg_status(struct task_struct *p) +{ + struct walt_related_thread_group *grp; + bool ret = false; + + rcu_read_lock(); + + grp = task_related_thread_group(p); + if (grp) + ret = grp->skip_min; + + rcu_read_unlock(); + + return ret; +} + #define CPU_RESERVED 1 static inline int is_reserved(int cpu) { diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 46960f30f70b..f3e725bdace4 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -92,22 +92,6 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) return min_t(unsigned long, util, capacity_orig_of(cpu)); } -bool walt_get_rtg_status(struct task_struct *p) -{ - struct walt_related_thread_group *grp; - bool ret = false; - - rcu_read_lock(); - - grp = task_related_thread_group(p); - if (grp) - ret = grp->skip_min; - - rcu_read_unlock(); - - return ret; -} - static inline bool walt_task_skip_min_cpu(struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -648,7 +632,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, done: trace_sched_task_util(p, cpumask_bits(candidates)[0], best_energy_cpu, sync, fbt_env.need_idle, fbt_env.fastpath, - start_t, boosted, is_rtg, start_cpu); + start_t, boosted, start_cpu); return best_energy_cpu; From 960f8d4f0f3b861d28809f69eb41838b6323ba6c Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 9 Apr 2021 09:41:37 +0530 Subject: [PATCH 069/346] sched/walt: Remove unused sched_dynamic_ravg_window_enable tunable sched_dynamic_ravg_window_enable tunable is not currently used. If 0 is written to this tunable, the dynamic window size feature is disabled in the kernel which results in bad performance. So remove this unused tunable. Change-Id: I64282146e526f98d049be25502ef638f538b0496 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/sysctl.c | 14 +------------- kernel/sched/walt/walt.h | 1 - 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 0f27133c7a6b..8cefdeeede32 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -47,7 +47,6 @@ unsigned int __read_mostly sysctl_sched_group_downmigrate_pct; unsigned int __read_mostly sysctl_sched_group_upmigrate_pct; unsigned int __read_mostly sysctl_sched_window_stats_policy; unsigned int sysctl_sched_ravg_window_nr_ticks; -unsigned int sysctl_sched_dynamic_ravg_window_enable; unsigned int sysctl_sched_walt_rotate_big_tasks; unsigned int sysctl_sched_task_unfilter_period; unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; @@ -182,7 +181,7 @@ static int sched_ravg_window_handler(struct ctl_table *table, mutex_lock(&mutex); - if (write && (HZ != 250 || !sysctl_sched_dynamic_ravg_window_enable)) + if (write && HZ != 250) goto unlock; ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); @@ -675,15 +674,6 @@ struct ctl_table walt_table[] = { .mode = 0644, .proc_handler = sched_ravg_window_handler, }, - { - .procname = "sched_dynamic_ravg_window_enable", - .data = &sysctl_sched_dynamic_ravg_window_enable, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, { .procname = "sched_upmigrate", .data = &sysctl_sched_capacity_margin_up_pct, @@ -830,8 +820,6 @@ void walt_tunables(void) sysctl_sched_ravg_window_nr_ticks = (HZ / NR_WINDOWS_PER_SEC); - sysctl_sched_dynamic_ravg_window_enable = (HZ == 250); - sched_load_granule = DEFAULT_SCHED_RAVG_WINDOW / NUM_LOAD_INDICES; for (i = 0; i < WALT_NR_CPUS; i++) { diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 0f1a71064769..1d87cdf8ef19 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -271,7 +271,6 @@ extern unsigned int __read_mostly sysctl_sched_group_downmigrate_pct; extern unsigned int __read_mostly sysctl_sched_group_upmigrate_pct; extern unsigned int __read_mostly sysctl_sched_window_stats_policy; extern unsigned int sysctl_sched_ravg_window_nr_ticks; -extern unsigned int sysctl_sched_dynamic_ravg_window_enable; extern unsigned int sysctl_sched_walt_rotate_big_tasks; extern unsigned int sysctl_sched_task_unfilter_period; extern unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; From 91bd0dce3b29738befd8912a569a2240df89a49e Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 9 Apr 2021 09:43:46 +0530 Subject: [PATCH 070/346] sched/walt: Read sysctl_sched_ravg_window_nr_ticks under mutex When sched_ravg_window_nr_ticks tunable is read and write concurrently, the read can return the previous value though the write is already happened. Fix this by reading the sysctl_sched_ravg_window_nr_ticks value under mutex. Change-Id: Ie1146c47ac1a184fc8bf4a2824bf6db28fd7156d Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/sysctl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 8cefdeeede32..4816d702c790 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -171,7 +171,7 @@ static int sched_ravg_window_handler(struct ctl_table *table, { int ret = -EPERM; static DEFINE_MUTEX(mutex); - int val = sysctl_sched_ravg_window_nr_ticks; + int val; struct ctl_table tmp = { .data = &val, @@ -184,6 +184,7 @@ static int sched_ravg_window_handler(struct ctl_table *table, if (write && HZ != 250) goto unlock; + val = sysctl_sched_ravg_window_nr_ticks; ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); if (ret || !write || (val == sysctl_sched_ravg_window_nr_ticks)) goto unlock; From fe4c36c11bedf8768e8589fdced23d120150ca63 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 9 Apr 2021 13:55:16 -0700 Subject: [PATCH 071/346] sched/walt: core_ctl init out of order core_ctl is launching a kthread which uses a spinlock, prior to initializing the spinlock. This creates a race, and a lockup. Change-Id: I170a77addc6e4c2cc90b6683b50723008760f2f4 Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index f1d2e09b95ae..d6e9d28298e7 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -1327,14 +1327,14 @@ int core_ctl_init(void) struct walt_sched_cluster *cluster; int ret; - /* initialize our single kthread */ + spin_lock_init(&core_ctl_pending_lock); + + /* initialize our single kthread, after spin lock init */ core_ctl_thread = kthread_run(try_core_ctl, NULL, "core_ctl"); if (IS_ERR(core_ctl_thread)) return PTR_ERR(core_ctl_thread); - spin_lock_init(&core_ctl_pending_lock); - sched_setscheduler_nocheck(core_ctl_thread, SCHED_FIFO, ¶m); for_each_sched_cluster(cluster) { From 8237cc8693bc26e3949830d6537f5c6b348ba1a0 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 12 Apr 2021 12:14:15 -0700 Subject: [PATCH 072/346] sched/walt: null ptr dereference fix in walt_lb_active_migration It is possible that wrq->push_task is null but walt_lb_active_migration assumes it is non-null. Local variable push_task is used after copying the value, and can be dereferenced. Update walt_lb_active_migration to safely check push_task, and ensure that the active migration flag is cleared. Change-Id: I3b555430f1ba940053756fcbf2cca5c7d19de9fb Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_lb.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 30202177a806..960da0dca5d4 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -45,7 +45,7 @@ static int walt_lb_active_migration(void *data) raw_spin_lock_irq(&busiest_rq->lock); /* sanity checks before initiating the pull */ - if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) + if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu) || !push_task) goto out_unlock; if (unlikely(busiest_cpu != raw_smp_processor_id() || @@ -73,15 +73,16 @@ static int walt_lb_active_migration(void *data) raw_spin_unlock(&busiest_rq->lock); if (push_task_detached) { - if (push_task_detached) { - raw_spin_lock(&target_rq->lock); - walt_attach_task(push_task, target_rq); - raw_spin_unlock(&target_rq->lock); - } + raw_spin_lock(&target_rq->lock); + walt_attach_task(push_task, target_rq); + raw_spin_unlock(&target_rq->lock); } - put_task_struct(push_task); + + if (push_task) + put_task_struct(push_task); local_irq_enable(); + return 0; } From f7fe6d674912f7ca4afa3f38cf7ec7ed26acb466 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 14 Apr 2021 16:32:18 -0700 Subject: [PATCH 073/346] sched/walt: add rt_time to dump_throttled_rt_tasks Add rq->rt.rt_time to debug output to better track the accumulated time that exceeded runtime. This will allow us to have a better understanding of the decision made to throttle RT threads. Change-Id: Icd8bc43583bddd61019f0d7f73a2315a8103d94a Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_debug.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_debug.c b/kernel/sched/walt/walt_debug.c index 0165b354d3a4..935c3626b427 100644 --- a/kernel/sched/walt/walt_debug.c +++ b/kernel/sched/walt/walt_debug.c @@ -4,6 +4,7 @@ */ #include +#include #include @@ -14,8 +15,9 @@ static void dump_throttled_rt_tasks(void *unused, int cpu, u64 clock, ktime_t rt_period, u64 rt_runtime, s64 rt_period_timer_expires) { printk_deferred("sched: RT throttling activated for cpu %d\n", cpu); - printk_deferred("rt_period_timer: expires=%lld now=%llu runtime=%llu period=%llu\n", - rt_period_timer_expires, ktime_get_ns(), rt_runtime, rt_period); + printk_deferred("rt_period_timer: expires=%lld now=%llu rt_time=%llu runtime=%llu period=%llu\n", + rt_period_timer_expires, ktime_get_ns(), + task_rq(current)->rt.rt_time, rt_runtime, rt_period); printk_deferred("potential CPU hogs:\n"); #ifdef CONFIG_SCHED_INFO if (sched_info_on()) From c87f1e133c1e48046ee4d0656f85b54bba1eac20 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 20 Apr 2021 13:19:38 -0700 Subject: [PATCH 074/346] sched: walt: remove extra function call In FBT, we are making an additional call to uclamp_task_util while we have already cached that value in a local variable. Use the cached value instead of calling the function again. Change-Id: I05828f02570056309256907aa7fee1396134706a Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index f3e725bdace4..f667f0fcf8ac 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -242,7 +242,7 @@ static void walt_find_best_target(struct sched_domain *sd, * accounting. However, the blocked utilization may be zero. */ wake_util = cpu_util_without(i, p); - new_util = wake_util + uclamp_task_util(p); + new_util = wake_util + min_util; spare_wake_cap = capacity_orig - wake_util; if (spare_wake_cap > most_spare_wake_cap) { From 4bc53e5700b3a11cfb5fff5c5e28797d2db510c9 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 20 Apr 2021 14:09:01 -0700 Subject: [PATCH 075/346] sched: walt: Move spare_cap calculation spare_cap is being calculated prior to idle_cpu evaluation. If an idle cpu is found, then we have no need to go through this operation. As such, move the computation to after idle cpu evaluation. Change-Id: I86ff39909f2f0cb096ac39d02b6171838c257357 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index f667f0fcf8ac..20cd6e800d69 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -263,13 +263,6 @@ static void walt_find_best_target(struct sched_domain *sd, if (new_util > capacity_orig) continue; - /* - * Pre-compute the maximum possible capacity we expect - * to have available on this CPU once the task is - * enqueued here. - */ - spare_cap = capacity_orig - new_util; - /* * Find an optimal backup IDLE CPU for non latency * sensitive tasks. @@ -313,6 +306,13 @@ static void walt_find_best_target(struct sched_domain *sd, continue; } + /* + * Compute the maximum possible capacity we expect + * to have available on this CPU once the task is + * enqueued here. + */ + spare_cap = capacity_orig - new_util; + /* * Try to spread the rtg high prio tasks so that they * don't preempt each other. This is a optimisitc From c79c75585329adf3c4a386bfcf7112bee5ec81cb Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 20 Apr 2021 14:25:17 -0700 Subject: [PATCH 076/346] sched: walt: Shift boost status of current task We check if the current task running on the CPU has the highest per task boost to avoid placing task on said CPU. However, if the CPU is idle, then this check becomes unnecessary. Change-Id: I257df862ee2b344c0c602f36f254b38f61b78e55 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 20cd6e800d69..5763afd50514 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -250,10 +250,6 @@ static void walt_find_best_target(struct sched_domain *sd, most_spare_cap_cpu = i; } - if (per_task_boost(cpu_rq(i)->curr) == - TASK_BOOST_STRICT_MAX) - continue; - /* * Ensure minimum capacity to grant the required boost. * The target CPU can be already at a capacity level higher @@ -306,6 +302,10 @@ static void walt_find_best_target(struct sched_domain *sd, continue; } + if (per_task_boost(cpu_rq(i)->curr) == + TASK_BOOST_STRICT_MAX) + continue; + /* * Compute the maximum possible capacity we expect * to have available on this CPU once the task is From 1226d14b061d20b922b4334c1e19e7929657cd29 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 20 Apr 2021 15:41:35 -0700 Subject: [PATCH 077/346] sched: walt: Improve FEEC energy variable nomenclature We are using the terms "prev_delta" and "best_delta" to represent energy consumption values. Change to "prev_energy" and "best_energy" to reflect the same. Change-Id: Ie63e3029c3be6ee5e8cc12867d690b0fc708416f Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 5763afd50514..359a9e16217b 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -501,7 +501,7 @@ static DEFINE_PER_CPU(cpumask_t, energy_cpus); int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync, int sibling_count_hint) { - unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; + unsigned long prev_energy = ULONG_MAX, best_energy = ULONG_MAX; struct root_domain *rd = cpu_rq(cpumask_first(cpu_active_mask))->rd; int weight, cpu = smp_processor_id(), best_energy_cpu = prev_cpu; struct perf_domain *pd; @@ -589,10 +589,10 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, } if (cpumask_test_cpu(prev_cpu, &p->cpus_mask)) - prev_delta = best_delta = + prev_energy = best_energy = walt_compute_energy(p, prev_cpu, pd); else - prev_delta = best_delta = ULONG_MAX; + prev_energy = best_energy = ULONG_MAX; /* Select the best candidate energy-wise. */ for_each_cpu(cpu, candidates) { @@ -601,15 +601,15 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, cur_energy = walt_compute_energy(p, cpu, pd); trace_sched_compute_energy(p, cpu, cur_energy, - prev_delta, best_delta, best_energy_cpu); + prev_energy, best_energy, best_energy_cpu); - if (cur_energy < best_delta) { - best_delta = cur_energy; + if (cur_energy < best_energy) { + best_energy = cur_energy; best_energy_cpu = cpu; - } else if (cur_energy == best_delta) { + } else if (cur_energy == best_energy) { if (select_cpu_same_energy(cpu, best_energy_cpu, prev_cpu)) { - best_delta = cur_energy; + best_energy = cur_energy; best_energy_cpu = cpu; } } @@ -624,8 +624,8 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, */ if (!(available_idle_cpu(best_energy_cpu) && walt_get_idle_exit_latency(cpu_rq(best_energy_cpu)) <= 1) && - (prev_delta != ULONG_MAX) && (best_energy_cpu != prev_cpu) && - ((prev_delta - best_delta) <= prev_delta >> 4) && + (prev_energy != ULONG_MAX) && (best_energy_cpu != prev_cpu) && + ((prev_energy - best_energy) <= prev_energy >> 4) && (capacity_orig_of(prev_cpu) <= capacity_orig_of(start_cpu))) best_energy_cpu = prev_cpu; From 04d77db8ef219ae9f53eefe2689104fca1b39377 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 20 Apr 2021 17:43:00 -0700 Subject: [PATCH 078/346] sched: walt: Cleanup fbt_env boosted member We are no longer using the boosted member in the fbt_env struct. Clean it up. Change-Id: If8d0a68b8fb090a889c59667eb0fd8534c62af65 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 359a9e16217b..68bc103e74e2 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -43,7 +43,6 @@ static inline bool task_demand_fits(struct task_struct *p, int cpu) struct find_best_target_env { bool is_rtg; int need_idle; - bool boosted; int fastpath; int start_cpu; int order_index; @@ -113,7 +112,7 @@ static inline bool walt_target_ok(int target_cpu, int order_index) } static void walt_get_indicies(struct task_struct *p, int *order_index, - int *end_index, int task_boost, bool boosted) + int *end_index, int task_boost, bool uclamp_boost) { int i = 0; @@ -136,7 +135,8 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, return; } - if (boosted || task_boost_policy(p) == SCHED_BOOST_ON_BIG || + if (uclamp_boost || task_boost || + task_boost_policy(p) == SCHED_BOOST_ON_BIG || walt_task_skip_min_cpu(p)) *order_index = 1; @@ -513,8 +513,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, u64 start_t = 0; int delta = 0; int task_boost = per_task_boost(p); - bool is_uclamp_boosted = uclamp_boosted(p); - bool boosted = is_uclamp_boosted || (task_boost > 0); + bool uclamp_boost = uclamp_boosted(p); int start_cpu, order_index, end_index; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && @@ -524,7 +523,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (unlikely(!cpu_array)) return -EPERM; - walt_get_indicies(p, &order_index, &end_index, task_boost, boosted); + walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost); start_cpu = cpumask_first(&cpu_array[order_index][0]); is_rtg = task_in_related_thread_group(p); @@ -558,7 +557,6 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, fbt_env.start_cpu = start_cpu; fbt_env.order_index = order_index; fbt_env.end_index = end_index; - fbt_env.boosted = boosted; fbt_env.strict_max = is_rtg && (task_boost == TASK_BOOST_STRICT_MAX); fbt_env.skip_cpu = walt_is_many_wakeup(sibling_count_hint) ? @@ -582,8 +580,9 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, delta = task_util(p); if (task_placement_boost_enabled(p) || fbt_env.need_idle || - boosted || is_rtg || __cpu_overutilized(prev_cpu, delta) || - !task_fits_max(p, prev_cpu) || !cpu_active(prev_cpu)) { + uclamp_boost || task_boost || is_rtg || + __cpu_overutilized(prev_cpu, delta) || !task_fits_max(p, prev_cpu) || + !cpu_active(prev_cpu)) { best_energy_cpu = cpu; goto unlock; } @@ -632,7 +631,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, done: trace_sched_task_util(p, cpumask_bits(candidates)[0], best_energy_cpu, sync, fbt_env.need_idle, fbt_env.fastpath, - start_t, boosted, start_cpu); + start_t, uclamp_boost || task_boost, start_cpu); return best_energy_cpu; From 216e06f3ddd8a94a49f431014816ebe1b4ab2e44 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Fri, 9 Apr 2021 12:29:43 +0530 Subject: [PATCH 079/346] sched/walt: Initialize task groups from vendor hook The current code does the task group initialization from a sysctl handler invoked via post-boot. But some tasks could have been added to the cgroups and walt would miss to act upon them. Depending on user space to notify when to populate task group pointers is racy. This patch moves away from this approach and use the cpu controller cgroup online vendor hook to initialize the task group pointers. During module load, iterate over all the existing cgroups if any and do the initialization. Change-Id: I133099a49f94de836b95403c5f1126587de84919 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/boost.c | 3 --- kernel/sched/walt/sysctl.c | 55 -------------------------------------- kernel/sched/walt/walt.c | 34 +++++++++++++++++++++++ kernel/sched/walt/walt.h | 2 -- 4 files changed, 34 insertions(+), 60 deletions(-) diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c index 5ac7dce8160c..ded1ce3d0ac5 100644 --- a/kernel/sched/walt/boost.c +++ b/kernel/sched/walt/boost.c @@ -18,9 +18,6 @@ static enum sched_boost_policy boost_policy_dt = SCHED_BOOST_NONE; static DEFINE_MUTEX(boost_mutex); -struct task_group *task_group_topapp; -struct task_group *task_group_foreground; - void walt_init_tg(struct task_group *tg) { struct walt_task_group *wtg; diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 4816d702c790..e7738f397f09 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -39,7 +39,6 @@ unsigned int sysctl_sched_wake_up_idle[2]; unsigned int sysctl_input_boost_ms; unsigned int sysctl_input_boost_freq[8]; unsigned int sysctl_sched_boost_on_input; -unsigned int sysctl_sched_init_stage; /* sysctl nodes accesed by other files */ unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; @@ -61,53 +60,6 @@ unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; -static void init_tg_pointers(void) -{ - struct cgroup_subsys_state *css = &root_task_group.css; - struct cgroup_subsys_state *top_css = css; - - /* ptrs are already initialized */ - if (task_group_topapp) - return; - - css_for_each_child(css, top_css) { - if (!strcmp(css->cgroup->kn->name, "top-app")) { - task_group_topapp = css_tg(css); - walt_init_topapp_tg(task_group_topapp); - } else if (!strcmp(css->cgroup->kn->name, "foreground")) { - task_group_foreground = css_tg(css); - walt_init_foreground_tg(task_group_foreground); - } else { - walt_init_tg(css_tg(css)); - } - } -} - -static int walt_init_stage_handler(struct ctl_table *table, - int write, void __user *buffer, size_t *lenp, - loff_t *ppos) -{ - int ret; - static DEFINE_MUTEX(mutex); - int old_value = sysctl_sched_init_stage; - - mutex_lock(&mutex); - - ret = proc_dointvec(table, write, buffer, lenp, ppos); - - if (ret || !write) - goto unlock; - - if (sysctl_sched_init_stage == 1 && - old_value != sysctl_sched_init_stage) { - init_tg_pointers(); - } - -unlock: - mutex_unlock(&mutex); - return ret; -} - static int walt_proc_group_thresholds_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) @@ -493,13 +445,6 @@ struct ctl_table input_boost_sysctls[] = { }; struct ctl_table walt_table[] = { - { - .procname = "sched_init_stage", - .data = &sysctl_sched_init_stage, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = walt_init_stage_handler, - }, { .procname = "sched_user_hint", .data = &sysctl_sched_user_hint, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index ab3f4a6d0b62..602eaead531c 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2984,6 +2984,24 @@ static int sync_cgroup_colocation(struct task_struct *p, bool insert) return __sched_set_group_id(p, grp_id); } +static void walt_update_tg_pointer(struct cgroup_subsys_state *css) +{ + if (!strcmp(css->cgroup->kn->name, "top-app")) + walt_init_topapp_tg(css_tg(css)); + else if (!strcmp(css->cgroup->kn->name, "foreground")) + walt_init_foreground_tg(css_tg(css)); + else + walt_init_tg(css_tg(css)); +} + +static void android_rvh_cpu_cgroup_online(void *unused, struct cgroup_subsys_state *css) +{ + if (unlikely(walt_disabled)) + return; + + walt_update_tg_pointer(css); +} + static void android_rvh_cpu_cgroup_attach(void *unused, struct cgroup_taskset *tset) { @@ -2993,6 +3011,9 @@ static void android_rvh_cpu_cgroup_attach(void *unused, struct task_group *tg; struct walt_task_group *wtg; + if (unlikely(walt_disabled)) + return; + cgroup_taskset_first(tset, &css); if (!css) return; @@ -3981,6 +4002,7 @@ static void register_walt_hooks(void) register_trace_android_rvh_schedule(android_rvh_schedule, NULL); register_trace_android_rvh_resume_cpus(android_rvh_resume_cpus, NULL); register_trace_android_rvh_cpu_cgroup_attach(android_rvh_cpu_cgroup_attach, NULL); + register_trace_android_rvh_cpu_cgroup_online(android_rvh_cpu_cgroup_online, NULL); register_trace_android_rvh_update_cpus_allowed(android_rvh_update_cpus_allowed, NULL); register_trace_android_rvh_sched_fork_init(android_rvh_sched_fork_init, NULL); register_trace_android_rvh_ttwu_cond(android_rvh_ttwu_cond, NULL); @@ -4039,6 +4061,17 @@ static int walt_init_stop_handler(void *data) return 0; } +static void walt_init_tg_pointers(void) +{ + struct cgroup_subsys_state *css = &root_task_group.css; + struct cgroup_subsys_state *top_css = css; + + rcu_read_lock(); + css_for_each_child(css, top_css) + walt_update_tg_pointer(css); + rcu_read_unlock(); +} + static void walt_init(void) { struct ctl_table_header *hdr; @@ -4054,6 +4087,7 @@ static void walt_init(void) BUG_ON(alloc_related_thread_groups()); walt_init_cycle_counter(); init_clusters(); + walt_init_tg_pointers(); register_walt_hooks(); walt_fixup_init(); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 1d87cdf8ef19..e94e50b4f507 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -282,8 +282,6 @@ extern void walt_update_group_thresholds(void); extern void sched_window_nr_ticks_change(void); extern unsigned long sched_user_hint_reset_time; extern struct irq_work walt_migration_irq_work; -extern struct task_group *task_group_topapp; -extern struct task_group *task_group_foreground; #define LIB_PATH_LENGTH 512 extern unsigned int cpuinfo_max_freq_cached; From b07ef98a47b54413a0fe43012d80f0946c7716ed Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 27 Apr 2021 17:50:56 -0700 Subject: [PATCH 080/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ife8de1eee2b7aa862979073c075466c97ba980e3 Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Stephen Dickey --- kernel/sched/walt/sched_avg.c | 4 ++-- kernel/sched/walt/walt.c | 6 +++--- kernel/sched/walt/walt.h | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index ededab2476e6..6bac78df55e7 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -189,7 +189,7 @@ int sched_busy_hyst_handler(struct ctl_table *table, int write, * * Update average with latest nr_running value for CPU */ -void sched_update_nr_prod(int cpu, bool enq) +void sched_update_nr_prod(int cpu, int enq) { u64 diff; u64 curr_time; @@ -201,7 +201,7 @@ void sched_update_nr_prod(int cpu, bool enq) diff = curr_time - per_cpu(last_time, cpu); BUG_ON((s64)diff < 0); per_cpu(last_time, cpu) = curr_time; - per_cpu(nr, cpu) = cpu_rq(cpu)->nr_running; + per_cpu(nr, cpu) = cpu_rq(cpu)->nr_running + enq; if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu)) per_cpu(nr_max, cpu) = per_cpu(nr, cpu); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 602eaead531c..2a7f6774358b 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3764,7 +3764,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (unlikely(walt_disabled)) return; wts->last_enqueued_ts = wallclock; - sched_update_nr_prod(rq->cpu, true); + sched_update_nr_prod(rq->cpu, 1); if (walt_fair_task(p)) { wts->misfit = !task_fits_max(p, rq->cpu); @@ -3784,7 +3784,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st if (p == wrq->ed_task) is_ed_task_present(rq, sched_ktime_clock(), p); - sched_update_nr_prod(rq->cpu, false); + sched_update_nr_prod(rq->cpu, -1); if (walt_fair_task(p)) dec_rq_walt_stats(rq, p); @@ -3823,7 +3823,7 @@ static void android_rvh_update_misfit_status(void *unused, struct task_struct *p change = misfit - old_misfit; if (change) { - sched_update_nr_prod(rq->cpu, true); + sched_update_nr_prod(rq->cpu, 0); wts->misfit = misfit; wrq->walt_stats.nr_big_tasks += change; BUG_ON(wrq->walt_stats.nr_big_tasks < 0); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index e94e50b4f507..4fe045436825 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -147,7 +147,7 @@ extern unsigned int sched_capacity_margin_down[WALT_NR_CPUS]; extern cpumask_t asym_cap_sibling_cpus; extern cpumask_t __read_mostly **cpu_array; -extern void sched_update_nr_prod(int cpu, bool enq); +extern void sched_update_nr_prod(int cpu, int enq); extern unsigned int walt_big_tasks(int cpu); extern void walt_rotate_work_init(void); extern void walt_rotation_checkpoint(int nr_big); From 2874629bd7871709b5ce7108760e396c81339dd5 Mon Sep 17 00:00:00 2001 From: jianzhou Date: Tue, 27 Apr 2021 13:54:39 +0800 Subject: [PATCH 081/346] sched/walt: Apply binder vendor hook performance tune Apply binder vendor hook performance tune for blocked binder transaction. Change-Id: Ia4a344fd260df12f4d5b364f7f4027c50c6f1a99 Signed-off-by: jianzhou --- kernel/sched/walt/walt_cfs.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 68bc103e74e2..808c1c8a5822 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -8,6 +8,7 @@ #include "walt.h" #include "trace.h" +#include <../../../drivers/android/binder_internal.h> #include "../../../drivers/android/binder_trace.h" /* Migration margins */ @@ -711,6 +712,34 @@ static void walt_binder_low_latency_clear(void *unused, struct binder_transactio wts->low_latency &= ~WALT_LOW_LATENCY_BINDER; } +static void binder_set_priority_hook(void *data, + struct binder_transaction *bndrtrans, struct task_struct *task) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; + struct walt_task_struct *current_wts = + (struct walt_task_struct *) current->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + if (bndrtrans && bndrtrans->need_reply && current_wts->boost == TASK_BOOST_STRICT_MAX) { + bndrtrans->android_vendor_data1 = wts->boost; + wts->boost = TASK_BOOST_STRICT_MAX; + } +} + +static void binder_restore_priority_hook(void *data, + struct binder_transaction *bndrtrans, struct task_struct *task) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + if (wts->boost == TASK_BOOST_STRICT_MAX) + wts->boost = bndrtrans->android_vendor_data1; +} + void walt_cfs_init(void) { register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL); @@ -718,4 +747,7 @@ void walt_cfs_init(void) register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL); register_trace_binder_transaction_received(walt_binder_low_latency_clear, NULL); + + register_trace_android_vh_binder_set_priority(binder_set_priority_hook, NULL); + register_trace_android_vh_binder_restore_priority(binder_restore_priority_hook, NULL); } From c7d56f981962344001b928bee53ce46991baf041 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 28 Apr 2021 15:56:28 -0700 Subject: [PATCH 082/346] sched/walt: improved debug for tracking accounting cases Introduce information into the debugging of fixup_cumulative_runnable_avg by ensuring that the data from the task of interest is dumped, and that the runqueue in question is locked. Change-Id: Idb6e3bd1b4ba67512fa6663555c22e933c9d0a6e Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2a7f6774358b..4d3f7077bfb3 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -285,15 +285,32 @@ static int in_sched_bug; }) static inline void -fixup_cumulative_runnable_avg(struct walt_sched_stats *stats, +fixup_cumulative_runnable_avg(struct task_struct *p, + struct walt_sched_stats *stats, s64 demand_scaled_delta, s64 pred_demand_scaled_delta) { - stats->cumulative_runnable_avg_scaled += demand_scaled_delta; - BUG_ON((s64)stats->cumulative_runnable_avg_scaled < 0); + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + s64 cumulative_runnable_avg_scaled = + stats->cumulative_runnable_avg_scaled + demand_scaled_delta; + s64 pred_demands_sum_scaled = + stats->pred_demands_sum_scaled + pred_demand_scaled_delta; - stats->pred_demands_sum_scaled += pred_demand_scaled_delta; - BUG_ON((s64)stats->pred_demands_sum_scaled < 0); + if (cumulative_runnable_avg_scaled < 0) { + printk_deferred("WALT-BUG task ds=%llu is higher than cra=%llu\n", + wts->demand_scaled, stats->cumulative_runnable_avg_scaled); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + stats->cumulative_runnable_avg_scaled = (u64)cumulative_runnable_avg_scaled; + + if (pred_demands_sum_scaled < 0) { + printk_deferred("WALT-BUG task pds=%llu is higher than pds_sum=%llu\n", + wts->pred_demand_scaled, stats->pred_demands_sum_scaled); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + stats->pred_demands_sum_scaled = (u64)pred_demands_sum_scaled; } static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, @@ -307,7 +324,7 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, wts->pred_demand_scaled; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - fixup_cumulative_runnable_avg(&wrq->walt_stats, task_load_delta, + fixup_cumulative_runnable_avg(p, &wrq->walt_stats, task_load_delta, pred_demand_delta); } @@ -3601,7 +3618,7 @@ walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - fixup_cumulative_runnable_avg(&wrq->walt_stats, wts->demand_scaled, + fixup_cumulative_runnable_avg(p, &wrq->walt_stats, wts->demand_scaled, wts->pred_demand_scaled); } @@ -3611,7 +3628,7 @@ walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - fixup_cumulative_runnable_avg(&wrq->walt_stats, + fixup_cumulative_runnable_avg(p, &wrq->walt_stats, -(s64)wts->demand_scaled, -(s64)wts->pred_demand_scaled); } From 717473462bc2d003b6e8b8da730b793586c645f1 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Fri, 23 Apr 2021 16:22:04 -0700 Subject: [PATCH 083/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I2435c4ecd4c6a978af188a6385d306691376bb0a Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4d3f7077bfb3..461198142b52 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -189,8 +189,11 @@ static inline void walt_task_dump(struct task_struct *p) int i, j = 0; int buffsz = WALT_NR_CPUS * 16; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + bool is_32bit_thread = is_compat_thread(task_thread_info(p)); - SCHED_PRINT(p->pid); + printk_deferred("Task: %.16s-%d\n", p->comm, p->pid); + SCHED_PRINT(p->policy); + SCHED_PRINT(p->prio); SCHED_PRINT(wts->mark_start); SCHED_PRINT(wts->demand); SCHED_PRINT(wts->coloc_demand); @@ -213,6 +216,10 @@ static inline void walt_task_dump(struct task_struct *p) SCHED_PRINT(wts->last_enqueued_ts); SCHED_PRINT(wts->misfit); SCHED_PRINT(wts->unfilter); + SCHED_PRINT(is_32bit_thread); + SCHED_PRINT(wts->grp); + SCHED_PRINT(p->on_cpu); + SCHED_PRINT(p->on_rq); } static inline void walt_rq_dump(int cpu) From 594217b0087b14b6cbad233c88cc04c2faad8d52 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 29 Apr 2021 18:15:37 -0700 Subject: [PATCH 084/346] sched/walt: debug for fixup_cumulative_runnable_avg Catch when the rq is not the same as this task's rq. Catch a double enqueue or dequeue of the same task. Change-Id: I322588f4c8ad59ee48c6d14bd87c5db6d842e6f6 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 461198142b52..356060cdd3c6 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -292,7 +292,8 @@ static int in_sched_bug; }) static inline void -fixup_cumulative_runnable_avg(struct task_struct *p, +fixup_cumulative_runnable_avg(struct rq *rq, + struct task_struct *p, struct walt_sched_stats *stats, s64 demand_scaled_delta, s64 pred_demand_scaled_delta) @@ -303,6 +304,14 @@ fixup_cumulative_runnable_avg(struct task_struct *p, s64 pred_demands_sum_scaled = stats->pred_demands_sum_scaled + pred_demand_scaled_delta; + lockdep_assert_held(&rq->lock); + + if (task_rq(p) != rq) { + printk_deferred("WALT-BUG task not on rq\n"); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + if (cumulative_runnable_avg_scaled < 0) { printk_deferred("WALT-BUG task ds=%llu is higher than cra=%llu\n", wts->demand_scaled, stats->cumulative_runnable_avg_scaled); @@ -331,7 +340,7 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, wts->pred_demand_scaled; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - fixup_cumulative_runnable_avg(p, &wrq->walt_stats, task_load_delta, + fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, task_load_delta, pred_demand_delta); } @@ -975,6 +984,9 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) wallclock = sched_ktime_clock(); + lockdep_assert_held(src_rq->lock); + lockdep_assert_held(dest_rq->lock); + walt_update_task_ravg(task_rq(p)->curr, task_rq(p), TASK_UPDATE, wallclock, 0); @@ -2205,6 +2217,7 @@ static void init_new_task_load(struct task_struct *p) wts->curr_window = 0; wts->prev_window = 0; wts->active_time = 0; + for (i = 0; i < NUM_BUSY_BUCKETS; ++i) wts->busy_buckets[i] = 0; @@ -3625,7 +3638,7 @@ walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - fixup_cumulative_runnable_avg(p, &wrq->walt_stats, wts->demand_scaled, + fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, wts->demand_scaled, wts->pred_demand_scaled); } @@ -3635,7 +3648,7 @@ walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - fixup_cumulative_runnable_avg(p, &wrq->walt_stats, + fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, -(s64)wts->demand_scaled, -(s64)wts->pred_demand_scaled); } From 2e720fa7e7423d920d74aaa59a2f904f079bf02e Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 29 Apr 2021 18:15:37 -0700 Subject: [PATCH 085/346] sched/walt: enhanced debug for double enqueue/dequeue attempt to catch double enqueue/dequeue of a task, which can impact walt accounting. Change-Id: I2ff83e75d65f083c8d2896dd10313c193a16fa7b Signed-off-by: Stephen Dickey --- include/linux/sched/walt.h | 4 ++++ kernel/sched/walt/walt.c | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 4c3a6ccdb834..38df32abd9cc 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -79,6 +79,9 @@ struct walt_task_struct { * used for prediction * * 'demand_scaled' represents task's demand scaled to 1024 + * + * 'prev_on_rq' tracks enqueue/dequeue of a task for error conditions + * 0 = nothing, 1 = enqueued, 2 = dequeued */ u64 mark_start; u32 sum, demand; @@ -110,6 +113,7 @@ struct walt_task_struct { u64 cpu_cycles; cpumask_t cpus_requested; bool iowaited; + int prev_on_rq; }; #define wts_to_ts(wts) ({ \ diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 356060cdd3c6..ab33f5caed14 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2217,6 +2217,7 @@ static void init_new_task_load(struct task_struct *p) wts->curr_window = 0; wts->prev_window = 0; wts->active_time = 0; + wts->prev_on_rq = 0; for (i = 0; i < NUM_BUSY_BUCKETS; ++i) wts->busy_buckets[i] = 0; @@ -3800,6 +3801,15 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (unlikely(walt_disabled)) return; + + /* catch double enqueue */ + if (wts->prev_on_rq == 1) { + printk_deferred("WALT-BUG double enqueue detected\n"); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + wts->prev_on_rq = 1; + wts->last_enqueued_ts = wallclock; sched_update_nr_prod(rq->cpu, 1); @@ -3815,9 +3825,19 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (unlikely(walt_disabled)) return; + + /* catch double deq */ + if (wts->prev_on_rq == 2) { + printk_deferred("WALT-BUG double dequeue detected\n"); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + + wts->prev_on_rq = 2; if (p == wrq->ed_task) is_ed_task_present(rq, sched_ktime_clock(), p); From 4dfdcf521a203d424226099bc19e53fdb2b24db9 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Tue, 20 Apr 2021 11:50:22 +0530 Subject: [PATCH 086/346] sched/walt: Cache the affinity of a user space task When a user space task move across cpuset cgroups, the affinity settings are lost. Fix this issue by caching the affinity of a user space task during affinity change system call and restore it during cpuset cgroup change when possible. The cpuset part is implemented in the code but affinity system call part is not implemented. This patch implements that. Change-Id: I9322400eb275aeeb35fb623210417df4730cc6ad Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index ab33f5caed14..16f97bc17b9f 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4003,6 +4003,27 @@ static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, *ret = set_cpus_allowed_ptr(p, &wts->cpus_requested); } +static void android_rvh_sched_setaffinity(void *unused, struct task_struct *p, + const struct cpumask *in_mask, + int *retval) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + /* nothing to do if the affinity call failed */ + if (*retval) + return; + + /* + * cache the affinity for user space tasks so that they + * can be restored during cpuset cgroup change. + */ + if (!(p->flags & PF_KTHREAD)) + cpumask_and(&wts->cpus_requested, in_mask, cpu_possible_mask); +} + static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -4061,6 +4082,7 @@ static void register_walt_hooks(void) register_trace_android_rvh_cpu_cgroup_attach(android_rvh_cpu_cgroup_attach, NULL); register_trace_android_rvh_cpu_cgroup_online(android_rvh_cpu_cgroup_online, NULL); register_trace_android_rvh_update_cpus_allowed(android_rvh_update_cpus_allowed, NULL); + register_trace_android_rvh_sched_setaffinity(android_rvh_sched_setaffinity, NULL); register_trace_android_rvh_sched_fork_init(android_rvh_sched_fork_init, NULL); register_trace_android_rvh_ttwu_cond(android_rvh_ttwu_cond, NULL); register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); From a822c7ebe19d055210665fa4612474b56082477d Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 22 Apr 2021 22:59:35 -0700 Subject: [PATCH 087/346] sched/walt: Add small task filtering to uclamp Currently, small tasks get placed on mid-cap CPUs if uclamp is active on them. This causes power numbers to increase. Introduce filtering such that small tasks stay on silver. Change-Id: I7e31d8a5ffd4c4f64d7bd3668970216550d23d24 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.h | 9 ++++++++- kernel/sched/walt/walt_cfs.c | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 4fe045436825..1c5cd2288fff 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -478,6 +478,13 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) return policy; } +static inline bool walt_uclamp_boosted(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return uclamp_eff_value(p, UCLAMP_MIN) > 0 && wts->unfilter; +} + static inline unsigned long capacity_of(int cpu) { return cpu_rq(cpu)->cpu_capacity; @@ -702,7 +709,7 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) if (is_min_capacity_cpu(cpu)) { if (task_boost_policy(p) == SCHED_BOOST_ON_BIG || task_boost > 0 || - uclamp_boosted(p) || + walt_uclamp_boosted(p) || walt_should_kick_upmigrate(p, cpu)) return false; } else { /* mid cap cpu */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 808c1c8a5822..0677b7a90b3c 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -514,7 +514,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, u64 start_t = 0; int delta = 0; int task_boost = per_task_boost(p); - bool uclamp_boost = uclamp_boosted(p); + bool uclamp_boost = walt_uclamp_boosted(p); int start_cpu, order_index, end_index; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && From 619670b06a59c6bfc5c59bdba7e85f0fca298e5e Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 6 May 2021 14:38:44 -0700 Subject: [PATCH 088/346] sched: Improve the scheduler There is a redundant check taking place in set_task_cpu hook. The cpu will never be negative from core kernel side, so remove it. Change-Id: I6d8ab62573f401d6aaeda9c5ddb1183c69e8c7d6 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 16f97bc17b9f..a60916730203 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3753,8 +3753,6 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign { if (unlikely(walt_disabled)) return; - if (new_cpu < 0) - return; fixup_busy_time(p, (int) new_cpu); } From 0fa8c0d22912a6f56c391f853839436b11a3655b Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 7 May 2021 13:21:28 -0700 Subject: [PATCH 089/346] sched/walt: cleanup references to pr_emerg In scheduler code printk_deferred should be used instead of pr_emerg, as this will avoid taking any locks and defer output to a safer time. Change-Id: I06b9060754d97f4ca141c561eb80216530e0e25e Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index a60916730203..5ffebd42d99d 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -180,7 +180,7 @@ static const unsigned int top_tasks_bitmap_size = static __read_mostly unsigned int walt_scale_demand_divisor; #define scale_demand(d) ((d)/walt_scale_demand_divisor) -#define SCHED_PRINT(arg) pr_emerg("%s=%llu", #arg, arg) +#define SCHED_PRINT(arg) printk_deferred("%s=%llu", #arg, arg) #define STRG(arg) #arg static inline void walt_task_dump(struct task_struct *p) @@ -238,7 +238,7 @@ static inline void walt_rq_dump(int cpu) * rq locks held. */ get_task_struct(tsk); - pr_emerg("CPU:%d nr_running:%u current: %d (%s)\n", + printk_deferred("CPU:%d nr_running:%u current: %d (%s)\n", cpu, rq->nr_running, tsk->pid, tsk->comm); printk_deferred("=========================================="); @@ -270,15 +270,15 @@ static inline void walt_dump(void) { int cpu; - pr_emerg("============ WALT RQ DUMP START ==============\n"); - pr_emerg("Sched ktime_get: %llu\n", sched_ktime_clock()); - pr_emerg("Time last window changed=%lu\n", + printk_deferred("============ WALT RQ DUMP START ==============\n"); + printk_deferred("Sched ktime_get: %llu\n", sched_ktime_clock()); + printk_deferred("Time last window changed=%lu\n", sched_ravg_window_change_time); for_each_online_cpu(cpu) walt_rq_dump(cpu); SCHED_PRINT(max_possible_capacity); SCHED_PRINT(min_max_possible_capacity); - pr_emerg("============ WALT RQ DUMP END ==============\n"); + printk_deferred("============ WALT RQ DUMP END ==============\n"); } static int in_sched_bug; @@ -399,7 +399,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) delta = wallclock - wrq->window_start; if (delta < 0) { - pr_emerg("WALT-BUG CPU%d; wallclock=%llu is lesser than window_start=%llu", + printk_deferred("WALT-BUG CPU%d; wallclock=%llu is lesser than window_start=%llu", rq->cpu, wallclock, wrq->window_start); SCHED_BUG_ON(1); } From 550838ff0a3ba1cffe3447b03b77c42c209954b4 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:48:58 -0700 Subject: [PATCH 090/346] sched/walt: Add MVP tasks feature MVP tasks feature is intended for improving general scheduler performance. Change-Id: I7262daf52c3ca4c627642cc5e2aa33a7b33a2305 Signed-off-by: Pavankumar Kondeti Signed-off-by: Sai Harshini Nimmala --- include/linux/sched/walt.h | 4 + kernel/sched/walt/trace.h | 66 ++++++++- kernel/sched/walt/walt.c | 16 ++- kernel/sched/walt/walt.h | 32 +++++ kernel/sched/walt/walt_cfs.c | 264 ++++++++++++++++++++++++++++++++++- kernel/sched/walt/walt_lb.c | 8 +- 6 files changed, 378 insertions(+), 12 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 38df32abd9cc..918466220ba0 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -114,6 +114,10 @@ struct walt_task_struct { cpumask_t cpus_requested; bool iowaited; int prev_on_rq; + struct list_head mvp_list; + u64 sum_exec_snapshot; + u64 total_exec; + int mvp_prio; }; #define wts_to_ts(wts) ({ \ diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 4effae0bbaad..b6f8a8738d3b 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1023,10 +1023,9 @@ TRACE_EVENT(sched_find_best_target, TRACE_EVENT(sched_enq_deq_task, - TP_PROTO(struct task_struct *p, bool enqueue, - unsigned int cpus_allowed), + TP_PROTO(struct task_struct *p, bool enqueue, unsigned int cpus_allowed, bool mvp), - TP_ARGS(p, enqueue, cpus_allowed), + TP_ARGS(p, enqueue, cpus_allowed, mvp), TP_STRUCT__entry( __array(char, comm, TASK_COMM_LEN) @@ -1039,7 +1038,8 @@ TRACE_EVENT(sched_enq_deq_task, __field(unsigned int, cpus_allowed) __field(unsigned int, demand) __field(unsigned int, pred_demand) - __field(bool, compat_thread) + __field(bool, compat_thread) + __field(bool, mvp) ), TP_fast_assign( @@ -1054,9 +1054,10 @@ TRACE_EVENT(sched_enq_deq_task, __entry->demand = task_load(p); __entry->pred_demand = task_pl(p); __entry->compat_thread = is_compat_thread(task_thread_info(p)); + __entry->mvp = mvp; ), - TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d", + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d mvp=%d", __entry->cpu, __entry->enqueue ? "enqueue" : "dequeue", __entry->comm, __entry->pid, @@ -1064,7 +1065,7 @@ TRACE_EVENT(sched_enq_deq_task, __entry->rt_nr_running, __entry->cpus_allowed, __entry->demand, __entry->pred_demand, - __entry->compat_thread) + __entry->compat_thread, __entry->mvp) ); TRACE_EVENT(walt_window_rollover, @@ -1083,6 +1084,59 @@ TRACE_EVENT(walt_window_rollover, TP_printk("window_start=%llu", __entry->window_start) ); + +DECLARE_EVENT_CLASS(walt_cfs_mvp_task_template, + + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + + TP_ARGS(p, wts, limit), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, prio) + __field(int, mvp_prio) + __field(int, cpu) + __field(u64, exec) + __field(unsigned int, limit) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->mvp_prio = wts->mvp_prio; + __entry->cpu = task_cpu(p); + __entry->exec = wts->total_exec; + __entry->limit = limit; + ), + + TP_printk("comm=%s pid=%d prio=%d mvp_prio=%d cpu=%d exec=%llu limit=%u", + __entry->comm, __entry->pid, __entry->prio, + __entry->mvp_prio, __entry->cpu, __entry->exec, + __entry->limit) +); + +/* called upon MVP task de-activation. exec will be more than limit */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_deactivate_mvp_task, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP is returned to run next */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_pick_next, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP (current) is not preempted by waking task */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_nopreempt, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP (waking task) preempts the current */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_preempt, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + #endif /* _TRACE_WALT_H */ #undef TRACE_INCLUDE_PATH diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 5ffebd42d99d..02bd142e11c1 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2243,6 +2243,11 @@ static void init_new_task_load(struct task_struct *p) wts->misfit = false; wts->rtg_high_prio = false; wts->unfilter = sysctl_sched_task_unfilter_period; + + INIT_LIST_HEAD(&wts->mvp_list); + wts->sum_exec_snapshot = 0; + wts->total_exec = 0; + wts->mvp_prio = WALT_NOT_MVP; } static void init_existing_task_load(struct task_struct *p) @@ -3621,6 +3626,8 @@ static void walt_sched_init_rq(struct rq *rq) clear_top_tasks_bitmap(wrq->top_tasks_bitmap[j]); } wrq->notif_pending = false; + + INIT_LIST_HEAD(&wrq->mvp_tasks); } void sched_window_nr_ticks_change(void) @@ -3814,10 +3821,11 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (walt_fair_task(p)) { wts->misfit = !task_fits_max(p, rq->cpu); inc_rq_walt_stats(rq, p); + walt_cfs_enqueue_task(rq, p); } walt_inc_cumulative_runnable_avg(rq, p); - trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0]); + trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); } static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) @@ -3841,11 +3849,13 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st sched_update_nr_prod(rq->cpu, -1); - if (walt_fair_task(p)) + if (walt_fair_task(p)) { dec_rq_walt_stats(rq, p); + walt_cfs_dequeue_task(rq, p); + } walt_dec_cumulative_runnable_avg(rq, p); - trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0]); + trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); } static void android_rvh_update_misfit_status(void *unused, struct task_struct *p, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 1c5cd2288fff..ab3584f9e7a3 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -118,6 +118,7 @@ struct walt_rq { bool high_irqload; u64 last_cc_update; u64 cycles; + struct list_head mvp_tasks; }; struct walt_sched_cluster { @@ -729,6 +730,22 @@ static inline bool walt_low_latency_task(struct task_struct *p) (task_util(p) < sysctl_walt_low_latency_task_threshold); } +static inline bool walt_binder_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_BINDER) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_procfs_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_PROCFS) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) { struct cpuidle_state *idle = idle_get_state(rq); @@ -861,4 +878,19 @@ static inline bool walt_fair_task(struct task_struct *p) { return p->prio >= MAX_RT_PRIO && !is_idle_task(p); } + + +#define WALT_MVP_SLICE 3000000U +#define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE) + +#define WALT_RTG_MVP 0 +#define WALT_BINDER_MVP 1 + +#define WALT_NOT_MVP -1 + +#define is_mvp(wts) (wts->mvp_prio != WALT_NOT_MVP) +void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p); +void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p); +void walt_cfs_tick(struct rq *rq); + #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 0677b7a90b3c..7b424d3b1d43 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -689,7 +689,8 @@ static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vrunti } } -static void walt_binder_low_latency_set(void *unused, struct task_struct *task) +static void walt_binder_low_latency_set(void *unused, struct task_struct *task, + bool sync, struct binder_proc *proc) { struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; @@ -738,16 +739,275 @@ static void binder_restore_priority_hook(void *data, if (wts->boost == TASK_BOOST_STRICT_MAX) wts->boost = bndrtrans->android_vendor_data1; + +} +/* + * Higher prio mvp can preempt lower prio mvp. + * + * However, the lower prio MVP slice will be more since we expect them to + * be the work horses. For example, binders will have higher prio MVP and + * they can preempt long running rtg prio tasks but binders loose their + * powers with in 3 msec where as rtg prio tasks can run more than that. + */ +static inline int walt_get_mvp_task_prio(struct task_struct *p) +{ + if ((per_task_boost(p) == TASK_BOOST_STRICT_MAX) || + task_rtg_high_prio(p) || + walt_procfs_low_latency_task(p)) + return WALT_RTG_MVP; + + if (walt_binder_low_latency_task(p)) + return WALT_BINDER_MVP; + + return WALT_NOT_MVP; +} + +static inline unsigned int walt_cfs_mvp_task_limit(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + /* Binder MVP tasks are high prio but have only single slice */ + if (wts->mvp_prio == WALT_BINDER_MVP) + return WALT_MVP_SLICE; + + return WALT_MVP_LIMIT; +} + +static void walt_cfs_insert_mvp_task(struct walt_rq *wrq, struct walt_task_struct *wts, + bool at_front) +{ + struct list_head *pos; + + list_for_each(pos, &wrq->mvp_tasks) { + struct walt_task_struct *tmp_wts = container_of(pos, struct walt_task_struct, + mvp_list); + + if (at_front) { + if (wts->mvp_prio >= tmp_wts->mvp_prio) + break; + } else { + if (wts->mvp_prio > tmp_wts->mvp_prio) + break; + } + } + + list_add(&wts->mvp_list, pos->prev); +} + +static void walt_cfs_deactivate_mvp_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + list_del_init(&wts->mvp_list); + wts->mvp_prio = WALT_NOT_MVP; + + /* + * Reset the exec time during sleep so that it starts + * from scratch upon next wakeup. total_exec should + * be preserved when task is enq/deq while it is on + * runqueue. + */ + if (p->state != TASK_RUNNING) + wts->total_exec = 0; +} + +/* + * MVP task runtime update happens here. Three possibilities: + * + * de-activated: The MVP consumed its runtime. Non MVP can preempt. + * slice expired: MVP slice is expired and other MVP can preempt. + * slice not expired: This MVP task can continue to run. + */ +static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1; + s64 delta; + unsigned int limit; + + /* sum_exec_snapshot can be ahead. See below increment */ + delta = curr->se.sum_exec_runtime - wts->sum_exec_snapshot; + if (delta < 0) + delta = 0; + else + delta += rq_clock_task(rq) - curr->se.exec_start; + + /* slice is not expired */ + if (delta < WALT_MVP_SLICE) + return; + + /* + * slice is expired, check if we have to deactivate the + * MVP task, otherwise requeue the task in the list so + * that other MVP tasks gets a chance. + */ + wts->sum_exec_snapshot += delta; + wts->total_exec += delta; + + limit = walt_cfs_mvp_task_limit(curr); + if (wts->total_exec > limit) { + walt_cfs_deactivate_mvp_task(curr); + trace_walt_cfs_deactivate_mvp_task(curr, wts, limit); + return; + } + + /* slice expired. re-queue the task */ + list_del(&wts->mvp_list); + walt_cfs_insert_mvp_task(wrq, wts, false); +} + +void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + int mvp_prio = walt_get_mvp_task_prio(p); + + if (mvp_prio == WALT_NOT_MVP) + return; + + /* + * This can happen during migration or enq/deq for prio/class change. + * it was once MVP but got demoted, it will not be MVP until + * it goes to sleep again. + */ + if (wts->total_exec > walt_cfs_mvp_task_limit(p)) + return; + + wts->mvp_prio = mvp_prio; + walt_cfs_insert_mvp_task(wrq, wts, task_running(rq, p)); + + /* + * We inserted the task at the appropriate position. Take the + * task runtime snapshot. From now onwards we use this point as a + * baseline to enforce the slice and demotion. + */ + if (!wts->total_exec) /* queue after sleep */ + wts->sum_exec_snapshot = p->se.sum_exec_runtime; + +} + +void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!list_empty(&wts->mvp_list)) + walt_cfs_deactivate_mvp_task(p); +} + +void walt_cfs_tick(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + if (list_empty(&wts->mvp_list)) + return; + + walt_cfs_account_mvp_runtime(rq, rq->curr); + /* + * If the current is not MVP means, we have to re-schedule to + * see if we can run any other task including MVP tasks. + */ + if ((wrq->mvp_tasks.next != &wts->mvp_list) && rq->cfs.h_nr_running > 1) + resched_curr(rq); +} + +/* + * When preempt = false and nopreempt = false, we leave the preemption + * decision to CFS. + */ +static void walt_cfs_check_preempt_wakeup(void *unused, struct rq *rq, struct task_struct *p, + bool *preempt, bool *nopreempt, int wake_flags, + struct sched_entity *se, struct sched_entity *pse, + int next_buddy_marked, unsigned int granularity) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts_p = (struct walt_task_struct *) p->android_vendor_data1; + struct task_struct *c = rq->curr; + struct walt_task_struct *wts_c = (struct walt_task_struct *) rq->curr->android_vendor_data1; + bool resched = false; + bool p_is_mvp, curr_is_mvp; + + if (unlikely(walt_disabled)) + return; + + p_is_mvp = !list_empty(&wts_p->mvp_list); + curr_is_mvp = !list_empty(&wts_c->mvp_list); + + /* + * current is not MVP, so preemption decision + * is simple. + */ + if (!curr_is_mvp) { + if (p_is_mvp) + goto preempt; + return; /* CFS decides preemption */ + } + + /* + * current is MVP. update its runtime before deciding the + * preemption. + */ + walt_cfs_account_mvp_runtime(rq, c); + resched = (wrq->mvp_tasks.next != &wts_c->mvp_list); + + /* + * current is no longer eligible to run. It must have been + * picked (because of MVP) ahead of other tasks in the CFS + * tree, so drive preemption to pick up the next task from + * the tree, which also includes picking up the first in + * the MVP queue. + */ + if (resched) + goto preempt; + + /* current is the first in the queue, so no preemption */ + *nopreempt = true; + trace_walt_cfs_mvp_wakeup_nopreempt(c, wts_c, walt_cfs_mvp_task_limit(c)); + return; +preempt: + *preempt = true; + trace_walt_cfs_mvp_wakeup_preempt(p, wts_p, walt_cfs_mvp_task_limit(p)); +} + +static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct task_struct **p, + struct sched_entity **se, bool *repick, bool simple, + struct task_struct *prev) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts; + struct task_struct *mvp; + + if (unlikely(walt_disabled)) + return; + + /* We don't have MVP tasks queued */ + if (list_empty(&wrq->mvp_tasks)) + return; + + /* Return the first task from MVP queue */ + wts = list_first_entry(&wrq->mvp_tasks, struct walt_task_struct, mvp_list); + mvp = wts_to_ts(wts); + + *p = mvp; + *se = &mvp->se; + *repick = true; + + trace_walt_cfs_mvp_pick_next(mvp, wts, walt_cfs_mvp_task_limit(mvp)); } void walt_cfs_init(void) { register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL); - register_trace_android_rvh_place_entity(walt_place_entity, NULL); register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL); register_trace_binder_transaction_received(walt_binder_low_latency_clear, NULL); register_trace_android_vh_binder_set_priority(binder_set_priority_hook, NULL); register_trace_android_vh_binder_restore_priority(binder_restore_priority_hook, NULL); + + register_trace_android_rvh_check_preempt_wakeup(walt_cfs_check_preempt_wakeup, NULL); + register_trace_android_rvh_replace_next_task_fair(walt_cfs_replace_next_task_fair, NULL); } diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 960da0dca5d4..921530c441ce 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -494,7 +494,13 @@ static void walt_lb_tick(void *unused, struct rq *rq) if (unlikely(walt_disabled)) return; - if (!rq->misfit_task_load || !walt_fair_task(p)) + + if (!walt_fair_task(p)) + return; + + walt_cfs_tick(rq); + + if (!rq->misfit_task_load) return; if (p->state != TASK_RUNNING || p->nr_cpus_allowed == 1) From c565963618c0a3941778bb99e641ad483a7a7e24 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 12 May 2021 12:17:57 -0700 Subject: [PATCH 091/346] walt: honor affinity in placement path Walt placement code could return the prev_cpu for placement without checking if it is in affinity mask. Fix it by checking task's cpus_ptr. Change-Id: I6edf2249bccffd1868dd546dc18b45af311cc91d Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 7b424d3b1d43..990fe008a9d5 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -200,7 +200,8 @@ static void walt_find_best_target(struct sched_domain *sd, if (((capacity_orig_of(prev_cpu) == capacity_orig_of(start_cpu)) || asym_cap_siblings(prev_cpu, start_cpu)) && cpu_active(prev_cpu) && cpu_online(prev_cpu) && - available_idle_cpu(prev_cpu)) { + available_idle_cpu(prev_cpu) && + cpumask_test_cpu(prev_cpu, p->cpus_ptr)) { target_cpu = prev_cpu; fbt_env->fastpath = PREV_CPU_FASTPATH; cpumask_set_cpu(target_cpu, candidates); From 4dcd5a2a02a2de20e873f46a3b775d61de6ea2a9 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 5 May 2021 22:47:14 -0700 Subject: [PATCH 092/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I2a10a999f94e2f21eb97761516c135aadbe305e7 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/boost.c | 8 ++------ kernel/sched/walt/sched_avg.c | 2 +- kernel/sched/walt/walt.c | 10 +++------- kernel/sched/walt/walt.h | 29 +++++++++-------------------- kernel/sched/walt/walt_cfs.c | 2 +- 5 files changed, 16 insertions(+), 35 deletions(-) diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c index ded1ce3d0ac5..eec2a5623987 100644 --- a/kernel/sched/walt/boost.c +++ b/kernel/sched/walt/boost.c @@ -14,8 +14,9 @@ * ended up with their load characteristics. Any entity enabling * boost is responsible for disabling it as well. */ +unsigned int sched_boost_type; +enum sched_boost_policy boost_policy; -static enum sched_boost_policy boost_policy_dt = SCHED_BOOST_NONE; static DEFINE_MUTEX(boost_mutex); void walt_init_tg(struct task_group *tg) @@ -75,11 +76,6 @@ static void set_boost_policy(int type) return; } - if (boost_policy_dt) { - boost_policy = boost_policy_dt; - return; - } - if (hmp_capable()) { boost_policy = SCHED_BOOST_ON_BIG; return; diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 6bac78df55e7..d670148f80e7 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -115,7 +115,7 @@ void sched_update_hyst_times(void) int cpu; unsigned long cpu_cap, coloc_busy_pct; - rtgb_active = is_rtgb_active() && (sched_boost() != CONSERVATIVE_BOOST) + rtgb_active = is_rtgb_active() && (sched_boost_type != CONSERVATIVE_BOOST) && (get_rtgb_active_time() < MAX_RTGB_TIME); for_each_possible_cpu(cpu) { diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 02bd142e11c1..9f892c525b63 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -60,8 +60,6 @@ static struct irq_work walt_cpufreq_irq_work; struct irq_work walt_migration_irq_work; unsigned int walt_rotation_enabled; cpumask_t asym_cap_sibling_cpus = CPU_MASK_NONE; -unsigned int sched_boost_type; -enum sched_boost_policy boost_policy; unsigned int __read_mostly sched_ravg_window = 20000000; unsigned int min_max_possible_capacity = 1024; @@ -441,8 +439,7 @@ static void update_task_cpu_cycles(struct task_struct *p, int cpu, static inline bool is_ed_enabled(void) { - return (walt_rotation_enabled || (sched_boost_policy() != - SCHED_BOOST_NONE)); + return (walt_rotation_enabled || (boost_policy != SCHED_BOOST_NONE)); } static inline bool is_ed_task(struct task_struct *p, u64 wallclock) @@ -3079,8 +3076,7 @@ static bool is_cluster_hosting_top_app(struct walt_sched_cluster *cluster) if (!grp) return false; - grp_on_min = !grp->skip_min && - (sched_boost_policy() != SCHED_BOOST_ON_BIG); + grp_on_min = !grp->skip_min && (boost_policy != SCHED_BOOST_ON_BIG); return (is_min_capacity_cluster(cluster) == grp_on_min); } @@ -3486,7 +3482,7 @@ void walt_rotation_checkpoint(int nr_big) if (!hmp_capable()) return; - if (!sysctl_sched_walt_rotate_big_tasks || sched_boost() != NO_BOOST) { + if (!sysctl_sched_walt_rotate_big_tasks || sched_boost_type != NO_BOOST) { walt_rotation_enabled = 0; return; } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index ab3584f9e7a3..57f2affa53bc 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -404,25 +404,15 @@ static inline unsigned long cpu_util_cum(int cpu) return READ_ONCE(cpu_rq(cpu)->cfs.avg.util_avg); } -static inline enum sched_boost_policy sched_boost_policy(void) -{ - return boost_policy; -} - -static inline int sched_boost(void) -{ - return sched_boost_type; -} - static inline bool rt_boost_on_big(void) { - return sched_boost() == FULL_THROTTLE_BOOST ? - (sched_boost_policy() == SCHED_BOOST_ON_BIG) : false; + return sched_boost_type == FULL_THROTTLE_BOOST ? + (boost_policy == SCHED_BOOST_ON_BIG) : false; } static inline bool is_full_throttle_boost(void) { - return sched_boost() == FULL_THROTTLE_BOOST; + return sched_boost_type == FULL_THROTTLE_BOOST; } static inline bool task_sched_boost(struct task_struct *p) @@ -433,7 +423,7 @@ static inline bool task_sched_boost(struct task_struct *p) struct walt_task_group *wtg; /* optimization for FT boost, skip looking at tg */ - if (sched_boost() == FULL_THROTTLE_BOOST) + if (sched_boost_type == FULL_THROTTLE_BOOST) return true; rcu_read_lock(); @@ -444,7 +434,7 @@ static inline bool task_sched_boost(struct task_struct *p) } tg = container_of(css, struct task_group, css); wtg = (struct walt_task_group *) tg->android_vendor_data1; - sched_boost_enabled = wtg->sched_boost_enable[sched_boost()]; + sched_boost_enabled = wtg->sched_boost_enable[sched_boost_type]; rcu_read_unlock(); return sched_boost_enabled; @@ -452,7 +442,7 @@ static inline bool task_sched_boost(struct task_struct *p) static inline bool task_placement_boost_enabled(struct task_struct *p) { - if (likely(sched_boost_policy() == SCHED_BOOST_NONE)) + if (likely(boost_policy == SCHED_BOOST_NONE)) return false; return task_sched_boost(p); @@ -462,16 +452,16 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) { enum sched_boost_policy policy; - if (likely(sched_boost_policy() == SCHED_BOOST_NONE)) + if (likely(boost_policy == SCHED_BOOST_NONE)) return SCHED_BOOST_NONE; - policy = task_sched_boost(p) ? sched_boost_policy() : SCHED_BOOST_NONE; + policy = task_sched_boost(p) ? boost_policy : SCHED_BOOST_NONE; if (policy == SCHED_BOOST_ON_BIG) { /* * Filter out tasks less than min task util threshold * under conservative boost. */ - if (sched_boost() == CONSERVATIVE_BOOST && + if (sched_boost_type == CONSERVATIVE_BOOST && task_util(p) <= sysctl_sched_min_task_util_for_boost) policy = SCHED_BOOST_NONE; } @@ -759,7 +749,6 @@ static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); extern void sched_update_hyst_times(void); -extern enum sched_boost_policy sched_boost_policy(void); extern void walt_rt_init(void); extern void walt_cfs_init(void); extern void walt_pause_init(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 990fe008a9d5..103fe55fbd92 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -96,7 +96,7 @@ static inline bool walt_task_skip_min_cpu(struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - return sched_boost() != CONSERVATIVE_BOOST && + return (sched_boost_type != CONSERVATIVE_BOOST) && walt_get_rtg_status(p) && wts->unfilter; } From fafe5c1f61af421adbb58b6914aacc2f747792ef Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Tue, 18 May 2021 12:10:54 +0530 Subject: [PATCH 093/346] sched/walt: Fix newline issues Fix newline issues in MVP tasks code. Change-Id: I0d0ee2ae5e50b5b4310544d4c1fb818bff998773 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt.h | 1 - kernel/sched/walt/walt_cfs.c | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 57f2affa53bc..861ea775ddf2 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -868,7 +868,6 @@ static inline bool walt_fair_task(struct task_struct *p) return p->prio >= MAX_RT_PRIO && !is_idle_task(p); } - #define WALT_MVP_SLICE 3000000U #define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 103fe55fbd92..ae996ed8fccc 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -742,6 +742,7 @@ static void binder_restore_priority_hook(void *data, wts->boost = bndrtrans->android_vendor_data1; } + /* * Higher prio mvp can preempt lower prio mvp. * From 42aab9f3640a5f273ae4cbbb4f20453e0902e9d5 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Wed, 12 May 2021 10:38:46 -0700 Subject: [PATCH 094/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I9a94b20b18bf5b78946fd26d3337f5227d735e03 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_lb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 921530c441ce..a0adcd7d6cd4 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -60,7 +60,8 @@ static int walt_lb_active_migration(void *data) if (task_on_rq_queued(push_task) && push_task->state == TASK_RUNNING && task_cpu(push_task) == busiest_cpu && - cpu_active(target_cpu)) { + cpu_active(target_cpu) && + cpumask_test_cpu(target_cpu, push_task->cpus_ptr)) { walt_detach_task(push_task, busiest_rq, target_rq); push_task_detached = 1; } From 2dbf2bb3b9fdbcdf67734c6f4757fe4538c0db6e Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 18 May 2021 17:10:51 -0700 Subject: [PATCH 095/346] sched/walt: fixup_busy_time: Correct typo in lockdep_assert_held The parameter to lockdep_assert_held should be a pointer to a lock, not the lock itself. Change-Id: I4586d86b8684b21370a2bc522862932481fef538 Signed-off-by: Elliot Berman --- kernel/sched/walt/walt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 9f892c525b63..b0079827eae9 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -981,8 +981,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) wallclock = sched_ktime_clock(); - lockdep_assert_held(src_rq->lock); - lockdep_assert_held(dest_rq->lock); + lockdep_assert_held(&src_rq->lock); + lockdep_assert_held(&dest_rq->lock); walt_update_task_ravg(task_rq(p)->curr, task_rq(p), TASK_UPDATE, From 5f5bd6a0d2aaaf10e9a27b1981c439a67cafdb4d Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 5 May 2021 22:47:28 -0700 Subject: [PATCH 096/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I5f08d40d6e9f18a905d87db078fa1a70abaa9030 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/boost.c | 6 ++++++ kernel/sched/walt/walt.c | 1 + kernel/sched/walt/walt.h | 1 + 3 files changed, 8 insertions(+) diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c index eec2a5623987..451b1090c60c 100644 --- a/kernel/sched/walt/boost.c +++ b/kernel/sched/walt/boost.c @@ -291,3 +291,9 @@ int sched_boost_handler(struct ctl_table *table, int write, mutex_unlock(&boost_mutex); return ret; } + +void walt_boost_init(void) +{ + /* force call the callbacks for default boost */ + sched_set_boost(FULL_THROTTLE_BOOST); +} diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index b0079827eae9..554fab2ee0a8 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4186,6 +4186,7 @@ static void walt_init(void) input_boost_init(); core_ctl_init(); + walt_boost_init(); waltgov_register(); i = match_string(sched_feat_names, __SCHED_FEAT_NR, "TTWU_QUEUE"); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 861ea775ddf2..43723dd53f8d 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -157,6 +157,7 @@ extern int sched_set_group_id(struct task_struct *p, unsigned int group_id); extern unsigned int sched_get_group_id(struct task_struct *p); extern void core_ctl_check(u64 wallclock); extern int sched_set_boost(int enable); +extern void walt_boost_init(void); extern int sched_wake_up_idle_show(struct seq_file *m, void *v); extern ssize_t sched_wake_up_idle_write(struct file *file, const char __user *buf, size_t count, loff_t *offset); From 7d53585557f541ad826ce5ee6d2c18e7be18eb98 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Wed, 19 May 2021 08:41:38 -0700 Subject: [PATCH 097/346] sched/walt: fix lock ordering issue between RTG lock and task's RQ lock lock ordering issue is observed in below scenario where preferred cluster for the task is being set on one CPU while task is being attached to related thread group which leads to deadlock. CPUX: scheduler_tick() --> acquire rq_lock() trace_android_rvh_tick_entry() android_rvh_tick_entry() set_preferred_cluster() --> try to acquire RTG lock taken by CPUY CPUY: cpu_cgroup_attach() trace_android_rvh_cpu_cgroup_attach() android_rvh_cpu_cgroup_attach() __sched_set_group_id() add_task_to_group() --> acquire RTG lock __task_rq_lock() --> try to acquire task's rq_lock() which is CPUX Fix the issue by moving set_preferred_cluster() related piece of code which is being called from scheduler_tick() to android_vh_scheduler_tick trace hook. Change-Id: Ib9e2d70ae45a304d21cde3de6b8a84f9049061ab Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt.c | 22 +++++++++++++++++----- kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_lb.c | 6 +----- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 554fab2ee0a8..00a902f22626 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3934,25 +3934,36 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct static void android_rvh_tick_entry(void *unused, struct rq *rq) { u64 wallclock; - u32 old_load; - struct walt_related_thread_group *grp; + lockdep_assert_held(&rq->lock); if (unlikely(walt_disabled)) return; + set_window_start(rq); wallclock = sched_ktime_clock(); - old_load = task_load(rq->curr); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + if (is_ed_task_present(rq, wallclock, NULL)) + waltgov_run_callback(rq, WALT_CPUFREQ_EARLY_DET); +} + +static void android_vh_scheduler_tick(void *unused, struct rq *rq) +{ + struct walt_related_thread_group *grp; + u32 old_load; + + if (unlikely(walt_disabled)) + return; + + old_load = task_load(rq->curr); rcu_read_lock(); grp = task_related_thread_group(rq->curr); if (update_preferred_cluster(grp, rq->curr, old_load, true)) set_preferred_cluster(grp); rcu_read_unlock(); - if (is_ed_task_present(rq, wallclock, NULL)) - waltgov_run_callback(rq, WALT_CPUFREQ_EARLY_DET); + walt_lb_tick(rq); } static void android_rvh_schedule(void *unused, struct task_struct *prev, @@ -4081,6 +4092,7 @@ static void register_walt_hooks(void) register_trace_android_rvh_try_to_wake_up(android_rvh_try_to_wake_up, NULL); register_trace_android_rvh_try_to_wake_up_success(android_rvh_try_to_wake_up_success, NULL); register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL); + register_trace_android_vh_scheduler_tick(android_vh_scheduler_tick, NULL); register_trace_android_rvh_schedule(android_rvh_schedule, NULL); register_trace_android_rvh_resume_cpus(android_rvh_resume_cpus, NULL); register_trace_android_rvh_cpu_cgroup_attach(android_rvh_cpu_cgroup_attach, NULL); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 43723dd53f8d..e4012126a0b7 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -881,5 +881,6 @@ static inline bool walt_fair_task(struct task_struct *p) void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p); void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p); void walt_cfs_tick(struct rq *rq); +void walt_lb_tick(struct rq *rq); #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index a0adcd7d6cd4..547428bcf1b2 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -485,7 +485,7 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask) } static DEFINE_RAW_SPINLOCK(walt_lb_migration_lock); -static void walt_lb_tick(void *unused, struct rq *rq) +void walt_lb_tick(struct rq *rq) { int prev_cpu = rq->cpu, new_cpu, ret; struct task_struct *p = rq->curr; @@ -493,9 +493,6 @@ static void walt_lb_tick(void *unused, struct rq *rq) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (unlikely(walt_disabled)) - return; - if (!walt_fair_task(p)) return; @@ -842,5 +839,4 @@ void walt_lb_init(void) register_trace_android_rvh_can_migrate_task(walt_can_migrate_task, NULL); register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); register_trace_android_rvh_sched_newidle_balance(walt_newidle_balance, NULL); - register_trace_android_vh_scheduler_tick(walt_lb_tick, NULL); } From a62a217748339e89c09e6252b05b452009823db2 Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Mon, 24 May 2021 14:22:28 +0800 Subject: [PATCH 098/346] sched/fair: Add sched_sync_hint_enable node Add sched_sync_hint_enable not to enable/disable sync. Change-Id: I455545487d002b4e3caf96c6cf8569549c3b1bbe Signed-off-by: Tengfei Fan --- kernel/sched/walt/sysctl.c | 10 ++++++++++ kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_cfs.c | 3 ++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index e7738f397f09..818117c03616 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -56,6 +56,7 @@ unsigned int sysctl_sched_min_task_util_for_colocation = 35; unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; const int sched_user_hint_max = 1000; unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ +unsigned int sysctl_sched_sync_hint_enable = 1; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -661,6 +662,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_sync_hint_enable", + .data = &sysctl_sched_sync_hint_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, { .procname = "sched_lib_name", .data = sched_lib_name, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index e4012126a0b7..5f27c897fe42 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -277,6 +277,7 @@ extern unsigned int sysctl_sched_walt_rotate_big_tasks; extern unsigned int sysctl_sched_task_unfilter_period; extern unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ +extern unsigned int sysctl_sched_sync_hint_enable; extern struct ctl_table walt_table[]; extern struct ctl_table walt_base_table[]; extern void walt_tunables(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index ae996ed8fccc..a647b7a726e7 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -544,7 +544,8 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (sync && (need_idle || (is_rtg && curr_is_rtg))) sync = 0; - if (sync && bias_to_this_cpu(p, cpu, start_cpu)) { + if (sysctl_sched_sync_hint_enable && sync + && bias_to_this_cpu(p, cpu, start_cpu)) { best_energy_cpu = cpu; fbt_env.fastpath = SYNC_WAKEUP; goto done; From 39605473ba582acd80596feb76739dd0fb0c58ab Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 24 May 2021 10:31:38 -0700 Subject: [PATCH 099/346] sched: walt: expand waltgov_util_update trace point When experiencing frequency related issues, it will be helpful to be aware of any potential clamping issues, as well as reasons why we might end up not updating to a different frequency. Change-Id: I0be24d713421357e077cd01d5c84f43138a17b24 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 3 ++- kernel/sched/walt/trace.h | 29 +++++++++++++++++++++-------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 4722a2a1dc96..57692b62949c 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -206,7 +206,8 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, unsigned int freq = policy->cpuinfo.max_freq; freq = map_util_freq(util, freq, max); - trace_waltgov_next_freq(policy->cpu, util, max, freq); + trace_waltgov_next_freq(policy->cpu, util, max, freq, policy->min, policy->max, + wg_policy->cached_raw_freq, wg_policy->need_freq_update); if (freq == wg_policy->cached_raw_freq && !wg_policy->need_freq_update) return wg_policy->next_freq; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index b6f8a8738d3b..3aa89daa9742 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -647,25 +647,38 @@ TRACE_EVENT(waltgov_util_update, TRACE_EVENT(waltgov_next_freq, TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, - unsigned int freq), - TP_ARGS(cpu, util, max, freq), + unsigned int freq, unsigned int min_freq, unsigned int max_freq, + unsigned int cached_raw_freq, bool need_freq_update), + TP_ARGS(cpu, util, max, freq, min_freq, max_freq, cached_raw_freq, need_freq_update), TP_STRUCT__entry( __field(unsigned int, cpu) __field(unsigned long, util) __field(unsigned long, max) __field(unsigned int, freq) + __field(unsigned int, min_freq) + __field(unsigned int, max_freq) + __field(unsigned int, cached_raw_freq) + __field(bool, need_freq_update) ), TP_fast_assign( - __entry->cpu = cpu; - __entry->util = util; - __entry->max = max; - __entry->freq = freq; + __entry->cpu = cpu; + __entry->util = util; + __entry->max = max; + __entry->freq = freq; + __entry->min_freq = min_freq; + __entry->max_freq = max_freq; + __entry->cached_raw_freq = cached_raw_freq; + __entry->need_freq_update = need_freq_update; ), - TP_printk("cpu=%u util=%lu max=%lu freq=%u", + TP_printk("cpu=%u util=%lu max=%lu freq=%u min_freq=%lu max_freq=%lu cached_raw_freq=%lu need_update=%d", __entry->cpu, __entry->util, __entry->max, - __entry->freq) + __entry->freq, + __entry->min_freq, + __entry->max_freq, + __entry->cached_raw_freq, + __entry->need_freq_update) ); TRACE_EVENT(walt_active_load_balance, From d804178181f3b46e36673b0363879723227cfd06 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 24 May 2021 18:03:52 -0700 Subject: [PATCH 100/346] sched: walt: Adjust nomenclature Change min_freq and max_freq to be policy_min_freq, policy_max_freq. Change-Id: Ief7ee31209338e6fc25f4ad8933b60256ac5eb95 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 3aa89daa9742..463f6ca24718 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -647,16 +647,17 @@ TRACE_EVENT(waltgov_util_update, TRACE_EVENT(waltgov_next_freq, TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, - unsigned int freq, unsigned int min_freq, unsigned int max_freq, + unsigned int freq, unsigned int policy_min_freq, unsigned int policy_max_freq, unsigned int cached_raw_freq, bool need_freq_update), - TP_ARGS(cpu, util, max, freq, min_freq, max_freq, cached_raw_freq, need_freq_update), + TP_ARGS(cpu, util, max, freq, policy_min_freq, policy_max_freq, + cached_raw_freq, need_freq_update), TP_STRUCT__entry( __field(unsigned int, cpu) __field(unsigned long, util) __field(unsigned long, max) __field(unsigned int, freq) - __field(unsigned int, min_freq) - __field(unsigned int, max_freq) + __field(unsigned int, policy_min_freq) + __field(unsigned int, policy_max_freq) __field(unsigned int, cached_raw_freq) __field(bool, need_freq_update) ), @@ -665,18 +666,18 @@ TRACE_EVENT(waltgov_next_freq, __entry->util = util; __entry->max = max; __entry->freq = freq; - __entry->min_freq = min_freq; - __entry->max_freq = max_freq; + __entry->policy_min_freq = policy_min_freq; + __entry->policy_max_freq = policy_max_freq; __entry->cached_raw_freq = cached_raw_freq; __entry->need_freq_update = need_freq_update; ), - TP_printk("cpu=%u util=%lu max=%lu freq=%u min_freq=%lu max_freq=%lu cached_raw_freq=%lu need_update=%d", + TP_printk("cpu=%u util=%lu max=%lu freq=%u policy_min_freq=%lu policy_max_freq=%lu cached_raw_freq=%lu need_update=%d", __entry->cpu, __entry->util, __entry->max, __entry->freq, - __entry->min_freq, - __entry->max_freq, + __entry->policy_min_freq, + __entry->policy_max_freq, __entry->cached_raw_freq, __entry->need_freq_update) ); From 8bbd8002edec3e9c19ac2a437f290d342fab7910 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 25 May 2021 14:34:17 -0700 Subject: [PATCH 101/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I3280c12b5446b8c12fd4269efc2c1dad70e4ae6a Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 62 +++++++++++++++++-------------------- kernel/sched/walt/walt.h | 4 +-- kernel/sched/walt/walt_lb.c | 4 +-- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 00a902f22626..9c84bccf835d 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -50,7 +50,7 @@ const char *migrate_type_names[] = { unsigned int sysctl_sched_user_hint; static ktime_t ktime_last; -static bool sched_ktime_suspended; +static bool walt_ktime_suspended; static bool use_cycle_counter; static DEFINE_MUTEX(cluster_lock); @@ -99,36 +99,30 @@ int set_task_boost(int boost, u64 period) } EXPORT_SYMBOL(set_task_boost); -u64 sched_ktime_clock(void) +u64 walt_ktime_get_ns(void) { - if (unlikely(sched_ktime_suspended)) + if (unlikely(walt_ktime_suspended)) return ktime_to_ns(ktime_last); return ktime_get_ns(); } -static void sched_resume(void) +static void walt_resume(void) { - sched_ktime_suspended = false; + walt_ktime_suspended = false; } -static int sched_suspend(void) +static int walt_suspend(void) { ktime_last = ktime_get(); - sched_ktime_suspended = true; + walt_ktime_suspended = true; return 0; } -static struct syscore_ops sched_syscore_ops = { - .resume = sched_resume, - .suspend = sched_suspend +static struct syscore_ops walt_syscore_ops = { + .resume = walt_resume, + .suspend = walt_suspend }; -static int sched_init_ops(void) -{ - register_syscore_ops(&sched_syscore_ops); - return 0; -} - static inline void acquire_rq_locks_irqsave(const cpumask_t *cpus, unsigned long *flags) { @@ -269,7 +263,7 @@ static inline void walt_dump(void) int cpu; printk_deferred("============ WALT RQ DUMP START ==============\n"); - printk_deferred("Sched ktime_get: %llu\n", sched_ktime_clock()); + printk_deferred("Sched ktime_get: %llu\n", walt_ktime_get_ns()); printk_deferred("Time last window changed=%lu\n", sched_ravg_window_change_time); for_each_online_cpu(cpu) @@ -488,7 +482,7 @@ static void walt_sched_account_irqstart(int cpu, struct task_struct *curr) /* We're here without rq->lock held, IRQ disabled */ raw_spin_lock(&rq->lock); - update_task_cpu_cycles(curr, cpu, sched_ktime_clock()); + update_task_cpu_cycles(curr, cpu, walt_ktime_get_ns()); raw_spin_unlock(&rq->lock); } @@ -500,7 +494,7 @@ static void walt_sched_account_irqend(int cpu, struct task_struct *curr, u64 del unsigned long flags; raw_spin_lock_irqsave(&rq->lock, flags); - walt_update_task_ravg(curr, rq, IRQ_UPDATE, sched_ktime_clock(), delta); + walt_update_task_ravg(curr, rq, IRQ_UPDATE, walt_ktime_get_ns(), delta); raw_spin_unlock_irqrestore(&rq->lock, flags); } @@ -979,7 +973,7 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) if (pstate == TASK_WAKING) double_rq_lock(src_rq, dest_rq); - wallclock = sched_ktime_clock(); + wallclock = walt_ktime_get_ns(); lockdep_assert_held(&src_rq->lock); lockdep_assert_held(&dest_rq->lock); @@ -2266,7 +2260,7 @@ static void mark_task_starting(struct task_struct *p) struct rq *rq = task_rq(p); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - wallclock = sched_ktime_clock(); + wallclock = walt_ktime_get_ns(); wts->mark_start = wts->last_wake_ts = wallclock; wts->last_enqueued_ts = wallclock; update_task_cpu_cycles(p, cpu_of(rq), wallclock); @@ -2615,7 +2609,7 @@ static int cpufreq_notifier_trans(struct notifier_block *nb, raw_spin_lock_irqsave(&rq->lock, flags); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, - sched_ktime_clock(), 0); + walt_ktime_get_ns(), 0); raw_spin_unlock_irqrestore(&rq->lock, flags); } @@ -2713,7 +2707,7 @@ static void _set_preferred_cluster(struct walt_related_thread_group *grp) goto out; } - wallclock = sched_ktime_clock(); + wallclock = walt_ktime_get_ns(); /* * wakeup of two or more related tasks could race with each other and @@ -2778,7 +2772,7 @@ static int update_preferred_cluster(struct walt_related_thread_group *grp, * has passed since we last updated preference */ if (abs(new_load - old_load) > sched_ravg_window / 4 || - sched_ktime_clock() - grp->last_update > sched_ravg_window) + walt_ktime_get_ns() - grp->last_update > sched_ravg_window) return 1; return 0; @@ -3113,7 +3107,7 @@ static void transfer_busy_time(struct rq *rq, struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - wallclock = sched_ktime_clock(); + wallclock = walt_ktime_get_ns(); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); @@ -3353,7 +3347,7 @@ static void walt_irq_work(struct irq_work *irq_work) level++; } - wc = sched_ktime_clock(); + wc = walt_ktime_get_ns(); walt_load_reported_window = atomic64_read(&walt_irq_work_lastq_ws); for_each_sched_cluster(cluster) { u64 aggr_grp_load = 0; @@ -3458,7 +3452,7 @@ static void walt_irq_work(struct irq_work *irq_work) wrq = (struct walt_rq *) this_rq()->android_vendor_data1; if ((sched_ravg_window != new_sched_ravg_window) && (wc < wrq->window_start + new_sched_ravg_window)) { - sched_ravg_window_change_time = sched_ktime_clock(); + sched_ravg_window_change_time = walt_ktime_get_ns(); trace_sched_ravg_window_change(sched_ravg_window, new_sched_ravg_window, sched_ravg_window_change_time); @@ -3508,7 +3502,7 @@ void walt_fill_ta_data(struct core_ctl_notif_data *data) goto fill_util; } - wallclock = sched_ktime_clock(); + wallclock = walt_ktime_get_ns(); list_for_each_entry(wts, &grp->tasks, grp_list) { if (wts->mark_start < wallclock - @@ -3797,7 +3791,7 @@ static void android_rvh_flush_task(void *unused, struct task_struct *p) static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) { - u64 wallclock = sched_ktime_clock(); + u64 wallclock = walt_ktime_get_ns(); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (unlikely(walt_disabled)) @@ -3841,7 +3835,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st wts->prev_on_rq = 2; if (p == wrq->ed_task) - is_ed_task_present(rq, sched_ktime_clock(), p); + is_ed_task_present(rq, walt_ktime_get_ns(), p); sched_update_nr_prod(rq->cpu, -1); @@ -3904,7 +3898,7 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) return; rq_lock_irqsave(rq, &rf); old_load = task_load(p); - wallclock = sched_ktime_clock(); + wallclock = walt_ktime_get_ns(); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); note_task_waking(p, wallclock); @@ -3940,7 +3934,7 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) return; set_window_start(rq); - wallclock = sched_ktime_clock(); + wallclock = walt_ktime_get_ns(); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); @@ -3969,7 +3963,7 @@ static void android_vh_scheduler_tick(void *unused, struct rq *rq) static void android_rvh_schedule(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq) { - u64 wallclock = sched_ktime_clock(); + u64 wallclock = walt_ktime_get_ns(); struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; if (unlikely(walt_disabled)) @@ -4178,7 +4172,7 @@ static void walt_init(void) walt_tunables(); - sched_init_ops(); + register_syscore_ops(&walt_syscore_ops); BUG_ON(alloc_related_thread_groups()); walt_init_cycle_counter(); init_clusters(); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 5f27c897fe42..61585a487379 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -182,7 +182,7 @@ extern int sched_boost_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); extern int sched_busy_hyst_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); -extern u64 sched_ktime_clock(void); +extern u64 walt_ktime_get_ns(void); extern void clear_walt_request(int cpu); extern void walt_init_tg(struct task_group *tg); extern void walt_init_topapp_tg(struct task_group *tg); @@ -365,7 +365,7 @@ static inline void waltgov_run_callback(struct rq *rq, unsigned int flags) cb = rcu_dereference_sched(*per_cpu_ptr(&waltgov_cb_data, cpu_of(rq))); if (cb) - cb->func(cb, sched_ktime_clock(), flags); + cb->func(cb, walt_ktime_get_ns(), flags); } extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 547428bcf1b2..29830f65a90c 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -135,7 +135,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) if (!is_min_capacity_cpu(src_cpu)) return; - wc = sched_ktime_clock(); + wc = walt_ktime_get_ns(); for_each_possible_cpu(i) { struct rq *rq = cpu_rq(i); @@ -594,7 +594,7 @@ static bool walt_balance_rt(struct rq *this_rq) goto unlock; wts = (struct walt_task_struct *) p->android_vendor_data1; - if (sched_ktime_clock() - wts->last_wake_ts < WALT_RT_PULL_THRESHOLD_NS) + if (walt_ktime_get_ns() - wts->last_wake_ts < WALT_RT_PULL_THRESHOLD_NS) goto unlock; pulled = true; From 5a4738726dd4c2663ab864c0d50eb586643dc7df Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 27 May 2021 07:14:06 +0530 Subject: [PATCH 102/346] sched/walt: Fix locking issue in MVP tasks update Protect MVP tasks list access in the tick path with rq lock. Change-Id: I387aa104cbacaf445bf739b8fb6d631283bd0ada Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_cfs.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index a647b7a726e7..93eaf6cbd584 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -828,6 +828,8 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr s64 delta; unsigned int limit; + lockdep_assert_held(&rq->lock); + /* sum_exec_snapshot can be ahead. See below increment */ delta = curr->se.sum_exec_runtime - wts->sum_exec_snapshot; if (delta < 0) @@ -905,8 +907,10 @@ void walt_cfs_tick(struct rq *rq) if (unlikely(walt_disabled)) return; + raw_spin_lock(&rq->lock); + if (list_empty(&wts->mvp_list)) - return; + goto out; walt_cfs_account_mvp_runtime(rq, rq->curr); /* @@ -915,6 +919,9 @@ void walt_cfs_tick(struct rq *rq) */ if ((wrq->mvp_tasks.next != &wts->mvp_list) && rq->cfs.h_nr_running > 1) resched_curr(rq); + +out: + raw_spin_unlock(&rq->lock); } /* From d233af280c6c7bf6513c5f1564564a3862d59b43 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Thu, 27 May 2021 10:24:50 +0530 Subject: [PATCH 103/346] sched/walt: Do the rq clock update before MVP tasks update The rq clock gets updated in the scheduler tick. However, if the lock is acquired remotely between the scheduler tick and the vendor hook, the clock update flags gets reset. Do the clock update in walt_cfs_account_mvp_runtime based on the clock update flags. Change-Id: Ie4254a191a18458cf1684070bc96ff63cf0103d3 Signed-off-by: Pavankumar Kondeti --- kernel/sched/walt/walt_cfs.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 93eaf6cbd584..02d1c891242a 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -830,6 +830,16 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr lockdep_assert_held(&rq->lock); + /* + * RQ clock update happens in tick path in the scheduler. + * Since we drop the lock in the scheduler before calling + * into vendor hook, it is possible that update flags are + * reset by another rq lock and unlock. Do the update here + * if required. + */ + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + /* sum_exec_snapshot can be ahead. See below increment */ delta = curr->se.sum_exec_runtime - wts->sum_exec_snapshot; if (delta < 0) From 9d8bdac63a9a4ac8b649250bcce2fef6f83b834f Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Thu, 27 May 2021 21:07:37 -0700 Subject: [PATCH 104/346] sched/walt: improve logging in the error paths Printing CPUs info is useful while going through error paths. Change-Id: If84794163b6c417d419785fdddd2eed98588f2eb Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 9c84bccf835d..c8647031402e 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -299,7 +299,8 @@ fixup_cumulative_runnable_avg(struct rq *rq, lockdep_assert_held(&rq->lock); if (task_rq(p) != rq) { - printk_deferred("WALT-BUG task not on rq\n"); + printk_deferred("WALT-BUG task not on rq: task_cpu=%d new_cpu=%d\n", + task_cpu(p), cpu_of(rq)); walt_task_dump(p); SCHED_BUG_ON(1); } @@ -3799,7 +3800,8 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st /* catch double enqueue */ if (wts->prev_on_rq == 1) { - printk_deferred("WALT-BUG double enqueue detected\n"); + printk_deferred("WALT-BUG double enqueue detected: task_cpu=%d new_cpu=%d\n", + task_cpu(p), cpu_of(rq)); walt_task_dump(p); SCHED_BUG_ON(1); } @@ -3828,7 +3830,8 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st /* catch double deq */ if (wts->prev_on_rq == 2) { - printk_deferred("WALT-BUG double dequeue detected\n"); + printk_deferred("WALT-BUG double dequeue detected: task_cpu=%d new_cpu=%d\n", + task_cpu(p), cpu_of(rq)); walt_task_dump(p); SCHED_BUG_ON(1); } From 13b036b5ef97877190f73284861f06164d3084cf Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Tue, 11 May 2021 10:43:20 -0700 Subject: [PATCH 105/346] sched: Add util busy hist sysctl tunables Add util_busy_hist sysctl tunables which solely look at the sum of cpu_util across CPUs as a metric for LPM biasing on any/all CPUs. Change-Id: Ib981c21a58059ef02772ee797808c5f30547a688 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/sysctl.c | 34 ++++++++++++++++++++++++++++++++++ kernel/sched/walt/walt.h | 3 +++ 2 files changed, 37 insertions(+) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 818117c03616..a005b3004413 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -34,6 +34,9 @@ unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus; unsigned int sysctl_sched_coloc_busy_hyst_cpu[WALT_NR_CPUS]; unsigned int sysctl_sched_coloc_busy_hyst_max_ms; unsigned int sysctl_sched_coloc_busy_hyst_cpu_busy_pct[WALT_NR_CPUS]; +unsigned int sysctl_sched_util_busy_hyst_enable_cpus; +unsigned int sysctl_sched_util_busy_hyst_cpu[WALT_NR_CPUS]; +unsigned int sysctl_sched_util_busy_hyst_cpu_util[WALT_NR_CPUS]; unsigned int sysctl_sched_boost; unsigned int sysctl_sched_wake_up_idle[2]; unsigned int sysctl_input_boost_ms; @@ -614,6 +617,33 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &one_hundred, }, + { + .procname = "sched_util_busy_hysteresis_enable_cpus", + .data = &sysctl_sched_util_busy_hyst_enable_cpus, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_util_busy_hyst_cpu_ns", + .data = &sysctl_sched_util_busy_hyst_cpu, + .maxlen = sizeof(unsigned int) * WALT_NR_CPUS, + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &ns_per_sec, + }, + { + .procname = "sched_util_busy_hyst_cpu_util", + .data = &sysctl_sched_util_busy_hyst_cpu_util, + .maxlen = sizeof(unsigned int) * WALT_NR_CPUS, + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, { .procname = "sched_ravg_window_nr_ticks", .data = &sysctl_sched_ravg_window_nr_ticks, @@ -781,10 +811,14 @@ void walt_tunables(void) for (i = 0; i < WALT_NR_CPUS; i++) { sysctl_sched_coloc_busy_hyst_cpu[i] = 39000000; sysctl_sched_coloc_busy_hyst_cpu_busy_pct[i] = 10; + sysctl_sched_util_busy_hyst_cpu[i] = 5000000; + sysctl_sched_util_busy_hyst_cpu_util[i] = 15; } sysctl_sched_coloc_busy_hyst_enable_cpus = 112; + sysctl_sched_util_busy_hyst_enable_cpus = 255; + sysctl_sched_coloc_busy_hyst_max_ms = 5000; sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 61585a487379..b0dc1690ae15 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -210,6 +210,9 @@ extern unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus; extern unsigned int sysctl_sched_coloc_busy_hyst_cpu[WALT_NR_CPUS]; extern unsigned int sysctl_sched_coloc_busy_hyst_max_ms; extern unsigned int sysctl_sched_coloc_busy_hyst_cpu_busy_pct[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_util_busy_hyst_enable_cpus; +extern unsigned int sysctl_sched_util_busy_hyst_cpu[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_util_busy_hyst_cpu_util[WALT_NR_CPUS]; extern unsigned int sysctl_sched_boost; /* To/from userspace */ extern unsigned int sysctl_sched_capacity_margin_up[MAX_MARGIN_LEVELS]; extern unsigned int sysctl_sched_capacity_margin_down[MAX_MARGIN_LEVELS]; From df94d80ceb31e6353f2019979e20fcec99e9c3ae Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Tue, 11 May 2021 10:59:31 -0700 Subject: [PATCH 106/346] sched: Add util_busy_hist feature Add util_busy_hist feature which looks at the total cpu_util across all CPUs as a metric for LPM biasing on CPUs. Change-Id: Ie148178dfdc8f2b13fd447b1a87d60176f53f4a7 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/sched_avg.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index d670148f80e7..934490b61b5b 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -29,6 +29,7 @@ static DEFINE_PER_CPU(atomic64_t, busy_hyst_end_time) = ATOMIC64_INIT(0); static DEFINE_PER_CPU(u64, hyst_time); static DEFINE_PER_CPU(u64, coloc_hyst_busy); static DEFINE_PER_CPU(u64, coloc_hyst_time); +static DEFINE_PER_CPU(u64, util_hyst_time); #define NR_THRESHOLD_PCT 15 #define MAX_RTGB_TIME (sysctl_sched_coloc_busy_hyst_max_ms * NSEC_PER_MSEC) @@ -130,6 +131,9 @@ void sched_update_hyst_times(void) sysctl_sched_coloc_busy_hyst_cpu[cpu] : 0; per_cpu(coloc_hyst_busy, cpu) = mult_frac(cpu_cap, coloc_busy_pct, 100); + per_cpu(util_hyst_time, cpu) = (BIT(cpu) + & sysctl_sched_util_busy_hyst_enable_cpus) ? + sysctl_sched_util_busy_hyst_cpu[cpu] : 0; } } @@ -140,9 +144,12 @@ static inline void update_busy_hyst_end_time(int cpu, bool dequeue, { bool nr_run_trigger = false; bool load_trigger = false, coloc_load_trigger = false; - u64 agg_hyst_time; + u64 agg_hyst_time, total_util = 0; + bool util_load_trigger = false; + int i; - if (!per_cpu(hyst_time, cpu) && !per_cpu(coloc_hyst_time, cpu)) + if (!per_cpu(hyst_time, cpu) && !per_cpu(coloc_hyst_time, cpu) && + !per_cpu(util_hyst_time, cpu)) return; if (prev_nr_run >= BUSY_NR_RUN && per_cpu(nr, cpu) < BUSY_NR_RUN) @@ -155,10 +162,22 @@ static inline void update_busy_hyst_end_time(int cpu, bool dequeue, if (dequeue && cpu_util(cpu) > per_cpu(coloc_hyst_busy, cpu)) coloc_load_trigger = true; - agg_hyst_time = max((nr_run_trigger || load_trigger) ? + if (dequeue) { + for_each_possible_cpu(i) { + total_util += cpu_util(i); + if (total_util >= sysctl_sched_util_busy_hyst_cpu_util[cpu]) { + util_load_trigger = true; + break; + } + } + } + + agg_hyst_time = max(max((nr_run_trigger || load_trigger) ? per_cpu(hyst_time, cpu) : 0, (nr_run_trigger || coloc_load_trigger) ? - per_cpu(coloc_hyst_time, cpu) : 0); + per_cpu(coloc_hyst_time, cpu) : 0), + (util_load_trigger) ? + per_cpu(util_hyst_time, cpu) : 0); if (agg_hyst_time) atomic64_set(&per_cpu(busy_hyst_end_time, cpu), From 004502ad2de5b23e77e9c22f4dfb4c136c5859ad Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Tue, 11 May 2021 11:42:49 -0700 Subject: [PATCH 107/346] sched/walt: add sched_busy_hyst_time trace event Create a trace event for tracking sched_busy_hyst behavior to improve debugging. Change-Id: I0921d8cc7e1f0781605a4963113a18e13827d6ac Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/sched_avg.c | 7 ++++++- kernel/sched/walt/trace.h | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 934490b61b5b..426bc30303cb 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -179,9 +179,14 @@ static inline void update_busy_hyst_end_time(int cpu, bool dequeue, (util_load_trigger) ? per_cpu(util_hyst_time, cpu) : 0); - if (agg_hyst_time) + if (agg_hyst_time) { atomic64_set(&per_cpu(busy_hyst_end_time, cpu), curr_time + agg_hyst_time); + trace_sched_busy_hyst_time(cpu, agg_hyst_time, prev_nr_run, + cpu_util(cpu), per_cpu(hyst_time, cpu), + per_cpu(coloc_hyst_time, cpu), + per_cpu(util_hyst_time, cpu)); + } } int sched_busy_hyst_handler(struct ctl_table *table, int write, diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 463f6ca24718..519ba4fd8aad 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -589,6 +589,41 @@ TRACE_EVENT(sched_get_nr_running_avg, __entry->nr_scaled) ); +TRACE_EVENT(sched_busy_hyst_time, + + TP_PROTO(int cpu, u64 hyst_time, unsigned long nr_run, + unsigned long cpu_util, u64 busy_hyst_time, + u64 coloc_hyst_time, u64 util_hyst_time), + + TP_ARGS(cpu, hyst_time, nr_run, cpu_util, busy_hyst_time, + coloc_hyst_time, util_hyst_time), + + TP_STRUCT__entry( + __field(int, cpu) + __field(u64, hyst_time) + __field(unsigned long, nr_run) + __field(unsigned long, cpu_util) + __field(u64, busy_hyst_time) + __field(u64, coloc_hyst_time) + __field(u64, util_hyst_time) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->hyst_time = hyst_time; + __entry->nr_run = nr_run; + __entry->cpu_util = cpu_util; + __entry->busy_hyst_time = busy_hyst_time; + __entry->coloc_hyst_time = coloc_hyst_time; + __entry->util_hyst_time = util_hyst_time; + ), + + TP_printk("cpu=%d hyst_time=%llu nr_run=%lu cpu_util=%lu busy_hyst_time=%llu coloc_hyst_time=%llu util_hyst_time=%llu", + __entry->cpu, __entry->hyst_time, __entry->nr_run, + __entry->cpu_util, __entry->busy_hyst_time, + __entry->coloc_hyst_time, __entry->util_hyst_time) +); + TRACE_EVENT(sched_ravg_window_change, TP_PROTO(unsigned int sched_ravg_window, unsigned int new_sched_ravg_window From 9fdd530d510f6878732110a85ae977d5ad5892e4 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 28 May 2021 10:08:41 -0700 Subject: [PATCH 108/346] sched/walt: improve debug of strange rq situations in walt When a task is not on a rq when it should be, halt and print some useful information. Change-Id: Iaecbf372c712acba98879c8afb4e073d50a962f7 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c8647031402e..4803b502e89c 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -184,6 +184,8 @@ static inline void walt_task_dump(struct task_struct *p) bool is_32bit_thread = is_compat_thread(task_thread_info(p)); printk_deferred("Task: %.16s-%d\n", p->comm, p->pid); + SCHED_PRINT(p->state); + SCHED_PRINT(p->cpu); SCHED_PRINT(p->policy); SCHED_PRINT(p->prio); SCHED_PRINT(wts->mark_start); @@ -299,23 +301,25 @@ fixup_cumulative_runnable_avg(struct rq *rq, lockdep_assert_held(&rq->lock); if (task_rq(p) != rq) { - printk_deferred("WALT-BUG task not on rq: task_cpu=%d new_cpu=%d\n", - task_cpu(p), cpu_of(rq)); + printk_deferred("WALT-BUG on CPU %d task %s(%d) not on rq %d", + raw_smp_processor_id(), p->comm, p->pid, rq->cpu); walt_task_dump(p); SCHED_BUG_ON(1); } if (cumulative_runnable_avg_scaled < 0) { - printk_deferred("WALT-BUG task ds=%llu is higher than cra=%llu\n", - wts->demand_scaled, stats->cumulative_runnable_avg_scaled); + printk_deferred("WALT-BUG on CPU %d task ds=%llu is higher than cra=%llu\n", + raw_smp_processor_id(), wts->demand_scaled, + stats->cumulative_runnable_avg_scaled); walt_task_dump(p); SCHED_BUG_ON(1); } stats->cumulative_runnable_avg_scaled = (u64)cumulative_runnable_avg_scaled; if (pred_demands_sum_scaled < 0) { - printk_deferred("WALT-BUG task pds=%llu is higher than pds_sum=%llu\n", - wts->pred_demand_scaled, stats->pred_demands_sum_scaled); + printk_deferred("WALT-BUG on CPU %d task pds=%llu is higher than pds_sum=%llu\n", + raw_smp_processor_id(), wts->pred_demand_scaled, + stats->pred_demands_sum_scaled); walt_task_dump(p); SCHED_BUG_ON(1); } @@ -979,6 +983,12 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) lockdep_assert_held(&src_rq->lock); lockdep_assert_held(&dest_rq->lock); + if (task_rq(p) != src_rq) { + printk_deferred("WALT-BUG on CPU %d task %s(%d) not on src_rq %d", + raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); + SCHED_BUG_ON(1); + } + walt_update_task_ravg(task_rq(p)->curr, task_rq(p), TASK_UPDATE, wallclock, 0); From ddb1c85d6042db5dcf37955619c88722135ac6f8 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 4 Jun 2021 15:09:14 -0700 Subject: [PATCH 109/346] sched: walt: Fix waltgov_next_freq format specifiers Certain unsigned int values are being specified as %lu instead of %u and it is leading to incorrectly displayed values. Change-Id: I26f65ef24a89c277e2da0f88c1e3d683ef545057 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 519ba4fd8aad..8b8780a2d97b 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -706,7 +706,7 @@ TRACE_EVENT(waltgov_next_freq, __entry->cached_raw_freq = cached_raw_freq; __entry->need_freq_update = need_freq_update; ), - TP_printk("cpu=%u util=%lu max=%lu freq=%u policy_min_freq=%lu policy_max_freq=%lu cached_raw_freq=%lu need_update=%d", + TP_printk("cpu=%u util=%lu max=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d", __entry->cpu, __entry->util, __entry->max, From efabb036d0068aedc9cb379eb8d2821b61f2dcfc Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Tue, 27 Apr 2021 18:46:40 -0700 Subject: [PATCH 110/346] sched: Disable LPM if CONSERVATIVE_BOOST is enabled Disable LPM for hysteresis time if CONSERVATIVE_BOOST is enabled. Change-Id: If88513b664c00ecb2cd48961b907bec719e822a9 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/Kconfig | 9 +++++++++ kernel/sched/walt/sched_avg.c | 17 +++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/Kconfig b/kernel/sched/walt/Kconfig index b9b2422f26f0..4157dffa013c 100644 --- a/kernel/sched/walt/Kconfig +++ b/kernel/sched/walt/Kconfig @@ -29,4 +29,13 @@ config SCHED_WALT_DEBUG This module also used to crash the system to catch issues in scenarios like RT throttling and sleeping while in atomic context etc. + +config SCHED_CONSERVATIVE_BOOST_LPM_BIAS + bool "Enable LPM bias if conservative boost is enabled" + default n + help + This feature will allow the scheduler to disable low power + modes on a cpu if conservative boost is active. The cpu + will not enter low power mode for a hysteresis time period, + which can be configured from userspace. endmenu diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 426bc30303cb..c8287e06b974 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -147,6 +147,7 @@ static inline void update_busy_hyst_end_time(int cpu, bool dequeue, u64 agg_hyst_time, total_util = 0; bool util_load_trigger = false; int i; + bool hyst_trigger, coloc_trigger; if (!per_cpu(hyst_time, cpu) && !per_cpu(coloc_hyst_time, cpu) && !per_cpu(util_hyst_time, cpu)) @@ -172,12 +173,16 @@ static inline void update_busy_hyst_end_time(int cpu, bool dequeue, } } - agg_hyst_time = max(max((nr_run_trigger || load_trigger) ? - per_cpu(hyst_time, cpu) : 0, - (nr_run_trigger || coloc_load_trigger) ? - per_cpu(coloc_hyst_time, cpu) : 0), - (util_load_trigger) ? - per_cpu(util_hyst_time, cpu) : 0); + coloc_trigger = nr_run_trigger || coloc_load_trigger; +#ifdef CONFIG_SCHED_CONSERVATIVE_BOOST_LPM_BIAS + hyst_trigger = nr_run_trigger || load_trigger || (sched_boost_type == CONSERVATIVE_BOOST); +#else + hyst_trigger = nr_run_trigger || load_trigger; +#endif + + agg_hyst_time = max(max(hyst_trigger ? per_cpu(hyst_time, cpu) : 0, + coloc_trigger ? per_cpu(coloc_hyst_time, cpu) : 0), + util_load_trigger ? per_cpu(util_hyst_time, cpu) : 0); if (agg_hyst_time) { atomic64_set(&per_cpu(busy_hyst_end_time, cpu), From 242434fe46d4d4e03569afe311da98a017b14eef Mon Sep 17 00:00:00 2001 From: Lingutla Chandrasekhar Date: Wed, 26 May 2021 12:23:22 +0530 Subject: [PATCH 111/346] sched: walt: support dynamic trace event registration Some trace events might not need to be enabled always, but needed for debug purpose. It is not optimal to register trace event, which is used for debug purpose only. So registering such trace events might add extra overhead in hotpaths. So add control knob for registering such trace events to trace points, which can be enabled whenever needed them. And can be unregister when not needed. To enable such event registration, write: echo 1 > /proc/sys/walt/sched_enable_tp And register sched_overutilized trace event with dynamic support, this event is copied from Qais's debug change in chrome project: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/2128628 Change-Id: I8983e31d405a635b38c7c985b56c2f564855d3e6 Signed-off-by: Lingutla Chandrasekhar --- kernel/sched/walt/Makefile | 2 +- kernel/sched/walt/sysctl.c | 9 +++++++ kernel/sched/walt/trace.h | 22 +++++++++++++++ kernel/sched/walt/walt.h | 3 +++ kernel/sched/walt/walt_tp.c | 54 +++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 kernel/sched/walt/walt_tp.c diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile index 870d9b2fcfe3..ed35366108ed 100644 --- a/kernel/sched/walt/Makefile +++ b/kernel/sched/walt/Makefile @@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n KCSAN_SANITIZE := n obj-$(CONFIG_SCHED_WALT) += sched-walt.o -sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index a005b3004413..578af892a5a8 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -773,6 +773,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_INT_MAX, }, + { + .procname = "sched_enable_tp", + .data = &sysctl_sched_dynamic_tp_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_dynamic_tp_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, { } }; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 8b8780a2d97b..dad667d8a2a9 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1186,6 +1186,28 @@ DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_preempt, TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), TP_ARGS(p, wts, limit)); +#define SPAN_SIZE (NR_CPUS/4) + +TRACE_EVENT(sched_overutilized, + + TP_PROTO(int overutilized, char *span), + + TP_ARGS(overutilized, span), + + TP_STRUCT__entry( + __field(int, overutilized) + __array(char, span, SPAN_SIZE) + ), + + TP_fast_assign( + __entry->overutilized = overutilized; + strscpy(__entry->span, span, SPAN_SIZE); + ), + + TP_printk("overutilized=%d span=0x%s", + __entry->overutilized, __entry->span) +); + #endif /* _TRACE_WALT_H */ #undef TRACE_INCLUDE_PATH diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index b0dc1690ae15..72e7616487f3 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -228,6 +228,9 @@ extern unsigned int sysctl_sched_many_wakeup_threshold; extern unsigned int sysctl_walt_rtg_cfs_boost_prio; extern __read_mostly unsigned int sysctl_sched_force_lb_enable; extern const int sched_user_hint_max; +extern unsigned int sysctl_sched_dynamic_tp_enable; +extern int sched_dynamic_tp_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); extern struct list_head cluster_head; #define for_each_sched_cluster(cluster) \ diff --git a/kernel/sched/walt/walt_tp.c b/kernel/sched/walt/walt_tp.c new file mode 100644 index 000000000000..f809128fddd0 --- /dev/null +++ b/kernel/sched/walt/walt_tp.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include "trace.h" + +unsigned int sysctl_sched_dynamic_tp_enable; + +static void sched_overutilized(void *data, struct root_domain *rd, + bool overutilized) +{ + if (trace_sched_overutilized_enabled()) { + char span[SPAN_SIZE]; + + cpumap_print_to_pagebuf(false, span, sched_trace_rd_span(rd)); + trace_sched_overutilized(overutilized, span); + } +} + +static void walt_register_dynamic_tp_events(void) +{ + register_trace_sched_overutilized_tp(sched_overutilized, NULL); +} + +static void walt_unregister_dynamic_tp_events(void) +{ + unregister_trace_sched_overutilized_tp(sched_overutilized, NULL); +} + +int sched_dynamic_tp_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + static DEFINE_MUTEX(mutex); + int ret = 0, *val = (unsigned int *)table->data; + unsigned int old_val; + + mutex_lock(&mutex); + old_val = sysctl_sched_dynamic_tp_enable; + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write || (old_val == sysctl_sched_dynamic_tp_enable)) + goto done; + + if (*val) + walt_register_dynamic_tp_events(); + else + walt_unregister_dynamic_tp_events(); +done: + mutex_unlock(&mutex); + return ret; +} From b7c86b3ea2aa9bbc3e57e9a020bead1acf165dbb Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:49:53 -0700 Subject: [PATCH 112/346] sched: walt: Move perf counter trace to dynamic trace point Currently the sched_switch_ctrs trace event is dependent on debugfs to get enabled, and not compatible with all builds. Move the sched_switch_ctrs to dynamic trace point registration under walt dynamic tp, as the trace event has considerable overhead and need to enabled on need basis. Change-Id: I6d9eb328ecf14a15e27d67dd5c1a46b0383aa860 Signed-off-by: Lingutla Chandrasekhar Signed-off-by: Stephen Dickey Signed-off-by: Sai Harshini Nimmala --- arch/arm64/kernel/Makefile | 2 +- kernel/sched/walt/perf_trace_counters.h | 105 ++++++++++++++++++++++++ kernel/sched/walt/walt_tp.c | 90 ++++++++++++++++++++ 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 kernel/sched/walt/perf_trace_counters.h diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fa7981d0d917..056c9d944d89 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -45,7 +45,7 @@ obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o -obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o perf_trace_user.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_CPU_PM) += sleep.o suspend.o obj-$(CONFIG_CPU_IDLE) += cpuidle.o diff --git a/kernel/sched/walt/perf_trace_counters.h b/kernel/sched/walt/perf_trace_counters.h new file mode 100644 index 000000000000..73d589a5628e --- /dev/null +++ b/kernel/sched/walt/perf_trace_counters.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2013-2014, 2017, 2021, The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM perf_trace_counters + +#if !defined(_PERF_TRACE_COUNTERS_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _PERF_TRACE_COUNTERS_H_ + +/* Ctr index for PMCNTENSET/CLR */ +#define CC 0x80000000 +#define C0 0x1 +#define C1 0x2 +#define C2 0x4 +#define C3 0x8 +#define C4 0x10 +#define C5 0x20 +#define C_ALL (CC | C0 | C1 | C2 | C3 | C4 | C5) +#define NUM_L1_CTRS 6 + +#include +#include +#include + +DECLARE_PER_CPU(u32, cntenset_val); +DECLARE_PER_CPU(u32, previous_ccnt); +DECLARE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts); +TRACE_EVENT(sched_switch_with_ctrs, + + TP_PROTO(pid_t prev, pid_t next), + + TP_ARGS(prev, next), + + TP_STRUCT__entry( + __field(pid_t, old_pid) + __field(pid_t, new_pid) + __field(u32, cctr) + __field(u32, ctr0) + __field(u32, ctr1) + __field(u32, ctr2) + __field(u32, ctr3) + __field(u32, ctr4) + __field(u32, ctr5) + ), + + TP_fast_assign( + u32 cpu = smp_processor_id(); + u32 i; + u32 cnten_val; + u32 total_ccnt = 0; + u32 total_cnt = 0; + u32 delta_l1_cnts[NUM_L1_CTRS]; + + __entry->old_pid = prev; + __entry->new_pid = next; + + cnten_val = per_cpu(cntenset_val, cpu); + + if (cnten_val & CC) { + /* Read value */ + total_ccnt = read_sysreg(pmccntr_el0); + __entry->cctr = total_ccnt - + per_cpu(previous_ccnt, cpu); + per_cpu(previous_ccnt, cpu) = total_ccnt; + } + for (i = 0; i < NUM_L1_CTRS; i++) { + if (cnten_val & (1 << i)) { + /* Select */ + write_sysreg(i, pmselr_el0); + isb(); + /* Read value */ + total_cnt = read_sysreg(pmxevcntr_el0); + delta_l1_cnts[i] = total_cnt - + per_cpu(previous_l1_cnts[i], cpu); + per_cpu(previous_l1_cnts[i], cpu) = + total_cnt; + } else + delta_l1_cnts[i] = 0; + } + + __entry->ctr0 = delta_l1_cnts[0]; + __entry->ctr1 = delta_l1_cnts[1]; + __entry->ctr2 = delta_l1_cnts[2]; + __entry->ctr3 = delta_l1_cnts[3]; + __entry->ctr4 = delta_l1_cnts[4]; + __entry->ctr5 = delta_l1_cnts[5]; + ), + + TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %u, CTR0: %u, CTR1: %u, CTR2: %u, CTR3: %u, CTR4: %u, CTR5: %u", + __entry->old_pid, __entry->new_pid, + __entry->cctr, + __entry->ctr0, __entry->ctr1, + __entry->ctr2, __entry->ctr3, + __entry->ctr4, __entry->ctr5) +); + +#endif +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../kernel/sched/walt + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE perf_trace_counters +#include diff --git a/kernel/sched/walt/walt_tp.c b/kernel/sched/walt/walt_tp.c index f809128fddd0..17708d91407e 100644 --- a/kernel/sched/walt/walt_tp.c +++ b/kernel/sched/walt/walt_tp.c @@ -3,12 +3,100 @@ * Copyright (c) 2021, The Linux Foundation. All rights reserved. */ +#include #include #include #include "trace.h" +#define CREATE_TRACE_POINTS +#include "perf_trace_counters.h" unsigned int sysctl_sched_dynamic_tp_enable; +#define USE_CPUHP_STATE CPUHP_AP_ONLINE_DYN + +DEFINE_PER_CPU(u32, cntenset_val); +DEFINE_PER_CPU(u32, previous_ccnt); +DEFINE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts); +DEFINE_PER_CPU(u32, old_pid); +DEFINE_PER_CPU(u32, hotplug_flag); + +static int tracectr_cpu_hotplug_coming_up(unsigned int cpu) +{ + per_cpu(hotplug_flag, cpu) = 1; + + return 0; +} + +static void setup_prev_cnts(u32 cpu, u32 cnten_val) +{ + int i; + + if (cnten_val & CC) + per_cpu(previous_ccnt, cpu) = + read_sysreg(pmccntr_el0); + + for (i = 0; i < NUM_L1_CTRS; i++) { + if (cnten_val & (1 << i)) { + /* Select */ + write_sysreg(i, pmselr_el0); + isb(); + /* Read value */ + per_cpu(previous_l1_cnts[i], cpu) = + read_sysreg(pmxevcntr_el0); + } + } +} + +void tracectr_notifier(void *ignore, bool preempt, + struct task_struct *prev, struct task_struct *next) +{ + u32 cnten_val; + int current_pid; + u32 cpu = task_cpu(next); + + if (!trace_sched_switch_with_ctrs_enabled()) + return; + + current_pid = next->pid; + if (per_cpu(old_pid, cpu) != -1) { + cnten_val = read_sysreg(pmcntenset_el0); + per_cpu(cntenset_val, cpu) = cnten_val; + /* Disable all the counters that were enabled */ + write_sysreg(cnten_val, pmcntenclr_el0); + + if (per_cpu(hotplug_flag, cpu) == 1) { + per_cpu(hotplug_flag, cpu) = 0; + setup_prev_cnts(cpu, cnten_val); + } else { + trace_sched_switch_with_ctrs(per_cpu(old_pid, cpu), + current_pid); + } + + /* Enable all the counters that were disabled */ + write_sysreg(cnten_val, pmcntenset_el0); + } + per_cpu(old_pid, cpu) = current_pid; +} + +static void register_sched_switch_ctrs(void) +{ + int cpu, rc; + + for_each_possible_cpu(cpu) + per_cpu(old_pid, cpu) = -1; + + rc = cpuhp_setup_state_nocalls(USE_CPUHP_STATE, "tracectr_cpu_hotplug", + tracectr_cpu_hotplug_coming_up, NULL); + if (rc >= 0) + register_trace_sched_switch(tracectr_notifier, NULL); +} + +static void unregister_sched_switch_ctrs(void) +{ + unregister_trace_sched_switch(tracectr_notifier, NULL); + cpuhp_remove_state_nocalls(USE_CPUHP_STATE); +} + static void sched_overutilized(void *data, struct root_domain *rd, bool overutilized) { @@ -23,11 +111,13 @@ static void sched_overutilized(void *data, struct root_domain *rd, static void walt_register_dynamic_tp_events(void) { register_trace_sched_overutilized_tp(sched_overutilized, NULL); + register_sched_switch_ctrs(); } static void walt_unregister_dynamic_tp_events(void) { unregister_trace_sched_overutilized_tp(sched_overutilized, NULL); + unregister_sched_switch_ctrs(); } int sched_dynamic_tp_handler(struct ctl_table *table, int write, From 1156ad0e3bf12da6eff50ea7c73e0359f2b952f1 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 25 May 2021 15:58:11 -0700 Subject: [PATCH 113/346] sched/walt: Track best target cpu for each cluster Currently, energy evaluation occurs only on one target cpu chosen across all CPU clusters. Change the algorithm to track target cpus for each cluster and subsequently perform energy calculations on these candidates. For a cluster, if both an idle cpu and a busy cpu is found, prefer idle cpu. That also means, once idle if found there is no need to visit upcoming busy cpus as we traverse the cpus in a cluster. Change-Id: Idd832748cdd01e0995783b1959332acd66e94cd6 Signed-off-by: Sai Harshini Nimmala Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/trace.h | 20 ++++---- kernel/sched/walt/walt_cfs.c | 99 ++++++++++++++++++++---------------- 2 files changed, 64 insertions(+), 55 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index dad667d8a2a9..7a4d8ace0d30 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1022,12 +1022,13 @@ TRACE_EVENT(sched_find_best_target, TP_PROTO(struct task_struct *tsk, unsigned long min_util, int start_cpu, - int best_idle, int most_spare_cap, int target, + unsigned long candidates, + int most_spare_cap, int order_index, int end_index, int skip, bool running), - TP_ARGS(tsk, min_util, start_cpu, - best_idle, most_spare_cap, target, + TP_ARGS(tsk, min_util, start_cpu, candidates, + most_spare_cap, order_index, end_index, skip, running), TP_STRUCT__entry( @@ -1035,9 +1036,8 @@ TRACE_EVENT(sched_find_best_target, __field(pid_t, pid) __field(unsigned long, min_util) __field(int, start_cpu) - __field(int, best_idle) + __field(unsigned long, candidates) __field(int, most_spare_cap) - __field(int, target) __field(int, order_index) __field(int, end_index) __field(int, skip) @@ -1049,21 +1049,19 @@ TRACE_EVENT(sched_find_best_target, __entry->pid = tsk->pid; __entry->min_util = min_util; __entry->start_cpu = start_cpu; - __entry->best_idle = best_idle; - __entry->most_spare_cap = most_spare_cap; - __entry->target = target; + __entry->candidates = candidates; + __entry->most_spare_cap = most_spare_cap; __entry->order_index = order_index; __entry->end_index = end_index; __entry->skip = skip; __entry->running = running; ), - TP_printk("pid=%d comm=%s start_cpu=%d best_idle=%d most_spare_cap=%d target=%d order_index=%d end_index=%d skip=%d running=%d", + TP_printk("pid=%d comm=%s start_cpu=%d candidates=%#lx most_spare_cap=%d order_index=%d end_index=%d skip=%d running=%d", __entry->pid, __entry->comm, __entry->start_cpu, - __entry->best_idle, + __entry->candidates, __entry->most_spare_cap, - __entry->target, __entry->order_index, __entry->end_index, __entry->skip, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 02d1c891242a..a6179725fdeb 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -164,8 +164,6 @@ static void walt_find_best_target(struct sched_domain *sd, long target_max_spare_cap = 0; unsigned long best_idle_cuml_util = ULONG_MAX; unsigned int min_exit_latency = UINT_MAX; - int best_idle_cpu = -1; - int target_cpu = -1; int i, start_cpu; long spare_wake_cap, most_spare_wake_cap = 0; int most_spare_cap_cpu = -1; @@ -202,13 +200,19 @@ static void walt_find_best_target(struct sched_domain *sd, cpu_active(prev_cpu) && cpu_online(prev_cpu) && available_idle_cpu(prev_cpu) && cpumask_test_cpu(prev_cpu, p->cpus_ptr)) { - target_cpu = prev_cpu; fbt_env->fastpath = PREV_CPU_FASTPATH; - cpumask_set_cpu(target_cpu, candidates); + cpumask_set_cpu(prev_cpu, candidates); goto out; } for (cluster = 0; cluster < num_sched_clusters; cluster++) { + int best_idle_cpu_cluster = -1; + int target_cpu_cluster = -1; + + target_max_spare_cap = 0; + min_exit_latency = INT_MAX; + best_idle_cuml_util = ULONG_MAX; + cpumask_and(&visit_cpus, &p->cpus_mask, &cpu_array[order_index][cluster]); for_each_cpu(i, &visit_cpus) { @@ -293,17 +297,21 @@ static void walt_find_best_target(struct sched_domain *sd, new_util_cuml = cpu_util_cum(i); if (min_exit_latency == idle_exit_latency && - (best_idle_cpu == prev_cpu || + (best_idle_cpu_cluster == prev_cpu || (i != prev_cpu && new_util_cuml > best_idle_cuml_util))) continue; min_exit_latency = idle_exit_latency; best_idle_cuml_util = new_util_cuml; - best_idle_cpu = i; + best_idle_cpu_cluster = i; continue; } + /* skip visiting any more busy if idle was found */ + if (best_idle_cpu_cluster != -1) + continue; + if (per_task_boost(cpu_rq(i)->curr) == TASK_BOOST_STRICT_MAX) continue; @@ -338,47 +346,40 @@ static void walt_find_best_target(struct sched_domain *sd, target_max_spare_cap = spare_cap; target_nr_rtg_high_prio = walt_nr_rtg_high_prio(i); - target_cpu = i; + target_cpu_cluster = i; } - if (best_idle_cpu != -1) - break; + if (best_idle_cpu_cluster != -1) + cpumask_set_cpu(best_idle_cpu_cluster, candidates); + else if (target_cpu_cluster != -1) + cpumask_set_cpu(target_cpu_cluster, candidates); - if ((cluster >= end_index) && (target_cpu != -1) && - walt_target_ok(target_cpu, order_index)) + if ((cluster >= end_index) && (!cpumask_empty(candidates)) && + walt_target_ok(target_cpu_cluster, order_index)) break; if (most_spare_cap_cpu != -1 && cluster >= stop_index) break; } - if (best_idle_cpu != -1) - target_cpu = -1; /* - * We set both idle and target as long as they are valid CPUs. + * We have set idle or target as long as they are valid CPUs. * If we don't find either, then we fallback to most_spare_cap, * If we don't find most spare cap, we fallback to prev_cpu, * provided that the prev_cpu is active. * If the prev_cpu is not active, we fallback to active_candidate. */ - if (unlikely(target_cpu == -1)) { - if (best_idle_cpu != -1) - target_cpu = best_idle_cpu; - else if (most_spare_cap_cpu != -1) - target_cpu = most_spare_cap_cpu; + if (unlikely(cpumask_empty(candidates))) { + if (most_spare_cap_cpu != -1) + cpumask_set_cpu(most_spare_cap_cpu, candidates); else if (!cpu_active(prev_cpu)) - target_cpu = active_candidate; + cpumask_set_cpu(active_candidate, candidates); } - if (target_cpu != -1) - cpumask_set_cpu(target_cpu, candidates); - if (best_idle_cpu != -1 && target_cpu != best_idle_cpu) - cpumask_set_cpu(best_idle_cpu, candidates); out: - trace_sched_find_best_target(p, min_util, start_cpu, - best_idle_cpu, most_spare_cap_cpu, - target_cpu, order_index, end_index, + trace_sched_find_best_target(p, min_util, start_cpu, cpumask_bits(candidates)[0], + most_spare_cap_cpu, order_index, end_index, fbt_env->skip_cpu, task_on_rq_queued(p)); } @@ -517,6 +518,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int task_boost = per_task_boost(p); bool uclamp_boost = walt_uclamp_boosted(p); int start_cpu, order_index, end_index; + int max_cap_cpu = -1; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && cpumask_test_cpu(prev_cpu, &p->cpus_mask)) @@ -569,28 +571,38 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, /* Bail out if no candidate was found. */ weight = cpumask_weight(candidates); - if (!weight) - goto unlock; + if (!weight) { + rcu_read_unlock(); + goto done; + } - /* If there is only one sensible candidate, select it now. */ - cpu = cpumask_first(candidates); - if (weight == 1 && (available_idle_cpu(cpu) || cpu == prev_cpu)) { - best_energy_cpu = cpu; - goto unlock; + max_cap_cpu = cpumask_first(candidates); + if (weight == 1) { + if (available_idle_cpu(max_cap_cpu) || max_cap_cpu == prev_cpu) { + best_energy_cpu = max_cap_cpu; + rcu_read_unlock(); + goto done; + } + } + + for_each_cpu(cpu, candidates) { + if (capacity_orig_of(max_cap_cpu) < capacity_orig_of(cpu)) + max_cap_cpu = cpu; + } + + if (task_placement_boost_enabled(p) || + task_boost || uclamp_boost || + !task_fits_max(p, prev_cpu) || !cpu_active(prev_cpu) || + walt_get_rtg_status(p)) { + best_energy_cpu = max_cap_cpu; + rcu_read_unlock(); + goto done; } if (p->state == TASK_WAKING) delta = task_util(p); - if (task_placement_boost_enabled(p) || fbt_env.need_idle || - uclamp_boost || task_boost || is_rtg || - __cpu_overutilized(prev_cpu, delta) || !task_fits_max(p, prev_cpu) || - !cpu_active(prev_cpu)) { - best_energy_cpu = cpu; - goto unlock; - } - - if (cpumask_test_cpu(prev_cpu, &p->cpus_mask)) + if (cpumask_test_cpu(prev_cpu, &p->cpus_mask) && !__cpu_overutilized(prev_cpu, delta)) prev_energy = best_energy = walt_compute_energy(p, prev_cpu, pd); else @@ -617,7 +629,6 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, } } -unlock: rcu_read_unlock(); /* From 71a5697adb52ba12aa16ce905ddba78028e98d61 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 4 May 2021 12:25:37 -0700 Subject: [PATCH 114/346] sched/walt: create walt friendly energy apis Create walt lookalikes for compute_energy (walt_pd_ compute_energy) and em_cpu_energy (walt_em_cpu_energy). Change-Id: Ib6cb0b90c1041b0d1c97729c1998c618fd60c81c Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 97 ++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index a6179725fdeb..ca43f9e01308 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -425,15 +425,104 @@ cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu) return min_t(unsigned long, util, capacity_orig_of(cpu)); } +/** + * walt_em_cpu_energy() - Estimates the energy consumed by the CPUs of a + performance domain + * @pd : performance domain for which energy has to be estimated + * @max_util : highest utilization among CPUs of the domain + * @sum_util : sum of the utilization of all CPUs in the domain + * + * This function must be used only for CPU devices. There is no validation, + * i.e. if the EM is a CPU type and has cpumask allocated. It is called from + * the scheduler code quite frequently and that is why there is not checks. + * + * Return: the sum of the energy consumed by the CPUs of the domain assuming + * a capacity state satisfying the max utilization of the domain. + */ +static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, + unsigned long max_util, unsigned long sum_util) +{ + unsigned long freq, scale_cpu; + struct em_perf_state *ps; + int i, cpu; + + if (!sum_util) + return 0; + + /* + * In order to predict the performance state, map the utilization of + * the most utilized CPU of the performance domain to a requested + * frequency, like schedutil. + */ + cpu = cpumask_first(to_cpumask(pd->cpus)); + scale_cpu = arch_scale_cpu_capacity(cpu); + ps = &pd->table[pd->nr_perf_states - 1]; + freq = map_util_freq(max_util, ps->frequency, scale_cpu); + + /* + * Find the lowest performance state of the Energy Model above the + * requested frequency. + */ + for (i = 0; i < pd->nr_perf_states; i++) { + ps = &pd->table[i]; + if (ps->frequency >= freq) + break; + } + + /* + * The capacity of a CPU in the domain at the performance state (ps) + * can be computed as: + * + * ps->freq * scale_cpu + * ps->cap = -------------------- (1) + * cpu_max_freq + * + * So, ignoring the costs of idle states (which are not available in + * the EM), the energy consumed by this CPU at that performance state + * is estimated as: + * + * ps->power * cpu_util + * cpu_nrg = -------------------- (2) + * ps->cap + * + * since 'cpu_util / ps->cap' represents its percentage of busy time. + * + * NOTE: Although the result of this computation actually is in + * units of power, it can be manipulated as an energy value + * over a scheduling period, since it is assumed to be + * constant during that interval. + * + * By injecting (1) in (2), 'cpu_nrg' can be re-expressed as a product + * of two terms: + * + * ps->power * cpu_max_freq cpu_util + * cpu_nrg = ------------------------ * --------- (3) + * ps->freq scale_cpu + * + * The first term is static, and is stored in the em_perf_state struct + * as 'ps->cost'. + * + * Since all CPUs of the domain have the same micro-architecture, they + * share the same 'ps->cost', and the same CPU capacity. Hence, the + * total energy of the domain (which is the simple sum of the energy of + * all of its CPUs) can be factorized as: + * + * ps->cost * \Sum cpu_util + * pd_nrg = ------------------------ (4) + * scale_cpu + */ + return ps->cost * sum_util / scale_cpu; +} + /* - * compute_energy(): Estimates the energy that @pd would consume if @p was + * walt_pd_compute_energy(): Estimates the energy that @pd would consume if @p was * migrated to @dst_cpu. compute_energy() predicts what will be the utilization * landscape of @pd's CPUs after the task migration, and uses the Energy Model * to compute what would be the energy if we decided to actually migrate that * task. */ static long -compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) +walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) { struct cpumask *pd_mask = perf_domain_span(pd); unsigned long max_util = 0, sum_util = 0; @@ -455,7 +544,7 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) max_util = max(max_util, cpu_util); } - return em_cpu_energy(pd->em_pd, max_util, sum_util); + return walt_em_cpu_energy(pd->em_pd, max_util, sum_util); } static inline long @@ -464,7 +553,7 @@ walt_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) long energy = 0; for (; pd; pd = pd->next) - energy += compute_energy(p, dst_cpu, pd); + energy += walt_pd_compute_energy(p, dst_cpu, pd); return energy; } From 31be7e0e1ec08264560090773f2dabf84ad8311a Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 4 May 2021 22:23:22 -0700 Subject: [PATCH 115/346] sched/walt: skip energy for uninvolved cluster A cluster that does not host prev cpu and or any of the candidates doesn't affect the energy values. Don't calculate their contributions. Change-Id: Id1b890dbf157fa27d90287202f7916c6ab102bba Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index ca43f9e01308..135b99a3af6d 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -548,12 +548,18 @@ walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *p } static inline long -walt_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) +walt_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, + cpumask_t *candidates) { long energy = 0; - for (; pd; pd = pd->next) - energy += walt_pd_compute_energy(p, dst_cpu, pd); + for (; pd; pd = pd->next) { + struct cpumask *pd_mask = perf_domain_span(pd); + + if (cpumask_intersects(candidates, pd_mask) + || cpumask_test_cpu(task_cpu(p), pd_mask)) + energy += walt_pd_compute_energy(p, dst_cpu, pd); + } return energy; } @@ -693,7 +699,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (cpumask_test_cpu(prev_cpu, &p->cpus_mask) && !__cpu_overutilized(prev_cpu, delta)) prev_energy = best_energy = - walt_compute_energy(p, prev_cpu, pd); + walt_compute_energy(p, prev_cpu, pd, candidates); else prev_energy = best_energy = ULONG_MAX; @@ -702,7 +708,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (cpu == prev_cpu) continue; - cur_energy = walt_compute_energy(p, cpu, pd); + cur_energy = walt_compute_energy(p, cpu, pd, candidates); trace_sched_compute_energy(p, cpu, cur_energy, prev_energy, best_energy, best_energy_cpu); From 11d8a970735bdb8fcd2128eb2122f2b9c064ea3b Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 25 May 2021 16:07:49 -0700 Subject: [PATCH 116/346] sched/walt: use prs for energy evaluation Currently, the code uses walt util for energy evaluation. That could be inaccurate because walt prs signals drive cpufreq. Change to using prs. Change-Id: Iea073c7963bf27ab1308950b07a2aead1c73805c Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 3 +- kernel/sched/walt/walt.h | 3 ++ kernel/sched/walt/walt_cfs.c | 74 ++++++++++++++++-------------------- 3 files changed, 37 insertions(+), 43 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4803b502e89c..bba3015024ba 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -169,8 +169,7 @@ static unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; static const unsigned int top_tasks_bitmap_size = BITS_TO_LONGS(NUM_LOAD_INDICES + 1) * sizeof(unsigned long); -static __read_mostly unsigned int walt_scale_demand_divisor; -#define scale_demand(d) ((d)/walt_scale_demand_divisor) +__read_mostly unsigned int walt_scale_demand_divisor; #define SCHED_PRINT(arg) printk_deferred("%s=%llu", #arg, arg) #define STRG(arg) #arg diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 72e7616487f3..01dba9e9865f 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -890,4 +890,7 @@ void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p); void walt_cfs_tick(struct rq *rq); void walt_lb_tick(struct rq *rq); +extern __read_mostly unsigned int walt_scale_demand_divisor; +#define scale_demand(d) ((d)/walt_scale_demand_divisor) + #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 135b99a3af6d..baf038bf6c57 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -50,6 +50,7 @@ struct find_best_target_env { int end_index; bool strict_max; int skip_cpu; + u64 prs[8]; }; /* @@ -220,8 +221,11 @@ static void walt_find_best_target(struct sched_domain *sd, unsigned long wake_util, new_util, new_util_cuml; long spare_cap; unsigned int idle_exit_latency = UINT_MAX; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; trace_sched_cpu_util(i); + /* record the prss as we visit cpus in a cluster */ + fbt_env->prs[i] = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; if (!cpu_active(i)) continue; @@ -383,46 +387,28 @@ static void walt_find_best_target(struct sched_domain *sd, fbt_env->skip_cpu, task_on_rq_queued(p)); } -static inline unsigned long -cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu) +static inline u64 +cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_dst_same_cluster, + u64 *prs) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; - unsigned long util = wrq->walt_stats.cumulative_runnable_avg_scaled; - bool queued = task_on_rq_queued(p); + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + long util = prs[cpu]; - /* - * When task is queued, - * (a) The evaluating CPU (cpu) is task's current CPU. If the - * task is migrating, discount the task contribution from the - * evaluation cpu. - * (b) The evaluating CPU (cpu) is task's current CPU. If the - * task is NOT migrating, nothing to do. The contribution is - * already present on the evaluation CPU. - * (c) The evaluating CPU (cpu) is not task's current CPU. But - * the task is migrating to the evaluating CPU. So add the - * task contribution to it. - * (d) The evaluating CPU (cpu) is neither the current CPU nor - * the destination CPU. don't care. - * - * When task is NOT queued i.e waking. Task contribution is not - * present on any CPU. - * - * (a) If the evaluating CPU is the destination CPU, add the task - * contribution. - * (b) The evaluation CPU is not the destination CPU, don't care. - */ - if (unlikely(queued)) { - if (task_cpu(p) == cpu) { - if (dst_cpu != cpu) - util = max_t(long, util - task_util(p), 0); - } else if (dst_cpu == cpu) { - util += task_util(p); + if (wts->prev_window) { + if (!prev_dst_same_cluster) { + /* intercluster migration of non rtg task - mimic fixups */ + util -= wts->prev_window_cpu[cpu]; + if (util < 0) + util = 0; + if (cpu == dst_cpu) + util += wts->prev_window; } - } else if (dst_cpu == cpu) { - util += task_util(p); + } else { + if (cpu == dst_cpu) + util += wts->demand; } - return min_t(unsigned long, util, capacity_orig_of(cpu)); + return util; } /** @@ -522,12 +508,16 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, * task. */ static long -walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) +walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, u64 *prs) { struct cpumask *pd_mask = perf_domain_span(pd); unsigned long max_util = 0, sum_util = 0; int cpu; unsigned long cpu_util; + bool prev_dst_same_cluster = false; + + if (same_cluster(task_cpu(p), dst_cpu)) + prev_dst_same_cluster = true; /* * The capacity state of CPUs of the current rd can be driven by CPUs @@ -539,17 +529,19 @@ walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *p * its pd list and will not be accounted by compute_energy(). */ for_each_cpu_and(cpu, pd_mask, cpu_online_mask) { - cpu_util = cpu_util_next_walt(cpu, p, dst_cpu); + cpu_util = cpu_util_next_walt_prs(cpu, p, dst_cpu, prev_dst_same_cluster, prs); sum_util += cpu_util; max_util = max(max_util, cpu_util); } + max_util = scale_demand(max_util); + sum_util = scale_demand(sum_util); return walt_em_cpu_energy(pd->em_pd, max_util, sum_util); } static inline long walt_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, - cpumask_t *candidates) + cpumask_t *candidates, u64 *prs) { long energy = 0; @@ -558,7 +550,7 @@ walt_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, if (cpumask_intersects(candidates, pd_mask) || cpumask_test_cpu(task_cpu(p), pd_mask)) - energy += walt_pd_compute_energy(p, dst_cpu, pd); + energy += walt_pd_compute_energy(p, dst_cpu, pd, prs); } return energy; @@ -699,7 +691,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (cpumask_test_cpu(prev_cpu, &p->cpus_mask) && !__cpu_overutilized(prev_cpu, delta)) prev_energy = best_energy = - walt_compute_energy(p, prev_cpu, pd, candidates); + walt_compute_energy(p, prev_cpu, pd, candidates, fbt_env.prs); else prev_energy = best_energy = ULONG_MAX; @@ -708,7 +700,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (cpu == prev_cpu) continue; - cur_energy = walt_compute_energy(p, cpu, pd, candidates); + cur_energy = walt_compute_energy(p, cpu, pd, candidates, fbt_env.prs); trace_sched_compute_energy(p, cpu, cur_energy, prev_energy, best_energy, best_energy_cpu); From 04f5df4cba3706965cfb9de7ec776723679f6240 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 4 May 2021 12:26:03 -0700 Subject: [PATCH 117/346] sched/walt: handle overlap between silver and gold When a task is being placed on silver and is not a filtered task and is not a small task, look for a gold for this task. Change-Id: Ic6f5a047e4f25c66f4b7be1f0b48ca49a2c5aa50 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index baf038bf6c57..ed6072969eaa 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -113,6 +113,7 @@ static inline bool walt_target_ok(int target_cpu, int order_index) (target_cpu == cpumask_first(&cpu_array[order_index][0]))); } +#define MIN_UTIL_FOR_ENERGY_EVAL 10 static void walt_get_indicies(struct task_struct *p, int *order_index, int *end_index, int task_boost, bool uclamp_boost) { @@ -148,6 +149,13 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, } *order_index = i; + + if (*order_index == 0 && + (task_util(p) >= MIN_UTIL_FOR_ENERGY_EVAL) && + !walt_get_rtg_status(p) && + !(sched_boost_type == CONSERVATIVE_BOOST && task_sched_boost(p)) + ) + *end_index = 1; } enum fastpaths { From b053fa9fdcde9e108061aab9c406b5fd560e1438 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 3 Jun 2021 23:07:00 -0700 Subject: [PATCH 118/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I9e3d2f76f07c2b2f2a5d4a6d9831489251de1455 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index ed6072969eaa..62cc366b89d6 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -115,7 +115,8 @@ static inline bool walt_target_ok(int target_cpu, int order_index) #define MIN_UTIL_FOR_ENERGY_EVAL 10 static void walt_get_indicies(struct task_struct *p, int *order_index, - int *end_index, int task_boost, bool uclamp_boost) + int *end_index, int per_task_boost, bool is_uclamp_boosted, + bool *energy_eval_needed) { int i = 0; @@ -125,12 +126,14 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, if (num_sched_clusters <= 1) return; - if (task_boost > TASK_BOOST_ON_MID) { + if (per_task_boost > TASK_BOOST_ON_MID) { *order_index = num_sched_clusters - 1; + *energy_eval_needed = false; return; } if (is_full_throttle_boost()) { + *energy_eval_needed = false; *order_index = num_sched_clusters - 1; if ((*order_index > 1) && task_demand_fits(p, cpumask_first(&cpu_array[*order_index][1]))) @@ -138,10 +141,12 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, return; } - if (uclamp_boost || task_boost || + if (is_uclamp_boosted || per_task_boost || task_boost_policy(p) == SCHED_BOOST_ON_BIG || - walt_task_skip_min_cpu(p)) + walt_task_skip_min_cpu(p)) { + *energy_eval_needed = false; *order_index = 1; + } for (i = *order_index ; i < num_sched_clusters - 1; i++) { if (task_demand_fits(p, cpumask_first(&cpu_array[i][0]))) @@ -152,10 +157,14 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, if (*order_index == 0 && (task_util(p) >= MIN_UTIL_FOR_ENERGY_EVAL) && + !(p->in_iowait && task_in_related_thread_group(p)) && !walt_get_rtg_status(p) && !(sched_boost_type == CONSERVATIVE_BOOST && task_sched_boost(p)) ) *end_index = 1; + + if (p->in_iowait && task_in_related_thread_group(p)) + *energy_eval_needed = false; } enum fastpaths { @@ -614,6 +623,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, bool uclamp_boost = walt_uclamp_boosted(p); int start_cpu, order_index, end_index; int max_cap_cpu = -1; + bool energy_eval_needed = true; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && cpumask_test_cpu(prev_cpu, &p->cpus_mask)) @@ -622,7 +632,8 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (unlikely(!cpu_array)) return -EPERM; - walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost); + walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost, + &energy_eval_needed); start_cpu = cpumask_first(&cpu_array[order_index][0]); is_rtg = task_in_related_thread_group(p); @@ -685,10 +696,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, max_cap_cpu = cpu; } - if (task_placement_boost_enabled(p) || - task_boost || uclamp_boost || - !task_fits_max(p, prev_cpu) || !cpu_active(prev_cpu) || - walt_get_rtg_status(p)) { + if (!energy_eval_needed) { best_energy_cpu = max_cap_cpu; rcu_read_unlock(); goto done; From 808c1b4e14eeddb278dd1ebb6f4a1208d3a3adae Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Mon, 24 May 2021 23:50:39 -0700 Subject: [PATCH 119/346] sched/walt: initiate initialization after topology The proper kernel provides tracehooks to indicate when its done initializing topology. Use that hook to simplify walt initialization. Change-Id: Ifa7540f2f65bf469c1f8007bb15d5e1317475929 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 39 +++++---------------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index bba3015024ba..8d0529bc677e 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -16,6 +16,7 @@ #include #include +#include #include #include "walt.h" @@ -4214,38 +4215,9 @@ static void walt_init(void) } } -static bool are_cpufreq_policies_available(void) -{ - int cpu; - - for_each_possible_cpu(cpu) { - if (!cpufreq_cpu_get_raw(cpu)) - return false; - } - - return true; -} - -struct work_struct walt_cpufreq_policy_work; - -static int walt_cpufreq_notifier_cb(struct notifier_block *nb, - unsigned long action, - void *data) -{ - if (are_cpufreq_policies_available()) - schedule_work(&walt_cpufreq_policy_work); - - return 0; -} - -static struct notifier_block walt_cpufreq_notifier_block = { - .notifier_call = walt_cpufreq_notifier_cb -}; - -static void walt_wq_cpufreq_policy_update(struct work_struct *work) +static void android_vh_update_topology_flags_workfn(void *unused, void *unused2) { walt_init(); - cpufreq_unregister_notifier(&walt_cpufreq_notifier_block, CPUFREQ_POLICY_NOTIFIER); } #define WALT_VENDOR_DATA_SIZE_TEST(wstruct, kstruct) \ @@ -4259,11 +4231,10 @@ static int walt_module_init(void) WALT_VENDOR_DATA_SIZE_TEST(struct walt_rq, struct rq); WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_group, struct task_group); - INIT_WORK(&walt_cpufreq_policy_work, walt_wq_cpufreq_policy_update); - cpufreq_register_notifier(&walt_cpufreq_notifier_block, - CPUFREQ_POLICY_NOTIFIER); + register_trace_android_vh_update_topology_flags_workfn( + android_vh_update_topology_flags_workfn, NULL); - if (are_cpufreq_policies_available()) + if (topology_update_done) walt_init(); return 0; From b1361e13a70bdda08c7f47d99123dc8493c4e096 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 8 Jun 2021 10:50:27 -0700 Subject: [PATCH 120/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ib4c5091436e0536bcb2ee63e9a31d9b1dd51dbe9 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 1 + kernel/sched/walt/walt.h | 5 ++- kernel/sched/walt/walt_cfs.c | 71 +++++++++++++++++++++++++++--------- 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 8d0529bc677e..bdf6e804d5bd 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2585,6 +2585,7 @@ static void walt_update_cluster_topology(void) init_cpu_array(); build_cpu_array(); + create_util_to_cost(); walt_clusters_parsed = true; } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 01dba9e9865f..00adeed30276 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -134,12 +134,12 @@ struct walt_sched_cluster { unsigned int max_possible_freq; unsigned int max_freq; u64 aggr_grp_load; + + u16 util_to_cost[1024]; }; extern struct walt_sched_cluster *sched_cluster[WALT_NR_CPUS]; -extern struct walt_sched_cluster *rq_cluster(struct rq *rq); - /*END SCHED.H PORT*/ extern int num_sched_clusters; @@ -893,4 +893,5 @@ void walt_lb_tick(struct rq *rq); extern __read_mostly unsigned int walt_scale_demand_divisor; #define scale_demand(d) ((d)/walt_scale_demand_divisor) +void create_util_to_cost(void); #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 62cc366b89d6..526916280e4e 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -11,6 +11,46 @@ #include <../../../drivers/android/binder_internal.h> #include "../../../drivers/android/binder_trace.h" +static void create_util_to_cost_pd(struct em_perf_domain *pd) +{ + int util, cpu = cpumask_first(to_cpumask(pd->cpus)); + unsigned long fmax; + unsigned long scale_cpu; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + + fmax = (u64)pd->table[pd->nr_perf_states - 1].frequency; + scale_cpu = arch_scale_cpu_capacity(cpu); + + for (util = 0; util < 1024; util++) { + int j; + + int f = (fmax * util) / scale_cpu; + struct em_perf_state *ps = &pd->table[0]; + + for (j = 0; j < pd->nr_perf_states; j++) { + ps = &pd->table[j]; + if (ps->frequency >= f) + break; + } + cluster->util_to_cost[util] = ps->cost; + } +} + +void create_util_to_cost(void) +{ + struct perf_domain *pd; + struct root_domain *rd = cpu_rq(smp_processor_id())->rd; + + rcu_read_lock(); + pd = rcu_dereference(rd->pd); + for (; pd; pd = pd->next) + create_util_to_cost_pd(pd->em_pd); + rcu_read_unlock(); +} + +DECLARE_PER_CPU(unsigned long, gov_last_util); + /* Migration margins */ unsigned int sched_capacity_margin_up[WALT_NR_CPUS] = { [0 ... WALT_NR_CPUS-1] = 1078 /* ~5% margin */ @@ -445,32 +485,23 @@ cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_ds static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, unsigned long max_util, unsigned long sum_util) { - unsigned long freq, scale_cpu; - struct em_perf_state *ps; - int i, cpu; + unsigned long scale_cpu; + int cpu; + struct walt_rq *wrq; if (!sum_util) return 0; /* - * In order to predict the performance state, map the utilization of - * the most utilized CPU of the performance domain to a requested - * frequency, like schedutil. + * In order to predict the capacity state, map the utilization of the + * most utilized CPU of the performance domain to a requested frequency, + * like schedutil. */ cpu = cpumask_first(to_cpumask(pd->cpus)); scale_cpu = arch_scale_cpu_capacity(cpu); - ps = &pd->table[pd->nr_perf_states - 1]; - freq = map_util_freq(max_util, ps->frequency, scale_cpu); - /* - * Find the lowest performance state of the Energy Model above the - * requested frequency. - */ - for (i = 0; i < pd->nr_perf_states; i++) { - ps = &pd->table[i]; - if (ps->frequency >= freq) - break; - } + max_util = max_util + (max_util >> 2); /* account for TARGET_LOAD usually 80 */ + max_util = max(max_util, arch_scale_freq_capacity(cpu)); /* * The capacity of a CPU in the domain at the performance state (ps) @@ -514,7 +545,11 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, * pd_nrg = ------------------------ (4) * scale_cpu */ - return ps->cost * sum_util / scale_cpu; + if (max_util >= 1024) + max_util = 1023; + + wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + return wrq->cluster->util_to_cost[max_util] * sum_util / scale_cpu; } /* From edad9aa29823d028b7866299345e038b349c6190 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 6 May 2021 15:05:34 -0700 Subject: [PATCH 121/346] sched/walt: use just 3% energy criteria Currently we use ~6% (right shift by 4 is really 6.25%) as a threshold of energy saving to move a task to a cpu other than the prev_cpu. Change it to 3%. Change-Id: I0d0177bbef2c22bcf37e6da42c3ed857f986c478 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 526916280e4e..c077627ad963 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -776,7 +776,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (!(available_idle_cpu(best_energy_cpu) && walt_get_idle_exit_latency(cpu_rq(best_energy_cpu)) <= 1) && (prev_energy != ULONG_MAX) && (best_energy_cpu != prev_cpu) && - ((prev_energy - best_energy) <= prev_energy >> 4) && + ((prev_energy - best_energy) <= prev_energy >> 5) && (capacity_orig_of(prev_cpu) <= capacity_orig_of(start_cpu))) best_energy_cpu = prev_cpu; From d5046c9052990b71d4ac8d3272d1f9314da75f98 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 8 Apr 2021 00:47:13 -0700 Subject: [PATCH 122/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ieca864be1e26e39c56fd0be6be135d4085e5d8aa Signed-off-by: Sai Harshini Nimmala Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index c077627ad963..9da4028c52af 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -213,6 +213,17 @@ enum fastpaths { PREV_CPU_FASTPATH, }; +static inline bool is_complex_sibling_idle(int cpu) +{ + if (is_min_capacity_cpu(cpu)) { + if (cpu % 2) + return available_idle_cpu(cpu - 1); + return available_idle_cpu(cpu + 1); + } + + return false; +} + static void walt_find_best_target(struct sched_domain *sd, cpumask_t *candidates, struct task_struct *p, @@ -266,6 +277,8 @@ static void walt_find_best_target(struct sched_domain *sd, for (cluster = 0; cluster < num_sched_clusters; cluster++) { int best_idle_cpu_cluster = -1; int target_cpu_cluster = -1; + int this_complex_idle = 0; + int best_complex_idle = 0; target_max_spare_cap = 0; min_exit_latency = INT_MAX; @@ -349,6 +362,10 @@ static void walt_find_best_target(struct sched_domain *sd, if (available_idle_cpu(i)) { idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(i)); + this_complex_idle = is_complex_sibling_idle(i) ? 1 : 0; + + if (this_complex_idle < best_complex_idle) + continue; /* * Prefer shallowest over deeper idle state cpu, * of same capacity cpus. @@ -366,6 +383,7 @@ static void walt_find_best_target(struct sched_domain *sd, min_exit_latency = idle_exit_latency; best_idle_cuml_util = new_util_cuml; best_idle_cpu_cluster = i; + best_complex_idle = this_complex_idle; continue; } From ee77eeeb739f8ec1fdb79fc2076cb842d616d65d Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 10 Jun 2021 09:31:39 -0700 Subject: [PATCH 123/346] sched: walt: Setup complex sibling hints at init Currently, we are determining the L2 cache sibling of silvers at runtime on every iteration FBT. It is better to do this setup once, and gather this information from hardware rather than hardcoding the results. Change-Id: I06ec2f590dc9ccc2ec9161f1a60eb47d77244153 Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 36 ++++++++++++++++++++++++++++++++++++ kernel/sched/walt/walt.h | 2 +- kernel/sched/walt/walt_cfs.c | 8 ++------ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index bdf6e804d5bd..d3ee5be4ae24 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2517,6 +2517,41 @@ static void walt_get_possible_siblings(int cpuid, struct cpumask *cluster_cpus) } } +int cpu_l2_sibling[WALT_NR_CPUS] = {[0 ... WALT_NR_CPUS-1] = -1}; +static void find_cache_siblings(void) +{ + int cpu, cpu2; + struct device_node *cpu_dev, *cpu_dev2, *cpu_l2_cache_node, *cpu_l2_cache_node2; + + for_each_possible_cpu(cpu) { + cpu_dev = of_get_cpu_node(cpu, NULL); + if (!cpu_dev) + continue; + + cpu_l2_cache_node = of_parse_phandle(cpu_dev, "next-level-cache", 0); + if (!cpu_l2_cache_node) + continue; + + for_each_possible_cpu(cpu2) { + if (cpu == cpu2) + continue; + + cpu_dev2 = of_get_cpu_node(cpu2, NULL); + if (!cpu_dev2) + continue; + + cpu_l2_cache_node2 = of_parse_phandle(cpu_dev2, "next-level-cache", 0); + if (!cpu_l2_cache_node2) + continue; + + if (cpu_l2_cache_node == cpu_l2_cache_node2) { + cpu_l2_sibling[cpu] = cpu2; + break; + } + } + } +} + static void walt_update_cluster_topology(void) { struct cpumask cpus = *cpu_possible_mask; @@ -2584,6 +2619,7 @@ static void walt_update_cluster_topology(void) init_cpu_array(); build_cpu_array(); + find_cache_siblings(); create_util_to_cost(); walt_clusters_parsed = true; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 00adeed30276..5a5e643b9aa3 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -147,7 +147,7 @@ extern unsigned int sched_capacity_margin_up[WALT_NR_CPUS]; extern unsigned int sched_capacity_margin_down[WALT_NR_CPUS]; extern cpumask_t asym_cap_sibling_cpus; extern cpumask_t __read_mostly **cpu_array; - +extern int cpu_l2_sibling[WALT_NR_CPUS]; extern void sched_update_nr_prod(int cpu, int enq); extern unsigned int walt_big_tasks(int cpu); extern void walt_rotate_work_init(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 9da4028c52af..1e65919138a7 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -215,12 +215,8 @@ enum fastpaths { static inline bool is_complex_sibling_idle(int cpu) { - if (is_min_capacity_cpu(cpu)) { - if (cpu % 2) - return available_idle_cpu(cpu - 1); - return available_idle_cpu(cpu + 1); - } - + if (cpu_l2_sibling[cpu] != -1) + return available_idle_cpu(cpu_l2_sibling[cpu]); return false; } From 32eccdc2bfe2b1626e6d4a9dacb0db1f500d6bdb Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Tue, 15 Jun 2021 00:40:16 -0700 Subject: [PATCH 124/346] sched/walt: consolidate multiple rcu unlock statements rcu unlock is being called in multiple places in walt_find_best_target(). Consolidate these calls. Change-Id: I86701bc975fcc7d68b6faa95b952b9dd40c2f8a9 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_cfs.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 1e65919138a7..2a06a0c3c19b 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -726,17 +726,14 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, /* Bail out if no candidate was found. */ weight = cpumask_weight(candidates); - if (!weight) { - rcu_read_unlock(); - goto done; - } + if (!weight) + goto unlock; max_cap_cpu = cpumask_first(candidates); if (weight == 1) { if (available_idle_cpu(max_cap_cpu) || max_cap_cpu == prev_cpu) { best_energy_cpu = max_cap_cpu; - rcu_read_unlock(); - goto done; + goto unlock; } } @@ -747,8 +744,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (!energy_eval_needed) { best_energy_cpu = max_cap_cpu; - rcu_read_unlock(); - goto done; + goto unlock; } if (p->state == TASK_WAKING) @@ -781,8 +777,6 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, } } - rcu_read_unlock(); - /* * Pick the prev CPU, if best energy CPU can't saves at least 6% of * the energy used by prev_cpu. @@ -794,6 +788,9 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, (capacity_orig_of(prev_cpu) <= capacity_orig_of(start_cpu))) best_energy_cpu = prev_cpu; +unlock: + rcu_read_unlock(); + done: trace_sched_task_util(p, cpumask_bits(candidates)[0], best_energy_cpu, sync, fbt_env.need_idle, fbt_env.fastpath, From 8d553f64719540d09118072510045f85d6fac249 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Mon, 14 Jun 2021 14:15:49 -0700 Subject: [PATCH 125/346] sched/walt: print per cluster energy Per cluster energy contribution is essential to debug energy evaluation. Add code to print it. Change-Id: Ifbe80d125b09facd358933ab3537260fea0584ac Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/trace.h | 43 ++++++++++++++++++++++---- kernel/sched/walt/walt.h | 6 ++++ kernel/sched/walt/walt_cfs.c | 58 ++++++++++++++++++++++++++++-------- 3 files changed, 89 insertions(+), 18 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 7a4d8ace0d30..0a53034c6917 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -881,9 +881,11 @@ TRACE_EVENT(sched_cpu_util, __field(int, reserved) __field(int, high_irq_load) __field(unsigned int, nr_rtg_high_prio_tasks) + __field(u64, prs_gprs) ), TP_fast_assign( + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; __entry->cpu = cpu; __entry->nr_running = cpu_rq(cpu)->nr_running; __entry->cpu_util = cpu_util(cpu); @@ -898,15 +900,16 @@ TRACE_EVENT(sched_cpu_util, __entry->reserved = is_reserved(cpu); __entry->high_irq_load = sched_cpu_high_irqload(cpu); __entry->nr_rtg_high_prio_tasks = walt_nr_rtg_high_prio(cpu); + __entry->prs_gprs = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; ), - TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%u capacity=%u capacity_orig=%u idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u", + TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%u capacity=%u capacity_orig=%u idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u prs_gprs=%llu", __entry->cpu, __entry->nr_running, __entry->cpu_util, __entry->cpu_util_cum, __entry->capacity_curr, __entry->capacity, __entry->capacity_orig, __entry->idle_exit_latency, __entry->irqload, __entry->online, __entry->inactive, __entry->reserved, __entry->high_irq_load, - __entry->nr_rtg_high_prio_tasks) + __entry->nr_rtg_high_prio_tasks, __entry->prs_gprs) ); TRACE_EVENT(sched_compute_energy, @@ -915,10 +918,11 @@ TRACE_EVENT(sched_compute_energy, unsigned long eval_energy, unsigned long prev_energy, unsigned long best_energy, - unsigned long best_energy_cpu), + unsigned long best_energy_cpu, + struct compute_energy_output *o), TP_ARGS(p, eval_cpu, eval_energy, prev_energy, best_energy, - best_energy_cpu), + best_energy_cpu, o), TP_STRUCT__entry( __field(int, pid) @@ -930,6 +934,18 @@ TRACE_EVENT(sched_compute_energy, __field(unsigned long, eval_energy) __field(int, best_energy_cpu) __field(unsigned long, best_energy) + __field(unsigned int, cluster_first_cpu0) + __field(unsigned int, cluster_first_cpu1) + __field(unsigned int, cluster_first_cpu2) + __field(unsigned long, s0) + __field(unsigned long, s1) + __field(unsigned long, s2) + __field(unsigned long, m0) + __field(unsigned long, m1) + __field(unsigned long, m2) + __field(u16, c0) + __field(u16, c1) + __field(u16, c2) ), TP_fast_assign( @@ -942,12 +958,27 @@ TRACE_EVENT(sched_compute_energy, __entry->eval_energy = eval_energy; __entry->best_energy_cpu = best_energy_cpu; __entry->best_energy = best_energy; + __entry->cluster_first_cpu0 = o->cluster_first_cpu[0]; + __entry->cluster_first_cpu1 = o->cluster_first_cpu[1]; + __entry->cluster_first_cpu2 = o->cluster_first_cpu[2]; + __entry->s0 = o->sum_util[0]; + __entry->s1 = o->sum_util[1]; + __entry->s2 = o->sum_util[2]; + __entry->m0 = o->max_util[0]; + __entry->m1 = o->max_util[1]; + __entry->m2 = o->max_util[2]; + __entry->c0 = o->cost[0]; + __entry->c1 = o->cost[1]; + __entry->c2 = o->cost[2]; ), - TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d prev_energy=%lu eval_cpu=%d eval_energy=%lu best_energy_cpu=%d best_energy=%lu", + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d prev_energy=%lu eval_cpu=%d eval_energy=%lu best_energy_cpu=%d best_energy=%lu, fcpu s m c = %u %u %u %u, %u %u %u %u, %u %u %u %u", __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, __entry->prev_energy, __entry->eval_cpu, __entry->eval_energy, - __entry->best_energy_cpu, __entry->best_energy) + __entry->best_energy_cpu, __entry->best_energy, + __entry->cluster_first_cpu0, __entry->s0, __entry->m0, __entry->c0, + __entry->cluster_first_cpu1, __entry->s1, __entry->m1, __entry->c1, + __entry->cluster_first_cpu2, __entry->s2, __entry->m2, __entry->c2) ) TRACE_EVENT(sched_task_util, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 5a5e643b9aa3..f1dca89e2f0e 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -894,4 +894,10 @@ extern __read_mostly unsigned int walt_scale_demand_divisor; #define scale_demand(d) ((d)/walt_scale_demand_divisor) void create_util_to_cost(void); +struct compute_energy_output { + unsigned long sum_util[MAX_CLUSTERS]; + unsigned long max_util[MAX_CLUSTERS]; + u16 cost[MAX_CLUSTERS]; + unsigned int cluster_first_cpu[MAX_CLUSTERS]; +}; #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 2a06a0c3c19b..0839c90c72bb 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -497,7 +497,8 @@ cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_ds * a capacity state satisfying the max utilization of the domain. */ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, - unsigned long max_util, unsigned long sum_util) + unsigned long max_util, unsigned long sum_util, + struct compute_energy_output *output, unsigned int x) { unsigned long scale_cpu; int cpu; @@ -563,6 +564,12 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, max_util = 1023; wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + if (output) { + output->cost[x] = wrq->cluster->util_to_cost[max_util]; + output->max_util[x] = max_util; + output->sum_util[x] = sum_util; + } return wrq->cluster->util_to_cost[max_util] * sum_util / scale_cpu; } @@ -574,7 +581,8 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, * task. */ static long -walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, u64 *prs) +walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, u64 *prs, + struct compute_energy_output *output, unsigned int x) { struct cpumask *pd_mask = perf_domain_span(pd); unsigned long max_util = 0, sum_util = 0; @@ -602,21 +610,28 @@ walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *p max_util = scale_demand(max_util); sum_util = scale_demand(sum_util); - return walt_em_cpu_energy(pd->em_pd, max_util, sum_util); + + if (output) + output->cluster_first_cpu[x] = cpumask_first(pd_mask); + + return walt_em_cpu_energy(pd->em_pd, max_util, sum_util, output, x); } static inline long walt_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, - cpumask_t *candidates, u64 *prs) + cpumask_t *candidates, u64 *prs, struct compute_energy_output *output) { long energy = 0; + unsigned int x = 0; for (; pd; pd = pd->next) { struct cpumask *pd_mask = perf_domain_span(pd); if (cpumask_intersects(candidates, pd_mask) - || cpumask_test_cpu(task_cpu(p), pd_mask)) - energy += walt_pd_compute_energy(p, dst_cpu, pd, prs); + || cpumask_test_cpu(task_cpu(p), pd_mask)) { + energy += walt_pd_compute_energy(p, dst_cpu, pd, prs, output, x); + x++; + } } return energy; @@ -673,6 +688,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int start_cpu, order_index, end_index; int max_cap_cpu = -1; bool energy_eval_needed = true; + struct compute_energy_output output; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && cpumask_test_cpu(prev_cpu, &p->cpus_mask)) @@ -750,20 +766,38 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (p->state == TASK_WAKING) delta = task_util(p); - if (cpumask_test_cpu(prev_cpu, &p->cpus_mask) && !__cpu_overutilized(prev_cpu, delta)) - prev_energy = best_energy = - walt_compute_energy(p, prev_cpu, pd, candidates, fbt_env.prs); - else + if (cpumask_test_cpu(prev_cpu, &p->cpus_mask) && !__cpu_overutilized(prev_cpu, delta)) { + if (trace_sched_compute_energy_enabled()) { + memset(&output, 0, sizeof(output)); + prev_energy = walt_compute_energy(p, prev_cpu, pd, candidates, fbt_env.prs, + &output); + } else { + prev_energy = walt_compute_energy(p, prev_cpu, pd, candidates, fbt_env.prs, + NULL); + } + + best_energy = prev_energy; + trace_sched_compute_energy(p, prev_cpu, prev_energy, 0, 0, 0, &output); + } else { prev_energy = best_energy = ULONG_MAX; + } /* Select the best candidate energy-wise. */ for_each_cpu(cpu, candidates) { if (cpu == prev_cpu) continue; - cur_energy = walt_compute_energy(p, cpu, pd, candidates, fbt_env.prs); + if (trace_sched_compute_energy_enabled()) { + memset(&output, 0, sizeof(output)); + cur_energy = walt_compute_energy(p, cpu, pd, candidates, fbt_env.prs, + &output); + } else { + cur_energy = walt_compute_energy(p, cpu, pd, candidates, fbt_env.prs, + NULL); + } + trace_sched_compute_energy(p, cpu, cur_energy, - prev_energy, best_energy, best_energy_cpu); + prev_energy, best_energy, best_energy_cpu, &output); if (cur_energy < best_energy) { best_energy = cur_energy; From cfb4162ea3a57bd32bf1eb62affc4555123e5fde Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 16 Jun 2021 17:53:24 -0700 Subject: [PATCH 126/346] sched: walt: Fix max_util calculation When calculating max_util for energy evaluations, we adjust the util on the basis of frequency, using arch_scale_freq_capacity. However, the return value is scaled to 1024 and not to the capacity of the CPU. Change-Id: I214ac4b40b88105147c2e5dcc350ff1601496d4c Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 0839c90c72bb..7e7649337611 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -516,7 +516,9 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, scale_cpu = arch_scale_cpu_capacity(cpu); max_util = max_util + (max_util >> 2); /* account for TARGET_LOAD usually 80 */ - max_util = max(max_util, arch_scale_freq_capacity(cpu)); + max_util = max(max_util, + (arch_scale_freq_capacity(cpu) * capacity_orig_of(cpu)) >> + SCHED_CAPACITY_SHIFT); /* * The capacity of a CPU in the domain at the performance state (ps) From 1b1ec2daf6a6d316f8f781e2e82b1f8c295dffb3 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:50:37 -0700 Subject: [PATCH 127/346] sched: Enable walt debug module in first stage To catch early errors, it's important to ensure that the walt debug module is insmodded early. Change-Id: I1418c16a30041469aa04025018a1f0a79065c390 Signed-off-by: Shaleen Agrawal Signed-off-by: Sai Harshini Nimmala Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index d3ee5be4ae24..9a7d157f17b6 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4279,3 +4279,7 @@ static int walt_module_init(void) module_init(walt_module_init); MODULE_LICENSE("GPL v2"); + +#if IS_ENABLED(CONFIG_SCHED_WALT_DEBUG) +MODULE_SOFTDEP("pre: sched-walt-debug"); +#endif From 4e9150549cb3a8126158cbe327ee1af3bfd8fbce Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 17 Jun 2021 13:47:31 -0700 Subject: [PATCH 128/346] sched/walt: Fix scheduling while atomic walt_init() is a function that could sleep and it should not be called in an atomic context. Introduce a work to do walt init. Change-Id: Ief3e9500d7675b998220693ac1908ac74e876f50 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 9a7d157f17b6..4f28f497e776 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4211,12 +4211,14 @@ static void walt_init_tg_pointers(void) rcu_read_unlock(); } -static void walt_init(void) +static void walt_init(struct work_struct *work) { struct ctl_table_header *hdr; static atomic_t already_inited = ATOMIC_INIT(0); int i; + might_sleep(); + if (atomic_cmpxchg(&already_inited, 0, 1)) return; @@ -4252,9 +4254,10 @@ static void walt_init(void) } } +static DECLARE_WORK(walt_init_work, walt_init); static void android_vh_update_topology_flags_workfn(void *unused, void *unused2) { - walt_init(); + schedule_work(&walt_init_work); } #define WALT_VENDOR_DATA_SIZE_TEST(wstruct, kstruct) \ @@ -4272,7 +4275,7 @@ static int walt_module_init(void) android_vh_update_topology_flags_workfn, NULL); if (topology_update_done) - walt_init(); + schedule_work(&walt_init_work); return 0; } From 4969204927cc1033860f32f3d3a6e32444b9b19b Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 3 Jun 2021 10:11:03 -0700 Subject: [PATCH 129/346] sched:walt: register tracehooks for race with pause The current code in force_compatible_cpus_allowed_ptr() holds just the cpus_read_lock, i.e. serializes with the second step of pause. However it doesn't serialize with the first step. IOW, a CPUX chosen in __set_cpus_allowed_ptr_locked() may end up getting deactivated in the pause first and stopper ends up returning early in __migrate_tasks() leaving the 32bit task on an invalid CPU. A simple fix is to serialize force_compatible_cpus_allowed_ptr() with pause first step. Add a cpu_maps_update_begin/done() in force_compatible_cpus_allowed_ptr(). Change-Id: If118aabf9e1252e58dcfe52609d59520c029bc41 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4f28f497e776..bd69de7cee8e 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4119,6 +4119,20 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } +static void android_vh_force_compatible_pre(void *unused, void *unused2) +{ + if (unlikely(walt_disabled)) + return; + cpu_maps_update_begin(); +} + +static void android_vh_force_compatible_post(void *unused, void *unused2) +{ + if (unlikely(walt_disabled)) + return; + cpu_maps_update_done(); +} + static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4148,6 +4162,8 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); + register_trace_android_vh_force_compatible_pre(android_vh_force_compatible_pre, NULL); + register_trace_android_vh_force_compatible_post(android_vh_force_compatible_post, NULL); } atomic64_t walt_irq_work_lastq_ws; From 72cb1a9422f9173a0994cd4649bf90e83fb9bc76 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 21 Jun 2021 13:37:25 -0700 Subject: [PATCH 130/346] sched/walt: prevent lockup when system is suspending When the system is suspending, the console itself can be locked up for a period of time. Unfortunately, pr_err will wait until the console is unlocked, causing significant delays. This causes a defect that seems to point to walt_pause. Fix this by ensuring that all pr_err statements are eliminated by using printk_deferred throughout walt_pause. This will ensure that the operation is not locked on console access. Change-Id: Id2e93a33d5c272fe0ee9b6f18bef5eb8ad4de07d Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_pause.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c index a80b6d72f9a0..474fd6dce21c 100644 --- a/kernel/sched/walt/walt_pause.c +++ b/kernel/sched/walt/walt_pause.c @@ -71,7 +71,7 @@ int walt_pause_cpus(struct cpumask *cpus) ret = pause_cpus(cpus); if (ret < 0) { - pr_err("pause_cpus failure ret=%d cpus=%*pbl\n", ret, + printk_deferred("pause_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); /* ref counts recorded, suppress failure */ @@ -103,7 +103,7 @@ int walt_resume_cpus(struct cpumask *cpus) ret = resume_cpus(cpus); if (ret < 0) { - pr_err("resume_cpus failure ret=%d cpus=%*pbl\n", ret, + printk_deferred("resume_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); /* ref counts recorded, suppress failure */ @@ -169,7 +169,7 @@ static void walt_pause_online_workfn(struct work_struct *work) mutex_unlock(&pause_lock); if (ret < 0) - pr_err("pause_cpus during online failure ret=%d cpus=%*pb1\n", ret, + printk_deferred("pause_cpus during online failure ret=%d cpus=%*pb1\n", ret, cpumask_pr_args(&re_pause_cpus)); } @@ -194,6 +194,6 @@ void walt_pause_init(void) walt_pause_hp_online, NULL); if (ret < 0) - pr_err("failure to register cpuhp online state ret=%d\n", ret); + printk_deferred("failure to register cpuhp online state ret=%d\n", ret); } #endif /* CONFIG_HOTPLUG_CPU */ From 2b51bbee3baf31dcd295284be8c9bf0fb2f70ddf Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 22 Jun 2021 16:15:36 -0700 Subject: [PATCH 131/346] Revert "sched:walt: register tracehooks for race with pause" This reverts commit 17e292f2531be14480932e1c1934ebb5bf42e720. Change-Id: I302081c5218cb55ce5d900189d7e3301d1a8c56e Signed-off-by: Elliot Berman --- kernel/sched/walt/walt.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index bd69de7cee8e..4f28f497e776 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4119,20 +4119,6 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } -static void android_vh_force_compatible_pre(void *unused, void *unused2) -{ - if (unlikely(walt_disabled)) - return; - cpu_maps_update_begin(); -} - -static void android_vh_force_compatible_post(void *unused, void *unused2) -{ - if (unlikely(walt_disabled)) - return; - cpu_maps_update_done(); -} - static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4162,8 +4148,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); - register_trace_android_vh_force_compatible_pre(android_vh_force_compatible_pre, NULL); - register_trace_android_vh_force_compatible_post(android_vh_force_compatible_post, NULL); } atomic64_t walt_irq_work_lastq_ws; From 228d91e408bd4c30141c7c42fb232c30c53228c6 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 22 Jun 2021 19:23:16 -0700 Subject: [PATCH 132/346] sched: walt: Add cgroup_attach trace point In order to debug tasks not being added to top-app group correctly, introduce a tracepoint to aid with such scenarios. Change-Id: Ibdfaf00f65661dc2700f30c951ab8be711cda2b2 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 25 +++++++++++++++++++++++++ kernel/sched/walt/walt.c | 18 +++++++----------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 0a53034c6917..d382b5fd018f 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1237,6 +1237,31 @@ TRACE_EVENT(sched_overutilized, __entry->overutilized, __entry->span) ); +TRACE_EVENT(sched_cgroup_attach, + + TP_PROTO(struct task_struct *p, unsigned int grp_id, int ret), + + TP_ARGS(p, grp_id, ret), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned int, grp_id) + __field(int, ret) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->grp_id = grp_id; + __entry->ret = ret; + ), + + TP_printk("comm=%s pid=%d grp_id=%u ret=%d", + __entry->comm, __entry->pid, + __entry->grp_id, __entry->ret) + +); #endif /* _TRACE_WALT_H */ #undef TRACE_INCLUDE_PATH diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4f28f497e776..dd925a739681 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3059,13 +3059,6 @@ static int create_default_coloc_group(void) return 0; } -static int sync_cgroup_colocation(struct task_struct *p, bool insert) -{ - unsigned int grp_id = insert ? DEFAULT_CGROUP_COLOC_ID : 0; - - return __sched_set_group_id(p, grp_id); -} - static void walt_update_tg_pointer(struct cgroup_subsys_state *css) { if (!strcmp(css->cgroup->kn->name, "top-app")) @@ -3089,9 +3082,10 @@ static void android_rvh_cpu_cgroup_attach(void *unused, { struct task_struct *task; struct cgroup_subsys_state *css; - bool colocate; struct task_group *tg; struct walt_task_group *wtg; + unsigned int grp_id; + int ret; if (unlikely(walt_disabled)) return; @@ -3102,10 +3096,12 @@ static void android_rvh_cpu_cgroup_attach(void *unused, tg = container_of(css, struct task_group, css); wtg = (struct walt_task_group *) tg->android_vendor_data1; - colocate = wtg->colocate; - cgroup_taskset_for_each(task, css, tset) - sync_cgroup_colocation(task, colocate); + cgroup_taskset_for_each(task, css, tset) { + grp_id = wtg->colocate ? DEFAULT_CGROUP_COLOC_ID : 0; + ret = __sched_set_group_id(task, grp_id); + trace_sched_cgroup_attach(task, grp_id, ret); + } } static bool is_cluster_hosting_top_app(struct walt_sched_cluster *cluster) From 5c3c385a68deb125d853ec9a25df901cfa2e4049 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 24 Jun 2021 14:57:44 -0700 Subject: [PATCH 133/346] sched: walt: Ensure only valid cpu selected Under FBT, we currently do not check if an active candidate is actually valid before setting it as part of the candidates mask. This can lead to page faults. Change-Id: I5aedb3bd1e076736270aa03b8889c67606c2a25d Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 7e7649337611..d7c86df25308 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -448,7 +448,7 @@ static void walt_find_best_target(struct sched_domain *sd, if (unlikely(cpumask_empty(candidates))) { if (most_spare_cap_cpu != -1) cpumask_set_cpu(most_spare_cap_cpu, candidates); - else if (!cpu_active(prev_cpu)) + else if (!cpu_active(prev_cpu) && active_candidate != -1) cpumask_set_cpu(active_candidate, candidates); } From 6c47bb351ead4e19c6e20c88d1c2a1c9ae40bc99 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 3 Jun 2021 10:11:03 -0700 Subject: [PATCH 134/346] sched:walt: register restricted tracehooks for race with pause The current code in force_compatible_cpus_allowed_ptr() holds just the cpus_read_lock, i.e. serializes with the second step of pause. However it doesn't serialize with the first step. IOW, a CPUX chosen in __set_cpus_allowed_ptr_locked() may end up getting deactivated in the pause first and stopper ends up returning early in __migrate_tasks() leaving the 32bit task on an invalid CPU. A simple fix is to serialize force_compatible_cpus_allowed_ptr() with pause first step. Add a cpu_maps_update_begin/done() in force_compatible_cpus_allowed_ptr(). Change-Id: I24e7838ee9813b4336bfcebc823e498c3606c793 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index dd925a739681..f27eaa2ccff4 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4115,6 +4115,20 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } +static void android_rvh_force_compatible_pre(void *unused, void *unused2) +{ + if (unlikely(walt_disabled)) + return; + cpu_maps_update_begin(); +} + +static void android_rvh_force_compatible_post(void *unused, void *unused2) +{ + if (unlikely(walt_disabled)) + return; + cpu_maps_update_done(); +} + static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4144,6 +4158,8 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); + register_trace_android_rvh_force_compatible_pre(android_rvh_force_compatible_pre, NULL); + register_trace_android_rvh_force_compatible_post(android_rvh_force_compatible_post, NULL); } atomic64_t walt_irq_work_lastq_ws; From a07c8977726560c3cc1a0bebfab2a1e99d96a191 Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Mon, 28 Jun 2021 19:49:01 -0700 Subject: [PATCH 135/346] sched: walt: Add AMU based instructions and cycle counts Add AMU based cycle and instruction counts to sched_switch_with_ctrs tracepoint. This will help get cycle and instruction counts if the AMUs are supported by hardware. Change-Id: Ia014b6b5afb143f4f790350843f37ae479075ae7 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/perf_trace_counters.h | 25 +++++++++++++++++++++++-- kernel/sched/walt/walt_tp.c | 1 + 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/perf_trace_counters.h b/kernel/sched/walt/perf_trace_counters.h index 73d589a5628e..1817e9e7937e 100644 --- a/kernel/sched/walt/perf_trace_counters.h +++ b/kernel/sched/walt/perf_trace_counters.h @@ -19,6 +19,7 @@ #define C5 0x20 #define C_ALL (CC | C0 | C1 | C2 | C3 | C4 | C5) #define NUM_L1_CTRS 6 +#define NUM_AMU_CTRS 2 #include #include @@ -27,6 +28,7 @@ DECLARE_PER_CPU(u32, cntenset_val); DECLARE_PER_CPU(u32, previous_ccnt); DECLARE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts); +DECLARE_PER_CPU(u64[NUM_AMU_CTRS], previous_amu_cnts); TRACE_EVENT(sched_switch_with_ctrs, TP_PROTO(pid_t prev, pid_t next), @@ -43,6 +45,8 @@ TRACE_EVENT(sched_switch_with_ctrs, __field(u32, ctr3) __field(u32, ctr4) __field(u32, ctr5) + __field(u64, amu0) + __field(u64, amu1) ), TP_fast_assign( @@ -51,7 +55,9 @@ TRACE_EVENT(sched_switch_with_ctrs, u32 cnten_val; u32 total_ccnt = 0; u32 total_cnt = 0; + u64 amu_cnt = 0; u32 delta_l1_cnts[NUM_L1_CTRS]; + u64 delta_amu_cnts[NUM_AMU_CTRS] = {0}; __entry->old_pid = prev; __entry->new_pid = next; @@ -80,20 +86,35 @@ TRACE_EVENT(sched_switch_with_ctrs, delta_l1_cnts[i] = 0; } + if (IS_ENABLED(CONFIG_ARM64_AMU_EXTN)) { + amu_cnt = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0); + delta_amu_cnts[0] = amu_cnt - + per_cpu(previous_amu_cnts[0], cpu); + per_cpu(previous_amu_cnts[0], cpu) = amu_cnt; + + amu_cnt = read_sysreg_s(SYS_AMEVCNTR0_INST_RET_EL0); + delta_amu_cnts[1] = amu_cnt - + per_cpu(previous_amu_cnts[1], cpu); + per_cpu(previous_amu_cnts[1], cpu) = amu_cnt; + } + __entry->ctr0 = delta_l1_cnts[0]; __entry->ctr1 = delta_l1_cnts[1]; __entry->ctr2 = delta_l1_cnts[2]; __entry->ctr3 = delta_l1_cnts[3]; __entry->ctr4 = delta_l1_cnts[4]; __entry->ctr5 = delta_l1_cnts[5]; + __entry->amu0 = delta_amu_cnts[0]; + __entry->amu1 = delta_amu_cnts[1]; ), - TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %u, CTR0: %u, CTR1: %u, CTR2: %u, CTR3: %u, CTR4: %u, CTR5: %u", + TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %u, CTR0: %u, CTR1: %u, CTR2: %u, CTR3: %u, CTR4: %u, CTR5: %u, CYC: %llu, INST: %llu", __entry->old_pid, __entry->new_pid, __entry->cctr, __entry->ctr0, __entry->ctr1, __entry->ctr2, __entry->ctr3, - __entry->ctr4, __entry->ctr5) + __entry->ctr4, __entry->ctr5, + __entry->amu0, __entry->amu1) ); #endif diff --git a/kernel/sched/walt/walt_tp.c b/kernel/sched/walt/walt_tp.c index 17708d91407e..88429ceb7ca3 100644 --- a/kernel/sched/walt/walt_tp.c +++ b/kernel/sched/walt/walt_tp.c @@ -17,6 +17,7 @@ unsigned int sysctl_sched_dynamic_tp_enable; DEFINE_PER_CPU(u32, cntenset_val); DEFINE_PER_CPU(u32, previous_ccnt); DEFINE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts); +DEFINE_PER_CPU(u64[NUM_AMU_CTRS], previous_amu_cnts); DEFINE_PER_CPU(u32, old_pid); DEFINE_PER_CPU(u32, hotplug_flag); From 0fade348a58ea2b412652106c61bc3afe9add7c8 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 11 May 2021 17:39:02 -0700 Subject: [PATCH 136/346] sched: suppress silver region frequency boost Currently, we boost frequencies by 25% across the board. However, due to region 3 silver CPUs energy concerns, we don't want to apply this boost provided that the utilization on silver CPUs is high enough. Additionally, there is the caveat that if the contribution from RT task is high enough (currently hard coded to 25% of silver capacity), we continue to apply the boost. To manipulate the silver threshold (set to 1000 by default, therefore this is disabled), use: echo x > /proc/sys/walt/sched_silver_thres . Change-Id: If689d0935f0c60f7ccc19e63780286c815eb801a Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 26 ++++++++++++++++++++++---- kernel/sched/walt/sysctl.c | 12 ++++++++++++ kernel/sched/walt/trace.h | 7 +++++-- kernel/sched/walt/walt.h | 1 + 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 57692b62949c..53b477fcbce0 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -199,13 +199,25 @@ static void waltgov_deferred_update(struct waltgov_policy *wg_policy, u64 time, } #define TARGET_LOAD 80 +static inline unsigned long walt_map_util_freq(unsigned long util, + unsigned long fmax, unsigned long cap, + int cpu) +{ + if (is_min_capacity_cpu(cpu) && + util >= sysctl_sched_silver_thres && + cpu_util_rt(cpu_rq(cpu)) < (cap >> 2)) + return (fmax + (fmax >> 4)) * util / cap; + return (fmax + (fmax >> 2)) * util / cap; +} + static unsigned int get_next_freq(struct waltgov_policy *wg_policy, - unsigned long util, unsigned long max) + unsigned long util, unsigned long max, + struct waltgov_cpu *wg_cpu) { struct cpufreq_policy *policy = wg_policy->policy; unsigned int freq = policy->cpuinfo.max_freq; - freq = map_util_freq(util, freq, max); + freq = walt_map_util_freq(util, freq, max, wg_cpu->cpu); trace_waltgov_next_freq(policy->cpu, util, max, freq, policy->min, policy->max, wg_policy->cached_raw_freq, wg_policy->need_freq_update); @@ -269,7 +281,13 @@ static inline unsigned long target_util(struct waltgov_policy *wg_policy, unsigned long util; util = freq_to_util(wg_policy, freq); - util = mult_frac(util, TARGET_LOAD, 100); + + if (wg_policy->max == min_max_possible_capacity && + util >= sysctl_sched_silver_thres) + util = mult_frac(util, 94, 100); + else + util = mult_frac(util, TARGET_LOAD, 100); + return util; } @@ -308,7 +326,7 @@ static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 tim waltgov_walt_adjust(j_wg_cpu, j_util, j_nl, &util, &max); } - return get_next_freq(wg_policy, util, max); + return get_next_freq(wg_policy, util, max, wg_cpu); } static void waltgov_update_freq(struct waltgov_callback *cb, u64 time, diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 578af892a5a8..8bfe41973a78 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -48,6 +48,7 @@ unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; unsigned int __read_mostly sysctl_sched_group_downmigrate_pct; unsigned int __read_mostly sysctl_sched_group_upmigrate_pct; unsigned int __read_mostly sysctl_sched_window_stats_policy; +unsigned int __read_mostly sysctl_sched_silver_thres; unsigned int sysctl_sched_ravg_window_nr_ticks; unsigned int sysctl_sched_walt_rotate_big_tasks; unsigned int sysctl_sched_task_unfilter_period; @@ -467,6 +468,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &four, }, + { + .procname = "sched_silver_thres", + .data = &sysctl_sched_silver_thres, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, { .procname = "sched_group_upmigrate", .data = &sysctl_sched_group_upmigrate_pct, @@ -813,6 +823,8 @@ void walt_tunables(void) sysctl_sched_window_stats_policy = WINDOW_STATS_MAX_RECENT_AVG; + sysctl_sched_silver_thres = 1000; + sysctl_sched_ravg_window_nr_ticks = (HZ / NR_WINDOWS_PER_SEC); sched_load_granule = DEFAULT_SCHED_RAVG_WINDOW / NUM_LOAD_INDICES; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index d382b5fd018f..199c882f61a1 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -695,6 +695,7 @@ TRACE_EVENT(waltgov_next_freq, __field(unsigned int, policy_max_freq) __field(unsigned int, cached_raw_freq) __field(bool, need_freq_update) + __field(unsigned int, rt_util) ), TP_fast_assign( __entry->cpu = cpu; @@ -705,8 +706,9 @@ TRACE_EVENT(waltgov_next_freq, __entry->policy_max_freq = policy_max_freq; __entry->cached_raw_freq = cached_raw_freq; __entry->need_freq_update = need_freq_update; + __entry->rt_util = cpu_util_rt(cpu_rq(cpu)); ), - TP_printk("cpu=%u util=%lu max=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d", + TP_printk("cpu=%u util=%lu max=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u", __entry->cpu, __entry->util, __entry->max, @@ -714,7 +716,8 @@ TRACE_EVENT(waltgov_next_freq, __entry->policy_min_freq, __entry->policy_max_freq, __entry->cached_raw_freq, - __entry->need_freq_update) + __entry->need_freq_update, + __entry->rt_util) ); TRACE_EVENT(walt_active_load_balance, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index f1dca89e2f0e..7eb09cf78116 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -204,6 +204,7 @@ extern unsigned int __read_mostly sched_load_granule; extern unsigned int sysctl_sched_min_task_util_for_boost; /* 0.68ms default for 20ms window size scaled to 1024 */ extern unsigned int sysctl_sched_min_task_util_for_colocation; +extern unsigned int __read_mostly sysctl_sched_silver_thres; extern unsigned int sysctl_sched_busy_hyst_enable_cpus; extern unsigned int sysctl_sched_busy_hyst; extern unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus; From 61bc5471503a7f539574a9fb47388a32bc0f1786 Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Fri, 9 Jul 2021 12:17:55 -0700 Subject: [PATCH 137/346] sched: Use IS_ENABLED macro to check if config is enabled Use IS_ENABLED macro to check for a config flag instead of using ifdef. Change-Id: I1c703f9abddf1f66e5b2795f4cf9a1b230f8da4f Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/sched_avg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index c8287e06b974..80e339a2016f 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -174,7 +174,7 @@ static inline void update_busy_hyst_end_time(int cpu, bool dequeue, } coloc_trigger = nr_run_trigger || coloc_load_trigger; -#ifdef CONFIG_SCHED_CONSERVATIVE_BOOST_LPM_BIAS +#if IS_ENABLED(CONFIG_SCHED_CONSERVATIVE_BOOST_LPM_BIAS) hyst_trigger = nr_run_trigger || load_trigger || (sched_boost_type == CONSERVATIVE_BOOST); #else hyst_trigger = nr_run_trigger || load_trigger; From d555a988e20a150e5140e9a25d5b3b53893e49ab Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Mon, 12 Jul 2021 11:08:23 -0700 Subject: [PATCH 138/346] sched: walt: Fix the logic for detecting enq/deq The dequeue boolean flag is used to calculate the busy hysteresis time which helps delay lpms. Fix the logic to detect whether its an enqueue or a dequeue. Change-Id: Ie7177bba80aa3d164a27587290194c0d13929384 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/sched_avg.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 80e339a2016f..7f4f6780ce68 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -139,7 +139,7 @@ void sched_update_hyst_times(void) #define BUSY_NR_RUN 3 #define BUSY_LOAD_FACTOR 10 -static inline void update_busy_hyst_end_time(int cpu, bool dequeue, +static inline void update_busy_hyst_end_time(int cpu, int enq, unsigned long prev_nr_run, u64 curr_time) { bool nr_run_trigger = false; @@ -148,6 +148,7 @@ static inline void update_busy_hyst_end_time(int cpu, bool dequeue, bool util_load_trigger = false; int i; bool hyst_trigger, coloc_trigger; + bool dequeue = (enq < 0); if (!per_cpu(hyst_time, cpu) && !per_cpu(coloc_hyst_time, cpu) && !per_cpu(util_hyst_time, cpu)) @@ -213,7 +214,7 @@ int sched_busy_hyst_handler(struct ctl_table *table, int write, /** * sched_update_nr_prod * @cpu: The core id of the nr running driver. - * @enq: enqueue/dequeue happening on this CPU. + * @enq: enqueue/dequeue/misfit happening on this CPU. * @return: N/A * * Update average with latest nr_running value for CPU @@ -235,7 +236,9 @@ void sched_update_nr_prod(int cpu, int enq) if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu)) per_cpu(nr_max, cpu) = per_cpu(nr, cpu); - update_busy_hyst_end_time(cpu, !enq, nr_running, curr_time); + /* Don't update hyst time for misfit tasks */ + if (enq) + update_busy_hyst_end_time(cpu, enq, nr_running, curr_time); per_cpu(nr_prod_sum, cpu) += nr_running * diff; per_cpu(nr_big_prod_sum, cpu) += walt_big_tasks(cpu) * diff; From 92074a3d0d7d21eea9460f422ea7e0de1167b28f Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Mon, 12 Jul 2021 22:39:30 -0700 Subject: [PATCH 139/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I306ff3e6fb571efd7ab6cfc36edcbb4dfa264c3d Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index f27eaa2ccff4..1530d34e1fcb 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -197,13 +197,13 @@ static inline void walt_task_dump(struct task_struct *p) for (i = 0 ; i < nr_cpu_ids; i++) j += scnprintf(buff + j, buffsz - j, "%u ", wts->curr_window_cpu[i]); - printk_deferred("%s=%d (%s)\n", STRG(wts->curr_window), + printk_deferred("%s=%u (%s)\n", STRG(wts->curr_window), wts->curr_window, buff); for (i = 0, j = 0 ; i < nr_cpu_ids; i++) j += scnprintf(buff + j, buffsz - j, "%u ", wts->prev_window_cpu[i]); - printk_deferred("%s=%d (%s)\n", STRG(wts->prev_window), + printk_deferred("%s=%u (%s)\n", STRG(wts->prev_window), wts->prev_window, buff); SCHED_PRINT(wts->last_wake_ts); From 8ce4bfae2ff8279079effdd8819e61ac07ee4e36 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Fri, 18 Jun 2021 02:42:26 -0700 Subject: [PATCH 140/346] sched/walt: correct return value for walt_pause/resume_cpus() It is possible that pause/resume of CPUs can fail, but walt_pause/resume_cpus() always returns zero which turns even failures into success. This would leave clients go into bad state in case of failures. To avoid such issues separate out ref count updates and clearing pause/resume masks (when the CPUs are already paused/resumed cases) and rearrange the code appropriately to make sure ref counting is properly handled in case of errors. Change-Id: I284752bf9372cada30a2b648f1c9c54446923117 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_pause.c | 47 +++++++++++++++------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c index 474fd6dce21c..96e89ed33810 100644 --- a/kernel/sched/walt/walt_pause.c +++ b/kernel/sched/walt/walt_pause.c @@ -16,33 +16,30 @@ struct pause_cpu_state { static DEFINE_PER_CPU(struct pause_cpu_state, pause_state); -/* increment ref count for cpus in passed mask */ -static void inc_ref_counts(struct cpumask *cpus) +/* increment/decrement ref count for cpus in pause/resume mask */ +static void update_ref_counts(struct cpumask *cpus, bool pause) { int cpu; struct pause_cpu_state *pause_cpu_state; for_each_cpu(cpu, cpus) { pause_cpu_state = per_cpu_ptr(&pause_state, cpu); - if (pause_cpu_state->ref_count) - cpumask_clear_cpu(cpu, cpus); - pause_cpu_state->ref_count++; + if (pause) + pause_cpu_state->ref_count++; + else { + WARN_ON_ONCE(pause_cpu_state->ref_count == 0); + pause_cpu_state->ref_count--; + } } } -/* - * decrement ref count for cpus in passed mask - * updates the cpus to include only cpus ready to be unpaused - */ -static void dec_test_ref_counts(struct cpumask *cpus) +static void update_pause_resume_cpus(struct cpumask *cpus) { int cpu; struct pause_cpu_state *pause_cpu_state; for_each_cpu(cpu, cpus) { pause_cpu_state = per_cpu_ptr(&pause_state, cpu); - WARN_ON_ONCE(pause_cpu_state->ref_count == 0); - pause_cpu_state->ref_count--; if (pause_cpu_state->ref_count) cpumask_clear_cpu(cpu, cpus); } @@ -57,26 +54,24 @@ int walt_pause_cpus(struct cpumask *cpus) mutex_lock(&pause_lock); cpumask_copy(&requested_cpus, cpus); - inc_ref_counts(cpus); + update_pause_resume_cpus(cpus); /* * Add ref counts for all cpus in mask, but * only actually pause online CPUs */ cpumask_and(cpus, cpus, cpu_online_mask); - - if (cpumask_empty(cpus)) + if (cpumask_empty(cpus)) { + update_ref_counts(&requested_cpus, true); goto unlock; + } ret = pause_cpus(cpus); - - if (ret < 0) { + if (ret < 0) printk_deferred("pause_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); - - /* ref counts recorded, suppress failure */ - ret = 0; - } + else + update_ref_counts(&requested_cpus, true); unlock: mutex_unlock(&pause_lock); @@ -92,22 +87,20 @@ int walt_resume_cpus(struct cpumask *cpus) mutex_lock(&pause_lock); cpumask_copy(&requested_cpus, cpus); - dec_test_ref_counts(cpus); + update_ref_counts(&requested_cpus, false); + update_pause_resume_cpus(cpus); /* only actually resume online CPUs */ cpumask_and(cpus, cpus, cpu_online_mask); - if (cpumask_empty(cpus)) goto unlock; ret = resume_cpus(cpus); - if (ret < 0) { printk_deferred("resume_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); - - /* ref counts recorded, suppress failure */ - ret = 0; + /* restore/increment ref counts in case of error */ + update_ref_counts(&requested_cpus, true); } unlock: mutex_unlock(&pause_lock); From f2c3618a6435110fb162783080ee991ecc75b448 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 14 Jul 2021 13:36:47 -0700 Subject: [PATCH 141/346] sched: walt: use scale_cpu for energy evals Currently, when adjusting the max util with respect to current cluster frequency, we use capacity_orig_of as the multiplier. This is incorrect, because capacity_orig_of will change depending on the current max freq. Change-Id: Ifb38c46a8dc68d18458f6b683f0e8e66857a0644 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index d7c86df25308..49918071d936 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -517,7 +517,7 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, max_util = max_util + (max_util >> 2); /* account for TARGET_LOAD usually 80 */ max_util = max(max_util, - (arch_scale_freq_capacity(cpu) * capacity_orig_of(cpu)) >> + (arch_scale_freq_capacity(cpu) * scale_cpu) >> SCHED_CAPACITY_SHIFT); /* From 51542a1e24def1997fcd71d7e0816aafb6f5b7a3 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 15 Jul 2021 16:02:16 -0700 Subject: [PATCH 142/346] sched/walt_rt: Honor sync wakeups The current code does not honor sync wakeup. Make it do that. Change-Id: I509fd704accdebd32ad2281c26a23e2044c6dc09 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_rt.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 5ae1936151ad..ce424b08e804 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -110,13 +110,26 @@ static inline bool walt_rt_task_fits_capacity(struct task_struct *p, int cpu) } #endif +/* + * walt specific should_honor_rt_sync (see rt.c). this will honor + * the sync flag regardless of whether the current waker is cfs or rt + */ +static inline bool walt_should_honor_rt_sync(struct rq *rq, struct task_struct *p, + bool sync) +{ + return sync && + p->prio <= rq->rt.highest_prio.next && + rq->rt.rt_nr_running <= 2; +} + static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int cpu, int sd_flag, int wake_flags, int *new_cpu) { struct task_struct *curr; - struct rq *rq; + struct rq *rq, *this_cpu_rq; bool may_not_preempt; - int ret, target = -1; + bool sync = !!(wake_flags && WF_SYNC); + int ret, target = -1, this_cpu; struct cpumask *lowest_mask; if (unlikely(walt_disabled)) @@ -126,6 +139,19 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) return; + this_cpu = raw_smp_processor_id(); + this_cpu_rq = cpu_rq(this_cpu); + + /* + * Respect the sync flag as long as the task can run on this CPU. + */ + if (sysctl_sched_sync_hint_enable && cpu_active(this_cpu) && + cpumask_test_cpu(this_cpu, task->cpus_ptr) && + walt_should_honor_rt_sync(this_cpu_rq, task, sync)) { + *new_cpu = this_cpu; + return; + } + *new_cpu = cpu; /* previous CPU as back up */ rq = cpu_rq(cpu); From 61cf79010e6d9de8c59bcadfca900678beb10ccf Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Thu, 22 Jul 2021 18:03:59 -0700 Subject: [PATCH 143/346] sched: walt: Use unsigned long type for counter values Use unsigned long data type for counter values since the pmu and amu counters can be 64 bit based on the hardware capability. Change-Id: I27f5e4d0741d7a0a1a7a6cc17edb9028a138708d Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/perf_trace_counters.h | 36 ++++++++++++------------- kernel/sched/walt/walt_tp.c | 6 ++--- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/kernel/sched/walt/perf_trace_counters.h b/kernel/sched/walt/perf_trace_counters.h index 1817e9e7937e..62f86b08a487 100644 --- a/kernel/sched/walt/perf_trace_counters.h +++ b/kernel/sched/walt/perf_trace_counters.h @@ -26,9 +26,9 @@ #include DECLARE_PER_CPU(u32, cntenset_val); -DECLARE_PER_CPU(u32, previous_ccnt); -DECLARE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts); -DECLARE_PER_CPU(u64[NUM_AMU_CTRS], previous_amu_cnts); +DECLARE_PER_CPU(unsigned long, previous_ccnt); +DECLARE_PER_CPU(unsigned long[NUM_L1_CTRS], previous_l1_cnts); +DECLARE_PER_CPU(unsigned long[NUM_AMU_CTRS], previous_amu_cnts); TRACE_EVENT(sched_switch_with_ctrs, TP_PROTO(pid_t prev, pid_t next), @@ -38,26 +38,26 @@ TRACE_EVENT(sched_switch_with_ctrs, TP_STRUCT__entry( __field(pid_t, old_pid) __field(pid_t, new_pid) - __field(u32, cctr) - __field(u32, ctr0) - __field(u32, ctr1) - __field(u32, ctr2) - __field(u32, ctr3) - __field(u32, ctr4) - __field(u32, ctr5) - __field(u64, amu0) - __field(u64, amu1) + __field(unsigned long, cctr) + __field(unsigned long, ctr0) + __field(unsigned long, ctr1) + __field(unsigned long, ctr2) + __field(unsigned long, ctr3) + __field(unsigned long, ctr4) + __field(unsigned long, ctr5) + __field(unsigned long, amu0) + __field(unsigned long, amu1) ), TP_fast_assign( u32 cpu = smp_processor_id(); u32 i; u32 cnten_val; - u32 total_ccnt = 0; - u32 total_cnt = 0; - u64 amu_cnt = 0; - u32 delta_l1_cnts[NUM_L1_CTRS]; - u64 delta_amu_cnts[NUM_AMU_CTRS] = {0}; + unsigned long total_ccnt = 0; + unsigned long total_cnt = 0; + unsigned long amu_cnt = 0; + unsigned long delta_l1_cnts[NUM_L1_CTRS] = {0}; + unsigned long delta_amu_cnts[NUM_AMU_CTRS] = {0}; __entry->old_pid = prev; __entry->new_pid = next; @@ -108,7 +108,7 @@ TRACE_EVENT(sched_switch_with_ctrs, __entry->amu1 = delta_amu_cnts[1]; ), - TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %u, CTR0: %u, CTR1: %u, CTR2: %u, CTR3: %u, CTR4: %u, CTR5: %u, CYC: %llu, INST: %llu", + TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %lu, CTR0: %lu, CTR1: %lu, CTR2: %lu, CTR3: %lu, CTR4: %lu, CTR5: %lu, CYC: %lu, INST: %lu", __entry->old_pid, __entry->new_pid, __entry->cctr, __entry->ctr0, __entry->ctr1, diff --git a/kernel/sched/walt/walt_tp.c b/kernel/sched/walt/walt_tp.c index 88429ceb7ca3..3c98161e889f 100644 --- a/kernel/sched/walt/walt_tp.c +++ b/kernel/sched/walt/walt_tp.c @@ -15,9 +15,9 @@ unsigned int sysctl_sched_dynamic_tp_enable; #define USE_CPUHP_STATE CPUHP_AP_ONLINE_DYN DEFINE_PER_CPU(u32, cntenset_val); -DEFINE_PER_CPU(u32, previous_ccnt); -DEFINE_PER_CPU(u32[NUM_L1_CTRS], previous_l1_cnts); -DEFINE_PER_CPU(u64[NUM_AMU_CTRS], previous_amu_cnts); +DEFINE_PER_CPU(unsigned long, previous_ccnt); +DEFINE_PER_CPU(unsigned long[NUM_L1_CTRS], previous_l1_cnts); +DEFINE_PER_CPU(unsigned long[NUM_AMU_CTRS], previous_amu_cnts); DEFINE_PER_CPU(u32, old_pid); DEFINE_PER_CPU(u32, hotplug_flag); From 55bfad9a226d7bfdf0f2cf8ac55a431b370eba17 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 27 Jul 2021 14:48:16 -0700 Subject: [PATCH 144/346] sched/walt: allow reference to walt debug capabilities Debug capabilities are needed throughout walt files, while the walt-specific debug/dumping capabilities are strictly in walt.c and dependent upon local variables. Support the ability to cause a crash across all walt functionality. Change-Id: I10fffcf028e39964f92b4a5c05dfd35f0eff63fb Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 16 ++++------------ kernel/sched/walt/walt.h | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 1530d34e1fcb..f2147d424a2b 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -175,7 +175,7 @@ __read_mostly unsigned int walt_scale_demand_divisor; #define SCHED_PRINT(arg) printk_deferred("%s=%llu", #arg, arg) #define STRG(arg) #arg -static inline void walt_task_dump(struct task_struct *p) +void walt_task_dump(struct task_struct *p) { char buff[WALT_NR_CPUS * 16]; int i, j = 0; @@ -216,7 +216,7 @@ static inline void walt_task_dump(struct task_struct *p) SCHED_PRINT(p->on_rq); } -static inline void walt_rq_dump(int cpu) +void walt_rq_dump(int cpu) { struct rq *rq = cpu_rq(cpu); struct task_struct *tsk = cpu_curr(cpu); @@ -260,7 +260,7 @@ static inline void walt_rq_dump(int cpu) SCHED_PRINT(sched_capacity_margin_down[cpu]); } -static inline void walt_dump(void) +void walt_dump(void) { int cpu; @@ -275,15 +275,7 @@ static inline void walt_dump(void) printk_deferred("============ WALT RQ DUMP END ==============\n"); } -static int in_sched_bug; -#define SCHED_BUG_ON(condition) \ -({ \ - if (unlikely(!!(condition)) && !in_sched_bug) { \ - in_sched_bug = 1; \ - walt_dump(); \ - BUG_ON(condition); \ - } \ -}) +int in_sched_bug; static inline void fixup_cumulative_runnable_avg(struct rq *rq, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 7eb09cf78116..cb992ba262a0 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -901,4 +901,19 @@ struct compute_energy_output { u16 cost[MAX_CLUSTERS]; unsigned int cluster_first_cpu[MAX_CLUSTERS]; }; + +extern void walt_task_dump(struct task_struct *p); +extern void walt_rq_dump(int cpu); +extern void walt_dump(void); +extern int in_sched_bug; + +#define SCHED_BUG_ON(condition) \ +({ \ + if (unlikely(!!(condition)) && !in_sched_bug) { \ + in_sched_bug = 1; \ + walt_dump(); \ + BUG_ON(condition); \ + } \ +}) + #endif /* _WALT_H */ From 2dd3a6b6652ba8ff626646c04730bd5a88c27a82 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Sun, 1 Aug 2021 21:53:25 -0700 Subject: [PATCH 145/346] sched/walt/core_ctl: use printk_deferred() in case of errors It is possible that the Pause and Resume operations fail and using pr_warn() could lead to core ctl task running for long period when it has to push the logbuf data to console which eventually cause RT throttling. Avoid this scenario by using printk_deferred() instead of pr_warn(). Change-Id: I2717205216caf994a8995d6f11178499a7015201 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/core_ctl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index d6e9d28298e7..25f284befd39 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -1155,7 +1155,7 @@ static void core_ctl_pause_cpus(struct cpumask *cpus_to_pause) if (cpumask_any(cpus_to_pause) < nr_cpu_ids) { if (walt_pause_cpus(cpus_to_pause) < 0) - pr_warn("core_ctl pause operation failed cpus=%*pbl paused_by_us=%*pbl\n", + printk_deferred("core_ctl pause operation failed cpus=%*pbl paused_by_us=%*pbl\n", cpumask_pr_args(cpus_to_pause), cpumask_pr_args(&cpus_paused_by_us)); else @@ -1184,7 +1184,7 @@ static void core_ctl_resume_cpus(struct cpumask *cpus_to_unpause) if (cpumask_any(cpus_to_unpause) < nr_cpu_ids) { if (walt_resume_cpus(cpus_to_unpause) < 0) - pr_warn("core_ctl resume operation failed cpus=%*pbl paused_by_us=%*pbl\n", + printk_deferred("core_ctl resume operation failed cpus=%*pbl paused_by_us=%*pbl\n", cpumask_pr_args(cpus_to_unpause), cpumask_pr_args(&cpus_paused_by_us)); else From 58862813fa7989abc826e2c8d386064e863a0166 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Mon, 26 Jul 2021 04:55:45 -0700 Subject: [PATCH 146/346] sched/walt: skip enery evaluation for need_idle when idle CPU is available When need_idle flag is set & lower capacity CPU is idle, there is no need to go for energy evaluation with higher capacity CPUs. Placement code used to skip energy evaluation in previous Kernels, this commit matches the need_idle functionality of previous Kernels. Change-Id: Ib21baa2b99e882e27550894ef814ed7114d01e8c Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_cfs.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 49918071d936..f481948772e2 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -755,6 +755,11 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, } } + if (need_idle && available_idle_cpu(max_cap_cpu)) { + best_energy_cpu = max_cap_cpu; + goto unlock; + } + for_each_cpu(cpu, candidates) { if (capacity_orig_of(max_cap_cpu) < capacity_orig_of(cpu)) max_cap_cpu = cpu; From 7bb38803c22019fb2a3ac8e0599d5a9c086dc1ed Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 28 Jul 2021 12:54:27 -0700 Subject: [PATCH 147/346] sched/walt: skip finding max when energy evaluating Currently the code is finding the max_cap_cpu amongst candidates but is unused if we are energy evaluating. Clean up the code. Change-Id: I317ed8328851b69534c375c69387647a94ada659 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index f481948772e2..16815a5ebf4e 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -688,7 +688,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int task_boost = per_task_boost(p); bool uclamp_boost = walt_uclamp_boosted(p); int start_cpu, order_index, end_index; - int max_cap_cpu = -1; + int first_cpu; bool energy_eval_needed = true; struct compute_energy_output output; @@ -747,25 +747,26 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (!weight) goto unlock; - max_cap_cpu = cpumask_first(candidates); + first_cpu = cpumask_first(candidates); if (weight == 1) { - if (available_idle_cpu(max_cap_cpu) || max_cap_cpu == prev_cpu) { - best_energy_cpu = max_cap_cpu; + if (available_idle_cpu(first_cpu) || first_cpu == prev_cpu) { + best_energy_cpu = first_cpu; goto unlock; } } - if (need_idle && available_idle_cpu(max_cap_cpu)) { - best_energy_cpu = max_cap_cpu; + if (need_idle && available_idle_cpu(first_cpu)) { + best_energy_cpu = first_cpu; goto unlock; } - for_each_cpu(cpu, candidates) { - if (capacity_orig_of(max_cap_cpu) < capacity_orig_of(cpu)) - max_cap_cpu = cpu; - } - if (!energy_eval_needed) { + int max_cap_cpu = first_cpu; + + for_each_cpu(cpu, candidates) { + if (capacity_orig_of(max_cap_cpu) < capacity_orig_of(cpu)) + max_cap_cpu = cpu; + } best_energy_cpu = max_cap_cpu; goto unlock; } From 452807ae7e53fa9141f3bf6e2cb186347eb65547 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Sun, 1 Aug 2021 23:48:36 -0700 Subject: [PATCH 148/346] sched/walt: choose best cpu based on spare cap When the placement code has to skip energy evaluation and quickly return the best cpu amongst the candidates, the current code chooses the max capacity cpu within them. This leads to overcrowding on the max cpu, as we could be operating in the skip energy eval mode for a long time. So instead of choosing max capacity cpu, choose a cpu with most spare capacity. Change-Id: I13e435cbfc802c5a1002703f928b5f56ef5d8e1f Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 16815a5ebf4e..e021524a4a12 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -670,6 +670,11 @@ static inline bool select_cpu_same_energy(int cpu, int best_cpu, int prev_cpu) return available_idle_cpu(best_cpu); } +static inline unsigned int capacity_spare_of(int cpu) +{ + return capacity_orig_of(cpu) - cpu_util(cpu); +} + static DEFINE_PER_CPU(cpumask_t, energy_cpus); int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync, int sibling_count_hint) @@ -761,13 +766,13 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, } if (!energy_eval_needed) { - int max_cap_cpu = first_cpu; + int max_spare_cpu = first_cpu; for_each_cpu(cpu, candidates) { - if (capacity_orig_of(max_cap_cpu) < capacity_orig_of(cpu)) - max_cap_cpu = cpu; + if (capacity_spare_of(max_spare_cpu) < capacity_spare_of(cpu)) + max_spare_cpu = cpu; } - best_energy_cpu = max_cap_cpu; + best_energy_cpu = max_spare_cpu; goto unlock; } From 5dc9dadb507e115bd825d21476097d6b9c72561c Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Tue, 3 Aug 2021 04:27:55 -0700 Subject: [PATCH 149/346] sched/walt/walt_rt: Improve the scheduler This change is for general scheduler improvemnts. Change-Id: I8e6d7afe12387d86f3721c6fd880812ba86d885e Signed-off-by: Pavankumar Kondeti [satyap@codeaurora.org: port to msm-5.10] Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_rt.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index ce424b08e804..23a3ce5d7e7a 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -23,6 +23,7 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task bool boost_on_big = rt_boost_on_big(); int cluster; int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0; + bool best_cpu_lt = true; if (unlikely(walt_disabled)) return; @@ -33,6 +34,8 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task rcu_read_lock(); for (cluster = 0; cluster < num_sched_clusters; cluster++) { for_each_cpu_and(cpu, lowest_mask, &cpu_array[order_index][cluster]) { + bool lt; + trace_sched_cpu_util(cpu); if (!cpu_active(cpu)) @@ -46,8 +49,21 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task util = cpu_util(cpu); - /* Find the least loaded CPU */ - if (util > best_cpu_util) + lt = (walt_low_latency_task(cpu_rq(cpu)->curr) || + walt_nr_rtg_high_prio(cpu)); + + /* + * When the best is suitable and the current is not, + * skip it + */ + if (lt && !best_cpu_lt) + continue; + + /* + * Either both are sutilable or unsuitable, load takes + * precedence. + */ + if (!(best_cpu_lt ^ lt) && (util > best_cpu_util)) continue; /* @@ -80,6 +96,7 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task best_cpu_util_cum = util_cum; best_cpu_util = util; *best_cpu = cpu; + best_cpu_lt = lt; } if (*best_cpu != -1) From 1a3d10c83189a283bea2926417701259de14f1cf Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 26 Jul 2021 14:16:47 -0700 Subject: [PATCH 150/346] sched/walt: test to ensure dequeue cpu same as enqueue cpu The dequeue cpu must always be the same as the enqueue cpu. Write a check to store off the enqueue cpu in the appropriate hook, and check the same at dequeue. Change-Id: I16bbc8e769b2a49c72b394456dde92213c6f3370 Signed-off-by: Stephen Dickey --- include/linux/sched/walt.h | 1 + kernel/sched/walt/walt.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 918466220ba0..ec39456e2e8b 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -114,6 +114,7 @@ struct walt_task_struct { cpumask_t cpus_requested; bool iowaited; int prev_on_rq; + int prev_on_rq_cpu; struct list_head mvp_list; u64 sum_exec_snapshot; u64 total_exec; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index f2147d424a2b..aeb7e0736144 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2212,6 +2212,7 @@ static void init_new_task_load(struct task_struct *p) wts->prev_window = 0; wts->active_time = 0; wts->prev_on_rq = 0; + wts->prev_on_rq_cpu = -1; for (i = 0; i < NUM_BUSY_BUCKETS; ++i) wts->busy_buckets[i] = 0; @@ -3833,6 +3834,15 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (unlikely(walt_disabled)) return; + lockdep_assert_held(&rq->lock); + + if (p->cpu != cpu_of(rq)) { + printk_deferred("WALT-BUG enqueuing on rq %d when task->cpu is %d\n", + cpu_of(rq), p->cpu); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + /* catch double enqueue */ if (wts->prev_on_rq == 1) { printk_deferred("WALT-BUG double enqueue detected: task_cpu=%d new_cpu=%d\n", @@ -3841,6 +3851,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st SCHED_BUG_ON(1); } wts->prev_on_rq = 1; + wts->prev_on_rq_cpu = cpu_of(rq); wts->last_enqueued_ts = wallclock; sched_update_nr_prod(rq->cpu, 1); @@ -3863,6 +3874,23 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st if (unlikely(walt_disabled)) return; + lockdep_assert_held(&rq->lock); + + /* + * a task can be enqueued before walt is started, and dequeued after. + * therefore the check to ensure that prev_on_rq_cpu is needed to prevent + * an invalid failure. + */ + if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq)) { + printk_deferred("WALT-BUG dequeue cpu %d not same as enqueue %d\n", + cpu_of(rq), wts->prev_on_rq_cpu); + walt_task_dump(p); + SCHED_BUG_ON(1); + } + + /* no longer on a cpu */ + wts->prev_on_rq_cpu = -1; + /* catch double deq */ if (wts->prev_on_rq == 2) { printk_deferred("WALT-BUG double dequeue detected: task_cpu=%d new_cpu=%d\n", From 1a9ac70efdb26e1f719d969322345b3964ad98db Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 27 Jul 2021 15:58:16 -0700 Subject: [PATCH 151/346] sched: walt: Register restricted trace hook for libunity fixup Need restricted hook so that it can sleep. Change-Id: I2b546f57f7ee58beed3dad84f201fd075fa0d21d Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/fixup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/fixup.c b/kernel/sched/walt/fixup.c index 6156b5827293..2b7bae6d00ef 100644 --- a/kernel/sched/walt/fixup.c +++ b/kernel/sched/walt/fixup.c @@ -77,7 +77,7 @@ static bool is_sched_lib_based_app(pid_t pid) return found; } -static void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy, +static void android_rvh_show_max_freq(void *unused, struct cpufreq_policy *policy, unsigned int *max_freq) { if (!cpuinfo_max_freq_cached) @@ -92,5 +92,5 @@ static void android_vh_show_max_freq(void *unused, struct cpufreq_policy *policy void walt_fixup_init(void) { - register_trace_android_vh_show_max_freq(android_vh_show_max_freq, NULL); + register_trace_android_rvh_show_max_freq(android_rvh_show_max_freq, NULL); } From 179319a53c035d26502dfd7a76ea701f1e82528c Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Tue, 3 Aug 2021 05:22:22 -0700 Subject: [PATCH 152/346] Sched/walt/walt_cfs: Improve the scheduler This change is for general scheduler improvemnts. Change-Id: Ic1b53470b3ebc9a8c2d34d34200c4b671af95840 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_cfs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index e021524a4a12..a08414829041 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -312,6 +312,10 @@ static void walt_find_best_target(struct sched_domain *sd, if (fbt_env->skip_cpu == i) continue; + if (per_task_boost(cpu_rq(i)->curr) == + TASK_BOOST_STRICT_MAX) + continue; + /* * p's blocked utilization is still accounted for on prev_cpu * so prev_cpu will receive a negative bias due to the double @@ -387,10 +391,6 @@ static void walt_find_best_target(struct sched_domain *sd, if (best_idle_cpu_cluster != -1) continue; - if (per_task_boost(cpu_rq(i)->curr) == - TASK_BOOST_STRICT_MAX) - continue; - /* * Compute the maximum possible capacity we expect * to have available on this CPU once the task is From 5f9174b2f311714c258a6bcb49ef4dcfe03f538f Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 27 Jul 2021 08:40:56 -0700 Subject: [PATCH 153/346] sched/walt: check next task state Validate that the task selected is in a good state. on_cpu must not be 1 when picking a task. on_cpu is only set in prepare_task() just before the task is switched in. on_rq must not be 0 or migrating when picking a task, because it must be active. Only active tasks are are enqueued on a runqueue, and a migrating task shouldn't be picked as the rq it is on should be locked doing the migration. cpu must be equal to the rq's cpu. pick_next_task must not be picking a task which is not enqueued in this rq's cpu. One exception to the above, is when the scheduler picks a task that is the same as the previous task. In this case the task might already be on_cpu. Change-Id: I07c94c3b2eab0a328222f29550e0515cdcd6e7ed Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_cfs.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index a08414829041..9a463143cb73 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1210,6 +1210,15 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct if (unlikely(walt_disabled)) return; + if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || + (*p)->on_rq == TASK_ON_RQ_MIGRATING || + (*p)->cpu != cpu_of(rq))) { + printk_deferred("WALT-BUG picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", + (*p)->comm, (*p)->pid, (*p)->on_cpu, + (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); + SCHED_BUG_ON(1); + } + /* We don't have MVP tasks queued */ if (list_empty(&wrq->mvp_tasks)) return; @@ -1222,6 +1231,15 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct *se = &mvp->se; *repick = true; + if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || + (*p)->on_rq == TASK_ON_RQ_MIGRATING || + (*p)->cpu != cpu_of(rq))) { + printk_deferred("WALT-BUG picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", + (*p)->comm, (*p)->pid, (*p)->on_cpu, + (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); + SCHED_BUG_ON(1); + } + trace_walt_cfs_mvp_pick_next(mvp, wts, walt_cfs_mvp_task_limit(mvp)); } From a7c3f86453945f747c7c32b933a86987cfeb631e Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 4 Aug 2021 21:35:46 -0700 Subject: [PATCH 154/346] sched/walt: when idle task is scheduled cra must be zero Validate that cumulative_runnable_avg is zero when the next scheduled task is the idle task. Change-Id: Ia14f9c773c48bf44dfb71080b9360dd473b73923 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index aeb7e0736144..128636ceb12a 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4031,6 +4031,7 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, { u64 wallclock = walt_ktime_get_ns(); struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; if (unlikely(walt_disabled)) return; @@ -4039,6 +4040,11 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, wts->last_sleep_ts = wallclock; walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); + if (is_idle_task(next)) + if (wrq->walt_stats.cumulative_runnable_avg_scaled != 0) { + printk_deferred("WALT-BUG next=idle cra!=0\n"); + SCHED_BUG_ON(1); + } } else { walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0); } From b634e6299e2ac8867a3478871fbad4cc0a7176b8 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 27 Jul 2021 15:52:10 -0700 Subject: [PATCH 155/346] sched/walt: sysctl node to allow capture of walt issues It is necessary to maintain debug checks in the walt code while preventing errant crashes in released code. Create a sysctl node that will control whether walt issues will crash the system, and create a macro to take advantage of it. Support the debug strings required by walt, so that this can be easily incorporated into walt files. Change-Id: Ie81cefe3c6d730574599f8d804d010f4a917bba7 Signed-off-by: Stephen Dickey --- kernel/sched/walt/sysctl.c | 10 ++++++ kernel/sched/walt/walt.c | 64 ++++++++++++++++++------------------ kernel/sched/walt/walt.h | 19 ++++++++++- kernel/sched/walt/walt_cfs.c | 4 +-- 4 files changed, 62 insertions(+), 35 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 8bfe41973a78..6e559502a28f 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -61,6 +61,7 @@ unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; const int sched_user_hint_max = 1000; unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ unsigned int sysctl_sched_sync_hint_enable = 1; +unsigned int sysctl_panic_on_walt_bug; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -711,6 +712,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "panic_on_walt_bug", + .data = &sysctl_panic_on_walt_bug, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, { .procname = "sched_lib_name", .data = sched_lib_name, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 128636ceb12a..fd35652ff5e0 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -296,7 +296,7 @@ fixup_cumulative_runnable_avg(struct rq *rq, printk_deferred("WALT-BUG on CPU %d task %s(%d) not on rq %d", raw_smp_processor_id(), p->comm, p->pid, rq->cpu); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } if (cumulative_runnable_avg_scaled < 0) { @@ -304,7 +304,7 @@ fixup_cumulative_runnable_avg(struct rq *rq, raw_smp_processor_id(), wts->demand_scaled, stats->cumulative_runnable_avg_scaled); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } stats->cumulative_runnable_avg_scaled = (u64)cumulative_runnable_avg_scaled; @@ -313,7 +313,7 @@ fixup_cumulative_runnable_avg(struct rq *rq, raw_smp_processor_id(), wts->pred_demand_scaled, stats->pred_demands_sum_scaled); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } stats->pred_demands_sum_scaled = (u64)pred_demands_sum_scaled; } @@ -390,7 +390,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) if (delta < 0) { printk_deferred("WALT-BUG CPU%d; wallclock=%llu is lesser than window_start=%llu", rq->cpu, wallclock, wrq->window_start); - SCHED_BUG_ON(1); + WALT_PANIC(1); } if (delta < sched_ravg_window) return old_window_start; @@ -699,10 +699,10 @@ static inline void account_load_subtractions(struct rq *rq) ls[i].new_subs = 0; } - SCHED_BUG_ON((s64)wrq->prev_runnable_sum < 0); - SCHED_BUG_ON((s64)wrq->curr_runnable_sum < 0); - SCHED_BUG_ON((s64)wrq->nt_prev_runnable_sum < 0); - SCHED_BUG_ON((s64)wrq->nt_curr_runnable_sum < 0); + WALT_PANIC((s64)wrq->prev_runnable_sum < 0); + WALT_PANIC((s64)wrq->curr_runnable_sum < 0); + WALT_PANIC((s64)wrq->nt_prev_runnable_sum < 0); + WALT_PANIC((s64)wrq->nt_curr_runnable_sum < 0); } static inline void create_subtraction_entry(struct rq *rq, u64 ws, int index) @@ -816,7 +816,7 @@ static inline void inter_cluster_migration_fixup src_wrq->curr_runnable_sum, wts->curr_window_cpu[task_cpu]); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } src_wrq->curr_runnable_sum -= wts->curr_window_cpu[task_cpu]; @@ -826,7 +826,7 @@ static inline void inter_cluster_migration_fixup src_wrq->prev_runnable_sum, wts->prev_window_cpu[task_cpu]); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } src_wrq->prev_runnable_sum -= wts->prev_window_cpu[task_cpu]; @@ -841,7 +841,7 @@ static inline void inter_cluster_migration_fixup src_wrq->nt_curr_runnable_sum, wts->curr_window_cpu[task_cpu]); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } src_wrq->nt_curr_runnable_sum -= wts->curr_window_cpu[task_cpu]; @@ -853,7 +853,7 @@ static inline void inter_cluster_migration_fixup src_wrq->nt_prev_runnable_sum, wts->prev_window_cpu[task_cpu]); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } src_wrq->nt_prev_runnable_sum -= wts->prev_window_cpu[task_cpu]; @@ -978,7 +978,7 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) if (task_rq(p) != src_rq) { printk_deferred("WALT-BUG on CPU %d task %s(%d) not on src_rq %d", raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); - SCHED_BUG_ON(1); + WALT_PANIC(1); } walt_update_task_ravg(task_rq(p)->curr, task_rq(p), @@ -1754,7 +1754,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * started at wallclock - irqtime. */ - SCHED_BUG_ON(!is_idle_task(p)); + WALT_PANIC(!is_idle_task(p)); mark_start = wallclock - irqtime; /* @@ -2123,7 +2123,7 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, printk_deferred("WALT-BUG pid=%u CPU%d wallclock=%llu < mark_start=%llu event=%d irqtime=%llu", p->pid, rq->cpu, wallclock, wts->mark_start, event, irqtime); - SCHED_BUG_ON((s64)time_delta < 0); + WALT_PANIC((s64)time_delta < 0); } wrq->task_exec_scale = DIV64_U64_ROUNDUP(cycles_delta * @@ -2448,13 +2448,13 @@ static void init_cpu_array(void) cpu_array = kcalloc(num_sched_clusters, sizeof(cpumask_t *), GFP_ATOMIC | __GFP_NOFAIL); if (!cpu_array) - SCHED_BUG_ON(1); + WALT_PANIC(1); for (i = 0; i < num_sched_clusters; i++) { cpu_array[i] = kcalloc(num_sched_clusters, sizeof(cpumask_t), GFP_ATOMIC | __GFP_NOFAIL); if (!cpu_array[i]) - SCHED_BUG_ON(1); + WALT_PANIC(1); } } @@ -2463,7 +2463,7 @@ static void build_cpu_array(void) int i; if (!cpu_array) - SCHED_BUG_ON(1); + WALT_PANIC(1); /* Construct cpu_array row by row */ for (i = 0; i < num_sched_clusters; i++) { int j, k = 1; @@ -2578,7 +2578,7 @@ static void walt_update_cluster_topology(void) * walt_update_cluster_topology() must be called AFTER policies * for all cpus are initialized. If not, simply BUG(). */ - SCHED_BUG_ON(!policy); + WALT_PANIC(!policy); if (policy) { cluster->max_possible_freq = policy->cpuinfo.max_freq; @@ -3169,7 +3169,7 @@ static void transfer_busy_time(struct rq *rq, p->pid, cpu, event, *src_curr_runnable_sum, wts->curr_window_cpu[cpu]); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } *src_curr_runnable_sum -= wts->curr_window_cpu[cpu]; @@ -3178,7 +3178,7 @@ static void transfer_busy_time(struct rq *rq, p->pid, cpu, event, *src_prev_runnable_sum, wts->prev_window_cpu[cpu]); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } *src_prev_runnable_sum -= wts->prev_window_cpu[cpu]; @@ -3190,7 +3190,7 @@ static void transfer_busy_time(struct rq *rq, *src_nt_curr_runnable_sum, wts->curr_window_cpu[cpu]); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } *src_nt_curr_runnable_sum -= wts->curr_window_cpu[cpu]; @@ -3202,7 +3202,7 @@ static void transfer_busy_time(struct rq *rq, *src_nt_prev_runnable_sum, wts->prev_window_cpu[cpu]); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } *src_nt_prev_runnable_sum -= wts->prev_window_cpu[cpu]; @@ -3229,7 +3229,7 @@ static void transfer_busy_time(struct rq *rq, p->pid, cpu, event, *src_curr_runnable_sum, wts->curr_window); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } *src_curr_runnable_sum -= wts->curr_window; @@ -3238,7 +3238,7 @@ static void transfer_busy_time(struct rq *rq, p->pid, cpu, event, *src_prev_runnable_sum, wts->prev_window); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } *src_prev_runnable_sum -= wts->prev_window; @@ -3249,7 +3249,7 @@ static void transfer_busy_time(struct rq *rq, *src_nt_curr_runnable_sum, wts->curr_window); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } *src_nt_curr_runnable_sum -= wts->curr_window; @@ -3259,7 +3259,7 @@ static void transfer_busy_time(struct rq *rq, *src_nt_prev_runnable_sum, wts->prev_window); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } *src_nt_prev_runnable_sum -= wts->prev_window; } @@ -3840,7 +3840,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st printk_deferred("WALT-BUG enqueuing on rq %d when task->cpu is %d\n", cpu_of(rq), p->cpu); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } /* catch double enqueue */ @@ -3848,7 +3848,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st printk_deferred("WALT-BUG double enqueue detected: task_cpu=%d new_cpu=%d\n", task_cpu(p), cpu_of(rq)); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } wts->prev_on_rq = 1; wts->prev_on_rq_cpu = cpu_of(rq); @@ -3885,7 +3885,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st printk_deferred("WALT-BUG dequeue cpu %d not same as enqueue %d\n", cpu_of(rq), wts->prev_on_rq_cpu); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } /* no longer on a cpu */ @@ -3896,7 +3896,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st printk_deferred("WALT-BUG double dequeue detected: task_cpu=%d new_cpu=%d\n", task_cpu(p), cpu_of(rq)); walt_task_dump(p); - SCHED_BUG_ON(1); + WALT_PANIC(1); } wts->prev_on_rq = 2; @@ -4043,7 +4043,7 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, if (is_idle_task(next)) if (wrq->walt_stats.cumulative_runnable_avg_scaled != 0) { printk_deferred("WALT-BUG next=idle cra!=0\n"); - SCHED_BUG_ON(1); + WALT_PANIC(1); } } else { walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index cb992ba262a0..f4fb1653472d 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -230,6 +230,7 @@ extern unsigned int sysctl_walt_rtg_cfs_boost_prio; extern __read_mostly unsigned int sysctl_sched_force_lb_enable; extern const int sched_user_hint_max; extern unsigned int sysctl_sched_dynamic_tp_enable; +extern unsigned int sysctl_panic_on_walt_bug; extern int sched_dynamic_tp_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); @@ -907,7 +908,7 @@ extern void walt_rq_dump(int cpu); extern void walt_dump(void); extern int in_sched_bug; -#define SCHED_BUG_ON(condition) \ +#define WALT_PANIC(condition) \ ({ \ if (unlikely(!!(condition)) && !in_sched_bug) { \ in_sched_bug = 1; \ @@ -916,4 +917,20 @@ extern int in_sched_bug; } \ }) +#define WALT_PANIC_SENTINEL 0x4544DEAD + +/* + * crash if walt bugs are fatal, otherwise return immediately. + * output format and arguments to console + */ +#define WALT_BUG(p, format, args...) \ +({ \ + if (unlikely(sysctl_panic_on_walt_bug == WALT_PANIC_SENTINEL)) {\ + printk_deferred("WALT-BUG " format, args); \ + if (p) \ + walt_task_dump(p); \ + WALT_PANIC(1); \ + } \ +}) + #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 9a463143cb73..6ada1faf6507 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1216,7 +1216,7 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct printk_deferred("WALT-BUG picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", (*p)->comm, (*p)->pid, (*p)->on_cpu, (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); - SCHED_BUG_ON(1); + WALT_PANIC(1); } /* We don't have MVP tasks queued */ @@ -1237,7 +1237,7 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct printk_deferred("WALT-BUG picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", (*p)->comm, (*p)->pid, (*p)->on_cpu, (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); - SCHED_BUG_ON(1); + WALT_PANIC(1); } trace_walt_cfs_mvp_pick_next(mvp, wts, walt_cfs_mvp_task_limit(mvp)); From 1efede7023718735f50427ee7d80c1d53790a854 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 27 Jul 2021 17:01:09 -0700 Subject: [PATCH 156/346] sched/walt: convert existing walt debug to use macro Walt debug code needs to go through the macro used to control whether or not a crash will happen on error. Additionally, update code so that when crashing is not supported, walt will attempt to correct the error. Change-Id: I2c9efb38e395db90b9fbfbf310329d2775ee7fae Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 223 ++++++++++++++++------------------- kernel/sched/walt/walt_cfs.c | 20 ++-- 2 files changed, 107 insertions(+), 136 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index fd35652ff5e0..d0bcb891f565 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -292,28 +292,23 @@ fixup_cumulative_runnable_avg(struct rq *rq, lockdep_assert_held(&rq->lock); - if (task_rq(p) != rq) { - printk_deferred("WALT-BUG on CPU %d task %s(%d) not on rq %d", - raw_smp_processor_id(), p->comm, p->pid, rq->cpu); - walt_task_dump(p); - WALT_PANIC(1); - } + if (task_rq(p) != rq) + WALT_BUG(p, "on CPU %d task %s(%d) not on rq %d", + raw_smp_processor_id(), p->comm, p->pid, rq->cpu); if (cumulative_runnable_avg_scaled < 0) { - printk_deferred("WALT-BUG on CPU %d task ds=%llu is higher than cra=%llu\n", - raw_smp_processor_id(), wts->demand_scaled, - stats->cumulative_runnable_avg_scaled); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "on CPU %d task ds=%llu is higher than cra=%llu\n", + raw_smp_processor_id(), wts->demand_scaled, + stats->cumulative_runnable_avg_scaled); + cumulative_runnable_avg_scaled = 0; } stats->cumulative_runnable_avg_scaled = (u64)cumulative_runnable_avg_scaled; if (pred_demands_sum_scaled < 0) { - printk_deferred("WALT-BUG on CPU %d task pds=%llu is higher than pds_sum=%llu\n", - raw_smp_processor_id(), wts->pred_demand_scaled, - stats->pred_demands_sum_scaled); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "on CPU %d task pds=%llu is higher than pds_sum=%llu\n", + raw_smp_processor_id(), wts->pred_demand_scaled, + stats->pred_demands_sum_scaled); + pred_demands_sum_scaled = 0; } stats->pred_demands_sum_scaled = (u64)pred_demands_sum_scaled; } @@ -389,7 +384,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) delta = wallclock - wrq->window_start; if (delta < 0) { printk_deferred("WALT-BUG CPU%d; wallclock=%llu is lesser than window_start=%llu", - rq->cpu, wallclock, wrq->window_start); + rq->cpu, wallclock, wrq->window_start); WALT_PANIC(1); } if (delta < sched_ravg_window) @@ -811,22 +806,20 @@ static inline void inter_cluster_migration_fixup dest_wrq->prev_runnable_sum += wts->prev_window; if (src_wrq->curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { - printk_deferred("WALT-BUG pid=%u CPU%d -> CPU%d src_crs=%llu is lesser than task_contrib=%llu", - p->pid, src_rq->cpu, dest_rq->cpu, - src_wrq->curr_runnable_sum, - wts->curr_window_cpu[task_cpu]); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "pid=%u CPU%d -> CPU%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, dest_rq->cpu, + src_wrq->curr_runnable_sum, + wts->curr_window_cpu[task_cpu]); + src_wrq->curr_runnable_sum = wts->curr_window_cpu[task_cpu]; } src_wrq->curr_runnable_sum -= wts->curr_window_cpu[task_cpu]; if (src_wrq->prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { - printk_deferred("WALT-BUG pid=%u CPU%d -> CPU%d src_prs=%llu is lesser than task_contrib=%llu", - p->pid, src_rq->cpu, dest_rq->cpu, - src_wrq->prev_runnable_sum, - wts->prev_window_cpu[task_cpu]); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "pid=%u CPU%d -> CPU%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, dest_rq->cpu, + src_wrq->prev_runnable_sum, + wts->prev_window_cpu[task_cpu]); + src_wrq->prev_runnable_sum = wts->prev_window_cpu[task_cpu]; } src_wrq->prev_runnable_sum -= wts->prev_window_cpu[task_cpu]; @@ -834,26 +827,22 @@ static inline void inter_cluster_migration_fixup dest_wrq->nt_curr_runnable_sum += wts->curr_window; dest_wrq->nt_prev_runnable_sum += wts->prev_window; - if (src_wrq->nt_curr_runnable_sum < - wts->curr_window_cpu[task_cpu]) { - printk_deferred("WALT-BUG pid=%u CPU%d -> CPU%d src_nt_crs=%llu is lesser than task_contrib=%llu", - p->pid, src_rq->cpu, dest_rq->cpu, - src_wrq->nt_curr_runnable_sum, - wts->curr_window_cpu[task_cpu]); - walt_task_dump(p); - WALT_PANIC(1); + if (src_wrq->nt_curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { + WALT_BUG(p, "pid=%u CPU%d -> CPU%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, dest_rq->cpu, + src_wrq->nt_curr_runnable_sum, + wts->curr_window_cpu[task_cpu]); + src_wrq->nt_curr_runnable_sum = wts->curr_window_cpu[task_cpu]; } src_wrq->nt_curr_runnable_sum -= wts->curr_window_cpu[task_cpu]; - if (src_wrq->nt_prev_runnable_sum < - wts->prev_window_cpu[task_cpu]) { - printk_deferred("WALT-BUG pid=%u CPU%d -> CPU%d src_nt_prs=%llu is lesser than task_contrib=%llu", - p->pid, src_rq->cpu, dest_rq->cpu, - src_wrq->nt_prev_runnable_sum, - wts->prev_window_cpu[task_cpu]); - walt_task_dump(p); - WALT_PANIC(1); + if (src_wrq->nt_prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { + WALT_BUG(p, "pid=%u CPU%d -> CPU%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, dest_rq->cpu, + src_wrq->nt_prev_runnable_sum, + wts->prev_window_cpu[task_cpu]); + src_wrq->nt_prev_runnable_sum = wts->prev_window_cpu[task_cpu]; } src_wrq->nt_prev_runnable_sum -= wts->prev_window_cpu[task_cpu]; @@ -975,11 +964,9 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) lockdep_assert_held(&src_rq->lock); lockdep_assert_held(&dest_rq->lock); - if (task_rq(p) != src_rq) { - printk_deferred("WALT-BUG on CPU %d task %s(%d) not on src_rq %d", + if (task_rq(p) != src_rq) + WALT_BUG(p, "on CPU %d task %s(%d) not on src_rq %d", raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); - WALT_PANIC(1); - } walt_update_task_ravg(task_rq(p)->curr, task_rq(p), TASK_UPDATE, @@ -2121,8 +2108,8 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, if ((s64)time_delta < 0) { printk_deferred("WALT-BUG pid=%u CPU%d wallclock=%llu < mark_start=%llu event=%d irqtime=%llu", - p->pid, rq->cpu, wallclock, - wts->mark_start, event, irqtime); + p->pid, rq->cpu, wallclock, + wts->mark_start, event, irqtime); WALT_PANIC((s64)time_delta < 0); } @@ -3165,44 +3152,38 @@ static void transfer_busy_time(struct rq *rq, dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; if (*src_curr_runnable_sum < wts->curr_window_cpu[cpu]) { - printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", - p->pid, cpu, event, *src_curr_runnable_sum, - wts->curr_window_cpu[cpu]); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_curr_runnable_sum, + wts->curr_window_cpu[cpu]); + *src_curr_runnable_sum = wts->curr_window_cpu[cpu]; } *src_curr_runnable_sum -= wts->curr_window_cpu[cpu]; if (*src_prev_runnable_sum < wts->prev_window_cpu[cpu]) { - printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", - p->pid, cpu, event, *src_prev_runnable_sum, - wts->prev_window_cpu[cpu]); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_prev_runnable_sum, + wts->prev_window_cpu[cpu]); + *src_prev_runnable_sum = wts->prev_window_cpu[cpu]; } *src_prev_runnable_sum -= wts->prev_window_cpu[cpu]; if (new_task) { - if (*src_nt_curr_runnable_sum < - wts->curr_window_cpu[cpu]) { - printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", - p->pid, cpu, event, - *src_nt_curr_runnable_sum, - wts->curr_window_cpu[cpu]); - walt_task_dump(p); - WALT_PANIC(1); + if (*src_nt_curr_runnable_sum < wts->curr_window_cpu[cpu]) { + WALT_BUG(p, "pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_curr_runnable_sum, + wts->curr_window_cpu[cpu]); + *src_nt_curr_runnable_sum = wts->curr_window_cpu[cpu]; } *src_nt_curr_runnable_sum -= wts->curr_window_cpu[cpu]; - if (*src_nt_prev_runnable_sum < - wts->prev_window_cpu[cpu]) { - printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", - p->pid, cpu, event, - *src_nt_prev_runnable_sum, - wts->prev_window_cpu[cpu]); - walt_task_dump(p); - WALT_PANIC(1); + if (*src_nt_prev_runnable_sum < wts->prev_window_cpu[cpu]) { + WALT_BUG(p, "pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_prev_runnable_sum, + wts->prev_window_cpu[cpu]); + *src_nt_prev_runnable_sum = wts->prev_window_cpu[cpu]; } *src_nt_prev_runnable_sum -= wts->prev_window_cpu[cpu]; @@ -3225,41 +3206,37 @@ static void transfer_busy_time(struct rq *rq, dst_nt_prev_runnable_sum = &wrq->nt_prev_runnable_sum; if (*src_curr_runnable_sum < wts->curr_window) { - printk_deferred("WALT-UG pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", - p->pid, cpu, event, *src_curr_runnable_sum, - wts->curr_window); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "WALT-UG pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_curr_runnable_sum, + wts->curr_window); + *src_curr_runnable_sum = wts->curr_window; } *src_curr_runnable_sum -= wts->curr_window; if (*src_prev_runnable_sum < wts->prev_window) { - printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", - p->pid, cpu, event, *src_prev_runnable_sum, - wts->prev_window); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_prev_runnable_sum, + wts->prev_window); + *src_prev_runnable_sum = wts->prev_window; } *src_prev_runnable_sum -= wts->prev_window; if (new_task) { if (*src_nt_curr_runnable_sum < wts->curr_window) { - printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", + WALT_BUG(p, "pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_nt_curr_runnable_sum, wts->curr_window); - walt_task_dump(p); - WALT_PANIC(1); + *src_nt_curr_runnable_sum = wts->curr_window; } *src_nt_curr_runnable_sum -= wts->curr_window; if (*src_nt_prev_runnable_sum < wts->prev_window) { - printk_deferred("WALT-BUG pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", - p->pid, cpu, event, - *src_nt_prev_runnable_sum, - wts->prev_window); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_prev_runnable_sum, + wts->prev_window); + *src_nt_prev_runnable_sum = wts->prev_window; } *src_nt_prev_runnable_sum -= wts->prev_window; } @@ -3830,26 +3807,24 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st { u64 wallclock = walt_ktime_get_ns(); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + bool double_enqueue = false; if (unlikely(walt_disabled)) return; lockdep_assert_held(&rq->lock); - if (p->cpu != cpu_of(rq)) { - printk_deferred("WALT-BUG enqueuing on rq %d when task->cpu is %d\n", + if (p->cpu != cpu_of(rq)) + WALT_BUG(p, "enqueuing on rq %d when task->cpu is %d\n", cpu_of(rq), p->cpu); - walt_task_dump(p); - WALT_PANIC(1); - } /* catch double enqueue */ if (wts->prev_on_rq == 1) { - printk_deferred("WALT-BUG double enqueue detected: task_cpu=%d new_cpu=%d\n", - task_cpu(p), cpu_of(rq)); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "double enqueue detected: task_cpu=%d new_cpu=%d\n", + task_cpu(p), cpu_of(rq)); + double_enqueue = true; } + wts->prev_on_rq = 1; wts->prev_on_rq_cpu = cpu_of(rq); @@ -3858,11 +3833,13 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (walt_fair_task(p)) { wts->misfit = !task_fits_max(p, rq->cpu); - inc_rq_walt_stats(rq, p); + if (!double_enqueue) + inc_rq_walt_stats(rq, p); walt_cfs_enqueue_task(rq, p); } - walt_inc_cumulative_runnable_avg(rq, p); + if (!double_enqueue) + walt_inc_cumulative_runnable_avg(rq, p); trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); } @@ -3870,6 +3847,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + bool double_dequeue = false; if (unlikely(walt_disabled)) return; @@ -3881,22 +3859,18 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st * therefore the check to ensure that prev_on_rq_cpu is needed to prevent * an invalid failure. */ - if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq)) { - printk_deferred("WALT-BUG dequeue cpu %d not same as enqueue %d\n", - cpu_of(rq), wts->prev_on_rq_cpu); - walt_task_dump(p); - WALT_PANIC(1); - } + if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq)) + WALT_BUG(p, "dequeue cpu %d not same as enqueue %d\n", + cpu_of(rq), wts->prev_on_rq_cpu); /* no longer on a cpu */ wts->prev_on_rq_cpu = -1; /* catch double deq */ if (wts->prev_on_rq == 2) { - printk_deferred("WALT-BUG double dequeue detected: task_cpu=%d new_cpu=%d\n", - task_cpu(p), cpu_of(rq)); - walt_task_dump(p); - WALT_PANIC(1); + WALT_BUG(p, "double dequeue detected: task_cpu=%d new_cpu=%d\n", + task_cpu(p), cpu_of(rq)); + double_dequeue = true; } wts->prev_on_rq = 2; @@ -3906,11 +3880,14 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st sched_update_nr_prod(rq->cpu, -1); if (walt_fair_task(p)) { - dec_rq_walt_stats(rq, p); + if (!double_dequeue) + dec_rq_walt_stats(rq, p); walt_cfs_dequeue_task(rq, p); } - walt_dec_cumulative_runnable_avg(rq, p); + if (!double_dequeue) + walt_dec_cumulative_runnable_avg(rq, p); + trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); } @@ -4040,11 +4017,9 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, wts->last_sleep_ts = wallclock; walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); - if (is_idle_task(next)) - if (wrq->walt_stats.cumulative_runnable_avg_scaled != 0) { - printk_deferred("WALT-BUG next=idle cra!=0\n"); - WALT_PANIC(1); - } + if (is_idle_task(next) && wrq->walt_stats.cumulative_runnable_avg_scaled != 0) + WALT_BUG(next, "next=idle cra non zero=%d\n", + wrq->walt_stats.cumulative_runnable_avg_scaled); } else { walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0); } diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 6ada1faf6507..821f84df76c5 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1212,12 +1212,10 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || (*p)->on_rq == TASK_ON_RQ_MIGRATING || - (*p)->cpu != cpu_of(rq))) { - printk_deferred("WALT-BUG picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", - (*p)->comm, (*p)->pid, (*p)->on_cpu, - (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); - WALT_PANIC(1); - } + (*p)->cpu != cpu_of(rq))) + WALT_BUG(*p, "picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", + (*p)->comm, (*p)->pid, (*p)->on_cpu, + (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); /* We don't have MVP tasks queued */ if (list_empty(&wrq->mvp_tasks)) @@ -1233,12 +1231,10 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || (*p)->on_rq == TASK_ON_RQ_MIGRATING || - (*p)->cpu != cpu_of(rq))) { - printk_deferred("WALT-BUG picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", - (*p)->comm, (*p)->pid, (*p)->on_cpu, - (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); - WALT_PANIC(1); - } + (*p)->cpu != cpu_of(rq))) + WALT_BUG(*p, "picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", + (*p)->comm, (*p)->pid, (*p)->on_cpu, + (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); trace_walt_cfs_mvp_pick_next(mvp, wts, walt_cfs_mvp_task_limit(mvp)); } From f6a9379748b20ea6af97cc999b5e54d8febc9004 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 9 Aug 2021 10:29:26 -0700 Subject: [PATCH 157/346] sched/walt: reduce severity of core_ctl warning When there is a failure with the call to pause_cpus() or resume_cpus() core_ctl will receive that error and retry on a window boundary. Currently this results in a printk_deferred message going to the console very frequently for a normal use case (hotplug disabled). Change these console messages to pr_debug() to eliminate the messages unless DEBUG is defined, and loglevel is increased. Change-Id: I99bd04cd71dbc0b365df07dea3776a11afa6b7d5 Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 25f284befd39..0f6fe64f58db 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -1155,9 +1155,9 @@ static void core_ctl_pause_cpus(struct cpumask *cpus_to_pause) if (cpumask_any(cpus_to_pause) < nr_cpu_ids) { if (walt_pause_cpus(cpus_to_pause) < 0) - printk_deferred("core_ctl pause operation failed cpus=%*pbl paused_by_us=%*pbl\n", - cpumask_pr_args(cpus_to_pause), - cpumask_pr_args(&cpus_paused_by_us)); + pr_debug("core_ctl pause operation failed cpus=%*pbl paused_by_us=%*pbl\n", + cpumask_pr_args(cpus_to_pause), + cpumask_pr_args(&cpus_paused_by_us)); else cpumask_or(&cpus_paused_by_us, &cpus_paused_by_us, &saved_cpus); } @@ -1184,9 +1184,9 @@ static void core_ctl_resume_cpus(struct cpumask *cpus_to_unpause) if (cpumask_any(cpus_to_unpause) < nr_cpu_ids) { if (walt_resume_cpus(cpus_to_unpause) < 0) - printk_deferred("core_ctl resume operation failed cpus=%*pbl paused_by_us=%*pbl\n", - cpumask_pr_args(cpus_to_unpause), - cpumask_pr_args(&cpus_paused_by_us)); + pr_debug("core_ctl resume operation failed cpus=%*pbl paused_by_us=%*pbl\n", + cpumask_pr_args(cpus_to_unpause), + cpumask_pr_args(&cpus_paused_by_us)); else cpumask_andnot(&cpus_paused_by_us, &cpus_paused_by_us, &saved_cpus); } From 04859b68388d7db9619fb33c2ec903e42613d968 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 10 Aug 2021 12:28:33 -0700 Subject: [PATCH 158/346] sched/walt: reduce severity of walt pause warning It is normal case to see hotplug become disabled and when this happens, pause_cpus and resume_cpus will fail. To walt_pause this is an issue as the requested action cannot be performed, but not from a system perspective. Reduce the severity of the walt_pause error upon failure of pause_cpus() and resume_cpus() to debug only. Corrective action is taken, so the warning doesn't need to be on the console. Change-Id: I5667a1efe46c07379e31bcfd9ff1cfeedd5be64c Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_pause.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c index 96e89ed33810..cca3d5904923 100644 --- a/kernel/sched/walt/walt_pause.c +++ b/kernel/sched/walt/walt_pause.c @@ -68,8 +68,8 @@ int walt_pause_cpus(struct cpumask *cpus) ret = pause_cpus(cpus); if (ret < 0) - printk_deferred("pause_cpus failure ret=%d cpus=%*pbl\n", ret, - cpumask_pr_args(&requested_cpus)); + pr_debug("pause_cpus failure ret=%d cpus=%*pbl\n", ret, + cpumask_pr_args(&requested_cpus)); else update_ref_counts(&requested_cpus, true); unlock: @@ -97,8 +97,8 @@ int walt_resume_cpus(struct cpumask *cpus) ret = resume_cpus(cpus); if (ret < 0) { - printk_deferred("resume_cpus failure ret=%d cpus=%*pbl\n", ret, - cpumask_pr_args(&requested_cpus)); + pr_debug("resume_cpus failure ret=%d cpus=%*pbl\n", ret, + cpumask_pr_args(&requested_cpus)); /* restore/increment ref counts in case of error */ update_ref_counts(&requested_cpus, true); } From a907a3cd42d7906172c24cde33073dcef3441617 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Mon, 9 Aug 2021 19:11:38 -0700 Subject: [PATCH 159/346] sched/walt/walt_cfs: Improve the scheduler This change is for general scheduler improvemnts. Change-Id: Iae112690c5cf125db5d197555d47244a63bfb32e Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_cfs.c | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index f4fb1653472d..cb6459eb6082 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -883,6 +883,7 @@ static inline bool walt_fair_task(struct task_struct *p) #define WALT_RTG_MVP 0 #define WALT_BINDER_MVP 1 +#define WALT_TASK_BOOST_MVP 2 #define WALT_NOT_MVP -1 diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 821f84df76c5..ee5b3ce30af3 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -961,14 +961,15 @@ static void binder_restore_priority_hook(void *data, */ static inline int walt_get_mvp_task_prio(struct task_struct *p) { - if ((per_task_boost(p) == TASK_BOOST_STRICT_MAX) || - task_rtg_high_prio(p) || - walt_procfs_low_latency_task(p)) - return WALT_RTG_MVP; + if (per_task_boost(p) == TASK_BOOST_STRICT_MAX) + return WALT_TASK_BOOST_MVP; if (walt_binder_low_latency_task(p)) return WALT_BINDER_MVP; + if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p)) + return WALT_RTG_MVP; + return WALT_NOT_MVP; } @@ -1010,15 +1011,6 @@ static void walt_cfs_deactivate_mvp_task(struct task_struct *p) list_del_init(&wts->mvp_list); wts->mvp_prio = WALT_NOT_MVP; - - /* - * Reset the exec time during sleep so that it starts - * from scratch upon next wakeup. total_exec should - * be preserved when task is enq/deq while it is on - * runqueue. - */ - if (p->state != TASK_RUNNING) - wts->total_exec = 0; } /* @@ -1114,6 +1106,15 @@ void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) if (!list_empty(&wts->mvp_list)) walt_cfs_deactivate_mvp_task(p); + + /* + * Reset the exec time during sleep so that it starts + * from scratch upon next wakeup. total_exec should + * be preserved when task is enq/deq while it is on + * runqueue. + */ + if (p->state != TASK_RUNNING) + wts->total_exec = 0; } void walt_cfs_tick(struct rq *rq) From 36183d4ec40806c4d27d856b66d4d3ebea8d3917 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 4 Aug 2021 15:56:08 -0700 Subject: [PATCH 160/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I8e5fbdb142a891f79b0073b2fc0ee7e81d46c845 Signed-off-by: Stephen Dickey --- kernel/sched/walt/sysctl.c | 10 ++++++++++ kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_cfs.c | 5 +++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 6e559502a28f..485ea5c858b4 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -62,6 +62,7 @@ const int sched_user_hint_max = 1000; unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ unsigned int sysctl_sched_sync_hint_enable = 1; unsigned int sysctl_panic_on_walt_bug; +unsigned int sysctl_sched_suppress_region2; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -712,6 +713,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_suppress_region2", + .data = &sysctl_sched_suppress_region2, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, { .procname = "panic_on_walt_bug", .data = &sysctl_panic_on_walt_bug, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index cb6459eb6082..b5ff6da7b34e 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -286,6 +286,7 @@ extern unsigned int sysctl_sched_task_unfilter_period; extern unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ extern unsigned int sysctl_sched_sync_hint_enable; +extern unsigned int sysctl_sched_suppress_region2; extern struct ctl_table walt_table[]; extern struct ctl_table walt_base_table[]; extern void walt_tunables(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index ee5b3ce30af3..286e653818d0 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -199,8 +199,9 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, (task_util(p) >= MIN_UTIL_FOR_ENERGY_EVAL) && !(p->in_iowait && task_in_related_thread_group(p)) && !walt_get_rtg_status(p) && - !(sched_boost_type == CONSERVATIVE_BOOST && task_sched_boost(p)) - ) + !(sched_boost_type == CONSERVATIVE_BOOST && task_sched_boost(p)) && + !sysctl_sched_suppress_region2 + ) *end_index = 1; if (p->in_iowait && task_in_related_thread_group(p)) From 03504cab3cd602864ad24e036f6ebe8bc8b0a855 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Fri, 20 Aug 2021 14:16:35 -0700 Subject: [PATCH 161/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I4ccf43485d46451837d3114de370b5ea9bc1f824 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/boost.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c index 451b1090c60c..521933dec5d8 100644 --- a/kernel/sched/walt/boost.c +++ b/kernel/sched/walt/boost.c @@ -174,7 +174,7 @@ static int sched_effective_boost(void) static void sched_boost_disable(int type) { struct sched_boost_data *sb = &sched_boosts[type]; - int next_boost; + int next_boost, prev_boost = sched_boost_type; if (sb->refcount <= 0) return; @@ -192,6 +192,9 @@ static void sched_boost_disable(int type) sb->exit(); next_boost = sched_effective_boost(); + if (next_boost == prev_boost) + return; + sched_boosts[next_boost].enter(); } From 925dd42831d2b4ec5c687f342d9ac65e06a809e4 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 26 Oct 2021 17:51:24 -0700 Subject: [PATCH 162/346] sched/walt: Improve the scheduler This change is for general scheduler improvemnts. Change-Id: I9c8c1a9c00534700b63834384c3553145ae8c935 Signed-off-by: Stephen Dickey Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/sysctl.c | 10 ++++++++++ kernel/sched/walt/walt.c | 18 ++++++++++++++++++ kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_debug.c | 1 - 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 485ea5c858b4..a0bbb0b30927 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -61,6 +61,7 @@ unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; const int sched_user_hint_max = 1000; unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ unsigned int sysctl_sched_sync_hint_enable = 1; +unsigned int sysctl_sched_bug_on_rt_throttle; unsigned int sysctl_panic_on_walt_bug; unsigned int sysctl_sched_suppress_region2; @@ -713,6 +714,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_bug_on_rt_throttle", + .data = &sysctl_sched_bug_on_rt_throttle, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, { .procname = "sched_suppress_region2", .data = &sysctl_sched_suppress_region2, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index d0bcb891f565..f3fa92fdbc7f 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4130,6 +4130,23 @@ static void android_rvh_force_compatible_post(void *unused, void *unused2) cpu_maps_update_done(); } +static void dump_throttled_rt_tasks(void *unused, int cpu, u64 clock, + ktime_t rt_period, u64 rt_runtime, s64 rt_period_timer_expires) +{ + printk_deferred("sched: RT throttling activated for cpu %d\n", cpu); + printk_deferred("rt_period_timer: expires=%lld now=%llu rt_time=%llu runtime=%llu period=%llu\n", + rt_period_timer_expires, ktime_get_ns(), + task_rq(current)->rt.rt_time, rt_runtime, rt_period); + printk_deferred("potential CPU hogs:\n"); +#ifdef CONFIG_SCHED_INFO + if (sched_info_on()) + printk_deferred("current %s (%d) is running for %llu nsec\n", + current->comm, current->pid, + clock - current->sched_info.last_arrival); +#endif + BUG_ON(sysctl_sched_bug_on_rt_throttle); +} + static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4161,6 +4178,7 @@ static void register_walt_hooks(void) register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); register_trace_android_rvh_force_compatible_pre(android_rvh_force_compatible_pre, NULL); register_trace_android_rvh_force_compatible_post(android_rvh_force_compatible_post, NULL); + register_trace_android_vh_dump_throttled_rt_tasks(dump_throttled_rt_tasks, NULL); } atomic64_t walt_irq_work_lastq_ws; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index b5ff6da7b34e..64c097f92ab3 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -286,6 +286,7 @@ extern unsigned int sysctl_sched_task_unfilter_period; extern unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ extern unsigned int sysctl_sched_sync_hint_enable; +extern unsigned int sysctl_sched_bug_on_rt_throttle; extern unsigned int sysctl_sched_suppress_region2; extern struct ctl_table walt_table[]; extern struct ctl_table walt_base_table[]; diff --git a/kernel/sched/walt/walt_debug.c b/kernel/sched/walt/walt_debug.c index 935c3626b427..59f6edbed53b 100644 --- a/kernel/sched/walt/walt_debug.c +++ b/kernel/sched/walt/walt_debug.c @@ -41,7 +41,6 @@ static int __init walt_debug_init(void) if (ret) return ret; - register_trace_android_vh_dump_throttled_rt_tasks(dump_throttled_rt_tasks, NULL); register_trace_android_rvh_schedule_bug(android_rvh_schedule_bug, NULL); return 0; From 9847c28cfd7e1dbe4ef99e0ddbb182ce7ebdb9a0 Mon Sep 17 00:00:00 2001 From: Maria Yu Date: Tue, 24 Aug 2021 18:35:07 +0800 Subject: [PATCH 163/346] sched/walt: Check if necessary for sched_boost modify Check if necessary to do exit and next_boost enter oprations instead of doing it regardlessly. Core control boost refcounts has been wrong if doing current sched_boost exit and next_boost enter by mistake. Change-Id: Ie40ebf0be0e1b99291d07d160ebef5964d44ba16 Signed-off-by: Maria Yu --- kernel/sched/walt/boost.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c index 521933dec5d8..8b6d5f721b26 100644 --- a/kernel/sched/walt/boost.c +++ b/kernel/sched/walt/boost.c @@ -184,17 +184,15 @@ static void sched_boost_disable(int type) if (sb->refcount) return; + next_boost = sched_effective_boost(); + if (next_boost == prev_boost) + return; /* * This boost's refcount becomes zero, so it must * be disabled. Disable it first and then apply * the next boost. */ sb->exit(); - - next_boost = sched_effective_boost(); - if (next_boost == prev_boost) - return; - sched_boosts[next_boost].enter(); } From c81145e1f42814bab9d90ba5a9d337e3466de466 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 24 Aug 2021 09:46:03 -0700 Subject: [PATCH 164/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I17cca453e5a4e6bbf4d0b55440d9f776dc2583cb Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/boost.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c index 8b6d5f721b26..dbab4634a988 100644 --- a/kernel/sched/walt/boost.c +++ b/kernel/sched/walt/boost.c @@ -192,7 +192,7 @@ static void sched_boost_disable(int type) * be disabled. Disable it first and then apply * the next boost. */ - sb->exit(); + sched_boosts[prev_boost].exit(); sched_boosts[next_boost].enter(); } From 318d73bd4dfd32bbde26d4d74a5fdba1eaa88c0b Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 24 Aug 2021 10:24:26 -0700 Subject: [PATCH 165/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I42edf6c6df05f0bf72e4c57ee701916860773428 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/boost.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c index dbab4634a988..ddfe83366013 100644 --- a/kernel/sched/walt/boost.c +++ b/kernel/sched/walt/boost.c @@ -225,12 +225,12 @@ static void sched_boost_enable(int type) static void sched_boost_disable_all(void) { int i; + int prev_boost = sched_boost_type; - for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) { - if (sched_boosts[i].refcount > 0) { - sched_boosts[i].exit(); + if (prev_boost != NO_BOOST) { + sched_boosts[prev_boost].exit(); + for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) sched_boosts[i].refcount = 0; - } } } From b7b0355db1a4eaff84fe8c7497b097198bac1b7d Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 24 Aug 2021 11:44:46 -0700 Subject: [PATCH 166/346] sched/walt: Improve the scheduler Improve core_ctl tracing such that the necessary flags for debugging eval_need are present in the trace, and so that all returns go through the same tracepoint, unlocking, and return functionality. Change-Id: I61d5ab86ba7650bea77e4416b0ffa9a07869bbf3 Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 19 +++++++++++-------- kernel/sched/walt/trace.h | 26 +++++++++++++++++++------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 0f6fe64f58db..f90948451d21 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -290,7 +290,7 @@ static ssize_t show_active_cpus(const struct cluster_data *state, char *buf) return scnprintf(buf, PAGE_SIZE, "%u\n", state->active_cpus); } -static unsigned int nr_paused_cpus(const struct cluster_data *cluster) +static unsigned int cluster_paused_cpus(const struct cluster_data *cluster) { cpumask_t cluster_paused_cpus; @@ -339,8 +339,8 @@ static ssize_t show_global_state(const struct cluster_data *state, char *buf) count += scnprintf(buf + count, PAGE_SIZE - count, "\tNeed CPUs: %u\n", cluster->need_cpus); count += scnprintf(buf + count, PAGE_SIZE - count, - "\tNr paused CPUs: %u\n", - nr_paused_cpus(cluster)); + "\tCluster paused CPUs: %u\n", + cluster_paused_cpus(cluster)); count += scnprintf(buf + count, PAGE_SIZE - count, "\tBoost: %u\n", (unsigned int) cluster->boost); } @@ -616,7 +616,7 @@ static int prev_cluster_nr_need_assist(int index) * Next cluster should not assist, while there are paused cpus * in this cluster. */ - if (nr_paused_cpus(prev_cluster)) + if (cluster_paused_cpus(prev_cluster)) return 0; for_each_cpu(cpu, &prev_cluster->cpu_mask) @@ -777,7 +777,7 @@ static bool adjustment_possible(const struct cluster_data *cluster, unsigned int need) { return (need < cluster->active_cpus || (need > cluster->active_cpus && - nr_paused_cpus(cluster))); + cluster_paused_cpus(cluster))); } static bool need_all_cpus(const struct cluster_data *cluster) @@ -837,8 +837,8 @@ static bool eval_need(struct cluster_data *cluster) */ if (new_need == last_need && new_need == cluster->active_cpus) { cluster->need_ts = now; - spin_unlock_irqrestore(&state_lock, flags); - return false; + ret = 0; + goto unlock; } elapsed = now - cluster->need_ts; @@ -849,8 +849,11 @@ static bool eval_need(struct cluster_data *cluster) cluster->need_ts = now; cluster->need_cpus = new_need; } + +unlock: trace_core_ctl_eval_need(cluster->first_cpu, last_need, new_need, - ret && need_flag); + cluster->active_cpus, ret, need_flag, + ret && need_flag, cluster->need_ts); spin_unlock_irqrestore(&state_lock, flags); return ret && need_flag; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 199c882f61a1..19cd65690f62 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -444,23 +444,35 @@ TRACE_EVENT(sched_load_to_gov, TRACE_EVENT(core_ctl_eval_need, - TP_PROTO(unsigned int cpu, unsigned int old_need, - unsigned int new_need, unsigned int updated), - TP_ARGS(cpu, old_need, new_need, updated), + TP_PROTO(unsigned int cpu, unsigned int last_need, + unsigned int new_need, unsigned int active_cpus, + unsigned int ret, unsigned int need_flag, + unsigned int updated, s64 need_ts), + TP_ARGS(cpu, last_need, new_need, active_cpus, ret, need_flag, updated, need_ts), TP_STRUCT__entry( __field(u32, cpu) - __field(u32, old_need) + __field(u32, last_need) __field(u32, new_need) + __field(u32, active_cpus) + __field(u32, ret) + __field(u32, need_flag) __field(u32, updated) + __field(s64, need_ts) ), TP_fast_assign( __entry->cpu = cpu; - __entry->old_need = old_need; + __entry->last_need = last_need; __entry->new_need = new_need; + __entry->active_cpus = active_cpus; + __entry->ret = ret; + __entry->need_flag = need_flag; __entry->updated = updated; + __entry->need_ts = need_ts; ), - TP_printk("cpu=%u, old_need=%u, new_need=%u, updated=%u", __entry->cpu, - __entry->old_need, __entry->new_need, __entry->updated) + TP_printk("cpu=%u last_need=%u new_need=%u active_cpus=%u ret=%u need_flag=%u updated=%u need_ts=%llu", + __entry->cpu, __entry->last_need, __entry->new_need, + __entry->active_cpus, __entry->ret, __entry->need_flag, + __entry->updated, __entry->need_ts) ); TRACE_EVENT(core_ctl_set_busy, From 7d1e74f709a0b88b27124bad1735d3f0dcedce7e Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Tue, 24 Aug 2021 10:45:35 -0700 Subject: [PATCH 167/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I6f953cd04ada8ccb36bbf1e2fe341375092b35c4 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/cpufreq_walt.c | 53 ++++++++++++++++++++++++++++++-- kernel/sched/walt/trace.h | 9 ++++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 53b477fcbce0..f13b9969bb46 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -22,6 +22,8 @@ struct waltgov_tunables { unsigned int hispeed_load; unsigned int hispeed_freq; unsigned int rtg_boost_freq; + unsigned int adaptive_min_freq; + unsigned int adaptive_max_freq; bool pl; int boost; }; @@ -215,10 +217,19 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, struct waltgov_cpu *wg_cpu) { struct cpufreq_policy *policy = wg_policy->policy; - unsigned int freq = policy->cpuinfo.max_freq; + unsigned int freq, raw_freq; - freq = walt_map_util_freq(util, freq, max, wg_cpu->cpu); - trace_waltgov_next_freq(policy->cpu, util, max, freq, policy->min, policy->max, + raw_freq = walt_map_util_freq(util, policy->cpuinfo.max_freq, max, wg_cpu->cpu); + freq = raw_freq; + + if (wg_policy->tunables->adaptive_max_freq) { + if (raw_freq < wg_policy->tunables->adaptive_min_freq) + freq = wg_policy->tunables->adaptive_min_freq; + else if (raw_freq <= wg_policy->tunables->adaptive_max_freq) + freq = wg_policy->tunables->adaptive_max_freq; + } + + trace_waltgov_next_freq(policy->cpu, util, max, raw_freq, freq, policy->min, policy->max, wg_policy->cached_raw_freq, wg_policy->need_freq_update); if (freq == wg_policy->cached_raw_freq && !wg_policy->need_freq_update) @@ -608,11 +619,41 @@ static ssize_t boost_store(struct gov_attr_set *attr_set, const char *buf, return count; } +#define WALTGOV_ATTR_RW(_name) \ +static struct governor_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) \ + +#define show_attr(name) \ +static ssize_t show_##name(struct gov_attr_set *attr_set, char *buf) \ +{ \ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); \ + return scnprintf(buf, PAGE_SIZE, "%lu\n", tunables->name); \ +} \ + +#define store_attr(name) \ +static ssize_t store_##name(struct gov_attr_set *attr_set, \ + const char *buf, size_t count) \ +{ \ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); \ + \ + if (kstrtouint(buf, 10, &tunables->name)) \ + return -EINVAL; \ + \ + return count; \ +} \ + +show_attr(adaptive_min_freq); +store_attr(adaptive_min_freq); +show_attr(adaptive_max_freq); +store_attr(adaptive_max_freq); + static struct governor_attr hispeed_load = __ATTR_RW(hispeed_load); static struct governor_attr hispeed_freq = __ATTR_RW(hispeed_freq); static struct governor_attr rtg_boost_freq = __ATTR_RW(rtg_boost_freq); static struct governor_attr pl = __ATTR_RW(pl); static struct governor_attr boost = __ATTR_RW(boost); +WALTGOV_ATTR_RW(adaptive_min_freq); +WALTGOV_ATTR_RW(adaptive_max_freq); static struct attribute *waltgov_attributes[] = { &up_rate_limit_us.attr, @@ -622,6 +663,8 @@ static struct attribute *waltgov_attributes[] = { &rtg_boost_freq.attr, &pl.attr, &boost.attr, + &adaptive_min_freq.attr, + &adaptive_max_freq.attr, NULL }; @@ -723,6 +766,8 @@ static void waltgov_tunables_save(struct cpufreq_policy *policy, cached->up_rate_limit_us = tunables->up_rate_limit_us; cached->down_rate_limit_us = tunables->down_rate_limit_us; cached->boost = tunables->boost; + cached->adaptive_min_freq = tunables->adaptive_min_freq; + cached->adaptive_max_freq = tunables->adaptive_max_freq; } static void waltgov_tunables_restore(struct cpufreq_policy *policy) @@ -741,6 +786,8 @@ static void waltgov_tunables_restore(struct cpufreq_policy *policy) tunables->up_rate_limit_us = cached->up_rate_limit_us; tunables->down_rate_limit_us = cached->down_rate_limit_us; tunables->boost = cached->boost; + tunables->adaptive_min_freq = cached->adaptive_min_freq; + tunables->adaptive_max_freq = cached->adaptive_max_freq; } static int waltgov_init(struct cpufreq_policy *policy) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 19cd65690f62..eeb9af0aac80 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -693,15 +693,16 @@ TRACE_EVENT(waltgov_util_update, ); TRACE_EVENT(waltgov_next_freq, - TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, + TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, unsigned int raw_freq, unsigned int freq, unsigned int policy_min_freq, unsigned int policy_max_freq, unsigned int cached_raw_freq, bool need_freq_update), - TP_ARGS(cpu, util, max, freq, policy_min_freq, policy_max_freq, + TP_ARGS(cpu, util, max, raw_freq, freq, policy_min_freq, policy_max_freq, cached_raw_freq, need_freq_update), TP_STRUCT__entry( __field(unsigned int, cpu) __field(unsigned long, util) __field(unsigned long, max) + __field(unsigned int, raw_freq) __field(unsigned int, freq) __field(unsigned int, policy_min_freq) __field(unsigned int, policy_max_freq) @@ -713,6 +714,7 @@ TRACE_EVENT(waltgov_next_freq, __entry->cpu = cpu; __entry->util = util; __entry->max = max; + __entry->raw_freq = raw_freq; __entry->freq = freq; __entry->policy_min_freq = policy_min_freq; __entry->policy_max_freq = policy_max_freq; @@ -720,10 +722,11 @@ TRACE_EVENT(waltgov_next_freq, __entry->need_freq_update = need_freq_update; __entry->rt_util = cpu_util_rt(cpu_rq(cpu)); ), - TP_printk("cpu=%u util=%lu max=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u", + TP_printk("cpu=%u util=%lu max=%lu raw_freq=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u", __entry->cpu, __entry->util, __entry->max, + __entry->raw_freq, __entry->freq, __entry->policy_min_freq, __entry->policy_max_freq, From 0e90c114dd08b24975329a4047c8d847f5d26ab7 Mon Sep 17 00:00:00 2001 From: Lingutla Chandrasekhar Date: Tue, 24 Aug 2021 14:00:06 +0530 Subject: [PATCH 168/346] sched: walt: Account rt scale to cpu_capacity While updating cpu_rq->cpu_capacity, we are ignoring rt pressure, and using same as cpu_capacity_orig, which might lead to unfair scheduler decisions. Fix it by accouting rt scale to cpu_capacity. Change-Id: Icead0af0d8a309cc5f93438a4d4d7e7e7c93653c Signed-off-by: Lingutla Chandrasekhar --- kernel/sched/walt/walt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index f3fa92fdbc7f..367521dea565 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3717,6 +3717,7 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); unsigned long thermal_cap; struct walt_sched_cluster *cluster; + unsigned long rt_pressure = max_capacity - *capacity; if (unlikely(walt_disabled)) return; @@ -3736,7 +3737,7 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long cluster->max_possible_freq); cpu_rq(cpu)->cpu_capacity_orig = min(max_capacity, thermal_cap); - *capacity = cpu_rq(cpu)->cpu_capacity_orig; + *capacity = cpu_rq(cpu)->cpu_capacity_orig - rt_pressure; } static void android_rvh_sched_cpu_starting(void *unused, int cpu) From 4717a648249726c8f4ee6332de6206f0ae5f20de Mon Sep 17 00:00:00 2001 From: Amir Vajid Date: Thu, 26 Aug 2021 14:54:07 -0700 Subject: [PATCH 169/346] sched: walt: add sched_switch_ctrs_cfg event Create an event to periodically dump the configuration of the pmu counter registers. This can be used in conjunction with the existing sched_switch_with_ctrs event to confirm which counter is configured in each register. Change-Id: Iaf11279e94d4af3f8aff522282585e680acd5f4d Signed-off-by: Amir Vajid --- kernel/sched/walt/perf_trace_counters.h | 52 +++++++++++++++++++++++++ kernel/sched/walt/walt_tp.c | 7 ++++ 2 files changed, 59 insertions(+) diff --git a/kernel/sched/walt/perf_trace_counters.h b/kernel/sched/walt/perf_trace_counters.h index 62f86b08a487..bcf318c2ebb1 100644 --- a/kernel/sched/walt/perf_trace_counters.h +++ b/kernel/sched/walt/perf_trace_counters.h @@ -18,6 +18,7 @@ #define C4 0x10 #define C5 0x20 #define C_ALL (CC | C0 | C1 | C2 | C3 | C4 | C5) +#define TYPE_MASK 0xFFFF #define NUM_L1_CTRS 6 #define NUM_AMU_CTRS 2 @@ -117,6 +118,57 @@ TRACE_EVENT(sched_switch_with_ctrs, __entry->amu0, __entry->amu1) ); +TRACE_EVENT(sched_switch_ctrs_cfg, + + TP_PROTO(int cpu), + + TP_ARGS(cpu), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, ctr0) + __field(unsigned long, ctr1) + __field(unsigned long, ctr2) + __field(unsigned long, ctr3) + __field(unsigned long, ctr4) + __field(unsigned long, ctr5) + ), + + TP_fast_assign( + u32 i; + u32 cnten_val; + u32 ctr_type[NUM_L1_CTRS] = {0}; + + cnten_val = per_cpu(cntenset_val, cpu); + + for (i = 0; i < NUM_L1_CTRS; i++) { + if (cnten_val & (1 << i)) { + /* Select */ + write_sysreg(i, pmselr_el0); + isb(); + /* Read type */ + ctr_type[i] = read_sysreg(pmxevtyper_el0) + & TYPE_MASK; + } else + ctr_type[i] = 0; + } + + __entry->cpu = cpu; + __entry->ctr0 = ctr_type[0]; + __entry->ctr1 = ctr_type[1]; + __entry->ctr2 = ctr_type[2]; + __entry->ctr3 = ctr_type[3]; + __entry->ctr4 = ctr_type[4]; + __entry->ctr5 = ctr_type[5]; + ), + + TP_printk("cpu=%d CTR0=%lu CTR1=%lu CTR2=%lu CTR3=%lu CTR4=%lu CTR5=%lu", + __entry->cpu, + __entry->ctr0, __entry->ctr1, + __entry->ctr2, __entry->ctr3, + __entry->ctr4, __entry->ctr5) +); + #endif #undef TRACE_INCLUDE_PATH #define TRACE_INCLUDE_PATH ../../kernel/sched/walt diff --git a/kernel/sched/walt/walt_tp.c b/kernel/sched/walt/walt_tp.c index 3c98161e889f..62d3acb2b482 100644 --- a/kernel/sched/walt/walt_tp.c +++ b/kernel/sched/walt/walt_tp.c @@ -20,6 +20,7 @@ DEFINE_PER_CPU(unsigned long[NUM_L1_CTRS], previous_l1_cnts); DEFINE_PER_CPU(unsigned long[NUM_AMU_CTRS], previous_amu_cnts); DEFINE_PER_CPU(u32, old_pid); DEFINE_PER_CPU(u32, hotplug_flag); +DEFINE_PER_CPU(u64, prev_time); static int tracectr_cpu_hotplug_coming_up(unsigned int cpu) { @@ -54,6 +55,7 @@ void tracectr_notifier(void *ignore, bool preempt, u32 cnten_val; int current_pid; u32 cpu = task_cpu(next); + u64 now; if (!trace_sched_switch_with_ctrs_enabled()) return; @@ -71,6 +73,11 @@ void tracectr_notifier(void *ignore, bool preempt, } else { trace_sched_switch_with_ctrs(per_cpu(old_pid, cpu), current_pid); + now = sched_clock(); + if ((now - per_cpu(prev_time, cpu)) > NSEC_PER_SEC) { + trace_sched_switch_ctrs_cfg(cpu); + per_cpu(prev_time, cpu) = now; + } } /* Enable all the counters that were disabled */ From 57cfe14de2e07d154cd444c0a91e2e2fbd73561c Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Fri, 27 Aug 2021 14:35:07 -0700 Subject: [PATCH 170/346] sched: walt: Increase nr_threshold to 40 percent Increase the nr_threshold percentage to 40 from 15. Change-Id: I32ce7246fc4cd32d4c8110bef63971c9a2dceb55 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/sched_avg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 7f4f6780ce68..981cd11891cc 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -31,7 +31,7 @@ static DEFINE_PER_CPU(u64, coloc_hyst_busy); static DEFINE_PER_CPU(u64, coloc_hyst_time); static DEFINE_PER_CPU(u64, util_hyst_time); -#define NR_THRESHOLD_PCT 15 +#define NR_THRESHOLD_PCT 40 #define MAX_RTGB_TIME (sysctl_sched_coloc_busy_hyst_max_ms * NSEC_PER_MSEC) /** From dc224f824d206bc6cbd9b58ef8a193d1b5228b0a Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Tue, 31 Aug 2021 14:49:26 +0800 Subject: [PATCH 171/346] sched/walt: update the value of active_cpus Because of online cpu not make sure active, but active make sure online, so when want to get active_cpus, make it and cpu_active_mask. Change-Id: Ie73b85506252edfad1f1258c9e059b0167dfe4a5 Signed-off-by: Tengfei Fan --- kernel/sched/walt/core_ctl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index f90948451d21..0b976655a280 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -287,7 +287,9 @@ static ssize_t show_need_cpus(const struct cluster_data *state, char *buf) static ssize_t show_active_cpus(const struct cluster_data *state, char *buf) { - return scnprintf(buf, PAGE_SIZE, "%u\n", state->active_cpus); + int active_cpus = get_active_cpu_count(state); + + return scnprintf(buf, PAGE_SIZE, "%u\n", active_cpus); } static unsigned int cluster_paused_cpus(const struct cluster_data *cluster) From 8fd0c42d94783cca4eebf56051230db1937e8f0c Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 31 Aug 2021 14:05:24 -0700 Subject: [PATCH 172/346] sched/walt: Improve the scheduler This change is for a general scheduler improvement. Change-Id: I68b72c7b557edfff685b58ec78525b806f0a7142 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_cfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 286e653818d0..5d4752ca6950 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -153,7 +153,7 @@ static inline bool walt_target_ok(int target_cpu, int order_index) (target_cpu == cpumask_first(&cpu_array[order_index][0]))); } -#define MIN_UTIL_FOR_ENERGY_EVAL 10 +#define MIN_UTIL_FOR_ENERGY_EVAL 52 static void walt_get_indicies(struct task_struct *p, int *order_index, int *end_index, int per_task_boost, bool is_uclamp_boosted, bool *energy_eval_needed) From 37d8832deeb3e9bfc30c5bcaa789fb203de623e4 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 31 Aug 2021 14:19:29 -0700 Subject: [PATCH 173/346] sched/walt: eval_need code and trace cleanup Improve the naming and types of variables in eval_need, reduce computation and adjust the tracepoint to reflect the new names of variables. Change-Id: Ifc19bc802bbc8382577b0e4ed4b4c9be77ffe99e Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 20 ++++++++++---------- kernel/sched/walt/trace.h | 16 ++++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 0b976655a280..a6f86e58bbcd 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -793,8 +793,8 @@ static bool eval_need(struct cluster_data *cluster) unsigned long flags; struct cpu_data *c; unsigned int need_cpus = 0, last_need, thres_idx; - int ret = 0; - bool need_flag = false; + bool adj_now = false; + bool adj_possible = false; unsigned int new_need; s64 now, elapsed; @@ -824,13 +824,12 @@ static bool eval_need(struct cluster_data *cluster) need_cpus = apply_task_need(cluster, need_cpus); } new_need = apply_limits(cluster, need_cpus); - need_flag = adjustment_possible(cluster, new_need); last_need = cluster->need_cpus; now = ktime_to_ms(ktime_get()); if (new_need > cluster->active_cpus) { - ret = 1; + adj_now = true; } else { /* * When there is no change in need and there are no more @@ -839,26 +838,27 @@ static bool eval_need(struct cluster_data *cluster) */ if (new_need == last_need && new_need == cluster->active_cpus) { cluster->need_ts = now; - ret = 0; + adj_now = false; goto unlock; } elapsed = now - cluster->need_ts; - ret = elapsed >= cluster->offline_delay_ms; + adj_now = elapsed >= cluster->offline_delay_ms; } - if (ret) { + if (adj_now) { + adj_possible = adjustment_possible(cluster, new_need); cluster->need_ts = now; cluster->need_cpus = new_need; } unlock: trace_core_ctl_eval_need(cluster->first_cpu, last_need, new_need, - cluster->active_cpus, ret, need_flag, - ret && need_flag, cluster->need_ts); + cluster->active_cpus, adj_now, adj_possible, + adj_now && adj_possible, cluster->need_ts); spin_unlock_irqrestore(&state_lock, flags); - return ret && need_flag; + return adj_now && adj_possible; } static void apply_need(struct cluster_data *cluster) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index eeb9af0aac80..aa7b56c95432 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -446,16 +446,16 @@ TRACE_EVENT(core_ctl_eval_need, TP_PROTO(unsigned int cpu, unsigned int last_need, unsigned int new_need, unsigned int active_cpus, - unsigned int ret, unsigned int need_flag, + unsigned int adj_now, unsigned int adj_possible, unsigned int updated, s64 need_ts), - TP_ARGS(cpu, last_need, new_need, active_cpus, ret, need_flag, updated, need_ts), + TP_ARGS(cpu, last_need, new_need, active_cpus, adj_now, adj_possible, updated, need_ts), TP_STRUCT__entry( __field(u32, cpu) __field(u32, last_need) __field(u32, new_need) __field(u32, active_cpus) - __field(u32, ret) - __field(u32, need_flag) + __field(u32, adj_now) + __field(u32, adj_possible) __field(u32, updated) __field(s64, need_ts) ), @@ -464,14 +464,14 @@ TRACE_EVENT(core_ctl_eval_need, __entry->last_need = last_need; __entry->new_need = new_need; __entry->active_cpus = active_cpus; - __entry->ret = ret; - __entry->need_flag = need_flag; + __entry->adj_now = adj_now; + __entry->adj_possible = adj_possible; __entry->updated = updated; __entry->need_ts = need_ts; ), - TP_printk("cpu=%u last_need=%u new_need=%u active_cpus=%u ret=%u need_flag=%u updated=%u need_ts=%llu", + TP_printk("cpu=%u last_need=%u new_need=%u active_cpus=%u adj_now=%u adj_possible=%u updated=%u need_ts=%llu", __entry->cpu, __entry->last_need, __entry->new_need, - __entry->active_cpus, __entry->ret, __entry->need_flag, + __entry->active_cpus, __entry->adj_now, __entry->adj_possible, __entry->updated, __entry->need_ts) ); From c4b8044564a8358acfe6bb8a860606c0f367bd19 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 18 Jun 2021 14:17:18 -0700 Subject: [PATCH 174/346] sched: walt: Use CRA for sum util Energy is defined as the cost * load. Currently, the cost is derived from prs and sum_util is also the sum of prs. This is unoptimal, as prs denotes frequency, but not load. Therefore, it is better to use CRA to denote the load instead. Change-Id: I503f0895951390ad111be24daab6cc76c289821b Signed-off-by: Shaleen Agrawal Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 45 ++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 5d4752ca6950..d5d85426949d 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -459,6 +459,48 @@ static void walt_find_best_target(struct sched_domain *sd, fbt_env->skip_cpu, task_on_rq_queued(p)); } +static inline unsigned long +cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + unsigned long util = wrq->walt_stats.cumulative_runnable_avg_scaled; + bool queued = task_on_rq_queued(p); + + /* + * When task is queued, + * (a) The evaluating CPU (cpu) is task's current CPU. If the + * task is migrating, discount the task contribution from the + * evaluation cpu. + * (b) The evaluating CPU (cpu) is task's current CPU. If the + * task is NOT migrating, nothing to do. The contribution is + * already present on the evaluation CPU. + * (c) The evaluating CPU (cpu) is not task's current CPU. But + * the task is migrating to the evaluating CPU. So add the + * task contribution to it. + * (d) The evaluating CPU (cpu) is neither the current CPU nor + * the destination CPU. don't care. + * + * When task is NOT queued i.e waking. Task contribution is not + * present on any CPU. + * + * (a) If the evaluating CPU is the destination CPU, add the task + * contribution. + * (b) The evaluation CPU is not the destination CPU, don't care. + */ + if (unlikely(queued)) { + if (task_cpu(p) == cpu) { + if (dst_cpu != cpu) + util = max_t(long, util - task_util(p), 0); + } else if (dst_cpu == cpu) { + util += task_util(p); + } + } else if (dst_cpu == cpu) { + util += task_util(p); + } + + return min_t(unsigned long, util, capacity_orig_of(cpu)); +} + static inline u64 cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_dst_same_cluster, u64 *prs) @@ -606,13 +648,12 @@ walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *p * its pd list and will not be accounted by compute_energy(). */ for_each_cpu_and(cpu, pd_mask, cpu_online_mask) { + sum_util += cpu_util_next_walt(cpu, p, dst_cpu); cpu_util = cpu_util_next_walt_prs(cpu, p, dst_cpu, prev_dst_same_cluster, prs); - sum_util += cpu_util; max_util = max(max_util, cpu_util); } max_util = scale_demand(max_util); - sum_util = scale_demand(sum_util); if (output) output->cluster_first_cpu[x] = cpumask_first(pd_mask); From beca103128416d4e7ee491c155b4118755621e6a Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 16 Jul 2021 12:31:15 -0700 Subject: [PATCH 175/346] sched: walt: Do dequeue task accounting after dequeue Until recently, we used to do all the dequeue task accounting business after the call the dequeue task call was done. However, due to an upstream change in the position of the trace hook, we modified our value adds to account for doing this work prior to dequeue task. However, there were unforeseen consequences as a result. As there is now an "after" dequeue hook available, it is more intuitive to do this work after dequeue task has taken place, and furthermore, this keeps the work in line with the manner in which it is done on prior chipsets ensuring stability. Change-Id: Iaa274ed39780316fd2d462ad22d4d8b1772def45 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/sched_avg.c | 2 +- kernel/sched/walt/walt.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 981cd11891cc..28c4cfc26f72 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -231,7 +231,7 @@ void sched_update_nr_prod(int cpu, int enq) diff = curr_time - per_cpu(last_time, cpu); BUG_ON((s64)diff < 0); per_cpu(last_time, cpu) = curr_time; - per_cpu(nr, cpu) = cpu_rq(cpu)->nr_running + enq; + per_cpu(nr, cpu) = cpu_rq(cpu)->nr_running; if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu)) per_cpu(nr_max, cpu) = per_cpu(nr, cpu); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 367521dea565..da5e617d7482 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3804,7 +3804,7 @@ static void android_rvh_flush_task(void *unused, struct task_struct *p) walt_task_dead(p); } -static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) +static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_struct *p) { u64 wallclock = walt_ktime_get_ns(); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -3844,7 +3844,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); } -static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p, int flags) +static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -4160,8 +4160,8 @@ static void register_walt_hooks(void) register_trace_android_rvh_account_irq(android_rvh_account_irq, NULL); register_trace_android_rvh_flush_task(android_rvh_flush_task, NULL); register_trace_android_rvh_update_misfit_status(android_rvh_update_misfit_status, NULL); - register_trace_android_rvh_enqueue_task(android_rvh_enqueue_task, NULL); - register_trace_android_rvh_dequeue_task(android_rvh_dequeue_task, NULL); + register_trace_android_rvh_after_enqueue_task(android_rvh_enqueue_task, NULL); + register_trace_android_rvh_after_dequeue_task(android_rvh_dequeue_task, NULL); register_trace_android_rvh_try_to_wake_up(android_rvh_try_to_wake_up, NULL); register_trace_android_rvh_try_to_wake_up_success(android_rvh_try_to_wake_up_success, NULL); register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL); From 00c0dd9fc15b0e26d3488aacc1e03d2b90f40da9 Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Thu, 9 Sep 2021 17:53:19 -0700 Subject: [PATCH 176/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Id0ed126e0d5bd18957438d4d79bce6c310791c00 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/cpufreq_walt.c | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index f13b9969bb46..709050681746 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -22,8 +22,8 @@ struct waltgov_tunables { unsigned int hispeed_load; unsigned int hispeed_freq; unsigned int rtg_boost_freq; - unsigned int adaptive_min_freq; - unsigned int adaptive_max_freq; + unsigned int adaptive_low_freq; + unsigned int adaptive_high_freq; bool pl; int boost; }; @@ -222,11 +222,11 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, raw_freq = walt_map_util_freq(util, policy->cpuinfo.max_freq, max, wg_cpu->cpu); freq = raw_freq; - if (wg_policy->tunables->adaptive_max_freq) { - if (raw_freq < wg_policy->tunables->adaptive_min_freq) - freq = wg_policy->tunables->adaptive_min_freq; - else if (raw_freq <= wg_policy->tunables->adaptive_max_freq) - freq = wg_policy->tunables->adaptive_max_freq; + if (wg_policy->tunables->adaptive_high_freq) { + if (raw_freq < wg_policy->tunables->adaptive_low_freq) + freq = wg_policy->tunables->adaptive_low_freq; + else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) + freq = wg_policy->tunables->adaptive_high_freq; } trace_waltgov_next_freq(policy->cpu, util, max, raw_freq, freq, policy->min, policy->max, @@ -642,18 +642,18 @@ static ssize_t store_##name(struct gov_attr_set *attr_set, \ return count; \ } \ -show_attr(adaptive_min_freq); -store_attr(adaptive_min_freq); -show_attr(adaptive_max_freq); -store_attr(adaptive_max_freq); +show_attr(adaptive_low_freq); +store_attr(adaptive_low_freq); +show_attr(adaptive_high_freq); +store_attr(adaptive_high_freq); static struct governor_attr hispeed_load = __ATTR_RW(hispeed_load); static struct governor_attr hispeed_freq = __ATTR_RW(hispeed_freq); static struct governor_attr rtg_boost_freq = __ATTR_RW(rtg_boost_freq); static struct governor_attr pl = __ATTR_RW(pl); static struct governor_attr boost = __ATTR_RW(boost); -WALTGOV_ATTR_RW(adaptive_min_freq); -WALTGOV_ATTR_RW(adaptive_max_freq); +WALTGOV_ATTR_RW(adaptive_low_freq); +WALTGOV_ATTR_RW(adaptive_high_freq); static struct attribute *waltgov_attributes[] = { &up_rate_limit_us.attr, @@ -663,8 +663,8 @@ static struct attribute *waltgov_attributes[] = { &rtg_boost_freq.attr, &pl.attr, &boost.attr, - &adaptive_min_freq.attr, - &adaptive_max_freq.attr, + &adaptive_low_freq.attr, + &adaptive_high_freq.attr, NULL }; @@ -766,8 +766,8 @@ static void waltgov_tunables_save(struct cpufreq_policy *policy, cached->up_rate_limit_us = tunables->up_rate_limit_us; cached->down_rate_limit_us = tunables->down_rate_limit_us; cached->boost = tunables->boost; - cached->adaptive_min_freq = tunables->adaptive_min_freq; - cached->adaptive_max_freq = tunables->adaptive_max_freq; + cached->adaptive_low_freq = tunables->adaptive_low_freq; + cached->adaptive_high_freq = tunables->adaptive_high_freq; } static void waltgov_tunables_restore(struct cpufreq_policy *policy) @@ -786,8 +786,8 @@ static void waltgov_tunables_restore(struct cpufreq_policy *policy) tunables->up_rate_limit_us = cached->up_rate_limit_us; tunables->down_rate_limit_us = cached->down_rate_limit_us; tunables->boost = cached->boost; - tunables->adaptive_min_freq = cached->adaptive_min_freq; - tunables->adaptive_max_freq = cached->adaptive_max_freq; + tunables->adaptive_low_freq = cached->adaptive_low_freq; + tunables->adaptive_high_freq = cached->adaptive_high_freq; } static int waltgov_init(struct cpufreq_policy *policy) From d61bdd0b0f99918033a07bad05f102e71024a755 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 13 Sep 2021 09:49:46 -0700 Subject: [PATCH 177/346] sched: walt: Improve the Scheduler This change is for general scheduler improvement. Change-Id: I78a13f15f7b95d7b6583222fbc68b828e651b802 Signed-off-by: Lingutla Chandrasekhar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/sysctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index a0bbb0b30927..96bb39bfe91d 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -112,12 +112,12 @@ static int walt_proc_user_hint_handler(struct ctl_table *table, mutex_lock(&mutex); - sched_user_hint_reset_time = jiffies + HZ; old_value = sysctl_sched_user_hint; ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); if (ret || !write || (old_value == sysctl_sched_user_hint)) goto unlock; + sched_user_hint_reset_time = jiffies + HZ; walt_irq_work_queue(&walt_migration_irq_work); unlock: From 60e7fe8dbade9e4da61cc3993daca41c994d65ce Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 13 Sep 2021 15:40:15 -0700 Subject: [PATCH 178/346] sched: walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I509129b041185e8f6b83eb586b7ce542889ebe5d Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index da5e617d7482..bad638335220 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2179,6 +2179,19 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even run_walt_irq_work(old_window_start, rq); } +static void __sched_fork_init(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->last_sleep_ts = 0; + wts->wake_up_idle = false; + wts->boost = 0; + wts->boost_expires = 0; + wts->boost_period = false; + wts->low_latency = false; + wts->iowaited = false; +} + static void init_new_task_load(struct task_struct *p) { int i; @@ -2230,6 +2243,7 @@ static void init_new_task_load(struct task_struct *p) wts->sum_exec_snapshot = 0; wts->total_exec = 0; wts->mvp_prio = WALT_NOT_MVP; + __sched_fork_init(p); } static void init_existing_task_load(struct task_struct *p) @@ -3696,6 +3710,7 @@ static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new) { if (unlikely(walt_disabled)) return; + init_new_task_load(new); add_new_task_to_grp(new); } @@ -3768,13 +3783,6 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign fixup_busy_time(p, (int) new_cpu); } -static void android_rvh_sched_fork(void *unused, struct task_struct *p) -{ - if (unlikely(walt_disabled)) - return; - init_new_task_load(p); -} - static void android_rvh_new_task_stats(void *unused, struct task_struct *p) { if (unlikely(walt_disabled)) @@ -4083,17 +4091,10 @@ static void android_rvh_sched_setaffinity(void *unused, struct task_struct *p, static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) { - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (unlikely(walt_disabled)) return; - wts->last_sleep_ts = 0; - wts->wake_up_idle = false; - wts->boost = 0; - wts->boost_expires = 0; - wts->boost_period = false; - wts->low_latency = false; - wts->iowaited = false; + + __sched_fork_init(p); } static void android_rvh_ttwu_cond(void *unused, bool *cond) @@ -4156,7 +4157,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_cpu_dying(android_rvh_sched_cpu_dying, NULL); register_trace_android_rvh_set_task_cpu(android_rvh_set_task_cpu, NULL); register_trace_android_rvh_new_task_stats(android_rvh_new_task_stats, NULL); - register_trace_android_rvh_sched_fork(android_rvh_sched_fork, NULL); register_trace_android_rvh_account_irq(android_rvh_account_irq, NULL); register_trace_android_rvh_flush_task(android_rvh_flush_task, NULL); register_trace_android_rvh_update_misfit_status(android_rvh_update_misfit_status, NULL); From 4a2c606e12a7f156786ed2b86c9a491bdc2dd89d Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 14 Sep 2021 01:15:03 -0700 Subject: [PATCH 179/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I7e2f06b0f93d6c3c2cc439f895083f4b7cb96212 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/sysctl.c | 10 ++++++++++ kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_lb.c | 7 +++++++ 3 files changed, 18 insertions(+) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 96bb39bfe91d..724a3849e4df 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -64,6 +64,7 @@ unsigned int sysctl_sched_sync_hint_enable = 1; unsigned int sysctl_sched_bug_on_rt_throttle; unsigned int sysctl_panic_on_walt_bug; unsigned int sysctl_sched_suppress_region2; +unsigned int sysctl_sched_skip_sp_newly_idle_lb; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -732,6 +733,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_skip_sp_newly_idle_lb", + .data = &sysctl_sched_skip_sp_newly_idle_lb, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, { .procname = "panic_on_walt_bug", .data = &sysctl_panic_on_walt_bug, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 64c097f92ab3..dd335d0cc299 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -288,6 +288,7 @@ extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by defau extern unsigned int sysctl_sched_sync_hint_enable; extern unsigned int sysctl_sched_bug_on_rt_throttle; extern unsigned int sysctl_sched_suppress_region2; +extern unsigned int sysctl_sched_skip_sp_newly_idle_lb; extern struct ctl_table walt_table[]; extern struct ctl_table walt_base_table[]; extern void walt_tunables(void); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 29830f65a90c..f8adcce301bd 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -690,6 +690,13 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][cluster]); + if (sysctl_sched_skip_sp_newly_idle_lb) { + if (busy_cpu == -1 && + ((order_index == 0 && cluster > 0) || + (order_index == 2 && cluster > 0 && !help_min_cap))) + break; + } + /* we got the busy/src cpu here. */ if (busy_cpu != -1 || this_rq->nr_running > 0) break; From 3da4f50cf498310866eaff61ab8004c4ea2b86ce Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 21 Sep 2021 13:00:36 -0700 Subject: [PATCH 180/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ia289e3ac635e22e66ae14c7cb2d171b4ad6bbf60 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/sysctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 724a3849e4df..76f929eba9dd 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -64,7 +64,7 @@ unsigned int sysctl_sched_sync_hint_enable = 1; unsigned int sysctl_sched_bug_on_rt_throttle; unsigned int sysctl_panic_on_walt_bug; unsigned int sysctl_sched_suppress_region2; -unsigned int sysctl_sched_skip_sp_newly_idle_lb; +unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; From 0808e3fa4d9ec1691d06f1dbf76317878fcbe6c6 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 14 Sep 2021 11:29:51 -0700 Subject: [PATCH 181/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ie81267e970c55da4f5f5f2a6ffb9b7f64e48503c Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/sysctl.c | 10 ++++++++++ kernel/sched/walt/walt.h | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 76f929eba9dd..85df1c2836dd 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -56,6 +56,7 @@ unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ unsigned int sysctl_sched_conservative_pl; unsigned int sysctl_sched_min_task_util_for_boost = 51; +unsigned int sysctl_sched_min_task_util_for_uclamp = 51; unsigned int sysctl_sched_min_task_util_for_colocation = 35; unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; const int sched_user_hint_max = 1000; @@ -543,6 +544,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &one_thousand, }, + { + .procname = "sched_min_task_util_for_uclamp", + .data = &sysctl_sched_min_task_util_for_uclamp, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, { .procname = "sched_min_task_util_for_colocation", .data = &sysctl_sched_min_task_util_for_colocation, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index dd335d0cc299..d6bd55f6e975 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -202,6 +202,7 @@ extern unsigned int __read_mostly sched_load_granule; /* 1ms default for 20ms window size scaled to 1024 */ extern unsigned int sysctl_sched_min_task_util_for_boost; +extern unsigned int sysctl_sched_min_task_util_for_uclamp; /* 0.68ms default for 20ms window size scaled to 1024 */ extern unsigned int sysctl_sched_min_task_util_for_colocation; extern unsigned int __read_mostly sysctl_sched_silver_thres; @@ -484,9 +485,8 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) static inline bool walt_uclamp_boosted(struct task_struct *p) { - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return uclamp_eff_value(p, UCLAMP_MIN) > 0 && wts->unfilter; + return ((uclamp_eff_value(p, UCLAMP_MIN) > 0) && + (task_util(p) > sysctl_sched_min_task_util_for_uclamp)); } static inline unsigned long capacity_of(int cpu) From b6b9ecaecb7ddd2f7ff7e3ad8cfc4f5cee0c624c Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 1 Sep 2021 17:32:14 -0700 Subject: [PATCH 182/346] sched: Improve the Scheduler This change is for general scheduler improvement. Change-Id: Id6818d7f79e8babb316aff3568cb65397df53c6f Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 2 +- kernel/sched/walt/sysctl.c | 9 +++++++++ kernel/sched/walt/trace.h | 22 ++++++++++++++++++---- kernel/sched/walt/walt.c | 25 +++++++++++++++---------- kernel/sched/walt/walt.h | 2 ++ 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index ec39456e2e8b..3d4106c4f4d9 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -43,7 +43,7 @@ struct walt_related_thread_group { struct rcu_head rcu; u64 last_update; u64 downmigrate_ts; - u64 start_ts; + u64 start_ktime_ts; }; struct walt_task_struct { diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 85df1c2836dd..091990b4b98c 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -66,6 +66,7 @@ unsigned int sysctl_sched_bug_on_rt_throttle; unsigned int sysctl_panic_on_walt_bug; unsigned int sysctl_sched_suppress_region2; unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; +unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -752,6 +753,14 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_hyst_min_coloc_ns", + .data = &sysctl_sched_hyst_min_coloc_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + }, { .procname = "panic_on_walt_bug", .data = &sysctl_panic_on_walt_bug, diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index aa7b56c95432..96f2cd1acfff 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -300,25 +300,39 @@ extern const char *migrate_type_names[]; TRACE_EVENT(sched_set_preferred_cluster, - TP_PROTO(struct walt_related_thread_group *grp, u64 total_demand), + TP_PROTO(struct walt_related_thread_group *grp, u64 total_demand, + bool prev_skip_min), - TP_ARGS(grp, total_demand), + TP_ARGS(grp, total_demand, prev_skip_min), TP_STRUCT__entry( __field(int, id) __field(u64, total_demand) __field(bool, skip_min) + __field(bool, prev_skip_min) + __field(u64, start_ktime_ts) + __field(u64, last_update) + __field(unsigned int, sysctl_sched_hyst_min_coloc_ns) + __field(u64, downmigrate_ts) ), TP_fast_assign( __entry->id = grp->id; __entry->total_demand = total_demand; __entry->skip_min = grp->skip_min; + __entry->prev_skip_min = prev_skip_min; + __entry->start_ktime_ts = grp->start_ktime_ts; + __entry->last_update = grp->last_update; + __entry->sysctl_sched_hyst_min_coloc_ns = sysctl_sched_hyst_min_coloc_ns; + __entry->downmigrate_ts = grp->downmigrate_ts; ), - TP_printk("group_id %d total_demand %llu skip_min %d", + TP_printk("group_id %d total_demand %llu skip_min %d prev_skip_min %d start_ktime_ts %llu last_update %llu min_coloc_ns %u downmigrate_ts %llu", __entry->id, __entry->total_demand, - __entry->skip_min) + __entry->skip_min, __entry->prev_skip_min, + __entry->start_ktime_ts, __entry->last_update, + __entry->sysctl_sched_hyst_min_coloc_ns, + __entry->downmigrate_ts) ); TRACE_EVENT(sched_migration_update_sum, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index bad638335220..e8dad614e7f3 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2693,7 +2693,7 @@ static DEFINE_RWLOCK(related_thread_group_lock); static inline void update_best_cluster(struct walt_related_thread_group *grp, - u64 demand, bool boost) + u64 combined_demand, bool boost) { if (boost) { /* @@ -2705,15 +2705,18 @@ void update_best_cluster(struct walt_related_thread_group *grp, } if (is_suh_max()) - demand = sched_group_upmigrate; + combined_demand = sched_group_upmigrate; if (!grp->skip_min) { - if (demand >= sched_group_upmigrate) + if (combined_demand >= sched_group_upmigrate) grp->skip_min = true; return; } - if (demand < sched_group_downmigrate) { - if (!sysctl_sched_coloc_downmigrate_ns) { + if (combined_demand < sched_group_downmigrate) { + if (!sysctl_sched_coloc_downmigrate_ns || + (grp->last_update - grp->start_ktime_ts) < + sysctl_sched_hyst_min_coloc_ns) { + grp->downmigrate_ts = 0; grp->skip_min = false; return; } @@ -2780,13 +2783,15 @@ static void _set_preferred_cluster(struct walt_related_thread_group *grp) grp->last_update = wallclock; update_best_cluster(grp, combined_demand, group_boost); - trace_sched_set_preferred_cluster(grp, combined_demand); out: + trace_sched_set_preferred_cluster(grp, combined_demand, prev_skip_min); if (grp->id == DEFAULT_CGROUP_COLOC_ID && grp->skip_min != prev_skip_min) { if (grp->skip_min) - grp->start_ts = sched_clock(); + grp->start_ktime_ts = wallclock; + else + grp->start_ktime_ts = 0; sched_update_hyst_times(); } } @@ -3297,12 +3302,12 @@ bool is_rtgb_active(void) u64 get_rtgb_active_time(void) { struct walt_related_thread_group *grp; - u64 now = sched_clock(); + u64 now = walt_ktime_get_ns(); grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); - if (grp && grp->skip_min && grp->start_ts) - return now - grp->start_ts; + if (grp && grp->skip_min && grp->start_ktime_ts) + return now - grp->start_ktime_ts; return 0; } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index d6bd55f6e975..243d16a5138d 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -225,6 +225,8 @@ extern unsigned int sysctl_input_boost_freq[8]; extern unsigned int sysctl_sched_boost_on_input; extern unsigned int sysctl_sched_user_hint; extern unsigned int sysctl_sched_conservative_pl; +extern unsigned int sysctl_sched_hyst_min_coloc_ns; + #define WALT_MANY_WAKEUP_DEFAULT 1000 extern unsigned int sysctl_sched_many_wakeup_threshold; extern unsigned int sysctl_walt_rtg_cfs_boost_prio; From a5116004fb63fb812e953986f04e8ca87955afee Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 7 Sep 2021 11:33:03 -0700 Subject: [PATCH 183/346] sched: walt: Imrpove the scheduler This change is for general scheduler improvement. Change-Id: I000f0d6aa486da31cf805f2aa1d21a821f6ebfdc Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 709050681746..c7b8e3458641 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -254,7 +254,7 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) #define NL_RATIO 75 #define DEFAULT_HISPEED_LOAD 90 #define DEFAULT_CPU0_RTG_BOOST_FREQ 1000000 -#define DEFAULT_CPU4_RTG_BOOST_FREQ 0 +#define DEFAULT_CPU4_RTG_BOOST_FREQ 768000 #define DEFAULT_CPU7_RTG_BOOST_FREQ 0 static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_util, unsigned long nl, unsigned long *util, From 71f74b8b42350d371166310ace01dd5e869cd86d Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 23 Sep 2021 13:16:45 -0700 Subject: [PATCH 184/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ib0003f97269920cb29f125da2ced34417a5a2229 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_lb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index f8adcce301bd..29e117b011ce 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -693,7 +693,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (sysctl_sched_skip_sp_newly_idle_lb) { if (busy_cpu == -1 && ((order_index == 0 && cluster > 0) || - (order_index == 2 && cluster > 0 && !help_min_cap))) + (order_index == 2 && cluster > 0))) break; } From a97a7d35149c3636421cd24fefe4c3eac2650332 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 27 Sep 2021 14:28:35 -0700 Subject: [PATCH 185/346] sched: Improve the scheduler This change is for general scheduler improvement. Change-Id: I6ba6408bd9934bde681452280c3be56cd8bc96e8 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index d5d85426949d..394487c4044e 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -988,9 +988,8 @@ static void binder_restore_priority_hook(void *data, if (unlikely(walt_disabled)) return; - if (wts->boost == TASK_BOOST_STRICT_MAX) + if (bndrtrans && wts->boost == TASK_BOOST_STRICT_MAX) wts->boost = bndrtrans->android_vendor_data1; - } /* From 52daa017317ef34deff05a0866faf775414c1a94 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 28 Sep 2021 09:51:37 -0700 Subject: [PATCH 186/346] sched: walt: Don't crash unless crash mode enabled When crashing is not enabled, let WALT attempt to correct behavior. Change-Id: Ica1c6a8aff465c5747db9acd0e16d99a7c2c4128 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index e8dad614e7f3..18117b421121 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -694,10 +694,26 @@ static inline void account_load_subtractions(struct rq *rq) ls[i].new_subs = 0; } - WALT_PANIC((s64)wrq->prev_runnable_sum < 0); - WALT_PANIC((s64)wrq->curr_runnable_sum < 0); - WALT_PANIC((s64)wrq->nt_prev_runnable_sum < 0); - WALT_PANIC((s64)wrq->nt_curr_runnable_sum < 0); + if ((s64)wrq->prev_runnable_sum < 0) { + WALT_BUG(NULL, "wrq->prev_runnable_sum=%llu < 0", + (s64)wrq->prev_runnable_sum); + wrq->prev_runnable_sum = 0; + } + if ((s64)wrq->curr_runnable_sum < 0) { + WALT_BUG(NULL, "wrq->curr_runnable_sum=%llu < 0", + (s64)wrq->curr_runnable_sum); + wrq->curr_runnable_sum = 0; + } + if ((s64)wrq->nt_prev_runnable_sum < 0) { + WALT_BUG(NULL, "wrq->nt_prev_runnable_sum=%llu < 0", + (s64)wrq->nt_prev_runnable_sum); + wrq->nt_prev_runnable_sum = 0; + } + if ((s64)wrq->nt_curr_runnable_sum < 0) { + WALT_BUG(NULL, "wrq->nt_curr_runnable_sum=%llu < 0", + (s64)wrq->nt_curr_runnable_sum); + wrq->nt_curr_runnable_sum = 0; + } } static inline void create_subtraction_entry(struct rq *rq, u64 ws, int index) From 544665ace607d82f4987c2944ddd345b555ec0cb Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 28 Sep 2021 16:13:48 -0700 Subject: [PATCH 187/346] sched/walt: improve task_util trace and variable names This change is for general scheduler improvement. Change-Id: Ia9be6164f683a5fa29aa5d1ce13633e1cc6016ba Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/trace.h | 7 +++++-- kernel/sched/walt/walt_cfs.c | 19 +++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 96f2cd1acfff..87c08f6d8eac 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1043,6 +1043,7 @@ TRACE_EVENT(sched_task_util, __field(unsigned long, cpus_allowed) __field(int, task_boost) __field(bool, low_latency) + __field(bool, iowaited) ), TP_fast_assign( @@ -1066,16 +1067,18 @@ TRACE_EVENT(sched_task_util, __entry->cpus_allowed = cpumask_bits(&p->cpus_mask)[0]; __entry->task_boost = per_task_boost(p); __entry->low_latency = walt_low_latency_task(p); + __entry->iowaited = + ((struct walt_task_struct *) p->android_vendor_data1)->iowaited; ), - TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d", + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d iowaited=%d", __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, __entry->candidates, __entry->best_energy_cpu, __entry->sync, __entry->need_idle, __entry->fastpath, __entry->placement_boost, __entry->latency, __entry->uclamp_boosted, __entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu, __entry->unfilter, __entry->cpus_allowed, __entry->task_boost, - __entry->low_latency) + __entry->low_latency, __entry->iowaited) ); /* diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 394487c4044e..6288bb794375 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -226,7 +226,7 @@ static void walt_find_best_target(struct sched_domain *sd, struct task_struct *p, struct find_best_target_env *fbt_env) { - unsigned long min_util = uclamp_task_util(p); + unsigned long min_task_util = uclamp_task_util(p); long target_max_spare_cap = 0; unsigned long best_idle_cuml_util = ULONG_MAX; unsigned int min_exit_latency = UINT_MAX; @@ -285,7 +285,7 @@ static void walt_find_best_target(struct sched_domain *sd, &cpu_array[order_index][cluster]); for_each_cpu(i, &visit_cpus) { unsigned long capacity_orig = capacity_orig_of(i); - unsigned long wake_util, new_util, new_util_cuml; + unsigned long wake_cpu_util, new_cpu_util, new_util_cuml; long spare_cap; unsigned int idle_exit_latency = UINT_MAX; struct walt_rq *wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; @@ -322,9 +322,8 @@ static void walt_find_best_target(struct sched_domain *sd, * so prev_cpu will receive a negative bias due to the double * accounting. However, the blocked utilization may be zero. */ - wake_util = cpu_util_without(i, p); - new_util = wake_util + min_util; - spare_wake_cap = capacity_orig - wake_util; + wake_cpu_util = cpu_util_without(i, p); + spare_wake_cap = capacity_orig - wake_cpu_util; if (spare_wake_cap > most_spare_wake_cap) { most_spare_wake_cap = spare_wake_cap; @@ -336,8 +335,8 @@ static void walt_find_best_target(struct sched_domain *sd, * The target CPU can be already at a capacity level higher * than the one required to boost the task. */ - new_util = max(min_util, new_util); - if (new_util > capacity_orig) + new_cpu_util = wake_cpu_util + min_task_util; + if (new_cpu_util > capacity_orig) continue; /* @@ -397,7 +396,7 @@ static void walt_find_best_target(struct sched_domain *sd, * to have available on this CPU once the task is * enqueued here. */ - spare_cap = capacity_orig - new_util; + spare_cap = capacity_orig - new_cpu_util; /* * Try to spread the rtg high prio tasks so that they @@ -454,7 +453,7 @@ static void walt_find_best_target(struct sched_domain *sd, } out: - trace_sched_find_best_target(p, min_util, start_cpu, cpumask_bits(candidates)[0], + trace_sched_find_best_target(p, min_task_util, start_cpu, cpumask_bits(candidates)[0], most_spare_cap_cpu, order_index, end_index, fbt_env->skip_cpu, task_on_rq_queued(p)); } @@ -883,7 +882,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, done: trace_sched_task_util(p, cpumask_bits(candidates)[0], best_energy_cpu, sync, fbt_env.need_idle, fbt_env.fastpath, - start_t, uclamp_boost || task_boost, start_cpu); + start_t, uclamp_boost, start_cpu); return best_energy_cpu; From d404d1993356ff3170b3d722f05af75366602887 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Sat, 18 Sep 2021 15:41:11 -0700 Subject: [PATCH 188/346] sched/walt: Fix caching freq earlier than down rate limit check Raw frequency was being cached earlier than the down rate limit check causing the calculated frequency to be cached even in a situation where the up/down rate limit check failed a frequency update. The following changes have been made wrt to the above observation: 1. The up/down rate limit check has been moved further up the frequency update path, where frequency is resolved and cached_raw_freq is utilized to ensure that all possible frequency checks are consolidated to one location. 2. Reset cached_raw_freq in the event of down rate limit check failing, thus forcing the frequency update path to resolve a new frequency value instead of using an invalid cached_raw_frequency and ensuring that the latest frequency is applied. Change-Id: I449ca6552aeb5f30be9d29ac472c0ba421f25da4 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/cpufreq_walt.c | 47 ++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index c7b8e3458641..192a395dd858 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -115,14 +115,18 @@ static bool waltgov_up_down_rate_limit(struct waltgov_policy *wg_policy, u64 tim } static bool waltgov_update_next_freq(struct waltgov_policy *wg_policy, u64 time, - unsigned int next_freq) + unsigned int next_freq, + unsigned int raw_freq) { if (wg_policy->next_freq == next_freq) return false; - if (waltgov_up_down_rate_limit(wg_policy, time, next_freq)) + if (waltgov_up_down_rate_limit(wg_policy, time, next_freq)) { + wg_policy->cached_raw_freq = 0; return false; + } + wg_policy->cached_raw_freq = raw_freq; wg_policy->next_freq = next_freq; wg_policy->last_freq_update_time = time; @@ -184,9 +188,6 @@ static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time, { struct cpufreq_policy *policy = wg_policy->policy; - if (!waltgov_update_next_freq(wg_policy, time, next_freq)) - return; - waltgov_track_cycles(wg_policy, wg_policy->policy->cur, time); cpufreq_driver_fast_switch(policy, next_freq); } @@ -194,9 +195,6 @@ static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time, static void waltgov_deferred_update(struct waltgov_policy *wg_policy, u64 time, unsigned int next_freq) { - if (!waltgov_update_next_freq(wg_policy, time, next_freq)) - return; - walt_irq_work_queue(&wg_policy->irq_work); } @@ -214,10 +212,10 @@ static inline unsigned long walt_map_util_freq(unsigned long util, static unsigned int get_next_freq(struct waltgov_policy *wg_policy, unsigned long util, unsigned long max, - struct waltgov_cpu *wg_cpu) + struct waltgov_cpu *wg_cpu, u64 time) { struct cpufreq_policy *policy = wg_policy->policy; - unsigned int freq, raw_freq; + unsigned int freq, raw_freq, final_freq; raw_freq = walt_map_util_freq(util, policy->cpuinfo.max_freq, max, wg_cpu->cpu); freq = raw_freq; @@ -232,12 +230,18 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, trace_waltgov_next_freq(policy->cpu, util, max, raw_freq, freq, policy->min, policy->max, wg_policy->cached_raw_freq, wg_policy->need_freq_update); - if (freq == wg_policy->cached_raw_freq && !wg_policy->need_freq_update) + if (wg_policy->cached_raw_freq && freq == wg_policy->cached_raw_freq && + !wg_policy->need_freq_update) return wg_policy->next_freq; wg_policy->need_freq_update = false; - wg_policy->cached_raw_freq = freq; - return cpufreq_driver_resolve_freq(policy, freq); + + final_freq = cpufreq_driver_resolve_freq(policy, freq); + + if (!waltgov_update_next_freq(wg_policy, time, final_freq, freq)) + return 0; + + return final_freq; } static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) @@ -337,7 +341,7 @@ static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 tim waltgov_walt_adjust(j_wg_cpu, j_util, j_nl, &util, &max); } - return get_next_freq(wg_policy, util, max, wg_cpu); + return get_next_freq(wg_policy, util, max, wg_cpu, time); } static void waltgov_update_freq(struct waltgov_callback *cb, u64 time, @@ -378,12 +382,16 @@ static void waltgov_update_freq(struct waltgov_callback *cb, u64 time, !(flags & WALT_CPUFREQ_CONTINUE)) { next_f = waltgov_next_freq_shared(wg_cpu, time); + if (!next_f) + goto out; + if (wg_policy->policy->fast_switch_enabled) waltgov_fast_switch(wg_policy, time, next_f); else waltgov_deferred_update(wg_policy, time, next_f); } +out: raw_spin_unlock(&wg_policy->update_lock); } @@ -935,7 +943,7 @@ static void waltgov_limits(struct cpufreq_policy *policy) { struct waltgov_policy *wg_policy = policy->governor_data; unsigned long flags, now; - unsigned int freq; + unsigned int freq, final_freq; if (!policy->fast_switch_enabled) { mutex_lock(&wg_policy->work_lock); @@ -954,9 +962,12 @@ static void waltgov_limits(struct cpufreq_policy *policy) * cpufreq_driver_resolve_freq() has a clamp, so we do not need * to do any sort of additional validation here. */ - freq = cpufreq_driver_resolve_freq(policy, freq); - wg_policy->cached_raw_freq = freq; - waltgov_fast_switch(wg_policy, now, freq); + final_freq = cpufreq_driver_resolve_freq(policy, freq); + + if (waltgov_update_next_freq(wg_policy, now, final_freq, + final_freq)) { + waltgov_fast_switch(wg_policy, now, final_freq); + } raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); } From e35de28c324413990279d92bf756cdb17044f67f Mon Sep 17 00:00:00 2001 From: Rishabh Bhatnagar Date: Tue, 24 Aug 2021 12:33:34 -0700 Subject: [PATCH 189/346] sched: walt: Add target load tunables Add target load threshold and target load shift tunables for each policy. This allows us to scale cpu frequency in a configurable way for each cluster. Change-Id: I2c41ce6ef8b5a91f0b55be6af075db70695efd98 Signed-off-by: Rishabh Bhatnagar --- kernel/sched/walt/cpufreq_walt.c | 36 +++++++++++++++++++++++++------- kernel/sched/walt/sysctl.c | 12 ----------- kernel/sched/walt/walt.h | 1 - 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 192a395dd858..f8fb81715bc4 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -24,6 +24,8 @@ struct waltgov_tunables { unsigned int rtg_boost_freq; unsigned int adaptive_low_freq; unsigned int adaptive_high_freq; + unsigned int target_load_thresh; + unsigned int target_load_shift; bool pl; int boost; }; @@ -200,13 +202,15 @@ static void waltgov_deferred_update(struct waltgov_policy *wg_policy, u64 time, #define TARGET_LOAD 80 static inline unsigned long walt_map_util_freq(unsigned long util, - unsigned long fmax, unsigned long cap, - int cpu) + struct waltgov_policy *wg_policy, + unsigned long cap, int cpu) { - if (is_min_capacity_cpu(cpu) && - util >= sysctl_sched_silver_thres && - cpu_util_rt(cpu_rq(cpu)) < (cap >> 2)) - return (fmax + (fmax >> 4)) * util / cap; + unsigned long fmax = wg_policy->policy->cpuinfo.max_freq; + unsigned int shift = wg_policy->tunables->target_load_shift; + + if (util >= wg_policy->tunables->target_load_thresh && + cpu_util_rt(cpu_rq(cpu)) < (cap >> 2)) + return (fmax + (fmax >> shift)) * util / cap; return (fmax + (fmax >> 2)) * util / cap; } @@ -217,7 +221,7 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, struct cpufreq_policy *policy = wg_policy->policy; unsigned int freq, raw_freq, final_freq; - raw_freq = walt_map_util_freq(util, policy->cpuinfo.max_freq, max, wg_cpu->cpu); + raw_freq = walt_map_util_freq(util, wg_policy, max, wg_cpu->cpu); freq = raw_freq; if (wg_policy->tunables->adaptive_high_freq) { @@ -260,6 +264,8 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) #define DEFAULT_CPU0_RTG_BOOST_FREQ 1000000 #define DEFAULT_CPU4_RTG_BOOST_FREQ 768000 #define DEFAULT_CPU7_RTG_BOOST_FREQ 0 +#define DEFAULT_TARGET_LOAD_THRESH 1024 +#define DEFAULT_TARGET_LOAD_SHIFT 4 static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_util, unsigned long nl, unsigned long *util, unsigned long *max) @@ -298,7 +304,7 @@ static inline unsigned long target_util(struct waltgov_policy *wg_policy, util = freq_to_util(wg_policy, freq); if (wg_policy->max == min_max_possible_capacity && - util >= sysctl_sched_silver_thres) + util >= wg_policy->tunables->target_load_thresh) util = mult_frac(util, 94, 100); else util = mult_frac(util, TARGET_LOAD, 100); @@ -654,6 +660,10 @@ show_attr(adaptive_low_freq); store_attr(adaptive_low_freq); show_attr(adaptive_high_freq); store_attr(adaptive_high_freq); +show_attr(target_load_thresh); +store_attr(target_load_thresh); +show_attr(target_load_shift); +store_attr(target_load_shift); static struct governor_attr hispeed_load = __ATTR_RW(hispeed_load); static struct governor_attr hispeed_freq = __ATTR_RW(hispeed_freq); @@ -662,6 +672,8 @@ static struct governor_attr pl = __ATTR_RW(pl); static struct governor_attr boost = __ATTR_RW(boost); WALTGOV_ATTR_RW(adaptive_low_freq); WALTGOV_ATTR_RW(adaptive_high_freq); +WALTGOV_ATTR_RW(target_load_thresh); +WALTGOV_ATTR_RW(target_load_shift); static struct attribute *waltgov_attributes[] = { &up_rate_limit_us.attr, @@ -673,6 +685,8 @@ static struct attribute *waltgov_attributes[] = { &boost.attr, &adaptive_low_freq.attr, &adaptive_high_freq.attr, + &target_load_thresh.attr, + &target_load_shift.attr, NULL }; @@ -776,6 +790,8 @@ static void waltgov_tunables_save(struct cpufreq_policy *policy, cached->boost = tunables->boost; cached->adaptive_low_freq = tunables->adaptive_low_freq; cached->adaptive_high_freq = tunables->adaptive_high_freq; + cached->target_load_thresh = tunables->target_load_thresh; + cached->target_load_shift = tunables->target_load_shift; } static void waltgov_tunables_restore(struct cpufreq_policy *policy) @@ -796,6 +812,8 @@ static void waltgov_tunables_restore(struct cpufreq_policy *policy) tunables->boost = cached->boost; tunables->adaptive_low_freq = cached->adaptive_low_freq; tunables->adaptive_high_freq = cached->adaptive_high_freq; + tunables->target_load_thresh = cached->target_load_thresh; + tunables->target_load_shift = cached->target_load_shift; } static int waltgov_init(struct cpufreq_policy *policy) @@ -831,6 +849,8 @@ static int waltgov_init(struct cpufreq_policy *policy) gov_attr_set_init(&tunables->attr_set, &wg_policy->tunables_hook); tunables->hispeed_load = DEFAULT_HISPEED_LOAD; + tunables->target_load_thresh = DEFAULT_TARGET_LOAD_THRESH; + tunables->target_load_shift = DEFAULT_TARGET_LOAD_SHIFT; switch (policy->cpu) { default: diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 091990b4b98c..5c0840029da3 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -48,7 +48,6 @@ unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; unsigned int __read_mostly sysctl_sched_group_downmigrate_pct; unsigned int __read_mostly sysctl_sched_group_upmigrate_pct; unsigned int __read_mostly sysctl_sched_window_stats_policy; -unsigned int __read_mostly sysctl_sched_silver_thres; unsigned int sysctl_sched_ravg_window_nr_ticks; unsigned int sysctl_sched_walt_rotate_big_tasks; unsigned int sysctl_sched_task_unfilter_period; @@ -474,15 +473,6 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &four, }, - { - .procname = "sched_silver_thres", - .data = &sysctl_sched_silver_thres, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = &one_thousand, - }, { .procname = "sched_group_upmigrate", .data = &sysctl_sched_group_upmigrate_pct, @@ -882,8 +872,6 @@ void walt_tunables(void) sysctl_sched_window_stats_policy = WINDOW_STATS_MAX_RECENT_AVG; - sysctl_sched_silver_thres = 1000; - sysctl_sched_ravg_window_nr_ticks = (HZ / NR_WINDOWS_PER_SEC); sched_load_granule = DEFAULT_SCHED_RAVG_WINDOW / NUM_LOAD_INDICES; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 243d16a5138d..ffd3e92c49e1 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -205,7 +205,6 @@ extern unsigned int sysctl_sched_min_task_util_for_boost; extern unsigned int sysctl_sched_min_task_util_for_uclamp; /* 0.68ms default for 20ms window size scaled to 1024 */ extern unsigned int sysctl_sched_min_task_util_for_colocation; -extern unsigned int __read_mostly sysctl_sched_silver_thres; extern unsigned int sysctl_sched_busy_hyst_enable_cpus; extern unsigned int sysctl_sched_busy_hyst; extern unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus; From 1900694cd024935ebd61d6b31d2941feccbadd02 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 6 Oct 2021 17:00:05 -0700 Subject: [PATCH 190/346] sched/walt: fix target load crossover This change is for general scheduler improvement. Change-Id: I28ccf141ccd52c0b8afc4548f83dd4e69b954670 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index f8fb81715bc4..689a4d3e080e 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -210,7 +210,10 @@ static inline unsigned long walt_map_util_freq(unsigned long util, if (util >= wg_policy->tunables->target_load_thresh && cpu_util_rt(cpu_rq(cpu)) < (cap >> 2)) - return (fmax + (fmax >> shift)) * util / cap; + return max( + (fmax + (fmax >> shift)) * util, + (fmax + (fmax >> 2)) * wg_policy->tunables->target_load_thresh + )/cap; return (fmax + (fmax >> 2)) * util / cap; } From 24a23474223eaabd692342c68af4383042fa264e Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 5 Oct 2021 11:56:31 -0700 Subject: [PATCH 191/346] sched: walt: Create pipeline "low latency" tasks This change is for general scheduler improvement. Change-Id: I94d790387a8d528fdfd954a8db2c8783eb090644 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/sysctl.c | 18 ++++++++ kernel/sched/walt/walt.h | 85 ++++++++++++++++++++---------------- kernel/sched/walt/walt_cfs.c | 6 ++- kernel/sched/walt/walt_lb.c | 2 + 4 files changed, 71 insertions(+), 40 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 5c0840029da3..03cef3c1c05c 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -186,6 +186,7 @@ enum { PER_TASK_BOOST, PER_TASK_BOOST_PERIOD_MS, LOW_LATENCY, + PIPELINE, }; static int sched_task_handler(struct ctl_table *table, int write, @@ -238,6 +239,10 @@ static int sched_task_handler(struct ctl_table *table, int write, pid_and_val[1] = wts->low_latency & WALT_LOW_LATENCY_PROCFS; break; + case PIPELINE: + pid_and_val[1] = wts->low_latency & + WALT_LOW_LATENCY_PIPELINE; + break; default: ret = -EINVAL; goto put_task; @@ -302,6 +307,12 @@ static int sched_task_handler(struct ctl_table *table, int write, else wts->low_latency &= ~WALT_LOW_LATENCY_PROCFS; break; + case PIPELINE: + if (val) + wts->low_latency |= WALT_LOW_LATENCY_PIPELINE; + else + wts->low_latency &= ~WALT_LOW_LATENCY_PIPELINE; + break; default: ret = -EINVAL; } @@ -823,6 +834,13 @@ struct ctl_table walt_table[] = { .mode = 0644, .proc_handler = sched_task_handler, }, + { + .procname = "sched_pipeline", + .data = (int *) PIPELINE, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, { .procname = "sched_task_read_pid", .data = &sysctl_task_read_pid, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index ffd3e92c49e1..0fb5a831cba7 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -49,8 +49,9 @@ enum migrate_types { RQ_TO_GROUP, }; -#define WALT_LOW_LATENCY_PROCFS BIT(0) -#define WALT_LOW_LATENCY_BINDER BIT(1) +#define WALT_LOW_LATENCY_PROCFS BIT(0) +#define WALT_LOW_LATENCY_BINDER BIT(1) +#define WALT_LOW_LATENCY_PIPELINE BIT(2) struct walt_cpu_load { unsigned long nl; @@ -419,6 +420,48 @@ static inline unsigned long cpu_util_cum(int cpu) return READ_ONCE(cpu_rq(cpu)->cfs.avg.util_avg); } +/* applying the task threshold for all types of low latency tasks. */ +static inline bool walt_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->low_latency && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_binder_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_BINDER) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_procfs_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_PROCFS) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_pipeline_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->low_latency & WALT_LOW_LATENCY_PIPELINE; +} + +static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) +{ + struct cpuidle_state *idle = idle_get_state(rq); + + if (idle) + return idle->exit_latency; + + return 0; /* CPU is not idle */ +} + static inline bool rt_boost_on_big(void) { return sched_boost_type == FULL_THROTTLE_BOOST ? @@ -477,7 +520,8 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) * under conservative boost. */ if (sched_boost_type == CONSERVATIVE_BOOST && - task_util(p) <= sysctl_sched_min_task_util_for_boost) + task_util(p) <= sysctl_sched_min_task_util_for_boost && + !walt_pipeline_low_latency_task(p)) policy = SCHED_BOOST_NONE; } @@ -725,41 +769,6 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) return task_fits_capacity(p, capacity, cpu); } -/* applying the task threshold for all types of low latency tasks. */ -static inline bool walt_low_latency_task(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return wts->low_latency && - (task_util(p) < sysctl_walt_low_latency_task_threshold); -} - -static inline bool walt_binder_low_latency_task(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return (wts->low_latency & WALT_LOW_LATENCY_BINDER) && - (task_util(p) < sysctl_walt_low_latency_task_threshold); -} - -static inline bool walt_procfs_low_latency_task(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return (wts->low_latency & WALT_LOW_LATENCY_PROCFS) && - (task_util(p) < sysctl_walt_low_latency_task_threshold); -} - -static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) -{ - struct cpuidle_state *idle = idle_get_state(rq); - - if (idle) - return idle->exit_latency; - - return 0; /* CPU is not idle */ -} - extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); extern void sched_update_hyst_times(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 6288bb794375..13086c193a71 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -138,7 +138,8 @@ static inline bool walt_task_skip_min_cpu(struct task_struct *p) struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; return (sched_boost_type != CONSERVATIVE_BOOST) && - walt_get_rtg_status(p) && wts->unfilter; + walt_get_rtg_status(p) && (wts->unfilter || + walt_pipeline_low_latency_task(p)); } static inline bool walt_is_many_wakeup(int sibling_count_hint) @@ -1007,7 +1008,8 @@ static inline int walt_get_mvp_task_prio(struct task_struct *p) if (walt_binder_low_latency_task(p)) return WALT_BINDER_MVP; - if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p)) + if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p) || + walt_pipeline_low_latency_task(p)) return WALT_RTG_MVP; return WALT_NOT_MVP; diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 29e117b011ce..e760f83cf3c3 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -225,6 +225,8 @@ static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, if (per_task_boost(p) == TASK_BOOST_STRICT_MAX && task_in_related_thread_group(p)) return false; + if (walt_pipeline_low_latency_task(p)) + return false; } /* Don't detach task if it is under active migration */ From e2d43d820a4814756b1e3fdc7f1e57dadbdf213b Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 8 Oct 2021 12:44:58 -0700 Subject: [PATCH 192/346] sched/walt: Remove newline in traces This change is for general scheduler improvement. Change-Id: I2873e2f0cb5a8e1d2349644c796bf188275882b6 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 87c08f6d8eac..ee8f4f7c99fe 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -815,7 +815,7 @@ TRACE_EVENT(walt_nohz_balance_kick, __entry->nr_cfs_running = rq->cfs.h_nr_running; ), - TP_printk("cpu=%d nr_running=%u nr_cfs_running=%u\n", + TP_printk("cpu=%d nr_running=%u nr_cfs_running=%u", __entry->cpu, __entry->nr_running, __entry->nr_cfs_running) ); @@ -852,7 +852,7 @@ TRACE_EVENT(walt_newidle_balance, __entry->overload = cpu_rq(this_cpu)->rd->overload; ), - TP_printk("cpu=%d busy_cpu=%d pulled=%d nr_running=%u rt_nr_running=%u nr_iowait=%d help_min_cap=%d avg_idle=%llu enough_idle=%d overload=%d\n", + TP_printk("cpu=%d busy_cpu=%d pulled=%d nr_running=%u rt_nr_running=%u nr_iowait=%d help_min_cap=%d avg_idle=%llu enough_idle=%d overload=%d", __entry->cpu, __entry->busy_cpu, __entry->pulled, __entry->nr_running, __entry->rt_nr_running, __entry->nr_iowait, __entry->help_min_cap, From e611070ee20749f6556770fa32df90be7e178137 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 22 Sep 2021 11:19:38 -0700 Subject: [PATCH 193/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I9cd7715f390d52163b7280ebb6e9f04bdfc38d4c Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_cfs.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 13086c193a71..27c5b825dc8f 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -693,23 +693,36 @@ static inline int wake_to_idle(struct task_struct *p) /* return true if cpu should be chosen over best_energy_cpu */ static inline bool select_cpu_same_energy(int cpu, int best_cpu, int prev_cpu) { + bool new_cpu_is_idle = available_idle_cpu(cpu); + bool best_cpu_is_idle = available_idle_cpu(best_cpu); + + if (capacity_orig_of(best_cpu) < capacity_orig_of(cpu)) + return false; if (capacity_orig_of(cpu) < capacity_orig_of(best_cpu)) return true; + if (best_cpu_is_idle && walt_get_idle_exit_latency(cpu_rq(best_cpu)) <= 1) + return false; + if (new_cpu_is_idle && walt_get_idle_exit_latency(cpu_rq(cpu)) <= 1) + return true; + + if (best_cpu_is_idle && !new_cpu_is_idle) + return false; + if (new_cpu_is_idle && !best_cpu_is_idle) + return true; + if (best_cpu == prev_cpu) return false; + if (cpu == prev_cpu) + return true; - if (available_idle_cpu(best_cpu) && walt_get_idle_exit_latency(cpu_rq(best_cpu)) <= 1) - return false; /* best_cpu is idle wfi or shallower */ + if (best_cpu_is_idle && new_cpu_is_idle) + return false; - if (available_idle_cpu(cpu) && walt_get_idle_exit_latency(cpu_rq(cpu)) <= 1) - return true; /* new cpu is idle wfi or shallower */ + if (cpu_util(best_cpu) <= cpu_util(cpu)) + return false; - /* - * If we are this far this must be a tie between a busy and deep idle, - * pick the busy. - */ - return available_idle_cpu(best_cpu); + return true; } static inline unsigned int capacity_spare_of(int cpu) From 822a7c683b3425447d40871332e2c3002bce60d9 Mon Sep 17 00:00:00 2001 From: Lingutla Chandrasekhar Date: Tue, 31 Aug 2021 19:31:04 +0530 Subject: [PATCH 194/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Ic5929726768f6a9ca2a5823e2795eeb5e172d93a Signed-off-by: Lingutla Chandrasekhar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/sysctl.c | 10 ++++++++++ kernel/sched/walt/walt.h | 3 +++ kernel/sched/walt/walt_cfs.c | 4 ++++ kernel/sched/walt/walt_lb.c | 13 ++++++++----- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 03cef3c1c05c..f4da057d1387 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -66,6 +66,7 @@ unsigned int sysctl_panic_on_walt_bug; unsigned int sysctl_sched_suppress_region2; unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000; +unsigned int sysctl_sched_asymcap_boost; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -859,6 +860,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_asymcap_boost", + .data = &sysctl_sched_asymcap_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, { } }; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 0fb5a831cba7..4008992f5315 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -292,6 +292,7 @@ extern unsigned int sysctl_sched_sync_hint_enable; extern unsigned int sysctl_sched_bug_on_rt_throttle; extern unsigned int sysctl_sched_suppress_region2; extern unsigned int sysctl_sched_skip_sp_newly_idle_lb; +extern unsigned int sysctl_sched_asymcap_boost; extern struct ctl_table walt_table[]; extern struct ctl_table walt_base_table[]; extern void walt_tunables(void); @@ -909,6 +910,8 @@ void walt_lb_tick(struct rq *rq); extern __read_mostly unsigned int walt_scale_demand_divisor; #define scale_demand(d) ((d)/walt_scale_demand_divisor) +#define ASYMCAP_BOOST(cpu) (sysctl_sched_asymcap_boost && !is_min_capacity_cpu(cpu)) + void create_util_to_cost(void); struct compute_energy_output { unsigned long sum_util[MAX_CLUSTERS]; diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 27c5b825dc8f..c40a400969be 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -187,6 +187,10 @@ static void walt_get_indicies(struct task_struct *p, int *order_index, walt_task_skip_min_cpu(p)) { *energy_eval_needed = false; *order_index = 1; + if (sysctl_sched_asymcap_boost) { + *end_index = 1; + return; + } } for (i = *order_index ; i < num_sched_clusters - 1; i++) { diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index e760f83cf3c3..f3d00fc03293 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -356,7 +356,7 @@ static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src unsigned long total_capacity = 0, total_util = 0, total_nr = 0; int total_cpus = 0; struct walt_rq *wrq; - + bool asymcap_boost = ASYMCAP_BOOST(dst_cpu); for_each_cpu(i, src_mask) { if (!cpu_active(i)) @@ -384,7 +384,8 @@ static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src * overutilized but for rotation, we have to * spread out. */ - if (!walt_rotation_enabled && !cpu_overutilized(i)) + if (!walt_rotation_enabled && !cpu_overutilized(i) && + !asymcap_boost) continue; if (util < busiest_util) @@ -398,7 +399,7 @@ static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src * Don't allow migrating to lower cluster unless this high * capacity cluster is sufficiently loaded. */ - if (!walt_rotation_enabled) { + if (!walt_rotation_enabled && !asymcap_boost) { if (total_nr <= total_cpus || total_util * 1280 < total_capacity * 1024) busiest_cpu = -1; } @@ -449,7 +450,8 @@ static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_ (!wrq->walt_stats.nr_big_tasks || !available_idle_cpu(dst_cpu))) continue; - if (!walt_rotation_enabled && !cpu_overutilized(i)) + if (!walt_rotation_enabled && !cpu_overutilized(i) && + !ASYMCAP_BOOST(i)) continue; if (util < busiest_util) @@ -460,7 +462,8 @@ static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_ busy_nr_big_tasks = wrq->walt_stats.nr_big_tasks; } - if (!walt_rotation_enabled && !busy_nr_big_tasks) { + if (!walt_rotation_enabled && !busy_nr_big_tasks && + !(busiest_cpu != -1 && ASYMCAP_BOOST(busiest_cpu))) { if (total_nr <= total_cpus || total_util * 1280 < total_capacity * 1024) busiest_cpu = -1; } From 03ee181e27775f12b040b3b5930cf4c344c62e49 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 14 Oct 2021 16:33:40 -0700 Subject: [PATCH 195/346] sched/walt: Change task state variable Upstream changed task's state variable in struct task_struct to non-volatile. Reflect these changes downstream. Change-Id: Icc9736e6b5f6fb5b7d02fde6d3117a9131d98b50 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 10 +++++----- kernel/sched/walt/walt_cfs.c | 6 +++--- kernel/sched/walt/walt_lb.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 18117b421121..941729ae8a76 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -184,7 +184,7 @@ void walt_task_dump(struct task_struct *p) bool is_32bit_thread = is_compat_thread(task_thread_info(p)); printk_deferred("Task: %.16s-%d\n", p->comm, p->pid); - SCHED_PRINT(p->state); + SCHED_PRINT(READ_ONCE(p->__state)); SCHED_PRINT(p->cpu); SCHED_PRINT(p->policy); SCHED_PRINT(p->prio); @@ -580,7 +580,7 @@ static inline u64 freq_policy_load(struct rq *rq) load = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; - if (cpu_ksoftirqd && cpu_ksoftirqd->state == TASK_RUNNING) + if (cpu_ksoftirqd && READ_ONCE(cpu_ksoftirqd->__state) == TASK_RUNNING) load = max_t(u64, load, task_load(cpu_ksoftirqd)); tt_load = top_task_load(rq); @@ -967,10 +967,10 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (!p->on_rq && p->state != TASK_WAKING) + if (!p->on_rq && READ_ONCE(p->__state) != TASK_WAKING) return; - pstate = p->state; + pstate = READ_ONCE(p->__state); if (pstate == TASK_WAKING) double_rq_lock(src_rq, dest_rq); @@ -2181,7 +2181,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even update_task_demand(p, rq, event, wallclock); update_cpu_busy_time(p, rq, event, wallclock, irqtime); update_task_pred_demand(rq, p, event); - if (event == PUT_PREV_TASK && p->state) + if (event == PUT_PREV_TASK && READ_ONCE(p->__state)) wts->iowaited = p->in_iowait; trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index c40a400969be..29d262d82549 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -116,7 +116,7 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) * utilization from cpu utilization. Instead just use * cpu_util for this case. */ - if (likely(p->state == TASK_WAKING)) + if (likely(READ_ONCE(p->__state) == TASK_WAKING)) return cpu_util(cpu); /* Task has no contribution or is new */ @@ -835,7 +835,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, goto unlock; } - if (p->state == TASK_WAKING) + if (READ_ONCE(p->__state) == TASK_WAKING) delta = task_util(p); if (cpumask_test_cpu(prev_cpu, &p->cpus_mask) && !__cpu_overutilized(prev_cpu, delta)) { @@ -1172,7 +1172,7 @@ void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) * be preserved when task is enq/deq while it is on * runqueue. */ - if (p->state != TASK_RUNNING) + if (READ_ONCE(p->__state) != TASK_RUNNING) wts->total_exec = 0; } diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index f3d00fc03293..01d1c15a1ac1 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -58,7 +58,7 @@ static int walt_lb_active_migration(void *data) BUG_ON(busiest_rq == target_rq); if (task_on_rq_queued(push_task) && - push_task->state == TASK_RUNNING && + READ_ONCE(push_task->__state) == TASK_RUNNING && task_cpu(push_task) == busiest_cpu && cpu_active(target_cpu) && cpumask_test_cpu(target_cpu, push_task->cpus_ptr)) { @@ -506,7 +506,7 @@ void walt_lb_tick(struct rq *rq) if (!rq->misfit_task_load) return; - if (p->state != TASK_RUNNING || p->nr_cpus_allowed == 1) + if (READ_ONCE(p->__state) != TASK_RUNNING || p->nr_cpus_allowed == 1) return; raw_spin_lock_irqsave(&walt_lb_migration_lock, flags); From d3e1802b11df9a11cb4042137f149083f422a84f Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 14 Oct 2021 16:39:17 -0700 Subject: [PATCH 196/346] sched_walt: Change from MAX_USER_RT_PRIO to MAX_RT_PRIO Upstream has done away with the MAX_USER_RT_PRIO macro. Reflect these changes downstream. Change-Id: I836bcae569af2e98db5cfc1330d04c80c5e68563 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/cpufreq_walt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 689a4d3e080e..689f47fe5d35 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -723,7 +723,7 @@ static void waltgov_policy_free(struct waltgov_policy *wg_policy) static int waltgov_kthread_create(struct waltgov_policy *wg_policy) { struct task_struct *thread; - struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 }; + struct sched_param param = { .sched_priority = MAX_RT_PRIO / 2 }; struct cpufreq_policy *policy = wg_policy->policy; int ret; From b634ecd65b605a9cf2786fb38dc314c3dad97946 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 14 Oct 2021 16:42:26 -0700 Subject: [PATCH 197/346] sched/walt: Remove redefinition of 'task_of' function 'task_of' function is defined in sched.h Remove redefinition in walt_cfs.c Change-Id: I07dd511fae0a74a03ebf2ae47f2b227404866972 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_cfs.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 29d262d82549..cd9901232973 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -928,11 +928,6 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, *target_cpu = prev_cpu; } -static inline struct task_struct *task_of(struct sched_entity *se) -{ - return container_of(se, struct task_struct, se); -} - static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vruntime) { if (unlikely(walt_disabled)) From 031f8e8239fe61a6e4b774a5059aa344aac856d8 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 14 Oct 2021 17:47:49 -0700 Subject: [PATCH 198/346] sched/walt: Change rq lock variable as per upstream The rq lock variable in struct rq is changed upstream. Reflect changes downstream. Change-Id: I8711d42dbe10f769b8568895993b92a6bcbcd69e Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/cpufreq_walt.c | 4 +- kernel/sched/walt/sched_avg.c | 4 +- kernel/sched/walt/sysctl.c | 4 +- kernel/sched/walt/walt.c | 64 ++++++++++++++++---------------- kernel/sched/walt/walt_cfs.c | 6 +-- kernel/sched/walt/walt_lb.c | 26 ++++++------- 6 files changed, 54 insertions(+), 54 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 689f47fe5d35..d1a46f3a59ee 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -629,9 +629,9 @@ static ssize_t boost_store(struct gov_attr_set *attr_set, const char *buf, struct rq *rq = cpu_rq(wg_policy->policy->cpu); unsigned long flags; - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); waltgov_run_callback(rq, WALT_CPUFREQ_BOOST_UPDATE); - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); } return count; } diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 28c4cfc26f72..cdf0e431498a 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -256,13 +256,13 @@ unsigned int sched_get_cpu_util(int cpu) unsigned int busy; struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); capacity = capacity_orig_of(cpu); util = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; util = div64_u64(util, sched_ravg_window >> SCHED_CAPACITY_SHIFT); - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); util = (util >= capacity) ? capacity : util; busy = div64_ul((util * 100), capacity); diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index f4da057d1387..c6eb6430b15b 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -96,9 +96,9 @@ static int walt_proc_group_thresholds_handler(struct ctl_table *table, int write * updating the thresholds is sufficient for * an atomic update. */ - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); walt_update_group_thresholds(); - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); mutex_unlock(&mutex); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 941729ae8a76..bda43f343f47 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -134,9 +134,9 @@ static inline void acquire_rq_locks_irqsave(const cpumask_t *cpus, for_each_cpu(cpu, cpus) { if (level == 0) - raw_spin_lock(&cpu_rq(cpu)->lock); + raw_spin_lock(&cpu_rq(cpu)->__lock); else - raw_spin_lock_nested(&cpu_rq(cpu)->lock, level); + raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); level++; } } @@ -147,7 +147,7 @@ static inline void release_rq_locks_irqrestore(const cpumask_t *cpus, int cpu; for_each_cpu(cpu, cpus) - raw_spin_unlock(&cpu_rq(cpu)->lock); + raw_spin_unlock(&cpu_rq(cpu)->__lock); local_irq_restore(*flags); } @@ -290,7 +290,7 @@ fixup_cumulative_runnable_avg(struct rq *rq, s64 pred_demands_sum_scaled = stats->pred_demands_sum_scaled + pred_demand_scaled_delta; - lockdep_assert_held(&rq->lock); + lockdep_assert_held(&rq->__lock); if (task_rq(p) != rq) WALT_BUG(p, "on CPU %d task %s(%d) not on rq %d", @@ -473,9 +473,9 @@ static void walt_sched_account_irqstart(int cpu, struct task_struct *curr) return; /* We're here without rq->lock held, IRQ disabled */ - raw_spin_lock(&rq->lock); + raw_spin_lock(&rq->__lock); update_task_cpu_cycles(curr, cpu, walt_ktime_get_ns()); - raw_spin_unlock(&rq->lock); + raw_spin_unlock(&rq->__lock); } static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, @@ -485,9 +485,9 @@ static void walt_sched_account_irqend(int cpu, struct task_struct *curr, u64 del struct rq *rq = cpu_rq(cpu); unsigned long flags; - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); walt_update_task_ravg(curr, rq, IRQ_UPDATE, walt_ktime_get_ns(), delta); - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); } /* @@ -510,14 +510,14 @@ void clear_walt_request(int cpu) if (wrq->push_task) { struct task_struct *push_task = NULL; - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); if (wrq->push_task) { clear_reserved(rq->push_cpu); push_task = wrq->push_task; wrq->push_task = NULL; } rq->active_balance = 0; - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); if (push_task) put_task_struct(push_task); } @@ -977,8 +977,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) wallclock = walt_ktime_get_ns(); - lockdep_assert_held(&src_rq->lock); - lockdep_assert_held(&dest_rq->lock); + lockdep_assert_held(&src_rq->__lock); + lockdep_assert_held(&dest_rq->__lock); if (task_rq(p) != src_rq) WALT_BUG(p, "on CPU %d task %s(%d) not on src_rq %d", @@ -1083,12 +1083,12 @@ static void set_window_start(struct rq *rq) struct rq *sync_rq = cpu_rq(cpumask_any(cpu_online_mask)); sync_wrq = (struct walt_rq *) sync_rq->android_vendor_data1; - raw_spin_unlock(&rq->lock); + raw_spin_unlock(&rq->__lock); double_rq_lock(rq, sync_rq); wrq->window_start = sync_wrq->window_start; wrq->curr_runnable_sum = wrq->prev_runnable_sum = 0; wrq->nt_curr_runnable_sum = wrq->nt_prev_runnable_sum = 0; - raw_spin_unlock(&sync_rq->lock); + raw_spin_unlock(&sync_rq->__lock); } wts->mark_start = wrq->window_start; @@ -2082,7 +2082,7 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - lockdep_assert_held(&rq->lock); + lockdep_assert_held(&rq->__lock); if (!use_cycle_counter) { wrq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * @@ -2168,7 +2168,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even if (!wrq->window_start || wts->mark_start == wallclock) return; - lockdep_assert_held(&rq->lock); + lockdep_assert_held(&rq->__lock); old_window_start = update_window_start(rq, wallclock, event); @@ -2665,10 +2665,10 @@ static int cpufreq_notifier_trans(struct notifier_block *nb, for_each_cpu(j, &cluster->cpus) { struct rq *rq = cpu_rq(j); - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, walt_ktime_get_ns(), 0); - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); } cluster->cur_freq = new_freq; @@ -3390,9 +3390,9 @@ static void walt_irq_work(struct irq_work *irq_work) for_each_cpu(cpu, cpu_possible_mask) { if (level == 0) - raw_spin_lock(&cpu_rq(cpu)->lock); + raw_spin_lock(&cpu_rq(cpu)->__lock); else - raw_spin_lock_nested(&cpu_rq(cpu)->lock, level); + raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); level++; } @@ -3512,7 +3512,7 @@ static void walt_irq_work(struct irq_work *irq_work) } for_each_cpu(cpu, cpu_possible_mask) - raw_spin_unlock(&cpu_rq(cpu)->lock); + raw_spin_unlock(&cpu_rq(cpu)->__lock); if (!is_migration) { wrq = (struct walt_rq *) this_rq()->android_vendor_data1; @@ -3783,9 +3783,9 @@ static void android_rvh_sched_cpu_starting(void *unused, int cpu) if (unlikely(walt_disabled)) return; - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); set_window_start(rq); - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); clear_walt_request(cpu); } @@ -3842,7 +3842,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (unlikely(walt_disabled)) return; - lockdep_assert_held(&rq->lock); + lockdep_assert_held(&rq->__lock); if (p->cpu != cpu_of(rq)) WALT_BUG(p, "enqueuing on rq %d when task->cpu is %d\n", @@ -3882,7 +3882,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st if (unlikely(walt_disabled)) return; - lockdep_assert_held(&rq->lock); + lockdep_assert_held(&rq->__lock); /* * a task can be enqueued before walt is started, and dequeued after. @@ -3992,17 +3992,17 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct if (unlikely(walt_disabled)) return; - raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags); + raw_spin_lock_irqsave(&cpu_rq(cpu)->__lock, flags); if (do_pl_notif(cpu_rq(cpu))) waltgov_run_callback(cpu_rq(cpu), WALT_CPUFREQ_PL); - raw_spin_unlock_irqrestore(&cpu_rq(cpu)->lock, flags); + raw_spin_unlock_irqrestore(&cpu_rq(cpu)->__lock, flags); } static void android_rvh_tick_entry(void *unused, struct rq *rq) { u64 wallclock; - lockdep_assert_held(&rq->lock); + lockdep_assert_held(&rq->__lock); if (unlikely(walt_disabled)) return; @@ -4069,9 +4069,9 @@ static void android_rvh_resume_cpus(void *unused, struct cpumask *resuming_cpus, */ for_each_cpu(i, resuming_cpus) { rq = cpu_rq(i); - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); resched_curr(rq); - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); } *err = 0; @@ -4215,7 +4215,7 @@ static int walt_init_stop_handler(void *data) read_lock(&tasklist_lock); for_each_possible_cpu(cpu) { - raw_spin_lock(&cpu_rq(cpu)->lock); + raw_spin_lock(&cpu_rq(cpu)->__lock); } do_each_thread(g, p) { @@ -4247,7 +4247,7 @@ static int walt_init_stop_handler(void *data) walt_disabled = false; for_each_possible_cpu(cpu) { - raw_spin_unlock(&cpu_rq(cpu)->lock); + raw_spin_unlock(&cpu_rq(cpu)->__lock); } read_unlock(&tasklist_lock); return 0; diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index cd9901232973..770360783133 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1081,7 +1081,7 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr s64 delta; unsigned int limit; - lockdep_assert_held(&rq->lock); + lockdep_assert_held(&rq->__lock); /* * RQ clock update happens in tick path in the scheduler. @@ -1179,7 +1179,7 @@ void walt_cfs_tick(struct rq *rq) if (unlikely(walt_disabled)) return; - raw_spin_lock(&rq->lock); + raw_spin_lock(&rq->__lock); if (list_empty(&wts->mvp_list)) goto out; @@ -1193,7 +1193,7 @@ void walt_cfs_tick(struct rq *rq) resched_curr(rq); out: - raw_spin_unlock(&rq->lock); + raw_spin_unlock(&rq->__lock); } /* diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 01d1c15a1ac1..20adabaf92c4 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -42,7 +42,7 @@ static int walt_lb_active_migration(void *data) struct task_struct *push_task = wrq->push_task; int push_task_detached = 0; - raw_spin_lock_irq(&busiest_rq->lock); + raw_spin_lock_irq(&busiest_rq->__lock); /* sanity checks before initiating the pull */ if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu) || !push_task) @@ -71,12 +71,12 @@ static int walt_lb_active_migration(void *data) target_cpu = busiest_rq->push_cpu; clear_reserved(target_cpu); wrq->push_task = NULL; - raw_spin_unlock(&busiest_rq->lock); + raw_spin_unlock(&busiest_rq->__lock); if (push_task_detached) { - raw_spin_lock(&target_rq->lock); + raw_spin_lock(&target_rq->__lock); walt_attach_task(push_task, target_rq); - raw_spin_unlock(&target_rq->lock); + raw_spin_unlock(&target_rq->__lock); } if (push_task) @@ -267,7 +267,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(src_cpu); - raw_spin_lock_irqsave(&src_rq->lock, flags); + raw_spin_lock_irqsave(&src_rq->__lock, flags); list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) @@ -298,7 +298,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) mark_reserved(dst_cpu); } /* lock must be dropped before waking the stopper */ - raw_spin_unlock_irqrestore(&src_rq->lock, flags); + raw_spin_unlock_irqrestore(&src_rq->__lock, flags); /* * Using our custom active load balance callback so that @@ -315,9 +315,9 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) if (!pulled_task) return 0; - raw_spin_lock_irqsave(&dst_rq->lock, flags); + raw_spin_lock_irqsave(&dst_rq->__lock, flags); walt_attach_task(p, dst_rq); - raw_spin_unlock_irqrestore(&dst_rq->lock, flags); + raw_spin_unlock_irqrestore(&dst_rq->__lock, flags); return 1; /* we pulled 1 task */ } @@ -525,16 +525,16 @@ void walt_lb_tick(struct rq *rq) capacity_orig_of(new_cpu) <= capacity_orig_of(prev_cpu)) goto out_unlock; - raw_spin_lock(&rq->lock); + raw_spin_lock(&rq->__lock); if (rq->active_balance) { - raw_spin_unlock(&rq->lock); + raw_spin_unlock(&rq->__lock); goto out_unlock; } rq->active_balance = 1; rq->push_cpu = new_cpu; get_task_struct(p); wrq->push_task = p; - raw_spin_unlock(&rq->lock); + raw_spin_unlock(&rq->__lock); mark_reserved(new_cpu); raw_spin_unlock_irqrestore(&walt_lb_migration_lock, flags); @@ -683,7 +683,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, goto repin; help_min_cap = should_help_min_cap(this_cpu); - raw_spin_unlock(&this_rq->lock); + raw_spin_unlock(&this_rq->__lock); /* * careful, we dropped the lock, and has to be acquired @@ -717,7 +717,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, *pulled_task = walt_lb_pull_tasks(this_cpu, busy_cpu); unlock: - raw_spin_lock(&this_rq->lock); + raw_spin_lock(&this_rq->__lock); rt_pulled: if (this_rq->cfs.h_nr_running && !*pulled_task) *pulled_task = 1; From beb88676bfc65b57e319338a4876539314b2a682 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Mon, 14 Jun 2021 14:51:49 -0700 Subject: [PATCH 199/346] sched/walt: complain if num clusters is high num_sched_clusters has to be less than MAX_CLUSTERS, complain otherwise. Change-Id: I0648585139d31dc948cf73aae7d093757a5762cc Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index bda43f343f47..8b6677c90ee2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2373,6 +2373,8 @@ static void add_cluster(const struct cpumask *cpus, struct list_head *head) int i; struct walt_rq *wrq; + BUG_ON(num_sched_clusters >= MAX_CLUSTERS); + for_each_cpu(i, cpus) { wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; wrq->cluster = cluster; From 9c062eae64dbf08c4f2a205a4888a084dd5b6ecd Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 1 Nov 2021 17:22:09 -0700 Subject: [PATCH 200/346] pause: port the users of pause Port the users of pause from the 5.10 kernel to ensure code is up to date. Change-Id: Ibe52cc37e0dca425f2f472b9cf254a2453bd9c47 Signed-off-by: Stephen Dickey --- drivers/soc/qcom/hyp_core_ctl.c | 2 + drivers/thermal/qcom/Kconfig | 11 ++ drivers/thermal/qcom/Makefile | 1 + drivers/thermal/qcom/thermal_pause.c | 168 ++++++++++++++++++--------- include/linux/thermal_pause.h | 33 ++++++ kernel/sched/walt/Makefile | 2 +- kernel/sched/walt/walt_pause.c | 6 +- 7 files changed, 162 insertions(+), 61 deletions(-) create mode 100644 include/linux/thermal_pause.h diff --git a/drivers/soc/qcom/hyp_core_ctl.c b/drivers/soc/qcom/hyp_core_ctl.c index ae9eac7a548c..c5a00da88e9a 100644 --- a/drivers/soc/qcom/hyp_core_ctl.c +++ b/drivers/soc/qcom/hyp_core_ctl.c @@ -29,6 +29,8 @@ #include +#include + #define MAX_RESERVE_CPUS (num_possible_cpus()/2) static DEFINE_PER_CPU(struct freq_qos_request, qos_min_req); diff --git a/drivers/thermal/qcom/Kconfig b/drivers/thermal/qcom/Kconfig index bfd889422dd3..53641c3470a1 100644 --- a/drivers/thermal/qcom/Kconfig +++ b/drivers/thermal/qcom/Kconfig @@ -41,3 +41,14 @@ config QCOM_LMH input from temperature and current sensors. On many newer Qualcomm SoCs LMh is configured in the firmware and this feature need not be enabled. However, on certain SoCs like sdm845 LMh has to be configured from kernel. + +config QTI_CPU_PAUSE_COOLING_DEVICE + tristate "QTI CPU Pause cooling devices" + depends on THERMAL_OF && HOTPLUG_CPU + depends on ARCH_QCOM || COMPILE_TEST + help + This enables the QTI CPU Pause cooling device. These cooling + devices will be used by QTI chipset to pause a CPU from being + scheduled and hence will let the CPU to power collapse. Pausing + a CPU will be used when the CPU frequency mitigation + is not good enough to achieve the necessary cooling. \ No newline at end of file diff --git a/drivers/thermal/qcom/Makefile b/drivers/thermal/qcom/Makefile index 0fa2512042e7..99a16fd8a22c 100644 --- a/drivers/thermal/qcom/Makefile +++ b/drivers/thermal/qcom/Makefile @@ -6,3 +6,4 @@ qcom_tsens-y += tsens.o tsens-v2.o tsens-v1.o tsens-v0_1.o \ obj-$(CONFIG_QCOM_SPMI_ADC_TM5) += qcom-spmi-adc-tm5.o obj-$(CONFIG_QCOM_SPMI_TEMP_ALARM) += qcom-spmi-temp-alarm.o obj-$(CONFIG_QCOM_LMH) += lmh.o +obj-$(CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE) += thermal_pause.o diff --git a/drivers/thermal/qcom/thermal_pause.c b/drivers/thermal/qcom/thermal_pause.c index f76db4033441..0c926c220c94 100644 --- a/drivers/thermal/qcom/thermal_pause.c +++ b/drivers/thermal/qcom/thermal_pause.c @@ -23,14 +23,18 @@ enum thermal_pause_levels { MAX_THERMAL_PAUSE_LEVEL }; +#define THERMAL_PAUSE_RETRY_COUNT 5 + struct thermal_pause_cdev { struct list_head node; cpumask_t cpu_mask; - bool thermal_pause_level; + enum thermal_pause_levels thermal_pause_level; + enum thermal_pause_levels thermal_pause_req; struct thermal_cooling_device *cdev; struct device_node *np; char cdev_name[THERMAL_NAME_LENGTH]; struct work_struct reg_work; + struct work_struct pause_update_work; }; static DEFINE_MUTEX(cpus_pause_lock); @@ -39,22 +43,25 @@ static LIST_HEAD(thermal_pause_cdev_list); static struct cpumask cpus_in_max_cooling_level; static enum cpuhp_state cpu_hp_online; -static BLOCKING_NOTIFIER_HEAD(multi_max_cooling_level_notifer); +static BLOCKING_NOTIFIER_HEAD(thermal_pause_notifier); -void cpu_cooling_multi_max_level_notifier_register(struct notifier_block *n) +void thermal_pause_notifier_register(struct notifier_block *n) { - blocking_notifier_chain_register(&multi_max_cooling_level_notifer, n); + blocking_notifier_chain_register(&thermal_pause_notifier, n); } +EXPORT_SYMBOL(thermal_pause_notifier_register); -void cpu_cooling_multi_max_level_notifier_unregister(struct notifier_block *n) +void thermal_pause_notifier_unregister(struct notifier_block *n) { - blocking_notifier_chain_unregister(&multi_max_cooling_level_notifer, n); + blocking_notifier_chain_unregister(&thermal_pause_notifier, n); } +EXPORT_SYMBOL(thermal_pause_notifier_unregister); -const struct cpumask *cpu_cooling_multi_get_max_level_cpumask(void) +const struct cpumask *thermal_paused_cpumask(void) { return &cpus_in_max_cooling_level; } +EXPORT_SYMBOL(thermal_paused_cpumask); static int thermal_pause_hp_online(unsigned int online_cpu) { @@ -76,26 +83,21 @@ static int thermal_pause_hp_online(unsigned int online_cpu) } /** - * thermal_pause_cpus_pause - function to pause a group of cpus at - * the specified level. + * thermal_pause_work - work function to pause a group of cpus at + * the specified level. * - * @thermal_pause_cdev: the pause device + * @thermal_pasue_cdev: the cdev currently being processed * - * function to handle setting the current cpus paused by + * Function to handle setting the current cpus paused by * this driver for the mask specified in the device. - * it assumes the mutex is locked. - * - * Returns 0 if CPUs were paused, error otherwise + * it assumes the mutex is locked upon entrance. */ -static int thermal_pause_cpus_pause(struct thermal_pause_cdev *thermal_pause_cdev) +static int thermal_pause_work(struct thermal_pause_cdev *thermal_pause_cdev) { int cpu = 0; int ret = -EBUSY; cpumask_t cpus_to_pause, cpus_to_notify; - if (thermal_pause_cdev->thermal_pause_level) - return ret; - cpumask_copy(&cpus_to_pause, &thermal_pause_cdev->cpu_mask); pr_debug("Pause:%*pbl\n", cpumask_pr_args(&thermal_pause_cdev->cpu_mask)); @@ -120,33 +122,29 @@ static int thermal_pause_cpus_pause(struct thermal_pause_cdev *thermal_pause_cde /* Failure. These cpus not paused by thermal */ pr_err("Error pausing CPU:%*pbl. err:%d\n", cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret); - return ret; } return ret; } /** - * thermal_pause_cpus_unpause - function to unpause a + * thermal_resume_work - work function to unpause a * group of cpus in the mask for this cdev * - * @thermal_pause_cdev: the pause device + * @thermal_pasue_cdev: the cdev currently being processed * - * function to handle enabling the group of cpus in the cdev - * - * Returns 0 if CPUs were unpaused, + * Function to handle enabling the group of cpus in the cdev. + * This is performed as a work function to avoid conflicts + * between a hotplug event invoking a pause cooling device, + * and a thermal event invoking a pause cooling device. */ -static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_cdev) +static int thermal_resume_work(struct thermal_pause_cdev *thermal_pause_cdev) { int cpu = 0; int ret = -ENODEV; cpumask_t cpus_to_unpause, new_paused_cpus, cpus_to_notify; struct thermal_pause_cdev *cdev; - /* do not unpause a cooling device not paused */ - if (!thermal_pause_cdev->thermal_pause_level) - return ret; - cpumask_copy(&cpus_to_unpause, &thermal_pause_cdev->cpu_mask); pr_debug("Unpause:%*pbl\n", cpumask_pr_args(&cpus_to_unpause)); @@ -177,12 +175,73 @@ static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_c /* Failure. Ref-count for cpus controlled by thermal still set */ pr_err("Error resuming CPU:%*pbl. err:%d\n", cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret); - return ret; } return ret; } +/** + * thermal_pause_set_update_work: Enforce Requested State + * + * @work: the work structure for this work + * + * Enforce the most recent requested cooling state, if + * it is a mismatch with the current state. Since the + * request is made in a different context from the + * enforcement, it is possible to have an updated request + * after completing the resume/pause request. + * + * Handle a post-operation mismatch between the cooling state + * and the requested state. + * + * This is performed as a work function to avoid conflicts + * between a hotplug event invoking a pause cooling device, + * and a thermal event invoking a pause cooling device. + */ +static void thermal_pause_update_work(struct work_struct *work) +{ + int ret = 0; + int retcnt = THERMAL_PAUSE_RETRY_COUNT; + struct thermal_pause_cdev *thermal_pause_cdev = + container_of(work, struct thermal_pause_cdev, pause_update_work); + + mutex_lock(&cpus_pause_lock); + +retry: + if (thermal_pause_cdev->thermal_pause_req != thermal_pause_cdev->thermal_pause_level) { + if (thermal_pause_cdev->thermal_pause_req == THERMAL_NO_CPU_PAUSE) { + ret = thermal_resume_work(thermal_pause_cdev); + if (ret >= 0) + thermal_pause_cdev->thermal_pause_level = THERMAL_NO_CPU_PAUSE; + } else { + ret = thermal_pause_work(thermal_pause_cdev); + if (ret >= 0) + thermal_pause_cdev->thermal_pause_level = THERMAL_GROUP_CPU_PAUSE; + } + if (ret < 0 && retcnt > 0) { + retcnt--; + goto retry; + } + } + + /* + * if the pause/resume operation itself failed (and failed THERMAL_PAUSE_RETRY_COUNT + * times) then ret will be negative here. Do not repeatedly retry if pause itself + * failed, because this can happen indefinitely. + * + * If instead the pause request has been toggled back and forth many times by + * the thermal framework, this can be handled here. + */ + if (ret >= 0 && + thermal_pause_cdev->thermal_pause_req != thermal_pause_cdev->thermal_pause_level) { + pr_debug("Pause: requested state changed while workfn running\n"); + retcnt = THERMAL_PAUSE_RETRY_COUNT; + goto retry; + } + + mutex_unlock(&cpus_pause_lock); +} + /** * thermal_pause_set_cur_state - callback function to set the current cooling * level. @@ -190,7 +249,8 @@ static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_c * @level: set this variable to the current cooling level. * * Callback for the thermal cooling device to change the cpu pause - * current cooling level. + * current cooling level, by making a requested and queueing the + * work to be done. * * Return: 0 on success, an error code otherwise. */ @@ -198,29 +258,22 @@ static int thermal_pause_set_cur_state(struct thermal_cooling_device *cdev, unsigned long level) { struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata; - int ret = 0; if (level >= MAX_THERMAL_PAUSE_LEVEL) return -EINVAL; - if (thermal_pause_cdev->thermal_pause_level == level) - return 0; - mutex_lock(&cpus_pause_lock); - if (level == THERMAL_GROUP_CPU_PAUSE) - ret = thermal_pause_cpus_pause(thermal_pause_cdev); + + if (level) + thermal_pause_cdev->thermal_pause_req = THERMAL_GROUP_CPU_PAUSE; else - ret = thermal_pause_cpus_unpause(thermal_pause_cdev); - /* - * only change the pause level if things were successful. Otherwise - * an unsuccessful pause operation can be followed by a resume - * operation, resuming cpus not paused by this cooling device. - */ - if (ret == 0) - thermal_pause_cdev->thermal_pause_level = level; + thermal_pause_cdev->thermal_pause_req = THERMAL_NO_CPU_PAUSE; + + queue_work(system_highpri_wq, &thermal_pause_cdev->pause_update_work); mutex_unlock(&cpus_pause_lock); - return ret; + + return 0; } /** @@ -307,6 +360,7 @@ static int thermal_pause_probe(struct platform_device *pdev) struct of_phandle_iterator it; cpumask_t cpu_mask; unsigned long mask = 0; + const char *alias; INIT_LIST_HEAD(&thermal_pause_cdev_list); cpumask_clear(&cpus_in_max_cooling_level); @@ -339,17 +393,22 @@ static int thermal_pause_probe(struct platform_device *pdev) of_node_put(subsys_np); return -ENOMEM; } - - snprintf(thermal_pause_cdev->cdev_name, THERMAL_NAME_LENGTH, + ret = of_property_read_string(subsys_np, + "qcom,cdev-alias", &alias); + if (ret) + snprintf(thermal_pause_cdev->cdev_name, THERMAL_NAME_LENGTH, "thermal-pause-%X", mask); + else + strscpy(thermal_pause_cdev->cdev_name, alias, + THERMAL_NAME_LENGTH); thermal_pause_cdev->thermal_pause_level = false; thermal_pause_cdev->cdev = NULL; thermal_pause_cdev->np = subsys_np; cpumask_copy(&thermal_pause_cdev->cpu_mask, &cpu_mask); - INIT_WORK(&thermal_pause_cdev->reg_work, - thermal_pause_register_cdev); + INIT_WORK(&thermal_pause_cdev->reg_work, thermal_pause_register_cdev); + INIT_WORK(&thermal_pause_cdev->pause_update_work, thermal_pause_update_work); list_add(&thermal_pause_cdev->node, &thermal_pause_cdev_list); } @@ -378,13 +437,8 @@ static int thermal_pause_remove(struct platform_device *pdev) /* for each asserted cooling device, resume the CPUs */ if (thermal_pause_cdev->thermal_pause_level) { - mutex_unlock(&cpus_pause_lock); - ret = walt_resume_cpus(&thermal_pause_cdev->cpu_mask); - - if (ret < 0) - pr_err("Error resuming CPU:%*pbl. err:%d\n", - cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret); - mutex_lock(&cpus_pause_lock); + thermal_pause_cdev->thermal_pause_req = THERMAL_NO_CPU_PAUSE; + queue_work(system_highpri_wq, &thermal_pause_cdev->pause_update_work); } if (thermal_pause_cdev->cdev) diff --git a/include/linux/thermal_pause.h b/include/linux/thermal_pause.h new file mode 100644 index 000000000000..0769cdd8d460 --- /dev/null +++ b/include/linux/thermal_pause.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _LINUX_THERMAL_PAUSE_H +#define _LINUX_THERMAL_PAUSE_H + +#include +#include + +#if IS_ENABLED(CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE) +extern void thermal_pause_notifier_register(struct notifier_block *n); +extern void thermal_pause_notifier_unregister(struct notifier_block *n); +extern const struct cpumask *thermal_paused_cpumask(void); +#else +static inline +void thermal_pause_notifier_register(struct notifier_block *n) +{ +} + +static inline +void thermal_pause_notifier_unregister(struct notifier_block *n) +{ +} + +static inline const struct cpumask *thermal_paused_cpumask(void) +{ + return cpu_none_mask; +} +#endif /* CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE */ + +#endif /* _LINUX_THERMAL_PAUSE_H */ diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile index ed35366108ed..870d9b2fcfe3 100644 --- a/kernel/sched/walt/Makefile +++ b/kernel/sched/walt/Makefile @@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n KCSAN_SANITIZE := n obj-$(CONFIG_SCHED_WALT) += sched-walt.o -sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c index cca3d5904923..ef6bdcbc897a 100644 --- a/kernel/sched/walt/walt_pause.c +++ b/kernel/sched/walt/walt_pause.c @@ -66,7 +66,7 @@ int walt_pause_cpus(struct cpumask *cpus) goto unlock; } - ret = pause_cpus(cpus); + // ret = pause_cpus(cpus); if (ret < 0) pr_debug("pause_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); @@ -95,7 +95,7 @@ int walt_resume_cpus(struct cpumask *cpus) if (cpumask_empty(cpus)) goto unlock; - ret = resume_cpus(cpus); + // ret = resume_cpus(cpus); if (ret < 0) { pr_debug("resume_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); @@ -156,7 +156,7 @@ static void walt_pause_online_workfn(struct work_struct *work) if (cpumask_empty(&re_pause_cpus)) goto unlock; - ret = pause_cpus(&re_pause_cpus); + //ret = pause_cpus(&re_pause_cpus); unlock: mutex_unlock(&pause_lock); From 7054cae73f2d0b0de8c076d28aa9a8926cfb528e Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Wed, 3 Nov 2021 22:46:44 -0700 Subject: [PATCH 201/346] sched/walt: Clear scale_frequency_data during init Change the FIE to use frequency scaling information provided by cpufreq driver as opposed to the higher priority arch specific counters. This is done by resetting the struct scale_freq_data for each CPU which holds the callback information specific to the HW counters, thus forcing the FIE to revert to default, the cpufreq driver. Change-Id: I23f508f9859be6a35155e5d000caf1c0b19f5584 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 8b6677c90ee2..cc800283f281 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -4307,6 +4308,8 @@ static void walt_init(struct work_struct *work) static_key_disable(&sched_feat_keys[i]); sysctl_sched_features &= ~(1UL << i); } + + topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_ARCH, cpu_online_mask); } static DECLARE_WORK(walt_init_work, walt_init); From 6d9fb971b7f0c34e6853a1386600a02152665577 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 22 Nov 2021 15:17:07 -0800 Subject: [PATCH 202/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Id758a4a0a836fe30b9f63b8f00c017247cf3b84c Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index cc800283f281..1d0d927d8e7b 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -384,8 +384,9 @@ update_window_start(struct rq *rq, u64 wallclock, int event) delta = wallclock - wrq->window_start; if (delta < 0) { - printk_deferred("WALT-BUG CPU%d; wallclock=%llu is lesser than window_start=%llu", - rq->cpu, wallclock, wrq->window_start); + printk_deferred("WALT-BUG CPU%d; wallclock=%llu(0x%llx) is lesser than window_start=%llu(0x%llx)", + rq->cpu, wallclock, wallclock, + wrq->window_start, wrq->window_start); WALT_PANIC(1); } if (delta < sched_ravg_window) @@ -2124,9 +2125,9 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, time_delta = wallclock - wts->mark_start; if ((s64)time_delta < 0) { - printk_deferred("WALT-BUG pid=%u CPU%d wallclock=%llu < mark_start=%llu event=%d irqtime=%llu", - p->pid, rq->cpu, wallclock, - wts->mark_start, event, irqtime); + printk_deferred("WALT-BUG pid=%u CPU%d wallclock=%llu(0x%llx) < mark_start=%llu(0x%llx) event=%d irqtime=%llu", + p->pid, rq->cpu, wallclock, wallclock, + wts->mark_start, wts->mark_start, event, irqtime); WALT_PANIC((s64)time_delta < 0); } From f9c2cd8a95ae4ccf000d6f43a4a4db4be3c60aac Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 23 Nov 2021 12:39:46 -0800 Subject: [PATCH 203/346] Revert "sched:walt: register tracehooks for race with pause" Aandoned upstream, reflecting it in kernel/msm-5.10. Change-Id: I0c1ebd96372803519b180641707562034996479b Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 1d0d927d8e7b..38b97d44412b 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4143,20 +4143,6 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } -static void android_rvh_force_compatible_pre(void *unused, void *unused2) -{ - if (unlikely(walt_disabled)) - return; - cpu_maps_update_begin(); -} - -static void android_rvh_force_compatible_post(void *unused, void *unused2) -{ - if (unlikely(walt_disabled)) - return; - cpu_maps_update_done(); -} - static void dump_throttled_rt_tasks(void *unused, int cpu, u64 clock, ktime_t rt_period, u64 rt_runtime, s64 rt_period_timer_expires) { @@ -4202,8 +4188,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); - register_trace_android_rvh_force_compatible_pre(android_rvh_force_compatible_pre, NULL); - register_trace_android_rvh_force_compatible_post(android_rvh_force_compatible_post, NULL); register_trace_android_vh_dump_throttled_rt_tasks(dump_throttled_rt_tasks, NULL); } From 5555cfa20504d7f5b839b0b96a1f0c1a428e41c3 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 23 Nov 2021 12:57:41 -0800 Subject: [PATCH 204/346] sched/walt: Remove vendor hook for resume_cpus Upstream does not carry pause feature. Remove related references. Change-Id: I7ac0352a761fa78af3918a5b3a61424df0d22a2f Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 38b97d44412b..97c3f55ebd9e 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4059,28 +4059,6 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, } } -static void android_rvh_resume_cpus(void *unused, struct cpumask *resuming_cpus, int *err) -{ - int i; - struct rq *rq; - unsigned long flags; - - if (unlikely(walt_disabled)) - return; - /* - * send a reschedule event on all resumed CPUs - * which trigger newly idle load balance. - */ - for_each_cpu(i, resuming_cpus) { - rq = cpu_rq(i); - raw_spin_lock_irqsave(&rq->__lock, flags); - resched_curr(rq); - raw_spin_unlock_irqrestore(&rq->__lock, flags); - } - - *err = 0; -} - static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, cpumask_var_t cpus_requested, const struct cpumask *new_mask, int *ret) @@ -4178,7 +4156,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL); register_trace_android_vh_scheduler_tick(android_vh_scheduler_tick, NULL); register_trace_android_rvh_schedule(android_rvh_schedule, NULL); - register_trace_android_rvh_resume_cpus(android_rvh_resume_cpus, NULL); register_trace_android_rvh_cpu_cgroup_attach(android_rvh_cpu_cgroup_attach, NULL); register_trace_android_rvh_cpu_cgroup_online(android_rvh_cpu_cgroup_online, NULL); register_trace_android_rvh_update_cpus_allowed(android_rvh_update_cpus_allowed, NULL); From 20ec0b66fbf45ede42a31192e8a2a7b72cf59d03 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 23 Nov 2021 14:47:02 -0800 Subject: [PATCH 205/346] include/linux/cpu: Remove get_online_cpus and put_online_cpus wrapper Upstream has done away with get_online_cpus and put_online_cpus wrappers around cpu_read_lock() and cpu_read_unlock(). Reflect these changes downstream. Change-Id: Ie7ae59b6faba93ea43ce1ba3a4b043ae87e1337f Signed-off-by: Sai Harshini Nimmala --- drivers/soc/qcom/msm_performance.c | 12 ++++++------ kernel/sched/walt/input-boost.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/soc/qcom/msm_performance.c b/drivers/soc/qcom/msm_performance.c index 9acd80761e53..a2dcfac2f267 100644 --- a/drivers/soc/qcom/msm_performance.c +++ b/drivers/soc/qcom/msm_performance.c @@ -317,7 +317,7 @@ static ssize_t set_cpu_min_freq(struct kobject *kobj, * of other CPUs in the cluster once it is done for at least one CPU * in the cluster */ - get_online_cpus(); + cpus_read_lock(); for_each_cpu(i, limit_mask_min) { i_cpu_stats = &per_cpu(msm_perf_cpu_stats, i); @@ -333,7 +333,7 @@ static ssize_t set_cpu_min_freq(struct kobject *kobj, for_each_cpu(j, policy.related_cpus) cpumask_clear_cpu(j, limit_mask_min); } - put_online_cpus(); + cpus_read_unlock(); return count; } @@ -397,7 +397,7 @@ static ssize_t set_cpu_max_freq(struct kobject *kobj, cp++; } - get_online_cpus(); + cpus_read_lock(); for_each_cpu(i, limit_mask_max) { i_cpu_stats = &per_cpu(msm_perf_cpu_stats, i); if (cpufreq_get_policy(&policy, i)) @@ -412,7 +412,7 @@ static ssize_t set_cpu_max_freq(struct kobject *kobj, for_each_cpu(j, policy.related_cpus) cpumask_clear_cpu(j, limit_mask_max); } - put_online_cpus(); + cpus_read_unlock(); return count; } @@ -1198,7 +1198,7 @@ static int __init msm_performance_init(void) free_cpumask_var(limit_mask_min); return -ENOMEM; } - get_online_cpus(); + cpus_read_lock(); for_each_possible_cpu(cpu) { if (!cpumask_test_cpu(cpu, cpu_online_mask)) per_cpu(cpu_is_hp, cpu) = true; @@ -1209,7 +1209,7 @@ static int __init msm_performance_init(void) hotplug_notify_up, hotplug_notify_down); - put_online_cpus(); + cpus_read_unlock(); msm_perf_kset = kset_create_and_add("msm_performance", NULL, kernel_kobj); if (!msm_perf_kset) { diff --git a/kernel/sched/walt/input-boost.c b/kernel/sched/walt/input-boost.c index c20454535dab..5372d899bd45 100644 --- a/kernel/sched/walt/input-boost.c +++ b/kernel/sched/walt/input-boost.c @@ -86,7 +86,7 @@ static void update_policy_online(void) struct cpumask online_cpus; /* Re-evaluate policy to trigger adjust notifier for online CPUs */ - get_online_cpus(); + cpus_read_lock(); online_cpus = *cpu_online_mask; for_each_cpu(i, &online_cpus) { policy = cpufreq_cpu_get(i); @@ -100,7 +100,7 @@ static void update_policy_online(void) policy->related_cpus); boost_adjust_notify(policy); } - put_online_cpus(); + cpus_read_unlock(); } static void do_input_boost_rem(struct work_struct *work) From ddd2ea392746691a18b1dc56da72c6e8f62091c4 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:24:58 -0800 Subject: [PATCH 206/346] sched/walt/walt_rt: Improve the scheduler This change is for general scheduler improvemnts Change-Id: I0a0b065ff2d1a563bccdd161dad890fae26f11c3 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_rt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 23a3ce5d7e7a..a26e6c278c9e 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -145,7 +145,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c struct task_struct *curr; struct rq *rq, *this_cpu_rq; bool may_not_preempt; - bool sync = !!(wake_flags && WF_SYNC); + bool sync = !!(wake_flags & WF_SYNC); int ret, target = -1, this_cpu; struct cpumask *lowest_mask; From 9a042aeb4157dbd6eceebba8329240b34cd2a47a Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 6 Dec 2021 15:01:28 -0800 Subject: [PATCH 207/346] sched/walt: improve preemptirq template based tracepoints To determine why a particular long preemption or long irqs off period has happened, without crashing the device, the tracepoints need improvement. The callers should be included. Change-Id: If147969717517563e56cd20d3c036f51835a5c73 Signed-off-by: Stephen Dickey --- kernel/sched/walt/preemptirq_long.c | 12 ++++++------ kernel/sched/walt/preemptirq_long.h | 28 +++++++++++++++++++++------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/kernel/sched/walt/preemptirq_long.c b/kernel/sched/walt/preemptirq_long.c index 45b77150bf0a..17b5ea24f049 100644 --- a/kernel/sched/walt/preemptirq_long.c +++ b/kernel/sched/walt/preemptirq_long.c @@ -51,7 +51,7 @@ static void note_irq_disable(void *u1, unsigned long u2, unsigned long u3) this_cpu_write(irq_disabled_ts, sched_clock()); } -static void test_irq_disable_long(void *u1, unsigned long u2, unsigned long u3) +static void test_irq_disable_long(void *u1, unsigned long ip, unsigned long parent_ip) { u64 ts = this_cpu_read(irq_disabled_ts); @@ -62,10 +62,10 @@ static void test_irq_disable_long(void *u1, unsigned long u2, unsigned long u3) ts = sched_clock() - ts; if (ts > sysctl_irqsoff_tracing_threshold_ns) { - trace_irq_disable_long(ts); + trace_irq_disable_long(ts, ip, parent_ip, CALLER_ADDR4, CALLER_ADDR5); if (sysctl_irqsoff_dmesg_output_enabled == IRQSOFF_SENTINEL) - printk_deferred("D=%llu C:(%ps<-%ps<-%ps<-%ps)\n", + printk_deferred("irqs off exceeds thresh delta=%llu C:(%ps<-%ps<-%ps<-%ps)\n", ts, (void *)CALLER_ADDR2, (void *)CALLER_ADDR3, (void *)CALLER_ADDR4, @@ -90,8 +90,8 @@ static void note_preempt_disable(void *u1, unsigned long u2, unsigned long u3) ps->ncsw = current->nvcsw + current->nivcsw; } -static void test_preempt_disable_long(void *u1, unsigned long u2, - unsigned long u3) +static void test_preempt_disable_long(void *u1, unsigned long ip, + unsigned long parent_ip) { struct preempt_store *ps = &per_cpu(the_ps, raw_smp_processor_id()); u64 delta = 0; @@ -114,7 +114,7 @@ static void test_preempt_disable_long(void *u1, unsigned long u2, ps->ts = 0; if (delta > sysctl_preemptoff_tracing_threshold_ns) - trace_preempt_disable_long(delta); + trace_preempt_disable_long(delta, ip, parent_ip, CALLER_ADDR4, CALLER_ADDR5); } static struct ctl_table preemptirq_long_table[] = { diff --git a/kernel/sched/walt/preemptirq_long.h b/kernel/sched/walt/preemptirq_long.h index 48ede42a0655..89dce56d79df 100644 --- a/kernel/sched/walt/preemptirq_long.h +++ b/kernel/sched/walt/preemptirq_long.h @@ -14,30 +14,44 @@ #include +/* reference preemptirq_template */ DECLARE_EVENT_CLASS(preemptirq_long_template, - TP_PROTO(u64 delta), + TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip, + unsigned long pparent_ip, unsigned long ppparent_ip), - TP_ARGS(delta), + TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip), TP_STRUCT__entry( __field(u64, delta) + __field(unsigned long, caller_offs) + __field(unsigned long, parent_offs) + __field(unsigned long, pparent_offs) + __field(unsigned long, ppparent_offs) ), TP_fast_assign( __entry->delta = delta; + __entry->caller_offs = ip; + __entry->parent_offs = parent_ip; + __entry->pparent_offs = pparent_ip; + __entry->ppparent_offs = ppparent_ip; ), - TP_printk("delta=%llu(ns)", __entry->delta) + TP_printk("delta=%llu(ns) caller=%ps <- %ps <- %ps <- %ps", + __entry->delta, __entry->caller_offs, + __entry->parent_offs, __entry->pparent_offs, __entry->ppparent_offs) ); DEFINE_EVENT(preemptirq_long_template, irq_disable_long, - TP_PROTO(u64 delta), - TP_ARGS(delta)); + TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip, + unsigned long pparent_ip, unsigned long ppparent_ip), + TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip)); DEFINE_EVENT(preemptirq_long_template, preempt_disable_long, - TP_PROTO(u64 delta), - TP_ARGS(delta)); + TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip, + unsigned long pparent_ip, unsigned long ppparent_ip), + TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip)); #endif /* _TRACE_PREEMPTIRQ_LONG_H */ From 2eecd3353fa649e54e8d6794b68fc461e5e3441d Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Mon, 1 Nov 2021 15:41:25 -0700 Subject: [PATCH 208/346] sched/walt: Remove duplicate get_task_struct get_pid_task internally increments the reference counter, which could cause unbalanced reference counter (always high). Only call get_task_struct when pid is 0. Change-Id: I409b477d0714ce35a0dc980782ee5878106ecffd Signed-off-by: Elliot Berman Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/fixup.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/fixup.c b/kernel/sched/walt/fixup.c index 2b7bae6d00ef..f3b5d7e258cd 100644 --- a/kernel/sched/walt/fixup.c +++ b/kernel/sched/walt/fixup.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2021 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -31,18 +32,13 @@ static bool is_sched_lib_based_app(pid_t pid) return false; rcu_read_lock(); - - p = pid ? get_pid_task(find_vpid(pid), PIDTYPE_PID) : current; + p = pid ? get_pid_task(find_vpid(pid), PIDTYPE_PID) : get_task_struct(current); + rcu_read_unlock(); if (!p) { - rcu_read_unlock(); kfree(tmp_lib_name); return false; } - /* Prevent p going away */ - get_task_struct(p); - rcu_read_unlock(); - mm = get_task_mm(p); if (!mm) goto put_task_struct; From 3fbfc998c9328fe526deded1a7d6f239431d081d Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Tue, 2 Nov 2021 09:30:31 -0700 Subject: [PATCH 209/346] walt_lb: Read push_task under lock push_task assignment is guarded by corresponding runqueue's lock. Read it only when lock is acquired. Change-Id: I74668df66b4e18286b919b136d912a3f8afda11c Signed-off-by: Elliot Berman Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_lb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 20adabaf92c4..46f1cbd82392 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021, Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -39,10 +40,11 @@ static int walt_lb_active_migration(void *data) int target_cpu = busiest_rq->push_cpu; struct rq *target_rq = cpu_rq(target_cpu); struct walt_rq *wrq = (struct walt_rq *) busiest_rq->android_vendor_data1; - struct task_struct *push_task = wrq->push_task; + struct task_struct *push_task; int push_task_detached = 0; raw_spin_lock_irq(&busiest_rq->__lock); + push_task = wrq->push_task; /* sanity checks before initiating the pull */ if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu) || !push_task) From b69d74c29eff1105935ec10b26b2274cac6ec27c Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 2 Nov 2021 10:29:17 -0700 Subject: [PATCH 210/346] sched: Improve the Scheduler This change is for general scheduler improvement. Change-Id: If9c4859601bdcb1ab92bf6d7125bcb74c9ec3f7b Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 4 ++-- kernel/sched/walt/walt.h | 19 ------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 97c3f55ebd9e..c291ef114857 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -502,7 +502,7 @@ unsigned int walt_big_tasks(int cpu) return wrq->walt_stats.nr_big_tasks; } -void clear_walt_request(int cpu) +static void clear_walt_request(int cpu) { struct rq *rq = cpu_rq(cpu); unsigned long flags; @@ -2197,7 +2197,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even run_walt_irq_work(old_window_start, rq); } -static void __sched_fork_init(struct task_struct *p) +static inline void __sched_fork_init(struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 4008992f5315..63404a15da48 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -151,7 +151,6 @@ extern cpumask_t __read_mostly **cpu_array; extern int cpu_l2_sibling[WALT_NR_CPUS]; extern void sched_update_nr_prod(int cpu, int enq); extern unsigned int walt_big_tasks(int cpu); -extern void walt_rotate_work_init(void); extern void walt_rotation_checkpoint(int nr_big); extern void walt_fill_ta_data(struct core_ctl_notif_data *data); extern int sched_set_group_id(struct task_struct *p, unsigned int group_id); @@ -159,38 +158,20 @@ extern unsigned int sched_get_group_id(struct task_struct *p); extern void core_ctl_check(u64 wallclock); extern int sched_set_boost(int enable); extern void walt_boost_init(void); -extern int sched_wake_up_idle_show(struct seq_file *m, void *v); -extern ssize_t sched_wake_up_idle_write(struct file *file, - const char __user *buf, size_t count, loff_t *offset); -extern int sched_wake_up_idle_open(struct inode *inode, struct file *filp); -extern int sched_init_task_load_show(struct seq_file *m, void *v); -extern ssize_t sched_init_task_load_write(struct file *file, const char __user *buf, - size_t count, loff_t *offset); -extern int sched_init_task_load_open(struct inode *inode, struct file *filp); -extern int sched_group_id_show(struct seq_file *m, void *v); -extern ssize_t sched_group_id_write(struct file *file, const char __user *buf, - size_t count, loff_t *offset); -extern int sched_group_id_open(struct inode *inode, struct file *filp); extern int sched_pause_cpus(struct cpumask *pause_cpus); extern int sched_unpause_cpus(struct cpumask *unpause_cpus); extern unsigned int sched_get_cpu_util(int cpu); extern void sched_update_hyst_times(void); -extern int -sched_updown_migrate_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, loff_t *ppos); extern int sched_boost_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); extern int sched_busy_hyst_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); extern u64 walt_ktime_get_ns(void); -extern void clear_walt_request(int cpu); extern void walt_init_tg(struct task_group *tg); extern void walt_init_topapp_tg(struct task_group *tg); extern void walt_init_foreground_tg(struct task_group *tg); extern int register_walt_callback(void); -extern void set_cpu_array(void); -extern int core_ctl_init(void); extern int input_boost_init(void); extern int core_ctl_init(void); From 7ba999c4e37f277eac2cf3646d53a08cd17ec6ce Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 3 Nov 2021 16:51:24 -0700 Subject: [PATCH 211/346] sched: walt: Fix bad casting in trace There is a bad cast from unsigned long to unsigned int that is taking place in sched_cpu_util tracepoint, and this is causing bad values to appear in the traces. Change-Id: I51561ac582b616e3319c8f43648cf5a4bfc632b6 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index ee8f4f7c99fe..554a27feec32 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -903,9 +903,9 @@ TRACE_EVENT(sched_cpu_util, __field(unsigned int, nr_running) __field(long, cpu_util) __field(long, cpu_util_cum) - __field(unsigned int, capacity_curr) - __field(unsigned int, capacity) - __field(unsigned int, capacity_orig) + __field(unsigned long, capacity_curr) + __field(unsigned long, capacity) + __field(unsigned long, capacity_orig) __field(unsigned int, idle_exit_latency) __field(u64, irqload) __field(int, online) @@ -935,7 +935,7 @@ TRACE_EVENT(sched_cpu_util, __entry->prs_gprs = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; ), - TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%u capacity=%u capacity_orig=%u idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u prs_gprs=%llu", + TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%lu capacity=%lu capacity_orig=%lu idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u prs_gprs=%llu", __entry->cpu, __entry->nr_running, __entry->cpu_util, __entry->cpu_util_cum, __entry->capacity_curr, __entry->capacity, __entry->capacity_orig, From 71f5d6022e02adecdac216de0ddd5f448ad21df1 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Tue, 27 Apr 2021 16:21:11 +0530 Subject: [PATCH 212/346] sched: walt: Fix stale walt CPU reservation flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When CPU trying to move a task to other cpu in active load balance or by other means, then the other helping cpu marked as reserved to avoid it for other scheduler decisions. Once the task moved successfully, the reservation will be cleared enables for other scheduler decisions. The reserved flag is been analogously protected with busy cpu’s rq->active_balance, which is protected with runqueue locks. So whenever rq->active_balance is set for busy cpu, then reserved flag would set for helping cpu. Sometimes, it is observed that, cpu is marked as reserved with no cpu's rq->active_balance set. There are some unlikely possible corner cases may cause this behavior: - On active load balance path, cpu stop machine returns queued status of active_balance work on cpu_stopper, which is not checked on active balance path. so when stop machine is not able to queue ( unlikely), then reserved flag wouldn't be cleared. So, catch the return value and on failure, clear reserved flag for cpu. - Clear_walt_request() called on the cpu to clear any pending walt works, it may possible that, push_task might have changed or cleared, then the reserved cpu would be left uncleared. So clear the push_cpu independent of push_task. Change-Id: I75d032bf399cb3da8e807186b1bc903114168a4e Signed-off-by: Pavankumar Kondeti Signed-off-by: Lingutla Chandrasekhar Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_lb.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 46f1cbd82392..cbf5c6300593 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -33,7 +33,7 @@ static void walt_attach_task(struct task_struct *p, struct rq *rq) check_preempt_curr(rq, p, 0); } -static int walt_lb_active_migration(void *data) +static int stop_walt_lb_active_migration(void *data) { struct rq *busiest_rq = data; int busiest_cpu = cpu_of(busiest_rq); @@ -307,10 +307,15 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) * the push_task is really pulled onto this CPU. */ if (active_balance) { + bool success; + wts = (struct walt_task_struct *) p->android_vendor_data1; trace_walt_active_load_balance(p, src_cpu, dst_cpu, wts); - stop_one_cpu_nowait(src_cpu, walt_lb_active_migration, + success = stop_one_cpu_nowait(src_cpu, stop_walt_lb_active_migration, src_rq, &src_rq->active_balance_work); + if (!success) + clear_reserved(dst_cpu); + return 0; /* we did not pull any task here */ } @@ -543,7 +548,7 @@ void walt_lb_tick(struct rq *rq) trace_walt_active_load_balance(p, prev_cpu, new_cpu, wts); ret = stop_one_cpu_nowait(prev_cpu, - walt_lb_active_migration, rq, + stop_walt_lb_active_migration, rq, &rq->active_balance_work); if (!ret) clear_reserved(new_cpu); From 6bff8ea1101b3d532ee853760b94f8809cb8b60e Mon Sep 17 00:00:00 2001 From: Lingutla Chandrasekhar Date: Tue, 1 Jun 2021 16:58:40 +0530 Subject: [PATCH 213/346] sched: walt: Improve the scheduler This change is for general scheduler improvements. Change-Id: Ia2854ae8701151761fe0780b6451133ab09a050b Signed-off-by: Lingutla Chandrasekhar Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_lb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index cbf5c6300593..e6da5e955a9e 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -505,6 +505,9 @@ void walt_lb_tick(struct rq *rq) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + if (available_idle_cpu(prev_cpu) && is_reserved(prev_cpu)) + clear_reserved(prev_cpu); + if (!walt_fair_task(p)) return; From 66317940e6ab7bafa4c1fd236502f39b522665d4 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 3 Nov 2021 14:15:57 -0700 Subject: [PATCH 214/346] sched: Improve the scheduler This change is for general scheduler improvements. Change-Id: I37d6cb75ca8b08d9ca155b86b7d71ff369f46e14 Signed-off-by: Lingutla Chandrasekhar Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_lb.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index e6da5e955a9e..cab632d7e724 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -103,12 +103,21 @@ static void walt_lb_rotate_work_func(struct work_struct *work) { struct walt_lb_rotate_work *wr = container_of(work, struct walt_lb_rotate_work, w); + struct rq *src_rq = cpu_rq(wr->src_cpu), *dst_rq = cpu_rq(wr->dst_cpu); + unsigned long flags; migrate_swap(wr->src_task, wr->dst_task, wr->dst_cpu, wr->src_cpu); put_task_struct(wr->src_task); put_task_struct(wr->dst_task); + local_irq_save(flags); + double_rq_lock(src_rq, dst_rq); + dst_rq->active_balance = 0; + src_rq->active_balance = 0; + double_rq_unlock(src_rq, dst_rq); + local_irq_restore(flags); + clear_reserved(wr->src_cpu); clear_reserved(wr->dst_cpu); } @@ -195,7 +204,8 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) dst_rq = cpu_rq(dst_cpu); double_rq_lock(src_rq, dst_rq); - if (walt_fair_task(dst_rq->curr)) { + if (walt_fair_task(dst_rq->curr) && + !src_rq->active_balance && !dst_rq->active_balance) { get_task_struct(src_rq->curr); get_task_struct(dst_rq->curr); @@ -208,6 +218,9 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) wr->src_cpu = src_cpu; wr->dst_cpu = dst_cpu; + + dst_rq->active_balance = 1; + src_rq->active_balance = 1; } double_rq_unlock(src_rq, dst_rq); @@ -505,8 +518,10 @@ void walt_lb_tick(struct rq *rq) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (available_idle_cpu(prev_cpu) && is_reserved(prev_cpu)) + raw_spin_lock(&rq->__lock); + if (available_idle_cpu(prev_cpu) && is_reserved(prev_cpu) && !rq->active_balance) clear_reserved(prev_cpu); + raw_spin_unlock(&rq->__lock); if (!walt_fair_task(p)) return; From a9b383db70c5948124a1c7af402e243e728a4b90 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 2 Nov 2021 17:09:09 -0700 Subject: [PATCH 215/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: Iffd4ae221581aaa4aeb244a0cddd40a8b6aac74d Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_lb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index cab632d7e724..7d09bb67f0c4 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -205,7 +205,9 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) double_rq_lock(src_rq, dst_rq); if (walt_fair_task(dst_rq->curr) && - !src_rq->active_balance && !dst_rq->active_balance) { + !src_rq->active_balance && !dst_rq->active_balance && + cpumask_test_cpu(dst_cpu, src_rq->curr->cpus_ptr) && + cpumask_test_cpu(src_cpu, dst_rq->curr->cpus_ptr)) { get_task_struct(src_rq->curr); get_task_struct(dst_rq->curr); From 863c755de0b6850d518e4c5b4b2552ba12837bcf Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Wed, 24 Nov 2021 11:11:31 -0800 Subject: [PATCH 216/346] sched/walt: Restore obj file in Makefile Restore obj file in Makefile that was removed in commit 3908a213bc56 ("pause: port the users of pause"). Change-Id: I2014b049ca7cb5e4188b07087cc22fa9ee40cf85 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile index 870d9b2fcfe3..ed35366108ed 100644 --- a/kernel/sched/walt/Makefile +++ b/kernel/sched/walt/Makefile @@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n KCSAN_SANITIZE := n obj-$(CONFIG_SCHED_WALT) += sched-walt.o -sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o From a58c301c82bd6201e266785e9fca5b5e1404fd32 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 23 Nov 2021 18:15:47 -0800 Subject: [PATCH 217/346] sched/walt: Remove unsused function Remove unused function Change-Id: I516d65f7fa18d82a5a414bf4b06cbf9d52661fa9 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_cfs.c | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 770360783133..68b2d887654e 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -928,30 +928,6 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, *target_cpu = prev_cpu; } -static void walt_place_entity(void *unused, struct sched_entity *se, u64 *vruntime) -{ - if (unlikely(walt_disabled)) - return; - if (entity_is_task(se)) { - unsigned long thresh = sysctl_sched_latency; - - /* - * Halve their sleep time's effect, to allow - * for a gentler effect of sleepers: - */ - if (sched_feat(GENTLE_FAIR_SLEEPERS)) - thresh >>= 1; - - if ((per_task_boost(task_of(se)) == TASK_BOOST_STRICT_MAX) || - walt_low_latency_task(task_of(se)) || - task_rtg_high_prio(task_of(se))) { - *vruntime -= sysctl_sched_latency; - *vruntime -= thresh; - se->vruntime = *vruntime; - } - } -} - static void walt_binder_low_latency_set(void *unused, struct task_struct *task, bool sync, struct binder_proc *proc) { From e2bc168ffb6d7787457b5e8fb90fe3a66f371e58 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Mon, 6 Dec 2021 16:47:43 -0800 Subject: [PATCH 218/346] sched/walt: Fix frequent frequency updates As a side effect of commit f64fafbec582dd46169fba528cad2341c0c2d906 the check to skip frequency update if the same frequency as the previous frequency is being applied, got removed at the point of update. Restore this check at the appropriate place. Change-Id: I7aee7f84f0a64747b01ac31c2fdcbb4440908deb Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/cpufreq_walt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index d1a46f3a59ee..377f0602c91e 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -239,7 +239,7 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, if (wg_policy->cached_raw_freq && freq == wg_policy->cached_raw_freq && !wg_policy->need_freq_update) - return wg_policy->next_freq; + return 0; wg_policy->need_freq_update = false; From ffce2634bcbcf2b8a75125d8fe91fdeec284d7bc Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Thu, 11 Nov 2021 09:38:18 +0800 Subject: [PATCH 219/346] sched/walt: remove task from mvp list if task in yield When a task yields, it relinquishes the cpu and scheduler is tasked to find another task. However he MVP implementation could return the same task leading to a loop where the yielded task gets to run back. To fix this, drop the MVP status of tasks that yield themselves. The MVP status is restored the next time they actually get enqueued - -likely because of a wakeup from sleep or task migration. Change-Id: Ia2a2074f18c4de56e0695ee2c5f1daf8a4eb5980 Signed-off-by: Tengfei Fan Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_cfs.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 68b2d887654e..2bfc05833d59 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1100,6 +1100,16 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr walt_cfs_insert_mvp_task(wrq, wts, false); } +static void walt_cfs_mvp_do_sched_yield(void *unused, struct rq *rq) +{ + struct task_struct *curr = rq->curr; + int mvp_prio = walt_get_mvp_task_prio(curr); + + lockdep_assert_held(&rq->__lock); + if (mvp_prio != WALT_NOT_MVP) + walt_cfs_deactivate_mvp_task(curr); +} + void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; @@ -1282,4 +1292,6 @@ void walt_cfs_init(void) register_trace_android_rvh_check_preempt_wakeup(walt_cfs_check_preempt_wakeup, NULL); register_trace_android_rvh_replace_next_task_fair(walt_cfs_replace_next_task_fair, NULL); + + register_trace_android_rvh_do_sched_yield(walt_cfs_mvp_do_sched_yield, NULL); } From 66f7fd76dfea3bbaa891483f17195c9f9763503b Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 6 Jan 2022 14:59:09 -0800 Subject: [PATCH 220/346] sched/walt: walt irq work reduce locking Optimize walt_irq_work to reduce locking for migrations. Reduce scheduler overheads by reducing the locks needed for the walt irq work migrations. Precisely track the cpus locked, and unlock them at conclusion of work function. Change-Id: I6ea6d24dc69e9876387253b489e91f82e45ed6d0 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 122 ++++++++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 26 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c291ef114857..91ddc2ebe007 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2143,7 +2143,7 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, wts->cpu_cycles = cur_cycles; } -static inline void run_walt_irq_work(u64 old_window_start, struct rq *rq) +static inline void run_walt_irq_work_rollover(u64 old_window_start, struct rq *rq) { u64 result; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; @@ -2194,7 +2194,7 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even done: wts->mark_start = wallclock; - run_walt_irq_work(old_window_start, rq); + run_walt_irq_work_rollover(old_window_start, rq); } static inline void __sched_fork_init(struct task_struct *p) @@ -3372,48 +3372,49 @@ static void walt_update_irqload(struct rq *rq) wrq->high_irqload = false; } -/* - * Runs in hard-irq context. This should ideally run just after the latest - * window roll-over. +/** + * __walt_irq_work_locked() - common function to process work + * @is_migration: if true, performing migration work, else rollover + * @lock_cpus: mask of the cpus involved in the operation. + * + * In rq locked context, update the cluster group load and find + * the load of the min cluster, while tracking the total aggregate + * work load. Update the cpufreq through the walt governor, + * based upon the new load calculated. + * + * For the window rollover case lock_cpus will be all possible cpus, + * and for migrations it will include the cpus from the two clusters + * involved in the migration. */ -static void walt_irq_work(struct irq_work *irq_work) +static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *lock_cpus) { struct walt_sched_cluster *cluster; struct rq *rq; int cpu; u64 wc; - bool is_migration = false, is_asym_migration = false; + bool is_asym_migration = false; u64 total_grp_load = 0, min_cluster_grp_load = 0; - int level = 0; unsigned long flags; struct walt_rq *wrq; - /* Am I the window rollover work or the migration work? */ - if (irq_work == &walt_migration_irq_work) - is_migration = true; - - for_each_cpu(cpu, cpu_possible_mask) { - if (level == 0) - raw_spin_lock(&cpu_rq(cpu)->__lock); - else - raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); - level++; - } - wc = walt_ktime_get_ns(); walt_load_reported_window = atomic64_read(&walt_irq_work_lastq_ws); for_each_sched_cluster(cluster) { u64 aggr_grp_load = 0; raw_spin_lock(&cluster->load_lock); - for_each_cpu(cpu, &cluster->cpus) { rq = cpu_rq(cpu); wrq = (struct walt_rq *) rq->android_vendor_data1; if (rq->curr) { - walt_update_task_ravg(rq->curr, rq, - TASK_UPDATE, wc, 0); - account_load_subtractions(rq); + /* only update ravg for locked cpus */ + if (cpumask_intersects(lock_cpus, &cluster->cpus)) { + walt_update_task_ravg(rq->curr, rq, + TASK_UPDATE, wc, 0); + account_load_subtractions(rq); + } + + /* update aggr_grp_load for all clusters, all cpus */ aggr_grp_load += wrq->grp_time.prev_runnable_sum; } @@ -3423,13 +3424,13 @@ static void walt_irq_work(struct irq_work *irq_work) wrq->notif_pending = false; } } + raw_spin_unlock(&cluster->load_lock); cluster->aggr_grp_load = aggr_grp_load; total_grp_load += aggr_grp_load; if (is_min_capacity_cluster(cluster)) min_cluster_grp_load = aggr_grp_load; - raw_spin_unlock(&cluster->load_lock); } if (total_grp_load) { @@ -3453,6 +3454,10 @@ static void walt_irq_work(struct irq_work *irq_work) cpumask_t cluster_online_cpus; unsigned int num_cpus, i = 1; + /* for migration, skip unnotified clusters */ + if (is_migration && !cpumask_intersects(lock_cpus, &cluster->cpus)) + continue; + cpumask_and(&cluster_online_cpus, &cluster->cpus, cpu_online_mask); num_cpus = cpumask_weight(&cluster_online_cpus); @@ -3514,8 +3519,73 @@ static void walt_irq_work(struct irq_work *irq_work) } spin_unlock_irqrestore(&sched_ravg_window_lock, flags); } +} - for_each_cpu(cpu, cpu_possible_mask) +/** + * irq_work_restrict_to_mig_clusters() - only allow notified clusters + * @lock_cpus: mask of the cpus for which the runque should be locked. + * + * Remove cpus in clusters that are not part of the migration, using + * the notif_pending flag to track. + * + * This is only valid for the migration irq work. + */ +static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus) +{ + struct walt_sched_cluster *cluster; + struct rq *rq; + struct walt_rq *wrq; + int cpu; + + for_each_sched_cluster(cluster) { + for_each_cpu(cpu, &cluster->cpus) { + rq = cpu_rq(cpu); + wrq = (struct walt_rq *)rq->android_vendor_data1; + + /* remove this cluster if it's not being notified */ + if (!wrq->notif_pending) { + cpumask_andnot(lock_cpus, lock_cpus, &cluster->cpus); + break; + } + } + } +} + +/** + * walt_irq_work() - perform walt irq work for rollover and migration + * + * Process a workqueue call scheduled, while running in a hard irq + * protected context. Handle migration and window rollover work + * with common funtionality, and on window rollover ask core control + * to decide if it needs to adjust the active cpus. + */ +static void walt_irq_work(struct irq_work *irq_work) +{ + cpumask_t lock_cpus; + struct walt_rq *wrq; + int level = 0; + int cpu; + bool is_migration = false; + + if (irq_work == &walt_migration_irq_work) + is_migration = true; + + cpumask_copy(&lock_cpus, cpu_possible_mask); + + if (is_migration) + irq_work_restrict_to_mig_clusters(&lock_cpus); + + for_each_cpu(cpu, &lock_cpus) { + if (level == 0) + raw_spin_lock(&cpu_rq(cpu)->__lock); + else + raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); + level++; + } + + __walt_irq_work_locked(is_migration, &lock_cpus); + + for_each_cpu(cpu, &lock_cpus) raw_spin_unlock(&cpu_rq(cpu)->__lock); if (!is_migration) { From 4e3c9e164089d7bbcb444c7353ecd23a8eea1c6f Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:26:49 -0800 Subject: [PATCH 221/346] sched/walt/halt: walt halt functionality Provide apis to the system in support of halting cpus, as a replacement for pause/isolation. Track the requests to halt a cpu, and only start the cpu again when all requests have been recanted. Perform a migration of tasks off of a halted cpu, if that cpu is online and not idle. Change-Id: I99340b12e01ff53499b4107d843b7d48e6d6dff1 Signed-off-by: Stephen Dickey --- include/linux/sched/walt.h | 2 + kernel/sched/walt/Makefile | 2 +- kernel/sched/walt/walt.c | 1 + kernel/sched/walt/walt.h | 13 ++ kernel/sched/walt/walt_halt.c | 325 ++++++++++++++++++++++++++++++++++ 5 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 kernel/sched/walt/walt_halt.c diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 3d4106c4f4d9..c03a3f30caa4 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -157,6 +157,8 @@ extern void core_ctl_notifier_unregister(struct notifier_block *n); extern int core_ctl_set_boost(bool boost); extern int walt_pause_cpus(struct cpumask *cpus); extern int walt_resume_cpus(struct cpumask *cpus); +extern int walt_halt_cpus(struct cpumask *cpus); +extern int walt_start_cpus(struct cpumask *cpus); #else static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout) { diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile index ed35366108ed..fbb464df6f38 100644 --- a/kernel/sched/walt/Makefile +++ b/kernel/sched/walt/Makefile @@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n KCSAN_SANITIZE := n obj-$(CONFIG_SCHED_WALT) += sched-walt.o -sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o walt_halt.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 91ddc2ebe007..6c7bda27f594 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4324,6 +4324,7 @@ static void walt_init(struct work_struct *work) walt_rt_init(); walt_cfs_init(); walt_pause_init(); + walt_halt_init(); stop_machine(walt_init_stop_handler, NULL, NULL); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 63404a15da48..e5a4da93afe3 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -757,6 +757,7 @@ extern void sched_update_hyst_times(void); extern void walt_rt_init(void); extern void walt_cfs_init(void); extern void walt_pause_init(void); +extern void walt_halt_init(void); extern void walt_fixup_init(void); extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync, int sibling_count_hint); @@ -901,11 +902,22 @@ struct compute_energy_output { unsigned int cluster_first_cpu[MAX_CLUSTERS]; }; +extern struct cpumask __cpu_halt_mask; +#define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask) +#define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask) + +extern struct cpumask __cpu_can_halt_mask; +#define cpu_can_halt_mask ((struct cpumask *)&__cpu_can_halt_mask) +#define cpu_can_halt(cpu) cpumask_test_cpu((cpu), cpu_can_halt_mask) + extern void walt_task_dump(struct task_struct *p); extern void walt_rq_dump(int cpu); extern void walt_dump(void); extern int in_sched_bug; +extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, + struct task_struct *p, int dest_cpu); + #define WALT_PANIC(condition) \ ({ \ if (unlikely(!!(condition)) && !in_sched_bug) { \ @@ -931,4 +943,5 @@ extern int in_sched_bug; } \ }) + #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c new file mode 100644 index 000000000000..d36c081ce25b --- /dev/null +++ b/kernel/sched/walt/walt_halt.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ +#include +#include +#include +#include "trace.h" + +#ifdef CONFIG_HOTPLUG_CPU + +/* if a cpu is halting */ +struct cpumask __cpu_halt_mask; +struct cpumask __cpu_can_halt_mask; + +static DEFINE_MUTEX(halt_lock); + +struct halt_cpu_state { + int ref_count; +}; + +static DEFINE_PER_CPU(struct halt_cpu_state, halt_state); + +/* + * Remove a task from the runqueue and pretend that it's migrating. This + * should prevent migrations for the detached task and disallow further + * changes to tsk_cpus_allowed. + */ +void +detach_one_task_core(struct task_struct *p, struct rq *rq, + struct list_head *tasks) +{ + lockdep_assert_held(&rq->__lock); + + p->on_rq = TASK_ON_RQ_MIGRATING; + deactivate_task(rq, p, 0); + list_add(&p->se.group_node, tasks); +} + +void attach_tasks_core(struct list_head *tasks, struct rq *rq) +{ + struct task_struct *p; + + lockdep_assert_held(&rq->__lock); + + while (!list_empty(tasks)) { + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); + + BUG_ON(task_rq(p) != rq); + activate_task(rq, p, 0); + p->on_rq = TASK_ON_RQ_QUEUED; + } +} + +/* + * Migrate all tasks from the rq, sleeping tasks will be migrated by + * try_to_wake_up()->select_task_rq(). + * + * Called with rq->__lock held even though we'er in stop_machine() and + * there's no concurrency possible, we hold the required locks anyway + * because of lock validation efforts. + * + * force: if false, the function will skip CPU pinned kthreads. + */ +static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, bool force) +{ + struct rq *rq = dead_rq; + struct task_struct *next, *stop = rq->stop; + LIST_HEAD(percpu_kthreads); + unsigned int num_pinned_kthreads = 1; + struct rq_flags orf = *rf; + int dest_cpu; + + /* + * Fudge the rq selection such that the below task selection loop + * doesn't get stuck on the currently eligible stop task. + * + * We're currently inside stop_machine() and the rq is either stuck + * in the stop_machine_cpu_stop() loop, or we're executing this code, + * either way we should never end up calling schedule() until we're + * done here. + */ + rq->stop = NULL; + + /* + * put_prev_task() and pick_next_task() sched + * class method both need to have an up-to-date + * value of rq->clock[_task] + */ + update_rq_clock(rq); + +#ifdef CONFIG_SCHED_DEBUG + /* note the clock update in orf */ + orf.clock_update_flags |= RQCF_UPDATED; +#endif + + for (;;) { + /* + * There's this thread running, bail when that's the only + * remaining thread: + */ + if (rq->nr_running == 1) + break; + + next = pick_migrate_task(rq); + + /* + * Argh ... no iterator for tasks, we need to remove the + * kthread from the run-queue to continue. + */ + + if (!force && is_per_cpu_kthread(next)) { + detach_one_task_core(next, rq, &percpu_kthreads); + num_pinned_kthreads += 1; + continue; + } + + /* + * Rules for changing task_struct::cpus_mask are holding + * both pi_lock and rq->__lock, such that holding either + * stabilizes the mask. + * + * Drop rq->__lock is not quite as disastrous as it usually is + * because !cpu_active at this point, which means load-balance + * will not interfere. Also, stop-machine. + */ + rq_unlock(rq, rf); + raw_spin_lock(&next->pi_lock); + rq_relock(rq, rf); + + /* + * Since we're inside stop-machine, _nothing_ should have + * changed the task, WARN if weird stuff happened, because in + * that case the above rq->__lock drop is a fail too. + */ + if (task_rq(next) != rq || !task_on_rq_queued(next)) { + /* + * In the !force case, there is a hole between + * rq_unlock() and rq_relock(), where another CPU might + * not observe an up to date cpu_active_mask and try to + * move tasks around. + */ + WARN_ON(force); + raw_spin_unlock(&next->pi_lock); + continue; + } + + /* Find suitable destination for @next, with force if needed. */ + dest_cpu = select_fallback_rq(dead_rq->cpu, next); + rq = __migrate_task(rq, rf, next, dest_cpu); + if (rq != dead_rq) { + rq_unlock(rq, rf); + rq = dead_rq; + *rf = orf; + rq_relock(rq, rf); + } + raw_spin_unlock(&next->pi_lock); + } + + if (num_pinned_kthreads > 1) + attach_tasks_core(&percpu_kthreads, rq); + + rq->stop = stop; +} + +static int drain_rq_cpu_stop(void *data) +{ + struct rq *rq = this_rq(); + struct rq_flags rf; + + rq_lock_irqsave(rq, &rf); + migrate_tasks(rq, &rf, false); + rq_unlock_irqrestore(rq, &rf); + + return 0; +} + +static int sched_cpu_drain_rq(unsigned int cpu) +{ + if (available_idle_cpu(cpu)) + return 0; + + return stop_one_cpu(cpu, drain_rq_cpu_stop, NULL); +} + +/* + * 1) add the cpus to the halt mask + * 2) migrate tasks off the cpu + * + */ +static int halt_cpus(struct cpumask *cpus) +{ + int cpu; + int ret = 0; + + for_each_cpu(cpu, cpus) { + + /* set the cpu as halted */ + cpumask_set_cpu(cpu, cpu_halt_mask); + + /* only drain online cpus */ + if (cpu_online(cpu)) { + /* drain the online CPU */ + ret = sched_cpu_drain_rq(cpu); + } + + if (ret < 0) { + /* cpu failed to drain, do not mark as halted */ + cpumask_clear_cpu(cpu, cpu_halt_mask); + break; + } + } + + return ret; +} + +/* + * 1) remove the cpus from the halt mask + * + */ +static int start_cpus(struct cpumask *cpus) +{ + if (!cpumask_empty(cpus)) { + + /* remove the cpus from the halt mask */ + cpumask_andnot(cpu_halt_mask, cpu_halt_mask, cpus); + } + + return 0; +} + +/* increment/decrement ref count for cpus in yield/halt mask */ +static void update_ref_counts(struct cpumask *cpus, bool halt) +{ + int cpu; + struct halt_cpu_state *halt_cpu_state; + + for_each_cpu(cpu, cpus) { + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); + if (halt) + halt_cpu_state->ref_count++; + else { + WARN_ON_ONCE(halt_cpu_state->ref_count == 0); + halt_cpu_state->ref_count--; + } + } +} + +static void update_halt_cpus(struct cpumask *cpus) +{ + int cpu; + struct halt_cpu_state *halt_cpu_state; + + for_each_cpu(cpu, cpus) { + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); + if (halt_cpu_state->ref_count) + cpumask_clear_cpu(cpu, cpus); + } +} + +/* cpus will be modified */ +int walt_halt_cpus(struct cpumask *cpus) +{ + int ret = 0; + cpumask_t requested_cpus; + + mutex_lock(&halt_lock); + + cpumask_copy(&requested_cpus, cpus); + update_halt_cpus(cpus); + + if (cpumask_empty(cpus)) { + update_ref_counts(&requested_cpus, true); + goto unlock; + } + + ret = halt_cpus(cpus); + + if (ret < 0) + pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret, + cpumask_pr_args(&requested_cpus)); + else + update_ref_counts(&requested_cpus, true); +unlock: + mutex_unlock(&halt_lock); + + return ret; +} +EXPORT_SYMBOL(walt_halt_cpus); + +/* cpus will be modified */ +int walt_start_cpus(struct cpumask *cpus) +{ + int ret = 0; + cpumask_t requested_cpus; + + mutex_lock(&halt_lock); + cpumask_copy(&requested_cpus, cpus); + update_ref_counts(&requested_cpus, false); + update_halt_cpus(cpus); + + ret = start_cpus(cpus); + + if (ret < 0) { + pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret, + cpumask_pr_args(&requested_cpus)); + /* restore/increment ref counts in case of error */ + update_ref_counts(&requested_cpus, true); + } + + mutex_unlock(&halt_lock); + + return ret; +} +EXPORT_SYMBOL(walt_start_cpus); + +void walt_halt_init(void) +{ + cpumask_set_cpu(5, cpu_can_halt_mask); + cpumask_set_cpu(6, cpu_can_halt_mask); + cpumask_set_cpu(7, cpu_can_halt_mask); +} + +#endif /* CONFIG_HOTPLUG_CPU */ From bb73350eac965ec1b6e74aa9c5610572e8617e9c Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:26:52 -0800 Subject: [PATCH 222/346] sched/walt/pause: remove pause functionality from build References to pause have been removed and replaced with halt. Remove walt pause functionality from the build. Change-Id: I07c412cfba4d9f13aff298aed43a0e208db9f046 Signed-off-by: Stephen Dickey --- kernel/sched/walt/Makefile | 2 +- kernel/sched/walt/walt.c | 1 - kernel/sched/walt/walt.h | 1 - kernel/sched/walt/walt_halt.c | 12 +++ kernel/sched/walt/walt_pause.c | 192 --------------------------------- 5 files changed, 13 insertions(+), 195 deletions(-) delete mode 100644 kernel/sched/walt/walt_pause.c diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile index fbb464df6f38..63f3f1b50e95 100644 --- a/kernel/sched/walt/Makefile +++ b/kernel/sched/walt/Makefile @@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n KCSAN_SANITIZE := n obj-$(CONFIG_SCHED_WALT) += sched-walt.o -sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o walt_halt.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_halt.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 6c7bda27f594..b479186d4f42 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4323,7 +4323,6 @@ static void walt_init(struct work_struct *work) walt_lb_init(); walt_rt_init(); walt_cfs_init(); - walt_pause_init(); walt_halt_init(); stop_machine(walt_init_stop_handler, NULL, NULL); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index e5a4da93afe3..2d00aaee997f 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -756,7 +756,6 @@ extern void sched_update_hyst_times(void); extern void walt_rt_init(void); extern void walt_cfs_init(void); -extern void walt_pause_init(void); extern void walt_halt_init(void); extern void walt_fixup_init(void); extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index d36c081ce25b..04b969f036ff 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -289,6 +289,12 @@ int walt_halt_cpus(struct cpumask *cpus) } EXPORT_SYMBOL(walt_halt_cpus); +int walt_pause_cpus(struct cpumask *cpus) +{ + return walt_halt_cpus(cpus); +} +EXPORT_SYMBOL(walt_pause_cpus); + /* cpus will be modified */ int walt_start_cpus(struct cpumask *cpus) { @@ -315,6 +321,12 @@ int walt_start_cpus(struct cpumask *cpus) } EXPORT_SYMBOL(walt_start_cpus); +int walt_resume_cpus(struct cpumask *cpus) +{ + return walt_start_cpus(cpus); +} +EXPORT_SYMBOL(walt_resume_cpus); + void walt_halt_init(void) { cpumask_set_cpu(5, cpu_can_halt_mask); diff --git a/kernel/sched/walt/walt_pause.c b/kernel/sched/walt/walt_pause.c deleted file mode 100644 index ef6bdcbc897a..000000000000 --- a/kernel/sched/walt/walt_pause.c +++ /dev/null @@ -1,192 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. - */ - -#include -#include - -#ifdef CONFIG_HOTPLUG_CPU - -static DEFINE_MUTEX(pause_lock); - -struct pause_cpu_state { - int ref_count; -}; - -static DEFINE_PER_CPU(struct pause_cpu_state, pause_state); - -/* increment/decrement ref count for cpus in pause/resume mask */ -static void update_ref_counts(struct cpumask *cpus, bool pause) -{ - int cpu; - struct pause_cpu_state *pause_cpu_state; - - for_each_cpu(cpu, cpus) { - pause_cpu_state = per_cpu_ptr(&pause_state, cpu); - if (pause) - pause_cpu_state->ref_count++; - else { - WARN_ON_ONCE(pause_cpu_state->ref_count == 0); - pause_cpu_state->ref_count--; - } - } -} - -static void update_pause_resume_cpus(struct cpumask *cpus) -{ - int cpu; - struct pause_cpu_state *pause_cpu_state; - - for_each_cpu(cpu, cpus) { - pause_cpu_state = per_cpu_ptr(&pause_state, cpu); - if (pause_cpu_state->ref_count) - cpumask_clear_cpu(cpu, cpus); - } -} - -/* cpus will be modified */ -int walt_pause_cpus(struct cpumask *cpus) -{ - int ret = 0; - cpumask_t requested_cpus; - - mutex_lock(&pause_lock); - - cpumask_copy(&requested_cpus, cpus); - update_pause_resume_cpus(cpus); - - /* - * Add ref counts for all cpus in mask, but - * only actually pause online CPUs - */ - cpumask_and(cpus, cpus, cpu_online_mask); - if (cpumask_empty(cpus)) { - update_ref_counts(&requested_cpus, true); - goto unlock; - } - - // ret = pause_cpus(cpus); - if (ret < 0) - pr_debug("pause_cpus failure ret=%d cpus=%*pbl\n", ret, - cpumask_pr_args(&requested_cpus)); - else - update_ref_counts(&requested_cpus, true); -unlock: - mutex_unlock(&pause_lock); - - return ret; -} -EXPORT_SYMBOL(walt_pause_cpus); - -/* cpus will be modified */ -int walt_resume_cpus(struct cpumask *cpus) -{ - int ret = 0; - cpumask_t requested_cpus; - - mutex_lock(&pause_lock); - cpumask_copy(&requested_cpus, cpus); - update_ref_counts(&requested_cpus, false); - update_pause_resume_cpus(cpus); - - /* only actually resume online CPUs */ - cpumask_and(cpus, cpus, cpu_online_mask); - if (cpumask_empty(cpus)) - goto unlock; - - // ret = resume_cpus(cpus); - if (ret < 0) { - pr_debug("resume_cpus failure ret=%d cpus=%*pbl\n", ret, - cpumask_pr_args(&requested_cpus)); - /* restore/increment ref counts in case of error */ - update_ref_counts(&requested_cpus, true); - } -unlock: - mutex_unlock(&pause_lock); - - return ret; -} -EXPORT_SYMBOL(walt_resume_cpus); - -struct work_struct walt_pause_online_work; - -/* - * With refcounting and online/offline operations of the CPU - * a recent and accurate value for the requested CPUs versus - * ref-counted CPUs, must be made. - * - * When a CPU is onlined, this chain of events gets out of order. - * The online workfn can be entered at the same time as the - * walt_resume. If both are resuming the same set of CPUs - * the call to walt_pause will decrement ref-counts and think that - * the CPU is unpaused. If the workfn has already found all the - * ref-counts (and they were still set) it will re-pause - * the CPUs thinking that is what the client intended. This - * leads to a conflict, because the client software is no longer - * tracking these CPUs, and the state doesn't match what the client - * intended. - * - * This case needs protection to maintain a valid state - * of the device (where ref-counts == # of pause requests) - * Use a mutex such that the values read at the start of walt_pause, - * walt_resume, or walt_pause_online_workfn remain valid until the - * operation is complete. A mutex must be used because pause_cpus - * (and resume_cpus) cannot be called with a spinlock held, and - * the operation is not complete - * until those routines return. - */ -static void walt_pause_online_workfn(struct work_struct *work) -{ - struct pause_cpu_state *pause_cpu_state; - cpumask_t re_pause_cpus; - int cpu, ret = 0; - - mutex_lock(&pause_lock); - - cpumask_clear(&re_pause_cpus); - - /* search and test all online cpus */ - for_each_online_cpu(cpu) { - pause_cpu_state = per_cpu_ptr(&pause_state, cpu); - if (pause_cpu_state->ref_count) - cpumask_set_cpu(cpu, &re_pause_cpus); - } - - if (cpumask_empty(&re_pause_cpus)) - goto unlock; - - //ret = pause_cpus(&re_pause_cpus); - -unlock: - mutex_unlock(&pause_lock); - - if (ret < 0) - printk_deferred("pause_cpus during online failure ret=%d cpus=%*pb1\n", ret, - cpumask_pr_args(&re_pause_cpus)); -} - -/* do not perform online work in hotplug context */ -static int walt_pause_hp_online(unsigned int online_cpu) -{ - struct pause_cpu_state *pause_cpu_state; - - pause_cpu_state = per_cpu_ptr(&pause_state, online_cpu); - if (pause_cpu_state->ref_count) - schedule_work(&walt_pause_online_work); - return 0; -} - -void walt_pause_init(void) -{ - int ret; - - INIT_WORK(&walt_pause_online_work, walt_pause_online_workfn); - - ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "walt-pause/online", - walt_pause_hp_online, NULL); - - if (ret < 0) - printk_deferred("failure to register cpuhp online state ret=%d\n", ret); -} -#endif /* CONFIG_HOTPLUG_CPU */ From 7a7432458d15c2cd31c46feb2d134ab0909ed50b Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 15 Dec 2021 13:34:06 -0800 Subject: [PATCH 223/346] sched/walt/halt: update tracepoints for halt/start halt/start tracepoints need a little enhancement, to ensure a measurement can be made from the start of an operation. Also to debug RT task placements, add the lowest_mask cpu mask which contains the cpus available for the RT task placement algorithm to trace_sched_cpu_util, and update the trace to show the halted state. Change-Id: Idd5ab03d8ca5af21d42415dce790993c8c5df1d5 Signed-off-by: Stephen Dickey --- kernel/sched/walt/trace.h | 68 ++++++++++++++++++++++++++++++++--- kernel/sched/walt/walt_cfs.c | 2 +- kernel/sched/walt/walt_halt.c | 10 ++++++ kernel/sched/walt/walt_rt.c | 2 +- 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 554a27feec32..1f566daa8ab0 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -894,9 +894,9 @@ TRACE_EVENT(walt_lb_cpu_util, TRACE_EVENT(sched_cpu_util, - TP_PROTO(int cpu), + TP_PROTO(int cpu, struct cpumask *lowest_mask), - TP_ARGS(cpu), + TP_ARGS(cpu, lowest_mask), TP_STRUCT__entry( __field(unsigned int, cpu) @@ -910,10 +910,12 @@ TRACE_EVENT(sched_cpu_util, __field(u64, irqload) __field(int, online) __field(int, inactive) + __field(int, halted) __field(int, reserved) __field(int, high_irq_load) __field(unsigned int, nr_rtg_high_prio_tasks) __field(u64, prs_gprs) + __field(unsigned int, lowest_mask) ), TP_fast_assign( @@ -929,19 +931,25 @@ TRACE_EVENT(sched_cpu_util, __entry->irqload = sched_irqload(cpu); __entry->online = cpu_online(cpu); __entry->inactive = !cpu_active(cpu); + __entry->halted = cpu_halted(cpu); __entry->reserved = is_reserved(cpu); __entry->high_irq_load = sched_cpu_high_irqload(cpu); __entry->nr_rtg_high_prio_tasks = walt_nr_rtg_high_prio(cpu); __entry->prs_gprs = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; + if (!lowest_mask) + __entry->lowest_mask = 0; + else + __entry->lowest_mask = cpumask_bits(lowest_mask)[0]; ), - TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%lu capacity=%lu capacity_orig=%lu idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u prs_gprs=%llu", + TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%lu capacity=%lu capacity_orig=%lu idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, halted=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u prs_gprs=%llu lowest_mask=0x%x", __entry->cpu, __entry->nr_running, __entry->cpu_util, __entry->cpu_util_cum, __entry->capacity_curr, __entry->capacity, __entry->capacity_orig, __entry->idle_exit_latency, __entry->irqload, __entry->online, - __entry->inactive, __entry->reserved, __entry->high_irq_load, - __entry->nr_rtg_high_prio_tasks, __entry->prs_gprs) + __entry->inactive, __entry->halted, __entry->reserved, __entry->high_irq_load, + __entry->nr_rtg_high_prio_tasks, __entry->prs_gprs, + __entry->lowest_mask) ); TRACE_EVENT(sched_compute_energy, @@ -1297,6 +1305,56 @@ TRACE_EVENT(sched_cgroup_attach, __entry->grp_id, __entry->ret) ); + +TRACE_EVENT(halt_cpus_start, + TP_PROTO(struct cpumask *cpus, unsigned char halt), + + TP_ARGS(cpus, halt), + + TP_STRUCT__entry( + __field(unsigned int, cpus) + __field(unsigned int, halted_cpus) + __field(unsigned char, halt) + ), + + TP_fast_assign( + __entry->cpus = cpumask_bits(cpus)[0]; + __entry->halted_cpus = cpumask_bits(cpu_halt_mask)[0]; + __entry->halt = halt; + ), + + TP_printk("req_cpus=0x%x halt_cpus=0x%x halt=%d", + __entry->cpus, __entry->halted_cpus, __entry->halt) + +); + +TRACE_EVENT(halt_cpus, + TP_PROTO(struct cpumask *cpus, u64 start_time, unsigned char halt, int err), + + TP_ARGS(cpus, start_time, halt, err), + + TP_STRUCT__entry( + __field(unsigned int, cpus) + __field(unsigned int, halted_cpus) + __field(unsigned int, time) + __field(unsigned char, halt) + __field(unsigned char, success) + ), + + TP_fast_assign( + __entry->cpus = cpumask_bits(cpus)[0]; + __entry->halted_cpus = cpumask_bits(cpu_halt_mask)[0]; + __entry->time = div64_u64(sched_clock() - start_time, 1000); + __entry->halt = halt; + __entry->success = ((err >= 0)?1:0); + ), + + TP_printk("req_cpus=0x%x halt_cpus=0x%x time=%u us halt=%d success=%d", + __entry->cpus, __entry->halted_cpus, + __entry->time, __entry->halt, __entry->success) + +); + #endif /* _TRACE_WALT_H */ #undef TRACE_INCLUDE_PATH diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 2bfc05833d59..61e63238e1e6 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -295,7 +295,7 @@ static void walt_find_best_target(struct sched_domain *sd, unsigned int idle_exit_latency = UINT_MAX; struct walt_rq *wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; - trace_sched_cpu_util(i); + trace_sched_cpu_util(i, NULL); /* record the prss as we visit cpus in a cluster */ fbt_env->prs[i] = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 04b969f036ff..cf770978f3c1 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -193,6 +193,9 @@ static int halt_cpus(struct cpumask *cpus) { int cpu; int ret = 0; + u64 start_time = sched_clock(); + + trace_halt_cpus_start(cpus, 1); for_each_cpu(cpu, cpus) { @@ -212,6 +215,8 @@ static int halt_cpus(struct cpumask *cpus) } } + trace_halt_cpus(cpus, start_time, 1, ret); + return ret; } @@ -221,12 +226,17 @@ static int halt_cpus(struct cpumask *cpus) */ static int start_cpus(struct cpumask *cpus) { + u64 start_time = sched_clock(); + + trace_halt_cpus_start(cpus, 0); if (!cpumask_empty(cpus)) { /* remove the cpus from the halt mask */ cpumask_andnot(cpu_halt_mask, cpu_halt_mask, cpus); } + trace_halt_cpus(cpus, start_time, 0, 0); + return 0; } diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index a26e6c278c9e..386266fed844 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -36,7 +36,7 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task for_each_cpu_and(cpu, lowest_mask, &cpu_array[order_index][cluster]) { bool lt; - trace_sched_cpu_util(cpu); + trace_sched_cpu_util(cpu, lowest_mask); if (!cpu_active(cpu)) continue; From f6b8e4e4149a7ba1a7127d8f1d1fed6f23565c2a Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 17 Dec 2021 16:34:00 -0800 Subject: [PATCH 224/346] sched/walt/core_ctl: minimum changes for halt Make the minimum change for core_ctl to work properly with halt instead of pause. Change-Id: I7d2a2b50daaffc96846c86b8fd9dedf7f604b0b6 Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index a6f86e58bbcd..7d48cb0ec4bf 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -323,7 +323,7 @@ static ssize_t show_global_state(const struct cluster_data *state, char *buf) cpu_online(c->cpu)); count += scnprintf(buf + count, PAGE_SIZE - count, "\tPaused: %u\n", - !cpu_active(c->cpu)); + cpu_halted(c->cpu)); count += scnprintf(buf + count, PAGE_SIZE - count, "\tFirst CPU: %u\n", cluster->first_cpu); @@ -766,13 +766,13 @@ static unsigned int get_active_cpu_count(const struct cluster_data *cluster) { cpumask_t cpus; - cpumask_and(&cpus, &cluster->cpu_mask, cpu_active_mask); + cpumask_andnot(&cpus, &cluster->cpu_mask, cpu_halt_mask); return cpumask_weight(&cpus); } static bool is_active(const struct cpu_data *state) { - return cpu_online(state->cpu) && cpu_active(state->cpu); + return cpu_online(state->cpu) && !cpu_halted(state->cpu); } static bool adjustment_possible(const struct cluster_data *cluster, @@ -1101,7 +1101,7 @@ static int __try_to_resume(struct cluster_data *cluster, unsigned int need, if (!cpumask_test_cpu(c->cpu, &cpus_paused_by_us)) continue; - if ((cpu_online(c->cpu) && cpu_active(c->cpu)) || + if ((cpu_online(c->cpu) && !cpu_halted(c->cpu)) || (!force && c->not_preferred)) continue; if (active_cpus + nr_pending == need) @@ -1159,7 +1159,7 @@ static void core_ctl_pause_cpus(struct cpumask *cpus_to_pause) cpumask_copy(&saved_cpus, cpus_to_pause); if (cpumask_any(cpus_to_pause) < nr_cpu_ids) { - if (walt_pause_cpus(cpus_to_pause) < 0) + if (walt_halt_cpus(cpus_to_pause) < 0) pr_debug("core_ctl pause operation failed cpus=%*pbl paused_by_us=%*pbl\n", cpumask_pr_args(cpus_to_pause), cpumask_pr_args(&cpus_paused_by_us)); @@ -1188,7 +1188,7 @@ static void core_ctl_resume_cpus(struct cpumask *cpus_to_unpause) cpumask_copy(&saved_cpus, cpus_to_unpause); if (cpumask_any(cpus_to_unpause) < nr_cpu_ids) { - if (walt_resume_cpus(cpus_to_unpause) < 0) + if (walt_start_cpus(cpus_to_unpause) < 0) pr_debug("core_ctl resume operation failed cpus=%*pbl paused_by_us=%*pbl\n", cpumask_pr_args(cpus_to_unpause), cpumask_pr_args(&cpus_paused_by_us)); From 79e3eab666285694656617d3a3073e35fa94683b Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:27:05 -0800 Subject: [PATCH 225/346] sched/walt: avoid halted cpus in newidle balance If a halted CPU goes idle, don't try to pull tasks into the CPU. Change-Id: I50383f736ae03992b8bb43f5aa12cc2803068f6f Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_lb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 7d09bb67f0c4..5f0bcf76da63 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -693,6 +693,9 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (!cpu_active(this_cpu)) return; + if (cpu_halted(this_cpu)) + return; + rq_unpin_lock(this_rq, rf); /* From 7a3fe1d52e04387f9286eb17574c98acca10da1e Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 15 Dec 2021 13:39:38 -0800 Subject: [PATCH 226/346] sched/walt: avoid halted cpus for prev cpu fastpath Currently walt_find_best_target can return a halted cpu. Prevent the PREV_CPU_FASTPATH for cpus that are currently halted, and reject them in general from being selected. Change-Id: Iced61bf9b1355703c6403d1dc9f90d6e64aefdc6 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_cfs.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 61e63238e1e6..798506277076 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -270,7 +270,9 @@ static void walt_find_best_target(struct sched_domain *sd, asym_cap_siblings(prev_cpu, start_cpu)) && cpu_active(prev_cpu) && cpu_online(prev_cpu) && available_idle_cpu(prev_cpu) && - cpumask_test_cpu(prev_cpu, p->cpus_ptr)) { + cpumask_test_cpu(prev_cpu, p->cpus_ptr) && + !cpu_halted(prev_cpu)) { + fbt_env->fastpath = PREV_CPU_FASTPATH; cpumask_set_cpu(prev_cpu, candidates); goto out; @@ -302,6 +304,9 @@ static void walt_find_best_target(struct sched_domain *sd, if (!cpu_active(i)) continue; + if (cpu_halted(i)) + continue; + if (active_candidate == -1) active_candidate = i; From ba4ec7d4861e91229c53331aa8c28cbda24ad63a Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:27:10 -0800 Subject: [PATCH 227/346] sched/walt/rt: check halted flag for rt task wakeups The halted flag must be checked when performing a wakeup of an rt task, to prevent tasks from running on halted CPUs. Change-Id: Ib4a5cff48611abb0d1f71fc10be45439d2b7dd64 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_rt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 386266fed844..07afcb39fdc6 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -41,6 +41,9 @@ static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task if (!cpu_active(cpu)) continue; + if (cpu_halted(cpu)) + continue; + if (sched_cpu_high_irqload(cpu)) continue; @@ -162,7 +165,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c /* * Respect the sync flag as long as the task can run on this CPU. */ - if (sysctl_sched_sync_hint_enable && cpu_active(this_cpu) && + if (sysctl_sched_sync_hint_enable && cpu_active(this_cpu) && !cpu_halted(this_cpu) && cpumask_test_cpu(this_cpu, task->cpus_ptr) && walt_should_honor_rt_sync(this_cpu_rq, task, sync)) { *new_cpu = this_cpu; From 80676560d271024eb9d016c8d605ebc2d3ebf8d9 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:27:07 -0800 Subject: [PATCH 228/346] sched/walt: do not allow migration to halted CPU rebalance_domains will call load_balance, and ultimately will call detach_tasks, walt_migrated_queued_task, and set_task_cpu, choosing a cpu that is halted. can_migrate_task will be used to validate whether a particular migration is allowed to happen. Reject migrations for tasks that are being migrated to a halted cpu. Change-Id: I0b2f5d3857a2b1db9a2fa5017d5238f143a85ed9 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_lb.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 5f0bcf76da63..7128a9ea26a7 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -250,6 +250,10 @@ static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, if (wrq->push_task == p) return false; + /* Don't detach task if dest cpu is halted */ + if (cpu_halted(dst_cpu)) + return false; + return true; } From a91f19065149172cb166ea61f9af6cd2b6365bfd Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:27:18 -0800 Subject: [PATCH 229/346] sched/walt: limited the dest_cpu with set_cpus_allowed_ptr_locked When set_cpus_allowed_ptr_locked runs, it chooses a new destination cpu for the task based upon the new mask for the task. Unfortunately this means that halted cpus can be included. Register for a trace hook such that dest_cpu can be recomputed, and a dest_cpu that is not halted, chosen. Change-Id: I0ab072eb395bece8c0ab896d7a1008f9cf55f468 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index b479186d4f42..3b2aed8a82d4 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4208,6 +4208,23 @@ static void dump_throttled_rt_tasks(void *unused, int cpu, u64 clock, BUG_ON(sysctl_sched_bug_on_rt_throttle); } +static void android_rvh_set_cpus_allowed_ptr_locked(void *unused, + const struct cpumask *cpu_valid_mask, + const struct cpumask *new_mask, + unsigned int *dest_cpu) +{ + cpumask_t allowed_cpus; + + if (unlikely(walt_disabled)) + return; + + if (cpu_halted(*dest_cpu)) { + /* remove halted cpus from the valid mask, and store locally */ + cpumask_andnot(&allowed_cpus, cpu_valid_mask, cpu_halt_mask); + *dest_cpu = cpumask_any_and_distribute(&allowed_cpus, new_mask); + } +} + static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4236,6 +4253,8 @@ static void register_walt_hooks(void) register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); register_trace_android_vh_dump_throttled_rt_tasks(dump_throttled_rt_tasks, NULL); + register_trace_android_rvh_set_cpus_allowed_ptr_locked( + android_rvh_set_cpus_allowed_ptr_locked, NULL); } atomic64_t walt_irq_work_lastq_ws; From dcbd6c724659dee354070eb48ffcf3e28351ef12 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:27:21 -0800 Subject: [PATCH 230/346] sched/walt: limit the cpu selected by rto_next_cpu rto_next_cpu can select any rd->rto_mask cpu, which may be halted. restrict rto_next_cpu to unhalted CPUs. Change-Id: I1c41c4c72b630d00cca080ed73609f6fdfb7143d Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 3b2aed8a82d4..de375ca6f537 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4225,6 +4225,23 @@ static void android_rvh_set_cpus_allowed_ptr_locked(void *unused, } } +static void android_rvh_rto_next_cpu(void *unused, + int rto_cpu, + struct cpumask *rto_mask, + int *cpu) +{ + cpumask_t allowed_cpus; + + if (unlikely(walt_disabled)) + return; + + if (cpu_halted(*cpu)) { + /* remove halted cpus from the valid mask, and store locally */ + cpumask_andnot(&allowed_cpus, rto_mask, cpu_halt_mask); + *cpu = cpumask_next(rto_cpu, &allowed_cpus); + } +} + static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4255,6 +4272,7 @@ static void register_walt_hooks(void) register_trace_android_vh_dump_throttled_rt_tasks(dump_throttled_rt_tasks, NULL); register_trace_android_rvh_set_cpus_allowed_ptr_locked( android_rvh_set_cpus_allowed_ptr_locked, NULL); + register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL); } atomic64_t walt_irq_work_lastq_ws; From 612b4f86c144527ca297bed5a505d344914dda0b Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:27:23 -0800 Subject: [PATCH 231/346] sched/walt: avoid halted cpus for rt in find_lowest_rq select_task_rq_rt() and push_rt_task() need to select a cpu, and ultimately attempt to get a good match on a cpu through find_lowest_rq. Walt hooks into this routine to make an adjustment such that the best energy aware cpu is found. It is possible that walt_rt_energy_aware_wake_cpu will be unable to find a non-halted cpu. find_lowest_rq at that point will use the lowest_mask, initialized from the task cpus_ptr, to find a CPU. At this point, find_lowest_rq might return a halted CPU. Instead, in the case where walt cannot find a best cpu to choose, force the lowest_mask to no longer include halted cpus, which will push find_lowest_rq to no longer choose a halted cpu. In the unlikely event that find_lowest_rq still fails to find a cpu it will pick anything from the lowest_mask. In the unlikely event that lowest_mask is empty after removing all of the halted cpus, find_lowest_rq will fail and the cpu chosen will be unconstrained by halt. Change-Id: I398bc804eb474396f6f06e2c3d4308fea71dafe8 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_rt.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 07afcb39fdc6..ec299ce106ef 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -10,8 +10,8 @@ static DEFINE_PER_CPU(cpumask_var_t, walt_local_cpu_mask); -static void walt_rt_energy_aware_wake_cpu(void *unused, struct task_struct *task, - struct cpumask *lowest_mask, int ret, int *best_cpu) +static void walt_rt_energy_aware_wake_cpu(struct task_struct *task, struct cpumask *lowest_mask, + int ret, int *best_cpu) { int cpu; unsigned long util, best_cpu_util = ULONG_MAX; @@ -205,7 +205,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, task, lowest_mask, walt_rt_task_fits_capacity); - walt_rt_energy_aware_wake_cpu(NULL, task, lowest_mask, ret, &target); + walt_rt_energy_aware_wake_cpu(task, lowest_mask, ret, &target); /* * If cpu is non-preemptible, prefer remote cpu @@ -220,6 +220,22 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c rcu_read_unlock(); } + +static void walt_rt_find_lowest_rq(void *unused, struct task_struct *task, + struct cpumask *lowest_mask, int ret, int *best_cpu) + +{ + walt_rt_energy_aware_wake_cpu(task, lowest_mask, ret, best_cpu); + + /* + * Walt was not able to find a non-halted best cpu. Ensure that + * find_lowest_rq doesn't use a halted cpu going forward, but + * does a best effort itself to find a good CPU. + */ + if (*best_cpu == -1) + cpumask_andnot(lowest_mask, lowest_mask, cpu_halt_mask); +} + void walt_rt_init(void) { unsigned int i; @@ -233,5 +249,5 @@ void walt_rt_init(void) } register_trace_android_rvh_select_task_rq_rt(walt_select_task_rq_rt, NULL); - register_trace_android_rvh_find_lowest_rq(walt_rt_energy_aware_wake_cpu, NULL); + register_trace_android_rvh_find_lowest_rq(walt_rt_find_lowest_rq, NULL); } From 324d02c0c5fa50a420b165e43b74c0be34524957 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 15 Dec 2021 13:27:40 -0800 Subject: [PATCH 232/346] sched/walt: avoid halted cpus in walt_find_energy_efficient_cpu select_task_rq_fair invokes the select_task_rq_fair hook, which ultimately invokes walt_find_energy_aware_cpu for cfs tasks. This can return a cpu that is halted. Update the one remaining case that can cause feec to return a halted cpu. Change-Id: Ic97345764ad8be1060868414869dc0e455f4bc54 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_cfs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 798506277076..f90befa107c6 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -788,8 +788,8 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (sync && (need_idle || (is_rtg && curr_is_rtg))) sync = 0; - if (sysctl_sched_sync_hint_enable && sync - && bias_to_this_cpu(p, cpu, start_cpu)) { + if (sysctl_sched_sync_hint_enable && sync && + bias_to_this_cpu(p, cpu, start_cpu) && !cpu_halted(cpu)) { best_energy_cpu = cpu; fbt_env.fastpath = SYNC_WAKEUP; goto done; From 9b29827decf04445d339b049c861dbc0d4e30784 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 30 Sep 2021 12:22:17 -0700 Subject: [PATCH 233/346] sched/walt: kick Gold if farthest cluster needs help This change is for general scheduler improvement. Change-Id: I997827d143a370e49e43ecba8494ee8e69652b57 Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_lb.c | 168 ++++++++++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 27 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 7128a9ea26a7..85c9558f7e01 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -348,7 +348,47 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) return 1; /* we pulled 1 task */ } -static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *src_mask) +#define SMALL_TASK_THRESHOLD 102 +/* + * find_first_idle_if_others_are_busy + * + * Get an idle cpu in the src_mask, iif the other cpus are busy + * with larger tasks or more than one task. + * + * Returns -1 if there are no idle cpus in the mask, or if there + * is a CPU that is about to be come newly idle. Returns + * if there is an idle cpu in a cluster that's not about to do + * newly idle load balancing. + */ +static int find_first_idle_if_others_are_busy(const cpumask_t *src_mask) +{ + int i, first_idle = -1; + + for_each_cpu(i, src_mask) { + if (available_idle_cpu(i)) + first_idle = i; + + if (cpu_util(i) < SMALL_TASK_THRESHOLD && cpu_rq(i)->nr_running == 1) { + /* + * there was one CPU in the mask that was almost idle, but not + * quite. when it becomes idle, it will do newidle load-balance, + * and start off with it's own cluster. So no reason to kick + * anything in this cluster. i.e. don't perform a wasted kick. + */ + first_idle = -1; + + /* return -1, a cpu in this cluster is about to do + * newly idle load balancing + */ + break; + } + } + + return first_idle; +} + +static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *src_mask, + int *has_misfit) { int i; int busiest_cpu = -1; @@ -373,8 +413,8 @@ static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *sr return busiest_cpu; } -#define SMALL_TASK_THRESHOLD 102 -static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src_mask) +static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t *src_mask, + int *has_misfit) { int i; int busiest_cpu = -1; @@ -433,7 +473,8 @@ static int walt_lb_find_busiest_higher_cap_cpu(int dst_cpu, const cpumask_t *src return busiest_cpu; } -static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_mask) +static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t *src_mask, + int *has_misfit) { int i; int busiest_cpu = -1; @@ -494,23 +535,26 @@ static int walt_lb_find_busiest_lower_cap_cpu(int dst_cpu, const cpumask_t *src_ busiest_cpu = -1; } + if (busy_nr_big_tasks && busiest_cpu != -1) + *has_misfit = true; + return busiest_cpu; } -static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask) +static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int *has_misfit) { int fsrc_cpu = cpumask_first(src_mask); int busiest_cpu; if (capacity_orig_of(dst_cpu) == capacity_orig_of(fsrc_cpu)) busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, - src_mask); + src_mask, has_misfit); else if (capacity_orig_of(dst_cpu) > capacity_orig_of(fsrc_cpu)) - busiest_cpu = walt_lb_find_busiest_lower_cap_cpu(dst_cpu, - src_mask); + busiest_cpu = walt_lb_find_busiest_from_lower_cap_cpu(dst_cpu, + src_mask, has_misfit); else - busiest_cpu = walt_lb_find_busiest_higher_cap_cpu(dst_cpu, - src_mask); + busiest_cpu = walt_lb_find_busiest_from_higher_cap_cpu(dst_cpu, + src_mask, has_misfit); return busiest_cpu; } @@ -660,6 +704,29 @@ static bool should_help_min_cap(int this_cpu) return false; } +static void kick_first_idle(int first_idle) +{ + unsigned int flags = NOHZ_KICK_MASK; + + if (first_idle == -1) + return; + + /* + * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets + * the first flag owns it; cleared by nohz_csd_func(). + */ + flags = atomic_fetch_or(flags, nohz_flags(first_idle)); + if (flags & NOHZ_KICK_MASK) + return; + + /* + * This way we generate an IPI on the target CPU which + * is idle. And the softirq performing nohz idle load balance + * will be run before returning from the IPI. + */ + smp_call_function_single_async(first_idle, &cpu_rq(first_idle)->nohz_csd); +} + /* similar to sysctl_sched_migration_cost */ #define NEWIDLE_BALANCE_THRESHOLD 500000 static void walt_newidle_balance(void *unused, struct rq *this_rq, @@ -669,10 +736,11 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, int this_cpu = this_rq->cpu; struct walt_rq *wrq = (struct walt_rq *) this_rq->android_vendor_data1; int order_index; - int cluster = 0; int busy_cpu = -1; bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); bool help_min_cap = false; + int first_idle; + int has_misfit = 0; if (unlikely(walt_disabled)) return; @@ -725,27 +793,72 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, * can be queued remotely, so keep a check on nr_running * and bail out. */ - do { - busy_cpu = walt_lb_find_busiest_cpu(this_cpu, - &cpu_array[order_index][cluster]); + if (order_index == 0) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; - if (sysctl_sched_skip_sp_newly_idle_lb) { - if (busy_cpu == -1 && - ((order_index == 0 && cluster > 0) || - (order_index == 2 && cluster > 0))) - break; + if (enough_idle) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; } - /* we got the busy/src cpu here. */ - if (busy_cpu != -1 || this_rq->nr_running > 0) - break; + /* help the farthest cluster indirectly if it needs help */ + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], + &has_misfit); + if (busy_cpu != -1) { + first_idle = + find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); + kick_first_idle(first_idle); + } + } else if (order_index == 2) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; - if (!enough_idle && !help_min_cap) - break; - } while (++cluster < num_sched_clusters); + /* help gold only if prime has had enough idle or gold has a misfit */ + has_misfit = false; + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], + &has_misfit); + if (busy_cpu != -1 && (enough_idle || has_misfit)) + goto found_busy_cpu; + /* help the farthest cluster indirectly if it needs help */ + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], + &has_misfit); + if (busy_cpu != -1) { + first_idle = + find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); + kick_first_idle(first_idle); + } + } else { + busy_cpu = + walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + + if (enough_idle) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + } + + has_misfit = false; + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], + &has_misfit); + if (busy_cpu != -1 && (enough_idle || has_misfit)) + goto found_busy_cpu; + } + goto unlock; + +found_busy_cpu: /* sanity checks before attempting the pull */ - if (busy_cpu == -1 || this_rq->nr_running > 0 || (busy_cpu == this_cpu)) + if (this_rq->nr_running > 0 || (busy_cpu == this_cpu)) goto unlock; *pulled_task = walt_lb_pull_tasks(this_cpu, busy_cpu); @@ -779,6 +892,7 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, int fsrc_cpu = group_first_cpu(group); int busiest_cpu = -1; struct cpumask src_mask; + int has_misfit; if (unlikely(walt_disabled)) return; @@ -805,7 +919,7 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, * remain same. */ cpumask_and(&src_mask, sched_group_span(group), env_cpus); - busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask); + busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask, &has_misfit); done: if (busiest_cpu != -1) *busiest = cpu_rq(busiest_cpu); From d0718d29fb337b51e61320705ae6c27c3864c320 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 12 Nov 2021 16:25:44 -0800 Subject: [PATCH 234/346] sched/walt: move trace_sched_compute_energy tracepoint In walt_find_energy_efficient_cpu, the trace for compute energy was being called just prior to an update to best energy cpu. The goal of the trace is to print the selected state of the routine, not an intermediate state. Change-Id: I8132b42e65b9b3bada9316c0ba738fa3baac9f49 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_cfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index f90befa107c6..3ab8b2a91ec0 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -873,9 +873,6 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, NULL); } - trace_sched_compute_energy(p, cpu, cur_energy, - prev_energy, best_energy, best_energy_cpu, &output); - if (cur_energy < best_energy) { best_energy = cur_energy; best_energy_cpu = cpu; @@ -886,6 +883,9 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, best_energy_cpu = cpu; } } + + trace_sched_compute_energy(p, cpu, cur_energy, + prev_energy, best_energy, best_energy_cpu, &output); } /* From 327b6ddd5ad7f1343786c4cde37af9cc9166c37d Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 23 Sep 2021 13:04:39 -0700 Subject: [PATCH 235/346] sched/walt: make cmt aware of boosted/colocated task This change is for general scheduler improvement. Change-Id: I6235ca8fce3d0c2ab3bcab1ecee0d97c6d0a9942 Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_lb.c | 102 +++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 35 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 85c9558f7e01..e0bfd5bab9e3 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -231,11 +231,16 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) } static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, - bool to_lower) + bool to_lower, bool force) { struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + /* Don't detach task if it is under active migration */ + if (wrq->push_task == p) + return false; + + if (to_lower) { if (wts->iowaited) return false; @@ -244,12 +249,12 @@ static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, return false; if (walt_pipeline_low_latency_task(p)) return false; + if (!force && walt_get_rtg_status(p)) + return false; + if (!force && !task_fits_max(p, dst_cpu)) + return false; } - /* Don't detach task if it is under active migration */ - if (wrq->push_task == p) - return false; - /* Don't detach task if dest cpu is halted */ if (cpu_halted(dst_cpu)) return false; @@ -289,55 +294,82 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(src_cpu); raw_spin_lock_irqsave(&src_rq->__lock, flags); + list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) continue; - if (!_walt_can_migrate_task(p, dst_cpu, to_lower)) + if (task_running(src_rq, p)) + continue; + + if (!_walt_can_migrate_task(p, dst_cpu, to_lower, false)) + continue; + + walt_detach_task(p, src_rq, dst_rq); + pulled_task = p; + goto unlock; + } + + list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { + + if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) + continue; + + if (task_running(src_rq, p)) + continue; + + if (!_walt_can_migrate_task(p, dst_cpu, to_lower, true)) + continue; + + walt_detach_task(p, src_rq, dst_rq); + pulled_task = p; + goto unlock; + } + + list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { + + if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) continue; if (task_running(src_rq, p)) { - if (need_active_lb(p, dst_cpu, src_cpu)) { + bool success; active_balance = true; - break; + src_rq->active_balance = 1; + src_rq->push_cpu = dst_cpu; + get_task_struct(p); + wrq->push_task = p; + mark_reserved(dst_cpu); + + /* lock must be dropped before waking the stopper */ + raw_spin_unlock_irqrestore(&src_rq->__lock, flags); + + /* + * Using our custom active load balance callback so that + * the push_task is really pulled onto this CPU. + */ + wts = (struct walt_task_struct *) p->android_vendor_data1; + trace_walt_active_load_balance(p, src_cpu, dst_cpu, wts); + success = stop_one_cpu_nowait(src_cpu, + stop_walt_lb_active_migration, + src_rq, &src_rq->active_balance_work); + if (!success) + clear_reserved(dst_cpu); + + return 0; /* we did not pull any task here */ } continue; } walt_detach_task(p, src_rq, dst_rq); pulled_task = p; - break; - } - - if (active_balance) { - src_rq->active_balance = 1; - src_rq->push_cpu = dst_cpu; - get_task_struct(p); - wrq->push_task = p; - mark_reserved(dst_cpu); + goto unlock; } +unlock: /* lock must be dropped before waking the stopper */ raw_spin_unlock_irqrestore(&src_rq->__lock, flags); - /* - * Using our custom active load balance callback so that - * the push_task is really pulled onto this CPU. - */ - if (active_balance) { - bool success; - - wts = (struct walt_task_struct *) p->android_vendor_data1; - trace_walt_active_load_balance(p, src_cpu, dst_cpu, wts); - success = stop_one_cpu_nowait(src_cpu, stop_walt_lb_active_migration, - src_rq, &src_rq->active_balance_work); - if (!success) - clear_reserved(dst_cpu); - - return 0; /* we did not pull any task here */ - } - if (!pulled_task) return 0; @@ -984,7 +1016,7 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, return; to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(task_cpu(p)); - if (_walt_can_migrate_task(p, dst_cpu, to_lower)) + if (_walt_can_migrate_task(p, dst_cpu, to_lower, true)) return; *can_migrate = 0; From 4735599f5108fa574b577084b39d0ac05c4abdf2 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 18 Nov 2021 11:56:55 -0800 Subject: [PATCH 236/346] sched: Remove references to min/max capacity cpus Adapt the min/max capacity cpu helper functions to use cluster IDs to guide decisions rather than capacities. Change-Id: Ic204f286c8eb95742ab64d366ce30d5ee780c340 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/core_ctl.c | 2 +- kernel/sched/walt/cpufreq_walt.c | 2 +- kernel/sched/walt/walt.c | 19 ++++++++++--------- kernel/sched/walt/walt.h | 29 ++++++++++++++++------------- kernel/sched/walt/walt_cfs.c | 3 +-- kernel/sched/walt/walt_lb.c | 8 ++++---- 6 files changed, 33 insertions(+), 30 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 7d48cb0ec4bf..9d36748fccf8 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -784,7 +784,7 @@ static bool adjustment_possible(const struct cluster_data *cluster, static bool need_all_cpus(const struct cluster_data *cluster) { - return (is_min_capacity_cpu(cluster->first_cpu) && + return (is_min_cluster_cpu(cluster->first_cpu) && sched_ravg_window < DEFAULT_SCHED_RAVG_WINDOW); } diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 377f0602c91e..53125b982600 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -306,7 +306,7 @@ static inline unsigned long target_util(struct waltgov_policy *wg_policy, util = freq_to_util(wg_policy, freq); - if (wg_policy->max == min_max_possible_capacity && + if (is_min_cluster_cpu(wg_policy->policy->cpu) && util >= wg_policy->tunables->target_load_thresh) util = mult_frac(util, 94, 100); else diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index de375ca6f537..8c0b909da209 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -64,8 +64,8 @@ unsigned int walt_rotation_enabled; cpumask_t asym_cap_sibling_cpus = CPU_MASK_NONE; unsigned int __read_mostly sched_ravg_window = 20000000; -unsigned int min_max_possible_capacity = 1024; -unsigned int max_possible_capacity = 1024; /* max(rq->max_possible_capacity) */ +int min_possible_cluster_id; +int max_possible_cluster_id; /* Initial task load. Newly created tasks are assigned this load. */ unsigned int __read_mostly sched_init_task_load_windows; /* @@ -271,8 +271,7 @@ void walt_dump(void) sched_ravg_window_change_time); for_each_online_cpu(cpu) walt_rq_dump(cpu); - SCHED_PRINT(max_possible_capacity); - SCHED_PRINT(min_max_possible_capacity); + SCHED_PRINT(max_possible_cluster_id); printk_deferred("============ WALT RQ DUMP END ==============\n"); } @@ -2446,16 +2445,18 @@ static void update_all_clusters_stats(void) for_each_sched_cluster(cluster) { u64 mpc = arch_scale_cpu_capacity( cluster_first_cpu(cluster)); + int cluster_id = cluster->id; - if (mpc > highest_mpc) + if (mpc > highest_mpc) { highest_mpc = mpc; + max_possible_cluster_id = cluster_id; + } - if (mpc < lowest_mpc) + if (mpc < lowest_mpc) { lowest_mpc = mpc; + min_possible_cluster_id = cluster_id; + } } - - max_possible_capacity = highest_mpc; - min_max_possible_capacity = lowest_mpc; walt_update_group_thresholds(); } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 2d00aaee997f..79f7045e02ac 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -177,8 +177,8 @@ extern int core_ctl_init(void); extern atomic64_t walt_irq_work_lastq_ws; extern unsigned int __read_mostly sched_ravg_window; -extern unsigned int min_max_possible_capacity; -extern unsigned int max_possible_capacity; +extern int min_possible_cluster_id; +extern int max_possible_cluster_id; extern unsigned int __read_mostly sched_init_task_load_windows; extern unsigned int __read_mostly sched_load_granule; @@ -622,22 +622,26 @@ static inline int cluster_first_cpu(struct walt_sched_cluster *cluster) static inline bool hmp_capable(void) { - return max_possible_capacity != min_max_possible_capacity; + return max_possible_cluster_id != min_possible_cluster_id; } -static inline bool is_max_capacity_cpu(int cpu) +static inline bool is_max_cluster_cpu(int cpu) { - return arch_scale_cpu_capacity(cpu) == max_possible_capacity; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster->id == max_possible_cluster_id; } -static inline bool is_min_capacity_cpu(int cpu) +static inline bool is_min_cluster_cpu(int cpu) { - return arch_scale_cpu_capacity(cpu) == min_max_possible_capacity; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster->id == min_possible_cluster_id; } static inline bool is_min_capacity_cluster(struct walt_sched_cluster *cluster) { - return is_min_capacity_cpu(cluster_first_cpu(cluster)); + return cluster->id == min_possible_cluster_id; } /* @@ -696,7 +700,7 @@ static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu) if (is_suh_max() && rtg && rtg->id == DEFAULT_CGROUP_COLOC_ID && rtg->skip_min && wts->unfilter) - return is_min_capacity_cpu(cpu); + return is_min_cluster_cpu(cpu); return false; } @@ -731,13 +735,12 @@ static inline bool task_fits_capacity(struct task_struct *p, static inline bool task_fits_max(struct task_struct *p, int cpu) { unsigned long capacity = capacity_orig_of(cpu); - unsigned long max_capacity = max_possible_capacity; unsigned long task_boost = per_task_boost(p); - if (capacity == max_capacity) + if (is_max_cluster_cpu(cpu)) return true; - if (is_min_capacity_cpu(cpu)) { + if (is_min_cluster_cpu(cpu)) { if (task_boost_policy(p) == SCHED_BOOST_ON_BIG || task_boost > 0 || walt_uclamp_boosted(p) || @@ -891,7 +894,7 @@ void walt_lb_tick(struct rq *rq); extern __read_mostly unsigned int walt_scale_demand_divisor; #define scale_demand(d) ((d)/walt_scale_demand_divisor) -#define ASYMCAP_BOOST(cpu) (sysctl_sched_asymcap_boost && !is_min_capacity_cpu(cpu)) +#define ASYMCAP_BOOST(cpu) (sysctl_sched_asymcap_boost && !is_min_cluster_cpu(cpu)) void create_util_to_cost(void); struct compute_energy_output { diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 3ab8b2a91ec0..a93bc35cbb2a 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -73,9 +73,8 @@ bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) static inline bool task_demand_fits(struct task_struct *p, int cpu) { unsigned long capacity = capacity_orig_of(cpu); - unsigned long max_capacity = max_possible_capacity; - if (capacity == max_capacity) + if (is_max_cluster_cpu(cpu)) return true; return task_fits_capacity(p, capacity, cpu); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index e0bfd5bab9e3..7e361c89e9f7 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -143,7 +143,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) struct walt_lb_rotate_work *wr = NULL; struct walt_task_struct *wts; - if (!is_min_capacity_cpu(src_cpu)) + if (!is_min_cluster_cpu(src_cpu)) return; wc = walt_ktime_get_ns(); @@ -151,7 +151,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) for_each_possible_cpu(i) { struct rq *rq = cpu_rq(i); - if (!is_min_capacity_cpu(i)) + if (!is_min_cluster_cpu(i)) break; if (is_reserved(i)) @@ -174,7 +174,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) for_each_possible_cpu(i) { struct rq *rq = cpu_rq(i); - if (is_min_capacity_cpu(i)) + if (is_min_cluster_cpu(i)) continue; if (is_reserved(i)) @@ -723,7 +723,7 @@ static bool should_help_min_cap(int this_cpu) { int cpu; - if (!sysctl_sched_force_lb_enable || is_min_capacity_cpu(this_cpu)) + if (!sysctl_sched_force_lb_enable || is_min_cluster_cpu(this_cpu)) return false; for_each_cpu(cpu, &cpu_array[0][0]) { From 4a531492c97a3874c9a46df850f93045c16a8b91 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 6 Dec 2021 16:36:44 -0800 Subject: [PATCH 237/346] sched/walt: Increase the number of history samples The number history samples should be raised to a higher value so that an active history of 100ms is considered in accounting task demand. Raise the history size from 5 to 8 given the default windows size is 8ms. This will capture 64ms of history for 8ms windows, and 128ms of history for 16ms windows, granting a compromise to prevent further code bloat. Change-Id: I63ad5b2a20c9a32d3a65448a78d6aec98afe6ad5 Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 2 +- kernel/sched/walt/walt.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index c03a3f30caa4..e7813c280140 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -31,7 +31,7 @@ enum task_boost_type { }; #define WALT_NR_CPUS 8 -#define RAVG_HIST_SIZE_MAX 5 +#define RAVG_HIST_SIZE_MAX 8 #define NUM_BUSY_BUCKETS 10 struct walt_related_thread_group { diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 8c0b909da209..2b904b127ce0 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -154,7 +154,7 @@ static inline void release_rq_locks_irqrestore(const cpumask_t *cpus, static unsigned int walt_cpu_high_irqload; -static __read_mostly unsigned int sched_ravg_hist_size = 5; +static __read_mostly unsigned int sched_ravg_hist_size = RAVG_HIST_SIZE_MAX; static __read_mostly unsigned int sched_io_is_busy = 1; From 229e297c1a97261dd6dadd48893b40dcd79c9a67 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 17 Dec 2021 16:57:55 -0800 Subject: [PATCH 238/346] sched/walt/halt: cleaup issues post-review Minor cleanup including renaming of the drain functions and commenting the update_halt_cpus function and usage. Change-Id: I44502cde850af8fc6124e7b8ae014c562b21d0d9 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_halt.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index cf770978f3c1..4a8fc0472d44 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -61,9 +61,9 @@ void attach_tasks_core(struct list_head *tasks, struct rq *rq) * there's no concurrency possible, we hold the required locks anyway * because of lock validation efforts. * - * force: if false, the function will skip CPU pinned kthreads. + * The function will skip CPU pinned kthreads. */ -static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, bool force) +static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) { struct rq *rq = dead_rq; struct task_struct *next, *stop = rq->stop; @@ -110,7 +110,7 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, bool force) * kthread from the run-queue to continue. */ - if (!force && is_per_cpu_kthread(next)) { + if (is_per_cpu_kthread(next)) { detach_one_task_core(next, rq, &percpu_kthreads); num_pinned_kthreads += 1; continue; @@ -135,18 +135,11 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, bool force) * that case the above rq->__lock drop is a fail too. */ if (task_rq(next) != rq || !task_on_rq_queued(next)) { - /* - * In the !force case, there is a hole between - * rq_unlock() and rq_relock(), where another CPU might - * not observe an up to date cpu_active_mask and try to - * move tasks around. - */ - WARN_ON(force); raw_spin_unlock(&next->pi_lock); continue; } - /* Find suitable destination for @next, with force if needed. */ + /* Find suitable destination for @next */ dest_cpu = select_fallback_rq(dead_rq->cpu, next); rq = __migrate_task(rq, rf, next, dest_cpu); if (rq != dead_rq) { @@ -170,13 +163,13 @@ static int drain_rq_cpu_stop(void *data) struct rq_flags rf; rq_lock_irqsave(rq, &rf); - migrate_tasks(rq, &rf, false); + migrate_tasks(rq, &rf); rq_unlock_irqrestore(rq, &rf); return 0; } -static int sched_cpu_drain_rq(unsigned int cpu) +static int cpu_drain_rq(unsigned int cpu) { if (available_idle_cpu(cpu)) return 0; @@ -205,7 +198,7 @@ static int halt_cpus(struct cpumask *cpus) /* only drain online cpus */ if (cpu_online(cpu)) { /* drain the online CPU */ - ret = sched_cpu_drain_rq(cpu); + ret = cpu_drain_rq(cpu); } if (ret < 0) { @@ -257,6 +250,7 @@ static void update_ref_counts(struct cpumask *cpus, bool halt) } } +/* remove cpus that are already halted */ static void update_halt_cpus(struct cpumask *cpus) { int cpu; @@ -278,6 +272,8 @@ int walt_halt_cpus(struct cpumask *cpus) mutex_lock(&halt_lock); cpumask_copy(&requested_cpus, cpus); + + /* remove cpus that are already halted */ update_halt_cpus(cpus); if (cpumask_empty(cpus)) { @@ -314,6 +310,8 @@ int walt_start_cpus(struct cpumask *cpus) mutex_lock(&halt_lock); cpumask_copy(&requested_cpus, cpus); update_ref_counts(&requested_cpus, false); + + /* remove cpus that should still be halted, due to ref-counts */ update_halt_cpus(cpus); ret = start_cpus(cpus); From 9c30510226d158626f49b1ec7dbc484c872959d4 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 11 Jan 2022 15:19:44 -0800 Subject: [PATCH 239/346] sched/walt: catch tasks running on disallowed cpus If a task is enqueued on a CPU for which it is not allowed as dictated by the cpus_ptr of the task then report the issue. Change-Id: Ib049e9ae07be1830d9ed6166c47aa78aff7e1b12 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2b904b127ce0..59b49318212f 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3930,6 +3930,10 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st double_enqueue = true; } + if (!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) + WALT_BUG(p, "enqueueing on rq=%d comm=%s(%d) affinity=0x%x", + cpu_of(rq), p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); + wts->prev_on_rq = 1; wts->prev_on_rq_cpu = cpu_of(rq); From 4a7f7c378c698e21691b81c002859d20c3683aa5 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 12 Jan 2022 12:50:24 -0800 Subject: [PATCH 240/346] sched/walt: cleanup cpu_active and cpu_online usage cpu_active mask is a super-set of cpu_online. Any cpus that are active are by definition online, so it is redundant to check both. Cleanup the usage of cpu_active and cpu_online, to eliminate redundancies and ensure usage of the super set (active) when needed. Change-Id: I085cb94a609819b6786c81d8abc784aae8b4ff54 Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 4 ++-- kernel/sched/walt/walt_cfs.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 9d36748fccf8..0f856727a16e 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -772,7 +772,7 @@ static unsigned int get_active_cpu_count(const struct cluster_data *cluster) static bool is_active(const struct cpu_data *state) { - return cpu_online(state->cpu) && !cpu_halted(state->cpu); + return cpu_active(state->cpu) && !cpu_halted(state->cpu); } static bool adjustment_possible(const struct cluster_data *cluster, @@ -1101,7 +1101,7 @@ static int __try_to_resume(struct cluster_data *cluster, unsigned int need, if (!cpumask_test_cpu(c->cpu, &cpus_paused_by_us)) continue; - if ((cpu_online(c->cpu) && !cpu_halted(c->cpu)) || + if ((cpu_active(c->cpu) && !cpu_halted(c->cpu)) || (!force && c->not_preferred)) continue; if (active_cpus + nr_pending == need) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index a93bc35cbb2a..5d6c3f53dd08 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -267,7 +267,7 @@ static void walt_find_best_target(struct sched_domain *sd, /* fast path for prev_cpu */ if (((capacity_orig_of(prev_cpu) == capacity_orig_of(start_cpu)) || asym_cap_siblings(prev_cpu, start_cpu)) && - cpu_active(prev_cpu) && cpu_online(prev_cpu) && + cpu_active(prev_cpu) && available_idle_cpu(prev_cpu) && cpumask_test_cpu(prev_cpu, p->cpus_ptr) && !cpu_halted(prev_cpu)) { From 42cb4efdabb0c7a1a7e789d755644222ffa23ae6 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 12 Jan 2022 17:39:19 -0800 Subject: [PATCH 241/346] sched/walt: remove unused can_halt cpu mask After reviewing, it was determind that the can_halt_mask is no longer needed. Remove it and cleanup code as appropriate. Change-Id: I5430e6f3eda505e0f8fe1cedb9fa7b20425f6900 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.h | 4 ---- kernel/sched/walt/walt_halt.c | 4 ---- 2 files changed, 8 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 79f7045e02ac..c2318e36264c 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -908,10 +908,6 @@ extern struct cpumask __cpu_halt_mask; #define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask) #define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask) -extern struct cpumask __cpu_can_halt_mask; -#define cpu_can_halt_mask ((struct cpumask *)&__cpu_can_halt_mask) -#define cpu_can_halt(cpu) cpumask_test_cpu((cpu), cpu_can_halt_mask) - extern void walt_task_dump(struct task_struct *p); extern void walt_rq_dump(int cpu); extern void walt_dump(void); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 4a8fc0472d44..16a71fe18e66 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -11,7 +11,6 @@ /* if a cpu is halting */ struct cpumask __cpu_halt_mask; -struct cpumask __cpu_can_halt_mask; static DEFINE_MUTEX(halt_lock); @@ -337,9 +336,6 @@ EXPORT_SYMBOL(walt_resume_cpus); void walt_halt_init(void) { - cpumask_set_cpu(5, cpu_can_halt_mask); - cpumask_set_cpu(6, cpu_can_halt_mask); - cpumask_set_cpu(7, cpu_can_halt_mask); } #endif /* CONFIG_HOTPLUG_CPU */ From f499ceb734c1bef6ac7f150d9cbb787285ebee6d Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Tue, 18 Jan 2022 11:00:13 -0800 Subject: [PATCH 242/346] sched: Simplify sliding window logic The sliding window logic in update_history is unnecessary complicated and involves moving each window entry into a new position. Instead, we can simply replace the older entries in place. Change-Id: I49a80184241393219f421421f07514e01b8a674e Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 1 + kernel/sched/walt/walt.c | 22 +++++++++------------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index e7813c280140..ce8e5bcee7cf 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -119,6 +119,7 @@ struct walt_task_struct { u64 sum_exec_snapshot; u64 total_exec; int mvp_prio; + int cidx; }; #define wts_to_ts(wts) ({ \ diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 59b49318212f..45e96ac17e00 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1857,7 +1857,7 @@ static void update_history(struct rq *rq, struct task_struct *p, { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u32 *hist = &wts->sum_history[0]; - int ridx, widx; + int i; u32 max = 0, avg, demand, pred_demand; u64 sum = 0; u16 demand_scaled, pred_demand_scaled; @@ -1868,20 +1868,15 @@ static void update_history(struct rq *rq, struct task_struct *p, goto done; /* Push new 'runtime' value onto stack */ - widx = sched_ravg_hist_size - 1; - ridx = widx - samples; - for (; ridx >= 0; --widx, --ridx) { - hist[widx] = hist[ridx]; - sum += hist[widx]; - if (hist[widx] > max) - max = hist[widx]; + for (; samples > 0; samples--) { + hist[wts->cidx] = runtime; + wts->cidx = ++(wts->cidx) % sched_ravg_hist_size; } - for (widx = 0; widx < samples && widx < sched_ravg_hist_size; widx++) { - hist[widx] = runtime; - sum += hist[widx]; - if (hist[widx] > max) - max = hist[widx]; + for (i = 0; i < sched_ravg_hist_size; i++) { + sum += hist[i]; + if (hist[i] > max) + max = hist[i]; } wts->sum = 0; @@ -2260,6 +2255,7 @@ static void init_new_task_load(struct task_struct *p) wts->sum_exec_snapshot = 0; wts->total_exec = 0; wts->mvp_prio = WALT_NOT_MVP; + wts->cidx = 0; __sched_fork_init(p); } From 0c44eb86b40dbaddae8428f7692e45321d982802 Mon Sep 17 00:00:00 2001 From: Maria Yu Date: Wed, 24 Nov 2021 15:31:25 +0800 Subject: [PATCH 243/346] sched/walt: Deal with mvp_list at the very initial stage When the android vendor hook is not registered yet, there is already tasks. And if those kind of tasks being killed without any wakeup after vendor hook registered, it will be a NULL referenced data. Change-Id: Ied2eb3805dd01740a01707e2016c24a4648b1b25 Signed-off-by: Maria Yu --- kernel/sched/walt/walt_cfs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 5d6c3f53dd08..82f54d187d0a 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1148,7 +1148,7 @@ void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (!list_empty(&wts->mvp_list)) + if (!list_empty(&wts->mvp_list) && wts->mvp_list.next) walt_cfs_deactivate_mvp_task(p); /* @@ -1171,7 +1171,7 @@ void walt_cfs_tick(struct rq *rq) raw_spin_lock(&rq->__lock); - if (list_empty(&wts->mvp_list)) + if (list_empty(&wts->mvp_list) || (wts->mvp_list.next == NULL)) goto out; walt_cfs_account_mvp_runtime(rq, rq->curr); @@ -1205,8 +1205,8 @@ static void walt_cfs_check_preempt_wakeup(void *unused, struct rq *rq, struct ta if (unlikely(walt_disabled)) return; - p_is_mvp = !list_empty(&wts_p->mvp_list); - curr_is_mvp = !list_empty(&wts_c->mvp_list); + p_is_mvp = !list_empty(&wts_p->mvp_list) && wts_p->mvp_list.next; + curr_is_mvp = !list_empty(&wts_c->mvp_list) && wts_c->mvp_list.next; /* * current is not MVP, so preemption decision From 5d6c9601a086df348ae5f8c41196e11d84e08b29 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 19 Jan 2022 11:31:11 -0800 Subject: [PATCH 244/346] sched/walt: update select_task_rq_rt to filter halted cpus walt_rt_energy_aware_wake_cpu could return -1, which makes walt_select_task_rq_rt use the task's prev cpu (the backup cpu). However, the backup_cpu could be halted. Address this by replacing the chosen cpu with the lowest order non-halted cpu in the system. Change-Id: I366bd2e9093b945a093ae4542a77983b0bc57229 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_rt.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index ec299ce106ef..594990a18291 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2022, The Linux Foundation. All rights reserved. */ #include @@ -217,6 +217,17 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c (may_not_preempt || task->prio < cpu_rq(target)->rt.highest_prio.curr)) *new_cpu = target; + /* if backup or chosen cpu is halted, pick something else */ + if (cpu_halted(*new_cpu)) { + cpumask_t non_halted; + + /* choose the lowest-order, unhalted, allowed CPU */ + cpumask_andnot(&non_halted, task->cpus_ptr, cpu_halt_mask); + target = cpumask_first(&non_halted); + if (target < nr_cpu_ids) + *new_cpu = target; + } + rcu_read_unlock(); } From 635f5f5611a9cbd2bc9195c3e4d162203c20b745 Mon Sep 17 00:00:00 2001 From: Ashay Jaiswal Date: Wed, 19 Jan 2022 17:32:29 +0530 Subject: [PATCH 245/346] sched: walt: fix rounding off error Update variable type to ensure all the valid values can be accomodated in variable. Change-Id: Ieaf0301d6ad524bfb87ff019aec41063ebc4c4c4 Signed-off-by: Ashay Jaiswal --- kernel/sched/walt/trace.h | 6 +++--- kernel/sched/walt/walt.h | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 1f566daa8ab0..c55d8cbfc218 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -983,9 +983,9 @@ TRACE_EVENT(sched_compute_energy, __field(unsigned long, m0) __field(unsigned long, m1) __field(unsigned long, m2) - __field(u16, c0) - __field(u16, c1) - __field(u16, c2) + __field(unsigned long, c0) + __field(unsigned long, c1) + __field(unsigned long, c2) ), TP_fast_assign( diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index c2318e36264c..32f570b603cc 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2022, The Linux Foundation. All rights reserved. */ #ifndef _WALT_H @@ -135,8 +135,7 @@ struct walt_sched_cluster { unsigned int max_possible_freq; unsigned int max_freq; u64 aggr_grp_load; - - u16 util_to_cost[1024]; + unsigned long util_to_cost[1024]; }; extern struct walt_sched_cluster *sched_cluster[WALT_NR_CPUS]; @@ -900,7 +899,7 @@ void create_util_to_cost(void); struct compute_energy_output { unsigned long sum_util[MAX_CLUSTERS]; unsigned long max_util[MAX_CLUSTERS]; - u16 cost[MAX_CLUSTERS]; + unsigned long cost[MAX_CLUSTERS]; unsigned int cluster_first_cpu[MAX_CLUSTERS]; }; From 2165ff1d71a3fb68588d14ef5fa98510ad4165e5 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 9 Apr 2021 09:02:18 -0700 Subject: [PATCH 246/346] sched/walt: Compare cluster IDs instead of capacities Currently, we make certain decisions based on capacities of individual CPUs. However, we may be interested in grouping certain cpus of different capacities together. In this case, we will need to use their cluster IDs to differentiate between two CPUs. Change-Id: I20d9c473a98e55dfdd658b96fc9285f0a0f51652 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.h | 4 +++- kernel/sched/walt/walt_cfs.c | 22 ++++++++++++++++------ kernel/sched/walt/walt_lb.c | 32 +++++++++++++++++++++----------- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 32f570b603cc..5792c86de8f2 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -719,11 +719,13 @@ static inline bool task_fits_capacity(struct task_struct *p, int cpu) { unsigned int margin; + struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; /* * Derive upmigration/downmigrate margin wrt the src/dest CPU. */ - if (capacity_orig_of(task_cpu(p)) > capacity_orig_of(cpu)) + if (src_wrq->cluster->id > dst_wrq->cluster->id) margin = sched_capacity_margin_down[cpu]; else margin = sched_capacity_margin_up[task_cpu(p)]; diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 82f54d187d0a..9cdbe5c59ad5 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -64,8 +64,10 @@ bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) { bool base_test = cpumask_test_cpu(cpu, &p->cpus_mask) && cpu_active(cpu); - bool start_cap_test = (capacity_orig_of(cpu) >= - capacity_orig_of(start_cpu)); + + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; + bool start_cap_test = (wrq->cluster->id >= start_wrq->cluster->id); return base_test && start_cap_test; } @@ -246,9 +248,12 @@ static void walt_find_best_target(struct sched_domain *sd, bool rtg_high_prio_task = task_rtg_high_prio(p); cpumask_t visit_cpus; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1; + struct walt_rq *start_wrq; /* Find start CPU based on boost value */ start_cpu = fbt_env->start_cpu; + start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; /* * For higher capacity worth I/O tasks, stop the search @@ -265,7 +270,7 @@ static void walt_find_best_target(struct sched_domain *sd, } /* fast path for prev_cpu */ - if (((capacity_orig_of(prev_cpu) == capacity_orig_of(start_cpu)) || + if (((prev_wrq->cluster->id == start_wrq->cluster->id) || asym_cap_siblings(prev_cpu, start_cpu)) && cpu_active(prev_cpu) && available_idle_cpu(prev_cpu) && @@ -703,10 +708,12 @@ static inline bool select_cpu_same_energy(int cpu, int best_cpu, int prev_cpu) { bool new_cpu_is_idle = available_idle_cpu(cpu); bool best_cpu_is_idle = available_idle_cpu(best_cpu); + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *best_wrq = (struct walt_rq *) cpu_rq(best_cpu)->android_vendor_data1; - if (capacity_orig_of(best_cpu) < capacity_orig_of(cpu)) + if (best_wrq->cluster->id < wrq->cluster->id) return false; - if (capacity_orig_of(cpu) < capacity_orig_of(best_cpu)) + if (wrq->cluster->id < best_wrq->cluster->id) return true; if (best_cpu_is_idle && walt_get_idle_exit_latency(cpu_rq(best_cpu)) <= 1) @@ -759,6 +766,8 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int first_cpu; bool energy_eval_needed = true; struct compute_energy_output output; + struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1; + struct walt_rq *start_wrq; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && cpumask_test_cpu(prev_cpu, &p->cpus_mask)) @@ -770,6 +779,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost, &energy_eval_needed); start_cpu = cpumask_first(&cpu_array[order_index][0]); + start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; is_rtg = task_in_related_thread_group(p); curr_is_rtg = task_in_related_thread_group(cpu_rq(cpu)->curr); @@ -895,7 +905,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, walt_get_idle_exit_latency(cpu_rq(best_energy_cpu)) <= 1) && (prev_energy != ULONG_MAX) && (best_energy_cpu != prev_cpu) && ((prev_energy - best_energy) <= prev_energy >> 5) && - (capacity_orig_of(prev_cpu) <= capacity_orig_of(start_cpu))) + (prev_wrq->cluster->id <= start_wrq->cluster->id)) best_energy_cpu = prev_cpu; unlock: diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 7e361c89e9f7..09cbd735e009 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -266,11 +266,13 @@ static inline bool need_active_lb(struct task_struct *p, int dst_cpu, int src_cpu) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; if (cpu_rq(src_cpu)->active_balance) return false; - if (capacity_orig_of(dst_cpu) <= capacity_orig_of(src_cpu)) + if (dst_wrq->cluster->id <= src_wrq->cluster->id) return false; if (!wts->misfit) @@ -286,12 +288,13 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) unsigned long flags; struct task_struct *pulled_task = NULL, *p; bool active_balance = false, to_lower; - struct walt_rq *wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; struct walt_task_struct *wts; BUG_ON(src_cpu == dst_cpu); - to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(src_cpu); + to_lower = dst_wrq->cluster->id < src_wrq->cluster->id; raw_spin_lock_irqsave(&src_rq->__lock, flags); @@ -339,7 +342,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) src_rq->active_balance = 1; src_rq->push_cpu = dst_cpu; get_task_struct(p); - wrq->push_task = p; + src_wrq->push_task = p; mark_reserved(dst_cpu); /* lock must be dropped before waking the stopper */ @@ -577,13 +580,15 @@ static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int { int fsrc_cpu = cpumask_first(src_mask); int busiest_cpu; + struct walt_rq *fsrc_wrq = (struct walt_rq *) cpu_rq(fsrc_cpu)->android_vendor_data1; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; - if (capacity_orig_of(dst_cpu) == capacity_orig_of(fsrc_cpu)) + if (dst_wrq->cluster->id == fsrc_wrq->cluster->id) busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, src_mask, has_misfit); - else if (capacity_orig_of(dst_cpu) > capacity_orig_of(fsrc_cpu)) + else if (dst_wrq->cluster->id > fsrc_wrq->cluster->id) busiest_cpu = walt_lb_find_busiest_from_lower_cap_cpu(dst_cpu, - src_mask, has_misfit); + src_mask, has_misfit); else busiest_cpu = walt_lb_find_busiest_from_higher_cap_cpu(dst_cpu, src_mask, has_misfit); @@ -597,7 +602,8 @@ void walt_lb_tick(struct rq *rq) int prev_cpu = rq->cpu, new_cpu, ret; struct task_struct *p = rq->curr; unsigned long flags; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *prev_wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *new_wrq; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; raw_spin_lock(&rq->__lock); @@ -627,9 +633,11 @@ void walt_lb_tick(struct rq *rq) new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1); rcu_read_unlock(); + new_wrq = (struct walt_rq *) cpu_rq(new_cpu)->android_vendor_data1; + /* prevent active task migration to busy or same/lower capacity CPU */ if (new_cpu < 0 || !available_idle_cpu(new_cpu) || - capacity_orig_of(new_cpu) <= capacity_orig_of(prev_cpu)) + new_wrq->cluster->id <= prev_wrq->cluster->id) goto out_unlock; raw_spin_lock(&rq->__lock); @@ -640,7 +648,7 @@ void walt_lb_tick(struct rq *rq) rq->active_balance = 1; rq->push_cpu = new_cpu; get_task_struct(p); - wrq->push_task = p; + prev_wrq->push_task = p; raw_spin_unlock(&rq->__lock); mark_reserved(new_cpu); @@ -1011,10 +1019,12 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, int dst_cpu, int *can_migrate) { bool to_lower; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + struct walt_rq *task_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1; if (unlikely(walt_disabled)) return; - to_lower = capacity_orig_of(dst_cpu) < capacity_orig_of(task_cpu(p)); + to_lower = dst_wrq->cluster->id < task_wrq->cluster->id; if (_walt_can_migrate_task(p, dst_cpu, to_lower, true)) return; From 2b8a21e9a417bd331f2f00f667717f7fde2bdd15 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 14 Jan 2022 10:57:06 -0800 Subject: [PATCH 247/346] sched/walt: create conditions for walt-bug and console output Allow for finer grain control over the walt-bug capabilities, such that different walt-bug types can be treated differently, either causing a panic, or simply printing out content. echo 0x4544DEXY > /proc/sys/walt/panic_on_walt_bug Y in the above represents the bitmask for the walt debug class that is enabled. If set, the device will panic on that class of issue. X in the above represents the bitmask for the walt debug class that is enabled, for printing to console. If set, device will printk on that class of issue. X and Y are independent, and maybe configured separately as desired. However, nothing will happen if the most significant 3 bytes of panic_on_walt_bug are not properly set to the SENTINEL value. Change-Id: I01382fdb5717625f42f45eac4151d47578f0b0c6 Signed-off-by: Stephen Dickey --- kernel/sched/walt/sysctl.c | 4 +-- kernel/sched/walt/walt.c | 66 +++++++++++++++++++++--------------- kernel/sched/walt/walt.h | 63 ++++++++++++++++++++++++++-------- kernel/sched/walt/walt_cfs.c | 8 +++-- 4 files changed, 95 insertions(+), 46 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index c6eb6430b15b..f52aa1b90f76 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2022, The Linux Foundation. All rights reserved. */ #include "walt.h" @@ -62,7 +62,7 @@ const int sched_user_hint_max = 1000; unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ unsigned int sysctl_sched_sync_hint_enable = 1; unsigned int sysctl_sched_bug_on_rt_throttle; -unsigned int sysctl_panic_on_walt_bug; +unsigned int sysctl_panic_on_walt_bug = walt_debug_initial_values(); unsigned int sysctl_sched_suppress_region2; unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 45e96ac17e00..f313c98b5d21 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2022, The Linux Foundation. All rights reserved. */ #include @@ -293,11 +293,11 @@ fixup_cumulative_runnable_avg(struct rq *rq, lockdep_assert_held(&rq->__lock); if (task_rq(p) != rq) - WALT_BUG(p, "on CPU %d task %s(%d) not on rq %d", + WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on rq %d", raw_smp_processor_id(), p->comm, p->pid, rq->cpu); if (cumulative_runnable_avg_scaled < 0) { - WALT_BUG(p, "on CPU %d task ds=%llu is higher than cra=%llu\n", + WALT_BUG(WALT_BUG_WALT, p, "on CPU %d task ds=%llu is higher than cra=%llu\n", raw_smp_processor_id(), wts->demand_scaled, stats->cumulative_runnable_avg_scaled); cumulative_runnable_avg_scaled = 0; @@ -305,7 +305,7 @@ fixup_cumulative_runnable_avg(struct rq *rq, stats->cumulative_runnable_avg_scaled = (u64)cumulative_runnable_avg_scaled; if (pred_demands_sum_scaled < 0) { - WALT_BUG(p, "on CPU %d task pds=%llu is higher than pds_sum=%llu\n", + WALT_BUG(WALT_BUG_WALT, p, "on CPU %d task pds=%llu is higher than pds_sum=%llu\n", raw_smp_processor_id(), wts->pred_demand_scaled, stats->pred_demands_sum_scaled); pred_demands_sum_scaled = 0; @@ -696,22 +696,22 @@ static inline void account_load_subtractions(struct rq *rq) } if ((s64)wrq->prev_runnable_sum < 0) { - WALT_BUG(NULL, "wrq->prev_runnable_sum=%llu < 0", + WALT_BUG(WALT_BUG_WALT, NULL, "wrq->prev_runnable_sum=%llu < 0", (s64)wrq->prev_runnable_sum); wrq->prev_runnable_sum = 0; } if ((s64)wrq->curr_runnable_sum < 0) { - WALT_BUG(NULL, "wrq->curr_runnable_sum=%llu < 0", + WALT_BUG(WALT_BUG_WALT, NULL, "wrq->curr_runnable_sum=%llu < 0", (s64)wrq->curr_runnable_sum); wrq->curr_runnable_sum = 0; } if ((s64)wrq->nt_prev_runnable_sum < 0) { - WALT_BUG(NULL, "wrq->nt_prev_runnable_sum=%llu < 0", + WALT_BUG(WALT_BUG_WALT, NULL, "wrq->nt_prev_runnable_sum=%llu < 0", (s64)wrq->nt_prev_runnable_sum); wrq->nt_prev_runnable_sum = 0; } if ((s64)wrq->nt_curr_runnable_sum < 0) { - WALT_BUG(NULL, "wrq->nt_curr_runnable_sum=%llu < 0", + WALT_BUG(WALT_BUG_WALT, NULL, "wrq->nt_curr_runnable_sum=%llu < 0", (s64)wrq->nt_curr_runnable_sum); wrq->nt_curr_runnable_sum = 0; } @@ -823,7 +823,8 @@ static inline void inter_cluster_migration_fixup dest_wrq->prev_runnable_sum += wts->prev_window; if (src_wrq->curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { - WALT_BUG(p, "pid=%u CPU%d -> CPU%d src_crs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU%d -> CPU%d src_crs=%llu is lesser than task_contrib=%llu", p->pid, src_rq->cpu, dest_rq->cpu, src_wrq->curr_runnable_sum, wts->curr_window_cpu[task_cpu]); @@ -832,7 +833,8 @@ static inline void inter_cluster_migration_fixup src_wrq->curr_runnable_sum -= wts->curr_window_cpu[task_cpu]; if (src_wrq->prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { - WALT_BUG(p, "pid=%u CPU%d -> CPU%d src_prs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU%d -> CPU%d src_prs=%llu is lesser than task_contrib=%llu", p->pid, src_rq->cpu, dest_rq->cpu, src_wrq->prev_runnable_sum, wts->prev_window_cpu[task_cpu]); @@ -845,7 +847,8 @@ static inline void inter_cluster_migration_fixup dest_wrq->nt_prev_runnable_sum += wts->prev_window; if (src_wrq->nt_curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { - WALT_BUG(p, "pid=%u CPU%d -> CPU%d src_nt_crs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU%d -> CPU%d src_nt_crs=%llu is lesser than task_contrib=%llu", p->pid, src_rq->cpu, dest_rq->cpu, src_wrq->nt_curr_runnable_sum, wts->curr_window_cpu[task_cpu]); @@ -855,7 +858,8 @@ static inline void inter_cluster_migration_fixup wts->curr_window_cpu[task_cpu]; if (src_wrq->nt_prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { - WALT_BUG(p, "pid=%u CPU%d -> CPU%d src_nt_prs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU%d -> CPU%d src_nt_prs=%llu is lesser than task_contrib=%llu", p->pid, src_rq->cpu, dest_rq->cpu, src_wrq->nt_prev_runnable_sum, wts->prev_window_cpu[task_cpu]); @@ -982,7 +986,7 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) lockdep_assert_held(&dest_rq->__lock); if (task_rq(p) != src_rq) - WALT_BUG(p, "on CPU %d task %s(%d) not on src_rq %d", + WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on src_rq %d", raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); walt_update_task_ravg(task_rq(p)->curr, task_rq(p), @@ -3188,7 +3192,8 @@ static void transfer_busy_time(struct rq *rq, dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; if (*src_curr_runnable_sum < wts->curr_window_cpu[cpu]) { - WALT_BUG(p, "pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_curr_runnable_sum, wts->curr_window_cpu[cpu]); *src_curr_runnable_sum = wts->curr_window_cpu[cpu]; @@ -3196,7 +3201,8 @@ static void transfer_busy_time(struct rq *rq, *src_curr_runnable_sum -= wts->curr_window_cpu[cpu]; if (*src_prev_runnable_sum < wts->prev_window_cpu[cpu]) { - WALT_BUG(p, "pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_prev_runnable_sum, wts->prev_window_cpu[cpu]); *src_prev_runnable_sum = wts->prev_window_cpu[cpu]; @@ -3205,7 +3211,8 @@ static void transfer_busy_time(struct rq *rq, if (new_task) { if (*src_nt_curr_runnable_sum < wts->curr_window_cpu[cpu]) { - WALT_BUG(p, "pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_nt_curr_runnable_sum, wts->curr_window_cpu[cpu]); @@ -3215,7 +3222,8 @@ static void transfer_busy_time(struct rq *rq, wts->curr_window_cpu[cpu]; if (*src_nt_prev_runnable_sum < wts->prev_window_cpu[cpu]) { - WALT_BUG(p, "pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_nt_prev_runnable_sum, wts->prev_window_cpu[cpu]); @@ -3242,7 +3250,8 @@ static void transfer_busy_time(struct rq *rq, dst_nt_prev_runnable_sum = &wrq->nt_prev_runnable_sum; if (*src_curr_runnable_sum < wts->curr_window) { - WALT_BUG(p, "WALT-UG pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "WALT-UG pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_curr_runnable_sum, wts->curr_window); *src_curr_runnable_sum = wts->curr_window; @@ -3250,7 +3259,8 @@ static void transfer_busy_time(struct rq *rq, *src_curr_runnable_sum -= wts->curr_window; if (*src_prev_runnable_sum < wts->prev_window) { - WALT_BUG(p, "pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_prev_runnable_sum, wts->prev_window); *src_prev_runnable_sum = wts->prev_window; @@ -3259,7 +3269,8 @@ static void transfer_busy_time(struct rq *rq, if (new_task) { if (*src_nt_curr_runnable_sum < wts->curr_window) { - WALT_BUG(p, "pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_nt_curr_runnable_sum, wts->curr_window); @@ -3268,7 +3279,8 @@ static void transfer_busy_time(struct rq *rq, *src_nt_curr_runnable_sum -= wts->curr_window; if (*src_nt_prev_runnable_sum < wts->prev_window) { - WALT_BUG(p, "pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", p->pid, cpu, event, *src_nt_prev_runnable_sum, wts->prev_window); @@ -3916,18 +3928,18 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st lockdep_assert_held(&rq->__lock); if (p->cpu != cpu_of(rq)) - WALT_BUG(p, "enqueuing on rq %d when task->cpu is %d\n", + WALT_BUG(WALT_BUG_UPSTREAM, p, "enqueuing on rq %d when task->cpu is %d\n", cpu_of(rq), p->cpu); /* catch double enqueue */ if (wts->prev_on_rq == 1) { - WALT_BUG(p, "double enqueue detected: task_cpu=%d new_cpu=%d\n", + WALT_BUG(WALT_BUG_UPSTREAM, p, "double enqueue detected: task_cpu=%d new_cpu=%d\n", task_cpu(p), cpu_of(rq)); double_enqueue = true; } if (!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) - WALT_BUG(p, "enqueueing on rq=%d comm=%s(%d) affinity=0x%x", + WALT_BUG(WALT_BUG_NONCRITICAL, p, "enqueueing on rq=%d comm=%s(%d) affinity=0x%x", cpu_of(rq), p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); wts->prev_on_rq = 1; @@ -3965,7 +3977,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st * an invalid failure. */ if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq)) - WALT_BUG(p, "dequeue cpu %d not same as enqueue %d\n", + WALT_BUG(WALT_BUG_UPSTREAM, p, "dequeue cpu %d not same as enqueue %d\n", cpu_of(rq), wts->prev_on_rq_cpu); /* no longer on a cpu */ @@ -3973,7 +3985,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st /* catch double deq */ if (wts->prev_on_rq == 2) { - WALT_BUG(p, "double dequeue detected: task_cpu=%d new_cpu=%d\n", + WALT_BUG(WALT_BUG_UPSTREAM, p, "double dequeue detected: task_cpu=%d new_cpu=%d\n", task_cpu(p), cpu_of(rq)); double_dequeue = true; } @@ -4123,7 +4135,7 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); if (is_idle_task(next) && wrq->walt_stats.cumulative_runnable_avg_scaled != 0) - WALT_BUG(next, "next=idle cra non zero=%d\n", + WALT_BUG(WALT_BUG_WALT, next, "next=idle cra non zero=%d\n", wrq->walt_stats.cumulative_runnable_avg_scaled); } else { walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 5792c86de8f2..a5f75363eeb0 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -917,6 +917,16 @@ extern int in_sched_bug; extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, struct task_struct *p, int dest_cpu); +enum WALT_DEBUG_FEAT { + WALT_BUG_UPSTREAM, + WALT_BUG_WALT, + WALT_BUG_NONCRITICAL, + WALT_BUG_UNUSED, + + /* maximum 4 entries allowed */ + WALT_DEBUG_FEAT_NR, +}; + #define WALT_PANIC(condition) \ ({ \ if (unlikely(!!(condition)) && !in_sched_bug) { \ @@ -926,21 +936,46 @@ extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, } \ }) -#define WALT_PANIC_SENTINEL 0x4544DEAD +/* the least signifcant byte is the bitmask for features and printk */ +#define WALT_PANIC_SENTINEL 0x4544DE00 -/* - * crash if walt bugs are fatal, otherwise return immediately. - * output format and arguments to console - */ -#define WALT_BUG(p, format, args...) \ -({ \ - if (unlikely(sysctl_panic_on_walt_bug == WALT_PANIC_SENTINEL)) {\ - printk_deferred("WALT-BUG " format, args); \ - if (p) \ - walt_task_dump(p); \ - WALT_PANIC(1); \ - } \ +#define walt_debug_bitmask_panic(x) (1UL << x) +#define walt_debug_bitmask_print(x) (1UL << (x + WALT_DEBUG_FEAT_NR)) + +/* setup initial values, bug and print on upstream and walt, ignore noncritical */ +#define walt_debug_initial_values() \ + (WALT_PANIC_SENTINEL | \ + walt_debug_bitmask_panic(WALT_BUG_UPSTREAM) | \ + walt_debug_bitmask_print(WALT_BUG_UPSTREAM) | \ + walt_debug_bitmask_panic(WALT_BUG_WALT) | \ + walt_debug_bitmask_print(WALT_BUG_WALT)) + +/* least significant nibble is the bug feature itself */ +#define walt_debug_feat_panic(x) (!!(sysctl_panic_on_walt_bug & (1UL << x))) + +/* 2nd least significant nibble is the print capability */ +#define walt_debug_feat_print(x) (!!(sysctl_panic_on_walt_bug & (1UL << (x + WALT_DEBUG_FEAT_NR)))) + +/* return true if the sentinel is set, regardless of feature set */ +static inline bool is_walt_sentinel(void) +{ + if (unlikely((sysctl_panic_on_walt_bug & 0xFFFFFF00) == WALT_PANIC_SENTINEL)) + return true; + return false; +} + +/* if the sentinel is properly set, print and/or panic as configured */ +#define WALT_BUG(feat, p, format, args...) \ +({ \ + if (is_walt_sentinel()) { \ + if (walt_debug_feat_print(feat)) \ + printk_deferred("WALT-BUG " format, args); \ + if (walt_debug_feat_panic(feat)) { \ + if (p) \ + walt_task_dump(p); \ + WALT_PANIC(1); \ + } \ + } \ }) - #endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 9cdbe5c59ad5..3b3bafe42f42 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2022, The Linux Foundation. All rights reserved. */ #include @@ -1268,7 +1268,8 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || (*p)->on_rq == TASK_ON_RQ_MIGRATING || (*p)->cpu != cpu_of(rq))) - WALT_BUG(*p, "picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", + WALT_BUG(WALT_BUG_UPSTREAM, *p, + "picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", (*p)->comm, (*p)->pid, (*p)->on_cpu, (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); @@ -1287,7 +1288,8 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || (*p)->on_rq == TASK_ON_RQ_MIGRATING || (*p)->cpu != cpu_of(rq))) - WALT_BUG(*p, "picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", + WALT_BUG(WALT_BUG_UPSTREAM, *p, + "picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", (*p)->comm, (*p)->pid, (*p)->on_cpu, (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); From 59b2ef40cbdf5c951717bf97bcb72da39b89a89d Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 11 Jan 2022 10:37:51 -0800 Subject: [PATCH 248/346] sched/walt: allow checking of last halt time against threshold Support the ability to check whether the current time is within a threshold since the last time the cpu was halted. If it's within that threshold, pass the check. Use that check to catch tasks being enqueued on halted cpus. Change-Id: I1716aefc970fc3dea7474f6321971bf00c3b76f4 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 5 ++++ kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_halt.c | 46 +++++++++++++++++++++++++++++++---- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index f313c98b5d21..b4c09317d2a1 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3942,6 +3942,11 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st WALT_BUG(WALT_BUG_NONCRITICAL, p, "enqueueing on rq=%d comm=%s(%d) affinity=0x%x", cpu_of(rq), p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); + if (cpu_halted(cpu_of(rq)) && !(p->flags & PF_KTHREAD) && !walt_halt_check_last(cpu_of(rq))) + WALT_BUG(WALT_BUG_NONCRITICAL, p, + "Non Kthread Started on halted cpu_of(rq)=%d comm=%s(%d) affinity=0x%x\n", + cpu_of(rq), p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); + wts->prev_on_rq = 1; wts->prev_on_rq_cpu = cpu_of(rq); diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index a5f75363eeb0..96d78ba245d3 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -905,6 +905,7 @@ struct compute_energy_output { unsigned int cluster_first_cpu[MAX_CLUSTERS]; }; +bool walt_halt_check_last(int cpu); extern struct cpumask __cpu_halt_mask; #define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask) #define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 16a71fe18e66..a8cf2ed2de8e 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -15,11 +15,17 @@ struct cpumask __cpu_halt_mask; static DEFINE_MUTEX(halt_lock); struct halt_cpu_state { + u64 last_halt; int ref_count; }; static DEFINE_PER_CPU(struct halt_cpu_state, halt_state); +/* the amount of time allowed for enqueue operations that happen + * just after a halt operation. + */ +#define WALT_HALT_CHECK_THRESHOLD_NS 400000 + /* * Remove a task from the runqueue and pretend that it's migrating. This * should prevent migrations for the detached task and disallow further @@ -176,6 +182,21 @@ static int cpu_drain_rq(unsigned int cpu) return stop_one_cpu(cpu, drain_rq_cpu_stop, NULL); } +/* + * returns true if last halt is within threshold + * note: do not take halt_lock, called from atomic context + */ +bool walt_halt_check_last(int cpu) +{ + u64 last_halt = per_cpu_ptr(&halt_state, cpu)->last_halt; + + /* last_halt is valid, check it against sched_clock */ + if (last_halt != 0 && sched_clock() - last_halt > WALT_HALT_CHECK_THRESHOLD_NS) + return false; + + return true; +} + /* * 1) add the cpus to the halt mask * 2) migrate tasks off the cpu @@ -186,14 +207,22 @@ static int halt_cpus(struct cpumask *cpus) int cpu; int ret = 0; u64 start_time = sched_clock(); + struct halt_cpu_state *halt_cpu_state; trace_halt_cpus_start(cpus, 1); for_each_cpu(cpu, cpus) { + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); + /* set the cpu as halted */ cpumask_set_cpu(cpu, cpu_halt_mask); + /* guarantee mask written before updating last_halt */ + wmb(); + + halt_cpu_state->last_halt = start_time; + /* only drain online cpus */ if (cpu_online(cpu)) { /* drain the online CPU */ @@ -219,12 +248,19 @@ static int halt_cpus(struct cpumask *cpus) static int start_cpus(struct cpumask *cpus) { u64 start_time = sched_clock(); + struct halt_cpu_state *halt_cpu_state; + int cpu; trace_halt_cpus_start(cpus, 0); - if (!cpumask_empty(cpus)) { - /* remove the cpus from the halt mask */ - cpumask_andnot(cpu_halt_mask, cpu_halt_mask, cpus); + for_each_cpu(cpu, cpus) { + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); + halt_cpu_state->last_halt = 0; + + /* guarantee zero'd last_halt before clearing from the mask */ + wmb(); + + cpumask_clear_cpu(cpu, cpu_halt_mask); } trace_halt_cpus(cpus, start_time, 0, 0); @@ -240,9 +276,9 @@ static void update_ref_counts(struct cpumask *cpus, bool halt) for_each_cpu(cpu, cpus) { halt_cpu_state = per_cpu_ptr(&halt_state, cpu); - if (halt) + if (halt) { halt_cpu_state->ref_count++; - else { + } else { WARN_ON_ONCE(halt_cpu_state->ref_count == 0); halt_cpu_state->ref_count--; } From 32db2ec6033183d6f702531d9fad911c2ae53527 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 27 Jan 2022 15:46:16 -0800 Subject: [PATCH 249/346] sched/walt: catch any set_task_cpu to unaffined cpus When a task's cpu affinity is changed, do_set_cpus_allowed() dequeues the task, updates the task's cpus_mask with new affinity but enqueues the task back on the current cpu, even if the current_cpu is not in the updated mask. This causes the affinity check in enqueue path to fail. Instead add this affinity check to set_task_cpu, this will catch any selection of an cpu outside allowed cpus. Change-Id: I6c1eaf77951f20aea961da463c1e8de0189d36e1 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index b4c09317d2a1..e81198c54f61 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3885,6 +3885,10 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign if (unlikely(walt_disabled)) return; fixup_busy_time(p, (int) new_cpu); + + if (!cpumask_test_cpu(new_cpu, p->cpus_ptr)) + WALT_BUG(WALT_BUG_WALT, p, "selecting unaffined cpu=%d comm=%s(%d) affinity=0x%x", + new_cpu, p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); } static void android_rvh_new_task_stats(void *unused, struct task_struct *p) @@ -3938,9 +3942,6 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st double_enqueue = true; } - if (!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr)) - WALT_BUG(WALT_BUG_NONCRITICAL, p, "enqueueing on rq=%d comm=%s(%d) affinity=0x%x", - cpu_of(rq), p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); if (cpu_halted(cpu_of(rq)) && !(p->flags & PF_KTHREAD) && !walt_halt_check_last(cpu_of(rq))) WALT_BUG(WALT_BUG_NONCRITICAL, p, From 87952a7cb165c593d4aba26a870f9d5645a1c2ae Mon Sep 17 00:00:00 2001 From: Ashay Jaiswal Date: Mon, 7 Feb 2022 14:32:27 +0530 Subject: [PATCH 250/346] sched: walt: update the copyright Update copyright of "walt.h". Change-Id: I2755f597f16904e54579c7e3a24917f41d7a35f7 Signed-off-by: Ashay Jaiswal --- kernel/sched/walt/walt.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 96d78ba245d3..a6036d8da2bb 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2019-2022, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _WALT_H From 97dc7487c40cf5f64d8902a1ae01af589aca9823 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 16 Feb 2022 19:43:32 -0800 Subject: [PATCH 251/346] sched/walt_cfs: temporarily disable MVP feature It is causing rb_tree corruption - disable it for now. Change-Id: I4bf8b47fd9ed0edf11a81706cbb2c0de5b817eb1 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 3b3bafe42f42..4dbd39afbce2 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1004,6 +1004,7 @@ static void binder_restore_priority_hook(void *data, */ static inline int walt_get_mvp_task_prio(struct task_struct *p) { +#if 0 if (per_task_boost(p) == TASK_BOOST_STRICT_MAX) return WALT_TASK_BOOST_MVP; @@ -1013,7 +1014,7 @@ static inline int walt_get_mvp_task_prio(struct task_struct *p) if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p) || walt_pipeline_low_latency_task(p)) return WALT_RTG_MVP; - +#endif return WALT_NOT_MVP; } From 790649850262010b510c024e5b94527fcaa281c0 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Sat, 19 Feb 2022 16:15:25 -0800 Subject: [PATCH 252/346] sched/walt: Fix in walt_lb_tick Currently, to compare cluster IDs of new and old CPUs, new cpu's runqueue is being accessed without checking if new cpu is valid. Fix this by checking new cpu's validity before it is used. Change-Id: I6a374de9690ede3d1deedb076959229b1e79b025 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_lb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 09cbd735e009..9a0502a59e7b 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -633,10 +633,13 @@ void walt_lb_tick(struct rq *rq) new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1); rcu_read_unlock(); + if (new_cpu < 0) + goto out_unlock; + new_wrq = (struct walt_rq *) cpu_rq(new_cpu)->android_vendor_data1; /* prevent active task migration to busy or same/lower capacity CPU */ - if (new_cpu < 0 || !available_idle_cpu(new_cpu) || + if (!available_idle_cpu(new_cpu) || new_wrq->cluster->id <= prev_wrq->cluster->id) goto out_unlock; From e8c7c3807756dc8dd6ff5ea0bf9b6bfad6e980f8 Mon Sep 17 00:00:00 2001 From: Ashay Jaiswal Date: Sat, 12 Feb 2022 00:06:03 +0530 Subject: [PATCH 253/346] sched: walt: take runnable into account for placement descision In case of fully loaded system with spare capacity 0 on all cores, placement logic selects previous cpu of the task for its placement. Update the logic to select cpu which has minimum number of runnables. Also, remove "active_candidate" as we always fallback to least runnable cpu. Change-Id: Ib698969d38dbe7fc7b08451f3c3637aba19a72bc Signed-off-by: Ashay Jaiswal --- kernel/sched/walt/trace.h | 18 ++++++++++++++---- kernel/sched/walt/walt_cfs.c | 32 +++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index c55d8cbfc218..925529525709 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #undef TRACE_SYSTEM @@ -1099,11 +1100,13 @@ TRACE_EVENT(sched_find_best_target, unsigned long candidates, int most_spare_cap, int order_index, int end_index, - int skip, bool running), + int skip, bool running, + int most_spare_rq_cpu, unsigned int cpu_rq_runnable_cnt), TP_ARGS(tsk, min_util, start_cpu, candidates, most_spare_cap, - order_index, end_index, skip, running), + order_index, end_index, skip, running, + most_spare_rq_cpu, cpu_rq_runnable_cnt), TP_STRUCT__entry( __array(char, comm, TASK_COMM_LEN) @@ -1116,6 +1119,8 @@ TRACE_EVENT(sched_find_best_target, __field(int, end_index) __field(int, skip) __field(bool, running) + __field(int, most_spare_rq_cpu) + __field(unsigned int, cpu_rq_runnable_cnt) ), TP_fast_assign( @@ -1129,9 +1134,11 @@ TRACE_EVENT(sched_find_best_target, __entry->end_index = end_index; __entry->skip = skip; __entry->running = running; + __entry->most_spare_rq_cpu = most_spare_rq_cpu; + __entry->cpu_rq_runnable_cnt = cpu_rq_runnable_cnt; ), - TP_printk("pid=%d comm=%s start_cpu=%d candidates=%#lx most_spare_cap=%d order_index=%d end_index=%d skip=%d running=%d", + TP_printk("pid=%d comm=%s start_cpu=%d candidates=%#lx most_spare_cap=%d order_index=%d end_index=%d skip=%d running=%d min_util=%lu spare_rq_cpu=%d min_runnable=%u", __entry->pid, __entry->comm, __entry->start_cpu, __entry->candidates, @@ -1139,7 +1146,10 @@ TRACE_EVENT(sched_find_best_target, __entry->order_index, __entry->end_index, __entry->skip, - __entry->running) + __entry->running, + __entry->min_util, + __entry->most_spare_rq_cpu, + __entry->cpu_rq_runnable_cnt) ); TRACE_EVENT(sched_enq_deq_task, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 4dbd39afbce2..c2bd0d064d18 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -227,6 +227,7 @@ static inline bool is_complex_sibling_idle(int cpu) return false; } +#define DIRE_STRAITS_PREV_NR_LIMIT 10 static void walt_find_best_target(struct sched_domain *sd, cpumask_t *candidates, struct task_struct *p, @@ -239,8 +240,9 @@ static void walt_find_best_target(struct sched_domain *sd, int i, start_cpu; long spare_wake_cap, most_spare_wake_cap = 0; int most_spare_cap_cpu = -1; + int least_nr_cpu = -1; + unsigned int cpu_rq_runnable_cnt = UINT_MAX; int prev_cpu = task_cpu(p); - int active_candidate = -1; int order_index = fbt_env->order_index, end_index = fbt_env->end_index; int stop_index = INT_MAX; int cluster; @@ -311,9 +313,6 @@ static void walt_find_best_target(struct sched_domain *sd, if (cpu_halted(i)) continue; - if (active_candidate == -1) - active_candidate = i; - /* * This CPU is the target of an active migration that's * yet to complete. Avoid placing another task on it. @@ -344,6 +343,16 @@ static void walt_find_best_target(struct sched_domain *sd, most_spare_cap_cpu = i; } + /* + * Keep track of runnables for each CPU, if none of the + * CPUs have spare capacity then use CPU with less + * number of runnables. + */ + if (cpu_rq(i)->nr_running < cpu_rq_runnable_cnt) { + cpu_rq_runnable_cnt = cpu_rq(i)->nr_running; + least_nr_cpu = i; + } + /* * Ensure minimum capacity to grant the required boost. * The target CPU can be already at a capacity level higher @@ -455,21 +464,26 @@ static void walt_find_best_target(struct sched_domain *sd, * We have set idle or target as long as they are valid CPUs. * If we don't find either, then we fallback to most_spare_cap, * If we don't find most spare cap, we fallback to prev_cpu, - * provided that the prev_cpu is active. - * If the prev_cpu is not active, we fallback to active_candidate. + * provided that the prev_cpu is active and has less than + * DIRE_STRAITS_PREV_NR_LIMIT runnables otherwise, we fallback to cpu + * with least number of runnables. */ if (unlikely(cpumask_empty(candidates))) { if (most_spare_cap_cpu != -1) cpumask_set_cpu(most_spare_cap_cpu, candidates); - else if (!cpu_active(prev_cpu) && active_candidate != -1) - cpumask_set_cpu(active_candidate, candidates); + else if (cpu_active(prev_cpu) + && (cpu_rq(prev_cpu)->nr_running < DIRE_STRAITS_PREV_NR_LIMIT)) + cpumask_set_cpu(prev_cpu, candidates); + else if (least_nr_cpu != -1) + cpumask_set_cpu(least_nr_cpu, candidates); } out: trace_sched_find_best_target(p, min_task_util, start_cpu, cpumask_bits(candidates)[0], most_spare_cap_cpu, order_index, end_index, - fbt_env->skip_cpu, task_on_rq_queued(p)); + fbt_env->skip_cpu, task_on_rq_queued(p), least_nr_cpu, + cpu_rq_runnable_cnt); } static inline unsigned long From 53ef4e9eedc44e7e6888b2688439147d23b4c8c6 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Sat, 19 Feb 2022 14:17:14 -0800 Subject: [PATCH 254/346] sched/walt: guarantee result from energy efficient cpu It is possible that walt_find_energy_efficient_cpu will return an error value, or a negative number for the cpu. In walt_lb_tick this was called without checking the return value. Update walt_find_energy_efficient_cpu to guarantee that it never returns a bad value, and that in all cases it returns prev_cpu if it cannot find an energy efficient cpu. Change-Id: Ifc08bdf42b8d54923f8a4ecc3ca8b2c0dba11f4f Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_cfs.c | 12 +++++++----- kernel/sched/walt/walt_lb.c | 8 ++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index c2bd0d064d18..d708710a3fc2 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2020-2022, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -788,7 +789,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, return prev_cpu; if (unlikely(!cpu_array)) - return -EPERM; + return prev_cpu; walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost, &energy_eval_needed); @@ -926,6 +927,9 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, rcu_read_unlock(); done: + if (best_energy_cpu < 0 || best_energy_cpu >= WALT_NR_CPUS) + best_energy_cpu = prev_cpu; + trace_sched_task_util(p, cpumask_bits(candidates)[0], best_energy_cpu, sync, fbt_env.need_idle, fbt_env.fastpath, start_t, uclamp_boost, start_cpu); @@ -934,7 +938,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, fail: rcu_read_unlock(); - return -EPERM; + return prev_cpu; } static void @@ -952,8 +956,6 @@ walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, p->wake_q_count = 0; *target_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, sync, sibling_count_hint); - if (unlikely(*target_cpu < 0)) - *target_cpu = prev_cpu; } static void walt_binder_low_latency_set(void *unused, struct task_struct *task, diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 9a0502a59e7b..ed6c2dc9781a 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2020-2021, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2020-2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -633,14 +633,10 @@ void walt_lb_tick(struct rq *rq) new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1); rcu_read_unlock(); - if (new_cpu < 0) - goto out_unlock; - new_wrq = (struct walt_rq *) cpu_rq(new_cpu)->android_vendor_data1; /* prevent active task migration to busy or same/lower capacity CPU */ - if (!available_idle_cpu(new_cpu) || - new_wrq->cluster->id <= prev_wrq->cluster->id) + if (!available_idle_cpu(new_cpu) || new_wrq->cluster->id <= prev_wrq->cluster->id) goto out_unlock; raw_spin_lock(&rq->__lock); From a63fa747f0eaa49ccbdba02818733f6a1e54f931 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 15 Feb 2022 13:19:10 -0800 Subject: [PATCH 255/346] sched/walt: rename to sched_get_cpu_util_pct Cleanup to give units to sched_get_cpu_util. Change-Id: If04a3b38d7ff0c6c0a53edb0ed6014e7698b5195 Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 13 +++++++------ kernel/sched/walt/sched_avg.c | 2 +- kernel/sched/walt/walt.h | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 0f856727a16e..52e78df1cd2d 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2014-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #define pr_fmt(fmt) "core_ctl: " fmt @@ -50,7 +51,7 @@ struct cluster_data { struct cpu_data { bool is_busy; - unsigned int busy; + unsigned int busy_pct; unsigned int cpu; bool not_preferred; struct cluster_data *cluster; @@ -328,7 +329,7 @@ static ssize_t show_global_state(const struct cluster_data *state, char *buf) "\tFirst CPU: %u\n", cluster->first_cpu); count += scnprintf(buf + count, PAGE_SIZE - count, - "\tBusy%%: %u\n", c->busy); + "\tBusy%%: %u\n", c->busy_pct); count += scnprintf(buf + count, PAGE_SIZE - count, "\tIs busy: %u\n", c->is_busy); count += scnprintf(buf + count, PAGE_SIZE - count, @@ -811,13 +812,13 @@ static bool eval_need(struct cluster_data *cluster) list_for_each_entry(c, &cluster->lru, sib) { bool old_is_busy = c->is_busy; - if (c->busy >= cluster->busy_up_thres[thres_idx] || + if (c->busy_pct >= cluster->busy_up_thres[thres_idx] || sched_cpu_high_irqload(c->cpu)) c->is_busy = true; - else if (c->busy < cluster->busy_down_thres[thres_idx]) + else if (c->busy_pct < cluster->busy_down_thres[thres_idx]) c->is_busy = false; - trace_core_ctl_set_busy(c->cpu, c->busy, old_is_busy, + trace_core_ctl_set_busy(c->cpu, c->busy_pct, old_is_busy, c->is_busy); need_cpus += c->is_busy; } @@ -984,7 +985,7 @@ void core_ctl_check(u64 window_start) if (!cluster || !cluster->inited) continue; - c->busy = sched_get_cpu_util(cpu); + c->busy_pct = sched_get_cpu_util_pct(cpu); } spin_unlock_irqrestore(&state_lock, flags); diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index cdf0e431498a..7fb3e0672e53 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -248,7 +248,7 @@ void sched_update_nr_prod(int cpu, int enq) /* * Returns the CPU utilization % in the last window. */ -unsigned int sched_get_cpu_util(int cpu) +unsigned int sched_get_cpu_util_pct(int cpu) { struct rq *rq = cpu_rq(cpu); u64 util; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index a6036d8da2bb..96ba533bc618 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -161,7 +161,7 @@ extern void walt_boost_init(void); extern int sched_pause_cpus(struct cpumask *pause_cpus); extern int sched_unpause_cpus(struct cpumask *unpause_cpus); -extern unsigned int sched_get_cpu_util(int cpu); +extern unsigned int sched_get_cpu_util_pct(int cpu); extern void sched_update_hyst_times(void); extern int sched_boost_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); From e27237ebec77f798e08f0facd80fa7ad4fe8047c Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Mon, 24 Jan 2022 20:04:49 -0800 Subject: [PATCH 256/346] sched/walt: Handle long running RT tasks Detect long running RT tasks by tapping into the scheduler_tick tracepoint, so that task can be detected while it is actively running on the CPU and cause panic on the same CPU to get relevant stack information for debug purposes. Change-Id: I205a5512ba61e1a92d64659b723e661e11c39dd1 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/sysctl.c | 12 +++++++ kernel/sched/walt/walt.h | 10 ++++++ kernel/sched/walt/walt_rt.c | 62 +++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index f52aa1b90f76..1f2ede153895 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -4,6 +4,7 @@ */ #include "walt.h" +#include static int neg_three = -3; static int three = 3; @@ -15,6 +16,7 @@ static int __maybe_unused two = 2; static int __maybe_unused four = 4; static int one_hundred = 100; static int one_thousand = 1000; +static int two_thousand = 2000; /* * CFS task prio range is [100 ... 139] @@ -67,6 +69,7 @@ unsigned int sysctl_sched_suppress_region2; unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000; unsigned int sysctl_sched_asymcap_boost; +unsigned int sysctl_sched_long_running_rt_task_ms = 1200; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -869,6 +872,15 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_long_running_rt_task_ms", + .data = &sysctl_sched_long_running_rt_task_ms, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_long_running_rt_task_ms_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_thousand, + }, { } }; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 96ba533bc618..ba34a2eea0d9 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -207,6 +207,7 @@ extern unsigned int sysctl_sched_boost_on_input; extern unsigned int sysctl_sched_user_hint; extern unsigned int sysctl_sched_conservative_pl; extern unsigned int sysctl_sched_hyst_min_coloc_ns; +extern unsigned int sysctl_sched_long_running_rt_task_ms; #define WALT_MANY_WAKEUP_DEFAULT 1000 extern unsigned int sysctl_sched_many_wakeup_threshold; @@ -878,6 +879,15 @@ static inline bool walt_fair_task(struct task_struct *p) return p->prio >= MAX_RT_PRIO && !is_idle_task(p); } +extern void rt_task_arrival_marker(void *unused, bool preempt, + struct task_struct *prev, struct task_struct *next); + +extern void long_running_rt_task_notifier(void *unused, struct rq *rq); + +extern int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); + #define WALT_MVP_SLICE 3000000U #define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 594990a18291..c8c8a5f690a7 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -8,7 +8,69 @@ #include "walt.h" #include "trace.h" +#define MSEC_TO_NSEC (1000 * 1000) + static DEFINE_PER_CPU(cpumask_var_t, walt_local_cpu_mask); +static DEFINE_PER_CPU(u64, rt_task_arrival_time); +static bool long_running_rt_task_trace_rgstrd; + +void rt_task_arrival_marker(void *unused, bool preempt, + struct task_struct *prev, struct task_struct *next) +{ + unsigned int cpu = raw_smp_processor_id(); + + if (rt_task(next)) + per_cpu(rt_task_arrival_time, cpu) = rq_clock_task(this_rq()); + else + per_cpu(rt_task_arrival_time, cpu) = 0; +} + +void long_running_rt_task_notifier(void *unused, struct rq *rq) +{ + struct task_struct *curr = rq->curr; + unsigned int cpu = raw_smp_processor_id(); + + if (!sysctl_sched_long_running_rt_task_ms) + return; + + if (!per_cpu(rt_task_arrival_time, cpu)) + return; + + if (rq_clock_task(rq) - + per_cpu(rt_task_arrival_time, cpu) + > sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC) { + printk_deferred("RT task %s (%d) runtime > %u now=%llu task arrival time=%llu runtime=%llu\n", + curr->comm, curr->pid, + sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC, + rq_clock_task(rq), + per_cpu(rt_task_arrival_time, cpu), + rq_clock_task(rq) - + per_cpu(rt_task_arrival_time, cpu)); + BUG(); + } +} + +int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + static DEFINE_MUTEX(mutex); + + mutex_lock(&mutex); + + ret = proc_douintvec_minmax(table, write, buffer, lenp, ppos); + + if (write && !long_running_rt_task_trace_rgstrd) { + register_trace_sched_switch(rt_task_arrival_marker, NULL); + register_trace_android_vh_scheduler_tick(long_running_rt_task_notifier, NULL); + long_running_rt_task_trace_rgstrd = true; + } + + mutex_unlock(&mutex); + + return ret; +} static void walt_rt_energy_aware_wake_cpu(struct task_struct *task, struct cpumask *lowest_mask, int ret, int *best_cpu) From e20d764ffad6b518a201c66a22dc51b370a6fa0f Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Mon, 28 Feb 2022 21:04:33 -0800 Subject: [PATCH 257/346] sched/walt: Deprecate RT task throttling Deprecate RT thottling mechanism in lieu of new feature that handles long running RT tasks. Change-Id: I1137c6deefc61bc311deac66788a06201e64146e Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 18 ------------------ kernel/sched/walt/walt_debug.c | 17 ----------------- 2 files changed, 35 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index e81198c54f61..53625f2ae91a 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4210,23 +4210,6 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } -static void dump_throttled_rt_tasks(void *unused, int cpu, u64 clock, - ktime_t rt_period, u64 rt_runtime, s64 rt_period_timer_expires) -{ - printk_deferred("sched: RT throttling activated for cpu %d\n", cpu); - printk_deferred("rt_period_timer: expires=%lld now=%llu rt_time=%llu runtime=%llu period=%llu\n", - rt_period_timer_expires, ktime_get_ns(), - task_rq(current)->rt.rt_time, rt_runtime, rt_period); - printk_deferred("potential CPU hogs:\n"); -#ifdef CONFIG_SCHED_INFO - if (sched_info_on()) - printk_deferred("current %s (%d) is running for %llu nsec\n", - current->comm, current->pid, - clock - current->sched_info.last_arrival); -#endif - BUG_ON(sysctl_sched_bug_on_rt_throttle); -} - static void android_rvh_set_cpus_allowed_ptr_locked(void *unused, const struct cpumask *cpu_valid_mask, const struct cpumask *new_mask, @@ -4288,7 +4271,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); - register_trace_android_vh_dump_throttled_rt_tasks(dump_throttled_rt_tasks, NULL); register_trace_android_rvh_set_cpus_allowed_ptr_locked( android_rvh_set_cpus_allowed_ptr_locked, NULL); register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL); diff --git a/kernel/sched/walt/walt_debug.c b/kernel/sched/walt/walt_debug.c index 59f6edbed53b..26458c7ee307 100644 --- a/kernel/sched/walt/walt_debug.c +++ b/kernel/sched/walt/walt_debug.c @@ -11,23 +11,6 @@ #include "walt.h" #include "walt_debug.h" -static void dump_throttled_rt_tasks(void *unused, int cpu, u64 clock, - ktime_t rt_period, u64 rt_runtime, s64 rt_period_timer_expires) -{ - printk_deferred("sched: RT throttling activated for cpu %d\n", cpu); - printk_deferred("rt_period_timer: expires=%lld now=%llu rt_time=%llu runtime=%llu period=%llu\n", - rt_period_timer_expires, ktime_get_ns(), - task_rq(current)->rt.rt_time, rt_runtime, rt_period); - printk_deferred("potential CPU hogs:\n"); -#ifdef CONFIG_SCHED_INFO - if (sched_info_on()) - printk_deferred("current %s (%d) is running for %llu nsec\n", - current->comm, current->pid, - clock - current->sched_info.last_arrival); -#endif - BUG(); -} - static void android_rvh_schedule_bug(void *unused, void *unused2) { BUG(); From b71791171e1559014879692b2f10f349dbe8602e Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 1 Mar 2022 10:41:41 -0800 Subject: [PATCH 258/346] sched/walt: do not perform unnecessary migrations It is possible that the fallback destination cpu is the same as the current cpu. In this case, this is an inappropriate migration request, causing a double lock unbalance error in walt_detach_task. Prevent this case by preventing a task that is not going to be migrated from being passed to __migrate_task, and instead detach the task and add it to the list to be added back later. Change-Id: I26f57bf52ca9a7bfe221d02d9ccb353d8c8d27ff Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_halt.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index a8cf2ed2de8e..4b83f7e71d7c 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -146,13 +146,22 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) /* Find suitable destination for @next */ dest_cpu = select_fallback_rq(dead_rq->cpu, next); - rq = __migrate_task(rq, rf, next, dest_cpu); - if (rq != dead_rq) { - rq_unlock(rq, rf); - rq = dead_rq; - *rf = orf; - rq_relock(rq, rf); + + if (cpu_of(rq) != dest_cpu) { + /* only perform a required migration */ + rq = __migrate_task(rq, rf, next, dest_cpu); + + if (rq != dead_rq) { + rq_unlock(rq, rf); + rq = dead_rq; + *rf = orf; + rq_relock(rq, rf); + } + } else { + detach_one_task_core(next, rq, &percpu_kthreads); + num_pinned_kthreads += 1; } + raw_spin_unlock(&next->pi_lock); } From 690109d81e7f8efe6218028a410a2ba5b4a4465f Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 16 Feb 2022 19:23:31 -0800 Subject: [PATCH 259/346] sched/walt_cfs: set_next_entity hierarchy Currently, in the simple case when replace_next_task_fair() hook picks a task it does not call set_next_entity on the newly picked task. Add that. Change-Id: I77074cf9a38ce1d2b659d8845cbaae362bd33441 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_cfs.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index d708710a3fc2..89019245b738 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1271,6 +1271,16 @@ static void walt_cfs_check_preempt_wakeup(void *unused, struct rq *rq, struct ta trace_walt_cfs_mvp_wakeup_preempt(p, wts_p, walt_cfs_mvp_task_limit(p)); } +#ifdef CONFIG_FAIR_GROUP_SCHED +/* Walk up scheduling entities hierarchy */ +#define for_each_sched_entity(se) \ + for (; se; se = se->parent) +#else /* !CONFIG_FAIR_GROUP_SCHED */ +#define for_each_sched_entity(se) \ + for (; se; se = NULL) +#endif + +extern void set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se); static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple, struct task_struct *prev) @@ -1278,6 +1288,7 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts; struct task_struct *mvp; + struct cfs_rq *cfs_rq; if (unlikely(walt_disabled)) return; @@ -1302,6 +1313,17 @@ static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct *se = &mvp->se; *repick = true; + if (simple) { + for_each_sched_entity((*se)) { + /* + * TODO If CFS_BANDWIDTH is enabled, we might pick + * from a throttled cfs_rq + */ + cfs_rq = cfs_rq_of(*se); + set_next_entity(cfs_rq, *se); + } + } + if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || (*p)->on_rq == TASK_ON_RQ_MIGRATING || (*p)->cpu != cpu_of(rq))) From 4d68dcf27d084050ace5be529a65e2c6d6483918 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 1 Mar 2022 12:52:44 -0800 Subject: [PATCH 260/346] Revert "sched/walt_cfs: temporarily disable MVP feature" This reverts commit 89fc117b538f9687ace7b399e12934cdfe601d66. WALT was missing set_next_entity() calls on mvp task hierarchy. Now that is addressed, let us revert the disabling of mvp feature. Change-Id: I50763e354fb1b885c3186078fa6db499b4a975e1 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_cfs.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 89019245b738..224d26bdde61 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1020,7 +1020,6 @@ static void binder_restore_priority_hook(void *data, */ static inline int walt_get_mvp_task_prio(struct task_struct *p) { -#if 0 if (per_task_boost(p) == TASK_BOOST_STRICT_MAX) return WALT_TASK_BOOST_MVP; @@ -1030,7 +1029,7 @@ static inline int walt_get_mvp_task_prio(struct task_struct *p) if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p) || walt_pipeline_low_latency_task(p)) return WALT_RTG_MVP; -#endif + return WALT_NOT_MVP; } From dcc31fca0c404eb6241f3f6eaf220dc2bf38341b Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 10 Jan 2022 14:17:03 -0800 Subject: [PATCH 261/346] sched: walt: use the is_cpu_allowed tracehook to restrict cpus is_cpu_allowed is critical to select_fallback_rq, select_task_rq, and __migrate_task, to ensure that only valid cpus are chosen for the execution of a task. Register for the trace hook, and restrict cpus selected by is_cpu_allowed to non-halted cpus. Change-Id: I72b2e90230603a8f97ce1da7014d72c0ebdee97e Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 53625f2ae91a..2914db7dcb00 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2016-2022, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -4244,6 +4245,14 @@ static void android_rvh_rto_next_cpu(void *unused, } } +static void android_rvh_is_cpu_allowed(void *unused, int cpu, bool *allowed) +{ + if (unlikely(walt_disabled)) + return; + + *allowed = !cpumask_test_cpu(cpu, cpu_halt_mask); +} + static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4274,6 +4283,7 @@ static void register_walt_hooks(void) register_trace_android_rvh_set_cpus_allowed_ptr_locked( android_rvh_set_cpus_allowed_ptr_locked, NULL); register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL); + register_trace_android_rvh_is_cpu_allowed(android_rvh_is_cpu_allowed, NULL); } atomic64_t walt_irq_work_lastq_ws; From 1284c21e067921777decc21d1dc857b4ff4f1b40 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 12 Jan 2022 17:23:55 -0800 Subject: [PATCH 262/346] sched/walt: avoid halted cpus in get_nohz_timer_target Halted cpus can still get assigned a timer through get_nohz_timer_target, as long as they are not isolcpus. Remove halted cpus from selection by get_nohz_timer_target, while maintaining the preference for idle cpus, and in-domain cpus. Change-Id: I2619b6cedc831c64857ed71461067b66873cb61a Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_halt.c | 60 ++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 4b83f7e71d7c..cf850e87750f 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -1,9 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021-2021 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include +#include +#include #include #include "trace.h" @@ -379,8 +382,63 @@ int walt_resume_cpus(struct cpumask *cpus) } EXPORT_SYMBOL(walt_resume_cpus); +static void android_rvh_get_nohz_timer_target(void *unused, int *cpu, bool *done) +{ + int i, default_cpu = -1; + struct sched_domain *sd; + cpumask_t unhalted; + + *done = true; + + if (housekeeping_cpu(*cpu, HK_FLAG_TIMER) && !cpu_halted(*cpu)) { + if (!available_idle_cpu(*cpu)) + return; + default_cpu = *cpu; + } + + rcu_read_lock(); + for_each_domain(*cpu, sd) { + for_each_cpu_and(i, sched_domain_span(sd), + housekeeping_cpumask(HK_FLAG_TIMER)) { + if (*cpu == i) + continue; + + if (!available_idle_cpu(i) && !cpu_halted(i)) { + *cpu = i; + goto unlock; + } + } + } + + if (default_cpu == -1) { + cpumask_complement(&unhalted, cpu_halt_mask); + for_each_cpu_and(i, &unhalted, + housekeeping_cpumask(HK_FLAG_TIMER)) { + if (*cpu == i) + continue; + + if (!available_idle_cpu(i)) { + *cpu = i; + goto unlock; + } + } + + /* no active, non-halted, not-idle, choose any */ + default_cpu = cpumask_any(&unhalted); + + if (unlikely(default_cpu >= nr_cpu_ids)) + goto unlock; + } + + *cpu = default_cpu; +unlock: + rcu_read_unlock(); +} + void walt_halt_init(void) { + register_trace_android_rvh_get_nohz_timer_target(android_rvh_get_nohz_timer_target, NULL); + } #endif /* CONFIG_HOTPLUG_CPU */ From 4be9ca4e32e8cd2b05637e16f8b345a17506df17 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 9 Apr 2021 08:34:31 -0700 Subject: [PATCH 263/346] sched: walt: Use rq_clock instead of sched_ktime_get Reduce the calls to clock hardware by utilizing cached rq_clock() values. Also track the latest clock seen on runqueue to workaround stale rq_clock() snapshots. Change-Id: Id4d905c36325ca1031231b55c697514dc4505658 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 7 +- kernel/sched/walt/walt.c | 127 +++++++++++++++++++------------ kernel/sched/walt/walt.h | 7 +- kernel/sched/walt/walt_lb.c | 21 ++++- 4 files changed, 103 insertions(+), 59 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 53125b982600..d05b38c4890a 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -5,6 +5,7 @@ * * Copyright (C) 2016, Intel Corporation * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -413,7 +414,7 @@ static void waltgov_work(struct kthread_work *work) raw_spin_lock_irqsave(&wg_policy->update_lock, flags); freq = wg_policy->next_freq; waltgov_track_cycles(wg_policy, wg_policy->policy->cur, - ktime_get_ns()); + walt_sched_clock()); raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); mutex_lock(&wg_policy->work_lock); @@ -972,14 +973,14 @@ static void waltgov_limits(struct cpufreq_policy *policy) mutex_lock(&wg_policy->work_lock); raw_spin_lock_irqsave(&wg_policy->update_lock, flags); waltgov_track_cycles(wg_policy, wg_policy->policy->cur, - ktime_get_ns()); + walt_sched_clock()); raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); cpufreq_policy_apply_limits(policy); mutex_unlock(&wg_policy->work_lock); } else { raw_spin_lock_irqsave(&wg_policy->update_lock, flags); freq = policy->cur; - now = ktime_get_ns(); + now = walt_sched_clock(); /* * cpufreq_driver_resolve_freq() has a clamp, so we do not need diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2914db7dcb00..4270d0d3d4c7 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -51,9 +50,8 @@ const char *migrate_type_names[] = { #define NEW_TASK_ACTIVE_TIME 100000000 unsigned int sysctl_sched_user_hint; - -static ktime_t ktime_last; -static bool walt_ktime_suspended; +static u64 sched_clock_last; +static bool walt_clock_suspended; static bool use_cycle_counter; static DEFINE_MUTEX(cluster_lock); @@ -79,6 +77,30 @@ unsigned int __read_mostly sched_init_task_load_windows; */ unsigned int __read_mostly sched_load_granule; +u64 walt_sched_clock(void) +{ + if (unlikely(walt_clock_suspended)) + return sched_clock_last; + return sched_clock(); +} + +static void walt_resume(void) +{ + walt_clock_suspended = false; +} + +static int walt_suspend(void) +{ + sched_clock_last = sched_clock(); + walt_clock_suspended = true; + return 0; +} + +static struct syscore_ops walt_syscore_ops = { + .resume = walt_resume, + .suspend = walt_suspend +}; + /* *@boost:should be 0,1,2. *@period:boost time based on ms units. @@ -92,7 +114,7 @@ int set_task_boost(int boost, u64 period) if (boost) { wts->boost = boost; wts->boost_period = (u64)period * 1000 * 1000; - wts->boost_expires = sched_clock() + wts->boost_period; + wts->boost_expires = walt_sched_clock() + wts->boost_period; } else { wts->boost = 0; wts->boost_expires = 0; @@ -102,30 +124,6 @@ int set_task_boost(int boost, u64 period) } EXPORT_SYMBOL(set_task_boost); -u64 walt_ktime_get_ns(void) -{ - if (unlikely(walt_ktime_suspended)) - return ktime_to_ns(ktime_last); - return ktime_get_ns(); -} - -static void walt_resume(void) -{ - walt_ktime_suspended = false; -} - -static int walt_suspend(void) -{ - ktime_last = ktime_get(); - walt_ktime_suspended = true; - return 0; -} - -static struct syscore_ops walt_syscore_ops = { - .resume = walt_resume, - .suspend = walt_suspend -}; - static inline void acquire_rq_locks_irqsave(const cpumask_t *cpus, unsigned long *flags) { @@ -153,6 +151,21 @@ static inline void release_rq_locks_irqrestore(const cpumask_t *cpus, local_irq_restore(*flags); } +static inline u64 walt_rq_clock(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (unlikely(walt_clock_suspended)) + return sched_clock_last; + + lockdep_assert_held(&rq->__lock); + + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + + return max(rq_clock(rq), wrq->latest_clock); +} + static unsigned int walt_cpu_high_irqload; static __read_mostly unsigned int sched_ravg_hist_size = RAVG_HIST_SIZE_MAX; @@ -267,7 +280,7 @@ void walt_dump(void) int cpu; printk_deferred("============ WALT RQ DUMP START ==============\n"); - printk_deferred("Sched ktime_get: %llu\n", walt_ktime_get_ns()); + printk_deferred("Sched clock: %llu\n", walt_sched_clock()); printk_deferred("Time last window changed=%lu\n", sched_ravg_window_change_time); for_each_online_cpu(cpu) @@ -382,6 +395,12 @@ update_window_start(struct rq *rq, u64 wallclock, int event) struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; u64 old_window_start = wrq->window_start; + if (wallclock < wrq->latest_clock) { + printk_deferred("WALT-BUG CPU%d; wallclock=%llu(0x%llx) is lesser than latest_clock=%llu(0x%llx)", + rq->cpu, wallclock, wallclock, wrq->latest_clock, + wrq->latest_clock); + WALT_PANIC(1); + } delta = wallclock - wrq->window_start; if (delta < 0) { printk_deferred("WALT-BUG CPU%d; wallclock=%llu(0x%llx) is lesser than window_start=%llu(0x%llx)", @@ -389,6 +408,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) wrq->window_start, wrq->window_start); WALT_PANIC(1); } + wrq->latest_clock = wallclock; if (delta < sched_ravg_window) return old_window_start; @@ -476,7 +496,7 @@ static void walt_sched_account_irqstart(int cpu, struct task_struct *curr) /* We're here without rq->lock held, IRQ disabled */ raw_spin_lock(&rq->__lock); - update_task_cpu_cycles(curr, cpu, walt_ktime_get_ns()); + update_task_cpu_cycles(curr, cpu, walt_sched_clock()); raw_spin_unlock(&rq->__lock); } @@ -488,7 +508,7 @@ static void walt_sched_account_irqend(int cpu, struct task_struct *curr, u64 del unsigned long flags; raw_spin_lock_irqsave(&rq->__lock, flags); - walt_update_task_ravg(curr, rq, IRQ_UPDATE, walt_ktime_get_ns(), delta); + walt_update_task_ravg(curr, rq, IRQ_UPDATE, walt_sched_clock(), delta); raw_spin_unlock_irqrestore(&rq->__lock, flags); } @@ -981,7 +1001,7 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) if (pstate == TASK_WAKING) double_rq_lock(src_rq, dest_rq); - wallclock = walt_ktime_get_ns(); + wallclock = walt_sched_clock(); lockdep_assert_held(&src_rq->__lock); lockdep_assert_held(&dest_rq->__lock); @@ -2283,7 +2303,7 @@ static void mark_task_starting(struct task_struct *p) struct rq *rq = task_rq(p); struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - wallclock = walt_ktime_get_ns(); + wallclock = walt_rq_clock(rq); wts->mark_start = wts->last_wake_ts = wallclock; wts->last_enqueued_ts = wallclock; update_task_cpu_cycles(p, cpu_of(rq), wallclock); @@ -2672,8 +2692,7 @@ static int cpufreq_notifier_trans(struct notifier_block *nb, struct rq *rq = cpu_rq(j); raw_spin_lock_irqsave(&rq->__lock, flags); - walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, - walt_ktime_get_ns(), 0); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, walt_sched_clock(), 0); raw_spin_unlock_irqrestore(&rq->__lock, flags); } @@ -2774,7 +2793,7 @@ static void _set_preferred_cluster(struct walt_related_thread_group *grp) goto out; } - wallclock = walt_ktime_get_ns(); + wallclock = walt_sched_clock(); /* * wakeup of two or more related tasks could race with each other and @@ -2840,8 +2859,11 @@ static int update_preferred_cluster(struct walt_related_thread_group *grp, * Update if task's load has changed significantly or a complete window * has passed since we last updated preference */ - if (abs(new_load - old_load) > sched_ravg_window / 4 || - walt_ktime_get_ns() - grp->last_update > sched_ravg_window) + + if (abs(new_load - old_load) > sched_ravg_window / 4) + return 1; + + if (walt_sched_clock() - grp->last_update > sched_ravg_window) return 1; return 0; @@ -3172,7 +3194,7 @@ static void transfer_busy_time(struct rq *rq, struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - wallclock = walt_ktime_get_ns(); + wallclock = walt_sched_clock(); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); @@ -3332,7 +3354,7 @@ bool is_rtgb_active(void) u64 get_rtgb_active_time(void) { struct walt_related_thread_group *grp; - u64 now = walt_ktime_get_ns(); + u64 now = walt_sched_clock(); grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); @@ -3407,7 +3429,7 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc unsigned long flags; struct walt_rq *wrq; - wc = walt_ktime_get_ns(); + wc = walt_sched_clock(); walt_load_reported_window = atomic64_read(&walt_irq_work_lastq_ws); for_each_sched_cluster(cluster) { u64 aggr_grp_load = 0; @@ -3520,7 +3542,7 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc wrq = (struct walt_rq *) this_rq()->android_vendor_data1; if ((sched_ravg_window != new_sched_ravg_window) && (wc < wrq->window_start + new_sched_ravg_window)) { - sched_ravg_window_change_time = walt_ktime_get_ns(); + sched_ravg_window_change_time = walt_sched_clock(); trace_sched_ravg_window_change(sched_ravg_window, new_sched_ravg_window, sched_ravg_window_change_time); @@ -3635,7 +3657,7 @@ void walt_fill_ta_data(struct core_ctl_notif_data *data) goto fill_util; } - wallclock = walt_ktime_get_ns(); + wallclock = walt_sched_clock(); list_for_each_entry(wts, &grp->tasks, grp_list) { if (wts->mark_start < wallclock - @@ -3923,7 +3945,7 @@ static void android_rvh_flush_task(void *unused, struct task_struct *p) static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_struct *p) { - u64 wallclock = walt_ktime_get_ns(); + u64 wallclock; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; bool double_enqueue = false; @@ -3932,6 +3954,8 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st lockdep_assert_held(&rq->__lock); + wallclock = walt_rq_clock(rq); + if (p->cpu != cpu_of(rq)) WALT_BUG(WALT_BUG_UPSTREAM, p, "enqueuing on rq %d when task->cpu is %d\n", cpu_of(rq), p->cpu); @@ -3999,7 +4023,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st wts->prev_on_rq = 2; if (p == wrq->ed_task) - is_ed_task_present(rq, walt_ktime_get_ns(), p); + is_ed_task_present(rq, walt_rq_clock(rq), p); sched_update_nr_prod(rq->cpu, -1); @@ -4065,7 +4089,7 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) return; rq_lock_irqsave(rq, &rf); old_load = task_load(p); - wallclock = walt_ktime_get_ns(); + wallclock = walt_sched_clock(); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); note_task_waking(p, wallclock); @@ -4101,7 +4125,7 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) return; set_window_start(rq); - wallclock = walt_ktime_get_ns(); + wallclock = walt_rq_clock(rq); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); @@ -4130,12 +4154,15 @@ static void android_vh_scheduler_tick(void *unused, struct rq *rq) static void android_rvh_schedule(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq) { - u64 wallclock = walt_ktime_get_ns(); + u64 wallclock; struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; if (unlikely(walt_disabled)) return; + + wallclock = walt_rq_clock(rq); + if (likely(prev != next)) { if (!prev->on_rq) wts->last_sleep_ts = wallclock; @@ -4305,7 +4332,7 @@ static int walt_init_stop_handler(void *data) init_existing_task_load(p); } while_each_thread(g, p); - window_start_ns = ktime_get_ns(); + window_start_ns = walt_sched_clock(); nr_windows = div64_u64(window_start_ns, sched_ravg_window); window_start_ns = (u64)nr_windows * (u64)sched_ravg_window; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index ba34a2eea0d9..8f059fe52197 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -121,6 +121,7 @@ struct walt_rq { u64 last_cc_update; u64 cycles; struct list_head mvp_tasks; + u64 latest_clock; }; struct walt_sched_cluster { @@ -167,7 +168,7 @@ extern int sched_boost_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); extern int sched_busy_hyst_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); -extern u64 walt_ktime_get_ns(void); +extern u64 walt_sched_clock(void); extern void walt_init_tg(struct task_group *tg); extern void walt_init_topapp_tg(struct task_group *tg); extern void walt_init_foreground_tg(struct task_group *tg); @@ -362,7 +363,7 @@ static inline void waltgov_run_callback(struct rq *rq, unsigned int flags) cb = rcu_dereference_sched(*per_cpu_ptr(&waltgov_cb_data, cpu_of(rq))); if (cb) - cb->func(cb, walt_ktime_get_ns(), flags); + cb->func(cb, walt_sched_clock(), flags); } extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load); @@ -607,7 +608,7 @@ static inline int per_task_boost(struct task_struct *p) struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (wts->boost_period) { - if (sched_clock() > wts->boost_expires) { + if (walt_sched_clock() > wts->boost_expires) { wts->boost_period = 0; wts->boost_expires = 0; wts->boost = 0; diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index ed6c2dc9781a..e8d182d90f18 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2020-2022, Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -146,7 +146,12 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) if (!is_min_cluster_cpu(src_cpu)) return; - wc = walt_ktime_get_ns(); + /* + * Use src_rq->clock directly instead of rq_clock() since + * we do not have the rq lock and + * src_rq->clock was updated in the tick callpath. + */ + wc = src_rq->clock; for_each_possible_cpu(i) { struct rq *rq = cpu_rq(i); @@ -681,6 +686,7 @@ static bool walt_balance_rt(struct rq *this_rq) struct task_struct *p; struct walt_task_struct *wts; bool pulled = false; + u64 wallclock; /* can't help if this has a runnable RT */ if (sched_rt_runnable(this_rq)) @@ -713,7 +719,16 @@ static bool walt_balance_rt(struct rq *this_rq) goto unlock; wts = (struct walt_task_struct *) p->android_vendor_data1; - if (walt_ktime_get_ns() - wts->last_wake_ts < WALT_RT_PULL_THRESHOLD_NS) + + /* + * Use rq->clock directly instead of rq_clock() since + * rq->clock was updated recently in the __schedule() -> pick_next_task() callpath. + * Time lost in grabbing rq locks will likely be corrected via max. + */ + wallclock = max(this_rq->clock, src_rq->clock); + + if (wallclock > wts->last_wake_ts && + wallclock - wts->last_wake_ts < WALT_RT_PULL_THRESHOLD_NS) goto unlock; pulled = true; From ef3ba55b554aebfe30ba8d5dfb8b3d6affee13e2 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 9 Feb 2022 14:09:30 -0800 Subject: [PATCH 264/346] sched: walt: Remove finding historical runtimes in pred When determining predictive demand, we currently look at the history samples in our past 8 windows that fall under the bucket range for which we are evaluating. However, this level of precision is not required, and can be dropped in favor of reduction of overheads. Change-Id: Icef5d23b1574c625ef1a6d6228603d2b18aa8a40 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4270d0d3d4c7..649857de467d 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1188,9 +1188,7 @@ static inline int busy_to_bucket(u32 normalized_rt) * A new predicted busy time is returned for task @p based on @runtime * passed in. The function searches through buckets that represent busy * time equal to or bigger than @runtime and attempts to find the bucket - * to use for prediction. Once found, it searches through historical busy - * time and returns the latest that falls into the bucket. If no such busy - * time exists, it returns the medium of that bucket. + * to use for prediction. Once found, it returns the midpoint of that bucket. */ static u32 get_pred_busy(struct task_struct *p, int start, u32 runtime) @@ -1198,7 +1196,6 @@ static u32 get_pred_busy(struct task_struct *p, int i; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u8 *buckets = wts->busy_buckets; - u32 *hist = wts->sum_history; u32 dmin, dmax; u64 cur_freq_runtime = 0; int first = NUM_BUSY_BUCKETS, final; @@ -1232,24 +1229,11 @@ static u32 get_pred_busy(struct task_struct *p, } dmax = mult_frac(final + 1, max_task_load(), NUM_BUSY_BUCKETS); - /* - * search through runtime history and return first runtime that falls - * into the range of predicted bucket. - */ - for (i = 0; i < sched_ravg_hist_size; i++) { - if (hist[i] >= dmin && hist[i] < dmax) { - ret = hist[i]; - break; - } - } - /* no historical runtime within bucket found, use average of the bin */ - if (ret < dmin) - ret = (dmin + dmax) / 2; /* * when updating in middle of a window, runtime could be higher * than all recorded history. Always predict at least runtime. */ - ret = max(runtime, ret); + ret = max(runtime, (dmin + dmax) / 2); out: trace_sched_update_pred_demand(p, runtime, mult_frac((unsigned int)cur_freq_runtime, 100, From a8e14cc9634a82dd198a5aaa7a315782559cf34a Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 9 Feb 2022 11:30:06 -0800 Subject: [PATCH 265/346] sched: walt: Introduce bucket bitmask In get_pred_busy, we attempt to find the closing bucket, which is nothing but the first bucket which is filled after the start bucket. We can optimize this computation by using a bitmask of buckets. Change-Id: I377be499fdd4876d5d3a43e8ebb34a489e6ae17b Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 1 + kernel/sched/walt/walt.c | 31 +++++++++++++++---------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index ce8e5bcee7cf..15860b721795 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -92,6 +92,7 @@ struct walt_task_struct { u32 curr_window, prev_window; u32 pred_demand; u8 busy_buckets[NUM_BUSY_BUCKETS]; + u16 bucket_bitmask; u16 demand_scaled; u16 pred_demand_scaled; u64 active_time; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 649857de467d..58f8efc96cf7 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1135,7 +1135,7 @@ static void set_window_start(struct rq *rq) * decayed. The rate of increase and decay could be different based * on current count in the bucket. */ -static inline void bucket_increase(u8 *buckets, int idx) +static inline void bucket_increase(u8 *buckets, u16 *bucket_bitmask, int idx) { int i, step; @@ -1143,8 +1143,10 @@ static inline void bucket_increase(u8 *buckets, int idx) if (idx != i) { if (buckets[i] > DEC_STEP) buckets[i] -= DEC_STEP; - else + else { buckets[i] = 0; + *bucket_bitmask &= ~BIT_MASK(i); + } } else { step = buckets[i] >= CONSISTENT_THRES ? INC_STEP_BIG : INC_STEP; @@ -1152,6 +1154,7 @@ static inline void bucket_increase(u8 *buckets, int idx) buckets[i] = U8_MAX; else buckets[i] += step; + *bucket_bitmask |= BIT_MASK(i); } } } @@ -1191,27 +1194,23 @@ static inline int busy_to_bucket(u32 normalized_rt) * to use for prediction. Once found, it returns the midpoint of that bucket. */ static u32 get_pred_busy(struct task_struct *p, - int start, u32 runtime) + int start, u32 runtime, u16 bucket_bitmask) { - int i; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - u8 *buckets = wts->busy_buckets; u32 dmin, dmax; u64 cur_freq_runtime = 0; - int first = NUM_BUSY_BUCKETS, final; + int first = NUM_BUSY_BUCKETS, final = NUM_BUSY_BUCKETS; u32 ret = runtime; + u16 next_mask = bucket_bitmask >> start; /* skip prediction for new tasks due to lack of history */ if (unlikely(is_new_task(p))) goto out; /* find minimal bucket index to pick */ - for (i = start; i < NUM_BUSY_BUCKETS; i++) { - if (buckets[i]) { - first = i; - break; - } - } + if (next_mask) + first = ffs(next_mask) - 1 + start; + /* if no higher buckets are filled, predict runtime */ if (first >= NUM_BUSY_BUCKETS) goto out; @@ -1249,7 +1248,7 @@ static inline u32 calc_pred_demand(struct task_struct *p) return wts->pred_demand; return get_pred_busy(p, busy_to_bucket(wts->curr_window), - wts->curr_window); + wts->curr_window, wts->bucket_bitmask); } /* @@ -1809,8 +1808,8 @@ static inline u32 predict_and_update_buckets( struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; bidx = busy_to_bucket(runtime); - pred_demand = get_pred_busy(p, bidx, runtime); - bucket_increase(wts->busy_buckets, bidx); + pred_demand = get_pred_busy(p, bidx, runtime, wts->bucket_bitmask); + bucket_increase(wts->busy_buckets, &wts->bucket_bitmask, bidx); return pred_demand; } @@ -2237,7 +2236,7 @@ static void init_new_task_load(struct task_struct *p) for (i = 0; i < NUM_BUSY_BUCKETS; ++i) wts->busy_buckets[i] = 0; - + wts->bucket_bitmask = 0; wts->cpu_cycles = 0; memset(wts->curr_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); From b53494ca462d2bd716d72d6dec3676c1a7ac75c5 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 9 Feb 2022 13:04:59 -0800 Subject: [PATCH 266/346] sched: walt: Use pred_demand_scaled for all predictive ops Firstly, we change the bucket size to 16. This has two effects: firstly, it helps improve power by predicting less aggressive midpoint values. Secondly, it allows us optimize multiplication/division operations by using bitshifts. To enable these further overhead optimizations, translate pred_demand to a 1024 unit value. Change-Id: Ieb00f5bbce8aedefc5a023ce1e354dbfb0055543 Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 8 +++-- kernel/sched/walt/trace.h | 55 ++++++++++++++++------------ kernel/sched/walt/walt.c | 73 +++++++++++++++----------------------- kernel/sched/walt/walt.h | 7 ---- 4 files changed, 65 insertions(+), 78 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 15860b721795..08777104a28a 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -32,7 +32,9 @@ enum task_boost_type { #define WALT_NR_CPUS 8 #define RAVG_HIST_SIZE_MAX 8 -#define NUM_BUSY_BUCKETS 10 +/* wts->bucket_bitmask needs to be updated if NUM_BUSY_BUCKETS > 16 */ +#define NUM_BUSY_BUCKETS 16 +#define NUM_BUSY_BUCKETS_SHIFT 4 struct walt_related_thread_group { int id; @@ -73,7 +75,8 @@ struct walt_task_struct { * * 'prev_window' represents the sum of all entries in prev_window_cpu * - * 'pred_demand' represents task's current predicted cpu busy time + * 'pred_demand_scaled' represents task's current predicted cpu busy time + * in terms of 1024 units * * 'busy_buckets' groups historical busy time into different buckets * used for prediction @@ -90,7 +93,6 @@ struct walt_task_struct { u32 curr_window_cpu[WALT_NR_CPUS]; u32 prev_window_cpu[WALT_NR_CPUS]; u32 curr_window, prev_window; - u32 pred_demand; u8 busy_buckets[NUM_BUSY_BUCKETS]; u16 bucket_bitmask; u16 demand_scaled; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 925529525709..26c38c06214c 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -24,39 +24,47 @@ extern const char *task_event_names[]; TRACE_EVENT(sched_update_pred_demand, - TP_PROTO(struct task_struct *p, u32 runtime, int pct, - unsigned int pred_demand, struct walt_task_struct *wts), + TP_PROTO(struct task_struct *p, u32 runtime, + unsigned int pred_demand_scaled, int start, + int first, int final, struct walt_task_struct *wts), - TP_ARGS(p, runtime, pct, pred_demand, wts), + TP_ARGS(p, runtime, pred_demand_scaled, start, first, final, wts), TP_STRUCT__entry( __array(char, comm, TASK_COMM_LEN) __field(pid_t, pid) __field(unsigned int, runtime) - __field(int, pct) - __field(unsigned int, pred_demand) + __field(unsigned int, pred_demand_scaled) __array(u8, bucket, NUM_BUSY_BUCKETS) __field(int, cpu) + __field(int, start) + __field(int, first) + __field(int, final) ), TP_fast_assign( memcpy(__entry->comm, p->comm, TASK_COMM_LEN); __entry->pid = p->pid; __entry->runtime = runtime; - __entry->pct = pct; - __entry->pred_demand = pred_demand; + __entry->pred_demand_scaled = pred_demand_scaled; memcpy(__entry->bucket, wts->busy_buckets, NUM_BUSY_BUCKETS * sizeof(u8)); __entry->cpu = task_cpu(p); + __entry->start = start; + __entry->first = first; + __entry->final = final; ), - TP_printk("%d (%s): runtime %u pct %d cpu %d pred_demand %u (buckets: %u %u %u %u %u %u %u %u %u %u)", + TP_printk("%d (%s): runtime %u cpu %d pred_demand_scaled %u start %d first %d final %d (buckets: %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u)", __entry->pid, __entry->comm, - __entry->runtime, __entry->pct, __entry->cpu, - __entry->pred_demand, __entry->bucket[0], __entry->bucket[1], + __entry->runtime, __entry->cpu, + __entry->pred_demand_scaled, __entry->start, __entry->first, __entry->final, + __entry->bucket[0], __entry->bucket[1], __entry->bucket[2], __entry->bucket[3], __entry->bucket[4], __entry->bucket[5], __entry->bucket[6], __entry->bucket[7], - __entry->bucket[8], __entry->bucket[9]) + __entry->bucket[8], __entry->bucket[9], __entry->bucket[10], + __entry->bucket[11], __entry->bucket[12], __entry->bucket[13], + __entry->bucket[14], __entry->bucket[15]) ); TRACE_EVENT(sched_update_history, @@ -74,7 +82,7 @@ TRACE_EVENT(sched_update_history, __field(enum task_event, evt) __field(unsigned int, demand) __field(unsigned int, coloc_demand) - __field(unsigned int, pred_demand) + __field(unsigned int, pred_demand_scaled) __array(u32, hist, RAVG_HIST_SIZE_MAX) __field(unsigned int, nr_big_tasks) __field(int, cpu) @@ -88,18 +96,18 @@ TRACE_EVENT(sched_update_history, __entry->evt = evt; __entry->demand = wts->demand; __entry->coloc_demand = wts->coloc_demand; - __entry->pred_demand = wts->pred_demand; + __entry->pred_demand_scaled = wts->pred_demand_scaled; memcpy(__entry->hist, wts->sum_history, RAVG_HIST_SIZE_MAX * sizeof(u32)); __entry->nr_big_tasks = wrq->walt_stats.nr_big_tasks; __entry->cpu = rq->cpu; ), - TP_printk("%d (%s): runtime %u samples %d event %s demand %u coloc_demand %u pred_demand %u (hist: %u %u %u %u %u) cpu %d nr_big %u", + TP_printk("%d (%s): runtime %u samples %d event %s demand %u coloc_demand %u pred_demand_scaled %u (hist: %u %u %u %u %u) cpu %d nr_big %u", __entry->pid, __entry->comm, __entry->runtime, __entry->samples, task_event_names[__entry->evt], - __entry->demand, __entry->coloc_demand, __entry->pred_demand, + __entry->demand, __entry->coloc_demand, __entry->pred_demand_scaled, __entry->hist[0], __entry->hist[1], __entry->hist[2], __entry->hist[3], __entry->hist[4], __entry->cpu, __entry->nr_big_tasks) @@ -167,7 +175,7 @@ TRACE_EVENT(sched_update_task_ravg, __field(unsigned int, coloc_demand) __field(unsigned int, sum) __field(int, cpu) - __field(unsigned int, pred_demand) + __field(unsigned int, pred_demand_scaled) __field(u64, rq_cs) __field(u64, rq_ps) __field(u64, grp_cs) @@ -201,7 +209,7 @@ TRACE_EVENT(sched_update_task_ravg, __entry->coloc_demand = wts->coloc_demand; __entry->sum = wts->sum; __entry->irqtime = irqtime; - __entry->pred_demand = wts->pred_demand; + __entry->pred_demand_scaled = wts->pred_demand_scaled; __entry->rq_cs = wrq->curr_runnable_sum; __entry->rq_ps = wrq->prev_runnable_sum; __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; @@ -223,13 +231,13 @@ TRACE_EVENT(sched_update_task_ravg, __entry->prev_top = wrq->prev_top; ), - TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u", + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand_scaled %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u", __entry->wallclock, __entry->win_start, __entry->delta, task_event_names[__entry->evt], __entry->cpu, __entry->cur_freq, __entry->cur_pid, __entry->pid, __entry->comm, __entry->mark_start, __entry->delta_m, __entry->demand, __entry->coloc_demand, - __entry->sum, __entry->irqtime, __entry->pred_demand, + __entry->sum, __entry->irqtime, __entry->pred_demand_scaled, __entry->rq_cs, __entry->rq_ps, __entry->curr_window, __window_print(p, __get_dynamic_array(curr_sum), nr_cpu_ids), __entry->prev_window, @@ -1168,7 +1176,7 @@ TRACE_EVENT(sched_enq_deq_task, __field(unsigned int, rt_nr_running) __field(unsigned int, cpus_allowed) __field(unsigned int, demand) - __field(unsigned int, pred_demand) + __field(unsigned int, pred_demand_scaled) __field(bool, compat_thread) __field(bool, mvp) ), @@ -1183,19 +1191,20 @@ TRACE_EVENT(sched_enq_deq_task, __entry->rt_nr_running = task_rq(p)->rt.rt_nr_running; __entry->cpus_allowed = cpus_allowed; __entry->demand = task_load(p); - __entry->pred_demand = task_pl(p); + __entry->pred_demand_scaled = + ((struct walt_task_struct *) p->android_vendor_data1)->pred_demand_scaled; __entry->compat_thread = is_compat_thread(task_thread_info(p)); __entry->mvp = mvp; ), - TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u is_compat_t=%d mvp=%d", + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand_scaled=%u is_compat_t=%d mvp=%d", __entry->cpu, __entry->enqueue ? "enqueue" : "dequeue", __entry->comm, __entry->pid, __entry->prio, __entry->nr_running, __entry->rt_nr_running, __entry->cpus_allowed, __entry->demand, - __entry->pred_demand, + __entry->pred_demand_scaled, __entry->compat_thread, __entry->mvp) ); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 58f8efc96cf7..ba4509306230 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1159,11 +1159,11 @@ static inline void bucket_increase(u8 *buckets, u16 *bucket_bitmask, int idx) } } -static inline int busy_to_bucket(u32 normalized_rt) +static inline int busy_to_bucket(u16 normalized_rt) { int bidx; - bidx = mult_frac(normalized_rt, NUM_BUSY_BUCKETS, max_task_load()); + bidx = normalized_rt >> (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); bidx = min(bidx, NUM_BUSY_BUCKETS - 1); /* @@ -1194,13 +1194,12 @@ static inline int busy_to_bucket(u32 normalized_rt) * to use for prediction. Once found, it returns the midpoint of that bucket. */ static u32 get_pred_busy(struct task_struct *p, - int start, u32 runtime, u16 bucket_bitmask) + int start, u16 runtime_scaled, u16 bucket_bitmask) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - u32 dmin, dmax; - u64 cur_freq_runtime = 0; + u16 dmin, dmax; int first = NUM_BUSY_BUCKETS, final = NUM_BUSY_BUCKETS; - u32 ret = runtime; + u16 ret = runtime_scaled; u16 next_mask = bucket_bitmask >> start; /* skip prediction for new tasks due to lack of history */ @@ -1224,43 +1223,31 @@ static u32 get_pred_busy(struct task_struct *p, dmin = 0; final = 1; } else { - dmin = mult_frac(final, max_task_load(), NUM_BUSY_BUCKETS); + dmin = final << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); } - dmax = mult_frac(final + 1, max_task_load(), NUM_BUSY_BUCKETS); + dmax = (final + 1) << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); /* * when updating in middle of a window, runtime could be higher * than all recorded history. Always predict at least runtime. */ - ret = max(runtime, (dmin + dmax) / 2); + ret = max(runtime_scaled, (u16) (((u32)dmin + dmax) / 2)); out: - trace_sched_update_pred_demand(p, runtime, - mult_frac((unsigned int)cur_freq_runtime, 100, - sched_ravg_window), ret, wts); + trace_sched_update_pred_demand(p, runtime_scaled, + ret, start, first, final, wts); return ret; } -static inline u32 calc_pred_demand(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - if (wts->pred_demand >= wts->curr_window) - return wts->pred_demand; - - return get_pred_busy(p, busy_to_bucket(wts->curr_window), - wts->curr_window, wts->bucket_bitmask); -} - /* - * predictive demand of a task is calculated at the window roll-over. + * predictive demand of a task was calculated at the last window roll-over. * if the task current window busy time exceeds the predicted * demand, update it here to reflect the task needs. */ static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int event) { - u32 new, old; - u16 new_scaled; + u16 new_pred_demand_scaled; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u16 curr_window_scaled; if (is_idle_task(p)) return; @@ -1280,21 +1267,20 @@ static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int ev return; } - new = calc_pred_demand(p); - old = wts->pred_demand; - - if (old >= new) + curr_window_scaled = scale_demand(wts->curr_window); + if (wts->pred_demand_scaled >= curr_window_scaled) return; - new_scaled = scale_demand(new); + new_pred_demand_scaled = get_pred_busy(p, busy_to_bucket(curr_window_scaled), + curr_window_scaled, wts->bucket_bitmask); + if (task_on_rq_queued(p) && (!task_has_dl_policy(p) || !p->dl.dl_throttled)) fixup_walt_sched_stats_common(rq, p, wts->demand_scaled, - new_scaled); + new_pred_demand_scaled); - wts->pred_demand = new; - wts->pred_demand_scaled = new_scaled; + wts->pred_demand_scaled = new_pred_demand_scaled; } static void clear_top_tasks_bitmap(unsigned long *bitmap) @@ -1801,17 +1787,17 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, new_window, full_window); } -static inline u32 predict_and_update_buckets( - struct task_struct *p, u32 runtime) { +static inline u16 predict_and_update_buckets( + struct task_struct *p, u16 runtime_scaled) { int bidx; - u32 pred_demand; + u32 pred_demand_scaled; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - bidx = busy_to_bucket(runtime); - pred_demand = get_pred_busy(p, bidx, runtime, wts->bucket_bitmask); + bidx = busy_to_bucket(runtime_scaled); + pred_demand_scaled = get_pred_busy(p, bidx, runtime_scaled, wts->bucket_bitmask); bucket_increase(wts->busy_buckets, &wts->bucket_bitmask, bidx); - return pred_demand; + return pred_demand_scaled; } static int @@ -1866,7 +1852,7 @@ static void update_history(struct rq *rq, struct task_struct *p, struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u32 *hist = &wts->sum_history[0]; int i; - u32 max = 0, avg, demand, pred_demand; + u32 max = 0, avg, demand; u64 sum = 0; u16 demand_scaled, pred_demand_scaled; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; @@ -1900,9 +1886,8 @@ static void update_history(struct rq *rq, struct task_struct *p, else demand = max(avg, runtime); } - pred_demand = predict_and_update_buckets(p, runtime); + pred_demand_scaled = predict_and_update_buckets(p, scale_demand(runtime)); demand_scaled = scale_demand(demand); - pred_demand_scaled = scale_demand(pred_demand); /* * A throttled deadline sched class task gets dequeued without @@ -1925,7 +1910,6 @@ static void update_history(struct rq *rq, struct task_struct *p, wts->demand = demand; wts->demand_scaled = demand_scaled; wts->coloc_demand = div64_u64(sum, sched_ravg_hist_size); - wts->pred_demand = pred_demand; wts->pred_demand_scaled = pred_demand_scaled; if (demand_scaled > sysctl_sched_min_task_util_for_colocation) @@ -2251,7 +2235,6 @@ static void init_new_task_load(struct task_struct *p) wts->demand = init_load_windows; wts->demand_scaled = init_load_windows_scaled; wts->coloc_demand = init_load_windows; - wts->pred_demand = 0; wts->pred_demand_scaled = 0; for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) wts->sum_history[i] = init_load_windows; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 8f059fe52197..5a973f5b12c7 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -788,13 +788,6 @@ static inline unsigned int task_load(struct task_struct *p) return wts->demand; } -static inline unsigned int task_pl(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return wts->pred_demand; -} - static inline bool task_in_related_thread_group(struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; From 8ecbc9e10618d6cc100dcb751bbe1bc9076253a6 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 24 Feb 2022 15:15:05 -0800 Subject: [PATCH 267/346] sched/walt: cleanup hooks for halt Sevearl hooks for halt were present in the main walt file, when it should be in the halt file. Move, and change initialization appropriately. Change-Id: Ibab5012a3728b70caebab24b19e25d5eb7771cb0 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 46 ----------------------------------- kernel/sched/walt/walt_halt.c | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index ba4509306230..fa2d3d3bf5df 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4204,48 +4204,6 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } -static void android_rvh_set_cpus_allowed_ptr_locked(void *unused, - const struct cpumask *cpu_valid_mask, - const struct cpumask *new_mask, - unsigned int *dest_cpu) -{ - cpumask_t allowed_cpus; - - if (unlikely(walt_disabled)) - return; - - if (cpu_halted(*dest_cpu)) { - /* remove halted cpus from the valid mask, and store locally */ - cpumask_andnot(&allowed_cpus, cpu_valid_mask, cpu_halt_mask); - *dest_cpu = cpumask_any_and_distribute(&allowed_cpus, new_mask); - } -} - -static void android_rvh_rto_next_cpu(void *unused, - int rto_cpu, - struct cpumask *rto_mask, - int *cpu) -{ - cpumask_t allowed_cpus; - - if (unlikely(walt_disabled)) - return; - - if (cpu_halted(*cpu)) { - /* remove halted cpus from the valid mask, and store locally */ - cpumask_andnot(&allowed_cpus, rto_mask, cpu_halt_mask); - *cpu = cpumask_next(rto_cpu, &allowed_cpus); - } -} - -static void android_rvh_is_cpu_allowed(void *unused, int cpu, bool *allowed) -{ - if (unlikely(walt_disabled)) - return; - - *allowed = !cpumask_test_cpu(cpu, cpu_halt_mask); -} - static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4273,10 +4231,6 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); - register_trace_android_rvh_set_cpus_allowed_ptr_locked( - android_rvh_set_cpus_allowed_ptr_locked, NULL); - register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL); - register_trace_android_rvh_is_cpu_allowed(android_rvh_is_cpu_allowed, NULL); } atomic64_t walt_irq_work_lastq_ws; diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index cf850e87750f..b7eb4dece2bb 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -435,9 +435,52 @@ static void android_rvh_get_nohz_timer_target(void *unused, int *cpu, bool *done rcu_read_unlock(); } +static void android_rvh_set_cpus_allowed_ptr_locked(void *unused, + const struct cpumask *cpu_valid_mask, + const struct cpumask *new_mask, + unsigned int *dest_cpu) +{ + cpumask_t allowed_cpus; + + if (unlikely(walt_disabled)) + return; + + if (cpu_halted(*dest_cpu)) { + /* remove halted cpus from the valid mask, and store locally */ + cpumask_andnot(&allowed_cpus, cpu_valid_mask, cpu_halt_mask); + *dest_cpu = cpumask_any_and_distribute(&allowed_cpus, new_mask); + } +} + +static void android_rvh_rto_next_cpu(void *unused, int rto_cpu, struct cpumask *rto_mask, int *cpu) +{ + cpumask_t allowed_cpus; + + if (unlikely(walt_disabled)) + return; + + if (cpu_halted(*cpu)) { + /* remove halted cpus from the valid mask, and store locally */ + cpumask_andnot(&allowed_cpus, rto_mask, cpu_halt_mask); + *cpu = cpumask_next(rto_cpu, &allowed_cpus); + } +} + +static void android_rvh_is_cpu_allowed(void *unused, int cpu, bool *allowed) +{ + if (unlikely(walt_disabled)) + return; + + *allowed = !cpumask_test_cpu(cpu, cpu_halt_mask); +} + void walt_halt_init(void) { register_trace_android_rvh_get_nohz_timer_target(android_rvh_get_nohz_timer_target, NULL); + register_trace_android_rvh_set_cpus_allowed_ptr_locked( + android_rvh_set_cpus_allowed_ptr_locked, NULL); + register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL); + register_trace_android_rvh_is_cpu_allowed(android_rvh_is_cpu_allowed, NULL); } From 31f357a5004e08f0ac19c6f46285561c32c41bf2 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 3 Mar 2022 13:58:06 -0800 Subject: [PATCH 268/346] sched/walt/halt: prevent migration of migration_disabled tasks Prior to the introduction of the PREEMPT_RT, migrate_disable() was introduced to support PREEMPT_RT. Many places throughout the code avoid using migrate_disable() if PREEMPT_RT is not set. However, bpf related code makes no such precautions, which is enabled. migrate_tasks() will forcefully migrate tasks off of this cpu without regard for migration_disabled state of the task. This causes a warning when attempting to migrate a migration_disabled task, and may have other unforseen consequences. Prevent migrate_tasks from moving tasks for which migration is disabled. Change-Id: I04aef81611c8d4df8e2abbd7833b8b12ec146548 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_halt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index b7eb4dece2bb..8e4925f7b439 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -150,7 +150,7 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) /* Find suitable destination for @next */ dest_cpu = select_fallback_rq(dead_rq->cpu, next); - if (cpu_of(rq) != dest_cpu) { + if (cpu_of(rq) != dest_cpu && !is_migration_disabled(next)) { /* only perform a required migration */ rq = __migrate_task(rq, rf, next, dest_cpu); From 3eeb00d31818e4d7464f8e8ba602edb962f9d8e9 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 2 Feb 2022 11:54:47 -0800 Subject: [PATCH 269/346] sched: Fix negative capacity issue under thermal conditions Currently, there is a issue in the system where the rt capacity could be greater than the thermal capacity, and as a consequence we could end up with a negative capacity in calculations. Change-Id: If81be87878ae7948b0278e30ac4a8618d434d070 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index fa2d3d3bf5df..4cc53c862870 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3821,31 +3821,32 @@ static void walt_cpu_frequency_limits(void *unused, struct cpufreq_policy *polic */ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity) { - unsigned long max_capacity = arch_scale_cpu_capacity(cpu); + unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu); unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); unsigned long thermal_cap; struct walt_sched_cluster *cluster; - unsigned long rt_pressure = max_capacity - *capacity; + unsigned long rt_pressure = fmax_capacity - *capacity; + struct rq *rq = cpu_rq(cpu); if (unlikely(walt_disabled)) return; /* - * thermal_pressure = max_capacity - curr_cap_as_per_thermal. + * thermal_pressure = cpu_scale - curr_cap_as_per_thermal. * so, - * curr_cap_as_per_thermal = max_capacity - thermal_pressure. + * curr_cap_as_per_thermal = cpu_scale - thermal_pressure. */ - thermal_cap = max_capacity - thermal_pressure; + thermal_cap = fmax_capacity - thermal_pressure; cluster = cpu_cluster(cpu); - /* reduce the max_capacity under cpufreq constraints */ + /* reduce the fmax_capacity under cpufreq constraints */ if (cluster->max_freq != cluster->max_possible_freq) - max_capacity = mult_frac(max_capacity, cluster->max_freq, + fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq, cluster->max_possible_freq); - cpu_rq(cpu)->cpu_capacity_orig = min(max_capacity, thermal_cap); - *capacity = cpu_rq(cpu)->cpu_capacity_orig - rt_pressure; + rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap); + *capacity = max(rq->cpu_capacity_orig - rt_pressure, 1UL); } static void android_rvh_sched_cpu_starting(void *unused, int cpu) From f9d4ca79371a8bec80aef38b0b4b8ba60fe3e823 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 3 Nov 2021 13:18:19 -0700 Subject: [PATCH 270/346] sched: walt: add trace in update_cpu_capacity When cpu capacities are updated, due to either thermal or frequency limits, print trace output to understand why the change happened. Change-Id: I5c09ddbcbfae5b26a23d6d2ca07b38c89159f878 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 35 +++++++++++++++++++++++++++++++++++ kernel/sched/walt/walt.c | 9 +++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 26c38c06214c..98e87c5c43d6 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1371,7 +1371,42 @@ TRACE_EVENT(halt_cpus, TP_printk("req_cpus=0x%x halt_cpus=0x%x time=%u us halt=%d success=%d", __entry->cpus, __entry->halted_cpus, __entry->time, __entry->halt, __entry->success) +); +TRACE_EVENT(update_cpu_capacity, + + TP_PROTO(int cpu, unsigned long rt_pressure, unsigned long capacity), + + TP_ARGS(cpu, rt_pressure, capacity), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, rt_pressure) + __field(unsigned long, capacity) + __field(unsigned long, arch_capacity) + __field(unsigned long, thermal_cap) + __field(unsigned long, max_possible_freq) + __field(unsigned long, max_freq) + ), + + TP_fast_assign( + struct walt_sched_cluster *cluster = cpu_cluster(cpu); + + __entry->cpu = cpu; + __entry->rt_pressure = rt_pressure; + __entry->capacity = capacity; + __entry->arch_capacity = arch_scale_cpu_capacity(cpu); + __entry->thermal_cap = arch_scale_cpu_capacity(cpu) - + arch_scale_thermal_pressure(cpu); + __entry->max_freq = cluster->max_freq; + __entry->max_possible_freq = cluster->max_possible_freq; + ), + + TP_printk("cpu=%d arch_capacity=%lu thermal_cap=%lu rt_pressure=%lu max_freq=%lu max_possible_freq=%lu capacity=%lu", + __entry->cpu, __entry->arch_capacity, + __entry->thermal_cap, __entry->rt_pressure, + __entry->max_freq, __entry->max_possible_freq, + __entry->capacity) ); #endif /* _TRACE_WALT_H */ diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 4cc53c862870..8ce20759a5d6 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3823,9 +3823,9 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long { unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu); unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); - unsigned long thermal_cap; - struct walt_sched_cluster *cluster; + unsigned long thermal_cap, old; unsigned long rt_pressure = fmax_capacity - *capacity; + struct walt_sched_cluster *cluster; struct rq *rq = cpu_rq(cpu); if (unlikely(walt_disabled)) @@ -3845,7 +3845,12 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq, cluster->max_possible_freq); + old = rq->cpu_capacity_orig; rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap); + + if (old != rq->cpu_capacity_orig) + trace_update_cpu_capacity(cpu, rt_pressure, *capacity); + *capacity = max(rq->cpu_capacity_orig - rt_pressure, 1UL); } From 2201f39e1cc7bba8b0733ae122293afc47b380ca Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 9 Mar 2022 10:23:12 -0800 Subject: [PATCH 271/346] sched/walt: Fix setting RTG Boost Freq Currently the tunable to set RTG Boost Frequency is set on the basis of a hardcoded policy->cpu. However, this is incorrect on two fronts - first, CPU4 is no longer the default CPU managing the Gold CPU policy (it is now CPU3). Secondly, the "typical" CPU managing the cluster's policy cpu may be offline, in which case, another CPU from the cluster would be managing the policy, and this implementation would break. Change-Id: I0b9d4b882c93f48ea73f4cfa20e547d4216e142c Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index d05b38c4890a..fa8b913bb2d6 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -265,9 +265,9 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) #define NL_RATIO 75 #define DEFAULT_HISPEED_LOAD 90 -#define DEFAULT_CPU0_RTG_BOOST_FREQ 1000000 -#define DEFAULT_CPU4_RTG_BOOST_FREQ 768000 -#define DEFAULT_CPU7_RTG_BOOST_FREQ 0 +#define DEFAULT_SILVER_RTG_BOOST_FREQ 1000000 +#define DEFAULT_GOLD_RTG_BOOST_FREQ 768000 +#define DEFAULT_PRIME_RTG_BOOST_FREQ 0 #define DEFAULT_TARGET_LOAD_THRESH 1024 #define DEFAULT_TARGET_LOAD_SHIFT 4 static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_util, @@ -856,18 +856,12 @@ static int waltgov_init(struct cpufreq_policy *policy) tunables->target_load_thresh = DEFAULT_TARGET_LOAD_THRESH; tunables->target_load_shift = DEFAULT_TARGET_LOAD_SHIFT; - switch (policy->cpu) { - default: - case 0: - tunables->rtg_boost_freq = DEFAULT_CPU0_RTG_BOOST_FREQ; - break; - case 4: - tunables->rtg_boost_freq = DEFAULT_CPU4_RTG_BOOST_FREQ; - break; - case 7: - tunables->rtg_boost_freq = DEFAULT_CPU7_RTG_BOOST_FREQ; - break; - } + if (is_min_cluster_cpu(policy->cpu)) + tunables->rtg_boost_freq = DEFAULT_SILVER_RTG_BOOST_FREQ; + else if (is_max_cluster_cpu(policy->cpu)) + tunables->rtg_boost_freq = DEFAULT_PRIME_RTG_BOOST_FREQ; + else + tunables->rtg_boost_freq = DEFAULT_GOLD_RTG_BOOST_FREQ; policy->governor_data = wg_policy; wg_policy->tunables = tunables; From 05cfb9a3e5ab6ca306407fc80974c4b3e1a95de3 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 8 Mar 2022 11:11:40 -0800 Subject: [PATCH 272/346] sched/walt: kick unhalted cpu for load-balancing When a cpu is unhalted it has no tasks enqueued on it. This means that nothing will be immediately scheduled on that cpu. Kick the cpu that has been unhalted, to ensure it can load-balance and help other cpus. Change-Id: I23fd12bd9967d6aabdbd13b868e0dc316f118342 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 24 ++++++++++++++++++++++++ kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_halt.c | 9 +++++++-- kernel/sched/walt/walt_lb.c | 27 ++------------------------- 4 files changed, 34 insertions(+), 27 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 8ce20759a5d6..09bf4f5ba25c 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3078,6 +3078,30 @@ static void walt_update_tg_pointer(struct cgroup_subsys_state *css) walt_init_tg(css_tg(css)); } +void walt_kick_cpu(int cpu) +{ + unsigned int flags = NOHZ_KICK_MASK; + + if (cpu == -1) + return; + + /* + * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets + * the first flag owns it; cleared by nohz_csd_func(). + */ + flags = atomic_fetch_or(flags, nohz_flags(cpu)); + if (flags & NOHZ_KICK_MASK) + return; + + /* + * This way we generate an IPI on the target CPU which + * is idle. And the softirq performing nohz idle load balance + * will be run before returning from the IPI. + */ + smp_call_function_single_async(cpu, &cpu_rq(cpu)->nohz_csd); +} + + static void android_rvh_cpu_cgroup_online(void *unused, struct cgroup_subsys_state *css) { if (unlikely(walt_disabled)) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 5a973f5b12c7..4a0ade0c0d60 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -161,6 +161,7 @@ extern int sched_set_boost(int enable); extern void walt_boost_init(void); extern int sched_pause_cpus(struct cpumask *pause_cpus); extern int sched_unpause_cpus(struct cpumask *unpause_cpus); +extern void walt_kick_cpu(int cpu); extern unsigned int sched_get_cpu_util_pct(int cpu); extern void sched_update_hyst_times(void); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 8e4925f7b439..8e89a82d26c3 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -268,11 +268,16 @@ static int start_cpus(struct cpumask *cpus) for_each_cpu(cpu, cpus) { halt_cpu_state = per_cpu_ptr(&halt_state, cpu); halt_cpu_state->last_halt = 0; - - /* guarantee zero'd last_halt before clearing from the mask */ wmb(); + /* wmb to guarantee zero'd last_halt before clearing from the mask */ + cpumask_clear_cpu(cpu, cpu_halt_mask); + + /* kick the cpu so it can pull tasks + * after the mask has been cleared. + */ + walt_kick_cpu(cpu); } trace_halt_cpus(cpus, start_time, 0, 0); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index e8d182d90f18..ecc873d69a30 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -758,29 +758,6 @@ static bool should_help_min_cap(int this_cpu) return false; } -static void kick_first_idle(int first_idle) -{ - unsigned int flags = NOHZ_KICK_MASK; - - if (first_idle == -1) - return; - - /* - * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets - * the first flag owns it; cleared by nohz_csd_func(). - */ - flags = atomic_fetch_or(flags, nohz_flags(first_idle)); - if (flags & NOHZ_KICK_MASK) - return; - - /* - * This way we generate an IPI on the target CPU which - * is idle. And the softirq performing nohz idle load balance - * will be run before returning from the IPI. - */ - smp_call_function_single_async(first_idle, &cpu_rq(first_idle)->nohz_csd); -} - /* similar to sysctl_sched_migration_cost */ #define NEWIDLE_BALANCE_THRESHOLD 500000 static void walt_newidle_balance(void *unused, struct rq *this_rq, @@ -866,7 +843,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - kick_first_idle(first_idle); + walt_kick_cpu(first_idle); } } else if (order_index == 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], @@ -887,7 +864,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - kick_first_idle(first_idle); + walt_kick_cpu(first_idle); } } else { busy_cpu = From 6acd93ad48602d122160bc68796ead10d2998102 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 9 Mar 2022 15:36:49 -0800 Subject: [PATCH 273/346] sched/walt: walt hooks skipped during rt throttling Currently, walt maintains cra and adds demand when enqueue task happens and subtracts from it when dequeue happens. However, when an rt task is throttled, walt is not informed of the dequeue. This leads to a residual demand when the idle task switches in post throttling leading to walt accounting error. Remove that specific accounting check since it is hard to inform WALT of the throttle dequeue and unthrottle enqueue, until rt can be fixed to support walt accounting in the throttling case. Change-Id: If1416a61fbe3979253a42cb82990b7b13ad2c4eb Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 09bf4f5ba25c..a7a54afe4c80 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4152,7 +4152,6 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, { u64 wallclock; struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; if (unlikely(walt_disabled)) return; @@ -4164,9 +4163,6 @@ static void android_rvh_schedule(void *unused, struct task_struct *prev, wts->last_sleep_ts = wallclock; walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); - if (is_idle_task(next) && wrq->walt_stats.cumulative_runnable_avg_scaled != 0) - WALT_BUG(WALT_BUG_WALT, next, "next=idle cra non zero=%d\n", - wrq->walt_stats.cumulative_runnable_avg_scaled); } else { walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0); } From 505af370aec32931d91534f268894b9825fab79f Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 10 Mar 2022 10:22:56 -0800 Subject: [PATCH 274/346] sched/walt: cleanup copyright issues Correct copyright issues and update copyrights for proper year. Change-Id: I2a8a407814c7124a6f0b7a257c1e26b82f018606 Signed-off-by: Stephen Dickey --- kernel/sched/walt/sysctl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 1f2ede153895..b7910b0a20b2 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2020-2022, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "walt.h" From 1de7b71d22a7f1956bdf7e1c85a07049d3921e9d Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 10 Mar 2022 14:21:21 -0800 Subject: [PATCH 275/346] sched/walt: Remove unused sysctl node While deprecating the RT throttling mechanism, associated sysctl node remained to be removed. Remove it now. Change-Id: Iac5a3da157d71bc878e1406f1f64f57e0ccb7103 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/sysctl.c | 10 ---------- kernel/sched/walt/walt.h | 1 - 2 files changed, 11 deletions(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index b7910b0a20b2..74df99d7f04f 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -64,7 +64,6 @@ unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; const int sched_user_hint_max = 1000; unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ unsigned int sysctl_sched_sync_hint_enable = 1; -unsigned int sysctl_sched_bug_on_rt_throttle; unsigned int sysctl_panic_on_walt_bug = walt_debug_initial_values(); unsigned int sysctl_sched_suppress_region2; unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; @@ -732,15 +731,6 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - { - .procname = "sched_bug_on_rt_throttle", - .data = &sysctl_sched_bug_on_rt_throttle, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, { .procname = "sched_suppress_region2", .data = &sysctl_sched_suppress_region2, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 4a0ade0c0d60..d1f8f3b0781a 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -273,7 +273,6 @@ extern unsigned int sysctl_sched_task_unfilter_period; extern unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ extern unsigned int sysctl_sched_sync_hint_enable; -extern unsigned int sysctl_sched_bug_on_rt_throttle; extern unsigned int sysctl_sched_suppress_region2; extern unsigned int sysctl_sched_skip_sp_newly_idle_lb; extern unsigned int sysctl_sched_asymcap_boost; From ef1fadeb4bcf3f12c52c58078571eb7402b09f4b Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Thu, 10 Mar 2022 15:20:11 -0800 Subject: [PATCH 276/346] sched/walt: Disable long running RT task detection Disable detection of long running RT tasks by default. Change-Id: Ia0224ebdfef5ee59523aeea80b25429d5b99f2be Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/sysctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 74df99d7f04f..7a59a743cdc2 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -69,7 +69,7 @@ unsigned int sysctl_sched_suppress_region2; unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000; unsigned int sysctl_sched_asymcap_boost; -unsigned int sysctl_sched_long_running_rt_task_ms = 1200; +unsigned int sysctl_sched_long_running_rt_task_ms; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; From ac78c5792b8a35a12381703a7b1eacee891e5faf Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 15 Mar 2022 14:42:21 -0700 Subject: [PATCH 277/346] sched/walt: Fix in long running RT task detection Ensure that long running RT task detection is set to a minimum of 800ms. Change-Id: I378c6b51ef9d417f9143e19e2938b625d1dd6a65 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_rt.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index c8c8a5f690a7..feb579f00c0f 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -61,6 +61,10 @@ int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int write, ret = proc_douintvec_minmax(table, write, buffer, lenp, ppos); + if (sysctl_sched_long_running_rt_task_ms > 0 && + sysctl_sched_long_running_rt_task_ms < 800) + sysctl_sched_long_running_rt_task_ms = 800; + if (write && !long_running_rt_task_trace_rgstrd) { register_trace_sched_switch(rt_task_arrival_marker, NULL); register_trace_android_vh_scheduler_tick(long_running_rt_task_notifier, NULL); From fb76acffb06a01facf59d20060703132d4445aa1 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Mon, 31 Jan 2022 21:59:23 -0800 Subject: [PATCH 278/346] sched: walt: properly account for migrating mvp task Currently, every time a task migrates while running, the enqueue resets the slice time accounted thus far, i.e. enqueue resets sum_exec_snapshot. There is a potential for a frequently active migrating task to get infinite slice. Fix this by ensuring total_exec gets updated regardless of SLICE threshold is crossed. Having a non zero total_exec will make enqueue skip taking a snapshot. Also we need to track two snapshots of sum_exec_runtime, one for total and one for slice. The total snapshot will be taken upon task's enqueue after it wakes up from sleep while the slice snapshot advances everytime the SLICE threshold is crossed. This scheme helps us track the total time the task ran since wakeup and the total time it ran in the slice. The snapshot however is the origin for accounting time towards SLICE, hence it should not be updated as long runtime is within SLICE. Once SLICE is crossed, snapshot should be updated for tracking the runtime towards the next SLICE period. Change-Id: I3cf82570c3224b2c9a40b278d2c9baa059216bc3 Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Sai Harshini Nimmala --- include/linux/sched/walt.h | 4 +++- kernel/sched/walt/walt.c | 3 ++- kernel/sched/walt/walt_cfs.c | 27 +++++++++++++++------------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 08777104a28a..87757ac85a4d 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _LINUX_SCHED_WALT_H @@ -119,7 +120,8 @@ struct walt_task_struct { int prev_on_rq; int prev_on_rq_cpu; struct list_head mvp_list; - u64 sum_exec_snapshot; + u64 sum_exec_snapshot_for_slice; + u64 sum_exec_snapshot_for_total; u64 total_exec; int mvp_prio; int cidx; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index a7a54afe4c80..6d97ac2995df 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2243,7 +2243,8 @@ static void init_new_task_load(struct task_struct *p) wts->unfilter = sysctl_sched_task_unfilter_period; INIT_LIST_HEAD(&wts->mvp_list); - wts->sum_exec_snapshot = 0; + wts->sum_exec_snapshot_for_slice = 0; + wts->sum_exec_snapshot_for_total = 0; wts->total_exec = 0; wts->mvp_prio = WALT_NOT_MVP; wts->cidx = 0; diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 224d26bdde61..3bb1bdc4edeb 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1084,7 +1084,7 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1; - s64 delta; + u64 slice; unsigned int limit; lockdep_assert_held(&rq->__lock); @@ -1099,24 +1099,26 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr if (!(rq->clock_update_flags & RQCF_UPDATED)) update_rq_clock(rq); - /* sum_exec_snapshot can be ahead. See below increment */ - delta = curr->se.sum_exec_runtime - wts->sum_exec_snapshot; - if (delta < 0) - delta = 0; + if (curr->se.sum_exec_runtime > wts->sum_exec_snapshot_for_total) + wts->total_exec = curr->se.sum_exec_runtime - wts->sum_exec_snapshot_for_total; else - delta += rq_clock_task(rq) - curr->se.exec_start; + wts->total_exec = 0; + + if (curr->se.sum_exec_runtime > wts->sum_exec_snapshot_for_slice) + slice = curr->se.sum_exec_runtime - wts->sum_exec_snapshot_for_slice; + else + slice = 0; /* slice is not expired */ - if (delta < WALT_MVP_SLICE) + if (slice < WALT_MVP_SLICE) return; + wts->sum_exec_snapshot_for_slice = curr->se.sum_exec_runtime; /* * slice is expired, check if we have to deactivate the * MVP task, otherwise requeue the task in the list so * that other MVP tasks gets a chance. */ - wts->sum_exec_snapshot += delta; - wts->total_exec += delta; limit = walt_cfs_mvp_task_limit(curr); if (wts->total_exec > limit) { @@ -1165,9 +1167,10 @@ void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p) * task runtime snapshot. From now onwards we use this point as a * baseline to enforce the slice and demotion. */ - if (!wts->total_exec) /* queue after sleep */ - wts->sum_exec_snapshot = p->se.sum_exec_runtime; - + if (!wts->total_exec) /* queue after sleep */ { + wts->sum_exec_snapshot_for_total = p->se.sum_exec_runtime; + wts->sum_exec_snapshot_for_slice = p->se.sum_exec_runtime; + } } void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) From adeb747e708dac8f1e030d165095cd309f89c90e Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Wed, 16 Mar 2022 23:38:03 -0700 Subject: [PATCH 279/346] sched: walt: fix stale window_start for walt load Currently the code uses walt_load_reported_window (wlrw in short) to mark the beginning of the new window. All the updates to cpufreq use the latest wlrw. With 'commit acd27bd371dd2 ("sched/walt: walt irq work reduce locking")' we could hit BUG(curr_ws < last_ws) in cpufreq_walt.c because of using a stale wlrw. An example of the race. cpu0 ED tick cpu1 ED tick cpu2 tick utra G->P migration rq_lock(0) rq_lock(2) cpu_util_freq_walt() detects rollover sets walt_irq_work_for_mgrtn read wlrw =760 walt_irq_work_lastq_ws =776 rq_lock 3,4,5,6,7 rq_unlock(2) UPDATE wlrw = 776 walt_irq_work_for_rlvr ... EL2/EL3 EXCEPTION ...waits for all rq lock ... OR SVM rq_unlock 3,4,5,6,7 rq_lock(1) RUNS cpu_util_freq_walt() read wlrw =776 write load->ws =776 waltgov_calc_avg_cap() HERE silver_policy->last_ws =776 rq_unlock(1) write load->ws=760 waltgov_calc_avg_cap() BUG(walt_load->ws 760 < silver_policy->last_ws 776) To fix this ensure that wlrw is updated ONLY when rq locks of all the cpus are held. Change-Id: I9a6649cbccdfdb5a70a0256703624dde47868a15 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 6d97ac2995df..2496935cc2fe 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3421,7 +3421,8 @@ static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *loc struct walt_rq *wrq; wc = walt_sched_clock(); - walt_load_reported_window = atomic64_read(&walt_irq_work_lastq_ws); + if (!is_migration) + walt_load_reported_window = atomic64_read(&walt_irq_work_lastq_ws); for_each_sched_cluster(cluster) { u64 aggr_grp_load = 0; From 5a69b42f6b277848ff2cf2fdcd6a7223415dbbbd Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 7 Mar 2022 12:12:40 -0800 Subject: [PATCH 280/346] sched/walt/halt: use kthread to launch cpu drain operations cpu_drain_rq() will use stop_machine to schedule the draining of a cpu. This must not be called within an atomic context, and therefore constrains us as to where a halt operation can be initiated. To prepare for initiating halt/resume operations directly from the core_control_check/walt_irq_work context, use a kthread to create a context that will allow scheduling, and perform the drain in that context. Change-Id: Ic2d9232bebbebf2173cb77be43cf5ba90ce875d5 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_halt.c | 67 ++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 8e89a82d26c3..c622115adae4 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -23,6 +23,7 @@ struct halt_cpu_state { }; static DEFINE_PER_CPU(struct halt_cpu_state, halt_state); +static DEFINE_RAW_SPINLOCK(walt_drain_pending_lock); /* the amount of time allowed for enqueue operations that happen * just after a halt operation. @@ -191,6 +192,10 @@ static int cpu_drain_rq(unsigned int cpu) if (available_idle_cpu(cpu)) return 0; + if (!cpu_online(cpu)) + return 0; + + /* this will schedule, must not be in atomic context */ return stop_one_cpu(cpu, drain_rq_cpu_stop, NULL); } @@ -209,10 +214,49 @@ bool walt_halt_check_last(int cpu) return true; } +struct drain_thread_data { + cpumask_t cpus_to_drain; +}; + +static struct drain_thread_data drain_data = { + .cpus_to_drain = { CPU_BITS_NONE } +}; + +static int __ref try_drain_rqs(void *data) +{ + cpumask_t *cpus_ptr = &((struct drain_thread_data *)data)->cpus_to_drain; + int cpu; + unsigned long flags; + + raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); + + /* continue while there are cpus to drain + * assume at least one cpu to start. + */ + do { + cpumask_t local_cpus; + + cpumask_copy(&local_cpus, cpus_ptr); + raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); + + for_each_cpu(cpu, &local_cpus) + cpu_drain_rq(cpu); + + raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); + cpumask_andnot(cpus_ptr, cpus_ptr, &local_cpus); + + } while (cpumask_weight(cpus_ptr)); + + raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); + + return 0; +} + +struct task_struct *walt_drain_thread; + /* * 1) add the cpus to the halt mask * 2) migrate tasks off the cpu - * */ static int halt_cpus(struct cpumask *cpus) { @@ -220,6 +264,7 @@ static int halt_cpus(struct cpumask *cpus) int ret = 0; u64 start_time = sched_clock(); struct halt_cpu_state *halt_cpu_state; + unsigned long flags; trace_halt_cpus_start(cpus, 1); @@ -234,20 +279,14 @@ static int halt_cpus(struct cpumask *cpus) wmb(); halt_cpu_state->last_halt = start_time; - - /* only drain online cpus */ - if (cpu_online(cpu)) { - /* drain the online CPU */ - ret = cpu_drain_rq(cpu); - } - - if (ret < 0) { - /* cpu failed to drain, do not mark as halted */ - cpumask_clear_cpu(cpu, cpu_halt_mask); - break; - } } + /* signal and wakeup the drain kthread */ + raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); + cpumask_or(&drain_data.cpus_to_drain, &drain_data.cpus_to_drain, cpus); + raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); + wake_up_process(walt_drain_thread); + trace_halt_cpus(cpus, start_time, 1, ret); return ret; @@ -481,6 +520,8 @@ static void android_rvh_is_cpu_allowed(void *unused, int cpu, bool *allowed) void walt_halt_init(void) { + walt_drain_thread = kthread_run(try_drain_rqs, &drain_data, "halt_drain_rqs"); + register_trace_android_rvh_get_nohz_timer_target(android_rvh_get_nohz_timer_target, NULL); register_trace_android_rvh_set_cpus_allowed_ptr_locked( android_rvh_set_cpus_allowed_ptr_locked, NULL); From 4240156af1af6aadaa6fd569ccd5246b256e3cc7 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Mon, 7 Mar 2022 12:16:43 -0800 Subject: [PATCH 281/346] sched/walt/core_ctl: do core control immediately The current code requires an execution of the core control kthread in order to perform any pause/unpause operations. Because the mechanism to pause or unpause a cpu (halt) is only a bit-flip operation, this can cause unnecessary delay in the unpause path. Use spinlocks in halt, and call the main core control functionality immediately from the walt_irq_work context, eliminating this potential delay. Change-Id: I74d82c445097c1074ad106501b101a06875da38e Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 3 ++- kernel/sched/walt/walt_halt.c | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 52e78df1cd2d..6cee4ad08efd 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -85,6 +85,7 @@ ATOMIC_NOTIFIER_HEAD(core_ctl_notifier); static unsigned int last_nr_big; static unsigned int get_active_cpu_count(const struct cluster_data *cluster); +static void __ref do_core_ctl(void); /* ========================= sysfs interface =========================== */ @@ -995,7 +996,7 @@ void core_ctl_check(u64 window_start) wakeup |= eval_need(cluster); if (wakeup) - wake_up_core_ctl_thread(); + do_core_ctl(); core_ctl_call_notifier(); } diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index c622115adae4..abf9b328c9b6 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -15,7 +15,8 @@ /* if a cpu is halting */ struct cpumask __cpu_halt_mask; -static DEFINE_MUTEX(halt_lock); +/* spin lock to allow calling from non-preemptible context */ +static DEFINE_RAW_SPINLOCK(halt_lock); struct halt_cpu_state { u64 last_halt; @@ -359,8 +360,9 @@ int walt_halt_cpus(struct cpumask *cpus) { int ret = 0; cpumask_t requested_cpus; + unsigned long flags; - mutex_lock(&halt_lock); + raw_spin_lock_irqsave(&halt_lock, flags); cpumask_copy(&requested_cpus, cpus); @@ -380,7 +382,7 @@ int walt_halt_cpus(struct cpumask *cpus) else update_ref_counts(&requested_cpus, true); unlock: - mutex_unlock(&halt_lock); + raw_spin_unlock_irqrestore(&halt_lock, flags); return ret; } @@ -397,8 +399,9 @@ int walt_start_cpus(struct cpumask *cpus) { int ret = 0; cpumask_t requested_cpus; + unsigned long flags; - mutex_lock(&halt_lock); + raw_spin_lock_irqsave(&halt_lock, flags); cpumask_copy(&requested_cpus, cpus); update_ref_counts(&requested_cpus, false); @@ -414,7 +417,7 @@ int walt_start_cpus(struct cpumask *cpus) update_ref_counts(&requested_cpus, true); } - mutex_unlock(&halt_lock); + raw_spin_unlock_irqrestore(&halt_lock, flags); return ret; } From 327df46119a36764fef5baa3d631cd52d6344146 Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 21 Mar 2022 11:27:51 -0700 Subject: [PATCH 282/346] sched/walt: Optimize update_task_ravg() calls When the window is advanced to the next boundary, the CPU busy time trackers need to be advanced and top-task trackers should be flipped. This rollover needs to be done just once for a cpu per window advancement, while it could be subject to numerous update_task_ravg (called utra henceforth) with various tasks spanning window cross over point. The current code achieves this one-time-rollover only when it sees a utra called for the currently running task and its time has crossed over in the new window. As a result of this rollover being done only when utra is called for current task, it became mandatory to call utra on current task before calling utra on a non-current task. This ensured a proper window rollover i.e. the trackers reset/flipped if needed prior to calling utra on non-current task which may have entered its execution in a new window. Explaining pictorially old new ws ms ws wc | | | | V V V V -----|----------------|-------- prev curr In the above example, a non current task wc may have entered the new window but prev and curr are still set as per the old window start. The time from new ws to wc cannot be accounted because curr(_runnable_sum) hasnt advanced i.e hasn't rolled over. Hence utra is called with the current task which causes a rollover i.e. it advances prev, curr(_runnable_sum) pointers amongst other things old new ws ms ws wc | | | | V V V V -----|----------------|-------- prev curr We see utra on current being called, when a task 'p' is woken up, two update_task_ravg() calls are needed. update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); when a task 'p' is migrating, three update_task_ravg() calls are needed. update_task_ravg(task_rq(p)->curr, task_rq(p), TASK_UPDATE, wallclock, 0); update_task_ravg(dest_rq->curr, dest_rq, TASK_UPDATE, wallclock, 0); update_task_ravg(p, task_rq(p), TASK_MIGRATE, wallclock, 0); when a task 'p' is tranferring in/out of rtg update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); We can optimize the number of update_task_ravg() calls by allowing to rollover regardless of utra on current task. The best place to do it is in update_window_start() right along the code which advances the window start. Since the update is not happening on the running task i.e current, the CPU utilization i.e prev_runnable_sum does not reflect the current task utilization immediately after the window is rolled over. However update_task_ravg() is called on the current before presenting the utilization to the governor. So there should not be any impact on the final frequency selection. A TASK_WAKE of a iowaited task needs to terminate the accumulation of iowait time on the cpu where the task had slept. The mark_start of the idle task is advanced and the "advanced" time is accounted as cpu busy time. For this situation alone call utra on idle task if an iowaited task is waking up. Change-Id: Ifb771b41bd62f3c748035bf1e5e5ad7912b400f4 Signed-off-by: Pavankumar Kondeti Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 70 ++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 2496935cc2fe..8a57b2c96f52 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -342,6 +342,9 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, pred_demand_delta); } +static void rollover_cpu_window(struct rq *rq, bool full_window); +static void rollover_top_tasks(struct rq *rq, bool full_window); + /* * Demand aggregation for frequency purpose: * @@ -394,6 +397,7 @@ update_window_start(struct rq *rq, u64 wallclock, int event) int nr_windows; struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; u64 old_window_start = wrq->window_start; + bool full_window; if (wallclock < wrq->latest_clock) { printk_deferred("WALT-BUG CPU%d; wallclock=%llu(0x%llx) is lesser than latest_clock=%llu(0x%llx)", @@ -417,6 +421,10 @@ update_window_start(struct rq *rq, u64 wallclock, int event) wrq->prev_window_size = sched_ravg_window; + full_window = nr_windows > 1; + rollover_cpu_window(rq, full_window); + rollover_top_tasks(rq, full_window); + return old_window_start; } @@ -1010,14 +1018,20 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on src_rq %d", raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); - walt_update_task_ravg(task_rq(p)->curr, task_rq(p), - TASK_UPDATE, - wallclock, 0); - walt_update_task_ravg(dest_rq->curr, dest_rq, - TASK_UPDATE, wallclock, 0); - - walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, - wallclock, 0); + walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, wallclock, 0); + /* + * The above update might have rolled over the + * window for this migrating task. Since we are + * going to adjust the destination CPU's busy time + * counters with the task busytime counters, roll over + * the window for the destination CPU also. + * + * The update_window_start() does nothing if the window + * is not rolled over, so there is no need to check for + * window boundary or if the counters will be accessed + * or not. + */ + update_window_start(dest_rq, wallclock, TASK_UPDATE); update_task_cpu_cycles(p, new_cpu, wallclock); @@ -1586,11 +1600,6 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, new_task = is_new_task(p); - if (p_is_curr_task && new_window) { - rollover_cpu_window(rq, full_window); - rollover_top_tasks(rq, full_window); - } - if (!account_busy_for_cpu_time(rq, p, irqtime, event)) goto done; @@ -1631,15 +1640,17 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, goto done; } + /* + * situations below this need window rollover, which should already be done + * in update_window_start() + */ + if (!p_is_curr_task) { /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has also started, but p is not the current task, so the - * window is not rolled over - just split up and account - * as necessary into curr and prev. The window is only - * rolled over when a new window is processed for the current - * task. + * must have been started in udpate_window_start() + * - just split up and account as necessary into curr and prev. * * Irqtime can't be accounted by a task that isn't the * currently running task. @@ -1684,8 +1695,8 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. If any of these three above conditions are true + * must have been started in udpate_window_start() + * If any of these three above conditions are true * then this busy time can't be accounted as irqtime. * * Busy time for the idle task need not be accounted. @@ -1717,10 +1728,6 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, } } - /* - * Rollover is done here by overwriting the values in - * prev_runnable_sum and curr_runnable_sum. - */ *prev_runnable_sum += delta; if (new_task) *nt_prev_runnable_sum += delta; @@ -1743,8 +1750,8 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* * account_busy_for_cpu_time() = 1 so busy time needs * to be accounted to the current window. A new window - * has started and p is the current task so rollover is - * needed. The current task must be the idle task because + * must have been started in udpate_window_start() + * The current task must be the idle task because * irqtime is not accounted for any other task. * * Irqtime will be accounted each time we process IRQ activity @@ -1756,11 +1763,11 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, mark_start = wallclock - irqtime; /* - * Roll window over. If IRQ busy time was just in the current + * If IRQ busy time was just in the current * window then that is all that need be accounted. */ if (mark_start > window_start) { - *curr_runnable_sum = scale_exec_time(irqtime, rq); + *curr_runnable_sum += scale_exec_time(irqtime, rq); return; } @@ -1776,7 +1783,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* Process the remaining IRQ busy time in the current window. */ delta = wallclock - window_start; - wrq->curr_runnable_sum = scale_exec_time(delta, rq); + wrq->curr_runnable_sum += scale_exec_time(delta, rq); return; } @@ -3187,7 +3194,6 @@ static void transfer_busy_time(struct rq *rq, wallclock = walt_sched_clock(); - walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); new_task = is_new_task(p); @@ -4088,7 +4094,9 @@ static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) rq_lock_irqsave(rq, &rf); old_load = task_load(p); wallclock = walt_sched_clock(); - walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + + if (is_idle_task(rq->curr) && p->in_iowait) + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); note_task_waking(p, wallclock); rq_unlock_irqrestore(rq, &rf); From df9aed75af1de2ace5098ed25c8c4e93285fb589 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Mon, 21 Mar 2022 18:23:24 -0700 Subject: [PATCH 283/346] sched/walt: Improve comments Document irqtime and cleanup window roll over explanation. Change-Id: I2fc092dd07926e35b1bf2f1839d56aa396fc666f Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Ashay Jaiswal --- kernel/sched/walt/walt.c | 55 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 8a57b2c96f52..81bb18aee352 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1564,6 +1564,50 @@ static void rollover_cpu_window(struct rq *rq, bool full_window) /* * Account cpu activity in its * busy time counters(wrq->curr/prev_runnable_sum) + * + * While the comments at the top of update_task_demand() apply, irqtime handling + * needs some explanation. + * + * Note that update_task_ravg() with irqtime is only called when idle, i.e. p is + * always idle + * + * ms_i = mark_start of idle task + * ws = wrq->window_start + * irq_s = start time of irq + * irq_e = end time of irq = wallclock + * + * note irqtime = irq_e - irq_s + * + * Similar to the explanation at update_task_demand() we have few sitautions for irqtime + * + * ws ms_i is ie + * | | | | + * V V V V + * --------|--------------------| + * prev curr + * + * In the above case, new_window is false and irqtime is accounted in curr_runnable_sum, this is + * done in the if (!new_window) block. + * + * ms_i ws is ie + * | | | | + * V V V V + * -------------|--------------------- + * prev curr + * + * In this case, new_window is true, however the irqtime falls within the current window, the + * entire irqtime is accounted in curr_runnable_sum. This is handled in the if (irqtime) block and + * within that if (mark_start > window_start) block + * + * ms_i is ws ie + * | | | | + * V V V V + * --------------------|--------------- + * prev curr + * + * In this case, new_window is true, portion of the irqtime needs to be accounted in + * prev_runnable_sum while the rest is in curr_runnable_sum. This is handled in the + * if (irqtime) block */ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, int event, u64 wallclock, u64 irqtime) @@ -1641,8 +1685,16 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, } /* - * situations below this need window rollover, which should already be done + * situations below this need window rollover, + * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done * in update_window_start() + * + * For task counters curr/prev_window[_cpu] are rolled over in the early part of + * this function. If full_window(s) have expired and time since last update needs + * to be accounted as busy time, set the prev to a complete window size time, else + * add the prev window portion. + * + * For task curr counters a new window has begun, always assign */ if (!p_is_curr_task) { @@ -1760,6 +1812,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, */ WALT_PANIC(!is_idle_task(p)); + /* mark_start here becomes the starting time of interrupt */ mark_start = wallclock - irqtime; /* From 40a3bf6633e8b71cc9f6851c3633d78045864b43 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 25 Mar 2022 15:48:33 -0700 Subject: [PATCH 284/346] sched/walt: stop accounting stopper threads as long running RT A stopper thread has priority 0, and therefore, it can be classified as an RT thread. However, in certain usecases, we may be executing for a prolonged duration under stopper context, and as such would hit long running RT task notifier unnecessarily. Change-Id: Ibe5c7aa447292a6a577d405f6e5ae8cf9cd2842c Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_rt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index feb579f00c0f..d4dd39edee5e 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -19,7 +19,7 @@ void rt_task_arrival_marker(void *unused, bool preempt, { unsigned int cpu = raw_smp_processor_id(); - if (rt_task(next)) + if (rt_task(next) && next != cpu_rq(cpu)->stop) per_cpu(rt_task_arrival_time, cpu) = rq_clock_task(this_rq()); else per_cpu(rt_task_arrival_time, cpu) = 0; From 9811092a16452beba2c21952c3b44c528c022168 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 24 Mar 2022 20:34:55 -0700 Subject: [PATCH 285/346] sched/walt: print all history samples The history samples were recently increased from 5 to 8. Reflect the same in the trace outputs to print out all 8 history samples. Change-Id: Ib79dbd193a643bf65c056d2f1cc0ba357ce3cdaa Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 98e87c5c43d6..cf32709f86c7 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -103,14 +103,17 @@ TRACE_EVENT(sched_update_history, __entry->cpu = rq->cpu; ), - TP_printk("%d (%s): runtime %u samples %d event %s demand %u coloc_demand %u pred_demand_scaled %u (hist: %u %u %u %u %u) cpu %d nr_big %u", + TP_printk("%d (%s): runtime %u samples %d event %s demand %u (hist: %u %u %u %u %u %u %u %u) coloc_demand %u pred_demand_scaled %u cpu %d nr_big %u", __entry->pid, __entry->comm, __entry->runtime, __entry->samples, task_event_names[__entry->evt], - __entry->demand, __entry->coloc_demand, __entry->pred_demand_scaled, + __entry->demand, __entry->hist[0], __entry->hist[1], __entry->hist[2], __entry->hist[3], - __entry->hist[4], __entry->cpu, __entry->nr_big_tasks) + __entry->hist[4], __entry->hist[5], + __entry->hist[6], __entry->hist[7], + __entry->coloc_demand, __entry->pred_demand_scaled, + __entry->cpu, __entry->nr_big_tasks) ); TRACE_EVENT(sched_get_task_cpu_cycles, From 0d9c99093d5a7c2eb4bf1fb12639ad8c90067e85 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Mon, 28 Mar 2022 15:58:38 -0700 Subject: [PATCH 286/346] sched/walt/rt: limit long running RT task detection to FIFO Right now, it is possible to detect the long running RT tasks false positively in scenarios where a DL or RR task could be running as well. The intention behind long running RT task detection is to catch any long running FIFO tasks. Update check to make sure RT arrival time is updated only for FIFO tasks to limit long running RT task detection. Change-Id: Ide59791b40acd5e9529af23217c16350278c5f60 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_rt.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index d4dd39edee5e..98f8974b8bba 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2020-2022, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -19,7 +20,7 @@ void rt_task_arrival_marker(void *unused, bool preempt, { unsigned int cpu = raw_smp_processor_id(); - if (rt_task(next) && next != cpu_rq(cpu)->stop) + if (next->policy == SCHED_FIFO) per_cpu(rt_task_arrival_time, cpu) = rq_clock_task(this_rq()); else per_cpu(rt_task_arrival_time, cpu) = 0; From c26c551ed20871d7a7ebb3031a8d1297bb8a1ad1 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 24 Mar 2022 11:59:33 -0700 Subject: [PATCH 287/346] sched/walt: Account for changing task affinity While performing a move of a task from one CPU to another, there's a possibility of a change in task's affinity occurring between detaching the task from previous CPU and setting new CPU of task. As a result, the new affinity of the task may not support the move. Account for this scenario by adding appropriate checks in the detaching task path and changing the destination of the move, if necessary. Change-Id: I25c456f4ed0131eabae48ab1fc82cc4a29ce5e7a Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_lb.c | 70 +++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index ecc873d69a30..2b60160b9d33 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -16,15 +16,53 @@ static inline unsigned long walt_lb_cpu_util(int cpu) return wrq->walt_stats.cumulative_runnable_avg_scaled; } -static void walt_detach_task(struct task_struct *p, struct rq *src_rq, - struct rq *dst_rq) + +static int walt_detach_task(struct task_struct *p, struct rq *src_rq, + struct rq *dst_rq, bool force_affinity_appropriate_cpu) { + int ret = -EINVAL; + int dest_cpu; + int retry_count = 0; + deactivate_task(src_rq, p, 0); +retry: double_lock_balance(src_rq, dst_rq); - if (!(src_rq->clock_update_flags & RQCF_UPDATED)) - update_rq_clock(src_rq); - set_task_cpu(p, dst_rq->cpu); + /* + * It's possible that src_rq lock was dropped while trying to acuire + * double rq lock. + * Task affinity could change then, recheck if moving the task to + * dst is still valid. + */ + if (!force_affinity_appropriate_cpu || + cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { + if (!(src_rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(src_rq); + set_task_cpu(p, dst_rq->cpu); + ret = 0; + } double_unlock_balance(src_rq, dst_rq); + + if (ret) { + /* + * Couldn't move the task, reactivate it on an appropriate cpu. + */ + if (cpumask_test_cpu(cpu_of(src_rq), p->cpus_ptr) && + cpu_active(cpu_of(src_rq)) && + !cpu_halted(cpu_of(src_rq))) { + activate_task(src_rq, p, 0); + } else { + dest_cpu = select_fallback_rq(cpu_of(src_rq), p); + dst_rq = cpu_rq(dest_cpu); + if (retry_count < 5) { + retry_count++; + goto retry; + } else { + printk_deferred("Failed to select CPU in task's affinity list after 5 tries\n"); + } + } + } + + return ret; } static void walt_attach_task(struct task_struct *p, struct rq *rq) @@ -64,8 +102,8 @@ static int stop_walt_lb_active_migration(void *data) task_cpu(push_task) == busiest_cpu && cpu_active(target_cpu) && cpumask_test_cpu(target_cpu, push_task->cpus_ptr)) { - walt_detach_task(push_task, busiest_rq, target_rq); - push_task_detached = 1; + if (!walt_detach_task(push_task, busiest_rq, target_rq, true)) + push_task_detached = 1; } out_unlock: /* called with busiest_rq lock */ @@ -314,7 +352,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) if (!_walt_can_migrate_task(p, dst_cpu, to_lower, false)) continue; - walt_detach_task(p, src_rq, dst_rq); + if (walt_detach_task(p, src_rq, dst_rq, true)) + continue; pulled_task = p; goto unlock; } @@ -330,7 +369,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) if (!_walt_can_migrate_task(p, dst_cpu, to_lower, true)) continue; - walt_detach_task(p, src_rq, dst_rq); + if (walt_detach_task(p, src_rq, dst_rq, true)) + continue; pulled_task = p; goto unlock; } @@ -370,7 +410,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) continue; } - walt_detach_task(p, src_rq, dst_rq); + if (walt_detach_task(p, src_rq, dst_rq, true)) + continue; pulled_task = p; goto unlock; } @@ -975,7 +1016,14 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, BUG_ON(!rf); rq_unpin_lock(rq, rf); - walt_detach_task(p, rq, cpu_rq(new_cpu)); + /* + * force_affinity_appropriate_cpu set to false allows p to be put on + * a non-affinity-appropriate cpu. We shouldn't move the task to any + * other cpu than new_cpu, even if affinity doesn't allow it, because + * calls to migrate_queued_task are accompanied with a + * activate_task on the new_cpu. + */ + walt_detach_task(p, rq, cpu_rq(new_cpu), false); rq_repin_lock(rq, rf); *detached = 1; From a2ed1ebb5d78f5116e32354fa65b1ba6ed1cddf6 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 4 Apr 2022 08:53:01 -0700 Subject: [PATCH 288/346] sched: Align window start with tick boundary As a side effect of the rq_clock optimization patch, the window_start initialized in the stop handler is no longer aligned with the tick boundary, which causes undesired effects as observed in APT usecases. Change-Id: I82a0c03fcdcdc18b47e382c47c94fd84680e978a Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 81bb18aee352..c2612cde457c 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -19,7 +19,6 @@ #include #include #include - #include "walt.h" #include "trace.h" @@ -4175,14 +4174,25 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct raw_spin_unlock_irqrestore(&cpu_rq(cpu)->__lock, flags); } +u64 tick_sched_clock; +static DECLARE_COMPLETION(tick_sched_clock_completion); + static void android_rvh_tick_entry(void *unused, struct rq *rq) { u64 wallclock; + if (!tick_sched_clock) { + /* + * Let the window begin 20us prior to the tick, + * that way we are guaranteed a rollover when the tick occurs. + */ + tick_sched_clock = rq_clock(rq) - 20000; + complete_all(&tick_sched_clock_completion); + } + lockdep_assert_held(&rq->__lock); if (unlikely(walt_disabled)) return; - set_window_start(rq); wallclock = walt_rq_clock(rq); @@ -4329,7 +4339,6 @@ static int walt_init_stop_handler(void *data) { int cpu; struct task_struct *g, *p; - u64 window_start_ns, nr_windows; struct walt_rq *wrq; read_lock(&tasklist_lock); @@ -4341,10 +4350,6 @@ static int walt_init_stop_handler(void *data) init_existing_task_load(p); } while_each_thread(g, p); - window_start_ns = walt_sched_clock(); - nr_windows = div64_u64(window_start_ns, sched_ravg_window); - window_start_ns = (u64)nr_windows * (u64)sched_ravg_window; - for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); @@ -4354,10 +4359,10 @@ static int walt_init_stop_handler(void *data) walt_sched_init_rq(rq); wrq = (struct walt_rq *) rq->android_vendor_data1; - wrq->window_start = window_start_ns; + wrq->window_start = tick_sched_clock; } - atomic64_set(&walt_irq_work_lastq_ws, window_start_ns); + atomic64_set(&walt_irq_work_lastq_ws, tick_sched_clock); create_default_coloc_group(); @@ -4408,7 +4413,7 @@ static void walt_init(struct work_struct *work) walt_rt_init(); walt_cfs_init(); walt_halt_init(); - + wait_for_completion(&tick_sched_clock_completion); stop_machine(walt_init_stop_handler, NULL, NULL); hdr = register_sysctl_table(walt_base_table); From 498e57c0280448edfb4767a8626d2ed682b4334a Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 18 Jan 2022 14:21:56 -0800 Subject: [PATCH 289/346] sched/walt: Add parameter for CPUFREQ reasons Currently, cpufreq traces contain a "flags" parameter, which helps understand why cpufreq updates were triggered. In this patch, a "reasons" parameter is introduced to provide an accurate guidance towards *why* the cpufreq chosen was selected, and the specific reason behind any applied value adds that have potentially modified the true load. Change-Id: I88a62569e0a1ec1555a89161aa532db6c72df334 Signed-off-by: Stephen Dickey Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 35 ++++++++++++++++++++++------- kernel/sched/walt/trace.h | 30 ++++++++++++++++--------- kernel/sched/walt/walt.c | 38 +++++++++++++++++++++----------- kernel/sched/walt/walt.h | 17 ++++++++++++-- 4 files changed, 87 insertions(+), 33 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index fa8b913bb2d6..54b22401b581 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -50,6 +50,7 @@ struct waltgov_policy { s64 down_rate_delay_ns; unsigned int next_freq; unsigned int cached_raw_freq; + unsigned int driving_cpu; /* The next fields are only needed if fast switch cannot be used: */ struct irq_work irq_work; @@ -70,6 +71,7 @@ struct waltgov_cpu { unsigned long util; unsigned long max; unsigned int flags; + unsigned int reasons; }; DEFINE_PER_CPU(struct waltgov_callback *, waltgov_cb_data); @@ -224,19 +226,24 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, { struct cpufreq_policy *policy = wg_policy->policy; unsigned int freq, raw_freq, final_freq; + struct waltgov_cpu *wg_driv_cpu = &per_cpu(waltgov_cpu, wg_policy->driving_cpu); raw_freq = walt_map_util_freq(util, wg_policy, max, wg_cpu->cpu); freq = raw_freq; if (wg_policy->tunables->adaptive_high_freq) { - if (raw_freq < wg_policy->tunables->adaptive_low_freq) + if (raw_freq < wg_policy->tunables->adaptive_low_freq) { freq = wg_policy->tunables->adaptive_low_freq; - else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) + wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_LOW; + } else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) { freq = wg_policy->tunables->adaptive_high_freq; + wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_HIGH; + } } trace_waltgov_next_freq(policy->cpu, util, max, raw_freq, freq, policy->min, policy->max, - wg_policy->cached_raw_freq, wg_policy->need_freq_update); + wg_policy->cached_raw_freq, wg_policy->need_freq_update, + wg_driv_cpu->cpu, wg_driv_cpu->reasons); if (wg_policy->cached_raw_freq && freq == wg_policy->cached_raw_freq && !wg_policy->need_freq_update) @@ -259,7 +266,8 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) unsigned long util; wg_cpu->max = max; - util = cpu_util_freq_walt(wg_cpu->cpu, &wg_cpu->walt_load); + wg_cpu->reasons = 0; + util = cpu_util_freq_walt(wg_cpu->cpu, &wg_cpu->walt_load, &wg_cpu->reasons); return uclamp_rq_util_with(rq, util, NULL); } @@ -270,6 +278,16 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) #define DEFAULT_PRIME_RTG_BOOST_FREQ 0 #define DEFAULT_TARGET_LOAD_THRESH 1024 #define DEFAULT_TARGET_LOAD_SHIFT 4 +static inline void max_and_reason(unsigned long *cur_util, unsigned long boost_util, + struct waltgov_cpu *wg_cpu, unsigned int reason) +{ + if (boost_util >= *cur_util) { + *cur_util = boost_util; + wg_cpu->reasons = reason; + wg_cpu->wg_policy->driving_cpu = wg_cpu->cpu; + } +} + static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_util, unsigned long nl, unsigned long *util, unsigned long *max) @@ -281,22 +299,22 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut unsigned long pl = wg_cpu->walt_load.pl; if (is_rtg_boost) - *util = max(*util, wg_policy->rtg_boost_util); + max_and_reason(util, wg_policy->rtg_boost_util, wg_cpu, CPUFREQ_REASON_RTG_BOOST); is_hiload = (cpu_util >= mult_frac(wg_policy->avg_cap, wg_policy->tunables->hispeed_load, 100)); if (is_hiload && !is_migration) - *util = max(*util, wg_policy->hispeed_util); + max_and_reason(util, wg_policy->hispeed_util, wg_cpu, CPUFREQ_REASON_HISPEED); if (is_hiload && nl >= mult_frac(cpu_util, NL_RATIO, 100)) - *util = *max; + max_and_reason(util, *max, wg_cpu, CPUFREQ_REASON_NWD); if (wg_policy->tunables->pl) { if (sysctl_sched_conservative_pl) pl = mult_frac(pl, TARGET_LOAD, 100); - *util = max(*util, pl); + max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL); } } @@ -346,6 +364,7 @@ static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 tim if (j_util * max >= j_max * util) { util = j_util; max = j_max; + wg_policy->driving_cpu = j; } waltgov_walt_adjust(j_wg_cpu, j_util, j_nl, &util, &max); diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index cf32709f86c7..2407d60c3ec1 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #undef TRACE_SYSTEM @@ -421,9 +421,10 @@ TRACE_EVENT(sched_load_to_gov, int freq_aggr, u64 load, int policy, int big_task_rotation, unsigned int user_hint, - struct walt_rq *wrq), + struct walt_rq *wrq, + unsigned int reasons), TP_ARGS(rq, aggr_grp_load, tt_load, freq_aggr, load, policy, - big_task_rotation, user_hint, wrq), + big_task_rotation, user_hint, wrq, reasons), TP_STRUCT__entry( __field(int, cpu) @@ -440,6 +441,7 @@ TRACE_EVENT(sched_load_to_gov, __field(u64, load) __field(int, big_task_rotation) __field(unsigned int, user_hint) + __field(unsigned int, reasons) ), TP_fast_assign( @@ -458,14 +460,15 @@ TRACE_EVENT(sched_load_to_gov, __entry->load = load; __entry->big_task_rotation = big_task_rotation; __entry->user_hint = user_hint; + __entry->reasons = reasons; ), - TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr=%d tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d user_hint=%u", + TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr=%d tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d user_hint=%u reasons=0x%x", __entry->cpu, __entry->policy, __entry->ed_task_pid, __entry->aggr_grp_load, __entry->freq_aggr, __entry->tt_load, __entry->rq_ps, __entry->grp_rq_ps, __entry->nt_ps, __entry->grp_nt_ps, __entry->pl, __entry->load, - __entry->big_task_rotation, __entry->user_hint) + __entry->big_task_rotation, __entry->user_hint, __entry->reasons) ); TRACE_EVENT(core_ctl_eval_need, @@ -721,9 +724,10 @@ TRACE_EVENT(waltgov_util_update, TRACE_EVENT(waltgov_next_freq, TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, unsigned int raw_freq, unsigned int freq, unsigned int policy_min_freq, unsigned int policy_max_freq, - unsigned int cached_raw_freq, bool need_freq_update), + unsigned int cached_raw_freq, bool need_freq_update, unsigned int driving_cpu, + unsigned int reason), TP_ARGS(cpu, util, max, raw_freq, freq, policy_min_freq, policy_max_freq, - cached_raw_freq, need_freq_update), + cached_raw_freq, need_freq_update, driving_cpu, reason), TP_STRUCT__entry( __field(unsigned int, cpu) __field(unsigned long, util) @@ -735,6 +739,8 @@ TRACE_EVENT(waltgov_next_freq, __field(unsigned int, cached_raw_freq) __field(bool, need_freq_update) __field(unsigned int, rt_util) + __field(unsigned int, driving_cpu) + __field(unsigned int, reason) ), TP_fast_assign( __entry->cpu = cpu; @@ -746,9 +752,11 @@ TRACE_EVENT(waltgov_next_freq, __entry->policy_max_freq = policy_max_freq; __entry->cached_raw_freq = cached_raw_freq; __entry->need_freq_update = need_freq_update; - __entry->rt_util = cpu_util_rt(cpu_rq(cpu)); + __entry->rt_util = cpu_util_rt(cpu_rq(cpu)); + __entry->driving_cpu = driving_cpu; + __entry->reason = reason; ), - TP_printk("cpu=%u util=%lu max=%lu raw_freq=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u", + TP_printk("cpu=%u util=%lu max=%lu raw_freq=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u driv_cpu=%u reason=0x%x", __entry->cpu, __entry->util, __entry->max, @@ -758,7 +766,9 @@ TRACE_EVENT(waltgov_next_freq, __entry->policy_max_freq, __entry->cached_raw_freq, __entry->need_freq_update, - __entry->rt_util) + __entry->rt_util, + __entry->driving_cpu, + __entry->reason) ); TRACE_EVENT(walt_active_load_balance, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c2612cde457c..30052ac15cd7 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -590,30 +590,41 @@ should_apply_suh_freq_boost(struct walt_sched_cluster *cluster) return is_cluster_hosting_top_app(cluster); } -static inline u64 freq_policy_load(struct rq *rq) +static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_sched_cluster *cluster = wrq->cluster; u64 aggr_grp_load = cluster->aggr_grp_load; - u64 load, tt_load = 0; + u64 load, tt_load = 0, kload = 0; struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu_of(rq)); if (wrq->ed_task != NULL) { load = sched_ravg_window; + *reason = CPUFREQ_REASON_EARLY_DET; goto done; } - if (sched_freq_aggr_en) + if (sched_freq_aggr_en) { load = wrq->prev_runnable_sum + aggr_grp_load; + *reason = CPUFREQ_REASON_FREQ_AGR; + } else load = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; - if (cpu_ksoftirqd && READ_ONCE(cpu_ksoftirqd->__state) == TASK_RUNNING) - load = max_t(u64, load, task_load(cpu_ksoftirqd)); + if (cpu_ksoftirqd && READ_ONCE(cpu_ksoftirqd->__state) == TASK_RUNNING) { + kload = task_load(cpu_ksoftirqd); + if (kload > load) { + load = kload; + *reason = CPUFREQ_REASON_KSOFTIRQD; + } + } tt_load = top_task_load(rq); - load = max_t(u64, load, tt_load); + if (tt_load > load) { + load = tt_load; + *reason = CPUFREQ_REASON_TT_LOAD; + } if (should_apply_suh_freq_boost(cluster)) { if (is_suh_max()) @@ -621,26 +632,27 @@ static inline u64 freq_policy_load(struct rq *rq) else load = div64_u64(load * sysctl_sched_user_hint, (u64)100); + *reason = CPUFREQ_REASON_SUH; } done: trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en, load, 0, walt_rotation_enabled, - sysctl_sched_user_hint, wrq); + sysctl_sched_user_hint, wrq, *reason); return load; } static bool rtgb_active; static inline unsigned long -__cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) +__cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reason) { u64 util; struct rq *rq = cpu_rq(cpu); unsigned long capacity = capacity_orig_of(cpu); struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - util = div64_u64(freq_policy_load(rq), + util = div64_u64(freq_policy_load(rq, reason), sched_ravg_window >> SCHED_CAPACITY_SHIFT); if (walt_load) { @@ -665,7 +677,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) (max(orig, mult_frac(other, x, 100))) unsigned long -cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) +cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reason) { struct walt_cpu_load wl_other = {0}; unsigned long util = 0, util_other = 0; @@ -673,13 +685,13 @@ cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) int i, mpct = sysctl_sched_asym_cap_sibling_freq_match_pct; if (!cpumask_test_cpu(cpu, &asym_cap_sibling_cpus)) - return __cpu_util_freq_walt(cpu, walt_load); + return __cpu_util_freq_walt(cpu, walt_load, reason); for_each_cpu(i, &asym_cap_sibling_cpus) { if (i == cpu) - util = __cpu_util_freq_walt(cpu, walt_load); + util = __cpu_util_freq_walt(cpu, walt_load, reason); else - util_other = __cpu_util_freq_walt(i, &wl_other); + util_other = __cpu_util_freq_walt(i, &wl_other, reason); } if (cpu == cpumask_last(&asym_cap_sibling_cpus)) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index d1f8f3b0781a..9e573452d642 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _WALT_H @@ -297,6 +297,18 @@ extern unsigned int sched_lib_mask_force; #define WALT_CPUFREQ_EARLY_DET (1U << 4) #define WALT_CPUFREQ_BOOST_UPDATE (1U << 5) +#define CPUFREQ_REASON_PL (1U << 1) +#define CPUFREQ_REASON_EARLY_DET (1U << 2) +#define CPUFREQ_REASON_RTG_BOOST (1U << 3) +#define CPUFREQ_REASON_HISPEED (1U << 4) +#define CPUFREQ_REASON_NWD (1U << 5) +#define CPUFREQ_REASON_FREQ_AGR (1U << 6) +#define CPUFREQ_REASON_KSOFTIRQD (1U << 7) +#define CPUFREQ_REASON_TT_LOAD (1U << 8) +#define CPUFREQ_REASON_SUH (1U << 9) +#define CPUFREQ_REASON_ADAPTIVE_LOW (1U << 10) +#define CPUFREQ_REASON_ADAPTIVE_HIGH (1U << 11) + #define NO_BOOST 0 #define FULL_THROTTLE_BOOST 1 #define CONSERVATIVE_BOOST 2 @@ -366,7 +378,8 @@ static inline void waltgov_run_callback(struct rq *rq, unsigned int flags) cb->func(cb, walt_sched_clock(), flags); } -extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load); +extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, + unsigned int *reason); int waltgov_register(void); extern void walt_lb_init(void); From b2f69d86bbe857b1b6445ec06437e2703e999759 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 5 Apr 2022 11:51:16 -0700 Subject: [PATCH 290/346] sched/walt/core_control: disable need_all_cpus() need_all_cpus() was put in place in past products, so that silver isolation could be enabled and disabled based upon the window size, as a work around for lack of user-space controls for disabling/enabling core_control. This blocks the ability to isolate silver cores for lower window sizes, which is a valid case that should be allowed for most products. Cleanup the code for need all cpus, such that silver core control may be used. Change-Id: I0e029a6c0df5eb1207c549cf843614c360e91b8e Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 6cee4ad08efd..12abfd26947f 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -784,12 +784,6 @@ static bool adjustment_possible(const struct cluster_data *cluster, cluster_paused_cpus(cluster))); } -static bool need_all_cpus(const struct cluster_data *cluster) -{ - return (is_min_cluster_cpu(cluster->first_cpu) && - sched_ravg_window < DEFAULT_SCHED_RAVG_WINDOW); -} - static bool eval_need(struct cluster_data *cluster) { unsigned long flags; @@ -805,7 +799,7 @@ static bool eval_need(struct cluster_data *cluster) spin_lock_irqsave(&state_lock, flags); - if (cluster->boost || !cluster->enable || need_all_cpus(cluster)) { + if (cluster->boost || !cluster->enable) { need_cpus = cluster->max_cpus; } else { cluster->active_cpus = get_active_cpu_count(cluster); From 851ffc330b360eca802bccdd95b4eb66c2e017e7 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 5 Apr 2022 13:47:56 -0700 Subject: [PATCH 291/346] sched/walt: Add yield case to LRRT task detection Reset the arrival time of the RT task when task yields. This way, time spent during and after yield is not conidered towards RT task runtime. Change-Id: I8f961bca1ee7bac67dafa0238594fc4a444fdcd1 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 14 ++++++++++++++ kernel/sched/walt/walt.h | 4 ++++ kernel/sched/walt/walt_cfs.c | 16 ++-------------- kernel/sched/walt/walt_rt.c | 2 +- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 30052ac15cd7..d3269cdbd883 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4315,6 +4315,19 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) *eas_check = true; } +static void walt_do_sched_yield(void *unused, struct rq *rq) +{ + struct task_struct *curr = rq->curr; + int mvp_prio = walt_get_mvp_task_prio(curr); + + lockdep_assert_held(&rq->__lock); + if (mvp_prio != WALT_NOT_MVP) + walt_cfs_deactivate_mvp_task(curr); + + if (rt_task(curr)) + per_cpu(rt_task_arrival_time, cpu_of(rq)) = 0; +} + static void register_walt_hooks(void) { register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); @@ -4342,6 +4355,7 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); + register_trace_android_rvh_do_sched_yield(walt_do_sched_yield, NULL); } atomic64_t walt_irq_work_lastq_ws; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 9e573452d642..df1307b45ed4 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -936,6 +936,10 @@ extern int in_sched_bug; extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, struct task_struct *p, int dest_cpu); +extern DEFINE_PER_CPU(u64, rt_task_arrival_time); +extern int walt_get_mvp_task_prio(struct task_struct *p); +extern void walt_cfs_deactivate_mvp_task(struct task_struct *p); + enum WALT_DEBUG_FEAT { WALT_BUG_UPSTREAM, WALT_BUG_WALT, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 3bb1bdc4edeb..a9d9faaf139b 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1018,7 +1018,7 @@ static void binder_restore_priority_hook(void *data, * they can preempt long running rtg prio tasks but binders loose their * powers with in 3 msec where as rtg prio tasks can run more than that. */ -static inline int walt_get_mvp_task_prio(struct task_struct *p) +int walt_get_mvp_task_prio(struct task_struct *p) { if (per_task_boost(p) == TASK_BOOST_STRICT_MAX) return WALT_TASK_BOOST_MVP; @@ -1065,7 +1065,7 @@ static void walt_cfs_insert_mvp_task(struct walt_rq *wrq, struct walt_task_struc list_add(&wts->mvp_list, pos->prev); } -static void walt_cfs_deactivate_mvp_task(struct task_struct *p) +void walt_cfs_deactivate_mvp_task(struct task_struct *p) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -1132,16 +1132,6 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr walt_cfs_insert_mvp_task(wrq, wts, false); } -static void walt_cfs_mvp_do_sched_yield(void *unused, struct rq *rq) -{ - struct task_struct *curr = rq->curr; - int mvp_prio = walt_get_mvp_task_prio(curr); - - lockdep_assert_held(&rq->__lock); - if (mvp_prio != WALT_NOT_MVP) - walt_cfs_deactivate_mvp_task(curr); -} - void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; @@ -1349,6 +1339,4 @@ void walt_cfs_init(void) register_trace_android_rvh_check_preempt_wakeup(walt_cfs_check_preempt_wakeup, NULL); register_trace_android_rvh_replace_next_task_fair(walt_cfs_replace_next_task_fair, NULL); - - register_trace_android_rvh_do_sched_yield(walt_cfs_mvp_do_sched_yield, NULL); } diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 98f8974b8bba..a48f81b93c06 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -12,7 +12,7 @@ #define MSEC_TO_NSEC (1000 * 1000) static DEFINE_PER_CPU(cpumask_var_t, walt_local_cpu_mask); -static DEFINE_PER_CPU(u64, rt_task_arrival_time); +DEFINE_PER_CPU(u64, rt_task_arrival_time); static bool long_running_rt_task_trace_rgstrd; void rt_task_arrival_marker(void *unused, bool preempt, From 3482549d47deb59552ec4a6c1e26a4c8fcb81225 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Wed, 6 Apr 2022 11:36:42 -0700 Subject: [PATCH 292/346] sched/walt/halt: ra-arrange the code re-arrange the code to avoid following issues: 1. Mark walt_drain_thread as RT thread with highest priority to avoid getting preempted by other tasks in the system. 2. Handle kthread_should_stop() in walt_drain_thread. Change-Id: I404674477948626503742a66fce657070b89b9e6 Signed-off-by: Satya Durga Srinivasu Prabhala Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_halt.c | 52 ++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index abf9b328c9b6..b91c1d6d28c7 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -190,10 +190,10 @@ static int drain_rq_cpu_stop(void *data) static int cpu_drain_rq(unsigned int cpu) { - if (available_idle_cpu(cpu)) + if (!cpu_online(cpu)) return 0; - if (!cpu_online(cpu)) + if (available_idle_cpu(cpu)) return 0; /* this will schedule, must not be in atomic context */ @@ -229,26 +229,26 @@ static int __ref try_drain_rqs(void *data) int cpu; unsigned long flags; - raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); - - /* continue while there are cpus to drain - * assume at least one cpu to start. - */ - do { - cpumask_t local_cpus; - - cpumask_copy(&local_cpus, cpus_ptr); - raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); - - for_each_cpu(cpu, &local_cpus) - cpu_drain_rq(cpu); - + while (!kthread_should_stop()) { raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); - cpumask_andnot(cpus_ptr, cpus_ptr, &local_cpus); + if (cpumask_weight(cpus_ptr)) { + cpumask_t local_cpus; - } while (cpumask_weight(cpus_ptr)); + cpumask_copy(&local_cpus, cpus_ptr); + raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); - raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); + for_each_cpu(cpu, &local_cpus) + cpu_drain_rq(cpu); + + raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); + cpumask_andnot(cpus_ptr, cpus_ptr, &local_cpus); + + } + raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + set_current_state(TASK_RUNNING); + } return 0; } @@ -286,7 +286,9 @@ static int halt_cpus(struct cpumask *cpus) raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); cpumask_or(&drain_data.cpus_to_drain, &drain_data.cpus_to_drain, cpus); raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); - wake_up_process(walt_drain_thread); + + if (!IS_ERR(walt_drain_thread)) + wake_up_process(walt_drain_thread); trace_halt_cpus(cpus, start_time, 1, ret); @@ -386,7 +388,6 @@ int walt_halt_cpus(struct cpumask *cpus) return ret; } -EXPORT_SYMBOL(walt_halt_cpus); int walt_pause_cpus(struct cpumask *cpus) { @@ -421,7 +422,6 @@ int walt_start_cpus(struct cpumask *cpus) return ret; } -EXPORT_SYMBOL(walt_start_cpus); int walt_resume_cpus(struct cpumask *cpus) { @@ -523,7 +523,15 @@ static void android_rvh_is_cpu_allowed(void *unused, int cpu, bool *allowed) void walt_halt_init(void) { + struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; + walt_drain_thread = kthread_run(try_drain_rqs, &drain_data, "halt_drain_rqs"); + if (IS_ERR(walt_drain_thread)) { + pr_err("Error creating walt drain thread\n"); + return; + } + + sched_setscheduler_nocheck(walt_drain_thread, SCHED_FIFO, ¶m); register_trace_android_rvh_get_nohz_timer_target(android_rvh_get_nohz_timer_target, NULL); register_trace_android_rvh_set_cpus_allowed_ptr_locked( From 2b39f2c898f5203a41534fc7c9f9caf52e17b159 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 7 Apr 2022 17:35:24 -0700 Subject: [PATCH 293/346] sched/walt: Fix lock_cpus mask setting during IC migrations In the existing code, we set the notifier pending flag only on the source and destination CPU. But, in irq_work_restrict_to_mig_clusters(), it clears the lock_cpus mask upon visiting non-involved CPUs in the cluster. This ends up not locking any CPUs, essentially not updating CPU frequencies upon inter cluster migrations. Fix this by only clearly lock_cpus if none of the CPUs in the cluster are involved in a migration. Change-Id: I486e6c62bf306de8fddbd3eab0537c419aedaff2 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index d3269cdbd883..994d92630bfd 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3632,16 +3632,19 @@ static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus) int cpu; for_each_sched_cluster(cluster) { + bool keep = false; for_each_cpu(cpu, &cluster->cpus) { rq = cpu_rq(cpu); wrq = (struct walt_rq *)rq->android_vendor_data1; /* remove this cluster if it's not being notified */ if (!wrq->notif_pending) { - cpumask_andnot(lock_cpus, lock_cpus, &cluster->cpus); + keep = true; break; } } + if (!keep) + cpumask_andnot(lock_cpus, lock_cpus, &cluster->cpus); } } From b64b350ef48efb670a0a6060b465f5c57bc5ddf1 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 8 Apr 2022 16:27:46 -0700 Subject: [PATCH 294/346] sched: Revert "sched/walt: Account for changing task affinity" This reverts commit bb44a08e52736ee05703b707348d96125d827acf. Change-Id: I1b32cbfc167d59b650090e5fdef98101ff9cf6a2 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_lb.c | 70 ++++++------------------------------- 1 file changed, 11 insertions(+), 59 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 2b60160b9d33..ecc873d69a30 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -16,53 +16,15 @@ static inline unsigned long walt_lb_cpu_util(int cpu) return wrq->walt_stats.cumulative_runnable_avg_scaled; } - -static int walt_detach_task(struct task_struct *p, struct rq *src_rq, - struct rq *dst_rq, bool force_affinity_appropriate_cpu) +static void walt_detach_task(struct task_struct *p, struct rq *src_rq, + struct rq *dst_rq) { - int ret = -EINVAL; - int dest_cpu; - int retry_count = 0; - deactivate_task(src_rq, p, 0); -retry: double_lock_balance(src_rq, dst_rq); - /* - * It's possible that src_rq lock was dropped while trying to acuire - * double rq lock. - * Task affinity could change then, recheck if moving the task to - * dst is still valid. - */ - if (!force_affinity_appropriate_cpu || - cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) { - if (!(src_rq->clock_update_flags & RQCF_UPDATED)) - update_rq_clock(src_rq); - set_task_cpu(p, dst_rq->cpu); - ret = 0; - } + if (!(src_rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(src_rq); + set_task_cpu(p, dst_rq->cpu); double_unlock_balance(src_rq, dst_rq); - - if (ret) { - /* - * Couldn't move the task, reactivate it on an appropriate cpu. - */ - if (cpumask_test_cpu(cpu_of(src_rq), p->cpus_ptr) && - cpu_active(cpu_of(src_rq)) && - !cpu_halted(cpu_of(src_rq))) { - activate_task(src_rq, p, 0); - } else { - dest_cpu = select_fallback_rq(cpu_of(src_rq), p); - dst_rq = cpu_rq(dest_cpu); - if (retry_count < 5) { - retry_count++; - goto retry; - } else { - printk_deferred("Failed to select CPU in task's affinity list after 5 tries\n"); - } - } - } - - return ret; } static void walt_attach_task(struct task_struct *p, struct rq *rq) @@ -102,8 +64,8 @@ static int stop_walt_lb_active_migration(void *data) task_cpu(push_task) == busiest_cpu && cpu_active(target_cpu) && cpumask_test_cpu(target_cpu, push_task->cpus_ptr)) { - if (!walt_detach_task(push_task, busiest_rq, target_rq, true)) - push_task_detached = 1; + walt_detach_task(push_task, busiest_rq, target_rq); + push_task_detached = 1; } out_unlock: /* called with busiest_rq lock */ @@ -352,8 +314,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) if (!_walt_can_migrate_task(p, dst_cpu, to_lower, false)) continue; - if (walt_detach_task(p, src_rq, dst_rq, true)) - continue; + walt_detach_task(p, src_rq, dst_rq); pulled_task = p; goto unlock; } @@ -369,8 +330,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) if (!_walt_can_migrate_task(p, dst_cpu, to_lower, true)) continue; - if (walt_detach_task(p, src_rq, dst_rq, true)) - continue; + walt_detach_task(p, src_rq, dst_rq); pulled_task = p; goto unlock; } @@ -410,8 +370,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) continue; } - if (walt_detach_task(p, src_rq, dst_rq, true)) - continue; + walt_detach_task(p, src_rq, dst_rq); pulled_task = p; goto unlock; } @@ -1016,14 +975,7 @@ static void walt_migrate_queued_task(void *unused, struct rq *rq, BUG_ON(!rf); rq_unpin_lock(rq, rf); - /* - * force_affinity_appropriate_cpu set to false allows p to be put on - * a non-affinity-appropriate cpu. We shouldn't move the task to any - * other cpu than new_cpu, even if affinity doesn't allow it, because - * calls to migrate_queued_task are accompanied with a - * activate_task on the new_cpu. - */ - walt_detach_task(p, rq, cpu_rq(new_cpu), false); + walt_detach_task(p, rq, cpu_rq(new_cpu)); rq_repin_lock(rq, rf); *detached = 1; From bd13df8ce93df28e0cc75c475daf912b15a96837 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 8 Apr 2022 15:48:00 -0700 Subject: [PATCH 295/346] sched/walt: streamline affinity mask Upstream tracks affinity mask in two ways, p->cpus_ptr and p->cpus_mask. However, we are currently using these two masks interchangeably. Currently cpus_ptr points to cpus_mask member in task_struct for most of the time. One exception being when the task is marked migration_disabled, during which cpus_ptr could point to a different mask of single_cpu. Streamline this to ensure WALT only relies on p->cpus_ptr to determine CPU affinity, which would be more accurate in migrate_disable situations. Change-Id: I437ffe531d7417eada71304414654341ce4a4ad9 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 2 +- kernel/sched/walt/walt.c | 4 ++-- kernel/sched/walt/walt_cfs.c | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 2407d60c3ec1..b1f1eb4e0688 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1094,7 +1094,7 @@ TRACE_EVENT(sched_task_util, __entry->start_cpu = start_cpu; __entry->unfilter = ((struct walt_task_struct *) p->android_vendor_data1)->unfilter; - __entry->cpus_allowed = cpumask_bits(&p->cpus_mask)[0]; + __entry->cpus_allowed = cpumask_bits(p->cpus_ptr)[0]; __entry->task_boost = per_task_boost(p); __entry->low_latency = walt_low_latency_task(p); __entry->iowaited = diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 994d92630bfd..c854281c888d 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4059,7 +4059,7 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st if (!double_enqueue) walt_inc_cumulative_runnable_avg(rq, p); - trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); + trace_sched_enq_deq_task(p, 1, cpumask_bits(p->cpus_ptr)[0], is_mvp(wts)); } static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p) @@ -4107,7 +4107,7 @@ static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_st if (!double_dequeue) walt_dec_cumulative_runnable_avg(rq, p); - trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0], is_mvp(wts)); + trace_sched_enq_deq_task(p, 0, cpumask_bits(p->cpus_ptr)[0], is_mvp(wts)); } static void android_rvh_update_misfit_status(void *unused, struct task_struct *p, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index a9d9faaf139b..651070279bb7 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -63,7 +63,7 @@ unsigned int sched_capacity_margin_down[WALT_NR_CPUS] = { static inline bool bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) { - bool base_test = cpumask_test_cpu(cpu, &p->cpus_mask) && + bool base_test = cpumask_test_cpu(cpu, p->cpus_ptr) && cpu_active(cpu); struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; @@ -295,7 +295,7 @@ static void walt_find_best_target(struct sched_domain *sd, min_exit_latency = INT_MAX; best_idle_cuml_util = ULONG_MAX; - cpumask_and(&visit_cpus, &p->cpus_mask, + cpumask_and(&visit_cpus, p->cpus_ptr, &cpu_array[order_index][cluster]); for_each_cpu(i, &visit_cpus) { unsigned long capacity_orig = capacity_orig_of(i); @@ -785,7 +785,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, struct walt_rq *start_wrq; if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && - cpumask_test_cpu(prev_cpu, &p->cpus_mask)) + cpumask_test_cpu(prev_cpu, p->cpus_ptr)) return prev_cpu; if (unlikely(!cpu_array)) @@ -867,7 +867,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, if (READ_ONCE(p->__state) == TASK_WAKING) delta = task_util(p); - if (cpumask_test_cpu(prev_cpu, &p->cpus_mask) && !__cpu_overutilized(prev_cpu, delta)) { + if (cpumask_test_cpu(prev_cpu, p->cpus_ptr) && !__cpu_overutilized(prev_cpu, delta)) { if (trace_sched_compute_energy_enabled()) { memset(&output, 0, sizeof(output)); prev_energy = walt_compute_energy(p, prev_cpu, pd, candidates, fbt_env.prs, From 97a7eb42925559b8b5b55e6fee65706e0aea4098 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Fri, 8 Apr 2022 16:01:56 -0700 Subject: [PATCH 296/346] sched/walt: fix incorrect migrations for RT task When a RT task is marked migration_disabled, i.e. its p->cpus_ptr points to a single cpu, walt_newidle_balance() could pull the task to an un-allowed cpu. To prevent that check p->cpus_ptr in addition to pick_highest_pushable_task(), as pick_highest_pushable_task() and pick_rt_task() only check for p->cpu_mask, which will be inaccurate in migration_disabled situation. Change-Id: If7b3cfcb357af7a3ff15de486e70b7b304e4f6ec Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_lb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index ecc873d69a30..212247934c17 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -718,6 +718,9 @@ static bool walt_balance_rt(struct rq *this_rq) if (!p) goto unlock; + if (!cpumask_test_cpu(this_cpu, p->cpus_ptr)) + goto unlock; + wts = (struct walt_task_struct *) p->android_vendor_data1; /* From 318a435b83a9fb79d233e2b2d83f85f1c43bcd34 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Thu, 14 Apr 2022 12:16:01 -0700 Subject: [PATCH 297/346] sched/walt: fix deadlock due to recursive rq locking complete_all() being called from android_rvh_tick_entry() trace hook implementation to wake up task (kworker) waiting on completion variable. While task waking if scheduler chooses to place the task on same CPU where complete_all() is invoked (from scheduler_tick()), system would run into deadlock as wake up path tries to acquire rq lock of the CPU from ttwu_queue() which was already acquired in scheduler_tick(). Fix the deadlock by moving related code chunk to android_vh_scheduler_tick() trace hook implementation. Call statck for info: [] queued_spin_lock_slowpath+0x88 [] try_to_wake_up[jt]+0x25c [] complete_all+0xb4 [] android_rvh_tick_entry[sched_walt]+0x5c [] scheduler_tick+0x4b0 Fixes: 4e3e7cc6133d2f7 ("sched: Align window start with tick boundary") Change-Id: I1e1310c6c5513f38c4fd95b933ff407d740c1112 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c854281c888d..a6c45b85fd0c 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4189,22 +4189,13 @@ static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct raw_spin_unlock_irqrestore(&cpu_rq(cpu)->__lock, flags); } -u64 tick_sched_clock; +static u64 tick_sched_clock; static DECLARE_COMPLETION(tick_sched_clock_completion); static void android_rvh_tick_entry(void *unused, struct rq *rq) { u64 wallclock; - if (!tick_sched_clock) { - /* - * Let the window begin 20us prior to the tick, - * that way we are guaranteed a rollover when the tick occurs. - */ - tick_sched_clock = rq_clock(rq) - 20000; - complete_all(&tick_sched_clock_completion); - } - lockdep_assert_held(&rq->__lock); if (unlikely(walt_disabled)) return; @@ -4222,6 +4213,15 @@ static void android_vh_scheduler_tick(void *unused, struct rq *rq) struct walt_related_thread_group *grp; u32 old_load; + if (!tick_sched_clock) { + /* + * Let the window begin 20us prior to the tick, + * that way we are guaranteed a rollover when the tick occurs. + */ + tick_sched_clock = rq_clock(rq) - 20000; + complete(&tick_sched_clock_completion); + } + if (unlikely(walt_disabled)) return; @@ -4442,7 +4442,7 @@ static void walt_init(struct work_struct *work) walt_rt_init(); walt_cfs_init(); walt_halt_init(); - wait_for_completion(&tick_sched_clock_completion); + wait_for_completion_interruptible(&tick_sched_clock_completion); stop_machine(walt_init_stop_handler, NULL, NULL); hdr = register_sysctl_table(walt_base_table); From d6ad86a43e3ed69352e799c364439de5e35198c8 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 14 Apr 2022 15:47:37 -0700 Subject: [PATCH 298/346] sched/walt: check for incorrect cpu assignment When a task is in execve, it can be assigned to any cpu regardless of its status as a 32bit task. However, once execve is done, the task needs to be assigned to a 32 bit cpu. Capture the case where a 32 bit task, not in execve, is assigned to an incorrect cpu. Change-Id: I7880521716f6ed16d81aad13e42c0ad4374bab15 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index a6c45b85fd0c..a578ecce0376 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3983,6 +3983,13 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign if (!cpumask_test_cpu(new_cpu, p->cpus_ptr)) WALT_BUG(WALT_BUG_WALT, p, "selecting unaffined cpu=%d comm=%s(%d) affinity=0x%x", new_cpu, p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); + + if (!p->in_execve && + is_compat_thread(task_thread_info(p)) && + !cpumask_test_cpu(new_cpu, system_32bit_el0_cpumask())) + WALT_BUG(WALT_BUG_WALT, p, + "selecting non 32 bit cpu=%d comm=%s(%d) 32bit_cpus=0x%x", + new_cpu, p->comm, p->pid, (*(cpumask_bits(system_32bit_el0_cpumask())))); } static void android_rvh_new_task_stats(void *unused, struct task_struct *p) From bb7363e104c92d1e1d2ac39b5fdeeffbc5d6a5e4 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Fri, 15 Apr 2022 18:37:02 -0700 Subject: [PATCH 299/346] sched/walt: remove duplicate definitions of rt_task_arrival_time rt_task_arrival_time is being defined in a header as well as in a source file which is incorrect, fix it by changing definition to declaration in header. While at it, clean-up related code and make sure long running RT detection happens for FIFO sched class only. Change-Id: Ia9f227ad3af0059d354eb0e4df3ab719c1348ac5 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt.c | 2 +- kernel/sched/walt/walt.h | 7 +------ kernel/sched/walt/walt_rt.c | 19 ++++++++++++++++--- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index a578ecce0376..ab0262b1e97e 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4334,7 +4334,7 @@ static void walt_do_sched_yield(void *unused, struct rq *rq) if (mvp_prio != WALT_NOT_MVP) walt_cfs_deactivate_mvp_task(curr); - if (rt_task(curr)) + if (per_cpu(rt_task_arrival_time, cpu_of(rq))) per_cpu(rt_task_arrival_time, cpu_of(rq)) = 0; } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index df1307b45ed4..03253a23cea9 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -886,11 +886,6 @@ static inline bool walt_fair_task(struct task_struct *p) return p->prio >= MAX_RT_PRIO && !is_idle_task(p); } -extern void rt_task_arrival_marker(void *unused, bool preempt, - struct task_struct *prev, struct task_struct *next); - -extern void long_running_rt_task_notifier(void *unused, struct rq *rq); - extern int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); @@ -936,7 +931,7 @@ extern int in_sched_bug; extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, struct task_struct *p, int dest_cpu); -extern DEFINE_PER_CPU(u64, rt_task_arrival_time); +DECLARE_PER_CPU(u64, rt_task_arrival_time); extern int walt_get_mvp_task_prio(struct task_struct *p); extern void walt_cfs_deactivate_mvp_task(struct task_struct *p); diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index a48f81b93c06..b245bca00ae4 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -12,10 +12,10 @@ #define MSEC_TO_NSEC (1000 * 1000) static DEFINE_PER_CPU(cpumask_var_t, walt_local_cpu_mask); -DEFINE_PER_CPU(u64, rt_task_arrival_time); +DEFINE_PER_CPU(u64, rt_task_arrival_time) = 0; static bool long_running_rt_task_trace_rgstrd; -void rt_task_arrival_marker(void *unused, bool preempt, +static void rt_task_arrival_marker(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next) { unsigned int cpu = raw_smp_processor_id(); @@ -26,7 +26,7 @@ void rt_task_arrival_marker(void *unused, bool preempt, per_cpu(rt_task_arrival_time, cpu) = 0; } -void long_running_rt_task_notifier(void *unused, struct rq *rq) +static void long_running_rt_task_notifier(void *unused, struct rq *rq) { struct task_struct *curr = rq->curr; unsigned int cpu = raw_smp_processor_id(); @@ -37,6 +37,19 @@ void long_running_rt_task_notifier(void *unused, struct rq *rq) if (!per_cpu(rt_task_arrival_time, cpu)) return; + if (per_cpu(rt_task_arrival_time, cpu) && curr->policy != SCHED_FIFO) { + /* This should never happen, trying to avoid any false positives */ + printk_deferred("Long running RT false positive detected for task %s (%d) runtime > %u now=%llu task arrival time=%llu runtime=%llu\n", + curr->comm, curr->pid, + sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC, + rq_clock_task(rq), + per_cpu(rt_task_arrival_time, cpu), + rq_clock_task(rq) - + per_cpu(rt_task_arrival_time, cpu)); + per_cpu(rt_task_arrival_time, cpu) = 0; + return; + } + if (rq_clock_task(rq) - per_cpu(rt_task_arrival_time, cpu) > sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC) { From 69bd1582b24d028c2fc50b6cbb965ea0487dd333 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 24 Jan 2022 17:02:45 -0800 Subject: [PATCH 300/346] sched: Use bitshift over division Now that our window history size is a power of 2, we can take advantage of this to replace division operations in a critical code path by using bit-shifts instead. Change-Id: Idc0ed37a87814783fcab2f4d5204b79fb9ce9996 Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 10 ++++++++-- kernel/sched/walt/trace.h | 4 ++-- kernel/sched/walt/walt.c | 16 +++++++--------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 87757ac85a4d..853c7c169795 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -32,7 +32,13 @@ enum task_boost_type { }; #define WALT_NR_CPUS 8 -#define RAVG_HIST_SIZE_MAX 8 +/* + * RAVG_HIST_SHIFT trick can only be used if RAVG_HIST_SIZE is a power of 2. + */ +#define RAVG_HIST_SIZE 8 +#define RAVG_HIST_SHIFT 3 +#define RAVG_HIST_MASK (RAVG_HIST_SIZE - 1) + /* wts->bucket_bitmask needs to be updated if NUM_BUSY_BUCKETS > 16 */ #define NUM_BUSY_BUCKETS 16 #define NUM_BUSY_BUCKETS_SHIFT 4 @@ -90,7 +96,7 @@ struct walt_task_struct { u64 mark_start; u32 sum, demand; u32 coloc_demand; - u32 sum_history[RAVG_HIST_SIZE_MAX]; + u32 sum_history[RAVG_HIST_SIZE]; u32 curr_window_cpu[WALT_NR_CPUS]; u32 prev_window_cpu[WALT_NR_CPUS]; u32 curr_window, prev_window; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index b1f1eb4e0688..4b48c88a5cd2 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -83,7 +83,7 @@ TRACE_EVENT(sched_update_history, __field(unsigned int, demand) __field(unsigned int, coloc_demand) __field(unsigned int, pred_demand_scaled) - __array(u32, hist, RAVG_HIST_SIZE_MAX) + __array(u32, hist, RAVG_HIST_SIZE) __field(unsigned int, nr_big_tasks) __field(int, cpu) ), @@ -98,7 +98,7 @@ TRACE_EVENT(sched_update_history, __entry->coloc_demand = wts->coloc_demand; __entry->pred_demand_scaled = wts->pred_demand_scaled; memcpy(__entry->hist, wts->sum_history, - RAVG_HIST_SIZE_MAX * sizeof(u32)); + RAVG_HIST_SIZE * sizeof(u32)); __entry->nr_big_tasks = wrq->walt_stats.nr_big_tasks; __entry->cpu = rq->cpu; ), diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index ab0262b1e97e..78036d5df2c2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -167,8 +167,6 @@ static inline u64 walt_rq_clock(struct rq *rq) static unsigned int walt_cpu_high_irqload; -static __read_mostly unsigned int sched_ravg_hist_size = RAVG_HIST_SIZE_MAX; - static __read_mostly unsigned int sched_io_is_busy = 1; /* Window size (in ns) */ @@ -1935,10 +1933,10 @@ static void update_history(struct rq *rq, struct task_struct *p, /* Push new 'runtime' value onto stack */ for (; samples > 0; samples--) { hist[wts->cidx] = runtime; - wts->cidx = ++(wts->cidx) % sched_ravg_hist_size; + wts->cidx = ++(wts->cidx) & RAVG_HIST_MASK; } - for (i = 0; i < sched_ravg_hist_size; i++) { + for (i = 0; i < RAVG_HIST_SIZE; i++) { sum += hist[i]; if (hist[i] > max) max = hist[i]; @@ -1951,7 +1949,7 @@ static void update_history(struct rq *rq, struct task_struct *p, } else if (sysctl_sched_window_stats_policy == WINDOW_STATS_MAX) { demand = max; } else { - avg = div64_u64(sum, sched_ravg_hist_size); + avg = sum >> RAVG_HIST_SHIFT; if (sysctl_sched_window_stats_policy == WINDOW_STATS_AVG) demand = avg; else @@ -1980,7 +1978,7 @@ static void update_history(struct rq *rq, struct task_struct *p, wts->demand = demand; wts->demand_scaled = demand_scaled; - wts->coloc_demand = div64_u64(sum, sched_ravg_hist_size); + wts->coloc_demand = sum >> RAVG_HIST_SHIFT; wts->pred_demand_scaled = pred_demand_scaled; if (demand_scaled > sysctl_sched_min_task_util_for_colocation) @@ -2307,7 +2305,7 @@ static void init_new_task_load(struct task_struct *p) wts->demand_scaled = init_load_windows_scaled; wts->coloc_demand = init_load_windows; wts->pred_demand_scaled = 0; - for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) + for (i = 0; i < RAVG_HIST_SIZE; ++i) wts->sum_history[i] = init_load_windows; wts->misfit = false; wts->rtg_high_prio = false; @@ -2850,7 +2848,7 @@ static void _set_preferred_cluster(struct walt_related_thread_group *grp) } if (wts->mark_start < wallclock - - (sched_ravg_window * sched_ravg_hist_size)) + (sched_ravg_window * RAVG_HIST_SIZE)) continue; combined_demand += wts->coloc_demand; @@ -3726,7 +3724,7 @@ void walt_fill_ta_data(struct core_ctl_notif_data *data) list_for_each_entry(wts, &grp->tasks, grp_list) { if (wts->mark_start < wallclock - - (sched_ravg_window * sched_ravg_hist_size)) + (sched_ravg_window * RAVG_HIST_SIZE)) continue; total_demand += wts->coloc_demand; From d0ba68e336a3783ff303d5387f7b7db039dcac1a Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 24 Jan 2022 16:52:41 -0800 Subject: [PATCH 301/346] sched: Improve the Scheduler This change is for general scheduler improvement. Change-Id: Iebc41134b83b1c83d02f433d92ec2b44ed972511 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/sched_avg.c | 2 +- kernel/sched/walt/walt.c | 17 ++++++++--------- kernel/sched/walt/walt.h | 11 ++++++++++- kernel/sched/walt/walt_cfs.c | 2 +- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 7fb3e0672e53..b2c0a737c107 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -261,7 +261,7 @@ unsigned int sched_get_cpu_util_pct(int cpu) capacity = capacity_orig_of(cpu); util = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; - util = div64_u64(util, sched_ravg_window >> SCHED_CAPACITY_SHIFT); + util = scale_time_to_util(util); raw_spin_unlock_irqrestore(&rq->__lock, flags); util = (util >= capacity) ? capacity : util; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 78036d5df2c2..beac36928eeb 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -650,8 +650,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea unsigned long capacity = capacity_orig_of(cpu); struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - util = div64_u64(freq_policy_load(rq, reason), - sched_ravg_window >> SCHED_CAPACITY_SHIFT); + util = scale_time_to_util(freq_policy_load(rq, reason)); if (walt_load) { u64 nl = wrq->nt_prev_runnable_sum + @@ -661,7 +660,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea wrq->old_busy_time = util; wrq->old_estimated_time = pl; - nl = div64_u64(nl, sched_ravg_window >> SCHED_CAPACITY_SHIFT); + nl = scale_time_to_util(nl); walt_load->nl = nl; walt_load->pl = pl; walt_load->ws = walt_load_reported_window; @@ -1290,7 +1289,7 @@ static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int ev return; } - curr_window_scaled = scale_demand(wts->curr_window); + curr_window_scaled = scale_time_to_util(wts->curr_window); if (wts->pred_demand_scaled >= curr_window_scaled) return; @@ -1515,7 +1514,7 @@ static inline u64 scale_exec_time(u64 delta, struct rq *rq) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - return (delta * wrq->task_exec_scale) >> 10; + return (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT; } /* Convert busy time to frequency equivalent @@ -1955,8 +1954,8 @@ static void update_history(struct rq *rq, struct task_struct *p, else demand = max(avg, runtime); } - pred_demand_scaled = predict_and_update_buckets(p, scale_demand(runtime)); - demand_scaled = scale_demand(demand); + pred_demand_scaled = predict_and_update_buckets(p, scale_time_to_util(runtime)); + demand_scaled = scale_time_to_util(demand); /* * A throttled deadline sched class task gets dequeued without @@ -2298,7 +2297,7 @@ static void init_new_task_load(struct task_struct *p) if (init_load_pct) { init_load_windows = div64_u64((u64)init_load_pct * (u64)sched_ravg_window, 100); - init_load_windows_scaled = scale_demand(init_load_windows); + init_load_windows_scaled = scale_time_to_util(init_load_windows); } wts->demand = init_load_windows; @@ -3771,7 +3770,7 @@ static void walt_init_window_dep(void) div64_u64((u64)sysctl_sched_init_task_load_pct * (u64)sched_ravg_window, 100); sched_init_task_load_windows_scaled = - scale_demand(sched_init_task_load_windows); + scale_time_to_util(sched_init_task_load_windows); walt_cpu_high_irqload = div64_u64((u64)sched_ravg_window * 95, (u64) 100); } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 03253a23cea9..717ae9f5f52f 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -906,7 +906,16 @@ void walt_cfs_tick(struct rq *rq); void walt_lb_tick(struct rq *rq); extern __read_mostly unsigned int walt_scale_demand_divisor; -#define scale_demand(d) ((d)/walt_scale_demand_divisor) + +static inline u64 scale_time_to_util(u64 d) +{ + /* + * The denominator at most could be (8 * tick_size) >> SCHED_CAPACITY_SHIFT, + * a value that easily fits a 32bit integer. + */ + do_div(d, walt_scale_demand_divisor); + return d; +} #define ASYMCAP_BOOST(cpu) (sysctl_sched_asymcap_boost && !is_min_cluster_cpu(cpu)) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 651070279bb7..7c10e3c2cb4a 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -681,7 +681,7 @@ walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *p max_util = max(max_util, cpu_util); } - max_util = scale_demand(max_util); + max_util = scale_time_to_util(max_util); if (output) output->cluster_first_cpu[x] = cpumask_first(pd_mask); From affc93eb8ab53ca94d86eab3e95dec51de362d71 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 14 Apr 2022 16:07:45 -0700 Subject: [PATCH 302/346] sched: Re-fix lock_cpus mask setting during IC migrations In an earlier fix, we intended to clear lock CPUs with a given cluster if and only all CPUs belonging to the cluster did not have a notification pending. However, the fix was incomplete, as we were continuing to do the opposite, resulting in undesired behavior. Fixes: 95d58bdfe8d ("sched/walt: Fix lock_cpus mask setting during IC migrations") Change-Id: Ia3c0d886a2a27b8f4cf8d6f60673aeb7213d5fb9 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index beac36928eeb..c4753e613bf0 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3629,18 +3629,18 @@ static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus) int cpu; for_each_sched_cluster(cluster) { - bool keep = false; + bool keep_locked = false; for_each_cpu(cpu, &cluster->cpus) { rq = cpu_rq(cpu); wrq = (struct walt_rq *)rq->android_vendor_data1; /* remove this cluster if it's not being notified */ - if (!wrq->notif_pending) { - keep = true; + if (wrq->notif_pending) { + keep_locked = true; break; } } - if (!keep) + if (!keep_locked) cpumask_andnot(lock_cpus, lock_cpus, &cluster->cpus); } } From 68dfacdb44fcdf5b47fa93d064cd3bc3db0766f1 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Mon, 18 Apr 2022 18:18:57 -0700 Subject: [PATCH 303/346] sched/walt: skip long running RT task detection for stopper tasks As stopper task's scheduling policy is set to FIFO and task may run long time depends on the number of tasks to be migrated etc., current long running RT task detection feature crashes system due to task's arrivial time stamp is taken when stopper task scheduled in during CPU hot-plug out operation and used it later when CPU gets hot-plugged in back. Fix this false positive issue by skipping update of arrivial time stamp for stopper tasks. Fixes: b429bee50fe2c8f ("sched/walt/rt: limit long running RT task detection to FIFO") Change-Id: I714494fee49b6e8e5d50885bd47b38ad0d535579 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/walt/walt_rt.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index b245bca00ae4..13926fcb5ab5 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -20,7 +20,7 @@ static void rt_task_arrival_marker(void *unused, bool preempt, { unsigned int cpu = raw_smp_processor_id(); - if (next->policy == SCHED_FIFO) + if (next->policy == SCHED_FIFO && next != cpu_rq(cpu)->stop) per_cpu(rt_task_arrival_time, cpu) = rq_clock_task(this_rq()); else per_cpu(rt_task_arrival_time, cpu) = 0; @@ -38,14 +38,12 @@ static void long_running_rt_task_notifier(void *unused, struct rq *rq) return; if (per_cpu(rt_task_arrival_time, cpu) && curr->policy != SCHED_FIFO) { - /* This should never happen, trying to avoid any false positives */ - printk_deferred("Long running RT false positive detected for task %s (%d) runtime > %u now=%llu task arrival time=%llu runtime=%llu\n", - curr->comm, curr->pid, - sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC, - rq_clock_task(rq), - per_cpu(rt_task_arrival_time, cpu), - rq_clock_task(rq) - - per_cpu(rt_task_arrival_time, cpu)); + /* + * It is possible that the scheduling policy for the current + * task might get changed after task arrival time stamp is + * noted during sched_switch of RT task. To avoid such false + * positives, reset arrival time stamp. + */ per_cpu(rt_task_arrival_time, cpu) = 0; return; } From eafe0b4e6fcd5c74508f9a4ec470446aa319c5fa Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Mon, 20 Dec 2021 17:02:46 +0800 Subject: [PATCH 304/346] sched/walt: check current task's mvp_list instead of mvp prio Current task's mvp prio get from walt_get_mvp_task_prio will still be there even if the task is not in the mvp list. For example task removed from mvp_list due to runtime slice limition. Change-Id: Iae5d6e420213111acd0b0e71095e0a6caffbe0aa Signed-off-by: Tengfei Fan --- kernel/sched/walt/walt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c4753e613bf0..7f36dff545c2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4325,10 +4325,10 @@ static void android_rvh_build_perf_domains(void *unused, bool *eas_check) static void walt_do_sched_yield(void *unused, struct rq *rq) { struct task_struct *curr = rq->curr; - int mvp_prio = walt_get_mvp_task_prio(curr); + struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1; lockdep_assert_held(&rq->__lock); - if (mvp_prio != WALT_NOT_MVP) + if (!list_empty(&wts->mvp_list) && wts->mvp_list.next) walt_cfs_deactivate_mvp_task(curr); if (per_cpu(rt_task_arrival_time, cpu_of(rq))) From 8b663c22fbfd223de7bfafdfeab62042e1efd95e Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Thu, 6 Jan 2022 17:13:42 +0800 Subject: [PATCH 305/346] sched/walt: get rcu lock before use task's cgroups uclamp_latency_sensitive() function will use struct task_struct's member of cgroups, but rcu lock should be got before use it. Change-Id: I1988ed7fe836f9f1ba99d59c5d46f26f3418b51e Signed-off-by: Tengfei Fan --- kernel/sched/walt/walt_cfs.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 7c10e3c2cb4a..5521856ac5d8 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -772,7 +772,7 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, cpumask_t *candidates; bool is_rtg, curr_is_rtg; struct find_best_target_env fbt_env; - bool need_idle = wake_to_idle(p) || uclamp_latency_sensitive(p); + bool need_idle = wake_to_idle(p); u64 start_t = 0; int delta = 0; int task_boost = per_task_boost(p); @@ -799,9 +799,6 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, is_rtg = task_in_related_thread_group(p); curr_is_rtg = task_in_related_thread_group(cpu_rq(cpu)->curr); - fbt_env.fastpath = 0; - fbt_env.need_idle = need_idle; - if (trace_sched_task_util_enabled()) start_t = sched_clock(); @@ -809,17 +806,22 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, candidates = this_cpu_ptr(&energy_cpus); cpumask_clear(candidates); + rcu_read_lock(); + need_idle |= uclamp_latency_sensitive(p); + + fbt_env.fastpath = 0; + fbt_env.need_idle = need_idle; + if (sync && (need_idle || (is_rtg && curr_is_rtg))) sync = 0; - if (sysctl_sched_sync_hint_enable && sync && - bias_to_this_cpu(p, cpu, start_cpu) && !cpu_halted(cpu)) { + if (sysctl_sched_sync_hint_enable && sync + && bias_to_this_cpu(p, cpu, start_cpu) && !cpu_halted(cpu)) { best_energy_cpu = cpu; fbt_env.fastpath = SYNC_WAKEUP; - goto done; + goto unlock; } - rcu_read_lock(); pd = rcu_dereference(rd->pd); if (!pd) goto fail; @@ -926,7 +928,6 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, unlock: rcu_read_unlock(); -done: if (best_energy_cpu < 0 || best_energy_cpu >= WALT_NR_CPUS) best_energy_cpu = prev_cpu; From 8c213b0ca37dcd977cbd4c10f52fcc44304e90a2 Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Tue, 21 Dec 2021 17:45:15 +0800 Subject: [PATCH 306/346] sched/walt: add num_mvp_tasks for get the number of mvp task in walt rq Add num_mvp_tasks for get the number of mvp task in walt rq. Change-Id: I4367723cd61f7ee5fd63c2afcfefd6eb3258c118 Signed-off-by: Tengfei Fan --- kernel/sched/walt/walt.c | 3 ++- kernel/sched/walt/walt.h | 3 ++- kernel/sched/walt/walt_cfs.c | 10 +++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 7f36dff545c2..c0a58e84e3f5 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3834,6 +3834,7 @@ static void walt_sched_init_rq(struct rq *rq) } wrq->notif_pending = false; + wrq->num_mvp_tasks = 0; INIT_LIST_HEAD(&wrq->mvp_tasks); } @@ -4329,7 +4330,7 @@ static void walt_do_sched_yield(void *unused, struct rq *rq) lockdep_assert_held(&rq->__lock); if (!list_empty(&wts->mvp_list) && wts->mvp_list.next) - walt_cfs_deactivate_mvp_task(curr); + walt_cfs_deactivate_mvp_task(rq, curr); if (per_cpu(rt_task_arrival_time, cpu_of(rq))) per_cpu(rt_task_arrival_time, cpu_of(rq)) = 0; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 717ae9f5f52f..47a4168cb00e 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -121,6 +121,7 @@ struct walt_rq { u64 last_cc_update; u64 cycles; struct list_head mvp_tasks; + int num_mvp_tasks; u64 latest_clock; }; @@ -942,7 +943,7 @@ extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, DECLARE_PER_CPU(u64, rt_task_arrival_time); extern int walt_get_mvp_task_prio(struct task_struct *p); -extern void walt_cfs_deactivate_mvp_task(struct task_struct *p); +extern void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p); enum WALT_DEBUG_FEAT { WALT_BUG_UPSTREAM, diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 5521856ac5d8..8f0ccda118b1 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1064,14 +1064,17 @@ static void walt_cfs_insert_mvp_task(struct walt_rq *wrq, struct walt_task_struc } list_add(&wts->mvp_list, pos->prev); + wrq->num_mvp_tasks++; } -void walt_cfs_deactivate_mvp_task(struct task_struct *p) +void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p) { + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; list_del_init(&wts->mvp_list); wts->mvp_prio = WALT_NOT_MVP; + wrq->num_mvp_tasks--; } /* @@ -1123,13 +1126,14 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr limit = walt_cfs_mvp_task_limit(curr); if (wts->total_exec > limit) { - walt_cfs_deactivate_mvp_task(curr); + walt_cfs_deactivate_mvp_task(rq, curr); trace_walt_cfs_deactivate_mvp_task(curr, wts, limit); return; } /* slice expired. re-queue the task */ list_del(&wts->mvp_list); + wrq->num_mvp_tasks--; walt_cfs_insert_mvp_task(wrq, wts, false); } @@ -1169,7 +1173,7 @@ void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; if (!list_empty(&wts->mvp_list) && wts->mvp_list.next) - walt_cfs_deactivate_mvp_task(p); + walt_cfs_deactivate_mvp_task(rq, p); /* * Reset the exec time during sleep so that it starts From 4bd7dbf6867f0e081ce7e9472f1636112b0c2893 Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Mon, 28 Feb 2022 14:09:10 +0800 Subject: [PATCH 307/346] sched/walt: don't requeue mvp task if only have one mvp task on rq Don't requeue mvp task if only have one mvp task on rq, because only have one mvp task on rq, current mvp task will be select for running if this mvp task runtime still less than limit. Change-Id: Idaf343e7471665c01fe711e86a3d7747e501ed76 Signed-off-by: Tengfei Fan --- kernel/sched/walt/walt_cfs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 8f0ccda118b1..10347ca9843f 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -1131,6 +1131,9 @@ static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr return; } + if (wrq->num_mvp_tasks == 1) + return; + /* slice expired. re-queue the task */ list_del(&wts->mvp_list); wrq->num_mvp_tasks--; From 52016a65f0c5ffddb0c644638532d3063989b6dc Mon Sep 17 00:00:00 2001 From: Tengfei Fan Date: Mon, 28 Feb 2022 14:13:38 +0800 Subject: [PATCH 308/346] sched/walt: use num_mvp_tasks for check if rq have mvp task There is mvp task running on rq if num_mvp_task not equal 0, so use num_mvp_tasks check if rq have mvp task. Change-Id: Ia653b8d1dbbaa642ccc06296338617252bf1225b Signed-off-by: Tengfei Fan --- kernel/sched/walt/walt_cfs.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 10347ca9843f..989ac84ad934 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -327,8 +327,7 @@ static void walt_find_best_target(struct sched_domain *sd, if (fbt_env->skip_cpu == i) continue; - if (per_task_boost(cpu_rq(i)->curr) == - TASK_BOOST_STRICT_MAX) + if (wrq->num_mvp_tasks > 0) continue; /* From ec2d7224f4ef0a2c811b584cdf2e691886db433a Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Thu, 7 Apr 2022 23:14:28 -0700 Subject: [PATCH 309/346] sched/walt: Improve the scheduler This change is for general scheduler improvements. Change-Id: I611eb9aa071e4fac1fad962fb98139f9aff79dcd Signed-off-by: Satya Durga Srinivasu Prabhala Signed-off-by: Sai Harshini Nimmala --- include/linux/sched/walt.h | 2 ++ kernel/sched/walt/sysctl.c | 34 ++++++++++++++++++++++++++-- kernel/sched/walt/trace.h | 45 ++++++++++++++++++++++++++++++++++++-- kernel/sched/walt/walt.c | 35 +++++++++++++++++------------ 4 files changed, 98 insertions(+), 18 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 853c7c169795..0bf24642e528 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -131,6 +131,8 @@ struct walt_task_struct { u64 total_exec; int mvp_prio; int cidx; + int load_boost; + int64_t boosted_task_load; }; #define wts_to_ts(wts) ({ \ diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 7a59a743cdc2..1be0123ebcbe 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -4,9 +4,11 @@ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ -#include "walt.h" #include +#include "walt.h" +#include "trace.h" + static int neg_three = -3; static int three = 3; static int two_hundred_fifty_five = 255; @@ -191,6 +193,7 @@ enum { PER_TASK_BOOST_PERIOD_MS, LOW_LATENCY, PIPELINE, + LOAD_BOOST, }; static int sched_task_handler(struct ctl_table *table, int write, @@ -247,6 +250,9 @@ static int sched_task_handler(struct ctl_table *table, int write, pid_and_val[1] = wts->low_latency & WALT_LOW_LATENCY_PIPELINE; break; + case LOAD_BOOST: + pid_and_val[1] = wts->load_boost; + break; default: ret = -EINVAL; goto put_task; @@ -259,7 +265,7 @@ static int sched_task_handler(struct ctl_table *table, int write, if (ret) goto unlock_mutex; - if (pid_and_val[0] <= 0 || pid_and_val[1] < 0) { + if (pid_and_val[0] <= 0) { ret = -ENOENT; goto unlock_mutex; } @@ -273,6 +279,10 @@ static int sched_task_handler(struct ctl_table *table, int write, wts = (struct walt_task_struct *) task->android_vendor_data1; param = (unsigned long)table->data; val = pid_and_val[1]; + if (param != LOAD_BOOST && val < 0) { + ret = -EINVAL; + goto put_task; + } switch (param) { case WAKE_UP_IDLE: wts->wake_up_idle = val; @@ -317,10 +327,23 @@ static int sched_task_handler(struct ctl_table *table, int write, else wts->low_latency &= ~WALT_LOW_LATENCY_PIPELINE; break; + case LOAD_BOOST: + if (pid_and_val[1] < -90 || pid_and_val[1] > 90) { + ret = -EINVAL; + goto put_task; + } + wts->load_boost = val; + if (val) + wts->boosted_task_load = mult_frac((int64_t)1024, (int64_t)val, 100); + else + wts->boosted_task_load = 0; + break; default: ret = -EINVAL; } + trace_sched_task_handler(task, param, val, CALLER_ADDR0, CALLER_ADDR1, + CALLER_ADDR2, CALLER_ADDR3, CALLER_ADDR4, CALLER_ADDR5); put_task: put_task_struct(task); unlock_mutex: @@ -836,6 +859,13 @@ struct ctl_table walt_table[] = { .mode = 0644, .proc_handler = sched_task_handler, }, + { + .procname = "task_load_boost", + .data = (int *) LOAD_BOOST, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, { .procname = "sched_task_read_pid", .data = &sysctl_task_read_pid, diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 4b48c88a5cd2..f780ea7cc85b 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1074,6 +1074,7 @@ TRACE_EVENT(sched_task_util, __field(int, task_boost) __field(bool, low_latency) __field(bool, iowaited) + __field(int, load_boost) ), TP_fast_assign( @@ -1099,16 +1100,18 @@ TRACE_EVENT(sched_task_util, __entry->low_latency = walt_low_latency_task(p); __entry->iowaited = ((struct walt_task_struct *) p->android_vendor_data1)->iowaited; + __entry->load_boost = + ((struct walt_task_struct *) p->android_vendor_data1)->load_boost; ), - TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d iowaited=%d", + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d iowaited=%d load_boost=%d", __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, __entry->candidates, __entry->best_energy_cpu, __entry->sync, __entry->need_idle, __entry->fastpath, __entry->placement_boost, __entry->latency, __entry->uclamp_boosted, __entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu, __entry->unfilter, __entry->cpus_allowed, __entry->task_boost, - __entry->low_latency, __entry->iowaited) + __entry->low_latency, __entry->iowaited, __entry->load_boost) ); /* @@ -1386,6 +1389,44 @@ TRACE_EVENT(halt_cpus, __entry->time, __entry->halt, __entry->success) ); +TRACE_EVENT(sched_task_handler, + TP_PROTO(struct task_struct *p, int param, int val, unsigned long c0, + unsigned long c1, unsigned long c2, unsigned long c3, + unsigned long c4, unsigned long c5), + + TP_ARGS(p, param, val, c0, c1, c2, c3, c4, c5), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, param) + __field(int, val) + __field(unsigned long, c0) + __field(unsigned long, c1) + __field(unsigned long, c2) + __field(unsigned long, c3) + __field(unsigned long, c4) + __field(unsigned long, c5) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->param = param; + __entry->val = val; + __entry->c0 = c0; + __entry->c1 = c1; + __entry->c2 = c2; + __entry->c3 = c3; + __entry->c4 = c4; + __entry->c5 = c5; + ), + + TP_printk("comm=%s pid=%d param=%d val=%d callers=%ps <- %ps <- %ps <- %ps <- %ps <- %ps", + __entry->comm, __entry->pid, __entry->param, __entry->val, __entry->c0, + __entry->c1, __entry->c2, __entry->c3, __entry->c4, __entry->c5) +); + TRACE_EVENT(update_cpu_capacity, TP_PROTO(int cpu, unsigned long rt_pressure, unsigned long capacity), diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c0a58e84e3f5..ea287de0c3fc 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1510,11 +1510,16 @@ static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, #define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) -static inline u64 scale_exec_time(u64 delta, struct rq *rq) +static inline u64 scale_exec_time(u64 delta, struct rq *rq, struct walt_task_struct *wts) { struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - return (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT; + delta = (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT; + + if (wts->load_boost && wts->grp && wts->grp->skip_min) + delta = (delta * (1024 + wts->boosted_task_load) >> 10); + + return delta; } /* Convert busy time to frequency equivalent @@ -1679,7 +1684,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, delta = wallclock - mark_start; else delta = irqtime; - delta = scale_exec_time(delta, rq); + delta = scale_exec_time(delta, rq, wts); *curr_runnable_sum += delta; if (new_task) *nt_curr_runnable_sum += delta; @@ -1721,7 +1726,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * A full window hasn't elapsed, account partial * contribution to previous completed window. */ - delta = scale_exec_time(window_start - mark_start, rq); + delta = scale_exec_time(window_start - mark_start, rq, wts); wts->prev_window += delta; wts->prev_window_cpu[cpu] += delta; } else { @@ -1730,7 +1735,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * the contribution to the previous window is the * full window (window_size). */ - delta = scale_exec_time(window_size, rq); + delta = scale_exec_time(window_size, rq, wts); wts->prev_window = delta; wts->prev_window_cpu[cpu] = delta; } @@ -1740,7 +1745,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, *nt_prev_runnable_sum += delta; /* Account piece of busy time in the current window. */ - delta = scale_exec_time(wallclock - window_start, rq); + delta = scale_exec_time(wallclock - window_start, rq, wts); *curr_runnable_sum += delta; if (new_task) *nt_curr_runnable_sum += delta; @@ -1770,7 +1775,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * A full window hasn't elapsed, account partial * contribution to previous completed window. */ - delta = scale_exec_time(window_start - mark_start, rq); + delta = scale_exec_time(window_start - mark_start, rq, wts); if (!is_idle_task(p)) { wts->prev_window += delta; wts->prev_window_cpu[cpu] += delta; @@ -1781,7 +1786,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * the contribution to the previous window is the * full window (window_size). */ - delta = scale_exec_time(window_size, rq); + delta = scale_exec_time(window_size, rq, wts); if (!is_idle_task(p)) { wts->prev_window = delta; wts->prev_window_cpu[cpu] = delta; @@ -1793,7 +1798,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, *nt_prev_runnable_sum += delta; /* Account piece of busy time in the current window. */ - delta = scale_exec_time(wallclock - window_start, rq); + delta = scale_exec_time(wallclock - window_start, rq, wts); *curr_runnable_sum += delta; if (new_task) *nt_curr_runnable_sum += delta; @@ -1828,7 +1833,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * window then that is all that need be accounted. */ if (mark_start > window_start) { - *curr_runnable_sum += scale_exec_time(irqtime, rq); + *curr_runnable_sum += scale_exec_time(irqtime, rq, wts); return; } @@ -1839,12 +1844,12 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, delta = window_start - mark_start; if (delta > window_size) delta = window_size; - delta = scale_exec_time(delta, rq); + delta = scale_exec_time(delta, rq, wts); *prev_runnable_sum += delta; /* Process the remaining IRQ busy time in the current window. */ delta = wallclock - window_start; - wrq->curr_runnable_sum += scale_exec_time(delta, rq); + wrq->curr_runnable_sum += scale_exec_time(delta, rq, wts); return; } @@ -1995,7 +2000,7 @@ static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - delta = scale_exec_time(delta, rq); + delta = scale_exec_time(delta, rq, wts); wts->sum += delta; if (unlikely(wts->sum > sched_ravg_window)) wts->sum = sched_ravg_window; @@ -2101,7 +2106,7 @@ static u64 update_task_demand(struct task_struct *p, struct rq *rq, /* Push new sample(s) into task's demand history */ update_history(rq, p, wts->sum, 1, event); if (nr_full_windows) { - u64 scaled_window = scale_exec_time(window_size, rq); + u64 scaled_window = scale_exec_time(window_size, rq, wts); update_history(rq, p, scaled_window, nr_full_windows, event); runtime += nr_full_windows * scaled_window; @@ -2262,6 +2267,8 @@ static inline void __sched_fork_init(struct task_struct *p) wts->boost_period = false; wts->low_latency = false; wts->iowaited = false; + wts->load_boost = 0; + wts->boosted_task_load = 0; } static void init_new_task_load(struct task_struct *p) From ec34dd43a0b708b58ff3aa3f4f579feefe2fd123 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 27 Apr 2022 17:07:35 -0700 Subject: [PATCH 310/346] sched/walt: use rq->clock_task from scheduler tick hook Since we are called from the main tick, rq clock task must have been updated very recently. Use it directly, instead of update_rq_clock_task() to avoid warnings. Change-Id: I8ea644cb28844b9ded6e45703aa72e72c21486f5 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_rt.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 13926fcb5ab5..8952f2504f8c 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -48,15 +48,20 @@ static void long_running_rt_task_notifier(void *unused, struct rq *rq) return; } - if (rq_clock_task(rq) - + /* + * Since we are called from the main tick, rq clock task must have + * been updated very recently. Use it directly, instead of + * update_rq_clock_task() to avoid warnings. + */ + if (rq->clock_task - per_cpu(rt_task_arrival_time, cpu) > sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC) { printk_deferred("RT task %s (%d) runtime > %u now=%llu task arrival time=%llu runtime=%llu\n", curr->comm, curr->pid, sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC, - rq_clock_task(rq), + rq->clock_task, per_cpu(rt_task_arrival_time, cpu), - rq_clock_task(rq) - + rq->clock_task - per_cpu(rt_task_arrival_time, cpu)); BUG(); } From 38d3a1c4e7b492eca24012100e6b5606c901ef38 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 29 Apr 2022 14:20:55 -0700 Subject: [PATCH 311/346] sched/walt: add wrq->latest_clock to the walt dump Tracking latest_clock from walt run queue structure. Change-Id: I50322b4efc91edbeaf61b72dd031e2a7e96edf28 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index ea287de0c3fc..a220a091d070 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -248,6 +248,7 @@ void walt_rq_dump(int cpu) cpu, rq->nr_running, tsk->pid, tsk->comm); printk_deferred("=========================================="); + SCHED_PRINT(wrq->latest_clock); SCHED_PRINT(wrq->window_start); SCHED_PRINT(wrq->prev_window_size); SCHED_PRINT(wrq->curr_runnable_sum); From dca8a7b03cb561af2d32b52f0f78d0a481917f43 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Fri, 29 Apr 2022 11:23:13 -0700 Subject: [PATCH 312/346] sched/walt: prevent main walt_irq_work execution for no notif If there are no notifications pending for performing migration work, there is no migration work to be done because the work must have already been handled. Optimize the walt_irq_work execution for migrations. If no cpus got locked, then no notifications were handled, and the irq work function should exit. Change-Id: I57730503ebb8f440a288666c79c4cb4f31ed1023 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index a220a091d070..127bd4d8e897 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -3674,9 +3674,18 @@ static void walt_irq_work(struct irq_work *irq_work) cpumask_copy(&lock_cpus, cpu_possible_mask); - if (is_migration) + if (is_migration) { irq_work_restrict_to_mig_clusters(&lock_cpus); + /* + * if the notif_pending was handled by a previous + * walt_irq_work invocation, there is no migration + * work. + */ + if (cpumask_empty(&lock_cpus)) + return; + } + for_each_cpu(cpu, &lock_cpus) { if (level == 0) raw_spin_lock(&cpu_rq(cpu)->__lock); From b8dab937cef37562765440393acf681c58e74fb5 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 28 Apr 2022 13:26:51 -0700 Subject: [PATCH 313/346] sched/walt: remove set_window_start set_window_start is a deprecated function that is never used. Remove it. Change-Id: I636a2a37e55c4d812606ea1f52c4ce80b70d44d2 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 43 ++-------------------------------------- 1 file changed, 2 insertions(+), 41 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 127bd4d8e897..c3e018a7da13 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1111,38 +1111,6 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) double_rq_unlock(src_rq, dest_rq); } -static void set_window_start(struct rq *rq) -{ - static int sync_cpu_available; - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - struct walt_rq *sync_wrq; - struct walt_task_struct *wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; - - if (likely(wrq->window_start)) - return; - - if (!sync_cpu_available) { - wrq->window_start = 1; - sync_cpu_available = 1; - atomic64_set(&walt_irq_work_lastq_ws, wrq->window_start); - walt_load_reported_window = - atomic64_read(&walt_irq_work_lastq_ws); - - } else { - struct rq *sync_rq = cpu_rq(cpumask_any(cpu_online_mask)); - - sync_wrq = (struct walt_rq *) sync_rq->android_vendor_data1; - raw_spin_unlock(&rq->__lock); - double_rq_lock(rq, sync_rq); - wrq->window_start = sync_wrq->window_start; - wrq->curr_runnable_sum = wrq->prev_runnable_sum = 0; - wrq->nt_curr_runnable_sum = wrq->nt_prev_runnable_sum = 0; - raw_spin_unlock(&sync_rq->__lock); - } - - wts->mark_start = wrq->window_start; -} - #define INC_STEP 8 #define DEC_STEP 2 #define CONSISTENT_THRES 16 @@ -3970,15 +3938,8 @@ static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long static void android_rvh_sched_cpu_starting(void *unused, int cpu) { - unsigned long flags; - struct rq *rq = cpu_rq(cpu); - if (unlikely(walt_disabled)) return; - raw_spin_lock_irqsave(&rq->__lock, flags); - set_window_start(rq); - raw_spin_unlock_irqrestore(&rq->__lock, flags); - clear_walt_request(cpu); } @@ -4218,10 +4179,10 @@ static void android_rvh_tick_entry(void *unused, struct rq *rq) { u64 wallclock; - lockdep_assert_held(&rq->__lock); if (unlikely(walt_disabled)) return; - set_window_start(rq); + + lockdep_assert_held(&rq->__lock); wallclock = walt_rq_clock(rq); walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); From db2bf2acc0d836dfe8eea94dbe8a139707ae1e21 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Sat, 30 Apr 2022 10:28:18 -0700 Subject: [PATCH 314/346] sched/walt: print global ws in UTRA traces Add a parameter in UTRA and UTRA_mini traces to help identify the global window_start. Change-Id: I1eb7432f47fe2f622ba1efc78047551b97f08952 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 21 +++++++++++++-------- kernel/sched/walt/walt.c | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index f780ea7cc85b..70de6d14d1bc 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -158,9 +158,9 @@ TRACE_EVENT(sched_update_task_ravg, TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, u64 wallclock, u64 irqtime, struct group_cpu_time *cpu_time, struct walt_rq *wrq, - struct walt_task_struct *wts), + struct walt_task_struct *wts, u64 walt_irq_work_lastq_ws), - TP_ARGS(p, rq, evt, wallclock, irqtime, cpu_time, wrq, wts), + TP_ARGS(p, rq, evt, wallclock, irqtime, cpu_time, wrq, wts, walt_irq_work_lastq_ws), TP_STRUCT__entry( __array(char, comm, TASK_COMM_LEN) @@ -194,6 +194,7 @@ TRACE_EVENT(sched_update_task_ravg, __field(u64, active_time) __field(u32, curr_top) __field(u32, prev_top) + __field(u64, walt_irq_work_lastq_ws) ), TP_fast_assign( @@ -232,9 +233,10 @@ TRACE_EVENT(sched_update_task_ravg, __entry->active_time = wts->active_time; __entry->curr_top = wrq->curr_top; __entry->prev_top = wrq->prev_top; + __entry->walt_irq_work_lastq_ws = walt_irq_work_lastq_ws; ), - TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand_scaled %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u", + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand_scaled %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u global_ws %llu", __entry->wallclock, __entry->win_start, __entry->delta, task_event_names[__entry->evt], __entry->cpu, __entry->cur_freq, __entry->cur_pid, @@ -248,7 +250,7 @@ TRACE_EVENT(sched_update_task_ravg, __entry->nt_cs, __entry->nt_ps, __entry->active_time, __entry->grp_cs, __entry->grp_ps, __entry->grp_nt_cs, __entry->grp_nt_ps, - __entry->curr_top, __entry->prev_top) + __entry->curr_top, __entry->prev_top, __entry->walt_irq_work_lastq_ws) ); TRACE_EVENT(sched_update_task_ravg_mini, @@ -256,9 +258,9 @@ TRACE_EVENT(sched_update_task_ravg_mini, TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, u64 wallclock, u64 irqtime, struct group_cpu_time *cpu_time, struct walt_rq *wrq, - struct walt_task_struct *wts), + struct walt_task_struct *wts, u64 walt_irq_work_lastq_ws), - TP_ARGS(p, rq, evt, wallclock, irqtime, cpu_time, wrq, wts), + TP_ARGS(p, rq, evt, wallclock, irqtime, cpu_time, wrq, wts, walt_irq_work_lastq_ws), TP_STRUCT__entry( __array(char, comm, TASK_COMM_LEN) @@ -277,6 +279,7 @@ TRACE_EVENT(sched_update_task_ravg_mini, __field(u64, grp_ps) __field(u32, curr_window) __field(u32, prev_window) + __field(u64, walt_irq_work_lastq_ws) ), TP_fast_assign( @@ -296,15 +299,17 @@ TRACE_EVENT(sched_update_task_ravg_mini, __entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0; __entry->curr_window = wts->curr_window; __entry->prev_window = wts->prev_window; + __entry->walt_irq_work_lastq_ws = walt_irq_work_lastq_ws; ), - TP_printk("wc %llu ws %llu delta %llu event %s cpu %d task %d (%s) ms %llu delta %llu demand %u rq_cs %llu rq_ps %llu cur_window %u prev_window %u grp_cs %lld grp_ps %lld", + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d task %d (%s) ms %llu delta %llu demand %u rq_cs %llu rq_ps %llu cur_window %u prev_window %u grp_cs %lld grp_ps %lld global_ws %llu", __entry->wallclock, __entry->win_start, __entry->delta, task_event_names[__entry->evt], __entry->cpu, __entry->pid, __entry->comm, __entry->mark_start, __entry->delta_m, __entry->demand, __entry->rq_cs, __entry->rq_ps, __entry->curr_window, - __entry->prev_window, __entry->grp_cs, __entry->grp_ps) + __entry->prev_window, __entry->grp_cs, __entry->grp_ps, + __entry->walt_irq_work_lastq_ws) ); struct migration_sum_data; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index c3e018a7da13..95cf1ed00be2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2215,9 +2215,9 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even wts->iowaited = p->in_iowait; trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime, - &wrq->grp_time, wrq, wts); + &wrq->grp_time, wrq, wts, atomic64_read(&walt_irq_work_lastq_ws)); trace_sched_update_task_ravg_mini(p, rq, event, wallclock, irqtime, - &wrq->grp_time, wrq, wts); + &wrq->grp_time, wrq, wts, atomic64_read(&walt_irq_work_lastq_ws)); done: wts->mark_start = wallclock; From 951ffacded2b9b2b48849ecc4f17d42eb738591c Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Tue, 26 Apr 2022 11:16:32 -0700 Subject: [PATCH 315/346] sched/walt: Ensure one 32bit capable cpu is never halted It is possible that all 32bit cpus are halted. In this case a 32 bit task can be woken, and the scheduler will not find a cpu for it to run on that is 32 bit capable. Prevent the least-significant 32bit capable cpu from ever being halted. Change-Id: I23f616a5988d7a7eaa6b0d7dd7ef82dbe6f5ec9b Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_halt.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index b91c1d6d28c7..f32ca51d3dff 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -271,6 +271,11 @@ static int halt_cpus(struct cpumask *cpus) for_each_cpu(cpu, cpus) { + if (cpu == cpumask_first(system_32bit_el0_cpumask())) { + ret = -EINVAL; + goto out; + } + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); /* set the cpu as halted */ @@ -290,6 +295,7 @@ static int halt_cpus(struct cpumask *cpus) if (!IS_ERR(walt_drain_thread)) wake_up_process(walt_drain_thread); +out: trace_halt_cpus(cpus, start_time, 1, ret); return ret; From 0e17af7bb06791216c31d92295e3cc2f6e1e4aa8 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 6 Apr 2022 19:54:58 -0700 Subject: [PATCH 316/346] sched/walt/halt: prevent selection of non-32 bit cpu When __migrate_task() is given a new cpu to run a 32bit task, it may be migrating to a halted cpu. is_cpu_allowed tracehook currently rejects halted cpus, forcing __migrate_task() to leave the task on its current cpu. The current cpu may not be 32 bit capable. If the halted state of the task is changed just prior to the migrate operation, the migration is rejected and the prev_cpu is chosen. If the prev_cpu is not 32 bit capable, then the task will be migrated to an improper cpu. This can only happen while a 32bit task is being execv'd. Detect the situation where a halted cpu is about to be rejected. If the task is in execve, and it's a 32 bit task, allow it to run on a halted cpu. Change-Id: I0a0ca79efeab08aed8136f92cc86dcc6b4179aab Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_halt.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index f32ca51d3dff..4a0c8f35f0af 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -519,12 +519,33 @@ static void android_rvh_rto_next_cpu(void *unused, int rto_cpu, struct cpumask * } } -static void android_rvh_is_cpu_allowed(void *unused, int cpu, bool *allowed) +/** + * android_rvh_is_cpu_allowed: disallow cpus that are halted + * + * Caveat: For 32 bit tasks that are being directed to a halted cpu, allow the halted cpu + * in a particular case (32 bit task, in execve, moving to a 32 bit cpu) + * This is to handle the call to is_cpu_allowed() from __migrate_task, in the + * event that a 32bit task is being execve'd. + */ +static void android_rvh_is_cpu_allowed(void *unused, struct task_struct *p, int cpu, bool *allowed) { if (unlikely(walt_disabled)) return; - *allowed = !cpumask_test_cpu(cpu, cpu_halt_mask); + if (cpumask_test_cpu(cpu, cpu_halt_mask)) { + + /* default reject for any halted cpu */ + *allowed = false; + + if (is_compat_thread(task_thread_info(p)) && + p->in_execve) { + /* + * the task is 32 bit capable and + * the context is execve. allow this cpu. + */ + *allowed = true; + } + } } void walt_halt_init(void) From 24982cc1a94c22b563f102ab4b476c10412ecbc3 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Sat, 30 Apr 2022 08:55:59 -0700 Subject: [PATCH 317/346] sched/walt: window rollover missing fix Currently, the code has a flow where a window rollover is not issued during migration. This could lead to a situation where no future window rollovers are handled. For example in a 4 + 3 + 1 topology, A B C indicate the window boundaries while X indicates the rq->window_start of each cpu and "GLB" indicates the timestamp of the global window tracker (walt_irq_work_lastq_ws) 1. A B C CPU0----X----------------------------------------------------------------- CPU1----X----------------------------------------------------------------- CPU2----X----------------------------------------------------------------- CPU3----X----------------------------------------------------------------- CPU4----X----------------------------------------------------------------- CPU5----X----------------------------------------------------------------- CPU6----X----------------------------------------------------------------- CPU7----X----------------------------------------------------------------- GLB X Now CPU1 gets an event(e) right before C, in which case it advances his rq1->window_start to B, also advances GLB to B and issues a rollover irq work, which will advance window_starts for all other cpus. 2. A B C CPU0----X----------------------------------------------------------------- CPU1----------------------------X---------------------e------------------- CPU2----X----------------------------------------------------------------- CPU3----X----------------------------------------------------------------- CPU4----X----------------------------------------------------------------- CPU5----X----------------------------------------------------------------- CPU6----X----------------------------------------------------------------- CPU7----X----------------------------------------------------------------- GLB X Since the event was very close to C, the irq_work(i) could run right after C, which advances all the window_starts to C, including that of cpu1. 3. A B C CPU0----------------------------------------------------X----------------- CPU1--------------------------------------------------e-X-i--------------- CPU2----------------------------------------------------X----------------- CPU3----------------------------------------------------X----------------- CPU4----------------------------------------------------X----------------- CPU5----------------------------------------------------X----------------- CPU6----------------------------------------------------X----------------- CPU7----------------------------------------------------X----------------- GLB X This advancement of window_start on cpu1 causes it to advance G (and reissue another irq_work, which is not relevant to this discussion) like below. CPU1 will advance the global because it matches its window_start (atomic_cmpxchg) 4. A B C CPU0----------------------------------------------------X----------------- CPU1--------------------------------------------------e-X-i--------------- CPU2----------------------------------------------------X----------------- CPU3----------------------------------------------------X----------------- CPU4----------------------------------------------------X----------------- CPU5----------------------------------------------------X----------------- CPU6----------------------------------------------------X----------------- CPU7----------------------------------------------------X----------------- GLB X However we could run in to a situation where the irq work (i) is held up because cpu0's rq lock isn't available (BTW a window rollover holds all the rqlocks) for some other reason. And while the irq work is held there, the rq locks of cpus1-7 are available, an inter cluster migration between cpu1 and cpu7 could happen causing cpu1,7 to advance their window_start. Redrawing the 3rd picture 3a. A B C CPU0----X----------------------------------------------------------------- CPU1--------------------------------------------------e-X---m-----i------- CPU2----X----------------------------------------------------------------- CPU3----X----------------------------------------------------------------- CPU4----X----------------------------------------------------------------- CPU5----X----------------------------------------------------------------- CPU6----X----------------------------------------------------------------- CPU7----------------------------------------------------X---m------------- GLB X Eventually when the irq work gets around to hold the locks on all the cpus, it will move the rest of the cpu's window_start to C. However the Global is left at B and cpu1 that should have advanced the Global, won't do it because of an unchecked advancement by the migration with cpu7. After this point the global is old and will never match any cpu's window_start and WALT will never do a window_rollover 4a. A B C CPU0----------------------------------------------------X----------------- CPU1--------------------------------------------------e-X---m-----i------- CPU2----------------------------------------------------X----------------- CPU3----------------------------------------------------X----------------- CPU4----------------------------------------------------X----------------- CPU5----------------------------------------------------X----------------- CPU6----------------------------------------------------X----------------- CPU7----------------------------------------------------X---m------------- GLB X Fix this by issuing migration walt_irq_work when an intercluster migration ends up rolling over the window in the dest rq. Change-Id: Ib3985a1cdf72342305398a9c648159195ca99658 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 95cf1ed00be2..d138ddc8e61d 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -993,6 +993,7 @@ static inline bool is_new_task(struct task_struct *p) return wts->active_time < NEW_TASK_ACTIVE_TIME; } +static inline void run_walt_irq_work_rollover(u64 old_window_start, struct rq *rq); static void fixup_busy_time(struct task_struct *p, int new_cpu) { @@ -1009,6 +1010,7 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u64 old_window_start; if (!p->on_rq && READ_ONCE(p->__state) != TASK_WAKING) return; @@ -1040,7 +1042,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) * window boundary or if the counters will be accessed * or not. */ - update_window_start(dest_rq, wallclock, TASK_UPDATE); + old_window_start = update_window_start(dest_rq, wallclock, TASK_UPDATE); + run_walt_irq_work_rollover(old_window_start, dest_rq); update_task_cpu_cycles(p, new_cpu, wallclock); From 549ffdbf7a59b0cbbfadde8847fb68c56211cfc6 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 5 May 2022 18:34:17 -0700 Subject: [PATCH 318/346] Revert "sched: walt: Remove finding historical runtimes in pred" This reverts commit 5c435ea559e452a09484fc20b65c3e128a77d8ab. Change-Id: Iabbe633fed376b0ca3bfb5dad78ce96f512f367c Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 1 + kernel/sched/walt/trace.h | 9 ++++++++- kernel/sched/walt/walt.c | 29 +++++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 0bf24642e528..9bb6ceada1a7 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -97,6 +97,7 @@ struct walt_task_struct { u32 sum, demand; u32 coloc_demand; u32 sum_history[RAVG_HIST_SIZE]; + u16 sum_history_util[RAVG_HIST_SIZE]; u32 curr_window_cpu[WALT_NR_CPUS]; u32 prev_window_cpu[WALT_NR_CPUS]; u32 curr_window, prev_window; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 70de6d14d1bc..0bfd17eea7fc 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -84,6 +84,7 @@ TRACE_EVENT(sched_update_history, __field(unsigned int, coloc_demand) __field(unsigned int, pred_demand_scaled) __array(u32, hist, RAVG_HIST_SIZE) + __array(u16, hist_util, RAVG_HIST_SIZE) __field(unsigned int, nr_big_tasks) __field(int, cpu) ), @@ -99,11 +100,13 @@ TRACE_EVENT(sched_update_history, __entry->pred_demand_scaled = wts->pred_demand_scaled; memcpy(__entry->hist, wts->sum_history, RAVG_HIST_SIZE * sizeof(u32)); + memcpy(__entry->hist_util, wts->sum_history_util, + RAVG_HIST_SIZE * sizeof(u16)); __entry->nr_big_tasks = wrq->walt_stats.nr_big_tasks; __entry->cpu = rq->cpu; ), - TP_printk("%d (%s): runtime %u samples %d event %s demand %u (hist: %u %u %u %u %u %u %u %u) coloc_demand %u pred_demand_scaled %u cpu %d nr_big %u", + TP_printk("%d (%s): runtime %u samples %d event %s demand %u (hist: %u %u %u %u %u %u %u %u) (hist_util: %u %u %u %u %u %u %u %u) coloc_demand %u pred_demand_scaled %u cpu %d nr_big %u", __entry->pid, __entry->comm, __entry->runtime, __entry->samples, task_event_names[__entry->evt], @@ -112,6 +115,10 @@ TRACE_EVENT(sched_update_history, __entry->hist[2], __entry->hist[3], __entry->hist[4], __entry->hist[5], __entry->hist[6], __entry->hist[7], + __entry->hist_util[0], __entry->hist_util[1], + __entry->hist_util[2], __entry->hist_util[3], + __entry->hist_util[4], __entry->hist_util[5], + __entry->hist_util[6], __entry->hist_util[7], __entry->coloc_demand, __entry->pred_demand_scaled, __entry->cpu, __entry->nr_big_tasks) ); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index d138ddc8e61d..1a7aa6fa6701 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1185,7 +1185,9 @@ static inline int busy_to_bucket(u16 normalized_rt) * A new predicted busy time is returned for task @p based on @runtime * passed in. The function searches through buckets that represent busy * time equal to or bigger than @runtime and attempts to find the bucket - * to use for prediction. Once found, it returns the midpoint of that bucket. + * to use for prediction. Once found, it searches through historical busy + * time and returns the latest that falls into the bucket. If no such busy + * time exists, it returns the medium of that bucket. */ static u32 get_pred_busy(struct task_struct *p, int start, u16 runtime_scaled, u16 bucket_bitmask) @@ -1195,6 +1197,8 @@ static u32 get_pred_busy(struct task_struct *p, int first = NUM_BUSY_BUCKETS, final = NUM_BUSY_BUCKETS; u16 ret = runtime_scaled; u16 next_mask = bucket_bitmask >> start; + u16 *hist_util = wts->sum_history_util; + int i; /* skip prediction for new tasks due to lack of history */ if (unlikely(is_new_task(p))) @@ -1221,11 +1225,24 @@ static u32 get_pred_busy(struct task_struct *p, } dmax = (final + 1) << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); + /* + * search through runtime history and return first runtime that falls + * into the range of predicted bucket. + */ + for (i = 0; i < RAVG_HIST_SIZE; i++) { + if (hist_util[i] >= dmin && hist_util[i] < dmax) { + ret = hist_util[i]; + break; + } + } + /* no historical runtime within bucket found, use average of the bin */ + if (ret < dmin) + ret = (u16) (((u32)dmin + dmax) / 2); /* * when updating in middle of a window, runtime could be higher * than all recorded history. Always predict at least runtime. */ - ret = max(runtime_scaled, (u16) (((u32)dmin + dmax) / 2)); + ret = max(runtime_scaled, ret); out: trace_sched_update_pred_demand(p, runtime_scaled, ret, start, first, final, wts); @@ -1896,19 +1913,23 @@ static void update_history(struct rq *rq, struct task_struct *p, { struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u32 *hist = &wts->sum_history[0]; + u16 *hist_util = &wts->sum_history_util[0]; int i; u32 max = 0, avg, demand; u64 sum = 0; - u16 demand_scaled, pred_demand_scaled; + u16 demand_scaled, pred_demand_scaled, runtime_scaled; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; /* Ignore windows where task had no activity */ if (!runtime || is_idle_task(p) || !samples) goto done; + runtime_scaled = scale_time_to_util(runtime); /* Push new 'runtime' value onto stack */ for (; samples > 0; samples--) { hist[wts->cidx] = runtime; + hist_util[wts->cidx] = runtime_scaled; wts->cidx = ++(wts->cidx) & RAVG_HIST_MASK; } @@ -1931,7 +1952,7 @@ static void update_history(struct rq *rq, struct task_struct *p, else demand = max(avg, runtime); } - pred_demand_scaled = predict_and_update_buckets(p, scale_time_to_util(runtime)); + pred_demand_scaled = predict_and_update_buckets(p, runtime_scaled); demand_scaled = scale_time_to_util(demand); /* From 86a2dd10daab1b9e74894649b15460ee3f668344 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 5 May 2022 11:08:38 -0700 Subject: [PATCH 319/346] sched/walt: Kick newidle balance after unhalting When a cpu is unhalted currently, it will attempt to perform a balance through the normal ipi kick interface. However, this can fail because a balance might have happened in the current 4mS tick, while the cpu was still halted. The next kick as a result of unhalting, fails to execute (min 1 tick between kicks). Instead, use walt_newidle_balance() to ensure that this cpu can take a task. Perform an smp async function call to the target cpu, invoke newidle balance, and allow the cpu to pull tasks immediately after unhalting. Change-Id: Ide1bb51f80dc8c4eba079588ffb6a7ed341e8628 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.h | 1 + kernel/sched/walt/walt_halt.c | 2 +- kernel/sched/walt/walt_lb.c | 37 +++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 47a4168cb00e..08d958eacaf0 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -163,6 +163,7 @@ extern void walt_boost_init(void); extern int sched_pause_cpus(struct cpumask *pause_cpus); extern int sched_unpause_cpus(struct cpumask *unpause_cpus); extern void walt_kick_cpu(int cpu); +extern void walt_smp_call_newidle_balance(int cpu); extern unsigned int sched_get_cpu_util_pct(int cpu); extern void sched_update_hyst_times(void); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 4a0c8f35f0af..346180098231 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -325,7 +325,7 @@ static int start_cpus(struct cpumask *cpus) /* kick the cpu so it can pull tasks * after the mask has been cleared. */ - walt_kick_cpu(cpu); + walt_smp_call_newidle_balance(cpu); } trace_halt_cpus(cpus, start_time, 0, 0); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 212247934c17..f6ccf195eec2 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -918,6 +918,34 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, help_min_cap, enough_idle); } +/* run newidle balance as a result of an unhalt operation */ +void walt_smp_newidle_balance(void *ignored) +{ + int cpu = raw_smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + int pulled_task; + int done = 0; + + rq_lock(rq, &rf); + update_rq_clock(rq); + walt_newidle_balance(NULL, rq, &rf, &pulled_task, &done); + resched_curr(rq); + rq_unlock(rq, &rf); +} + +static DEFINE_PER_CPU(call_single_data_t, nib_csd); + +void walt_smp_call_newidle_balance(int cpu) +{ + call_single_data_t *csd = &per_cpu(nib_csd, cpu); + + if (unlikely(walt_disabled)) + return; + + smp_call_function_single_async(cpu, csd); +} + static void walt_find_busiest_queue(void *unused, int dst_cpu, struct sched_group *group, struct cpumask *env_cpus, @@ -1028,6 +1056,8 @@ static void walt_can_migrate_task(void *unused, struct task_struct *p, void walt_lb_init(void) { + int cpu; + walt_lb_rotate_work_init(); register_trace_android_rvh_migrate_queued_task(walt_migrate_queued_task, NULL); @@ -1035,4 +1065,11 @@ void walt_lb_init(void) register_trace_android_rvh_can_migrate_task(walt_can_migrate_task, NULL); register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); register_trace_android_rvh_sched_newidle_balance(walt_newidle_balance, NULL); + + for_each_cpu(cpu, cpu_possible_mask) { + call_single_data_t *csd; + + csd = &per_cpu(nib_csd, cpu); + INIT_CSD(csd, walt_smp_newidle_balance, (void *)(unsigned long)cpu); + } } From bd66d3a04f724b981cb1660bc2851d7e172ddef1 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 17 Mar 2022 15:52:15 -0700 Subject: [PATCH 320/346] sched/walt/avg: cache window based statistics Previously it was possible to update core control with new statistics and those statistics would be unavailable to any other decisions that need to be made. Update sched_avg to cache the statistics locally and simply pass a reference to that information back to core control. Provide a cluster util function to provide window based load. Create a sysctl node and threshold to use to compare against cluster util. Change-Id: I8cc40603e49e29e7f8a343f6ff667ce9abe09cb5 Signed-off-by: Stephen Dickey --- kernel/sched/walt/core_ctl.c | 14 +++++++-- kernel/sched/walt/sched_avg.c | 55 +++++++++++++++++++++++++++++++++-- kernel/sched/walt/walt.c | 6 ++++ kernel/sched/walt/walt.h | 4 ++- 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 12abfd26947f..3bcb59d8f646 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -482,7 +482,7 @@ static struct kobj_type ktype_core_ctl = { /* ==================== runqueue based core count =================== */ -static struct sched_avg_stats nr_stats[WALT_NR_CPUS]; +static struct sched_avg_stats *nr_stats; /* * nr_need: @@ -686,7 +686,7 @@ static void update_running_avg(void) unsigned long flags; int big_avg = 0; - sched_get_nr_running_avg(nr_stats); + nr_stats = sched_get_nr_running_avg(); spin_lock_irqsave(&state_lock, flags); for_each_cluster(cluster, index) { @@ -954,6 +954,14 @@ static void core_ctl_call_notifier(void) atomic_notifier_call_chain(&core_ctl_notifier, 0, &ndata); } +/* + * sched_get_nr_running_avg will wipe out previous statistics and + * update it to the values computed since the last call. + * + * core_ctl_check assumes that the statistics are stable, hence + * window based. Therefore core_ctl_check must only be called from + * window rollover, or walt_irq_work for not migration. + */ void core_ctl_check(u64 window_start) { int cpu; @@ -1330,6 +1338,8 @@ int core_ctl_init(void) spin_lock_init(&core_ctl_pending_lock); + nr_stats = sched_get_nr_running_avg(); + /* initialize our single kthread, after spin lock init */ core_ctl_thread = kthread_run(try_core_ctl, NULL, "core_ctl"); diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index b2c0a737c107..c18e5c2234a9 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -34,6 +34,28 @@ static DEFINE_PER_CPU(u64, util_hyst_time); #define NR_THRESHOLD_PCT 40 #define MAX_RTGB_TIME (sysctl_sched_coloc_busy_hyst_max_ms * NSEC_PER_MSEC) +struct sched_avg_stats stats[WALT_NR_CPUS]; +unsigned int cstats_util_pct[MAX_CLUSTERS]; + +/** + * sched_get_cluster_util_pct + * @return: provide the percentage of this cluter that was used in the + * previous window. + * + * This routine may be called any number of times as needed during + * a window, but will always return the same result until window + * rollover. + */ +unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster) +{ + unsigned int cluster_util_pct = 0; + + if (cluster->id < MAX_CLUSTERS) + cluster_util_pct = cstats_util_pct[cluster->id]; + + return cluster_util_pct; +} + /** * sched_get_nr_running_avg * @return: Average nr_running, iowait and nr_big_tasks value since last poll. @@ -41,18 +63,22 @@ static DEFINE_PER_CPU(u64, util_hyst_time); * of accuracy. * * Obtains the average nr_running value since the last poll. - * This function may not be called concurrently with itself + * This function may not be called concurrently with itself. + * + * It is assumed that this function is called at most once per window + * rollover. */ -void sched_get_nr_running_avg(struct sched_avg_stats *stats) +struct sched_avg_stats *sched_get_nr_running_avg(void) { int cpu; u64 curr_time = sched_clock(); u64 period = curr_time - last_get_time; u64 tmp_nr, tmp_misfit; bool any_hyst_time = false; + struct walt_sched_cluster *cluster; if (!period) - return; + goto done; /* read and reset nr_running counts */ for_each_possible_cpu(cpu) { @@ -96,6 +122,27 @@ void sched_get_nr_running_avg(struct sched_avg_stats *stats) spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); } + /* collect cluster load stats */ + for_each_sched_cluster(cluster) { + unsigned int num_cpus = cpumask_weight(&cluster->cpus); + unsigned int sum_util_pct = 0; + + /* load is already scaled, see freq_policy_load/prev_runnable_sum */ + for_each_cpu(cpu, &cluster->cpus) { + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + /* compute the % this cpu's utilization of the cpu capacity, + * and sum it across all cpus + */ + sum_util_pct += + (wrq->util * 100) / arch_scale_cpu_capacity(cpu); + } + + /* calculate the averge per-cpu utilization */ + cstats_util_pct[cluster->id] = sum_util_pct / num_cpus; + } + for_each_possible_cpu(cpu) { if (per_cpu(coloc_hyst_time, cpu)) { any_hyst_time = true; @@ -107,6 +154,8 @@ void sched_get_nr_running_avg(struct sched_avg_stats *stats) last_get_time = curr_time; +done: + return &stats[0]; } EXPORT_SYMBOL(sched_get_nr_running_avg); diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 1a7aa6fa6701..696ad1a99d68 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -653,6 +653,12 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea util = scale_time_to_util(freq_policy_load(rq, reason)); + /* + * util is on a scale of 0 to 1024. this is the utilization + * of the cpu in the last window + */ + wrq->util = util; + if (walt_load) { u64 nl = wrq->nt_prev_runnable_sum + wrq->grp_time.nt_prev_runnable_sum; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 08d958eacaf0..2c1487df0181 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -120,6 +120,7 @@ struct walt_rq { bool high_irqload; u64 last_cc_update; u64 cycles; + u64 util; struct list_head mvp_tasks; int num_mvp_tasks; u64 latest_clock; @@ -773,7 +774,8 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) return task_fits_capacity(p, capacity, cpu); } -extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); +extern struct sched_avg_stats *sched_get_nr_running_avg(void); +extern unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster); extern void sched_update_hyst_times(void); extern void walt_rt_init(void); From e7853f5841bae6d0ba9499e4ee1fd9bd9bb9019d Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Thu, 31 Mar 2022 20:18:46 -0700 Subject: [PATCH 321/346] sched/walt: add functionality for choosing a packing cpu Very small tasks can end up spread across all cpus in a cluster, and cause excessive wfi and lpm exit latencies. This can be reduced significantly by combining these tasks on to a single cpu. Create the functionality to be used in cfs/rt that can make the decision as to whether a packing cpu should be chosen. Add supporting sysctl nodes to tune the thresholds for this. Change-Id: I68723b8dc461efffeba3069bfd5d41724f071bd2 Signed-off-by: Stephen Dickey --- kernel/sched/walt/cpufreq_walt.c | 19 +++++++ kernel/sched/walt/sched_avg.c | 11 ++++ kernel/sched/walt/sysctl.c | 20 +++++++ kernel/sched/walt/walt.c | 92 ++++++++++++++++++++++++++++++++ kernel/sched/walt/walt.h | 8 +++ 5 files changed, 150 insertions(+) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 54b22401b581..3b5d7109ca15 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -188,6 +188,25 @@ static void waltgov_calc_avg_cap(struct waltgov_policy *wg_policy, u64 curr_ws, wg_policy->last_ws = curr_ws; } +/* + * if waltgov is initialized, return the avg_cap seen + * over the last window. return 0 otherwise. + */ +unsigned int waltgov_get_avg_cap(unsigned int cpu) +{ + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + + /* if the policy is not initialized, or the callback + * is not initialized. callback is initialized on + * start of waltgov, erased on stop of waltgov. + */ + if (!wg_policy || !wg_cpu->cb.func) + return 0; + + return wg_policy->avg_cap; +} + static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time, unsigned int next_freq) { diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index c18e5c2234a9..4e5a267d2b17 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -56,6 +56,16 @@ unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster) return cluster_util_pct; } +unsigned int sched_get_cpu_avg_cap(int cpu) +{ + unsigned int cpu_avg_cap = 0; + + if (cpu < WALT_NR_CPUS) + cpu_avg_cap = stats[cpu].avg_cap; + + return cpu_avg_cap; +} + /** * sched_get_nr_running_avg * @return: Average nr_running, iowait and nr_big_tasks value since last poll. @@ -109,6 +119,7 @@ struct sched_avg_stats *sched_get_nr_running_avg(void) NR_THRESHOLD_PCT), 100); stats[cpu].nr_max = per_cpu(nr_max, cpu); stats[cpu].nr_scaled = tmp_nr; + stats[cpu].avg_cap = waltgov_get_avg_cap(cpu); trace_sched_get_nr_running_avg(cpu, stats[cpu].nr, stats[cpu].nr_misfit, stats[cpu].nr_max, diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 1be0123ebcbe..310956da94e0 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -72,6 +72,8 @@ unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000; unsigned int sysctl_sched_asymcap_boost; unsigned int sysctl_sched_long_running_rt_task_ms; +unsigned int sysctl_sched_idle_enough; +unsigned int sysctl_sched_cluster_util_thres_pct; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -893,6 +895,24 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "sched_cluster_util_thres_pct", + .data = &sysctl_sched_cluster_util_thres_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sched_idle_enough", + .data = &sysctl_sched_idle_enough, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, { .procname = "sched_long_running_rt_task_ms", .data = &sysctl_sched_long_running_rt_task_ms, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 696ad1a99d68..bf159a4f3fb3 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -343,6 +343,98 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, static void rollover_cpu_window(struct rq *rq, bool full_window); static void rollover_top_tasks(struct rq *rq, bool full_window); +/* + * if last window's average capacity is less than or + * equal to the current capacity, return true. + */ +static inline bool is_cpufreq_avg_or_higher(int cpu) +{ + unsigned int avg_cap = sched_get_cpu_avg_cap(cpu); + unsigned int cur_cap = capacity_curr_of(cpu); + + if (cur_cap >= avg_cap && avg_cap != 0) + return true; + + return false; +} + +/* walt_find_cluster_packing_cpu - Return a packing_cpu choice common for this cluster. + * @start_cpu: The cpu from the cluster to choose from + * + * If the cluster has a 32bit capable cpu return it regardless + * of whether it is halted or not. + * + * If the cluster does not have a 32 bit capable cpu, find the + * first unhalted, active cpu in this cluster. + */ +int walt_find_cluster_packing_cpu(int start_cpu) +{ + struct rq *rq = cpu_rq(start_cpu); + struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + cpumask_t unhalted_cpus; + cpumask_t cluster_32bit_cpus; + + /* find all 32 bit capable cpus in this cluster */ + cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask()); + + /* pack 32 bit and 64 bit tasks on the same cpu, if possible */ + if (cpumask_weight(&cluster_32bit_cpus) > 0) + return cpumask_first(&cluster_32bit_cpus); + + /* find all unhalted active cpus */ + cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); + + /* find all unhalted active cpus in this cluster */ + cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); + + /* return the first found unhalted, active cpu, in this cluster */ + return cpumask_first(&unhalted_cpus); +} + +/* for cfs and rt, determine if packing_cpu should be used */ +bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p) +{ + struct rq *rq; + struct walt_rq *wrq; + struct walt_sched_cluster *cluster; + + /* packing cpu must be a valid cpu for runqueue lookup */ + if (packing_cpu >= nr_cpu_ids) + return false; + + rq = cpu_rq(packing_cpu); + wrq = (struct walt_rq *)rq->android_vendor_data1; + cluster = wrq->cluster; + + /* if idle_enough feature is not enabled */ + if (!sysctl_sched_idle_enough) + return false; + + /* if cpu is not allowed for this task */ + if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr)) + return false; + + /* if cluster util is high */ + if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct) + return false; + + /* if cpu utilization is high */ + if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough) + return false; + + /* don't pack big tasks */ + if (task_util(p) >= sysctl_sched_idle_enough) + return false; + + /* if cpufreq is lower than the previous window */ + if (!is_cpufreq_avg_or_higher(packing_cpu)) + return false; + + /* the packing cpu can be used, so pack! */ + return true; +} + /* * Demand aggregation for frequency purpose: * diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 2c1487df0181..7c58e3d549b8 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -187,6 +187,9 @@ extern int max_possible_cluster_id; extern unsigned int __read_mostly sched_init_task_load_windows; extern unsigned int __read_mostly sched_load_granule; +extern unsigned int sysctl_sched_idle_enough; +extern unsigned int sysctl_sched_cluster_util_thres_pct; + /* 1ms default for 20ms window size scaled to 1024 */ extern unsigned int sysctl_sched_min_task_util_for_boost; extern unsigned int sysctl_sched_min_task_util_for_uclamp; @@ -345,6 +348,7 @@ struct sched_avg_stats { int nr_misfit; int nr_max; int nr_scaled; + u32 avg_cap; }; struct waltgov_callback { @@ -776,6 +780,8 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) extern struct sched_avg_stats *sched_get_nr_running_avg(void); extern unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster); +extern unsigned int sched_get_cpu_avg_cap(int cpu); +extern unsigned int waltgov_get_avg_cap(unsigned int cpu); extern void sched_update_hyst_times(void); extern void walt_rt_init(void); @@ -784,6 +790,8 @@ extern void walt_halt_init(void); extern void walt_fixup_init(void); extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync, int sibling_count_hint); +extern int walt_find_cluster_packing_cpu(int start_cpu); +extern bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p); static inline unsigned int cpu_max_possible_freq(int cpu) { From 36c37e32cebba7e1464c6ff439a3e91a34f98844 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 6 Apr 2022 14:45:39 -0700 Subject: [PATCH 322/346] sched/walt cfs: rt: find a packing cpu Under low-util conditions, small tasks can be packed onto a single dedicated cpu per cluster, the first cpu in the cluster. This can improve performance by ensuring fewer exits from wfi or deep-idle, and reduce power by keeping more of the other cpus in wfi or deep-idle, for longer. Add the functionality to perform choosing of the packing cpu and incorporate this into the select_rq path for fair and rt in walt hooks. Change-Id: I02675ceb6dd2e5fb40f4b14308159cc1fdde245e Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt_cfs.c | 16 ++++++++++++++++ kernel/sched/walt/walt_rt.c | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 989ac84ad934..b6378331fd61 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -219,6 +219,7 @@ enum fastpaths { NONE = 0, SYNC_WAKEUP, PREV_CPU_FASTPATH, + CLUSTER_PACKING_FASTPATH, }; static inline bool is_complex_sibling_idle(int cpu) @@ -251,6 +252,7 @@ static void walt_find_best_target(struct sched_domain *sd, bool rtg_high_prio_task = task_rtg_high_prio(p); cpumask_t visit_cpus; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + int packing_cpu; struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1; struct walt_rq *start_wrq; @@ -272,6 +274,14 @@ static void walt_find_best_target(struct sched_domain *sd, most_spare_wake_cap = LONG_MIN; } + /* fast path for packing_cpu */ + packing_cpu = walt_find_cluster_packing_cpu(start_cpu); + if (walt_choose_packing_cpu(packing_cpu, p)) { + fbt_env->fastpath = CLUSTER_PACKING_FASTPATH; + cpumask_set_cpu(packing_cpu, candidates); + goto out; + } + /* fast path for prev_cpu */ if (((prev_wrq->cluster->id == start_wrq->cluster->id) || asym_cap_siblings(prev_cpu, start_cpu)) && @@ -842,6 +852,12 @@ int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, goto unlock; first_cpu = cpumask_first(candidates); + + if (fbt_env.fastpath == CLUSTER_PACKING_FASTPATH) { + best_energy_cpu = first_cpu; + goto unlock; + } + if (weight == 1) { if (available_idle_cpu(first_cpu) || first_cpu == prev_cpu) { best_energy_cpu = first_cpu; diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 8952f2504f8c..02ba1d2200c1 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -234,6 +234,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c bool sync = !!(wake_flags & WF_SYNC); int ret, target = -1, this_cpu; struct cpumask *lowest_mask; + int packing_cpu; if (unlikely(walt_disabled)) return; @@ -288,6 +289,13 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, task, lowest_mask, walt_rt_task_fits_capacity); + /* create a fastpath for finding a packing cpu */ + packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); + if (walt_choose_packing_cpu(packing_cpu, task)) { + *new_cpu = packing_cpu; + goto unlock; + } + walt_rt_energy_aware_wake_cpu(task, lowest_mask, ret, &target); /* @@ -310,7 +318,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c if (target < nr_cpu_ids) *new_cpu = target; } - +unlock: rcu_read_unlock(); } @@ -319,6 +327,15 @@ static void walt_rt_find_lowest_rq(void *unused, struct task_struct *task, struct cpumask *lowest_mask, int ret, int *best_cpu) { + int packing_cpu; + + /* create a fastpath for finding a packing cpu */ + packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); + if (walt_choose_packing_cpu(packing_cpu, task)) { + *best_cpu = packing_cpu; + return; + } + walt_rt_energy_aware_wake_cpu(task, lowest_mask, ret, best_cpu); /* From 4c409fadbe79f9cfd4f96d485db73f8c190568f5 Mon Sep 17 00:00:00 2001 From: Ashay Jaiswal Date: Tue, 10 May 2022 14:33:44 +0530 Subject: [PATCH 323/346] sched: walt: allow silver to help prime during newidle balance Present newidle balance design doesn't allow farthest cluster to help during newidle balance (silver's newidle balance cannot help prime and prime cannot help silver). For Example: If silver is entering idle and it find's prime core busy then newidle balance will try to find and kick an idle gold core but it will not participate in pulling task from prime to silver. Update the logic to allow silver cluster to help and pull task during newidle balance if none of the cores in the nearest cluster (Gold here) are idle. Change-Id: I0908058d55f344467f559b50209a6bf8bb18b0ef Signed-off-by: Ashay Jaiswal --- kernel/sched/walt/walt_lb.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index f6ccf195eec2..500ec59807c4 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -840,13 +840,19 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, goto found_busy_cpu; } - /* help the farthest cluster indirectly if it needs help */ + /* + * help the farthest cluster by kicking an idle cpu in the next + * cluster. In case no idle is found, pull it in. + */ busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], &has_misfit); if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - walt_kick_cpu(first_idle); + if (first_idle != 1) + walt_kick_cpu(first_idle); + else + goto found_busy_cpu; } } else if (order_index == 2) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], From 7adf761c84b82987b1d41d599a96a370624066b0 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 12 May 2022 16:15:03 -0700 Subject: [PATCH 324/346] sched/walt: Add global_ws to walt dump This change is for general scheduler improvement. Change-Id: I88cf8678b918aed530f3af122edea2bca5b60503 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index bf159a4f3fb3..e8c7ecf82dc2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -281,6 +281,8 @@ void walt_dump(void) printk_deferred("Sched clock: %llu\n", walt_sched_clock()); printk_deferred("Time last window changed=%lu\n", sched_ravg_window_change_time); + printk_deferred("global_ws=%llu\n", + atomic64_read(&walt_irq_work_lastq_ws)); for_each_online_cpu(cpu) walt_rq_dump(cpu); SCHED_PRINT(max_possible_cluster_id); From 8075477f59d5c9771f596c7e98e9ebe992fb1c90 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 16 May 2022 11:26:39 -0700 Subject: [PATCH 325/346] sched/walt: Add thermal pressure to cpu_util TP In the event that thermal throttling is under way, and user has not enabled update_cpu_capacity trace point, add a backup way of quickly determining scheduler under thermal constraints by displaying the thermal pressure in the sched_cpu_util trace point. Change-Id: I36b8f23e37b4e9b42770e1535feabb9d513fb730 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 0bfd17eea7fc..30d1a1fffb73 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -950,6 +950,7 @@ TRACE_EVENT(sched_cpu_util, __field(unsigned int, nr_rtg_high_prio_tasks) __field(u64, prs_gprs) __field(unsigned int, lowest_mask) + __field(unsigned long, thermal_pressure) ), TP_fast_assign( @@ -974,16 +975,17 @@ TRACE_EVENT(sched_cpu_util, __entry->lowest_mask = 0; else __entry->lowest_mask = cpumask_bits(lowest_mask)[0]; + __entry->thermal_pressure = arch_scale_thermal_pressure(cpu); ), - TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%lu capacity=%lu capacity_orig=%lu idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, halted=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u prs_gprs=%llu lowest_mask=0x%x", + TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%lu capacity=%lu capacity_orig=%lu idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, halted=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u prs_gprs=%llu lowest_mask=0x%x thermal_pressure=%llu", __entry->cpu, __entry->nr_running, __entry->cpu_util, __entry->cpu_util_cum, __entry->capacity_curr, __entry->capacity, __entry->capacity_orig, __entry->idle_exit_latency, __entry->irqload, __entry->online, __entry->inactive, __entry->halted, __entry->reserved, __entry->high_irq_load, __entry->nr_rtg_high_prio_tasks, __entry->prs_gprs, - __entry->lowest_mask) + __entry->lowest_mask, __entry->thermal_pressure) ); TRACE_EVENT(sched_compute_energy, From 301ac5032acb0a4b848a24c06076d21f7b0fc31b Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 18 May 2022 09:55:17 -0700 Subject: [PATCH 326/346] sched/walt: Cleanup CPUFREQ flags and reasons Currently, the flags and reasons are defined based off left shifts, which requires manual translation when printed in traces. Change those definitions to be directly in hex format, maintaining congruence. Additionally, in the event that a boost_util is 0, ensure that a reason is not inappropriately assigned. Change-Id: Ia8e9c1628373f3355a3fcd9707a5950a9757e42b Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 2 +- kernel/sched/walt/walt.h | 35 ++++++++++++++++---------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 3b5d7109ca15..61c839aae70b 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -300,7 +300,7 @@ static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) static inline void max_and_reason(unsigned long *cur_util, unsigned long boost_util, struct waltgov_cpu *wg_cpu, unsigned int reason) { - if (boost_util >= *cur_util) { + if (boost_util && boost_util >= *cur_util) { *cur_util = boost_util; wg_cpu->reasons = reason; wg_cpu->wg_policy->driving_cpu = wg_cpu->cpu; diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 7c58e3d549b8..4b6154a8c32d 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -296,24 +296,25 @@ extern char sched_lib_name[LIB_PATH_LENGTH]; extern unsigned int sched_lib_mask_force; /* WALT cpufreq interface */ -#define WALT_CPUFREQ_ROLLOVER (1U << 0) -#define WALT_CPUFREQ_CONTINUE (1U << 1) -#define WALT_CPUFREQ_IC_MIGRATION (1U << 2) -#define WALT_CPUFREQ_PL (1U << 3) -#define WALT_CPUFREQ_EARLY_DET (1U << 4) -#define WALT_CPUFREQ_BOOST_UPDATE (1U << 5) +#define WALT_CPUFREQ_ROLLOVER 0x1 +#define WALT_CPUFREQ_CONTINUE 0x2 +#define WALT_CPUFREQ_IC_MIGRATION 0x4 +#define WALT_CPUFREQ_PL 0x8 +#define WALT_CPUFREQ_EARLY_DET 0x10 +#define WALT_CPUFREQ_BOOST_UPDATE 0x20 -#define CPUFREQ_REASON_PL (1U << 1) -#define CPUFREQ_REASON_EARLY_DET (1U << 2) -#define CPUFREQ_REASON_RTG_BOOST (1U << 3) -#define CPUFREQ_REASON_HISPEED (1U << 4) -#define CPUFREQ_REASON_NWD (1U << 5) -#define CPUFREQ_REASON_FREQ_AGR (1U << 6) -#define CPUFREQ_REASON_KSOFTIRQD (1U << 7) -#define CPUFREQ_REASON_TT_LOAD (1U << 8) -#define CPUFREQ_REASON_SUH (1U << 9) -#define CPUFREQ_REASON_ADAPTIVE_LOW (1U << 10) -#define CPUFREQ_REASON_ADAPTIVE_HIGH (1U << 11) +#define CPUFREQ_REASON_LOAD 0 +#define CPUFREQ_REASON_PL 0x2 +#define CPUFREQ_REASON_EARLY_DET 0x4 +#define CPUFREQ_REASON_RTG_BOOST 0x8 +#define CPUFREQ_REASON_HISPEED 0x10 +#define CPUFREQ_REASON_NWD 0x20 +#define CPUFREQ_REASON_FREQ_AGR 0x40 +#define CPUFREQ_REASON_KSOFTIRQD 0x80 +#define CPUFREQ_REASON_TT_LOAD 0x100 +#define CPUFREQ_REASON_SUH 0x200 +#define CPUFREQ_REASON_ADAPTIVE_LOW 0x400 +#define CPUFREQ_REASON_ADAPTIVE_HIGH 0x800 #define NO_BOOST 0 #define FULL_THROTTLE_BOOST 1 From 89feaab758aaeb0c279008d1213a4f743ccac9f1 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Mon, 23 May 2022 16:31:44 -0700 Subject: [PATCH 327/346] sched/walt: Prevent within cluster load balance of misfits A misfit task can be pulled to a CPU within the same cluster in the newly-idle balance path. Since that just increases migration count and does not show any significant benefits, prevent pulling of a task if it is going to be a misfit on the destination CPU. Only pull if there are many such tasks and the source CPU needs help from a newly idle CPU. Change-Id: Ief55e3916d5902e473c0eaa101e8aefa01c04398 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt_lb.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 500ec59807c4..72b385c3b4e7 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -236,7 +236,7 @@ static void walt_lb_check_for_rotation(struct rq *src_rq) } static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, - bool to_lower, bool force) + bool to_lower, bool to_higher, bool force) { struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; @@ -245,7 +245,6 @@ static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, if (wrq->push_task == p) return false; - if (to_lower) { if (wts->iowaited) return false; @@ -258,6 +257,10 @@ static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, return false; if (!force && !task_fits_max(p, dst_cpu)) return false; + } else if (!to_higher) { + if (!task_fits_max(p, dst_cpu) && + wrq->walt_stats.nr_big_tasks < 2) + return false; } /* Don't detach task if dest cpu is halted */ @@ -292,7 +295,7 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) struct rq *src_rq = cpu_rq(src_cpu); unsigned long flags; struct task_struct *pulled_task = NULL, *p; - bool active_balance = false, to_lower; + bool active_balance = false, to_lower, to_higher; struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; struct walt_task_struct *wts; @@ -301,6 +304,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) to_lower = dst_wrq->cluster->id < src_wrq->cluster->id; + to_higher = dst_wrq->cluster->id > src_wrq->cluster->id; + raw_spin_lock_irqsave(&src_rq->__lock, flags); list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { @@ -311,7 +316,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) if (task_running(src_rq, p)) continue; - if (!_walt_can_migrate_task(p, dst_cpu, to_lower, false)) + if (!_walt_can_migrate_task(p, dst_cpu, to_lower, to_higher, + false)) continue; walt_detach_task(p, src_rq, dst_rq); @@ -327,7 +333,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) if (task_running(src_rq, p)) continue; - if (!_walt_can_migrate_task(p, dst_cpu, to_lower, true)) + if (!_walt_can_migrate_task(p, dst_cpu, to_lower, to_higher, + true)) continue; walt_detach_task(p, src_rq, dst_rq); @@ -1046,15 +1053,17 @@ static void walt_nohz_balancer_kick(void *unused, struct rq *rq, static void walt_can_migrate_task(void *unused, struct task_struct *p, int dst_cpu, int *can_migrate) { - bool to_lower; + bool to_lower, to_higher; struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; struct walt_rq *task_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1; if (unlikely(walt_disabled)) return; to_lower = dst_wrq->cluster->id < task_wrq->cluster->id; + to_higher = dst_wrq->cluster->id > task_wrq->cluster->id; - if (_walt_can_migrate_task(p, dst_cpu, to_lower, true)) + if (_walt_can_migrate_task(p, dst_cpu, to_lower, + to_higher, true)) return; *can_migrate = 0; From 99a0b1fcbc28d0da545d3cc6863c986d6719c38a Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Tue, 3 May 2022 14:53:01 -0700 Subject: [PATCH 328/346] sched/walt: properly account irq entry/exit Due to restructuring of upstream logic, it is necessary to properly deduce start and end times of irqs to maintain correct accounting. Change-Id: I8f0ff9b0f9137283315e5b67cdf2cfb1dede033f Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 74 ++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index e8c7ecf82dc2..456db6f34d0f 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -586,32 +586,8 @@ static bool is_ed_task_present(struct rq *rq, u64 wallclock, struct task_struct return false; } -static void walt_sched_account_irqstart(int cpu, struct task_struct *curr) -{ - struct rq *rq = cpu_rq(cpu); - struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; - - if (!wrq->window_start) - return; - - /* We're here without rq->lock held, IRQ disabled */ - raw_spin_lock(&rq->__lock); - update_task_cpu_cycles(curr, cpu, walt_sched_clock()); - raw_spin_unlock(&rq->__lock); -} - static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, u64 wallclock, u64 irqtime); -static void walt_sched_account_irqend(int cpu, struct task_struct *curr, u64 delta) -{ - struct rq *rq = cpu_rq(cpu); - unsigned long flags; - - raw_spin_lock_irqsave(&rq->__lock, flags); - walt_update_task_ravg(curr, rq, IRQ_UPDATE, walt_sched_clock(), delta); - raw_spin_unlock_irqrestore(&rq->__lock, flags); -} - /* * Return total number of tasks "eligible" to run on higher capacity cpus */ @@ -4099,18 +4075,49 @@ static void android_rvh_new_task_stats(void *unused, struct task_struct *p) mark_task_starting(p); } -static void android_rvh_account_irq(void *unused, struct task_struct *curr, int cpu, s64 delta) +static void android_rvh_account_irq_start(void *unused, struct task_struct *curr, int cpu, + s64 delta) { - struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct rq *rq; + struct walt_rq *wrq; if (unlikely(walt_disabled)) return; - if (!!(curr->flags & PF_IDLE)) { - if (hardirq_count() || in_serving_softirq()) - walt_sched_account_irqend(cpu, curr, delta); - else - walt_sched_account_irqstart(cpu, curr); - } + + if (!is_idle_task(curr)) + return; + + rq = cpu_rq(cpu); + wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (!wrq->window_start) + return; + + /* We're here without rq->lock held, IRQ disabled */ + raw_spin_lock(&rq->__lock); + update_task_cpu_cycles(curr, cpu, walt_sched_clock()); + raw_spin_unlock(&rq->__lock); +} + +static void android_rvh_account_irq_end(void *unused, struct task_struct *curr, int cpu, s64 delta) +{ + struct rq *rq; + unsigned long flags; + struct walt_rq *wrq; + + if (unlikely(walt_disabled)) + return; + + if (!is_idle_task(curr)) + return; + + rq = cpu_rq(cpu); + wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + raw_spin_lock_irqsave(&rq->__lock, flags); + walt_update_task_ravg(curr, rq, IRQ_UPDATE, walt_sched_clock(), delta); + raw_spin_unlock_irqrestore(&rq->__lock, flags); + wrq->last_irq_window = wrq->window_start; } @@ -4446,7 +4453,8 @@ static void register_walt_hooks(void) register_trace_android_rvh_sched_cpu_dying(android_rvh_sched_cpu_dying, NULL); register_trace_android_rvh_set_task_cpu(android_rvh_set_task_cpu, NULL); register_trace_android_rvh_new_task_stats(android_rvh_new_task_stats, NULL); - register_trace_android_rvh_account_irq(android_rvh_account_irq, NULL); + register_trace_android_rvh_account_irq_start(android_rvh_account_irq_start, NULL); + register_trace_android_rvh_account_irq_end(android_rvh_account_irq_end, NULL); register_trace_android_rvh_flush_task(android_rvh_flush_task, NULL); register_trace_android_rvh_update_misfit_status(android_rvh_update_misfit_status, NULL); register_trace_android_rvh_after_enqueue_task(android_rvh_enqueue_task, NULL); From 401453342220f3f47414709cbcbbf87b269b68c6 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 27 May 2022 18:51:52 -0700 Subject: [PATCH 329/346] sched/walt/halt: track halt reasons Currently, there are no protections under conditions where CPUs may be halted or unhalted twice, which could lead to undesired behavior. Track reasons for refcounts per CPU to ensure proper accounting. Change-Id: Ic85a13f3536e3287fc76a02004d2b4601ada7a1a Signed-off-by: Shaleen Agrawal --- drivers/soc/qcom/hyp_core_ctl.c | 4 +-- drivers/thermal/qcom/thermal_pause.c | 4 +-- include/linux/sched/walt.h | 19 ++++++++----- kernel/sched/walt/core_ctl.c | 4 +-- kernel/sched/walt/walt_halt.c | 40 +++++++++++++--------------- 5 files changed, 38 insertions(+), 33 deletions(-) diff --git a/drivers/soc/qcom/hyp_core_ctl.c b/drivers/soc/qcom/hyp_core_ctl.c index c5a00da88e9a..2e490c09b561 100644 --- a/drivers/soc/qcom/hyp_core_ctl.c +++ b/drivers/soc/qcom/hyp_core_ctl.c @@ -109,7 +109,7 @@ static inline int pause_cpu(int cpu) cpumask_clear(&cpus_to_pause); cpumask_set_cpu(cpu, &cpus_to_pause); - ret = walt_pause_cpus(&cpus_to_pause); + ret = walt_pause_cpus(&cpus_to_pause, PAUSE_HYP); return ret; } @@ -122,7 +122,7 @@ static inline int resume_cpu(int cpu) cpumask_clear(&cpus_to_resume); cpumask_set_cpu(cpu, &cpus_to_resume); - ret = walt_resume_cpus(&cpus_to_resume); + ret = walt_resume_cpus(&cpus_to_resume, PAUSE_HYP); return ret; } diff --git a/drivers/thermal/qcom/thermal_pause.c b/drivers/thermal/qcom/thermal_pause.c index 0c926c220c94..5e367cb59cd4 100644 --- a/drivers/thermal/qcom/thermal_pause.c +++ b/drivers/thermal/qcom/thermal_pause.c @@ -102,7 +102,7 @@ static int thermal_pause_work(struct thermal_pause_cdev *thermal_pause_cdev) pr_debug("Pause:%*pbl\n", cpumask_pr_args(&thermal_pause_cdev->cpu_mask)); mutex_unlock(&cpus_pause_lock); - ret = walt_pause_cpus(&cpus_to_pause); + ret = walt_pause_cpus(&cpus_to_pause, PAUSE_THERMAL); mutex_lock(&cpus_pause_lock); if (ret == 0) { @@ -149,7 +149,7 @@ static int thermal_resume_work(struct thermal_pause_cdev *thermal_pause_cdev) pr_debug("Unpause:%*pbl\n", cpumask_pr_args(&cpus_to_unpause)); mutex_unlock(&cpus_pause_lock); - ret = walt_resume_cpus(&cpus_to_unpause); + ret = walt_resume_cpus(&cpus_to_unpause, PAUSE_THERMAL); mutex_lock(&cpus_pause_lock); if (ret == 0) { diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 9bb6ceada1a7..34f771ace147 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -11,6 +11,12 @@ #include #include +enum pause_reason { + PAUSE_CORE_CTL = 0x01, + PAUSE_THERMAL = 0x02, + PAUSE_HYP = 0x04, +}; + #if IS_ENABLED(CONFIG_SCHED_WALT) #define MAX_CPUS_PER_CLUSTER 6 @@ -170,10 +176,11 @@ struct notifier_block; extern void core_ctl_notifier_register(struct notifier_block *n); extern void core_ctl_notifier_unregister(struct notifier_block *n); extern int core_ctl_set_boost(bool boost); -extern int walt_pause_cpus(struct cpumask *cpus); -extern int walt_resume_cpus(struct cpumask *cpus); -extern int walt_halt_cpus(struct cpumask *cpus); -extern int walt_start_cpus(struct cpumask *cpus); + +extern int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason); +extern int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason); +extern int walt_halt_cpus(struct cpumask *cpus, enum pause_reason reason); +extern int walt_start_cpus(struct cpumask *cpus, enum pause_reason reason); #else static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout) { @@ -211,11 +218,11 @@ static inline void core_ctl_notifier_unregister(struct notifier_block *n) { } -inline int walt_pause_cpus(struct cpumask *cpus) +inline int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason) { return 0; } -inline int walt_resume_cpus(struct cpumask *cpus) +inline int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason) { return 0; } diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index 3bcb59d8f646..cdb123d90b8f 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -1163,7 +1163,7 @@ static void core_ctl_pause_cpus(struct cpumask *cpus_to_pause) cpumask_copy(&saved_cpus, cpus_to_pause); if (cpumask_any(cpus_to_pause) < nr_cpu_ids) { - if (walt_halt_cpus(cpus_to_pause) < 0) + if (walt_halt_cpus(cpus_to_pause, PAUSE_CORE_CTL) < 0) pr_debug("core_ctl pause operation failed cpus=%*pbl paused_by_us=%*pbl\n", cpumask_pr_args(cpus_to_pause), cpumask_pr_args(&cpus_paused_by_us)); @@ -1192,7 +1192,7 @@ static void core_ctl_resume_cpus(struct cpumask *cpus_to_unpause) cpumask_copy(&saved_cpus, cpus_to_unpause); if (cpumask_any(cpus_to_unpause) < nr_cpu_ids) { - if (walt_start_cpus(cpus_to_unpause) < 0) + if (walt_start_cpus(cpus_to_unpause, PAUSE_CORE_CTL) < 0) pr_debug("core_ctl resume operation failed cpus=%*pbl paused_by_us=%*pbl\n", cpumask_pr_args(cpus_to_unpause), cpumask_pr_args(&cpus_paused_by_us)); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 346180098231..f61528ef7cb2 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -20,7 +20,7 @@ static DEFINE_RAW_SPINLOCK(halt_lock); struct halt_cpu_state { u64 last_halt; - int ref_count; + u8 reason; }; static DEFINE_PER_CPU(struct halt_cpu_state, halt_state); @@ -333,20 +333,18 @@ static int start_cpus(struct cpumask *cpus) return 0; } -/* increment/decrement ref count for cpus in yield/halt mask */ -static void update_ref_counts(struct cpumask *cpus, bool halt) +/* update reason for cpus in yield/halt mask */ +static void update_reasons(struct cpumask *cpus, bool halt, enum pause_reason reason) { int cpu; struct halt_cpu_state *halt_cpu_state; for_each_cpu(cpu, cpus) { halt_cpu_state = per_cpu_ptr(&halt_state, cpu); - if (halt) { - halt_cpu_state->ref_count++; - } else { - WARN_ON_ONCE(halt_cpu_state->ref_count == 0); - halt_cpu_state->ref_count--; - } + if (halt) + halt_cpu_state->reason |= reason; + else + halt_cpu_state->reason &= ~reason; } } @@ -358,13 +356,13 @@ static void update_halt_cpus(struct cpumask *cpus) for_each_cpu(cpu, cpus) { halt_cpu_state = per_cpu_ptr(&halt_state, cpu); - if (halt_cpu_state->ref_count) + if (halt_cpu_state->reason) cpumask_clear_cpu(cpu, cpus); } } /* cpus will be modified */ -int walt_halt_cpus(struct cpumask *cpus) +int walt_halt_cpus(struct cpumask *cpus, enum pause_reason reason) { int ret = 0; cpumask_t requested_cpus; @@ -378,7 +376,7 @@ int walt_halt_cpus(struct cpumask *cpus) update_halt_cpus(cpus); if (cpumask_empty(cpus)) { - update_ref_counts(&requested_cpus, true); + update_reasons(&requested_cpus, true, reason); goto unlock; } @@ -388,21 +386,21 @@ int walt_halt_cpus(struct cpumask *cpus) pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); else - update_ref_counts(&requested_cpus, true); + update_reasons(&requested_cpus, true, reason); unlock: raw_spin_unlock_irqrestore(&halt_lock, flags); return ret; } -int walt_pause_cpus(struct cpumask *cpus) +int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason) { - return walt_halt_cpus(cpus); + return walt_halt_cpus(cpus, reason); } EXPORT_SYMBOL(walt_pause_cpus); /* cpus will be modified */ -int walt_start_cpus(struct cpumask *cpus) +int walt_start_cpus(struct cpumask *cpus, enum pause_reason reason) { int ret = 0; cpumask_t requested_cpus; @@ -410,9 +408,9 @@ int walt_start_cpus(struct cpumask *cpus) raw_spin_lock_irqsave(&halt_lock, flags); cpumask_copy(&requested_cpus, cpus); - update_ref_counts(&requested_cpus, false); + update_reasons(&requested_cpus, false, reason); - /* remove cpus that should still be halted, due to ref-counts */ + /* remove cpus that should still be halted */ update_halt_cpus(cpus); ret = start_cpus(cpus); @@ -421,7 +419,7 @@ int walt_start_cpus(struct cpumask *cpus) pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret, cpumask_pr_args(&requested_cpus)); /* restore/increment ref counts in case of error */ - update_ref_counts(&requested_cpus, true); + update_reasons(&requested_cpus, true, reason); } raw_spin_unlock_irqrestore(&halt_lock, flags); @@ -429,9 +427,9 @@ int walt_start_cpus(struct cpumask *cpus) return ret; } -int walt_resume_cpus(struct cpumask *cpus) +int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason) { - return walt_start_cpus(cpus); + return walt_start_cpus(cpus, reason); } EXPORT_SYMBOL(walt_resume_cpus); From 0a354ab6ac5a11935bdef88acd9838bf57d8b7f1 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 23 Mar 2022 13:43:05 -0700 Subject: [PATCH 330/346] sched/walt: Improve the scheduler This change is for general scheduler improvement. Change-Id: I2b95dc15c0dbe88966c90acfa4dca6aebf1a3f95 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.h | 17 ++++++++++------- kernel/sched/walt/walt_cfs.c | 5 +++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 4b6154a8c32d..1663bbf281e1 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -757,6 +757,13 @@ static inline bool task_fits_capacity(struct task_struct *p, return capacity * 1024 > uclamp_task_util(p) * margin; } +static inline bool task_in_related_thread_group(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (rcu_access_pointer(wts->grp) != NULL); +} + static inline bool task_fits_max(struct task_struct *p, int cpu) { unsigned long capacity = capacity_orig_of(cpu); @@ -774,6 +781,9 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) } else { /* mid cap cpu */ if (task_boost > TASK_BOOST_ON_MID) return false; + if (!task_in_related_thread_group(p) && p->prio >= 124) + /* a non topapp low prio task fits on gold */ + return true; } return task_fits_capacity(p, capacity, cpu); @@ -814,13 +824,6 @@ static inline unsigned int task_load(struct task_struct *p) return wts->demand; } -static inline bool task_in_related_thread_group(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return (rcu_access_pointer(wts->grp) != NULL); -} - static inline bool task_rtg_high_prio(struct task_struct *p) { return task_in_related_thread_group(p) && diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index b6378331fd61..da6489938e69 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -80,6 +80,11 @@ static inline bool task_demand_fits(struct task_struct *p, int cpu) if (is_max_cluster_cpu(cpu)) return true; + if (!task_in_related_thread_group(p) && p->prio >= 124 && + !is_min_cluster_cpu(cpu) && !is_max_cluster_cpu(cpu)) { + /* a non topapp low prio task fits on gold */ + return true; + } return task_fits_capacity(p, capacity, cpu); } From 421ed6261aaf30ce3e0aedc229ba8b45077c6909 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Thu, 26 May 2022 15:29:42 -0700 Subject: [PATCH 331/346] sched/walt: reject queue_wakelist() is cpu is same as waker cpu The upstream code has a WARN when wakelist is being enforced on the same cpu where the waker runs. Although WALT tries its best to skip the current cpu during many_wakeups situation, it may have to put the task on it during highload/offline or 32 bit situations. Simply disable queue_wakelist() if we wakeup the task on the waker cpu. Change-Id: Ie13f1156bf0389570dbe0fbac70e271db2b9b211 Signed-off-by: Abhijeet Dharmapurikar Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 456db6f34d0f..8efa849ecb15 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4411,11 +4411,12 @@ static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) __sched_fork_init(p); } -static void android_rvh_ttwu_cond(void *unused, bool *cond) +static void android_rvh_ttwu_cond(void *unused, int cpu, bool *cond) { if (unlikely(walt_disabled)) return; - *cond = sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT; + *cond = (sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT) && + (cpu != smp_processor_id()); } static void android_rvh_sched_exec(void *unused, bool *cond) From 2fc529c88a31e6fdcc3aa4570770f928a3f8cf6f Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 22 Apr 2022 12:53:52 -0700 Subject: [PATCH 332/346] sched/walt: handle misaligned task window start Introduce awareness for task window start and identify if task window_start doesn't match that of the CPUs just before accounting. Change-Id: I58d806e2a75bc647dc28b1f95d00352d29c5c447 Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 1 + kernel/sched/walt/walt.c | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 34f771ace147..1db7e4df8f74 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -100,6 +100,7 @@ struct walt_task_struct { * 0 = nothing, 1 = enqueued, 2 = dequeued */ u64 mark_start; + u64 window_start; u32 sum, demand; u32 coloc_demand; u32 sum_history[RAVG_HIST_SIZE]; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 8efa849ecb15..ebace5ca44af 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -1121,6 +1121,17 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) old_window_start = update_window_start(dest_rq, wallclock, TASK_UPDATE); run_walt_irq_work_rollover(old_window_start, dest_rq); + if (wts->window_start != src_wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s task %s(%d)'s ws=%llu not equal to src_rq %d's ws=%llu", + __func__, raw_smp_processor_id(), p->comm, p->pid, + wts->window_start, src_rq->cpu, src_wrq->window_start); + if (wts->window_start != dest_wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s task %s(%d)'s ws=%llu not equal to dest_rq %d's ws=%llu", + __func__, raw_smp_processor_id(), p->comm, p->pid, + wts->window_start, dest_rq->cpu, dest_wrq->window_start); + update_task_cpu_cycles(p, new_cpu, wallclock); new_task = is_new_task(p); @@ -1715,9 +1726,10 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, * Handle per-task window rollover. We don't care about the * idle task. */ - if (!is_idle_task(p)) { - if (new_window) + if (new_window) { + if (!is_idle_task(p)) rollover_task_window(p, full_window); + wts->window_start = window_start; } new_task = is_new_task(p); @@ -1736,6 +1748,12 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; } + if (wts->window_start != wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s task %s(%d)'s ws=%llu not equal to rq %d's ws=%llu", + __func__, raw_smp_processor_id(), p->comm, p->pid, + wts->window_start, rq->cpu, wrq->window_start); + if (!new_window) { /* * account_busy_for_cpu_time() = 1 so busy time needs @@ -2302,6 +2320,9 @@ static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int even old_window_start = update_window_start(rq, wallclock, event); + if (!wts->window_start) + wts->window_start = wrq->window_start; + if (!wts->mark_start) { update_task_cpu_cycles(p, cpu_of(rq), wallclock); goto done; @@ -2355,6 +2376,7 @@ static void init_new_task_load(struct task_struct *p) INIT_LIST_HEAD(&wts->grp_list); wts->mark_start = 0; + wts->window_start = 0; wts->sum = 0; wts->curr_window = 0; wts->prev_window = 0; @@ -3332,6 +3354,13 @@ static void transfer_busy_time(struct rq *rq, wallclock = walt_sched_clock(); walt_update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); + + if (wts->window_start != wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s event=%d task %s(%d)'s ws=%llu not equal to rq %d's ws=%llu", + __func__, raw_smp_processor_id(), event, p->comm, p->pid, + wts->window_start, rq->cpu, wrq->window_start); + new_task = is_new_task(p); cpu_time = &wrq->grp_time; From c42b241161b0ffafe09479d353bfad1c7346b080 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Thu, 21 Apr 2022 11:47:25 -0700 Subject: [PATCH 333/346] sched: walt: Drop double rq locks for migrations Currently, WALT holds rq lock for both the cpus when handling task migration. Upstream code however (PELT) holds rq locks consecutively for migrations. This results in cumbersome tricks in WALT to hold them at the same time - rq locks need to be acquired in order of cpu numbers. Besides that, holding two rq locks for every migration results in contentions in busy situations. Split the migration accounting in two steps - subtracting time from src cpu and adding time to the dest cpu. Subtract when we know the task is moving i.e. the migration hook and add on the dest cpu during subsequent enqueue. Upstream code guarantees the rq lock for src cpu is held during migration hook for queued or running task. For a waking task grab the src cpu lock. No need to grab rq lock for dest cpu as enqueue hook is always run with that lock held. The basic idea of walt migration accounting, non group tasks move their contributions only for inter cluster migration while group tasks do that for every migration, stays. Change-Id: I5d319e26bcbc088448954fd83bbcd1981ee7fb5c Signed-off-by: Shaleen Agrawal Signed-off-by: Sai Harshini Nimmala --- include/linux/sched/walt.h | 2 + kernel/sched/walt/walt.c | 265 +++++++++++++++++++++--------------- kernel/sched/walt/walt_lb.c | 29 +--- 3 files changed, 162 insertions(+), 134 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 1db7e4df8f74..12c93451f85a 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -141,6 +141,8 @@ struct walt_task_struct { int cidx; int load_boost; int64_t boosted_task_load; + int prev_cpu; + u8 enqueue_after_migration; }; #define wts_to_ts(wts) ({ \ diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index ebace5ca44af..f0ae0e603513 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -918,28 +918,17 @@ static void update_cluster_load_subtractions(struct task_struct *p, raw_spin_unlock(&cluster->load_lock); } -static inline void inter_cluster_migration_fixup - (struct task_struct *p, int new_cpu, int task_cpu, bool new_task) +static inline void migrate_inter_cluster_subtraction(struct task_struct *p, int task_cpu, + bool new_task) { - struct rq *dest_rq = cpu_rq(new_cpu); struct rq *src_rq = cpu_rq(task_cpu); - struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - if (same_freq_domain(new_cpu, task_cpu)) - return; - - wts->curr_window_cpu[new_cpu] = wts->curr_window; - wts->prev_window_cpu[new_cpu] = wts->prev_window; - - dest_wrq->curr_runnable_sum += wts->curr_window; - dest_wrq->prev_runnable_sum += wts->prev_window; - if (src_wrq->curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { WALT_BUG(WALT_BUG_WALT, p, - "pid=%u CPU%d -> CPU%d src_crs=%llu is lesser than task_contrib=%llu", - p->pid, src_rq->cpu, dest_rq->cpu, + "pid=%u CPU%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, src_wrq->curr_runnable_sum, wts->curr_window_cpu[task_cpu]); src_wrq->curr_runnable_sum = wts->curr_window_cpu[task_cpu]; @@ -948,8 +937,8 @@ static inline void inter_cluster_migration_fixup if (src_wrq->prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { WALT_BUG(WALT_BUG_WALT, p, - "pid=%u CPU%d -> CPU%d src_prs=%llu is lesser than task_contrib=%llu", - p->pid, src_rq->cpu, dest_rq->cpu, + "pid=%u CPU%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, src_wrq->prev_runnable_sum, wts->prev_window_cpu[task_cpu]); src_wrq->prev_runnable_sum = wts->prev_window_cpu[task_cpu]; @@ -957,13 +946,10 @@ static inline void inter_cluster_migration_fixup src_wrq->prev_runnable_sum -= wts->prev_window_cpu[task_cpu]; if (new_task) { - dest_wrq->nt_curr_runnable_sum += wts->curr_window; - dest_wrq->nt_prev_runnable_sum += wts->prev_window; - if (src_wrq->nt_curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { WALT_BUG(WALT_BUG_WALT, p, - "pid=%u CPU%d -> CPU%d src_nt_crs=%llu is lesser than task_contrib=%llu", - p->pid, src_rq->cpu, dest_rq->cpu, + "pid=%u CPU%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, src_wrq->nt_curr_runnable_sum, wts->curr_window_cpu[task_cpu]); src_wrq->nt_curr_runnable_sum = wts->curr_window_cpu[task_cpu]; @@ -973,8 +959,8 @@ static inline void inter_cluster_migration_fixup if (src_wrq->nt_prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { WALT_BUG(WALT_BUG_WALT, p, - "pid=%u CPU%d -> CPU%d src_nt_prs=%llu is lesser than task_contrib=%llu", - p->pid, src_rq->cpu, dest_rq->cpu, + "pid=%u CPU%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, src_wrq->nt_prev_runnable_sum, wts->prev_window_cpu[task_cpu]); src_wrq->nt_prev_runnable_sum = wts->prev_window_cpu[task_cpu]; @@ -990,6 +976,26 @@ static inline void inter_cluster_migration_fixup src_wrq->window_start, new_task); } +static inline void migrate_inter_cluster_addition(struct task_struct *p, int new_cpu, + bool new_task) +{ + struct rq *dest_rq = cpu_rq(new_cpu); + struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + + wts->curr_window_cpu[new_cpu] = wts->curr_window; + wts->prev_window_cpu[new_cpu] = wts->prev_window; + + dest_wrq->curr_runnable_sum += wts->curr_window; + dest_wrq->prev_runnable_sum += wts->prev_window; + + if (new_task) { + dest_wrq->nt_curr_runnable_sum += wts->curr_window; + dest_wrq->nt_prev_runnable_sum += wts->prev_window; + } +} + static u32 load_to_index(u32 load) { u32 index = load / sched_load_granule; @@ -997,39 +1003,26 @@ static u32 load_to_index(u32 load) return min(index, (u32)(NUM_LOAD_INDICES - 1)); } -static void -migrate_top_tasks(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) +static void migrate_top_tasks_subtraction(struct task_struct *p, struct rq *src_rq) { int index; int top_index; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; u32 curr_window = wts->curr_window; u32 prev_window = wts->prev_window; - struct walt_rq *dst_wrq = (struct walt_rq *) dst_rq->android_vendor_data1; struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; u8 src = src_wrq->curr_table; - u8 dst = dst_wrq->curr_table; u8 *src_table; - u8 *dst_table; if (curr_window) { src_table = src_wrq->top_tasks[src]; - dst_table = dst_wrq->top_tasks[dst]; index = load_to_index(curr_window); src_table[index] -= 1; - dst_table[index] += 1; if (!src_table[index]) __clear_bit(NUM_LOAD_INDICES - index - 1, src_wrq->top_tasks_bitmap[src]); - if (dst_table[index] == 1) - __set_bit(NUM_LOAD_INDICES - index - 1, - dst_wrq->top_tasks_bitmap[dst]); - - if (index > dst_wrq->curr_top) - dst_wrq->curr_top = index; - top_index = src_wrq->curr_top; if (index == top_index && !src_table[index]) src_wrq->curr_top = get_top_index( @@ -1038,28 +1031,56 @@ migrate_top_tasks(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) if (prev_window) { src = 1 - src; - dst = 1 - dst; src_table = src_wrq->top_tasks[src]; - dst_table = dst_wrq->top_tasks[dst]; index = load_to_index(prev_window); src_table[index] -= 1; - dst_table[index] += 1; if (!src_table[index]) __clear_bit(NUM_LOAD_INDICES - index - 1, src_wrq->top_tasks_bitmap[src]); + top_index = src_wrq->prev_top; + if (index == top_index && !src_table[index]) + src_wrq->prev_top = get_top_index( + src_wrq->top_tasks_bitmap[src], top_index); + } +} + +static void migrate_top_tasks_addition(struct task_struct *p, struct rq *rq) +{ + int index; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u32 curr_window = wts->curr_window; + u32 prev_window = wts->prev_window; + struct walt_rq *dst_wrq = (struct walt_rq *) rq->android_vendor_data1; + u8 dst = dst_wrq->curr_table; + u8 *dst_table; + + if (curr_window) { + dst_table = dst_wrq->top_tasks[dst]; + index = load_to_index(curr_window); + dst_table[index] += 1; + + if (dst_table[index] == 1) + __set_bit(NUM_LOAD_INDICES - index - 1, + dst_wrq->top_tasks_bitmap[dst]); + + if (index > dst_wrq->curr_top) + dst_wrq->curr_top = index; + } + + if (prev_window) { + dst = 1 - dst; + dst_table = dst_wrq->top_tasks[dst]; + index = load_to_index(prev_window); + dst_table[index] += 1; + if (dst_table[index] == 1) __set_bit(NUM_LOAD_INDICES - index - 1, dst_wrq->top_tasks_bitmap[dst]); if (index > dst_wrq->prev_top) dst_wrq->prev_top = index; - - top_index = src_wrq->prev_top; - if (index == top_index && !src_table[index]) - src_wrq->prev_top = get_top_index( - src_wrq->top_tasks_bitmap[src], top_index); } } @@ -1071,22 +1092,17 @@ static inline bool is_new_task(struct task_struct *p) } static inline void run_walt_irq_work_rollover(u64 old_window_start, struct rq *rq); -static void fixup_busy_time(struct task_struct *p, int new_cpu) +static void migrate_busy_time_subtraction(struct task_struct *p, int new_cpu) { struct rq *src_rq = task_rq(p); - struct rq *dest_rq = cpu_rq(new_cpu); u64 wallclock; - u64 *src_curr_runnable_sum, *dst_curr_runnable_sum; - u64 *src_prev_runnable_sum, *dst_prev_runnable_sum; - u64 *src_nt_curr_runnable_sum, *dst_nt_curr_runnable_sum; - u64 *src_nt_prev_runnable_sum, *dst_nt_prev_runnable_sum; + u64 *src_curr_runnable_sum, *src_prev_runnable_sum; + u64 *src_nt_curr_runnable_sum, *src_nt_prev_runnable_sum; bool new_task; struct walt_related_thread_group *grp; long pstate; - struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - u64 old_window_start; if (!p->on_rq && READ_ONCE(p->__state) != TASK_WAKING) return; @@ -1094,44 +1110,30 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) pstate = READ_ONCE(p->__state); if (pstate == TASK_WAKING) - double_rq_lock(src_rq, dest_rq); - - wallclock = walt_sched_clock(); + raw_spin_rq_lock(src_rq); lockdep_assert_held(&src_rq->__lock); - lockdep_assert_held(&dest_rq->__lock); if (task_rq(p) != src_rq) WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on src_rq %d", raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); + if (!same_freq_domain(task_cpu(p), new_cpu)) + wts->enqueue_after_migration = 2; /* 2 is intercluster */ + else + wts->enqueue_after_migration = 1; /* 1 is within cluster */ + + wallclock = walt_sched_clock(); walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, wallclock, 0); - /* - * The above update might have rolled over the - * window for this migrating task. Since we are - * going to adjust the destination CPU's busy time - * counters with the task busytime counters, roll over - * the window for the destination CPU also. - * - * The update_window_start() does nothing if the window - * is not rolled over, so there is no need to check for - * window boundary or if the counters will be accessed - * or not. - */ - old_window_start = update_window_start(dest_rq, wallclock, TASK_UPDATE); - run_walt_irq_work_rollover(old_window_start, dest_rq); if (wts->window_start != src_wrq->window_start) WALT_BUG(WALT_BUG_WALT, p, "CPU%d: %s task %s(%d)'s ws=%llu not equal to src_rq %d's ws=%llu", __func__, raw_smp_processor_id(), p->comm, p->pid, wts->window_start, src_rq->cpu, src_wrq->window_start); - if (wts->window_start != dest_wrq->window_start) - WALT_BUG(WALT_BUG_WALT, p, - "CPU%d: %s task %s(%d)'s ws=%llu not equal to dest_rq %d's ws=%llu", - __func__, raw_smp_processor_id(), p->comm, p->pid, - wts->window_start, dest_rq->cpu, dest_wrq->window_start); + + /* safe to update the task cyc cntr for new_cpu without the new_cpu rq_lock */ update_task_cpu_cycles(p, new_cpu, wallclock); new_task = is_new_task(p); @@ -1144,61 +1146,106 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) * load has to reported on a single CPU regardless. */ if (grp) { - struct group_cpu_time *cpu_time; + struct group_cpu_time *cpu_time = &src_wrq->grp_time; - cpu_time = &src_wrq->grp_time; src_curr_runnable_sum = &cpu_time->curr_runnable_sum; src_prev_runnable_sum = &cpu_time->prev_runnable_sum; src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; - cpu_time = &dest_wrq->grp_time; + if (wts->curr_window) { + *src_curr_runnable_sum -= wts->curr_window; + if (new_task) + *src_nt_curr_runnable_sum -= wts->curr_window; + } + + if (wts->prev_window) { + *src_prev_runnable_sum -= wts->prev_window; + if (new_task) + *src_nt_prev_runnable_sum -= wts->prev_window; + } + } else { + if (wts->enqueue_after_migration == 2) + migrate_inter_cluster_subtraction(p, task_cpu(p), new_task); + } + + migrate_top_tasks_subtraction(p, src_rq); + + if (is_ed_enabled() && (p == src_wrq->ed_task)) + src_wrq->ed_task = NULL; + + wts->prev_cpu = task_cpu(p); + + if (pstate == TASK_WAKING) + raw_spin_rq_unlock(src_rq); +} + +static void migrate_busy_time_addition(struct task_struct *p, int new_cpu, u64 wallclock) +{ + struct rq *dest_rq = cpu_rq(new_cpu); + u64 *dst_curr_runnable_sum, *dst_prev_runnable_sum; + u64 *dst_nt_curr_runnable_sum, *dst_nt_prev_runnable_sum; + bool new_task; + struct walt_related_thread_group *grp; + struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + int src_cpu = wts->prev_cpu; + struct rq *src_rq = cpu_rq(src_cpu); + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + + walt_update_task_ravg(p, dest_rq, TASK_UPDATE, wallclock, 0); + + if (wts->window_start != dest_wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s task %s(%d)'s ws=%llu not equal to dest_rq %d's ws=%llu", + __func__, raw_smp_processor_id(), p->comm, p->pid, + wts->window_start, dest_rq->cpu, dest_wrq->window_start); + + new_task = is_new_task(p); + /* Protected by rq_lock */ + grp = wts->grp; + + /* + * For frequency aggregation, we continue to do migration fixups + * even for intra cluster migrations. This is because, the aggregated + * load has to reported on a single CPU regardless. + */ + if (grp) { + struct group_cpu_time *cpu_time = &dest_wrq->grp_time; + dst_curr_runnable_sum = &cpu_time->curr_runnable_sum; dst_prev_runnable_sum = &cpu_time->prev_runnable_sum; dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; if (wts->curr_window) { - *src_curr_runnable_sum -= wts->curr_window; *dst_curr_runnable_sum += wts->curr_window; - if (new_task) { - *src_nt_curr_runnable_sum -= wts->curr_window; + if (new_task) *dst_nt_curr_runnable_sum += wts->curr_window; - } } if (wts->prev_window) { - *src_prev_runnable_sum -= wts->prev_window; *dst_prev_runnable_sum += wts->prev_window; - if (new_task) { - *src_nt_prev_runnable_sum -= wts->prev_window; + if (new_task) *dst_nt_prev_runnable_sum += wts->prev_window; - } } } else { - inter_cluster_migration_fixup(p, new_cpu, - task_cpu(p), new_task); + if (wts->enqueue_after_migration == 2) + migrate_inter_cluster_addition(p, new_cpu, new_task); } - migrate_top_tasks(p, src_rq, dest_rq); + migrate_top_tasks_addition(p, dest_rq); - if (!same_freq_domain(new_cpu, task_cpu(p))) { + if (wts->enqueue_after_migration == 2) { src_wrq->notif_pending = true; dest_wrq->notif_pending = true; walt_irq_work_queue(&walt_migration_irq_work); } - if (is_ed_enabled()) { - if (p == src_wrq->ed_task) { - src_wrq->ed_task = NULL; - dest_wrq->ed_task = p; - } else if (is_ed_task(p, wallclock)) { - dest_wrq->ed_task = p; - } - } + if (is_ed_enabled() && is_ed_task(p, wallclock)) + dest_wrq->ed_task = p; - if (pstate == TASK_WAKING) - double_rq_unlock(src_rq, dest_rq); + wts->enqueue_after_migration = 0; } #define INC_STEP 8 @@ -2375,6 +2422,8 @@ static void init_new_task_load(struct task_struct *p) rcu_assign_pointer(wts->grp, NULL); INIT_LIST_HEAD(&wts->grp_list); + wts->prev_cpu = raw_smp_processor_id(); + wts->enqueue_after_migration = 0; wts->mark_start = 0; wts->window_start = 0; wts->sum = 0; @@ -4083,7 +4132,8 @@ static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsign { if (unlikely(walt_disabled)) return; - fixup_busy_time(p, (int) new_cpu); + + migrate_busy_time_subtraction(p, (int) new_cpu); if (!cpumask_test_cpu(new_cpu, p->cpus_ptr)) WALT_BUG(WALT_BUG_WALT, p, "selecting unaffined cpu=%d comm=%s(%d) affinity=0x%x", @@ -4168,8 +4218,6 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st lockdep_assert_held(&rq->__lock); - wallclock = walt_rq_clock(rq); - if (p->cpu != cpu_of(rq)) WALT_BUG(WALT_BUG_UPSTREAM, p, "enqueuing on rq %d when task->cpu is %d\n", cpu_of(rq), p->cpu); @@ -4181,12 +4229,17 @@ static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_st double_enqueue = true; } - if (cpu_halted(cpu_of(rq)) && !(p->flags & PF_KTHREAD) && !walt_halt_check_last(cpu_of(rq))) WALT_BUG(WALT_BUG_NONCRITICAL, p, "Non Kthread Started on halted cpu_of(rq)=%d comm=%s(%d) affinity=0x%x\n", cpu_of(rq), p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); + wallclock = walt_rq_clock(rq); + if (wts->enqueue_after_migration != 0) { + wallclock = walt_sched_clock(); + migrate_busy_time_addition(p, cpu_of(rq), wallclock); + } + wts->prev_on_rq = 1; wts->prev_on_rq_cpu = cpu_of(rq); diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 72b385c3b4e7..4d03bee065f5 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -19,12 +19,9 @@ static inline unsigned long walt_lb_cpu_util(int cpu) static void walt_detach_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) { + //TODO can we just replace with detach_task in fair.c?? deactivate_task(src_rq, p, 0); - double_lock_balance(src_rq, dst_rq); - if (!(src_rq->clock_update_flags & RQCF_UPDATED)) - update_rq_clock(src_rq); set_task_cpu(p, dst_rq->cpu); - double_unlock_balance(src_rq, dst_rq); } static void walt_attach_task(struct task_struct *p, struct rq *rq) @@ -1002,29 +999,6 @@ static void walt_find_busiest_queue(void *unused, int dst_cpu, trace_walt_find_busiest_queue(dst_cpu, busiest_cpu, src_mask.bits[0]); } -static void walt_migrate_queued_task(void *unused, struct rq *rq, - struct rq_flags *rf, - struct task_struct *p, - int new_cpu, int *detached) -{ - if (unlikely(walt_disabled)) - return; - /* - * WALT expects both source and destination rqs to be - * held when set_task_cpu() is called on a queued task. - * so implementing this detach hook. unpin the lock - * before detaching and repin it later to make lockdep - * happy. - */ - BUG_ON(!rf); - - rq_unpin_lock(rq, rf); - walt_detach_task(p, rq, cpu_rq(new_cpu)); - rq_repin_lock(rq, rf); - - *detached = 1; -} - /* * we only decide if nohz balance kick is needed or not. the * first CPU in the nohz.idle will come out of idle and do @@ -1075,7 +1049,6 @@ void walt_lb_init(void) walt_lb_rotate_work_init(); - register_trace_android_rvh_migrate_queued_task(walt_migrate_queued_task, NULL); register_trace_android_rvh_sched_nohz_balancer_kick(walt_nohz_balancer_kick, NULL); register_trace_android_rvh_can_migrate_task(walt_can_migrate_task, NULL); register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); From f8314d13b2ea74194b0cf6364f137f9a26ff33a2 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Mon, 6 Jun 2022 15:33:55 -0700 Subject: [PATCH 334/346] sched/walt: Ensure driving cpu is used for region3 Currently, wg_cpu is used, however, this is incorrect, as the driving CPU should be used when evaluating region3 conditions. Change-Id: I4f86ec6021b718e35d84f589dc9be68a53ab9523 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/cpufreq_walt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 61c839aae70b..650fff45845b 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -247,7 +247,7 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, unsigned int freq, raw_freq, final_freq; struct waltgov_cpu *wg_driv_cpu = &per_cpu(waltgov_cpu, wg_policy->driving_cpu); - raw_freq = walt_map_util_freq(util, wg_policy, max, wg_cpu->cpu); + raw_freq = walt_map_util_freq(util, wg_policy, max, wg_driv_cpu->cpu); freq = raw_freq; if (wg_policy->tunables->adaptive_high_freq) { From 7f927567ff3c642c569c7be52dc82258d9552542 Mon Sep 17 00:00:00 2001 From: Ashay Jaiswal Date: Thu, 7 Apr 2022 14:58:38 +0530 Subject: [PATCH 335/346] sched: walt: add nested spinlock api Use nested spin lock api to avoid deadlock warning Change-Id: Ia2aed8f025bb209876ff87ba5a9e32b2b80718d8 Signed-off-by: Ashay Jaiswal Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index f0ae0e603513..f277d26a92f2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4567,10 +4567,15 @@ static int walt_init_stop_handler(void *data) int cpu; struct task_struct *g, *p; struct walt_rq *wrq; + int level = 0; read_lock(&tasklist_lock); for_each_possible_cpu(cpu) { - raw_spin_lock(&cpu_rq(cpu)->__lock); + if (level == 0) + raw_spin_lock(&cpu_rq(cpu)->__lock); + else + raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); + level++; } do_each_thread(g, p) { From 492decfdc48c120b51dd5e7616ca5db81a489f25 Mon Sep 17 00:00:00 2001 From: Stephen Dickey Date: Wed, 13 Apr 2022 09:39:26 -0700 Subject: [PATCH 336/346] sched: walt: handle CTS failure for sched_set/getaffinity During CTS testing, the user-space code can get an affinity that includes a halted cpu, and immediately use that to set the affinity. Currently sched_setaffinity will cause an error if a halted cpu is selected. Port the change that was in use prior to the introduction of pause, correcting the CTS test case. Change-Id: I9bfc819e4a6904494c199931e76f2702b7990758 Signed-off-by: Stephen Dickey --- kernel/sched/walt/walt.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index f277d26a92f2..da36b06cabae 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4464,6 +4464,16 @@ static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, *ret = set_cpus_allowed_ptr(p, &wts->cpus_requested); } +static void android_rvh_sched_getaffinity(void *unused, struct task_struct *p, + struct cpumask *in_mask) +{ + if (unlikely(walt_disabled)) + return; + + if (!(p->flags & PF_KTHREAD)) + cpumask_andnot(in_mask, in_mask, cpu_halt_mask); +} + static void android_rvh_sched_setaffinity(void *unused, struct task_struct *p, const struct cpumask *in_mask, int *retval) @@ -4551,6 +4561,7 @@ static void register_walt_hooks(void) register_trace_android_rvh_cpu_cgroup_online(android_rvh_cpu_cgroup_online, NULL); register_trace_android_rvh_update_cpus_allowed(android_rvh_update_cpus_allowed, NULL); register_trace_android_rvh_sched_setaffinity(android_rvh_sched_setaffinity, NULL); + register_trace_android_rvh_sched_getaffinity(android_rvh_sched_getaffinity, NULL); register_trace_android_rvh_sched_fork_init(android_rvh_sched_fork_init, NULL); register_trace_android_rvh_ttwu_cond(android_rvh_ttwu_cond, NULL); register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); From 181cb9d5608bad99123aaf057485b34b5bf51b50 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 18 May 2022 12:58:13 -0700 Subject: [PATCH 337/346] Revert "sched: Use bitshift over division" This reverts commit bec4f0b0ebe82f2c99c519b424794c17ae2ff8b7. Change-Id: Ib7f02388dfcb5b263a636e261c4e43ee5f4f93b6 Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 6 ------ kernel/sched/walt/walt.c | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index 12c93451f85a..bb636d930aad 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -38,13 +38,7 @@ enum task_boost_type { }; #define WALT_NR_CPUS 8 -/* - * RAVG_HIST_SHIFT trick can only be used if RAVG_HIST_SIZE is a power of 2. - */ #define RAVG_HIST_SIZE 8 -#define RAVG_HIST_SHIFT 3 -#define RAVG_HIST_MASK (RAVG_HIST_SIZE - 1) - /* wts->bucket_bitmask needs to be updated if NUM_BUSY_BUCKETS > 16 */ #define NUM_BUSY_BUCKETS 16 #define NUM_BUSY_BUCKETS_SHIFT 4 diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index da36b06cabae..ab43e0820bc2 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -2071,7 +2071,7 @@ static void update_history(struct rq *rq, struct task_struct *p, for (; samples > 0; samples--) { hist[wts->cidx] = runtime; hist_util[wts->cidx] = runtime_scaled; - wts->cidx = ++(wts->cidx) & RAVG_HIST_MASK; + wts->cidx = ++(wts->cidx) % RAVG_HIST_SIZE; } for (i = 0; i < RAVG_HIST_SIZE; i++) { @@ -2087,7 +2087,7 @@ static void update_history(struct rq *rq, struct task_struct *p, } else if (sysctl_sched_window_stats_policy == WINDOW_STATS_MAX) { demand = max; } else { - avg = sum >> RAVG_HIST_SHIFT; + avg = div64_u64(sum, RAVG_HIST_SIZE); if (sysctl_sched_window_stats_policy == WINDOW_STATS_AVG) demand = avg; else @@ -2116,7 +2116,7 @@ static void update_history(struct rq *rq, struct task_struct *p, wts->demand = demand; wts->demand_scaled = demand_scaled; - wts->coloc_demand = sum >> RAVG_HIST_SHIFT; + wts->coloc_demand = div64_u64(sum, RAVG_HIST_SIZE); wts->pred_demand_scaled = pred_demand_scaled; if (demand_scaled > sysctl_sched_min_task_util_for_colocation) From bd98fea5e6382f122cff9f0724c63fad1b39340f Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 18 May 2022 12:59:59 -0700 Subject: [PATCH 338/346] Revert "sched/walt: print all history samples" This reverts commit feb7fa207560e9b759964df7b6566a8758bb535c. Change-Id: I992beae0327b2e44eade80c4f50f18eda125d16e Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/trace.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index 30d1a1fffb73..d260c63a9609 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -106,19 +106,17 @@ TRACE_EVENT(sched_update_history, __entry->cpu = rq->cpu; ), - TP_printk("%d (%s): runtime %u samples %d event %s demand %u (hist: %u %u %u %u %u %u %u %u) (hist_util: %u %u %u %u %u %u %u %u) coloc_demand %u pred_demand_scaled %u cpu %d nr_big %u", + TP_printk("%d (%s): runtime %u samples %d event %s demand %u (hist: %u %u %u %u %u) (hist_util: %u %u %u %u %u) coloc_demand %u pred_demand_scaled %u cpu %d nr_big %u", __entry->pid, __entry->comm, __entry->runtime, __entry->samples, task_event_names[__entry->evt], __entry->demand, __entry->hist[0], __entry->hist[1], __entry->hist[2], __entry->hist[3], - __entry->hist[4], __entry->hist[5], - __entry->hist[6], __entry->hist[7], + __entry->hist[4], __entry->hist_util[0], __entry->hist_util[1], __entry->hist_util[2], __entry->hist_util[3], - __entry->hist_util[4], __entry->hist_util[5], - __entry->hist_util[6], __entry->hist_util[7], + __entry->hist_util[4], __entry->coloc_demand, __entry->pred_demand_scaled, __entry->cpu, __entry->nr_big_tasks) ); From 626213d61b1b5b7668095afaa1968ccad8e1c197 Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Wed, 15 Jun 2022 18:31:48 -0700 Subject: [PATCH 339/346] treewide: Fixup copyright attribution Clean up missing LF copyright attribution. Change-Id: Ic8c44271696144adc4555b5c7ff6573c166de5c3 Signed-off-by: Elliot Berman Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/fixup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/fixup.c b/kernel/sched/walt/fixup.c index f3b5d7e258cd..c0fc19acb347 100644 --- a/kernel/sched/walt/fixup.c +++ b/kernel/sched/walt/fixup.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. - * Copyright (c) 2016-2021 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. */ #include From b18d15cb16d08d750b2d0d6e77c81acf2c5b6cce Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 15 Jun 2022 17:39:30 -0700 Subject: [PATCH 340/346] sched/walt: Check reserved CPU during walt_newidle_balance Currently, it is observed that multiple calls to active_load_balance can take place. For example, say two Big threads are running on Golds, and become misfits at the same time. Then, Prime which is currently say halted, will end up pulling both threads. This is because first, as Prime is unhalted, the CPU is kicked to enable it to pull tasks. However, while Prime is kicked, it goes back to idle briefly, causing the upstream newidle_balance path to come into play. As a consequence, both threads are queued on Prime, and eventually, the one sitting in the runqueue is put back on a Gold. This behavior is unoptimal and unnecessary. Ensure that walt_newidle_balance only takes when the CPU is not marked as reserved. Change-Id: I00feb8df3c4433333a356f2e855fb7a3ee848789 Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_lb.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 4d03bee065f5..e9aa3770cad6 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -783,9 +783,6 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (unlikely(walt_disabled)) return; - /*Cluster isn't initialized until after WALT is enabled*/ - order_index = wrq->cluster->id; - /* * newly idle load balance is completely handled here, so * set done to skip the load balance by the caller. @@ -806,6 +803,12 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (cpu_halted(this_cpu)) return; + if (is_reserved(this_cpu)) + return; + + /*Cluster isn't initialized until after WALT is enabled*/ + order_index = wrq->cluster->id; + rq_unpin_lock(this_rq, rf); /* From f41f3a4e0c0f026e6d00f9d353bea54cf188b26e Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Wed, 15 Jun 2022 15:30:11 -0700 Subject: [PATCH 341/346] sched/walt: Fix excess detach_task in walt_lb_pull_tasks There's three loops to find a task to pull. In the first iteration, we look for a task that is not running, which fits a strict criteria. In the second iteration, we look for a task that is not running, which fits a lax criteria. If we have still not found a valid task to pull, we intended to pull the running task servicing an active load migration. However, there is some leftover code, wherein a task that is not running will end up being detached and pulled. This was not intended, and leads to incorrect behavior. Essentially, this is negating the effects of the first two iterations. Change-Id: Ia225d89043c76ec6e661771a39d058498346920d Signed-off-by: Shaleen Agrawal --- kernel/sched/walt/walt_lb.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index e9aa3770cad6..62f92715236b 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -341,11 +341,9 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { - if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) - continue; - if (task_running(src_rq, p)) { - if (need_active_lb(p, dst_cpu, src_cpu)) { + if (cpumask_test_cpu(dst_cpu, p->cpus_ptr) + && need_active_lb(p, dst_cpu, src_cpu)) { bool success; active_balance = true; src_rq->active_balance = 1; @@ -371,12 +369,8 @@ static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) return 0; /* we did not pull any task here */ } - continue; + goto unlock; } - - walt_detach_task(p, src_rq, dst_rq); - pulled_task = p; - goto unlock; } unlock: /* lock must be dropped before waking the stopper */ From 84bb530ed34a831569e74b166cd58508c3ea1341 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Wed, 15 Jun 2022 19:08:19 -0700 Subject: [PATCH 342/346] sched/walt: Add walt_disabled checks Entry into WALT code via tracehooks should be guarded based on the completion of WALT initialization. Add check for the same in functions where it is missing. Change-Id: I2a04003977832087be681108c15bd6bbb24fc7e8 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/walt.c | 3 +++ kernel/sched/walt/walt_rt.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index ab43e0820bc2..1fcb74c80208 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -4530,6 +4530,9 @@ static void walt_do_sched_yield(void *unused, struct rq *rq) struct task_struct *curr = rq->curr; struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1; + if (unlikely(walt_disabled)) + return; + lockdep_assert_held(&rq->__lock); if (!list_empty(&wts->mvp_list) && wts->mvp_list.next) walt_cfs_deactivate_mvp_task(rq, curr); diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 02ba1d2200c1..e69fe2a9e9de 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -329,6 +329,9 @@ static void walt_rt_find_lowest_rq(void *unused, struct task_struct *task, { int packing_cpu; + if (unlikely(walt_disabled)) + return; + /* create a fastpath for finding a packing cpu */ packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); if (walt_choose_packing_cpu(packing_cpu, task)) { From ca90b4c36e5641272803113b298da01b61f859b2 Mon Sep 17 00:00:00 2001 From: Sai Harshini Nimmala Date: Tue, 21 Jun 2022 19:50:52 -0700 Subject: [PATCH 343/346] sched/walt: Remove avg_cap check in packing CPU feature The current scheduler code uses avg_cap that is calculated by waltgov. This is stored in a private structure wg_policy in waltgov. The problem with this is wg_policy could be freed when the governor changes or all the CPUs in the policy get offlined. This leads to a use-after-free access of wg_policy->avg_cap. Besides, WALT introduced checks between avg_cap and cur_freq to identify if scheduler guided frequency is overridden by CPU frequency min/max limits. However, avg_cap doesn't reflect the scheduler guided frequency and instead represents the frequency from min/max limits. IOW, cur_freq will always be equal to avg_cap when CPU frequency limits are active, rendering the check useless and thus allowing packing. Remove the code that retrieves and checks avg_cap. Change-Id: I9b32c6798fb209d63d688ed89dc17674a7859475 Signed-off-by: Sai Harshini Nimmala --- kernel/sched/walt/cpufreq_walt.c | 19 ------------------- kernel/sched/walt/sched_avg.c | 11 ----------- kernel/sched/walt/walt.c | 19 ------------------- kernel/sched/walt/walt.h | 3 --- 4 files changed, 52 deletions(-) diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index 650fff45845b..7de85593f288 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -188,25 +188,6 @@ static void waltgov_calc_avg_cap(struct waltgov_policy *wg_policy, u64 curr_ws, wg_policy->last_ws = curr_ws; } -/* - * if waltgov is initialized, return the avg_cap seen - * over the last window. return 0 otherwise. - */ -unsigned int waltgov_get_avg_cap(unsigned int cpu) -{ - struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); - struct waltgov_policy *wg_policy = wg_cpu->wg_policy; - - /* if the policy is not initialized, or the callback - * is not initialized. callback is initialized on - * start of waltgov, erased on stop of waltgov. - */ - if (!wg_policy || !wg_cpu->cb.func) - return 0; - - return wg_policy->avg_cap; -} - static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time, unsigned int next_freq) { diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c index 4e5a267d2b17..c18e5c2234a9 100644 --- a/kernel/sched/walt/sched_avg.c +++ b/kernel/sched/walt/sched_avg.c @@ -56,16 +56,6 @@ unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster) return cluster_util_pct; } -unsigned int sched_get_cpu_avg_cap(int cpu) -{ - unsigned int cpu_avg_cap = 0; - - if (cpu < WALT_NR_CPUS) - cpu_avg_cap = stats[cpu].avg_cap; - - return cpu_avg_cap; -} - /** * sched_get_nr_running_avg * @return: Average nr_running, iowait and nr_big_tasks value since last poll. @@ -119,7 +109,6 @@ struct sched_avg_stats *sched_get_nr_running_avg(void) NR_THRESHOLD_PCT), 100); stats[cpu].nr_max = per_cpu(nr_max, cpu); stats[cpu].nr_scaled = tmp_nr; - stats[cpu].avg_cap = waltgov_get_avg_cap(cpu); trace_sched_get_nr_running_avg(cpu, stats[cpu].nr, stats[cpu].nr_misfit, stats[cpu].nr_max, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 1fcb74c80208..a1f2145f2345 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -345,21 +345,6 @@ static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, static void rollover_cpu_window(struct rq *rq, bool full_window); static void rollover_top_tasks(struct rq *rq, bool full_window); -/* - * if last window's average capacity is less than or - * equal to the current capacity, return true. - */ -static inline bool is_cpufreq_avg_or_higher(int cpu) -{ - unsigned int avg_cap = sched_get_cpu_avg_cap(cpu); - unsigned int cur_cap = capacity_curr_of(cpu); - - if (cur_cap >= avg_cap && avg_cap != 0) - return true; - - return false; -} - /* walt_find_cluster_packing_cpu - Return a packing_cpu choice common for this cluster. * @start_cpu: The cpu from the cluster to choose from * @@ -429,10 +414,6 @@ bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p) if (task_util(p) >= sysctl_sched_idle_enough) return false; - /* if cpufreq is lower than the previous window */ - if (!is_cpufreq_avg_or_higher(packing_cpu)) - return false; - /* the packing cpu can be used, so pack! */ return true; } diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 1663bbf281e1..b8a0e5bf8096 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -349,7 +349,6 @@ struct sched_avg_stats { int nr_misfit; int nr_max; int nr_scaled; - u32 avg_cap; }; struct waltgov_callback { @@ -791,8 +790,6 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) extern struct sched_avg_stats *sched_get_nr_running_avg(void); extern unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster); -extern unsigned int sched_get_cpu_avg_cap(int cpu); -extern unsigned int waltgov_get_avg_cap(unsigned int cpu); extern void sched_update_hyst_times(void); extern void walt_rt_init(void); From 336aaa54a799e9ffea2836d8b66f4b0cb493f2f6 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 22 Jun 2022 13:13:39 -0700 Subject: [PATCH 344/346] sched/walt: check for -1 for first_idle cpu There is a typo in the current code it is checking a value of 1 for unavailability of idle kick cpu. It should check for -1 instead. Fix it. Change-Id: I8796b4c6315d157834ecc8dc2263bcfb7f50e582 Signed-off-by: Abhijeet Dharmapurikar --- kernel/sched/walt/walt_lb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 62f92715236b..43beea4b55ab 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -850,7 +850,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, if (busy_cpu != -1) { first_idle = find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); - if (first_idle != 1) + if (first_idle != -1) walt_kick_cpu(first_idle); else goto found_busy_cpu; From 4bb9750f22b9e453a5ec04d454f5b06a3519239a Mon Sep 17 00:00:00 2001 From: Ashay Jaiswal Date: Tue, 12 Apr 2022 12:42:13 +0530 Subject: [PATCH 345/346] sched: walt: handle support for single and two cluster system Update walt_newidle load balance to support single and two cluster topology. Change-Id: Ic2bbbe7a273267c1a7aa340f9908b5e5325d21ba Signed-off-by: Ashay Jaiswal --- kernel/sched/walt/walt_lb.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 43beea4b55ab..a3020210bbc3 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -770,7 +770,7 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, int order_index; int busy_cpu = -1; bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); - bool help_min_cap = false; + bool help_min_cap = false, find_next_cluster = false; int first_idle; int has_misfit = 0; @@ -828,6 +828,26 @@ static void walt_newidle_balance(void *unused, struct rq *this_rq, * can be queued remotely, so keep a check on nr_running * and bail out. */ + if (num_sched_clusters <= 2) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + + if (num_sched_clusters == 2) { + has_misfit = false; + find_next_cluster = (order_index == 0) ? enough_idle : 1; + if (find_next_cluster) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, + &cpu_array[order_index][1], &has_misfit); + if (busy_cpu != -1 && (enough_idle || has_misfit)) + goto found_busy_cpu; + } + } + + goto unlock; + } + if (order_index == 0) { busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], &has_misfit); From 5204e041242eb01da10760d52bd8ed7d46e5e872 Mon Sep 17 00:00:00 2001 From: Shaleen Agrawal Date: Fri, 24 Jun 2022 13:58:51 -0700 Subject: [PATCH 346/346] sched: improve the scheduler This change is for general scheduler improvement. Change-Id: I98af51b7efd49aeec5a7b405625a1db12c654cd5 Signed-off-by: Shaleen Agrawal --- include/linux/sched/walt.h | 1 + kernel/sched/walt/walt.c | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h index bb636d930aad..85eda2491683 100644 --- a/include/linux/sched/walt.h +++ b/include/linux/sched/walt.h @@ -136,6 +136,7 @@ struct walt_task_struct { int load_boost; int64_t boosted_task_load; int prev_cpu; + int new_cpu; u8 enqueue_after_migration; }; diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index a1f2145f2345..702e6a492269 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -203,6 +203,15 @@ void walt_task_dump(struct task_struct *p) SCHED_PRINT(wts->mark_start); SCHED_PRINT(wts->demand); SCHED_PRINT(wts->coloc_demand); + SCHED_PRINT(wts->enqueue_after_migration); + SCHED_PRINT(wts->last_sleep_ts); + SCHED_PRINT(wts->prev_cpu); + SCHED_PRINT(wts->new_cpu); + SCHED_PRINT(wts->misfit); + SCHED_PRINT(wts->prev_on_rq); + SCHED_PRINT(wts->prev_on_rq_cpu); + SCHED_PRINT(wts->mvp_prio); + SCHED_PRINT(wts->iowaited); SCHED_PRINT(sched_ravg_window); SCHED_PRINT(new_sched_ravg_window); @@ -1099,6 +1108,8 @@ static void migrate_busy_time_subtraction(struct task_struct *p, int new_cpu) WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on src_rq %d", raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); + wts->new_cpu = new_cpu; + if (!same_freq_domain(task_cpu(p), new_cpu)) wts->enqueue_after_migration = 2; /* 2 is intercluster */ else @@ -1227,6 +1238,7 @@ static void migrate_busy_time_addition(struct task_struct *p, int new_cpu, u64 w dest_wrq->ed_task = p; wts->enqueue_after_migration = 0; + wts->new_cpu = -1; } #define INC_STEP 8 @@ -2404,6 +2416,7 @@ static void init_new_task_load(struct task_struct *p) INIT_LIST_HEAD(&wts->grp_list); wts->prev_cpu = raw_smp_processor_id(); + wts->new_cpu = -1; wts->enqueue_after_migration = 0; wts->mark_start = 0; wts->window_start = 0;