mirror of
https://github.com/torvalds/linux.git
synced 2026-07-29 18:51:21 +02:00
Merge "Merge scheduler changes from 5.15 into msm-pineapple"
This commit is contained in:
qctecmdr
Gerrit - the friendly Code Review server
commit
e092e6d615
2
Kconfig
2
Kconfig
|
|
@ -33,3 +33,5 @@ source "Documentation/Kconfig"
|
|||
|
||||
# ANDROID: Set KCONFIG_EXT_PREFIX to decend into an external project.
|
||||
source "$(KCONFIG_EXT_PREFIX)Kconfig.ext"
|
||||
|
||||
source "kernel/sched/walt/Kconfig"
|
||||
|
|
|
|||
1101
drivers/soc/qcom/hyp_core_ctl.c
Normal file
1101
drivers/soc/qcom/hyp_core_ctl.c
Normal file
File diff suppressed because it is too large
Load Diff
1231
drivers/soc/qcom/msm_performance.c
Normal file
1231
drivers/soc/qcom/msm_performance.c
Normal file
File diff suppressed because it is too large
Load Diff
|
|
@ -41,3 +41,14 @@ config QCOM_LMH
|
|||
input from temperature and current sensors. On many newer Qualcomm SoCs
|
||||
LMh is configured in the firmware and this feature need not be enabled.
|
||||
However, on certain SoCs like sdm845 LMh has to be configured from kernel.
|
||||
|
||||
config QTI_CPU_PAUSE_COOLING_DEVICE
|
||||
tristate "QTI CPU Pause cooling devices"
|
||||
depends on THERMAL_OF && HOTPLUG_CPU
|
||||
depends on ARCH_QCOM || COMPILE_TEST
|
||||
help
|
||||
This enables the QTI CPU Pause cooling device. These cooling
|
||||
devices will be used by QTI chipset to pause a CPU from being
|
||||
scheduled and hence will let the CPU to power collapse. Pausing
|
||||
a CPU will be used when the CPU frequency mitigation
|
||||
is not good enough to achieve the necessary cooling.
|
||||
|
|
@ -6,3 +6,4 @@ qcom_tsens-y += tsens.o tsens-v2.o tsens-v1.o tsens-v0_1.o \
|
|||
obj-$(CONFIG_QCOM_SPMI_ADC_TM5) += qcom-spmi-adc-tm5.o
|
||||
obj-$(CONFIG_QCOM_SPMI_TEMP_ALARM) += qcom-spmi-temp-alarm.o
|
||||
obj-$(CONFIG_QCOM_LMH) += lmh.o
|
||||
obj-$(CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE) += thermal_pause.o
|
||||
|
|
|
|||
473
drivers/thermal/qcom/thermal_pause.c
Normal file
473
drivers/thermal/qcom/thermal_pause.c
Normal file
|
|
@ -0,0 +1,473 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#define pr_fmt(fmt) "%s:%s " fmt, KBUILD_MODNAME, __func__
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/thermal.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/of_device.h>
|
||||
#include <linux/suspend.h>
|
||||
#include <linux/cpumask.h>
|
||||
#include <linux/sched/walt.h>
|
||||
|
||||
enum thermal_pause_levels {
|
||||
THERMAL_NO_CPU_PAUSE,
|
||||
THERMAL_GROUP_CPU_PAUSE,
|
||||
|
||||
/* define new pause levels above this line */
|
||||
MAX_THERMAL_PAUSE_LEVEL
|
||||
};
|
||||
|
||||
#define THERMAL_PAUSE_RETRY_COUNT 5
|
||||
|
||||
struct thermal_pause_cdev {
|
||||
struct list_head node;
|
||||
cpumask_t cpu_mask;
|
||||
enum thermal_pause_levels thermal_pause_level;
|
||||
enum thermal_pause_levels thermal_pause_req;
|
||||
struct thermal_cooling_device *cdev;
|
||||
struct device_node *np;
|
||||
char cdev_name[THERMAL_NAME_LENGTH];
|
||||
struct work_struct reg_work;
|
||||
struct work_struct pause_update_work;
|
||||
};
|
||||
|
||||
static DEFINE_MUTEX(cpus_pause_lock);
|
||||
static LIST_HEAD(thermal_pause_cdev_list);
|
||||
|
||||
static struct cpumask cpus_in_max_cooling_level;
|
||||
static enum cpuhp_state cpu_hp_online;
|
||||
|
||||
static BLOCKING_NOTIFIER_HEAD(thermal_pause_notifier);
|
||||
|
||||
void thermal_pause_notifier_register(struct notifier_block *n)
|
||||
{
|
||||
blocking_notifier_chain_register(&thermal_pause_notifier, n);
|
||||
}
|
||||
EXPORT_SYMBOL(thermal_pause_notifier_register);
|
||||
|
||||
void thermal_pause_notifier_unregister(struct notifier_block *n)
|
||||
{
|
||||
blocking_notifier_chain_unregister(&thermal_pause_notifier, n);
|
||||
}
|
||||
EXPORT_SYMBOL(thermal_pause_notifier_unregister);
|
||||
|
||||
const struct cpumask *thermal_paused_cpumask(void)
|
||||
{
|
||||
return &cpus_in_max_cooling_level;
|
||||
}
|
||||
EXPORT_SYMBOL(thermal_paused_cpumask);
|
||||
|
||||
static int thermal_pause_hp_online(unsigned int online_cpu)
|
||||
{
|
||||
struct thermal_pause_cdev *thermal_pause_cdev;
|
||||
int ret = 0;
|
||||
|
||||
pr_debug("online entry CPU:%d\n", online_cpu);
|
||||
|
||||
mutex_lock(&cpus_pause_lock);
|
||||
list_for_each_entry(thermal_pause_cdev, &thermal_pause_cdev_list, node) {
|
||||
if (cpumask_test_cpu(online_cpu, &thermal_pause_cdev->cpu_mask)
|
||||
&& !thermal_pause_cdev->cdev)
|
||||
queue_work(system_highpri_wq,
|
||||
&thermal_pause_cdev->reg_work);
|
||||
}
|
||||
mutex_unlock(&cpus_pause_lock);
|
||||
pr_debug("online exit CPU:%d\n", online_cpu);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* thermal_pause_work - work function to pause a group of cpus at
|
||||
* the specified level.
|
||||
*
|
||||
* @thermal_pasue_cdev: the cdev currently being processed
|
||||
*
|
||||
* Function to handle setting the current cpus paused by
|
||||
* this driver for the mask specified in the device.
|
||||
* it assumes the mutex is locked upon entrance.
|
||||
*/
|
||||
static int thermal_pause_work(struct thermal_pause_cdev *thermal_pause_cdev)
|
||||
{
|
||||
int cpu = 0;
|
||||
int ret = -EBUSY;
|
||||
cpumask_t cpus_to_pause, cpus_to_notify;
|
||||
|
||||
cpumask_copy(&cpus_to_pause, &thermal_pause_cdev->cpu_mask);
|
||||
pr_debug("Pause:%*pbl\n", cpumask_pr_args(&thermal_pause_cdev->cpu_mask));
|
||||
|
||||
mutex_unlock(&cpus_pause_lock);
|
||||
ret = walt_pause_cpus(&cpus_to_pause, PAUSE_THERMAL);
|
||||
mutex_lock(&cpus_pause_lock);
|
||||
|
||||
if (ret == 0) {
|
||||
/* remove CPUs that have already been notified */
|
||||
cpumask_andnot(&cpus_to_notify, &thermal_pause_cdev->cpu_mask,
|
||||
&cpus_in_max_cooling_level);
|
||||
|
||||
for_each_cpu(cpu, &cpus_to_notify)
|
||||
blocking_notifier_call_chain(&thermal_pause_notifier, 1,
|
||||
(void *)(long)cpu);
|
||||
|
||||
/* track CPUs currently in cooling level */
|
||||
cpumask_or(&cpus_in_max_cooling_level,
|
||||
&cpus_in_max_cooling_level,
|
||||
&thermal_pause_cdev->cpu_mask);
|
||||
} else {
|
||||
/* Failure. These cpus not paused by thermal */
|
||||
pr_err("Error pausing CPU:%*pbl. err:%d\n",
|
||||
cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* thermal_resume_work - work function to unpause a
|
||||
* group of cpus in the mask for this cdev
|
||||
*
|
||||
* @thermal_pasue_cdev: the cdev currently being processed
|
||||
*
|
||||
* Function to handle enabling the group of cpus in the cdev.
|
||||
* This is performed as a work function to avoid conflicts
|
||||
* between a hotplug event invoking a pause cooling device,
|
||||
* and a thermal event invoking a pause cooling device.
|
||||
*/
|
||||
static int thermal_resume_work(struct thermal_pause_cdev *thermal_pause_cdev)
|
||||
{
|
||||
int cpu = 0;
|
||||
int ret = -ENODEV;
|
||||
cpumask_t cpus_to_unpause, new_paused_cpus, cpus_to_notify;
|
||||
struct thermal_pause_cdev *cdev;
|
||||
|
||||
cpumask_copy(&cpus_to_unpause, &thermal_pause_cdev->cpu_mask);
|
||||
pr_debug("Unpause:%*pbl\n", cpumask_pr_args(&cpus_to_unpause));
|
||||
|
||||
mutex_unlock(&cpus_pause_lock);
|
||||
ret = walt_resume_cpus(&cpus_to_unpause, PAUSE_THERMAL);
|
||||
mutex_lock(&cpus_pause_lock);
|
||||
|
||||
if (ret == 0) {
|
||||
/* gather up the cpus paused state from all the cdevs */
|
||||
cpumask_clear(&new_paused_cpus);
|
||||
list_for_each_entry(cdev, &thermal_pause_cdev_list, node) {
|
||||
if (!cdev->thermal_pause_level || cdev == thermal_pause_cdev)
|
||||
continue;
|
||||
cpumask_or(&new_paused_cpus, &new_paused_cpus, &cdev->cpu_mask);
|
||||
}
|
||||
|
||||
/* remove CPUs that will remain paused */
|
||||
cpumask_andnot(&cpus_to_notify, &cpus_in_max_cooling_level, &new_paused_cpus);
|
||||
|
||||
/* Notify for each CPU that we intended to resume */
|
||||
for_each_cpu(cpu, &cpus_to_notify)
|
||||
blocking_notifier_call_chain(&thermal_pause_notifier, 0,
|
||||
(void *)(long)cpu);
|
||||
|
||||
/* update the cpus cooling mask */
|
||||
cpumask_copy(&cpus_in_max_cooling_level, &new_paused_cpus);
|
||||
} else {
|
||||
/* Failure. Ref-count for cpus controlled by thermal still set */
|
||||
pr_err("Error resuming CPU:%*pbl. err:%d\n",
|
||||
cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* thermal_pause_set_update_work: Enforce Requested State
|
||||
*
|
||||
* @work: the work structure for this work
|
||||
*
|
||||
* Enforce the most recent requested cooling state, if
|
||||
* it is a mismatch with the current state. Since the
|
||||
* request is made in a different context from the
|
||||
* enforcement, it is possible to have an updated request
|
||||
* after completing the resume/pause request.
|
||||
*
|
||||
* Handle a post-operation mismatch between the cooling state
|
||||
* and the requested state.
|
||||
*
|
||||
* This is performed as a work function to avoid conflicts
|
||||
* between a hotplug event invoking a pause cooling device,
|
||||
* and a thermal event invoking a pause cooling device.
|
||||
*/
|
||||
static void thermal_pause_update_work(struct work_struct *work)
|
||||
{
|
||||
int ret = 0;
|
||||
int retcnt = THERMAL_PAUSE_RETRY_COUNT;
|
||||
struct thermal_pause_cdev *thermal_pause_cdev =
|
||||
container_of(work, struct thermal_pause_cdev, pause_update_work);
|
||||
|
||||
mutex_lock(&cpus_pause_lock);
|
||||
|
||||
retry:
|
||||
if (thermal_pause_cdev->thermal_pause_req != thermal_pause_cdev->thermal_pause_level) {
|
||||
if (thermal_pause_cdev->thermal_pause_req == THERMAL_NO_CPU_PAUSE) {
|
||||
ret = thermal_resume_work(thermal_pause_cdev);
|
||||
if (ret >= 0)
|
||||
thermal_pause_cdev->thermal_pause_level = THERMAL_NO_CPU_PAUSE;
|
||||
} else {
|
||||
ret = thermal_pause_work(thermal_pause_cdev);
|
||||
if (ret >= 0)
|
||||
thermal_pause_cdev->thermal_pause_level = THERMAL_GROUP_CPU_PAUSE;
|
||||
}
|
||||
if (ret < 0 && retcnt > 0) {
|
||||
retcnt--;
|
||||
goto retry;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* if the pause/resume operation itself failed (and failed THERMAL_PAUSE_RETRY_COUNT
|
||||
* times) then ret will be negative here. Do not repeatedly retry if pause itself
|
||||
* failed, because this can happen indefinitely.
|
||||
*
|
||||
* If instead the pause request has been toggled back and forth many times by
|
||||
* the thermal framework, this can be handled here.
|
||||
*/
|
||||
if (ret >= 0 &&
|
||||
thermal_pause_cdev->thermal_pause_req != thermal_pause_cdev->thermal_pause_level) {
|
||||
pr_debug("Pause: requested state changed while workfn running\n");
|
||||
retcnt = THERMAL_PAUSE_RETRY_COUNT;
|
||||
goto retry;
|
||||
}
|
||||
|
||||
mutex_unlock(&cpus_pause_lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* thermal_pause_set_cur_state - callback function to set the current cooling
|
||||
* level.
|
||||
* @cdev: thermal cooling device pointer.
|
||||
* @level: set this variable to the current cooling level.
|
||||
*
|
||||
* Callback for the thermal cooling device to change the cpu pause
|
||||
* current cooling level, by making a requested and queueing the
|
||||
* work to be done.
|
||||
*
|
||||
* Return: 0 on success, an error code otherwise.
|
||||
*/
|
||||
static int thermal_pause_set_cur_state(struct thermal_cooling_device *cdev,
|
||||
unsigned long level)
|
||||
{
|
||||
struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata;
|
||||
|
||||
if (level >= MAX_THERMAL_PAUSE_LEVEL)
|
||||
return -EINVAL;
|
||||
|
||||
mutex_lock(&cpus_pause_lock);
|
||||
|
||||
if (level)
|
||||
thermal_pause_cdev->thermal_pause_req = THERMAL_GROUP_CPU_PAUSE;
|
||||
else
|
||||
thermal_pause_cdev->thermal_pause_req = THERMAL_NO_CPU_PAUSE;
|
||||
|
||||
queue_work(system_highpri_wq, &thermal_pause_cdev->pause_update_work);
|
||||
|
||||
mutex_unlock(&cpus_pause_lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* thermal_pause_get_cur_state - callback function to get the current cooling
|
||||
* state.
|
||||
* @cdev: thermal cooling device pointer.
|
||||
* @state: fill this variable with the current cooling state.
|
||||
*
|
||||
* Callback for the thermal cooling device to return the cpu pause
|
||||
* current cooling level
|
||||
*
|
||||
* Return: 0 on success, an error code otherwise.
|
||||
*/
|
||||
static int thermal_pause_get_cur_state(struct thermal_cooling_device *cdev,
|
||||
unsigned long *level)
|
||||
{
|
||||
struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata;
|
||||
|
||||
*level = thermal_pause_cdev->thermal_pause_level;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* thermal_pause_get_max_state - callback function to get the max cooling state.
|
||||
* @cdev: thermal cooling device pointer.
|
||||
* @level: fill this variable with the max cooling level
|
||||
*
|
||||
* Callback for the thermal cooling device to return the cpu
|
||||
* pause max cooling state.
|
||||
*
|
||||
* Return: 0 on success, an error code otherwise.
|
||||
*/
|
||||
static int thermal_pause_get_max_state(struct thermal_cooling_device *cdev,
|
||||
unsigned long *level)
|
||||
{
|
||||
*level = MAX_THERMAL_PAUSE_LEVEL - 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct thermal_cooling_device_ops thermal_pause_cooling_ops = {
|
||||
.get_max_state = thermal_pause_get_max_state,
|
||||
.get_cur_state = thermal_pause_get_cur_state,
|
||||
.set_cur_state = thermal_pause_set_cur_state,
|
||||
};
|
||||
|
||||
static void thermal_pause_register_cdev(struct work_struct *work)
|
||||
{
|
||||
struct thermal_pause_cdev *thermal_pause_cdev =
|
||||
container_of(work, struct thermal_pause_cdev, reg_work);
|
||||
int ret = 0;
|
||||
cpumask_t cpus_online;
|
||||
|
||||
cpumask_and(&cpus_online,
|
||||
&thermal_pause_cdev->cpu_mask,
|
||||
cpu_online_mask);
|
||||
if (!cpumask_equal(&thermal_pause_cdev->cpu_mask, &cpus_online))
|
||||
return;
|
||||
|
||||
thermal_pause_cdev->cdev = thermal_of_cooling_device_register(
|
||||
thermal_pause_cdev->np,
|
||||
thermal_pause_cdev->cdev_name,
|
||||
thermal_pause_cdev,
|
||||
&thermal_pause_cooling_ops);
|
||||
|
||||
if (IS_ERR(thermal_pause_cdev->cdev)) {
|
||||
ret = PTR_ERR(thermal_pause_cdev->cdev);
|
||||
pr_err("Cooling register failed for %s, ret:%d\n",
|
||||
thermal_pause_cdev->cdev_name, ret);
|
||||
thermal_pause_cdev->cdev = NULL;
|
||||
return;
|
||||
}
|
||||
pr_debug("Cooling device [%s] registered.\n",
|
||||
thermal_pause_cdev->cdev_name);
|
||||
}
|
||||
|
||||
static int thermal_pause_probe(struct platform_device *pdev)
|
||||
{
|
||||
int ret = 0, cpu = 0;
|
||||
struct device_node *subsys_np = NULL, *cpu_phandle = NULL;
|
||||
struct device *cpu_dev;
|
||||
struct thermal_pause_cdev *thermal_pause_cdev = NULL;
|
||||
struct device_node *np = pdev->dev.of_node;
|
||||
struct of_phandle_iterator it;
|
||||
cpumask_t cpu_mask;
|
||||
unsigned long mask = 0;
|
||||
const char *alias;
|
||||
|
||||
INIT_LIST_HEAD(&thermal_pause_cdev_list);
|
||||
cpumask_clear(&cpus_in_max_cooling_level);
|
||||
|
||||
for_each_available_child_of_node(np, subsys_np) {
|
||||
|
||||
cpumask_clear(&cpu_mask);
|
||||
mask = 0;
|
||||
of_phandle_iterator_init(&it, subsys_np, "qcom,cpus", NULL, 0);
|
||||
while (of_phandle_iterator_next(&it) == 0) {
|
||||
cpu_phandle = it.node;
|
||||
for_each_possible_cpu(cpu) {
|
||||
cpu_dev = get_cpu_device(cpu);
|
||||
if (cpu_dev && cpu_dev->of_node
|
||||
== cpu_phandle) {
|
||||
cpumask_set_cpu(cpu, &cpu_mask);
|
||||
mask = mask | BIT(cpu);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (cpumask_empty(&cpu_mask))
|
||||
continue;
|
||||
|
||||
thermal_pause_cdev = devm_kzalloc(&pdev->dev,
|
||||
sizeof(*thermal_pause_cdev), GFP_KERNEL);
|
||||
|
||||
if (!thermal_pause_cdev) {
|
||||
of_node_put(subsys_np);
|
||||
return -ENOMEM;
|
||||
}
|
||||
ret = of_property_read_string(subsys_np,
|
||||
"qcom,cdev-alias", &alias);
|
||||
if (ret)
|
||||
snprintf(thermal_pause_cdev->cdev_name, THERMAL_NAME_LENGTH,
|
||||
"thermal-pause-%X", mask);
|
||||
else
|
||||
strscpy(thermal_pause_cdev->cdev_name, alias,
|
||||
THERMAL_NAME_LENGTH);
|
||||
|
||||
thermal_pause_cdev->thermal_pause_level = false;
|
||||
thermal_pause_cdev->cdev = NULL;
|
||||
thermal_pause_cdev->np = subsys_np;
|
||||
cpumask_copy(&thermal_pause_cdev->cpu_mask, &cpu_mask);
|
||||
|
||||
INIT_WORK(&thermal_pause_cdev->reg_work, thermal_pause_register_cdev);
|
||||
INIT_WORK(&thermal_pause_cdev->pause_update_work, thermal_pause_update_work);
|
||||
list_add(&thermal_pause_cdev->node, &thermal_pause_cdev_list);
|
||||
}
|
||||
|
||||
ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal-pause/cdev:online",
|
||||
thermal_pause_hp_online, NULL);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
cpu_hp_online = ret;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int thermal_pause_remove(struct platform_device *pdev)
|
||||
{
|
||||
struct thermal_pause_cdev *thermal_pause_cdev = NULL, *next = NULL;
|
||||
int ret = 0;
|
||||
|
||||
if (cpu_hp_online) {
|
||||
cpuhp_remove_state_nocalls(cpu_hp_online);
|
||||
cpu_hp_online = 0;
|
||||
}
|
||||
|
||||
mutex_lock(&cpus_pause_lock);
|
||||
list_for_each_entry_safe(thermal_pause_cdev, next,
|
||||
&thermal_pause_cdev_list, node) {
|
||||
|
||||
/* for each asserted cooling device, resume the CPUs */
|
||||
if (thermal_pause_cdev->thermal_pause_level) {
|
||||
thermal_pause_cdev->thermal_pause_req = THERMAL_NO_CPU_PAUSE;
|
||||
queue_work(system_highpri_wq, &thermal_pause_cdev->pause_update_work);
|
||||
}
|
||||
|
||||
if (thermal_pause_cdev->cdev)
|
||||
thermal_cooling_device_unregister(
|
||||
thermal_pause_cdev->cdev);
|
||||
list_del(&thermal_pause_cdev->node);
|
||||
}
|
||||
|
||||
mutex_unlock(&cpus_pause_lock);
|
||||
|
||||
/* if the resume failed, thermal still controls the CPUs.
|
||||
* ensure that the error is passed to the caller.
|
||||
*/
|
||||
return ret;
|
||||
}
|
||||
|
||||
static const struct of_device_id thermal_pause_match[] = {
|
||||
{ .compatible = "qcom,thermal-pause", },
|
||||
{},
|
||||
};
|
||||
|
||||
static struct platform_driver thermal_pause_driver = {
|
||||
.probe = thermal_pause_probe,
|
||||
.remove = thermal_pause_remove,
|
||||
.driver = {
|
||||
.name = KBUILD_MODNAME,
|
||||
.of_match_table = thermal_pause_match,
|
||||
},
|
||||
};
|
||||
|
||||
module_platform_driver(thermal_pause_driver);
|
||||
MODULE_LICENSE("GPL v2");
|
||||
229
include/linux/sched/walt.h
Normal file
229
include/linux/sched/walt.h
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _LINUX_SCHED_WALT_H
|
||||
#define _LINUX_SCHED_WALT_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/spinlock_types.h>
|
||||
#include <linux/cpumask.h>
|
||||
|
||||
enum pause_reason {
|
||||
PAUSE_CORE_CTL = 0x01,
|
||||
PAUSE_THERMAL = 0x02,
|
||||
PAUSE_HYP = 0x04,
|
||||
};
|
||||
|
||||
#if IS_ENABLED(CONFIG_SCHED_WALT)
|
||||
|
||||
#define MAX_CPUS_PER_CLUSTER 6
|
||||
#define MAX_CLUSTERS 3
|
||||
|
||||
struct core_ctl_notif_data {
|
||||
unsigned int nr_big;
|
||||
unsigned int coloc_load_pct;
|
||||
unsigned int ta_util_pct[MAX_CLUSTERS];
|
||||
unsigned int cur_cap_pct[MAX_CLUSTERS];
|
||||
};
|
||||
|
||||
enum task_boost_type {
|
||||
TASK_BOOST_NONE = 0,
|
||||
TASK_BOOST_ON_MID,
|
||||
TASK_BOOST_ON_MAX,
|
||||
TASK_BOOST_STRICT_MAX,
|
||||
TASK_BOOST_END,
|
||||
};
|
||||
|
||||
#define WALT_NR_CPUS 8
|
||||
#define RAVG_HIST_SIZE 8
|
||||
/* wts->bucket_bitmask needs to be updated if NUM_BUSY_BUCKETS > 16 */
|
||||
#define NUM_BUSY_BUCKETS 16
|
||||
#define NUM_BUSY_BUCKETS_SHIFT 4
|
||||
|
||||
struct walt_related_thread_group {
|
||||
int id;
|
||||
raw_spinlock_t lock;
|
||||
struct list_head tasks;
|
||||
struct list_head list;
|
||||
bool skip_min;
|
||||
struct rcu_head rcu;
|
||||
u64 last_update;
|
||||
u64 downmigrate_ts;
|
||||
u64 start_ktime_ts;
|
||||
};
|
||||
|
||||
struct walt_task_struct {
|
||||
/*
|
||||
* 'mark_start' marks the beginning of an event (task waking up, task
|
||||
* starting to execute, task being preempted) within a window
|
||||
*
|
||||
* 'sum' represents how runnable a task has been within current
|
||||
* window. It incorporates both running time and wait time and is
|
||||
* frequency scaled.
|
||||
*
|
||||
* 'sum_history' keeps track of history of 'sum' seen over previous
|
||||
* RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are
|
||||
* ignored.
|
||||
*
|
||||
* 'demand' represents maximum sum seen over previous
|
||||
* sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency
|
||||
* demand for tasks.
|
||||
*
|
||||
* 'curr_window_cpu' represents task's contribution to cpu busy time on
|
||||
* various CPUs in the current window
|
||||
*
|
||||
* 'prev_window_cpu' represents task's contribution to cpu busy time on
|
||||
* various CPUs in the previous window
|
||||
*
|
||||
* 'curr_window' represents the sum of all entries in curr_window_cpu
|
||||
*
|
||||
* 'prev_window' represents the sum of all entries in prev_window_cpu
|
||||
*
|
||||
* 'pred_demand_scaled' represents task's current predicted cpu busy time
|
||||
* in terms of 1024 units
|
||||
*
|
||||
* 'busy_buckets' groups historical busy time into different buckets
|
||||
* used for prediction
|
||||
*
|
||||
* 'demand_scaled' represents task's demand scaled to 1024
|
||||
*
|
||||
* 'prev_on_rq' tracks enqueue/dequeue of a task for error conditions
|
||||
* 0 = nothing, 1 = enqueued, 2 = dequeued
|
||||
*/
|
||||
u64 mark_start;
|
||||
u64 window_start;
|
||||
u32 sum, demand;
|
||||
u32 coloc_demand;
|
||||
u32 sum_history[RAVG_HIST_SIZE];
|
||||
u16 sum_history_util[RAVG_HIST_SIZE];
|
||||
u32 curr_window_cpu[WALT_NR_CPUS];
|
||||
u32 prev_window_cpu[WALT_NR_CPUS];
|
||||
u32 curr_window, prev_window;
|
||||
u8 busy_buckets[NUM_BUSY_BUCKETS];
|
||||
u16 bucket_bitmask;
|
||||
u16 demand_scaled;
|
||||
u16 pred_demand_scaled;
|
||||
u64 active_time;
|
||||
u64 last_win_size;
|
||||
int boost;
|
||||
bool wake_up_idle;
|
||||
bool misfit;
|
||||
bool rtg_high_prio;
|
||||
u8 low_latency;
|
||||
u64 boost_period;
|
||||
u64 boost_expires;
|
||||
u64 last_sleep_ts;
|
||||
u32 init_load_pct;
|
||||
u32 unfilter;
|
||||
u64 last_wake_ts;
|
||||
u64 last_enqueued_ts;
|
||||
struct walt_related_thread_group __rcu *grp;
|
||||
struct list_head grp_list;
|
||||
u64 cpu_cycles;
|
||||
cpumask_t cpus_requested;
|
||||
bool iowaited;
|
||||
int prev_on_rq;
|
||||
int prev_on_rq_cpu;
|
||||
struct list_head mvp_list;
|
||||
u64 sum_exec_snapshot_for_slice;
|
||||
u64 sum_exec_snapshot_for_total;
|
||||
u64 total_exec;
|
||||
int mvp_prio;
|
||||
int cidx;
|
||||
int load_boost;
|
||||
int64_t boosted_task_load;
|
||||
int prev_cpu;
|
||||
int new_cpu;
|
||||
u8 enqueue_after_migration;
|
||||
};
|
||||
|
||||
#define wts_to_ts(wts) ({ \
|
||||
void *__mptr = (void *)(wts); \
|
||||
((struct task_struct *)(__mptr - \
|
||||
offsetof(struct task_struct, android_vendor_data1))); })
|
||||
|
||||
static inline bool sched_get_wake_up_idle(struct task_struct *p)
|
||||
{
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
|
||||
|
||||
return wts->wake_up_idle;
|
||||
}
|
||||
|
||||
static inline int sched_set_wake_up_idle(struct task_struct *p, bool wake_up_idle)
|
||||
{
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
|
||||
|
||||
wts->wake_up_idle = wake_up_idle;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void set_wake_up_idle(bool wake_up_idle)
|
||||
{
|
||||
struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1;
|
||||
|
||||
wts->wake_up_idle = wake_up_idle;
|
||||
}
|
||||
|
||||
extern int sched_lpm_disallowed_time(int cpu, u64 *timeout);
|
||||
extern int set_task_boost(int boost, u64 period);
|
||||
|
||||
struct notifier_block;
|
||||
extern void core_ctl_notifier_register(struct notifier_block *n);
|
||||
extern void core_ctl_notifier_unregister(struct notifier_block *n);
|
||||
extern int core_ctl_set_boost(bool boost);
|
||||
|
||||
extern int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason);
|
||||
extern int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason);
|
||||
extern int walt_halt_cpus(struct cpumask *cpus, enum pause_reason reason);
|
||||
extern int walt_start_cpus(struct cpumask *cpus, enum pause_reason reason);
|
||||
#else
|
||||
static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout)
|
||||
{
|
||||
return INT_MAX;
|
||||
}
|
||||
static inline int set_task_boost(int boost, u64 period)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline bool sched_get_wake_up_idle(struct task_struct *p)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline int sched_set_wake_up_idle(struct task_struct *p, bool wake_up_idle)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void set_wake_up_idle(bool wake_up_idle)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int core_ctl_set_boost(bool boost)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void core_ctl_notifier_register(struct notifier_block *n)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void core_ctl_notifier_unregister(struct notifier_block *n)
|
||||
{
|
||||
}
|
||||
|
||||
inline int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
inline int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _LINUX_SCHED_WALT_H */
|
||||
33
include/linux/thermal_pause.h
Normal file
33
include/linux/thermal_pause.h
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _LINUX_THERMAL_PAUSE_H
|
||||
#define _LINUX_THERMAL_PAUSE_H
|
||||
|
||||
#include <linux/notifier.h>
|
||||
#include <linux/cpumask.h>
|
||||
|
||||
#if IS_ENABLED(CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE)
|
||||
extern void thermal_pause_notifier_register(struct notifier_block *n);
|
||||
extern void thermal_pause_notifier_unregister(struct notifier_block *n);
|
||||
extern const struct cpumask *thermal_paused_cpumask(void);
|
||||
#else
|
||||
static inline
|
||||
void thermal_pause_notifier_register(struct notifier_block *n)
|
||||
{
|
||||
}
|
||||
|
||||
static inline
|
||||
void thermal_pause_notifier_unregister(struct notifier_block *n)
|
||||
{
|
||||
}
|
||||
|
||||
static inline const struct cpumask *thermal_paused_cpumask(void)
|
||||
{
|
||||
return cpu_none_mask;
|
||||
}
|
||||
#endif /* CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE */
|
||||
|
||||
#endif /* _LINUX_THERMAL_PAUSE_H */
|
||||
|
|
@ -32,3 +32,5 @@ obj-y += core.o
|
|||
obj-y += fair.o
|
||||
obj-y += build_policy.o
|
||||
obj-y += build_utility.o
|
||||
|
||||
obj-$(CONFIG_SCHED_WALT) += walt/
|
||||
|
|
|
|||
41
kernel/sched/walt/Kconfig
Normal file
41
kernel/sched/walt/Kconfig
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
# QTI WALT based scheduler
|
||||
#
|
||||
menu "QTI WALT based scheduler features"
|
||||
|
||||
config SCHED_WALT
|
||||
tristate "Support window based load tracking"
|
||||
depends on SMP
|
||||
help
|
||||
This feature will allow the scheduler to maintain a tunable window
|
||||
based set of metrics for tasks and runqueues. These metrics can be
|
||||
used to guide task placement as well as task frequency requirements
|
||||
for cpufreq governors.
|
||||
|
||||
config SCHED_WALT_DEBUG
|
||||
tristate "WALT debug module"
|
||||
depends on SCHED_WALT
|
||||
select TRACE_PREEMPT_TOGGLE
|
||||
select TRACE_IRQFLAGS
|
||||
help
|
||||
This module provides the means of debugging long preempt and
|
||||
irq disable code. This helps in identifying the scheduling
|
||||
latencies. The module rely on preemptirq trace hooks and
|
||||
print the stacktrace to the ftrace upon long preempt and irq
|
||||
events. Sysctl knobs are available for the user to configure
|
||||
the thresholds.
|
||||
|
||||
This module also used to crash the system to catch issues
|
||||
in scenarios like RT throttling and sleeping while in atomic
|
||||
context etc.
|
||||
|
||||
config SCHED_CONSERVATIVE_BOOST_LPM_BIAS
|
||||
bool "Enable LPM bias if conservative boost is enabled"
|
||||
default n
|
||||
help
|
||||
This feature will allow the scheduler to disable low power
|
||||
modes on a cpu if conservative boost is active. The cpu
|
||||
will not enter low power mode for a hysteresis time period,
|
||||
which can be configured from userspace.
|
||||
endmenu
|
||||
10
kernel/sched/walt/Makefile
Normal file
10
kernel/sched/walt/Makefile
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
|
||||
KCOV_INSTRUMENT := n
|
||||
KCSAN_SANITIZE := n
|
||||
|
||||
obj-$(CONFIG_SCHED_WALT) += sched-walt.o
|
||||
sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_halt.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o
|
||||
|
||||
obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o
|
||||
sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o
|
||||
300
kernel/sched/walt/boost.c
Normal file
300
kernel/sched/walt/boost.c
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/of.h>
|
||||
|
||||
#include "walt.h"
|
||||
#include "trace.h"
|
||||
|
||||
/*
|
||||
* Scheduler boost is a mechanism to temporarily place tasks on CPUs
|
||||
* with higher capacity than those where a task would have normally
|
||||
* ended up with their load characteristics. Any entity enabling
|
||||
* boost is responsible for disabling it as well.
|
||||
*/
|
||||
unsigned int sched_boost_type;
|
||||
enum sched_boost_policy boost_policy;
|
||||
|
||||
static DEFINE_MUTEX(boost_mutex);
|
||||
|
||||
void walt_init_tg(struct task_group *tg)
|
||||
{
|
||||
struct walt_task_group *wtg;
|
||||
|
||||
wtg = (struct walt_task_group *) tg->android_vendor_data1;
|
||||
|
||||
wtg->colocate = false;
|
||||
wtg->sched_boost_enable[NO_BOOST] = false;
|
||||
wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true;
|
||||
wtg->sched_boost_enable[CONSERVATIVE_BOOST] = false;
|
||||
wtg->sched_boost_enable[RESTRAINED_BOOST] = false;
|
||||
}
|
||||
|
||||
void walt_init_topapp_tg(struct task_group *tg)
|
||||
{
|
||||
struct walt_task_group *wtg;
|
||||
|
||||
wtg = (struct walt_task_group *) tg->android_vendor_data1;
|
||||
|
||||
wtg->colocate = true;
|
||||
wtg->sched_boost_enable[NO_BOOST] = false;
|
||||
wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true;
|
||||
wtg->sched_boost_enable[CONSERVATIVE_BOOST] = true;
|
||||
wtg->sched_boost_enable[RESTRAINED_BOOST] = false;
|
||||
}
|
||||
|
||||
void walt_init_foreground_tg(struct task_group *tg)
|
||||
{
|
||||
struct walt_task_group *wtg;
|
||||
|
||||
wtg = (struct walt_task_group *) tg->android_vendor_data1;
|
||||
|
||||
wtg->colocate = false;
|
||||
wtg->sched_boost_enable[NO_BOOST] = false;
|
||||
wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true;
|
||||
wtg->sched_boost_enable[CONSERVATIVE_BOOST] = true;
|
||||
wtg->sched_boost_enable[RESTRAINED_BOOST] = false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Scheduler boost type and boost policy might at first seem unrelated,
|
||||
* however, there exists a connection between them that will allow us
|
||||
* to use them interchangeably during placement decisions. We'll explain
|
||||
* the connection here in one possible way so that the implications are
|
||||
* clear when looking at placement policies.
|
||||
*
|
||||
* When policy = SCHED_BOOST_NONE, type is either none or RESTRAINED
|
||||
* When policy = SCHED_BOOST_ON_ALL or SCHED_BOOST_ON_BIG, type can
|
||||
* neither be none nor RESTRAINED.
|
||||
*/
|
||||
static void set_boost_policy(int type)
|
||||
{
|
||||
if (type == NO_BOOST || type == RESTRAINED_BOOST) {
|
||||
boost_policy = SCHED_BOOST_NONE;
|
||||
return;
|
||||
}
|
||||
|
||||
if (hmp_capable()) {
|
||||
boost_policy = SCHED_BOOST_ON_BIG;
|
||||
return;
|
||||
}
|
||||
|
||||
boost_policy = SCHED_BOOST_ON_ALL;
|
||||
}
|
||||
|
||||
static bool verify_boost_params(int type)
|
||||
{
|
||||
return type >= RESTRAINED_BOOST_DISABLE && type <= RESTRAINED_BOOST;
|
||||
}
|
||||
|
||||
static void sched_no_boost_nop(void)
|
||||
{
|
||||
}
|
||||
|
||||
static void sched_full_throttle_boost_enter(void)
|
||||
{
|
||||
core_ctl_set_boost(true);
|
||||
walt_enable_frequency_aggregation(true);
|
||||
}
|
||||
|
||||
static void sched_full_throttle_boost_exit(void)
|
||||
{
|
||||
core_ctl_set_boost(false);
|
||||
walt_enable_frequency_aggregation(false);
|
||||
}
|
||||
|
||||
static void sched_conservative_boost_enter(void)
|
||||
{
|
||||
}
|
||||
|
||||
static void sched_conservative_boost_exit(void)
|
||||
{
|
||||
}
|
||||
|
||||
static void sched_restrained_boost_enter(void)
|
||||
{
|
||||
walt_enable_frequency_aggregation(true);
|
||||
}
|
||||
|
||||
static void sched_restrained_boost_exit(void)
|
||||
{
|
||||
walt_enable_frequency_aggregation(false);
|
||||
}
|
||||
|
||||
struct sched_boost_data {
|
||||
int refcount;
|
||||
void (*enter)(void);
|
||||
void (*exit)(void);
|
||||
};
|
||||
|
||||
static struct sched_boost_data sched_boosts[] = {
|
||||
[NO_BOOST] = {
|
||||
.refcount = 0,
|
||||
.enter = sched_no_boost_nop,
|
||||
.exit = sched_no_boost_nop,
|
||||
},
|
||||
[FULL_THROTTLE_BOOST] = {
|
||||
.refcount = 0,
|
||||
.enter = sched_full_throttle_boost_enter,
|
||||
.exit = sched_full_throttle_boost_exit,
|
||||
},
|
||||
[CONSERVATIVE_BOOST] = {
|
||||
.refcount = 0,
|
||||
.enter = sched_conservative_boost_enter,
|
||||
.exit = sched_conservative_boost_exit,
|
||||
},
|
||||
[RESTRAINED_BOOST] = {
|
||||
.refcount = 0,
|
||||
.enter = sched_restrained_boost_enter,
|
||||
.exit = sched_restrained_boost_exit,
|
||||
},
|
||||
};
|
||||
|
||||
#define SCHED_BOOST_START FULL_THROTTLE_BOOST
|
||||
#define SCHED_BOOST_END (RESTRAINED_BOOST + 1)
|
||||
|
||||
static int sched_effective_boost(void)
|
||||
{
|
||||
int i;
|
||||
|
||||
/*
|
||||
* The boosts are sorted in descending order by
|
||||
* priority.
|
||||
*/
|
||||
for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) {
|
||||
if (sched_boosts[i].refcount >= 1)
|
||||
return i;
|
||||
}
|
||||
|
||||
return NO_BOOST;
|
||||
}
|
||||
|
||||
static void sched_boost_disable(int type)
|
||||
{
|
||||
struct sched_boost_data *sb = &sched_boosts[type];
|
||||
int next_boost, prev_boost = sched_boost_type;
|
||||
|
||||
if (sb->refcount <= 0)
|
||||
return;
|
||||
|
||||
sb->refcount--;
|
||||
|
||||
if (sb->refcount)
|
||||
return;
|
||||
|
||||
next_boost = sched_effective_boost();
|
||||
if (next_boost == prev_boost)
|
||||
return;
|
||||
/*
|
||||
* This boost's refcount becomes zero, so it must
|
||||
* be disabled. Disable it first and then apply
|
||||
* the next boost.
|
||||
*/
|
||||
sched_boosts[prev_boost].exit();
|
||||
sched_boosts[next_boost].enter();
|
||||
}
|
||||
|
||||
static void sched_boost_enable(int type)
|
||||
{
|
||||
struct sched_boost_data *sb = &sched_boosts[type];
|
||||
int next_boost, prev_boost = sched_boost_type;
|
||||
|
||||
sb->refcount++;
|
||||
|
||||
if (sb->refcount != 1)
|
||||
return;
|
||||
|
||||
/*
|
||||
* This boost enable request did not come before.
|
||||
* Take this new request and find the next boost
|
||||
* by aggregating all the enabled boosts. If there
|
||||
* is a change, disable the previous boost and enable
|
||||
* the next boost.
|
||||
*/
|
||||
|
||||
next_boost = sched_effective_boost();
|
||||
if (next_boost == prev_boost)
|
||||
return;
|
||||
|
||||
sched_boosts[prev_boost].exit();
|
||||
sched_boosts[next_boost].enter();
|
||||
}
|
||||
|
||||
static void sched_boost_disable_all(void)
|
||||
{
|
||||
int i;
|
||||
int prev_boost = sched_boost_type;
|
||||
|
||||
if (prev_boost != NO_BOOST) {
|
||||
sched_boosts[prev_boost].exit();
|
||||
for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++)
|
||||
sched_boosts[i].refcount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void _sched_set_boost(int type)
|
||||
{
|
||||
if (type == 0)
|
||||
sched_boost_disable_all();
|
||||
else if (type > 0)
|
||||
sched_boost_enable(type);
|
||||
else
|
||||
sched_boost_disable(-type);
|
||||
|
||||
/*
|
||||
* sysctl_sched_boost holds the boost request from
|
||||
* user space which could be different from the
|
||||
* effectively enabled boost. Update the effective
|
||||
* boost here.
|
||||
*/
|
||||
|
||||
sched_boost_type = sched_effective_boost();
|
||||
sysctl_sched_boost = sched_boost_type;
|
||||
set_boost_policy(sysctl_sched_boost);
|
||||
trace_sched_set_boost(sysctl_sched_boost);
|
||||
}
|
||||
|
||||
int sched_set_boost(int type)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
mutex_lock(&boost_mutex);
|
||||
if (verify_boost_params(type))
|
||||
_sched_set_boost(type);
|
||||
else
|
||||
ret = -EINVAL;
|
||||
mutex_unlock(&boost_mutex);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int sched_boost_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *lenp,
|
||||
loff_t *ppos)
|
||||
{
|
||||
int ret;
|
||||
unsigned int *data = (unsigned int *)table->data;
|
||||
|
||||
mutex_lock(&boost_mutex);
|
||||
|
||||
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
||||
|
||||
if (ret || !write)
|
||||
goto done;
|
||||
|
||||
if (verify_boost_params(*data))
|
||||
_sched_set_boost(*data);
|
||||
else
|
||||
ret = -EINVAL;
|
||||
|
||||
done:
|
||||
mutex_unlock(&boost_mutex);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void walt_boost_init(void)
|
||||
{
|
||||
/* force call the callbacks for default boost */
|
||||
sched_set_boost(FULL_THROTTLE_BOOST);
|
||||
}
|
||||
1360
kernel/sched/walt/core_ctl.c
Normal file
1360
kernel/sched/walt/core_ctl.c
Normal file
File diff suppressed because it is too large
Load Diff
1027
kernel/sched/walt/cpufreq_walt.c
Normal file
1027
kernel/sched/walt/cpufreq_walt.c
Normal file
File diff suppressed because it is too large
Load Diff
92
kernel/sched/walt/fixup.c
Normal file
92
kernel/sched/walt/fixup.c
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <trace/hooks/cpufreq.h>
|
||||
|
||||
#include "walt.h"
|
||||
|
||||
unsigned int cpuinfo_max_freq_cached;
|
||||
|
||||
char sched_lib_name[LIB_PATH_LENGTH];
|
||||
unsigned int sched_lib_mask_force;
|
||||
|
||||
static bool is_sched_lib_based_app(pid_t pid)
|
||||
{
|
||||
const char *name = NULL;
|
||||
char *libname, *lib_list;
|
||||
struct vm_area_struct *vma;
|
||||
char path_buf[LIB_PATH_LENGTH];
|
||||
char *tmp_lib_name;
|
||||
bool found = false;
|
||||
struct task_struct *p;
|
||||
struct mm_struct *mm;
|
||||
|
||||
if (strnlen(sched_lib_name, LIB_PATH_LENGTH) == 0)
|
||||
return false;
|
||||
|
||||
tmp_lib_name = kmalloc(LIB_PATH_LENGTH, GFP_KERNEL);
|
||||
if (!tmp_lib_name)
|
||||
return false;
|
||||
|
||||
rcu_read_lock();
|
||||
p = pid ? get_pid_task(find_vpid(pid), PIDTYPE_PID) : get_task_struct(current);
|
||||
rcu_read_unlock();
|
||||
if (!p) {
|
||||
kfree(tmp_lib_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
mm = get_task_mm(p);
|
||||
if (!mm)
|
||||
goto put_task_struct;
|
||||
|
||||
down_read(&mm->mmap_lock);
|
||||
for (vma = mm->mmap; vma ; vma = vma->vm_next) {
|
||||
if (vma->vm_file && vma->vm_flags & VM_EXEC) {
|
||||
name = d_path(&vma->vm_file->f_path,
|
||||
path_buf, LIB_PATH_LENGTH);
|
||||
if (IS_ERR(name))
|
||||
goto release_sem;
|
||||
|
||||
strlcpy(tmp_lib_name, sched_lib_name, LIB_PATH_LENGTH);
|
||||
lib_list = tmp_lib_name;
|
||||
while ((libname = strsep(&lib_list, ","))) {
|
||||
libname = skip_spaces(libname);
|
||||
if (strnstr(name, libname,
|
||||
strnlen(name, LIB_PATH_LENGTH))) {
|
||||
found = true;
|
||||
goto release_sem;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
release_sem:
|
||||
up_read(&mm->mmap_lock);
|
||||
mmput(mm);
|
||||
put_task_struct:
|
||||
put_task_struct(p);
|
||||
kfree(tmp_lib_name);
|
||||
return found;
|
||||
}
|
||||
|
||||
static void android_rvh_show_max_freq(void *unused, struct cpufreq_policy *policy,
|
||||
unsigned int *max_freq)
|
||||
{
|
||||
if (!cpuinfo_max_freq_cached)
|
||||
return;
|
||||
|
||||
if (!(BIT(policy->cpu) & sched_lib_mask_force))
|
||||
return;
|
||||
|
||||
if (is_sched_lib_based_app(current->pid))
|
||||
*max_freq = cpuinfo_max_freq_cached << 1;
|
||||
}
|
||||
|
||||
void walt_fixup_init(void)
|
||||
{
|
||||
register_trace_android_rvh_show_max_freq(android_rvh_show_max_freq, NULL);
|
||||
}
|
||||
300
kernel/sched/walt/input-boost.c
Normal file
300
kernel/sched/walt/input-boost.c
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2013-2015,2017,2019-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#define pr_fmt(fmt) "input-boost: " fmt
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/cpufreq.h>
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/input.h>
|
||||
#include <linux/time.h>
|
||||
#include <linux/sysfs.h>
|
||||
#include <linux/pm_qos.h>
|
||||
|
||||
#include "walt.h"
|
||||
|
||||
#define input_boost_attr_rw(_name) \
|
||||
static struct kobj_attribute _name##_attr = \
|
||||
__ATTR(_name, 0644, show_##_name, store_##_name)
|
||||
|
||||
#define show_one(file_name) \
|
||||
static ssize_t show_##file_name \
|
||||
(struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
|
||||
{ \
|
||||
return scnprintf(buf, PAGE_SIZE, "%u\n", file_name); \
|
||||
}
|
||||
|
||||
#define store_one(file_name) \
|
||||
static ssize_t store_##file_name \
|
||||
(struct kobject *kobj, struct kobj_attribute *attr, \
|
||||
const char *buf, size_t count) \
|
||||
{ \
|
||||
\
|
||||
sscanf(buf, "%u", &file_name); \
|
||||
return count; \
|
||||
}
|
||||
|
||||
struct cpu_sync {
|
||||
int cpu;
|
||||
unsigned int input_boost_min;
|
||||
unsigned int input_boost_freq;
|
||||
};
|
||||
|
||||
static DEFINE_PER_CPU(struct cpu_sync, sync_info);
|
||||
static struct workqueue_struct *input_boost_wq;
|
||||
|
||||
static struct work_struct input_boost_work;
|
||||
|
||||
static bool sched_boost_active;
|
||||
|
||||
static struct delayed_work input_boost_rem;
|
||||
static u64 last_input_time;
|
||||
#define MIN_INPUT_INTERVAL (150 * USEC_PER_MSEC)
|
||||
|
||||
static DEFINE_PER_CPU(struct freq_qos_request, qos_req);
|
||||
|
||||
static void boost_adjust_notify(struct cpufreq_policy *policy)
|
||||
{
|
||||
unsigned int cpu = policy->cpu;
|
||||
struct cpu_sync *s = &per_cpu(sync_info, cpu);
|
||||
unsigned int ib_min = s->input_boost_min;
|
||||
struct freq_qos_request *req = &per_cpu(qos_req, cpu);
|
||||
int ret;
|
||||
|
||||
pr_debug("CPU%u policy min before boost: %u kHz\n",
|
||||
cpu, policy->min);
|
||||
pr_debug("CPU%u boost min: %u kHz\n", cpu, ib_min);
|
||||
|
||||
ret = freq_qos_update_request(req, ib_min);
|
||||
|
||||
if (ret < 0)
|
||||
pr_err("Failed to update freq constraint in boost_adjust: %d\n",
|
||||
ib_min);
|
||||
|
||||
pr_debug("CPU%u policy min after boost: %u kHz\n", cpu, policy->min);
|
||||
}
|
||||
|
||||
static void update_policy_online(void)
|
||||
{
|
||||
unsigned int i;
|
||||
struct cpufreq_policy *policy;
|
||||
struct cpumask online_cpus;
|
||||
|
||||
/* Re-evaluate policy to trigger adjust notifier for online CPUs */
|
||||
cpus_read_lock();
|
||||
online_cpus = *cpu_online_mask;
|
||||
for_each_cpu(i, &online_cpus) {
|
||||
policy = cpufreq_cpu_get(i);
|
||||
if (!policy) {
|
||||
pr_err("%s: cpufreq policy not found for cpu%d\n",
|
||||
__func__, i);
|
||||
return;
|
||||
}
|
||||
|
||||
cpumask_andnot(&online_cpus, &online_cpus,
|
||||
policy->related_cpus);
|
||||
boost_adjust_notify(policy);
|
||||
}
|
||||
cpus_read_unlock();
|
||||
}
|
||||
|
||||
static void do_input_boost_rem(struct work_struct *work)
|
||||
{
|
||||
unsigned int i, ret;
|
||||
struct cpu_sync *i_sync_info;
|
||||
|
||||
/* Reset the input_boost_min for all CPUs in the system */
|
||||
pr_debug("Resetting input boost min for all CPUs\n");
|
||||
for_each_possible_cpu(i) {
|
||||
i_sync_info = &per_cpu(sync_info, i);
|
||||
i_sync_info->input_boost_min = 0;
|
||||
}
|
||||
|
||||
/* Update policies for all online CPUs */
|
||||
update_policy_online();
|
||||
|
||||
if (sched_boost_active) {
|
||||
ret = sched_set_boost(0);
|
||||
if (!ret)
|
||||
pr_err("input-boost: sched boost disable failed\n");
|
||||
sched_boost_active = false;
|
||||
}
|
||||
}
|
||||
|
||||
static void do_input_boost(struct work_struct *work)
|
||||
{
|
||||
unsigned int i, ret;
|
||||
struct cpu_sync *i_sync_info;
|
||||
|
||||
cancel_delayed_work_sync(&input_boost_rem);
|
||||
if (sched_boost_active) {
|
||||
sched_set_boost(0);
|
||||
sched_boost_active = false;
|
||||
}
|
||||
|
||||
/* Set the input_boost_min for all CPUs in the system */
|
||||
pr_debug("Setting input boost min for all CPUs\n");
|
||||
for (i = 0; i < 8; i++) {
|
||||
i_sync_info = &per_cpu(sync_info, i);
|
||||
i_sync_info->input_boost_min = sysctl_input_boost_freq[i];
|
||||
}
|
||||
|
||||
/* Update policies for all online CPUs */
|
||||
update_policy_online();
|
||||
|
||||
/* Enable scheduler boost to migrate tasks to big cluster */
|
||||
if (sysctl_sched_boost_on_input > 0) {
|
||||
ret = sched_set_boost(sysctl_sched_boost_on_input);
|
||||
if (ret)
|
||||
pr_err("input-boost: sched boost enable failed\n");
|
||||
else
|
||||
sched_boost_active = true;
|
||||
}
|
||||
|
||||
queue_delayed_work(input_boost_wq, &input_boost_rem,
|
||||
msecs_to_jiffies(sysctl_input_boost_ms));
|
||||
}
|
||||
|
||||
static void inputboost_input_event(struct input_handle *handle,
|
||||
unsigned int type, unsigned int code, int value)
|
||||
{
|
||||
u64 now;
|
||||
int cpu;
|
||||
int enabled = 0;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
if (sysctl_input_boost_freq[cpu] > 0) {
|
||||
enabled = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!enabled)
|
||||
return;
|
||||
|
||||
now = ktime_to_us(ktime_get());
|
||||
if (now - last_input_time < MIN_INPUT_INTERVAL)
|
||||
return;
|
||||
|
||||
if (work_pending(&input_boost_work))
|
||||
return;
|
||||
|
||||
queue_work(input_boost_wq, &input_boost_work);
|
||||
last_input_time = ktime_to_us(ktime_get());
|
||||
}
|
||||
|
||||
static int inputboost_input_connect(struct input_handler *handler,
|
||||
struct input_dev *dev, const struct input_device_id *id)
|
||||
{
|
||||
struct input_handle *handle;
|
||||
int error;
|
||||
|
||||
handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL);
|
||||
if (!handle)
|
||||
return -ENOMEM;
|
||||
|
||||
handle->dev = dev;
|
||||
handle->handler = handler;
|
||||
handle->name = "cpufreq";
|
||||
|
||||
error = input_register_handle(handle);
|
||||
if (error)
|
||||
goto err2;
|
||||
|
||||
error = input_open_device(handle);
|
||||
if (error)
|
||||
goto err1;
|
||||
|
||||
return 0;
|
||||
err1:
|
||||
input_unregister_handle(handle);
|
||||
err2:
|
||||
kfree(handle);
|
||||
return error;
|
||||
}
|
||||
|
||||
static void inputboost_input_disconnect(struct input_handle *handle)
|
||||
{
|
||||
input_close_device(handle);
|
||||
input_unregister_handle(handle);
|
||||
kfree(handle);
|
||||
}
|
||||
|
||||
static const struct input_device_id inputboost_ids[] = {
|
||||
/* multi-touch touchscreen */
|
||||
{
|
||||
.flags = INPUT_DEVICE_ID_MATCH_EVBIT |
|
||||
INPUT_DEVICE_ID_MATCH_ABSBIT,
|
||||
.evbit = { BIT_MASK(EV_ABS) },
|
||||
.absbit = { [BIT_WORD(ABS_MT_POSITION_X)] =
|
||||
BIT_MASK(ABS_MT_POSITION_X) |
|
||||
BIT_MASK(ABS_MT_POSITION_Y)
|
||||
},
|
||||
},
|
||||
/* touchpad */
|
||||
{
|
||||
.flags = INPUT_DEVICE_ID_MATCH_KEYBIT |
|
||||
INPUT_DEVICE_ID_MATCH_ABSBIT,
|
||||
.keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) },
|
||||
.absbit = { [BIT_WORD(ABS_X)] =
|
||||
BIT_MASK(ABS_X) | BIT_MASK(ABS_Y)
|
||||
},
|
||||
},
|
||||
/* Keypad */
|
||||
{
|
||||
.flags = INPUT_DEVICE_ID_MATCH_EVBIT,
|
||||
.evbit = { BIT_MASK(EV_KEY) },
|
||||
},
|
||||
{ },
|
||||
};
|
||||
|
||||
static struct input_handler inputboost_input_handler = {
|
||||
.event = inputboost_input_event,
|
||||
.connect = inputboost_input_connect,
|
||||
.disconnect = inputboost_input_disconnect,
|
||||
.name = "input-boost",
|
||||
.id_table = inputboost_ids,
|
||||
};
|
||||
|
||||
struct kobject *input_boost_kobj;
|
||||
int input_boost_init(void)
|
||||
{
|
||||
int cpu, ret;
|
||||
struct cpu_sync *s;
|
||||
struct cpufreq_policy *policy;
|
||||
struct freq_qos_request *req;
|
||||
|
||||
input_boost_wq = alloc_workqueue("inputboost_wq", WQ_HIGHPRI, 0);
|
||||
if (!input_boost_wq)
|
||||
return -EFAULT;
|
||||
|
||||
INIT_WORK(&input_boost_work, do_input_boost);
|
||||
INIT_DELAYED_WORK(&input_boost_rem, do_input_boost_rem);
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
s = &per_cpu(sync_info, cpu);
|
||||
s->cpu = cpu;
|
||||
req = &per_cpu(qos_req, cpu);
|
||||
policy = cpufreq_cpu_get(cpu);
|
||||
if (!policy) {
|
||||
pr_err("%s: cpufreq policy not found for cpu%d\n",
|
||||
__func__, cpu);
|
||||
return -ESRCH;
|
||||
}
|
||||
|
||||
ret = freq_qos_add_request(&policy->constraints, req,
|
||||
FREQ_QOS_MIN, policy->min);
|
||||
if (ret < 0) {
|
||||
pr_err("%s: Failed to add freq constraint (%d)\n",
|
||||
__func__, ret);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
ret = input_register_handler(&inputboost_input_handler);
|
||||
return 0;
|
||||
}
|
||||
178
kernel/sched/walt/perf_trace_counters.h
Normal file
178
kernel/sched/walt/perf_trace_counters.h
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2013-2014, 2017, 2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#undef TRACE_SYSTEM
|
||||
#define TRACE_SYSTEM perf_trace_counters
|
||||
|
||||
#if !defined(_PERF_TRACE_COUNTERS_H_) || defined(TRACE_HEADER_MULTI_READ)
|
||||
#define _PERF_TRACE_COUNTERS_H_
|
||||
|
||||
/* Ctr index for PMCNTENSET/CLR */
|
||||
#define CC 0x80000000
|
||||
#define C0 0x1
|
||||
#define C1 0x2
|
||||
#define C2 0x4
|
||||
#define C3 0x8
|
||||
#define C4 0x10
|
||||
#define C5 0x20
|
||||
#define C_ALL (CC | C0 | C1 | C2 | C3 | C4 | C5)
|
||||
#define TYPE_MASK 0xFFFF
|
||||
#define NUM_L1_CTRS 6
|
||||
#define NUM_AMU_CTRS 2
|
||||
|
||||
#include <linux/sched.h>
|
||||
#include <linux/cpumask.h>
|
||||
#include <linux/tracepoint.h>
|
||||
|
||||
DECLARE_PER_CPU(u32, cntenset_val);
|
||||
DECLARE_PER_CPU(unsigned long, previous_ccnt);
|
||||
DECLARE_PER_CPU(unsigned long[NUM_L1_CTRS], previous_l1_cnts);
|
||||
DECLARE_PER_CPU(unsigned long[NUM_AMU_CTRS], previous_amu_cnts);
|
||||
TRACE_EVENT(sched_switch_with_ctrs,
|
||||
|
||||
TP_PROTO(pid_t prev, pid_t next),
|
||||
|
||||
TP_ARGS(prev, next),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(pid_t, old_pid)
|
||||
__field(pid_t, new_pid)
|
||||
__field(unsigned long, cctr)
|
||||
__field(unsigned long, ctr0)
|
||||
__field(unsigned long, ctr1)
|
||||
__field(unsigned long, ctr2)
|
||||
__field(unsigned long, ctr3)
|
||||
__field(unsigned long, ctr4)
|
||||
__field(unsigned long, ctr5)
|
||||
__field(unsigned long, amu0)
|
||||
__field(unsigned long, amu1)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
u32 cpu = smp_processor_id();
|
||||
u32 i;
|
||||
u32 cnten_val;
|
||||
unsigned long total_ccnt = 0;
|
||||
unsigned long total_cnt = 0;
|
||||
unsigned long amu_cnt = 0;
|
||||
unsigned long delta_l1_cnts[NUM_L1_CTRS] = {0};
|
||||
unsigned long delta_amu_cnts[NUM_AMU_CTRS] = {0};
|
||||
|
||||
__entry->old_pid = prev;
|
||||
__entry->new_pid = next;
|
||||
|
||||
cnten_val = per_cpu(cntenset_val, cpu);
|
||||
|
||||
if (cnten_val & CC) {
|
||||
/* Read value */
|
||||
total_ccnt = read_sysreg(pmccntr_el0);
|
||||
__entry->cctr = total_ccnt -
|
||||
per_cpu(previous_ccnt, cpu);
|
||||
per_cpu(previous_ccnt, cpu) = total_ccnt;
|
||||
}
|
||||
for (i = 0; i < NUM_L1_CTRS; i++) {
|
||||
if (cnten_val & (1 << i)) {
|
||||
/* Select */
|
||||
write_sysreg(i, pmselr_el0);
|
||||
isb();
|
||||
/* Read value */
|
||||
total_cnt = read_sysreg(pmxevcntr_el0);
|
||||
delta_l1_cnts[i] = total_cnt -
|
||||
per_cpu(previous_l1_cnts[i], cpu);
|
||||
per_cpu(previous_l1_cnts[i], cpu) =
|
||||
total_cnt;
|
||||
} else
|
||||
delta_l1_cnts[i] = 0;
|
||||
}
|
||||
|
||||
if (IS_ENABLED(CONFIG_ARM64_AMU_EXTN)) {
|
||||
amu_cnt = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0);
|
||||
delta_amu_cnts[0] = amu_cnt -
|
||||
per_cpu(previous_amu_cnts[0], cpu);
|
||||
per_cpu(previous_amu_cnts[0], cpu) = amu_cnt;
|
||||
|
||||
amu_cnt = read_sysreg_s(SYS_AMEVCNTR0_INST_RET_EL0);
|
||||
delta_amu_cnts[1] = amu_cnt -
|
||||
per_cpu(previous_amu_cnts[1], cpu);
|
||||
per_cpu(previous_amu_cnts[1], cpu) = amu_cnt;
|
||||
}
|
||||
|
||||
__entry->ctr0 = delta_l1_cnts[0];
|
||||
__entry->ctr1 = delta_l1_cnts[1];
|
||||
__entry->ctr2 = delta_l1_cnts[2];
|
||||
__entry->ctr3 = delta_l1_cnts[3];
|
||||
__entry->ctr4 = delta_l1_cnts[4];
|
||||
__entry->ctr5 = delta_l1_cnts[5];
|
||||
__entry->amu0 = delta_amu_cnts[0];
|
||||
__entry->amu1 = delta_amu_cnts[1];
|
||||
),
|
||||
|
||||
TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %lu, CTR0: %lu, CTR1: %lu, CTR2: %lu, CTR3: %lu, CTR4: %lu, CTR5: %lu, CYC: %lu, INST: %lu",
|
||||
__entry->old_pid, __entry->new_pid,
|
||||
__entry->cctr,
|
||||
__entry->ctr0, __entry->ctr1,
|
||||
__entry->ctr2, __entry->ctr3,
|
||||
__entry->ctr4, __entry->ctr5,
|
||||
__entry->amu0, __entry->amu1)
|
||||
);
|
||||
|
||||
TRACE_EVENT(sched_switch_ctrs_cfg,
|
||||
|
||||
TP_PROTO(int cpu),
|
||||
|
||||
TP_ARGS(cpu),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(int, cpu)
|
||||
__field(unsigned long, ctr0)
|
||||
__field(unsigned long, ctr1)
|
||||
__field(unsigned long, ctr2)
|
||||
__field(unsigned long, ctr3)
|
||||
__field(unsigned long, ctr4)
|
||||
__field(unsigned long, ctr5)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
u32 i;
|
||||
u32 cnten_val;
|
||||
u32 ctr_type[NUM_L1_CTRS] = {0};
|
||||
|
||||
cnten_val = per_cpu(cntenset_val, cpu);
|
||||
|
||||
for (i = 0; i < NUM_L1_CTRS; i++) {
|
||||
if (cnten_val & (1 << i)) {
|
||||
/* Select */
|
||||
write_sysreg(i, pmselr_el0);
|
||||
isb();
|
||||
/* Read type */
|
||||
ctr_type[i] = read_sysreg(pmxevtyper_el0)
|
||||
& TYPE_MASK;
|
||||
} else
|
||||
ctr_type[i] = 0;
|
||||
}
|
||||
|
||||
__entry->cpu = cpu;
|
||||
__entry->ctr0 = ctr_type[0];
|
||||
__entry->ctr1 = ctr_type[1];
|
||||
__entry->ctr2 = ctr_type[2];
|
||||
__entry->ctr3 = ctr_type[3];
|
||||
__entry->ctr4 = ctr_type[4];
|
||||
__entry->ctr5 = ctr_type[5];
|
||||
),
|
||||
|
||||
TP_printk("cpu=%d CTR0=%lu CTR1=%lu CTR2=%lu CTR3=%lu CTR4=%lu CTR5=%lu",
|
||||
__entry->cpu,
|
||||
__entry->ctr0, __entry->ctr1,
|
||||
__entry->ctr2, __entry->ctr3,
|
||||
__entry->ctr4, __entry->ctr5)
|
||||
);
|
||||
|
||||
#endif
|
||||
#undef TRACE_INCLUDE_PATH
|
||||
#define TRACE_INCLUDE_PATH ../../kernel/sched/walt
|
||||
|
||||
#undef TRACE_INCLUDE_FILE
|
||||
#define TRACE_INCLUDE_FILE perf_trace_counters
|
||||
#include <trace/define_trace.h>
|
||||
177
kernel/sched/walt/preemptirq_long.c
Normal file
177
kernel/sched/walt/preemptirq_long.c
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/ftrace.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/sysctl.h>
|
||||
#include <linux/printk.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/sched/clock.h>
|
||||
#include <trace/hooks/preemptirq.h>
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include "preemptirq_long.h"
|
||||
|
||||
#define IRQSOFF_SENTINEL 0x0fffDEAD
|
||||
|
||||
static unsigned int sysctl_preemptoff_tracing_threshold_ns = 1000000;
|
||||
static unsigned int sysctl_irqsoff_tracing_threshold_ns = 5000000;
|
||||
static unsigned int sysctl_irqsoff_dmesg_output_enabled;
|
||||
static unsigned int sysctl_irqsoff_crash_sentinel_value;
|
||||
static unsigned int sysctl_irqsoff_crash_threshold_ns = 10000000;
|
||||
|
||||
static unsigned int half_million = 500000;
|
||||
static unsigned int one_hundred_million = 100000000;
|
||||
static unsigned int one_million = 1000000;
|
||||
|
||||
static DEFINE_PER_CPU(u64, irq_disabled_ts);
|
||||
|
||||
/*
|
||||
* preemption disable tracking require additional context
|
||||
* to rule out false positives. see the comment in
|
||||
* test_preempt_disable_long() for more details.
|
||||
*/
|
||||
struct preempt_store {
|
||||
u64 ts;
|
||||
int pid;
|
||||
unsigned long ncsw;
|
||||
};
|
||||
static DEFINE_PER_CPU(struct preempt_store, the_ps);
|
||||
|
||||
static void note_irq_disable(void *u1, unsigned long u2, unsigned long u3)
|
||||
{
|
||||
if (is_idle_task(current))
|
||||
return;
|
||||
|
||||
/*
|
||||
* We just have to note down the time stamp here. We
|
||||
* use stacktrace trigger feature to print the stacktrace.
|
||||
*/
|
||||
this_cpu_write(irq_disabled_ts, sched_clock());
|
||||
}
|
||||
|
||||
static void test_irq_disable_long(void *u1, unsigned long ip, unsigned long parent_ip)
|
||||
{
|
||||
u64 ts = this_cpu_read(irq_disabled_ts);
|
||||
|
||||
if (!ts)
|
||||
return;
|
||||
|
||||
this_cpu_write(irq_disabled_ts, 0);
|
||||
ts = sched_clock() - ts;
|
||||
|
||||
if (ts > sysctl_irqsoff_tracing_threshold_ns) {
|
||||
trace_irq_disable_long(ts, ip, parent_ip, CALLER_ADDR4, CALLER_ADDR5);
|
||||
|
||||
if (sysctl_irqsoff_dmesg_output_enabled == IRQSOFF_SENTINEL)
|
||||
printk_deferred("irqs off exceeds thresh delta=%llu C:(%ps<-%ps<-%ps<-%ps)\n",
|
||||
ts, (void *)CALLER_ADDR2,
|
||||
(void *)CALLER_ADDR3,
|
||||
(void *)CALLER_ADDR4,
|
||||
(void *)CALLER_ADDR5);
|
||||
}
|
||||
|
||||
if (sysctl_irqsoff_crash_sentinel_value == IRQSOFF_SENTINEL &&
|
||||
ts > sysctl_irqsoff_crash_threshold_ns) {
|
||||
printk_deferred("delta=%llu(ns) > crash_threshold=%u(ns) Task=%s\n",
|
||||
ts, sysctl_irqsoff_crash_threshold_ns,
|
||||
current->comm);
|
||||
BUG_ON(1);
|
||||
}
|
||||
}
|
||||
|
||||
static void note_preempt_disable(void *u1, unsigned long u2, unsigned long u3)
|
||||
{
|
||||
struct preempt_store *ps = &per_cpu(the_ps, raw_smp_processor_id());
|
||||
|
||||
ps->ts = sched_clock();
|
||||
ps->pid = current->pid;
|
||||
ps->ncsw = current->nvcsw + current->nivcsw;
|
||||
}
|
||||
|
||||
static void test_preempt_disable_long(void *u1, unsigned long ip,
|
||||
unsigned long parent_ip)
|
||||
{
|
||||
struct preempt_store *ps = &per_cpu(the_ps, raw_smp_processor_id());
|
||||
u64 delta = 0;
|
||||
|
||||
if (!ps->ts)
|
||||
return;
|
||||
|
||||
/*
|
||||
* schedule() calls __schedule() with preemption disabled.
|
||||
* if we had entered idle and exiting idle now, we think
|
||||
* preemption is disabled the whole time. Detect this by
|
||||
* checking if the preemption is disabled across the same
|
||||
* task. There is a possiblity that the same task is scheduled
|
||||
* after idle. To rule out this possibility, compare the
|
||||
* context switch count also.
|
||||
*/
|
||||
if (ps->pid == current->pid && (ps->ncsw == current->nvcsw +
|
||||
current->nivcsw))
|
||||
delta = sched_clock() - ps->ts;
|
||||
|
||||
ps->ts = 0;
|
||||
if (delta > sysctl_preemptoff_tracing_threshold_ns)
|
||||
trace_preempt_disable_long(delta, ip, parent_ip, CALLER_ADDR4, CALLER_ADDR5);
|
||||
}
|
||||
|
||||
static struct ctl_table preemptirq_long_table[] = {
|
||||
{
|
||||
.procname = "preemptoff_tracing_threshold_ns",
|
||||
.data = &sysctl_preemptoff_tracing_threshold_ns,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec,
|
||||
},
|
||||
{
|
||||
.procname = "irqsoff_tracing_threshold_ns",
|
||||
.data = &sysctl_irqsoff_tracing_threshold_ns,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_douintvec_minmax,
|
||||
.extra1 = &half_million,
|
||||
.extra2 = &one_hundred_million,
|
||||
},
|
||||
{
|
||||
.procname = "irqsoff_dmesg_output_enabled",
|
||||
.data = &sysctl_irqsoff_dmesg_output_enabled,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec,
|
||||
},
|
||||
{
|
||||
.procname = "irqsoff_crash_sentinel_value",
|
||||
.data = &sysctl_irqsoff_crash_sentinel_value,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec,
|
||||
},
|
||||
{
|
||||
.procname = "irqsoff_crash_threshold_ns",
|
||||
.data = &sysctl_irqsoff_crash_threshold_ns,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_douintvec_minmax,
|
||||
.extra1 = &one_million,
|
||||
.extra2 = &one_hundred_million,
|
||||
},
|
||||
{ }
|
||||
};
|
||||
|
||||
int preemptirq_long_init(void)
|
||||
{
|
||||
if (!register_sysctl("preemptirq", preemptirq_long_table)) {
|
||||
pr_err("Fail to register sysctl table\n");
|
||||
return -EPERM;
|
||||
}
|
||||
|
||||
register_trace_android_rvh_irqs_disable(note_irq_disable, NULL);
|
||||
register_trace_android_rvh_irqs_enable(test_irq_disable_long, NULL);
|
||||
register_trace_android_rvh_preempt_disable(note_preempt_disable, NULL);
|
||||
register_trace_android_rvh_preempt_enable(test_preempt_disable_long,
|
||||
NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
59
kernel/sched/walt/preemptirq_long.h
Normal file
59
kernel/sched/walt/preemptirq_long.h
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2021 The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#undef TRACE_SYSTEM
|
||||
#define TRACE_SYSTEM preemptirq_long
|
||||
|
||||
#undef TRACE_INCLUDE_PATH
|
||||
#define TRACE_INCLUDE_PATH .
|
||||
|
||||
#if !defined(_TRACE_PREEMPTIRQ_LONG_H) || defined(TRACE_HEADER_MULTI_READ)
|
||||
#define _TRACE_PREEMPTIRQ_LONG_H
|
||||
|
||||
#include <linux/tracepoint.h>
|
||||
|
||||
/* reference preemptirq_template */
|
||||
DECLARE_EVENT_CLASS(preemptirq_long_template,
|
||||
|
||||
TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip,
|
||||
unsigned long pparent_ip, unsigned long ppparent_ip),
|
||||
|
||||
TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(u64, delta)
|
||||
__field(unsigned long, caller_offs)
|
||||
__field(unsigned long, parent_offs)
|
||||
__field(unsigned long, pparent_offs)
|
||||
__field(unsigned long, ppparent_offs)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->delta = delta;
|
||||
__entry->caller_offs = ip;
|
||||
__entry->parent_offs = parent_ip;
|
||||
__entry->pparent_offs = pparent_ip;
|
||||
__entry->ppparent_offs = ppparent_ip;
|
||||
),
|
||||
|
||||
TP_printk("delta=%llu(ns) caller=%ps <- %ps <- %ps <- %ps",
|
||||
__entry->delta, __entry->caller_offs,
|
||||
__entry->parent_offs, __entry->pparent_offs, __entry->ppparent_offs)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(preemptirq_long_template, irq_disable_long,
|
||||
TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip,
|
||||
unsigned long pparent_ip, unsigned long ppparent_ip),
|
||||
TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip));
|
||||
|
||||
DEFINE_EVENT(preemptirq_long_template, preempt_disable_long,
|
||||
TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip,
|
||||
unsigned long pparent_ip, unsigned long ppparent_ip),
|
||||
TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip));
|
||||
|
||||
#endif /* _TRACE_PREEMPTIRQ_LONG_H */
|
||||
|
||||
/* This part must be outside protection */
|
||||
#include <trace/define_trace.h>
|
||||
338
kernel/sched/walt/sched_avg.c
Normal file
338
kernel/sched/walt/sched_avg.c
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2012, 2015-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Scheduler hook for average runqueue determination
|
||||
*/
|
||||
#include <linux/module.h>
|
||||
#include <linux/percpu.h>
|
||||
#include <linux/hrtimer.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/math64.h>
|
||||
|
||||
#include "walt.h"
|
||||
#include "trace.h"
|
||||
|
||||
static DEFINE_PER_CPU(u64, nr_prod_sum);
|
||||
static DEFINE_PER_CPU(u64, last_time);
|
||||
static DEFINE_PER_CPU(u64, nr_big_prod_sum);
|
||||
static DEFINE_PER_CPU(u64, nr);
|
||||
static DEFINE_PER_CPU(u64, nr_max);
|
||||
|
||||
static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock);
|
||||
static s64 last_get_time;
|
||||
|
||||
static DEFINE_PER_CPU(atomic64_t, busy_hyst_end_time) = ATOMIC64_INIT(0);
|
||||
|
||||
static DEFINE_PER_CPU(u64, hyst_time);
|
||||
static DEFINE_PER_CPU(u64, coloc_hyst_busy);
|
||||
static DEFINE_PER_CPU(u64, coloc_hyst_time);
|
||||
static DEFINE_PER_CPU(u64, util_hyst_time);
|
||||
|
||||
#define NR_THRESHOLD_PCT 40
|
||||
#define MAX_RTGB_TIME (sysctl_sched_coloc_busy_hyst_max_ms * NSEC_PER_MSEC)
|
||||
|
||||
struct sched_avg_stats stats[WALT_NR_CPUS];
|
||||
unsigned int cstats_util_pct[MAX_CLUSTERS];
|
||||
|
||||
/**
|
||||
* sched_get_cluster_util_pct
|
||||
* @return: provide the percentage of this cluter that was used in the
|
||||
* previous window.
|
||||
*
|
||||
* This routine may be called any number of times as needed during
|
||||
* a window, but will always return the same result until window
|
||||
* rollover.
|
||||
*/
|
||||
unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster)
|
||||
{
|
||||
unsigned int cluster_util_pct = 0;
|
||||
|
||||
if (cluster->id < MAX_CLUSTERS)
|
||||
cluster_util_pct = cstats_util_pct[cluster->id];
|
||||
|
||||
return cluster_util_pct;
|
||||
}
|
||||
|
||||
/**
|
||||
* sched_get_nr_running_avg
|
||||
* @return: Average nr_running, iowait and nr_big_tasks value since last poll.
|
||||
* Returns the avg * 100 to return up to two decimal points
|
||||
* of accuracy.
|
||||
*
|
||||
* Obtains the average nr_running value since the last poll.
|
||||
* This function may not be called concurrently with itself.
|
||||
*
|
||||
* It is assumed that this function is called at most once per window
|
||||
* rollover.
|
||||
*/
|
||||
struct sched_avg_stats *sched_get_nr_running_avg(void)
|
||||
{
|
||||
int cpu;
|
||||
u64 curr_time = sched_clock();
|
||||
u64 period = curr_time - last_get_time;
|
||||
u64 tmp_nr, tmp_misfit;
|
||||
bool any_hyst_time = false;
|
||||
struct walt_sched_cluster *cluster;
|
||||
|
||||
if (!period)
|
||||
goto done;
|
||||
|
||||
/* read and reset nr_running counts */
|
||||
for_each_possible_cpu(cpu) {
|
||||
unsigned long flags;
|
||||
u64 diff;
|
||||
|
||||
spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);
|
||||
curr_time = sched_clock();
|
||||
diff = curr_time - per_cpu(last_time, cpu);
|
||||
BUG_ON((s64)diff < 0);
|
||||
|
||||
tmp_nr = per_cpu(nr_prod_sum, cpu);
|
||||
tmp_nr += per_cpu(nr, cpu) * diff;
|
||||
tmp_nr = div64_u64((tmp_nr * 100), period);
|
||||
|
||||
tmp_misfit = per_cpu(nr_big_prod_sum, cpu);
|
||||
tmp_misfit += walt_big_tasks(cpu) * diff;
|
||||
tmp_misfit = div64_u64((tmp_misfit * 100), period);
|
||||
|
||||
/*
|
||||
* NR_THRESHOLD_PCT is to make sure that the task ran
|
||||
* at least 85% in the last window to compensate any
|
||||
* over estimating being done.
|
||||
*/
|
||||
stats[cpu].nr = (int)div64_u64((tmp_nr + NR_THRESHOLD_PCT),
|
||||
100);
|
||||
stats[cpu].nr_misfit = (int)div64_u64((tmp_misfit +
|
||||
NR_THRESHOLD_PCT), 100);
|
||||
stats[cpu].nr_max = per_cpu(nr_max, cpu);
|
||||
stats[cpu].nr_scaled = tmp_nr;
|
||||
|
||||
trace_sched_get_nr_running_avg(cpu, stats[cpu].nr,
|
||||
stats[cpu].nr_misfit, stats[cpu].nr_max,
|
||||
stats[cpu].nr_scaled);
|
||||
|
||||
per_cpu(last_time, cpu) = curr_time;
|
||||
per_cpu(nr_prod_sum, cpu) = 0;
|
||||
per_cpu(nr_big_prod_sum, cpu) = 0;
|
||||
per_cpu(nr_max, cpu) = per_cpu(nr, cpu);
|
||||
|
||||
spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
|
||||
}
|
||||
|
||||
/* collect cluster load stats */
|
||||
for_each_sched_cluster(cluster) {
|
||||
unsigned int num_cpus = cpumask_weight(&cluster->cpus);
|
||||
unsigned int sum_util_pct = 0;
|
||||
|
||||
/* load is already scaled, see freq_policy_load/prev_runnable_sum */
|
||||
for_each_cpu(cpu, &cluster->cpus) {
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
|
||||
|
||||
/* compute the % this cpu's utilization of the cpu capacity,
|
||||
* and sum it across all cpus
|
||||
*/
|
||||
sum_util_pct +=
|
||||
(wrq->util * 100) / arch_scale_cpu_capacity(cpu);
|
||||
}
|
||||
|
||||
/* calculate the averge per-cpu utilization */
|
||||
cstats_util_pct[cluster->id] = sum_util_pct / num_cpus;
|
||||
}
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
if (per_cpu(coloc_hyst_time, cpu)) {
|
||||
any_hyst_time = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (any_hyst_time && get_rtgb_active_time() >= MAX_RTGB_TIME)
|
||||
sched_update_hyst_times();
|
||||
|
||||
last_get_time = curr_time;
|
||||
|
||||
done:
|
||||
return &stats[0];
|
||||
}
|
||||
EXPORT_SYMBOL(sched_get_nr_running_avg);
|
||||
|
||||
void sched_update_hyst_times(void)
|
||||
{
|
||||
bool rtgb_active;
|
||||
int cpu;
|
||||
unsigned long cpu_cap, coloc_busy_pct;
|
||||
|
||||
rtgb_active = is_rtgb_active() && (sched_boost_type != CONSERVATIVE_BOOST)
|
||||
&& (get_rtgb_active_time() < MAX_RTGB_TIME);
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
cpu_cap = arch_scale_cpu_capacity(cpu);
|
||||
coloc_busy_pct = sysctl_sched_coloc_busy_hyst_cpu_busy_pct[cpu];
|
||||
per_cpu(hyst_time, cpu) = (BIT(cpu)
|
||||
& sysctl_sched_busy_hyst_enable_cpus) ?
|
||||
sysctl_sched_busy_hyst : 0;
|
||||
per_cpu(coloc_hyst_time, cpu) = ((BIT(cpu)
|
||||
& sysctl_sched_coloc_busy_hyst_enable_cpus)
|
||||
&& rtgb_active) ?
|
||||
sysctl_sched_coloc_busy_hyst_cpu[cpu] : 0;
|
||||
per_cpu(coloc_hyst_busy, cpu) = mult_frac(cpu_cap,
|
||||
coloc_busy_pct, 100);
|
||||
per_cpu(util_hyst_time, cpu) = (BIT(cpu)
|
||||
& sysctl_sched_util_busy_hyst_enable_cpus) ?
|
||||
sysctl_sched_util_busy_hyst_cpu[cpu] : 0;
|
||||
}
|
||||
}
|
||||
|
||||
#define BUSY_NR_RUN 3
|
||||
#define BUSY_LOAD_FACTOR 10
|
||||
static inline void update_busy_hyst_end_time(int cpu, int enq,
|
||||
unsigned long prev_nr_run, u64 curr_time)
|
||||
{
|
||||
bool nr_run_trigger = false;
|
||||
bool load_trigger = false, coloc_load_trigger = false;
|
||||
u64 agg_hyst_time, total_util = 0;
|
||||
bool util_load_trigger = false;
|
||||
int i;
|
||||
bool hyst_trigger, coloc_trigger;
|
||||
bool dequeue = (enq < 0);
|
||||
|
||||
if (!per_cpu(hyst_time, cpu) && !per_cpu(coloc_hyst_time, cpu) &&
|
||||
!per_cpu(util_hyst_time, cpu))
|
||||
return;
|
||||
|
||||
if (prev_nr_run >= BUSY_NR_RUN && per_cpu(nr, cpu) < BUSY_NR_RUN)
|
||||
nr_run_trigger = true;
|
||||
|
||||
if (dequeue && (cpu_util(cpu) * BUSY_LOAD_FACTOR) >
|
||||
capacity_orig_of(cpu))
|
||||
load_trigger = true;
|
||||
|
||||
if (dequeue && cpu_util(cpu) > per_cpu(coloc_hyst_busy, cpu))
|
||||
coloc_load_trigger = true;
|
||||
|
||||
if (dequeue) {
|
||||
for_each_possible_cpu(i) {
|
||||
total_util += cpu_util(i);
|
||||
if (total_util >= sysctl_sched_util_busy_hyst_cpu_util[cpu]) {
|
||||
util_load_trigger = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
coloc_trigger = nr_run_trigger || coloc_load_trigger;
|
||||
#if IS_ENABLED(CONFIG_SCHED_CONSERVATIVE_BOOST_LPM_BIAS)
|
||||
hyst_trigger = nr_run_trigger || load_trigger || (sched_boost_type == CONSERVATIVE_BOOST);
|
||||
#else
|
||||
hyst_trigger = nr_run_trigger || load_trigger;
|
||||
#endif
|
||||
|
||||
agg_hyst_time = max(max(hyst_trigger ? per_cpu(hyst_time, cpu) : 0,
|
||||
coloc_trigger ? per_cpu(coloc_hyst_time, cpu) : 0),
|
||||
util_load_trigger ? per_cpu(util_hyst_time, cpu) : 0);
|
||||
|
||||
if (agg_hyst_time) {
|
||||
atomic64_set(&per_cpu(busy_hyst_end_time, cpu),
|
||||
curr_time + agg_hyst_time);
|
||||
trace_sched_busy_hyst_time(cpu, agg_hyst_time, prev_nr_run,
|
||||
cpu_util(cpu), per_cpu(hyst_time, cpu),
|
||||
per_cpu(coloc_hyst_time, cpu),
|
||||
per_cpu(util_hyst_time, cpu));
|
||||
}
|
||||
}
|
||||
|
||||
int sched_busy_hyst_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *lenp, loff_t *ppos)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (table->maxlen > (sizeof(unsigned int) * num_possible_cpus()))
|
||||
table->maxlen = sizeof(unsigned int) * num_possible_cpus();
|
||||
|
||||
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
||||
|
||||
if (!ret && write)
|
||||
sched_update_hyst_times();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* sched_update_nr_prod
|
||||
* @cpu: The core id of the nr running driver.
|
||||
* @enq: enqueue/dequeue/misfit happening on this CPU.
|
||||
* @return: N/A
|
||||
*
|
||||
* Update average with latest nr_running value for CPU
|
||||
*/
|
||||
void sched_update_nr_prod(int cpu, int enq)
|
||||
{
|
||||
u64 diff;
|
||||
u64 curr_time;
|
||||
unsigned long flags, nr_running;
|
||||
|
||||
spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);
|
||||
nr_running = per_cpu(nr, cpu);
|
||||
curr_time = sched_clock();
|
||||
diff = curr_time - per_cpu(last_time, cpu);
|
||||
BUG_ON((s64)diff < 0);
|
||||
per_cpu(last_time, cpu) = curr_time;
|
||||
per_cpu(nr, cpu) = cpu_rq(cpu)->nr_running;
|
||||
|
||||
if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu))
|
||||
per_cpu(nr_max, cpu) = per_cpu(nr, cpu);
|
||||
|
||||
/* Don't update hyst time for misfit tasks */
|
||||
if (enq)
|
||||
update_busy_hyst_end_time(cpu, enq, nr_running, curr_time);
|
||||
|
||||
per_cpu(nr_prod_sum, cpu) += nr_running * diff;
|
||||
per_cpu(nr_big_prod_sum, cpu) += walt_big_tasks(cpu) * diff;
|
||||
spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns the CPU utilization % in the last window.
|
||||
*/
|
||||
unsigned int sched_get_cpu_util_pct(int cpu)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
u64 util;
|
||||
unsigned long capacity, flags;
|
||||
unsigned int busy;
|
||||
struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1;
|
||||
|
||||
raw_spin_lock_irqsave(&rq->__lock, flags);
|
||||
|
||||
capacity = capacity_orig_of(cpu);
|
||||
|
||||
util = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum;
|
||||
util = scale_time_to_util(util);
|
||||
raw_spin_unlock_irqrestore(&rq->__lock, flags);
|
||||
|
||||
util = (util >= capacity) ? capacity : util;
|
||||
busy = div64_ul((util * 100), capacity);
|
||||
return busy;
|
||||
}
|
||||
|
||||
int sched_lpm_disallowed_time(int cpu, u64 *timeout)
|
||||
{
|
||||
u64 now = sched_clock();
|
||||
u64 bias_end_time = atomic64_read(&per_cpu(busy_hyst_end_time, cpu));
|
||||
|
||||
if (unlikely(is_reserved(cpu))) {
|
||||
*timeout = 10 * NSEC_PER_MSEC;
|
||||
return 0; /* shallowest c-state */
|
||||
}
|
||||
|
||||
if (now < bias_end_time) {
|
||||
*timeout = bias_end_time - now;
|
||||
return 0; /* shallowest c-state */
|
||||
}
|
||||
|
||||
return INT_MAX; /* don't care */
|
||||
}
|
||||
EXPORT_SYMBOL(sched_lpm_disallowed_time);
|
||||
979
kernel/sched/walt/sysctl.c
Normal file
979
kernel/sched/walt/sysctl.c
Normal file
|
|
@ -0,0 +1,979 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <trace/hooks/sched.h>
|
||||
|
||||
#include "walt.h"
|
||||
#include "trace.h"
|
||||
|
||||
static int neg_three = -3;
|
||||
static int three = 3;
|
||||
static int two_hundred_fifty_five = 255;
|
||||
static unsigned int ns_per_sec = NSEC_PER_SEC;
|
||||
static unsigned int one_hundred_thousand = 100000;
|
||||
static unsigned int two_hundred_million = 200000000;
|
||||
static int __maybe_unused two = 2;
|
||||
static int __maybe_unused four = 4;
|
||||
static int one_hundred = 100;
|
||||
static int one_thousand = 1000;
|
||||
static int two_thousand = 2000;
|
||||
|
||||
/*
|
||||
* CFS task prio range is [100 ... 139]
|
||||
* 120 is the default prio.
|
||||
* RTG boost range is [100 ... 119] because giving
|
||||
* boost for [120 .. 139] does not make sense.
|
||||
* 99 means disabled and it is the default value.
|
||||
*/
|
||||
static unsigned int min_cfs_boost_prio = 99;
|
||||
static unsigned int max_cfs_boost_prio = 119;
|
||||
|
||||
unsigned int sysctl_sched_capacity_margin_up_pct[MAX_MARGIN_LEVELS];
|
||||
unsigned int sysctl_sched_capacity_margin_dn_pct[MAX_MARGIN_LEVELS];
|
||||
unsigned int sysctl_sched_busy_hyst_enable_cpus;
|
||||
unsigned int sysctl_sched_busy_hyst;
|
||||
unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus;
|
||||
unsigned int sysctl_sched_coloc_busy_hyst_cpu[WALT_NR_CPUS];
|
||||
unsigned int sysctl_sched_coloc_busy_hyst_max_ms;
|
||||
unsigned int sysctl_sched_coloc_busy_hyst_cpu_busy_pct[WALT_NR_CPUS];
|
||||
unsigned int sysctl_sched_util_busy_hyst_enable_cpus;
|
||||
unsigned int sysctl_sched_util_busy_hyst_cpu[WALT_NR_CPUS];
|
||||
unsigned int sysctl_sched_util_busy_hyst_cpu_util[WALT_NR_CPUS];
|
||||
unsigned int sysctl_sched_boost;
|
||||
unsigned int sysctl_sched_wake_up_idle[2];
|
||||
unsigned int sysctl_input_boost_ms;
|
||||
unsigned int sysctl_input_boost_freq[8];
|
||||
unsigned int sysctl_sched_boost_on_input;
|
||||
|
||||
/* sysctl nodes accesed by other files */
|
||||
unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns;
|
||||
unsigned int __read_mostly sysctl_sched_group_downmigrate_pct;
|
||||
unsigned int __read_mostly sysctl_sched_group_upmigrate_pct;
|
||||
unsigned int __read_mostly sysctl_sched_window_stats_policy;
|
||||
unsigned int sysctl_sched_ravg_window_nr_ticks;
|
||||
unsigned int sysctl_sched_walt_rotate_big_tasks;
|
||||
unsigned int sysctl_sched_task_unfilter_period;
|
||||
unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct;
|
||||
unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */
|
||||
unsigned int sysctl_sched_conservative_pl;
|
||||
unsigned int sysctl_sched_min_task_util_for_boost = 51;
|
||||
unsigned int sysctl_sched_min_task_util_for_uclamp = 51;
|
||||
unsigned int sysctl_sched_min_task_util_for_colocation = 35;
|
||||
unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT;
|
||||
const int sched_user_hint_max = 1000;
|
||||
unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */
|
||||
unsigned int sysctl_sched_sync_hint_enable = 1;
|
||||
unsigned int sysctl_panic_on_walt_bug = walt_debug_initial_values();
|
||||
unsigned int sysctl_sched_suppress_region2;
|
||||
unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1;
|
||||
unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000;
|
||||
unsigned int sysctl_sched_asymcap_boost;
|
||||
unsigned int sysctl_sched_long_running_rt_task_ms;
|
||||
unsigned int sysctl_sched_idle_enough;
|
||||
unsigned int sysctl_sched_cluster_util_thres_pct;
|
||||
|
||||
/* range is [1 .. INT_MAX] */
|
||||
static int sysctl_task_read_pid = 1;
|
||||
|
||||
static int walt_proc_group_thresholds_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *lenp,
|
||||
loff_t *ppos)
|
||||
{
|
||||
int ret;
|
||||
static DEFINE_MUTEX(mutex);
|
||||
struct rq *rq = cpu_rq(cpumask_first(cpu_possible_mask));
|
||||
unsigned long flags;
|
||||
|
||||
if (unlikely(num_sched_clusters <= 0))
|
||||
return -EPERM;
|
||||
|
||||
mutex_lock(&mutex);
|
||||
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
||||
if (ret || !write) {
|
||||
mutex_unlock(&mutex);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* The load scale factor update happens with all
|
||||
* rqs locked. so acquiring 1 CPU rq lock and
|
||||
* updating the thresholds is sufficient for
|
||||
* an atomic update.
|
||||
*/
|
||||
raw_spin_lock_irqsave(&rq->__lock, flags);
|
||||
walt_update_group_thresholds();
|
||||
raw_spin_unlock_irqrestore(&rq->__lock, flags);
|
||||
|
||||
mutex_unlock(&mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int walt_proc_user_hint_handler(struct ctl_table *table,
|
||||
int write, void __user *buffer, size_t *lenp,
|
||||
loff_t *ppos)
|
||||
{
|
||||
int ret;
|
||||
unsigned int old_value;
|
||||
static DEFINE_MUTEX(mutex);
|
||||
|
||||
mutex_lock(&mutex);
|
||||
|
||||
old_value = sysctl_sched_user_hint;
|
||||
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
||||
if (ret || !write || (old_value == sysctl_sched_user_hint))
|
||||
goto unlock;
|
||||
|
||||
sched_user_hint_reset_time = jiffies + HZ;
|
||||
walt_irq_work_queue(&walt_migration_irq_work);
|
||||
|
||||
unlock:
|
||||
mutex_unlock(&mutex);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int sched_ravg_window_handler(struct ctl_table *table,
|
||||
int write, void __user *buffer, size_t *lenp,
|
||||
loff_t *ppos)
|
||||
{
|
||||
int ret = -EPERM;
|
||||
static DEFINE_MUTEX(mutex);
|
||||
int val;
|
||||
|
||||
struct ctl_table tmp = {
|
||||
.data = &val,
|
||||
.maxlen = sizeof(val),
|
||||
.mode = table->mode,
|
||||
};
|
||||
|
||||
mutex_lock(&mutex);
|
||||
|
||||
if (write && HZ != 250)
|
||||
goto unlock;
|
||||
|
||||
val = sysctl_sched_ravg_window_nr_ticks;
|
||||
ret = proc_dointvec(&tmp, write, buffer, lenp, ppos);
|
||||
if (ret || !write || (val == sysctl_sched_ravg_window_nr_ticks))
|
||||
goto unlock;
|
||||
|
||||
if (val != 2 && val != 3 && val != 4 && val != 5 && val != 8) {
|
||||
ret = -EINVAL;
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
sysctl_sched_ravg_window_nr_ticks = val;
|
||||
sched_window_nr_ticks_change();
|
||||
|
||||
unlock:
|
||||
mutex_unlock(&mutex);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static DEFINE_MUTEX(sysctl_pid_mutex);
|
||||
static int sched_task_read_pid_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *lenp,
|
||||
loff_t *ppos)
|
||||
{
|
||||
int ret;
|
||||
|
||||
mutex_lock(&sysctl_pid_mutex);
|
||||
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
||||
mutex_unlock(&sysctl_pid_mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
enum {
|
||||
TASK_BEGIN = 0,
|
||||
WAKE_UP_IDLE,
|
||||
INIT_TASK_LOAD,
|
||||
GROUP_ID,
|
||||
PER_TASK_BOOST,
|
||||
PER_TASK_BOOST_PERIOD_MS,
|
||||
LOW_LATENCY,
|
||||
PIPELINE,
|
||||
LOAD_BOOST,
|
||||
};
|
||||
|
||||
static int sched_task_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *lenp,
|
||||
loff_t *ppos)
|
||||
{
|
||||
int ret, param;
|
||||
struct task_struct *task;
|
||||
int pid_and_val[2] = {-1, -1};
|
||||
int val;
|
||||
struct walt_task_struct *wts;
|
||||
|
||||
struct ctl_table tmp = {
|
||||
.data = &pid_and_val,
|
||||
.maxlen = sizeof(pid_and_val),
|
||||
.mode = table->mode,
|
||||
};
|
||||
|
||||
mutex_lock(&sysctl_pid_mutex);
|
||||
|
||||
if (!write) {
|
||||
task = get_pid_task(find_vpid(sysctl_task_read_pid),
|
||||
PIDTYPE_PID);
|
||||
if (!task) {
|
||||
ret = -ENOENT;
|
||||
goto unlock_mutex;
|
||||
}
|
||||
wts = (struct walt_task_struct *) task->android_vendor_data1;
|
||||
pid_and_val[0] = sysctl_task_read_pid;
|
||||
param = (unsigned long)table->data;
|
||||
switch (param) {
|
||||
case WAKE_UP_IDLE:
|
||||
pid_and_val[1] = wts->wake_up_idle;
|
||||
break;
|
||||
case INIT_TASK_LOAD:
|
||||
pid_and_val[1] = wts->init_load_pct;
|
||||
break;
|
||||
case GROUP_ID:
|
||||
pid_and_val[1] = sched_get_group_id(task);
|
||||
break;
|
||||
case PER_TASK_BOOST:
|
||||
pid_and_val[1] = wts->boost;
|
||||
break;
|
||||
case PER_TASK_BOOST_PERIOD_MS:
|
||||
pid_and_val[1] =
|
||||
div64_ul(wts->boost_period,
|
||||
1000000UL);
|
||||
break;
|
||||
case LOW_LATENCY:
|
||||
pid_and_val[1] = wts->low_latency &
|
||||
WALT_LOW_LATENCY_PROCFS;
|
||||
break;
|
||||
case PIPELINE:
|
||||
pid_and_val[1] = wts->low_latency &
|
||||
WALT_LOW_LATENCY_PIPELINE;
|
||||
break;
|
||||
case LOAD_BOOST:
|
||||
pid_and_val[1] = wts->load_boost;
|
||||
break;
|
||||
default:
|
||||
ret = -EINVAL;
|
||||
goto put_task;
|
||||
}
|
||||
ret = proc_dointvec(&tmp, write, buffer, lenp, ppos);
|
||||
goto put_task;
|
||||
}
|
||||
|
||||
ret = proc_dointvec(&tmp, write, buffer, lenp, ppos);
|
||||
if (ret)
|
||||
goto unlock_mutex;
|
||||
|
||||
if (pid_and_val[0] <= 0) {
|
||||
ret = -ENOENT;
|
||||
goto unlock_mutex;
|
||||
}
|
||||
|
||||
/* parsed the values successfully in pid_and_val[] array */
|
||||
task = get_pid_task(find_vpid(pid_and_val[0]), PIDTYPE_PID);
|
||||
if (!task) {
|
||||
ret = -ENOENT;
|
||||
goto unlock_mutex;
|
||||
}
|
||||
wts = (struct walt_task_struct *) task->android_vendor_data1;
|
||||
param = (unsigned long)table->data;
|
||||
val = pid_and_val[1];
|
||||
if (param != LOAD_BOOST && val < 0) {
|
||||
ret = -EINVAL;
|
||||
goto put_task;
|
||||
}
|
||||
switch (param) {
|
||||
case WAKE_UP_IDLE:
|
||||
wts->wake_up_idle = val;
|
||||
break;
|
||||
case INIT_TASK_LOAD:
|
||||
if (pid_and_val[1] < 0 || pid_and_val[1] > 100) {
|
||||
ret = -EINVAL;
|
||||
goto put_task;
|
||||
}
|
||||
wts->init_load_pct = val;
|
||||
break;
|
||||
case GROUP_ID:
|
||||
ret = sched_set_group_id(task, val);
|
||||
break;
|
||||
case PER_TASK_BOOST:
|
||||
if (val < TASK_BOOST_NONE || val >= TASK_BOOST_END) {
|
||||
ret = -EINVAL;
|
||||
goto put_task;
|
||||
}
|
||||
wts->boost = val;
|
||||
if (val == 0)
|
||||
wts->boost_period = 0;
|
||||
break;
|
||||
case PER_TASK_BOOST_PERIOD_MS:
|
||||
if (wts->boost == 0 && val) {
|
||||
/* setting boost period w/o boost is invalid */
|
||||
ret = -EINVAL;
|
||||
goto put_task;
|
||||
}
|
||||
wts->boost_period = (u64)val * 1000 * 1000;
|
||||
wts->boost_expires = sched_clock() + wts->boost_period;
|
||||
break;
|
||||
case LOW_LATENCY:
|
||||
if (val)
|
||||
wts->low_latency |= WALT_LOW_LATENCY_PROCFS;
|
||||
else
|
||||
wts->low_latency &= ~WALT_LOW_LATENCY_PROCFS;
|
||||
break;
|
||||
case PIPELINE:
|
||||
if (val)
|
||||
wts->low_latency |= WALT_LOW_LATENCY_PIPELINE;
|
||||
else
|
||||
wts->low_latency &= ~WALT_LOW_LATENCY_PIPELINE;
|
||||
break;
|
||||
case LOAD_BOOST:
|
||||
if (pid_and_val[1] < -90 || pid_and_val[1] > 90) {
|
||||
ret = -EINVAL;
|
||||
goto put_task;
|
||||
}
|
||||
wts->load_boost = val;
|
||||
if (val)
|
||||
wts->boosted_task_load = mult_frac((int64_t)1024, (int64_t)val, 100);
|
||||
else
|
||||
wts->boosted_task_load = 0;
|
||||
break;
|
||||
default:
|
||||
ret = -EINVAL;
|
||||
}
|
||||
|
||||
trace_sched_task_handler(task, param, val, CALLER_ADDR0, CALLER_ADDR1,
|
||||
CALLER_ADDR2, CALLER_ADDR3, CALLER_ADDR4, CALLER_ADDR5);
|
||||
put_task:
|
||||
put_task_struct(task);
|
||||
unlock_mutex:
|
||||
mutex_unlock(&sysctl_pid_mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PROC_SYSCTL
|
||||
static void sched_update_updown_migrate_values(bool up)
|
||||
{
|
||||
int i = 0, cpu;
|
||||
struct walt_sched_cluster *cluster;
|
||||
int cap_margin_levels = num_sched_clusters - 1;
|
||||
|
||||
if (cap_margin_levels > 1) {
|
||||
/*
|
||||
* No need to worry about CPUs in last cluster
|
||||
* if there are more than 2 clusters in the system
|
||||
*/
|
||||
for_each_sched_cluster(cluster) {
|
||||
for_each_cpu(cpu, &cluster->cpus) {
|
||||
if (up)
|
||||
sched_capacity_margin_up[cpu] =
|
||||
SCHED_FIXEDPOINT_SCALE * 100 /
|
||||
sysctl_sched_capacity_margin_up_pct[i];
|
||||
else
|
||||
sched_capacity_margin_down[cpu] =
|
||||
SCHED_FIXEDPOINT_SCALE * 100 /
|
||||
sysctl_sched_capacity_margin_dn_pct[i];
|
||||
}
|
||||
|
||||
if (++i >= cap_margin_levels)
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
for_each_possible_cpu(cpu) {
|
||||
if (up)
|
||||
sched_capacity_margin_up[cpu] =
|
||||
|
||||
SCHED_FIXEDPOINT_SCALE * 100 /
|
||||
sysctl_sched_capacity_margin_up_pct[0];
|
||||
else
|
||||
sched_capacity_margin_down[cpu] =
|
||||
sysctl_sched_capacity_margin_dn_pct[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int sched_updown_migrate_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *lenp,
|
||||
loff_t *ppos)
|
||||
{
|
||||
int ret, i;
|
||||
unsigned int *data = (unsigned int *)table->data;
|
||||
static DEFINE_MUTEX(mutex);
|
||||
int cap_margin_levels = num_sched_clusters ? num_sched_clusters - 1 : 0;
|
||||
int val[MAX_MARGIN_LEVELS];
|
||||
struct ctl_table tmp = {
|
||||
.data = &val,
|
||||
.maxlen = sizeof(int) * cap_margin_levels,
|
||||
.mode = table->mode,
|
||||
};
|
||||
|
||||
if (cap_margin_levels <= 0)
|
||||
return -EINVAL;
|
||||
|
||||
mutex_lock(&mutex);
|
||||
|
||||
if (!write) {
|
||||
ret = proc_dointvec(table, write, buffer, lenp, ppos);
|
||||
goto unlock_mutex;
|
||||
}
|
||||
|
||||
ret = proc_dointvec(&tmp, write, buffer, lenp, ppos);
|
||||
if (ret)
|
||||
goto unlock_mutex;
|
||||
|
||||
/* check if valid pct values are passed in */
|
||||
for (i = 0; i < cap_margin_levels; i++) {
|
||||
if (val[i] <= 0 || val[i] > 100) {
|
||||
ret = -EINVAL;
|
||||
goto unlock_mutex;
|
||||
}
|
||||
}
|
||||
|
||||
/* check up pct is greater than dn pct */
|
||||
if (data == &sysctl_sched_capacity_margin_up_pct[0]) {
|
||||
for (i = 0; i < cap_margin_levels; i++) {
|
||||
if (val[i] < sysctl_sched_capacity_margin_dn_pct[i]) {
|
||||
ret = -EINVAL;
|
||||
goto unlock_mutex;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (i = 0; i < cap_margin_levels; i++) {
|
||||
if (sysctl_sched_capacity_margin_up_pct[i] < val[i]) {
|
||||
ret = -EINVAL;
|
||||
goto unlock_mutex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* all things checkout update the value */
|
||||
for (i = 0; i < cap_margin_levels; i++)
|
||||
data[i] = val[i];
|
||||
|
||||
/* update individual cpu thresholds */
|
||||
sched_update_updown_migrate_values(data == &sysctl_sched_capacity_margin_up_pct[0]);
|
||||
|
||||
unlock_mutex:
|
||||
mutex_unlock(&mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
#endif /* CONFIG_PROC_SYSCTL */
|
||||
|
||||
struct ctl_table input_boost_sysctls[] = {
|
||||
{
|
||||
.procname = "input_boost_ms",
|
||||
.data = &sysctl_input_boost_ms,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_hundred_thousand,
|
||||
},
|
||||
{
|
||||
.procname = "input_boost_freq",
|
||||
.data = &sysctl_input_boost_freq,
|
||||
.maxlen = sizeof(unsigned int) * 8,
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_INT_MAX,
|
||||
},
|
||||
{
|
||||
.procname = "sched_boost_on_input",
|
||||
.data = &sysctl_sched_boost_on_input,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_INT_MAX,
|
||||
},
|
||||
{ }
|
||||
};
|
||||
|
||||
struct ctl_table walt_table[] = {
|
||||
{
|
||||
.procname = "sched_user_hint",
|
||||
.data = &sysctl_sched_user_hint,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = walt_proc_user_hint_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = (void *)&sched_user_hint_max,
|
||||
},
|
||||
{
|
||||
.procname = "sched_window_stats_policy",
|
||||
.data = &sysctl_sched_window_stats_policy,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &four,
|
||||
},
|
||||
{
|
||||
.procname = "sched_group_upmigrate",
|
||||
.data = &sysctl_sched_group_upmigrate_pct,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = walt_proc_group_thresholds_handler,
|
||||
.extra1 = &sysctl_sched_group_downmigrate_pct,
|
||||
},
|
||||
{
|
||||
.procname = "sched_group_downmigrate",
|
||||
.data = &sysctl_sched_group_downmigrate_pct,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = walt_proc_group_thresholds_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &sysctl_sched_group_upmigrate_pct,
|
||||
},
|
||||
{
|
||||
.procname = "sched_boost",
|
||||
.data = &sysctl_sched_boost,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_boost_handler,
|
||||
.extra1 = &neg_three,
|
||||
.extra2 = &three,
|
||||
},
|
||||
{
|
||||
.procname = "sched_conservative_pl",
|
||||
.data = &sysctl_sched_conservative_pl,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "sched_many_wakeup_threshold",
|
||||
.data = &sysctl_sched_many_wakeup_threshold,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = &two,
|
||||
.extra2 = &one_thousand,
|
||||
},
|
||||
{
|
||||
.procname = "sched_walt_rotate_big_tasks",
|
||||
.data = &sysctl_sched_walt_rotate_big_tasks,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "sched_min_task_util_for_boost",
|
||||
.data = &sysctl_sched_min_task_util_for_boost,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_thousand,
|
||||
},
|
||||
{
|
||||
.procname = "sched_min_task_util_for_uclamp",
|
||||
.data = &sysctl_sched_min_task_util_for_uclamp,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_thousand,
|
||||
},
|
||||
{
|
||||
.procname = "sched_min_task_util_for_colocation",
|
||||
.data = &sysctl_sched_min_task_util_for_colocation,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_thousand,
|
||||
},
|
||||
{
|
||||
.procname = "sched_asym_cap_sibling_freq_match_pct",
|
||||
.data = &sysctl_sched_asym_cap_sibling_freq_match_pct,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ONE,
|
||||
.extra2 = &one_hundred,
|
||||
},
|
||||
{
|
||||
.procname = "sched_coloc_downmigrate_ns",
|
||||
.data = &sysctl_sched_coloc_downmigrate_ns,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_douintvec_minmax,
|
||||
},
|
||||
{
|
||||
.procname = "sched_task_unfilter_period",
|
||||
.data = &sysctl_sched_task_unfilter_period,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ONE,
|
||||
.extra2 = &two_hundred_million,
|
||||
},
|
||||
{
|
||||
.procname = "sched_busy_hysteresis_enable_cpus",
|
||||
.data = &sysctl_sched_busy_hyst_enable_cpus,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &two_hundred_fifty_five,
|
||||
},
|
||||
{
|
||||
.procname = "sched_busy_hyst_ns",
|
||||
.data = &sysctl_sched_busy_hyst,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &ns_per_sec,
|
||||
},
|
||||
{
|
||||
.procname = "sched_coloc_busy_hysteresis_enable_cpus",
|
||||
.data = &sysctl_sched_coloc_busy_hyst_enable_cpus,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &two_hundred_fifty_five,
|
||||
},
|
||||
{
|
||||
.procname = "sched_coloc_busy_hyst_cpu_ns",
|
||||
.data = &sysctl_sched_coloc_busy_hyst_cpu,
|
||||
.maxlen = sizeof(unsigned int) * WALT_NR_CPUS,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &ns_per_sec,
|
||||
},
|
||||
{
|
||||
.procname = "sched_coloc_busy_hyst_max_ms",
|
||||
.data = &sysctl_sched_coloc_busy_hyst_max_ms,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_hundred_thousand,
|
||||
},
|
||||
{
|
||||
.procname = "sched_coloc_busy_hyst_cpu_busy_pct",
|
||||
.data = &sysctl_sched_coloc_busy_hyst_cpu_busy_pct,
|
||||
.maxlen = sizeof(unsigned int) * WALT_NR_CPUS,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_hundred,
|
||||
},
|
||||
{
|
||||
.procname = "sched_util_busy_hysteresis_enable_cpus",
|
||||
.data = &sysctl_sched_util_busy_hyst_enable_cpus,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &two_hundred_fifty_five,
|
||||
},
|
||||
{
|
||||
.procname = "sched_util_busy_hyst_cpu_ns",
|
||||
.data = &sysctl_sched_util_busy_hyst_cpu,
|
||||
.maxlen = sizeof(unsigned int) * WALT_NR_CPUS,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &ns_per_sec,
|
||||
},
|
||||
{
|
||||
.procname = "sched_util_busy_hyst_cpu_util",
|
||||
.data = &sysctl_sched_util_busy_hyst_cpu_util,
|
||||
.maxlen = sizeof(unsigned int) * WALT_NR_CPUS,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_busy_hyst_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_thousand,
|
||||
},
|
||||
{
|
||||
.procname = "sched_ravg_window_nr_ticks",
|
||||
.data = &sysctl_sched_ravg_window_nr_ticks,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_ravg_window_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_upmigrate",
|
||||
.data = &sysctl_sched_capacity_margin_up_pct,
|
||||
.maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_updown_migrate_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_downmigrate",
|
||||
.data = &sysctl_sched_capacity_margin_dn_pct,
|
||||
.maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_updown_migrate_handler,
|
||||
},
|
||||
{
|
||||
.procname = "walt_rtg_cfs_boost_prio",
|
||||
.data = &sysctl_walt_rtg_cfs_boost_prio,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = &min_cfs_boost_prio,
|
||||
.extra2 = &max_cfs_boost_prio,
|
||||
},
|
||||
{
|
||||
.procname = "walt_low_latency_task_threshold",
|
||||
.data = &sysctl_walt_low_latency_task_threshold,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_thousand,
|
||||
},
|
||||
{
|
||||
.procname = "sched_force_lb_enable",
|
||||
.data = &sysctl_sched_force_lb_enable,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "sched_sync_hint_enable",
|
||||
.data = &sysctl_sched_sync_hint_enable,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "sched_suppress_region2",
|
||||
.data = &sysctl_sched_suppress_region2,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "sched_skip_sp_newly_idle_lb",
|
||||
.data = &sysctl_sched_skip_sp_newly_idle_lb,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "sched_hyst_min_coloc_ns",
|
||||
.data = &sysctl_sched_hyst_min_coloc_ns,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
},
|
||||
{
|
||||
.procname = "panic_on_walt_bug",
|
||||
.data = &sysctl_panic_on_walt_bug,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_INT_MAX,
|
||||
},
|
||||
{
|
||||
.procname = "sched_lib_name",
|
||||
.data = sched_lib_name,
|
||||
.maxlen = LIB_PATH_LENGTH,
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_dostring,
|
||||
},
|
||||
{
|
||||
.procname = "sched_lib_mask_force",
|
||||
.data = &sched_lib_mask_force,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_douintvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &two_hundred_fifty_five,
|
||||
},
|
||||
{
|
||||
.procname = "input_boost",
|
||||
.mode = 0555,
|
||||
.child = input_boost_sysctls,
|
||||
},
|
||||
{
|
||||
.procname = "sched_wake_up_idle",
|
||||
.data = (int *) WAKE_UP_IDLE,
|
||||
.maxlen = sizeof(unsigned int) * 2,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_init_task_load",
|
||||
.data = (int *) INIT_TASK_LOAD,
|
||||
.maxlen = sizeof(unsigned int) * 2,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_group_id",
|
||||
.data = (int *) GROUP_ID,
|
||||
.maxlen = sizeof(unsigned int) * 2,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_per_task_boost",
|
||||
.data = (int *) PER_TASK_BOOST,
|
||||
.maxlen = sizeof(unsigned int) * 2,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_per_task_boost_period_ms",
|
||||
.data = (int *) PER_TASK_BOOST_PERIOD_MS,
|
||||
.maxlen = sizeof(unsigned int) * 2,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_low_latency",
|
||||
.data = (int *) LOW_LATENCY,
|
||||
.maxlen = sizeof(unsigned int) * 2,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_pipeline",
|
||||
.data = (int *) PIPELINE,
|
||||
.maxlen = sizeof(unsigned int) * 2,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_handler,
|
||||
},
|
||||
{
|
||||
.procname = "task_load_boost",
|
||||
.data = (int *) LOAD_BOOST,
|
||||
.maxlen = sizeof(unsigned int) * 2,
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_handler,
|
||||
},
|
||||
{
|
||||
.procname = "sched_task_read_pid",
|
||||
.data = &sysctl_task_read_pid,
|
||||
.maxlen = sizeof(int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_task_read_pid_handler,
|
||||
.extra1 = SYSCTL_ONE,
|
||||
.extra2 = SYSCTL_INT_MAX,
|
||||
},
|
||||
{
|
||||
.procname = "sched_enable_tp",
|
||||
.data = &sysctl_sched_dynamic_tp_enable,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_dynamic_tp_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "sched_asymcap_boost",
|
||||
.data = &sysctl_sched_asymcap_boost,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_douintvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "sched_cluster_util_thres_pct",
|
||||
.data = &sysctl_sched_cluster_util_thres_pct,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_douintvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_INT_MAX,
|
||||
},
|
||||
{
|
||||
.procname = "sched_idle_enough",
|
||||
.data = &sysctl_sched_idle_enough,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = proc_douintvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_INT_MAX,
|
||||
},
|
||||
{
|
||||
.procname = "sched_long_running_rt_task_ms",
|
||||
.data = &sysctl_sched_long_running_rt_task_ms,
|
||||
.maxlen = sizeof(unsigned int),
|
||||
.mode = 0644,
|
||||
.proc_handler = sched_long_running_rt_task_ms_handler,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &two_thousand,
|
||||
},
|
||||
{ }
|
||||
};
|
||||
|
||||
struct ctl_table walt_base_table[] = {
|
||||
{
|
||||
.procname = "walt",
|
||||
.mode = 0555,
|
||||
.child = walt_table,
|
||||
},
|
||||
{ },
|
||||
};
|
||||
|
||||
void walt_tunables(void)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < MAX_MARGIN_LEVELS; i++) {
|
||||
sysctl_sched_capacity_margin_up_pct[i] = 95; /* ~5% margin */
|
||||
sysctl_sched_capacity_margin_dn_pct[i] = 85; /* ~15% margin */
|
||||
}
|
||||
|
||||
sysctl_sched_group_upmigrate_pct = 100;
|
||||
|
||||
sysctl_sched_group_downmigrate_pct = 95;
|
||||
|
||||
sysctl_sched_asym_cap_sibling_freq_match_pct = 100;
|
||||
|
||||
sysctl_sched_task_unfilter_period = 100000000;
|
||||
|
||||
sysctl_sched_window_stats_policy = WINDOW_STATS_MAX_RECENT_AVG;
|
||||
|
||||
sysctl_sched_ravg_window_nr_ticks = (HZ / NR_WINDOWS_PER_SEC);
|
||||
|
||||
sched_load_granule = DEFAULT_SCHED_RAVG_WINDOW / NUM_LOAD_INDICES;
|
||||
|
||||
for (i = 0; i < WALT_NR_CPUS; i++) {
|
||||
sysctl_sched_coloc_busy_hyst_cpu[i] = 39000000;
|
||||
sysctl_sched_coloc_busy_hyst_cpu_busy_pct[i] = 10;
|
||||
sysctl_sched_util_busy_hyst_cpu[i] = 5000000;
|
||||
sysctl_sched_util_busy_hyst_cpu_util[i] = 15;
|
||||
}
|
||||
|
||||
sysctl_sched_coloc_busy_hyst_enable_cpus = 112;
|
||||
|
||||
sysctl_sched_util_busy_hyst_enable_cpus = 255;
|
||||
|
||||
sysctl_sched_coloc_busy_hyst_max_ms = 5000;
|
||||
|
||||
sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW;
|
||||
|
||||
sysctl_input_boost_ms = 40;
|
||||
|
||||
for (i = 0; i < 8; i++)
|
||||
sysctl_input_boost_freq[i] = 0;
|
||||
}
|
||||
84
kernel/sched/walt/trace.c
Normal file
84
kernel/sched/walt/trace.c
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include "walt.h"
|
||||
|
||||
static inline void __window_data(u32 *dst, u32 *src)
|
||||
{
|
||||
if (src)
|
||||
memcpy(dst, src, nr_cpu_ids * sizeof(u32));
|
||||
else
|
||||
memset(dst, 0, nr_cpu_ids * sizeof(u32));
|
||||
}
|
||||
|
||||
struct trace_seq;
|
||||
const char *__window_print(struct trace_seq *p, const u32 *buf, int buf_len)
|
||||
{
|
||||
int i;
|
||||
const char *ret = p->buffer + seq_buf_used(&p->seq);
|
||||
|
||||
for (i = 0; i < buf_len; i++)
|
||||
trace_seq_printf(p, "%u ", buf[i]);
|
||||
|
||||
trace_seq_putc(p, 0);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline s64 __rq_update_sum(struct rq *rq, bool curr, bool new)
|
||||
{
|
||||
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
|
||||
|
||||
if (curr)
|
||||
if (new)
|
||||
return wrq->nt_curr_runnable_sum;
|
||||
else
|
||||
return wrq->curr_runnable_sum;
|
||||
else
|
||||
if (new)
|
||||
return wrq->nt_prev_runnable_sum;
|
||||
else
|
||||
return wrq->prev_runnable_sum;
|
||||
}
|
||||
|
||||
static inline s64 __grp_update_sum(struct rq *rq, bool curr, bool new)
|
||||
{
|
||||
struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1;
|
||||
|
||||
if (curr)
|
||||
if (new)
|
||||
return wrq->grp_time.nt_curr_runnable_sum;
|
||||
else
|
||||
return wrq->grp_time.curr_runnable_sum;
|
||||
else
|
||||
if (new)
|
||||
return wrq->grp_time.nt_prev_runnable_sum;
|
||||
else
|
||||
return wrq->grp_time.prev_runnable_sum;
|
||||
}
|
||||
|
||||
static inline s64
|
||||
__get_update_sum(struct rq *rq, enum migrate_types migrate_type,
|
||||
bool src, bool new, bool curr)
|
||||
{
|
||||
switch (migrate_type) {
|
||||
case RQ_TO_GROUP:
|
||||
if (src)
|
||||
return __rq_update_sum(rq, curr, new);
|
||||
else
|
||||
return __grp_update_sum(rq, curr, new);
|
||||
case GROUP_TO_RQ:
|
||||
if (src)
|
||||
return __grp_update_sum(rq, curr, new);
|
||||
else
|
||||
return __rq_update_sum(rq, curr, new);
|
||||
default:
|
||||
WARN_ON_ONCE(1);
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include "trace.h"
|
||||
1484
kernel/sched/walt/trace.h
Normal file
1484
kernel/sched/walt/trace.h
Normal file
File diff suppressed because it is too large
Load Diff
4707
kernel/sched/walt/walt.c
Normal file
4707
kernel/sched/walt/walt.c
Normal file
File diff suppressed because it is too large
Load Diff
1021
kernel/sched/walt/walt.h
Normal file
1021
kernel/sched/walt/walt.h
Normal file
File diff suppressed because it is too large
Load Diff
1370
kernel/sched/walt/walt_cfs.c
Normal file
1370
kernel/sched/walt/walt_cfs.c
Normal file
File diff suppressed because it is too large
Load Diff
34
kernel/sched/walt/walt_debug.c
Normal file
34
kernel/sched/walt/walt_debug.c
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/sched.h>
|
||||
|
||||
#include <trace/hooks/sched.h>
|
||||
|
||||
#include "walt.h"
|
||||
#include "walt_debug.h"
|
||||
|
||||
static void android_rvh_schedule_bug(void *unused, void *unused2)
|
||||
{
|
||||
BUG();
|
||||
}
|
||||
|
||||
static int __init walt_debug_init(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = preemptirq_long_init();
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
register_trace_android_rvh_schedule_bug(android_rvh_schedule_bug, NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
module_init(walt_debug_init);
|
||||
|
||||
MODULE_DESCRIPTION("QTI WALT Debug Module");
|
||||
MODULE_LICENSE("GPL v2");
|
||||
5
kernel/sched/walt/walt_debug.h
Normal file
5
kernel/sched/walt/walt_debug.h
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
int preemptirq_long_init(void);
|
||||
569
kernel/sched/walt/walt_halt.c
Normal file
569
kernel/sched/walt/walt_halt.c
Normal file
|
|
@ -0,0 +1,569 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2021-2021 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/cpumask.h>
|
||||
#include <linux/sched/isolation.h>
|
||||
#include <trace/hooks/sched.h>
|
||||
#include <walt.h>
|
||||
#include "trace.h"
|
||||
|
||||
#ifdef CONFIG_HOTPLUG_CPU
|
||||
|
||||
/* if a cpu is halting */
|
||||
struct cpumask __cpu_halt_mask;
|
||||
|
||||
/* spin lock to allow calling from non-preemptible context */
|
||||
static DEFINE_RAW_SPINLOCK(halt_lock);
|
||||
|
||||
struct halt_cpu_state {
|
||||
u64 last_halt;
|
||||
u8 reason;
|
||||
};
|
||||
|
||||
static DEFINE_PER_CPU(struct halt_cpu_state, halt_state);
|
||||
static DEFINE_RAW_SPINLOCK(walt_drain_pending_lock);
|
||||
|
||||
/* the amount of time allowed for enqueue operations that happen
|
||||
* just after a halt operation.
|
||||
*/
|
||||
#define WALT_HALT_CHECK_THRESHOLD_NS 400000
|
||||
|
||||
/*
|
||||
* Remove a task from the runqueue and pretend that it's migrating. This
|
||||
* should prevent migrations for the detached task and disallow further
|
||||
* changes to tsk_cpus_allowed.
|
||||
*/
|
||||
void
|
||||
detach_one_task_core(struct task_struct *p, struct rq *rq,
|
||||
struct list_head *tasks)
|
||||
{
|
||||
lockdep_assert_held(&rq->__lock);
|
||||
|
||||
p->on_rq = TASK_ON_RQ_MIGRATING;
|
||||
deactivate_task(rq, p, 0);
|
||||
list_add(&p->se.group_node, tasks);
|
||||
}
|
||||
|
||||
void attach_tasks_core(struct list_head *tasks, struct rq *rq)
|
||||
{
|
||||
struct task_struct *p;
|
||||
|
||||
lockdep_assert_held(&rq->__lock);
|
||||
|
||||
while (!list_empty(tasks)) {
|
||||
p = list_first_entry(tasks, struct task_struct, se.group_node);
|
||||
list_del_init(&p->se.group_node);
|
||||
|
||||
BUG_ON(task_rq(p) != rq);
|
||||
activate_task(rq, p, 0);
|
||||
p->on_rq = TASK_ON_RQ_QUEUED;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Migrate all tasks from the rq, sleeping tasks will be migrated by
|
||||
* try_to_wake_up()->select_task_rq().
|
||||
*
|
||||
* Called with rq->__lock held even though we'er in stop_machine() and
|
||||
* there's no concurrency possible, we hold the required locks anyway
|
||||
* because of lock validation efforts.
|
||||
*
|
||||
* The function will skip CPU pinned kthreads.
|
||||
*/
|
||||
static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
|
||||
{
|
||||
struct rq *rq = dead_rq;
|
||||
struct task_struct *next, *stop = rq->stop;
|
||||
LIST_HEAD(percpu_kthreads);
|
||||
unsigned int num_pinned_kthreads = 1;
|
||||
struct rq_flags orf = *rf;
|
||||
int dest_cpu;
|
||||
|
||||
/*
|
||||
* Fudge the rq selection such that the below task selection loop
|
||||
* doesn't get stuck on the currently eligible stop task.
|
||||
*
|
||||
* We're currently inside stop_machine() and the rq is either stuck
|
||||
* in the stop_machine_cpu_stop() loop, or we're executing this code,
|
||||
* either way we should never end up calling schedule() until we're
|
||||
* done here.
|
||||
*/
|
||||
rq->stop = NULL;
|
||||
|
||||
/*
|
||||
* put_prev_task() and pick_next_task() sched
|
||||
* class method both need to have an up-to-date
|
||||
* value of rq->clock[_task]
|
||||
*/
|
||||
update_rq_clock(rq);
|
||||
|
||||
#ifdef CONFIG_SCHED_DEBUG
|
||||
/* note the clock update in orf */
|
||||
orf.clock_update_flags |= RQCF_UPDATED;
|
||||
#endif
|
||||
|
||||
for (;;) {
|
||||
/*
|
||||
* There's this thread running, bail when that's the only
|
||||
* remaining thread:
|
||||
*/
|
||||
if (rq->nr_running == 1)
|
||||
break;
|
||||
|
||||
next = pick_migrate_task(rq);
|
||||
|
||||
/*
|
||||
* Argh ... no iterator for tasks, we need to remove the
|
||||
* kthread from the run-queue to continue.
|
||||
*/
|
||||
|
||||
if (is_per_cpu_kthread(next)) {
|
||||
detach_one_task_core(next, rq, &percpu_kthreads);
|
||||
num_pinned_kthreads += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* Rules for changing task_struct::cpus_mask are holding
|
||||
* both pi_lock and rq->__lock, such that holding either
|
||||
* stabilizes the mask.
|
||||
*
|
||||
* Drop rq->__lock is not quite as disastrous as it usually is
|
||||
* because !cpu_active at this point, which means load-balance
|
||||
* will not interfere. Also, stop-machine.
|
||||
*/
|
||||
rq_unlock(rq, rf);
|
||||
raw_spin_lock(&next->pi_lock);
|
||||
rq_relock(rq, rf);
|
||||
|
||||
/*
|
||||
* Since we're inside stop-machine, _nothing_ should have
|
||||
* changed the task, WARN if weird stuff happened, because in
|
||||
* that case the above rq->__lock drop is a fail too.
|
||||
*/
|
||||
if (task_rq(next) != rq || !task_on_rq_queued(next)) {
|
||||
raw_spin_unlock(&next->pi_lock);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Find suitable destination for @next */
|
||||
dest_cpu = select_fallback_rq(dead_rq->cpu, next);
|
||||
|
||||
if (cpu_of(rq) != dest_cpu && !is_migration_disabled(next)) {
|
||||
/* only perform a required migration */
|
||||
rq = __migrate_task(rq, rf, next, dest_cpu);
|
||||
|
||||
if (rq != dead_rq) {
|
||||
rq_unlock(rq, rf);
|
||||
rq = dead_rq;
|
||||
*rf = orf;
|
||||
rq_relock(rq, rf);
|
||||
}
|
||||
} else {
|
||||
detach_one_task_core(next, rq, &percpu_kthreads);
|
||||
num_pinned_kthreads += 1;
|
||||
}
|
||||
|
||||
raw_spin_unlock(&next->pi_lock);
|
||||
}
|
||||
|
||||
if (num_pinned_kthreads > 1)
|
||||
attach_tasks_core(&percpu_kthreads, rq);
|
||||
|
||||
rq->stop = stop;
|
||||
}
|
||||
|
||||
static int drain_rq_cpu_stop(void *data)
|
||||
{
|
||||
struct rq *rq = this_rq();
|
||||
struct rq_flags rf;
|
||||
|
||||
rq_lock_irqsave(rq, &rf);
|
||||
migrate_tasks(rq, &rf);
|
||||
rq_unlock_irqrestore(rq, &rf);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int cpu_drain_rq(unsigned int cpu)
|
||||
{
|
||||
if (!cpu_online(cpu))
|
||||
return 0;
|
||||
|
||||
if (available_idle_cpu(cpu))
|
||||
return 0;
|
||||
|
||||
/* this will schedule, must not be in atomic context */
|
||||
return stop_one_cpu(cpu, drain_rq_cpu_stop, NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
* returns true if last halt is within threshold
|
||||
* note: do not take halt_lock, called from atomic context
|
||||
*/
|
||||
bool walt_halt_check_last(int cpu)
|
||||
{
|
||||
u64 last_halt = per_cpu_ptr(&halt_state, cpu)->last_halt;
|
||||
|
||||
/* last_halt is valid, check it against sched_clock */
|
||||
if (last_halt != 0 && sched_clock() - last_halt > WALT_HALT_CHECK_THRESHOLD_NS)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
struct drain_thread_data {
|
||||
cpumask_t cpus_to_drain;
|
||||
};
|
||||
|
||||
static struct drain_thread_data drain_data = {
|
||||
.cpus_to_drain = { CPU_BITS_NONE }
|
||||
};
|
||||
|
||||
static int __ref try_drain_rqs(void *data)
|
||||
{
|
||||
cpumask_t *cpus_ptr = &((struct drain_thread_data *)data)->cpus_to_drain;
|
||||
int cpu;
|
||||
unsigned long flags;
|
||||
|
||||
while (!kthread_should_stop()) {
|
||||
raw_spin_lock_irqsave(&walt_drain_pending_lock, flags);
|
||||
if (cpumask_weight(cpus_ptr)) {
|
||||
cpumask_t local_cpus;
|
||||
|
||||
cpumask_copy(&local_cpus, cpus_ptr);
|
||||
raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags);
|
||||
|
||||
for_each_cpu(cpu, &local_cpus)
|
||||
cpu_drain_rq(cpu);
|
||||
|
||||
raw_spin_lock_irqsave(&walt_drain_pending_lock, flags);
|
||||
cpumask_andnot(cpus_ptr, cpus_ptr, &local_cpus);
|
||||
|
||||
}
|
||||
raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags);
|
||||
set_current_state(TASK_INTERRUPTIBLE);
|
||||
schedule();
|
||||
set_current_state(TASK_RUNNING);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct task_struct *walt_drain_thread;
|
||||
|
||||
/*
|
||||
* 1) add the cpus to the halt mask
|
||||
* 2) migrate tasks off the cpu
|
||||
*/
|
||||
static int halt_cpus(struct cpumask *cpus)
|
||||
{
|
||||
int cpu;
|
||||
int ret = 0;
|
||||
u64 start_time = sched_clock();
|
||||
struct halt_cpu_state *halt_cpu_state;
|
||||
unsigned long flags;
|
||||
|
||||
trace_halt_cpus_start(cpus, 1);
|
||||
|
||||
for_each_cpu(cpu, cpus) {
|
||||
|
||||
if (cpu == cpumask_first(system_32bit_el0_cpumask())) {
|
||||
ret = -EINVAL;
|
||||
goto out;
|
||||
}
|
||||
|
||||
halt_cpu_state = per_cpu_ptr(&halt_state, cpu);
|
||||
|
||||
/* set the cpu as halted */
|
||||
cpumask_set_cpu(cpu, cpu_halt_mask);
|
||||
|
||||
/* guarantee mask written before updating last_halt */
|
||||
wmb();
|
||||
|
||||
halt_cpu_state->last_halt = start_time;
|
||||
}
|
||||
|
||||
/* signal and wakeup the drain kthread */
|
||||
raw_spin_lock_irqsave(&walt_drain_pending_lock, flags);
|
||||
cpumask_or(&drain_data.cpus_to_drain, &drain_data.cpus_to_drain, cpus);
|
||||
raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags);
|
||||
|
||||
if (!IS_ERR(walt_drain_thread))
|
||||
wake_up_process(walt_drain_thread);
|
||||
|
||||
out:
|
||||
trace_halt_cpus(cpus, start_time, 1, ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* 1) remove the cpus from the halt mask
|
||||
*
|
||||
*/
|
||||
static int start_cpus(struct cpumask *cpus)
|
||||
{
|
||||
u64 start_time = sched_clock();
|
||||
struct halt_cpu_state *halt_cpu_state;
|
||||
int cpu;
|
||||
|
||||
trace_halt_cpus_start(cpus, 0);
|
||||
|
||||
for_each_cpu(cpu, cpus) {
|
||||
halt_cpu_state = per_cpu_ptr(&halt_state, cpu);
|
||||
halt_cpu_state->last_halt = 0;
|
||||
wmb();
|
||||
|
||||
/* wmb to guarantee zero'd last_halt before clearing from the mask */
|
||||
|
||||
cpumask_clear_cpu(cpu, cpu_halt_mask);
|
||||
|
||||
/* kick the cpu so it can pull tasks
|
||||
* after the mask has been cleared.
|
||||
*/
|
||||
walt_smp_call_newidle_balance(cpu);
|
||||
}
|
||||
|
||||
trace_halt_cpus(cpus, start_time, 0, 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* update reason for cpus in yield/halt mask */
|
||||
static void update_reasons(struct cpumask *cpus, bool halt, enum pause_reason reason)
|
||||
{
|
||||
int cpu;
|
||||
struct halt_cpu_state *halt_cpu_state;
|
||||
|
||||
for_each_cpu(cpu, cpus) {
|
||||
halt_cpu_state = per_cpu_ptr(&halt_state, cpu);
|
||||
if (halt)
|
||||
halt_cpu_state->reason |= reason;
|
||||
else
|
||||
halt_cpu_state->reason &= ~reason;
|
||||
}
|
||||
}
|
||||
|
||||
/* remove cpus that are already halted */
|
||||
static void update_halt_cpus(struct cpumask *cpus)
|
||||
{
|
||||
int cpu;
|
||||
struct halt_cpu_state *halt_cpu_state;
|
||||
|
||||
for_each_cpu(cpu, cpus) {
|
||||
halt_cpu_state = per_cpu_ptr(&halt_state, cpu);
|
||||
if (halt_cpu_state->reason)
|
||||
cpumask_clear_cpu(cpu, cpus);
|
||||
}
|
||||
}
|
||||
|
||||
/* cpus will be modified */
|
||||
int walt_halt_cpus(struct cpumask *cpus, enum pause_reason reason)
|
||||
{
|
||||
int ret = 0;
|
||||
cpumask_t requested_cpus;
|
||||
unsigned long flags;
|
||||
|
||||
raw_spin_lock_irqsave(&halt_lock, flags);
|
||||
|
||||
cpumask_copy(&requested_cpus, cpus);
|
||||
|
||||
/* remove cpus that are already halted */
|
||||
update_halt_cpus(cpus);
|
||||
|
||||
if (cpumask_empty(cpus)) {
|
||||
update_reasons(&requested_cpus, true, reason);
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
ret = halt_cpus(cpus);
|
||||
|
||||
if (ret < 0)
|
||||
pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret,
|
||||
cpumask_pr_args(&requested_cpus));
|
||||
else
|
||||
update_reasons(&requested_cpus, true, reason);
|
||||
unlock:
|
||||
raw_spin_unlock_irqrestore(&halt_lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason)
|
||||
{
|
||||
return walt_halt_cpus(cpus, reason);
|
||||
}
|
||||
EXPORT_SYMBOL(walt_pause_cpus);
|
||||
|
||||
/* cpus will be modified */
|
||||
int walt_start_cpus(struct cpumask *cpus, enum pause_reason reason)
|
||||
{
|
||||
int ret = 0;
|
||||
cpumask_t requested_cpus;
|
||||
unsigned long flags;
|
||||
|
||||
raw_spin_lock_irqsave(&halt_lock, flags);
|
||||
cpumask_copy(&requested_cpus, cpus);
|
||||
update_reasons(&requested_cpus, false, reason);
|
||||
|
||||
/* remove cpus that should still be halted */
|
||||
update_halt_cpus(cpus);
|
||||
|
||||
ret = start_cpus(cpus);
|
||||
|
||||
if (ret < 0) {
|
||||
pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret,
|
||||
cpumask_pr_args(&requested_cpus));
|
||||
/* restore/increment ref counts in case of error */
|
||||
update_reasons(&requested_cpus, true, reason);
|
||||
}
|
||||
|
||||
raw_spin_unlock_irqrestore(&halt_lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason)
|
||||
{
|
||||
return walt_start_cpus(cpus, reason);
|
||||
}
|
||||
EXPORT_SYMBOL(walt_resume_cpus);
|
||||
|
||||
static void android_rvh_get_nohz_timer_target(void *unused, int *cpu, bool *done)
|
||||
{
|
||||
int i, default_cpu = -1;
|
||||
struct sched_domain *sd;
|
||||
cpumask_t unhalted;
|
||||
|
||||
*done = true;
|
||||
|
||||
if (housekeeping_cpu(*cpu, HK_FLAG_TIMER) && !cpu_halted(*cpu)) {
|
||||
if (!available_idle_cpu(*cpu))
|
||||
return;
|
||||
default_cpu = *cpu;
|
||||
}
|
||||
|
||||
rcu_read_lock();
|
||||
for_each_domain(*cpu, sd) {
|
||||
for_each_cpu_and(i, sched_domain_span(sd),
|
||||
housekeeping_cpumask(HK_FLAG_TIMER)) {
|
||||
if (*cpu == i)
|
||||
continue;
|
||||
|
||||
if (!available_idle_cpu(i) && !cpu_halted(i)) {
|
||||
*cpu = i;
|
||||
goto unlock;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (default_cpu == -1) {
|
||||
cpumask_complement(&unhalted, cpu_halt_mask);
|
||||
for_each_cpu_and(i, &unhalted,
|
||||
housekeeping_cpumask(HK_FLAG_TIMER)) {
|
||||
if (*cpu == i)
|
||||
continue;
|
||||
|
||||
if (!available_idle_cpu(i)) {
|
||||
*cpu = i;
|
||||
goto unlock;
|
||||
}
|
||||
}
|
||||
|
||||
/* no active, non-halted, not-idle, choose any */
|
||||
default_cpu = cpumask_any(&unhalted);
|
||||
|
||||
if (unlikely(default_cpu >= nr_cpu_ids))
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
*cpu = default_cpu;
|
||||
unlock:
|
||||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
static void android_rvh_set_cpus_allowed_ptr_locked(void *unused,
|
||||
const struct cpumask *cpu_valid_mask,
|
||||
const struct cpumask *new_mask,
|
||||
unsigned int *dest_cpu)
|
||||
{
|
||||
cpumask_t allowed_cpus;
|
||||
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
if (cpu_halted(*dest_cpu)) {
|
||||
/* remove halted cpus from the valid mask, and store locally */
|
||||
cpumask_andnot(&allowed_cpus, cpu_valid_mask, cpu_halt_mask);
|
||||
*dest_cpu = cpumask_any_and_distribute(&allowed_cpus, new_mask);
|
||||
}
|
||||
}
|
||||
|
||||
static void android_rvh_rto_next_cpu(void *unused, int rto_cpu, struct cpumask *rto_mask, int *cpu)
|
||||
{
|
||||
cpumask_t allowed_cpus;
|
||||
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
if (cpu_halted(*cpu)) {
|
||||
/* remove halted cpus from the valid mask, and store locally */
|
||||
cpumask_andnot(&allowed_cpus, rto_mask, cpu_halt_mask);
|
||||
*cpu = cpumask_next(rto_cpu, &allowed_cpus);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* android_rvh_is_cpu_allowed: disallow cpus that are halted
|
||||
*
|
||||
* Caveat: For 32 bit tasks that are being directed to a halted cpu, allow the halted cpu
|
||||
* in a particular case (32 bit task, in execve, moving to a 32 bit cpu)
|
||||
* This is to handle the call to is_cpu_allowed() from __migrate_task, in the
|
||||
* event that a 32bit task is being execve'd.
|
||||
*/
|
||||
static void android_rvh_is_cpu_allowed(void *unused, struct task_struct *p, int cpu, bool *allowed)
|
||||
{
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
if (cpumask_test_cpu(cpu, cpu_halt_mask)) {
|
||||
|
||||
/* default reject for any halted cpu */
|
||||
*allowed = false;
|
||||
|
||||
if (is_compat_thread(task_thread_info(p)) &&
|
||||
p->in_execve) {
|
||||
/*
|
||||
* the task is 32 bit capable and
|
||||
* the context is execve. allow this cpu.
|
||||
*/
|
||||
*allowed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void walt_halt_init(void)
|
||||
{
|
||||
struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
|
||||
|
||||
walt_drain_thread = kthread_run(try_drain_rqs, &drain_data, "halt_drain_rqs");
|
||||
if (IS_ERR(walt_drain_thread)) {
|
||||
pr_err("Error creating walt drain thread\n");
|
||||
return;
|
||||
}
|
||||
|
||||
sched_setscheduler_nocheck(walt_drain_thread, SCHED_FIFO, ¶m);
|
||||
|
||||
register_trace_android_rvh_get_nohz_timer_target(android_rvh_get_nohz_timer_target, NULL);
|
||||
register_trace_android_rvh_set_cpus_allowed_ptr_locked(
|
||||
android_rvh_set_cpus_allowed_ptr_locked, NULL);
|
||||
register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL);
|
||||
register_trace_android_rvh_is_cpu_allowed(android_rvh_is_cpu_allowed, NULL);
|
||||
|
||||
}
|
||||
|
||||
#endif /* CONFIG_HOTPLUG_CPU */
|
||||
1080
kernel/sched/walt/walt_lb.c
Normal file
1080
kernel/sched/walt/walt_lb.c
Normal file
File diff suppressed because it is too large
Load Diff
367
kernel/sched/walt/walt_rt.c
Normal file
367
kernel/sched/walt/walt_rt.c
Normal file
|
|
@ -0,0 +1,367 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <trace/hooks/sched.h>
|
||||
|
||||
#include "walt.h"
|
||||
#include "trace.h"
|
||||
|
||||
#define MSEC_TO_NSEC (1000 * 1000)
|
||||
|
||||
static DEFINE_PER_CPU(cpumask_var_t, walt_local_cpu_mask);
|
||||
DEFINE_PER_CPU(u64, rt_task_arrival_time) = 0;
|
||||
static bool long_running_rt_task_trace_rgstrd;
|
||||
|
||||
static void rt_task_arrival_marker(void *unused, bool preempt,
|
||||
struct task_struct *prev, struct task_struct *next)
|
||||
{
|
||||
unsigned int cpu = raw_smp_processor_id();
|
||||
|
||||
if (next->policy == SCHED_FIFO && next != cpu_rq(cpu)->stop)
|
||||
per_cpu(rt_task_arrival_time, cpu) = rq_clock_task(this_rq());
|
||||
else
|
||||
per_cpu(rt_task_arrival_time, cpu) = 0;
|
||||
}
|
||||
|
||||
static void long_running_rt_task_notifier(void *unused, struct rq *rq)
|
||||
{
|
||||
struct task_struct *curr = rq->curr;
|
||||
unsigned int cpu = raw_smp_processor_id();
|
||||
|
||||
if (!sysctl_sched_long_running_rt_task_ms)
|
||||
return;
|
||||
|
||||
if (!per_cpu(rt_task_arrival_time, cpu))
|
||||
return;
|
||||
|
||||
if (per_cpu(rt_task_arrival_time, cpu) && curr->policy != SCHED_FIFO) {
|
||||
/*
|
||||
* It is possible that the scheduling policy for the current
|
||||
* task might get changed after task arrival time stamp is
|
||||
* noted during sched_switch of RT task. To avoid such false
|
||||
* positives, reset arrival time stamp.
|
||||
*/
|
||||
per_cpu(rt_task_arrival_time, cpu) = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Since we are called from the main tick, rq clock task must have
|
||||
* been updated very recently. Use it directly, instead of
|
||||
* update_rq_clock_task() to avoid warnings.
|
||||
*/
|
||||
if (rq->clock_task -
|
||||
per_cpu(rt_task_arrival_time, cpu)
|
||||
> sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC) {
|
||||
printk_deferred("RT task %s (%d) runtime > %u now=%llu task arrival time=%llu runtime=%llu\n",
|
||||
curr->comm, curr->pid,
|
||||
sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC,
|
||||
rq->clock_task,
|
||||
per_cpu(rt_task_arrival_time, cpu),
|
||||
rq->clock_task -
|
||||
per_cpu(rt_task_arrival_time, cpu));
|
||||
BUG();
|
||||
}
|
||||
}
|
||||
|
||||
int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *lenp,
|
||||
loff_t *ppos)
|
||||
{
|
||||
int ret;
|
||||
static DEFINE_MUTEX(mutex);
|
||||
|
||||
mutex_lock(&mutex);
|
||||
|
||||
ret = proc_douintvec_minmax(table, write, buffer, lenp, ppos);
|
||||
|
||||
if (sysctl_sched_long_running_rt_task_ms > 0 &&
|
||||
sysctl_sched_long_running_rt_task_ms < 800)
|
||||
sysctl_sched_long_running_rt_task_ms = 800;
|
||||
|
||||
if (write && !long_running_rt_task_trace_rgstrd) {
|
||||
register_trace_sched_switch(rt_task_arrival_marker, NULL);
|
||||
register_trace_android_vh_scheduler_tick(long_running_rt_task_notifier, NULL);
|
||||
long_running_rt_task_trace_rgstrd = true;
|
||||
}
|
||||
|
||||
mutex_unlock(&mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void walt_rt_energy_aware_wake_cpu(struct task_struct *task, struct cpumask *lowest_mask,
|
||||
int ret, int *best_cpu)
|
||||
{
|
||||
int cpu;
|
||||
unsigned long util, best_cpu_util = ULONG_MAX;
|
||||
unsigned long best_cpu_util_cum = ULONG_MAX;
|
||||
unsigned long util_cum;
|
||||
unsigned long tutil = task_util(task);
|
||||
unsigned int best_idle_exit_latency = UINT_MAX;
|
||||
unsigned int cpu_idle_exit_latency = UINT_MAX;
|
||||
bool boost_on_big = rt_boost_on_big();
|
||||
int cluster;
|
||||
int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0;
|
||||
bool best_cpu_lt = true;
|
||||
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
if (!ret)
|
||||
return; /* No targets found */
|
||||
|
||||
rcu_read_lock();
|
||||
for (cluster = 0; cluster < num_sched_clusters; cluster++) {
|
||||
for_each_cpu_and(cpu, lowest_mask, &cpu_array[order_index][cluster]) {
|
||||
bool lt;
|
||||
|
||||
trace_sched_cpu_util(cpu, lowest_mask);
|
||||
|
||||
if (!cpu_active(cpu))
|
||||
continue;
|
||||
|
||||
if (cpu_halted(cpu))
|
||||
continue;
|
||||
|
||||
if (sched_cpu_high_irqload(cpu))
|
||||
continue;
|
||||
|
||||
if (__cpu_overutilized(cpu, tutil))
|
||||
continue;
|
||||
|
||||
util = cpu_util(cpu);
|
||||
|
||||
lt = (walt_low_latency_task(cpu_rq(cpu)->curr) ||
|
||||
walt_nr_rtg_high_prio(cpu));
|
||||
|
||||
/*
|
||||
* When the best is suitable and the current is not,
|
||||
* skip it
|
||||
*/
|
||||
if (lt && !best_cpu_lt)
|
||||
continue;
|
||||
|
||||
/*
|
||||
* Either both are sutilable or unsuitable, load takes
|
||||
* precedence.
|
||||
*/
|
||||
if (!(best_cpu_lt ^ lt) && (util > best_cpu_util))
|
||||
continue;
|
||||
|
||||
/*
|
||||
* If the previous CPU has same load, keep it as
|
||||
* best_cpu.
|
||||
*/
|
||||
if (best_cpu_util == util && *best_cpu == task_cpu(task))
|
||||
continue;
|
||||
|
||||
/*
|
||||
* If candidate CPU is the previous CPU, select it.
|
||||
* Otherwise, if its load is same with best_cpu and in
|
||||
* a shallower C-state, select it. If all above
|
||||
* conditions are same, select the least cumulative
|
||||
* window demand CPU.
|
||||
*/
|
||||
cpu_idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(cpu));
|
||||
|
||||
util_cum = cpu_util_cum(cpu);
|
||||
if (cpu != task_cpu(task) && best_cpu_util == util) {
|
||||
if (best_idle_exit_latency < cpu_idle_exit_latency)
|
||||
continue;
|
||||
|
||||
if (best_idle_exit_latency == cpu_idle_exit_latency &&
|
||||
best_cpu_util_cum < util_cum)
|
||||
continue;
|
||||
}
|
||||
|
||||
best_idle_exit_latency = cpu_idle_exit_latency;
|
||||
best_cpu_util_cum = util_cum;
|
||||
best_cpu_util = util;
|
||||
*best_cpu = cpu;
|
||||
best_cpu_lt = lt;
|
||||
}
|
||||
|
||||
if (*best_cpu != -1)
|
||||
break;
|
||||
}
|
||||
|
||||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
#ifdef CONFIG_UCLAMP_TASK
|
||||
static inline bool walt_rt_task_fits_capacity(struct task_struct *p, int cpu)
|
||||
{
|
||||
unsigned int min_cap;
|
||||
unsigned int max_cap;
|
||||
unsigned int cpu_cap;
|
||||
|
||||
min_cap = uclamp_eff_value(p, UCLAMP_MIN);
|
||||
max_cap = uclamp_eff_value(p, UCLAMP_MAX);
|
||||
|
||||
cpu_cap = capacity_orig_of(cpu);
|
||||
|
||||
return cpu_cap >= min(min_cap, max_cap);
|
||||
}
|
||||
#else
|
||||
static inline bool walt_rt_task_fits_capacity(struct task_struct *p, int cpu)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* walt specific should_honor_rt_sync (see rt.c). this will honor
|
||||
* the sync flag regardless of whether the current waker is cfs or rt
|
||||
*/
|
||||
static inline bool walt_should_honor_rt_sync(struct rq *rq, struct task_struct *p,
|
||||
bool sync)
|
||||
{
|
||||
return sync &&
|
||||
p->prio <= rq->rt.highest_prio.next &&
|
||||
rq->rt.rt_nr_running <= 2;
|
||||
}
|
||||
|
||||
static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int cpu,
|
||||
int sd_flag, int wake_flags, int *new_cpu)
|
||||
{
|
||||
struct task_struct *curr;
|
||||
struct rq *rq, *this_cpu_rq;
|
||||
bool may_not_preempt;
|
||||
bool sync = !!(wake_flags & WF_SYNC);
|
||||
int ret, target = -1, this_cpu;
|
||||
struct cpumask *lowest_mask;
|
||||
int packing_cpu;
|
||||
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
/* For anything but wake ups, just return the task_cpu */
|
||||
if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK)
|
||||
return;
|
||||
|
||||
this_cpu = raw_smp_processor_id();
|
||||
this_cpu_rq = cpu_rq(this_cpu);
|
||||
|
||||
/*
|
||||
* Respect the sync flag as long as the task can run on this CPU.
|
||||
*/
|
||||
if (sysctl_sched_sync_hint_enable && cpu_active(this_cpu) && !cpu_halted(this_cpu) &&
|
||||
cpumask_test_cpu(this_cpu, task->cpus_ptr) &&
|
||||
walt_should_honor_rt_sync(this_cpu_rq, task, sync)) {
|
||||
*new_cpu = this_cpu;
|
||||
return;
|
||||
}
|
||||
|
||||
*new_cpu = cpu; /* previous CPU as back up */
|
||||
rq = cpu_rq(cpu);
|
||||
|
||||
rcu_read_lock();
|
||||
curr = READ_ONCE(rq->curr); /* unlocked access */
|
||||
|
||||
/*
|
||||
* If the current task on @p's runqueue is a softirq task,
|
||||
* it may run without preemption for a time that is
|
||||
* ill-suited for a waiting RT task. Therefore, try to
|
||||
* wake this RT task on another runqueue.
|
||||
*
|
||||
* Otherwise, just let it ride on the affined RQ and the
|
||||
* post-schedule router will push the preempted task away
|
||||
*
|
||||
* This test is optimistic, if we get it wrong the load-balancer
|
||||
* will have to sort it out.
|
||||
*
|
||||
* We take into account the capacity of the CPU to ensure it fits the
|
||||
* requirement of the task - which is only important on heterogeneous
|
||||
* systems like big.LITTLE.
|
||||
*/
|
||||
may_not_preempt = task_may_not_preempt(curr, cpu);
|
||||
|
||||
lowest_mask = this_cpu_cpumask_var_ptr(walt_local_cpu_mask);
|
||||
|
||||
/*
|
||||
* If we're on asym system ensure we consider the different capacities
|
||||
* of the CPUs when searching for the lowest_mask.
|
||||
*/
|
||||
ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, task,
|
||||
lowest_mask, walt_rt_task_fits_capacity);
|
||||
|
||||
/* create a fastpath for finding a packing cpu */
|
||||
packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task));
|
||||
if (walt_choose_packing_cpu(packing_cpu, task)) {
|
||||
*new_cpu = packing_cpu;
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
walt_rt_energy_aware_wake_cpu(task, lowest_mask, ret, &target);
|
||||
|
||||
/*
|
||||
* If cpu is non-preemptible, prefer remote cpu
|
||||
* even if it's running a higher-prio task.
|
||||
* Otherwise: Don't bother moving it if the destination CPU is
|
||||
* not running a lower priority task.
|
||||
*/
|
||||
if (target != -1 &&
|
||||
(may_not_preempt || task->prio < cpu_rq(target)->rt.highest_prio.curr))
|
||||
*new_cpu = target;
|
||||
|
||||
/* if backup or chosen cpu is halted, pick something else */
|
||||
if (cpu_halted(*new_cpu)) {
|
||||
cpumask_t non_halted;
|
||||
|
||||
/* choose the lowest-order, unhalted, allowed CPU */
|
||||
cpumask_andnot(&non_halted, task->cpus_ptr, cpu_halt_mask);
|
||||
target = cpumask_first(&non_halted);
|
||||
if (target < nr_cpu_ids)
|
||||
*new_cpu = target;
|
||||
}
|
||||
unlock:
|
||||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
|
||||
static void walt_rt_find_lowest_rq(void *unused, struct task_struct *task,
|
||||
struct cpumask *lowest_mask, int ret, int *best_cpu)
|
||||
|
||||
{
|
||||
int packing_cpu;
|
||||
|
||||
if (unlikely(walt_disabled))
|
||||
return;
|
||||
|
||||
/* create a fastpath for finding a packing cpu */
|
||||
packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task));
|
||||
if (walt_choose_packing_cpu(packing_cpu, task)) {
|
||||
*best_cpu = packing_cpu;
|
||||
return;
|
||||
}
|
||||
|
||||
walt_rt_energy_aware_wake_cpu(task, lowest_mask, ret, best_cpu);
|
||||
|
||||
/*
|
||||
* Walt was not able to find a non-halted best cpu. Ensure that
|
||||
* find_lowest_rq doesn't use a halted cpu going forward, but
|
||||
* does a best effort itself to find a good CPU.
|
||||
*/
|
||||
if (*best_cpu == -1)
|
||||
cpumask_andnot(lowest_mask, lowest_mask, cpu_halt_mask);
|
||||
}
|
||||
|
||||
void walt_rt_init(void)
|
||||
{
|
||||
unsigned int i;
|
||||
|
||||
for_each_possible_cpu(i) {
|
||||
if(!(zalloc_cpumask_var_node(&per_cpu(walt_local_cpu_mask, i),
|
||||
GFP_KERNEL, cpu_to_node(i)))) {
|
||||
pr_err("walt_local_cpu_mask alloc failed for cpu%d\n", i);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
register_trace_android_rvh_select_task_rq_rt(walt_select_task_rq_rt, NULL);
|
||||
register_trace_android_rvh_find_lowest_rq(walt_rt_find_lowest_rq, NULL);
|
||||
}
|
||||
152
kernel/sched/walt/walt_tp.c
Normal file
152
kernel/sched/walt/walt_tp.c
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/tracepoint.h>
|
||||
#include <trace/hooks/sched.h>
|
||||
#include "trace.h"
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include "perf_trace_counters.h"
|
||||
|
||||
unsigned int sysctl_sched_dynamic_tp_enable;
|
||||
|
||||
#define USE_CPUHP_STATE CPUHP_AP_ONLINE_DYN
|
||||
|
||||
DEFINE_PER_CPU(u32, cntenset_val);
|
||||
DEFINE_PER_CPU(unsigned long, previous_ccnt);
|
||||
DEFINE_PER_CPU(unsigned long[NUM_L1_CTRS], previous_l1_cnts);
|
||||
DEFINE_PER_CPU(unsigned long[NUM_AMU_CTRS], previous_amu_cnts);
|
||||
DEFINE_PER_CPU(u32, old_pid);
|
||||
DEFINE_PER_CPU(u32, hotplug_flag);
|
||||
DEFINE_PER_CPU(u64, prev_time);
|
||||
|
||||
static int tracectr_cpu_hotplug_coming_up(unsigned int cpu)
|
||||
{
|
||||
per_cpu(hotplug_flag, cpu) = 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void setup_prev_cnts(u32 cpu, u32 cnten_val)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (cnten_val & CC)
|
||||
per_cpu(previous_ccnt, cpu) =
|
||||
read_sysreg(pmccntr_el0);
|
||||
|
||||
for (i = 0; i < NUM_L1_CTRS; i++) {
|
||||
if (cnten_val & (1 << i)) {
|
||||
/* Select */
|
||||
write_sysreg(i, pmselr_el0);
|
||||
isb();
|
||||
/* Read value */
|
||||
per_cpu(previous_l1_cnts[i], cpu) =
|
||||
read_sysreg(pmxevcntr_el0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void tracectr_notifier(void *ignore, bool preempt,
|
||||
struct task_struct *prev, struct task_struct *next)
|
||||
{
|
||||
u32 cnten_val;
|
||||
int current_pid;
|
||||
u32 cpu = task_cpu(next);
|
||||
u64 now;
|
||||
|
||||
if (!trace_sched_switch_with_ctrs_enabled())
|
||||
return;
|
||||
|
||||
current_pid = next->pid;
|
||||
if (per_cpu(old_pid, cpu) != -1) {
|
||||
cnten_val = read_sysreg(pmcntenset_el0);
|
||||
per_cpu(cntenset_val, cpu) = cnten_val;
|
||||
/* Disable all the counters that were enabled */
|
||||
write_sysreg(cnten_val, pmcntenclr_el0);
|
||||
|
||||
if (per_cpu(hotplug_flag, cpu) == 1) {
|
||||
per_cpu(hotplug_flag, cpu) = 0;
|
||||
setup_prev_cnts(cpu, cnten_val);
|
||||
} else {
|
||||
trace_sched_switch_with_ctrs(per_cpu(old_pid, cpu),
|
||||
current_pid);
|
||||
now = sched_clock();
|
||||
if ((now - per_cpu(prev_time, cpu)) > NSEC_PER_SEC) {
|
||||
trace_sched_switch_ctrs_cfg(cpu);
|
||||
per_cpu(prev_time, cpu) = now;
|
||||
}
|
||||
}
|
||||
|
||||
/* Enable all the counters that were disabled */
|
||||
write_sysreg(cnten_val, pmcntenset_el0);
|
||||
}
|
||||
per_cpu(old_pid, cpu) = current_pid;
|
||||
}
|
||||
|
||||
static void register_sched_switch_ctrs(void)
|
||||
{
|
||||
int cpu, rc;
|
||||
|
||||
for_each_possible_cpu(cpu)
|
||||
per_cpu(old_pid, cpu) = -1;
|
||||
|
||||
rc = cpuhp_setup_state_nocalls(USE_CPUHP_STATE, "tracectr_cpu_hotplug",
|
||||
tracectr_cpu_hotplug_coming_up, NULL);
|
||||
if (rc >= 0)
|
||||
register_trace_sched_switch(tracectr_notifier, NULL);
|
||||
}
|
||||
|
||||
static void unregister_sched_switch_ctrs(void)
|
||||
{
|
||||
unregister_trace_sched_switch(tracectr_notifier, NULL);
|
||||
cpuhp_remove_state_nocalls(USE_CPUHP_STATE);
|
||||
}
|
||||
|
||||
static void sched_overutilized(void *data, struct root_domain *rd,
|
||||
bool overutilized)
|
||||
{
|
||||
if (trace_sched_overutilized_enabled()) {
|
||||
char span[SPAN_SIZE];
|
||||
|
||||
cpumap_print_to_pagebuf(false, span, sched_trace_rd_span(rd));
|
||||
trace_sched_overutilized(overutilized, span);
|
||||
}
|
||||
}
|
||||
|
||||
static void walt_register_dynamic_tp_events(void)
|
||||
{
|
||||
register_trace_sched_overutilized_tp(sched_overutilized, NULL);
|
||||
register_sched_switch_ctrs();
|
||||
}
|
||||
|
||||
static void walt_unregister_dynamic_tp_events(void)
|
||||
{
|
||||
unregister_trace_sched_overutilized_tp(sched_overutilized, NULL);
|
||||
unregister_sched_switch_ctrs();
|
||||
}
|
||||
|
||||
int sched_dynamic_tp_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *lenp, loff_t *ppos)
|
||||
{
|
||||
static DEFINE_MUTEX(mutex);
|
||||
int ret = 0, *val = (unsigned int *)table->data;
|
||||
unsigned int old_val;
|
||||
|
||||
mutex_lock(&mutex);
|
||||
old_val = sysctl_sched_dynamic_tp_enable;
|
||||
|
||||
ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
||||
if (ret || !write || (old_val == sysctl_sched_dynamic_tp_enable))
|
||||
goto done;
|
||||
|
||||
if (*val)
|
||||
walt_register_dynamic_tp_events();
|
||||
else
|
||||
walt_unregister_dynamic_tp_events();
|
||||
done:
|
||||
mutex_unlock(&mutex);
|
||||
return ret;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user