github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-30 19:21:28 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge "defconfig: pineapple-gki: Enable governor driver"

Browse Source
This commit is contained in:
qctecmdr 2022-06-08 02:27:44 -07:00 committed by Gerrit - the friendly Code Review server
commit f829779b80
9 changed files with 1906 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/arm64/configs/vendor/pineapple_GKI.config vendored
View File

@ -3,6 +3,7 @@ CONFIG_ARM_SMMU=m
CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT=y
CONFIG_ARM_SMMU_QCOM=m
CONFIG_COMMON_CLK_QCOM=m
CONFIG_CPU_IDLE_GOV_QCOM_LPM=m
CONFIG_HWSPINLOCK_QCOM=m
CONFIG_INTERCONNECT_QCOM=m
CONFIG_INTERCONNECT_QCOM_DEBUG=m

13
drivers/cpuidle/Kconfig
View File

@ -44,6 +44,19 @@ config CPU_IDLE_GOV_HALTPOLL
Some virtualized workloads benefit from using it.
config CPU_IDLE_GOV_QCOM_LPM
tristate "Qualcomm Technologies, Inc. CPU and Cluster governor"
depends on ARCH_QCOM
depends on ARM_PSCI_CPUIDLE
help
This governor implements effective cpu and cluster idle state
selection with help of scheduler inputs, cpu idle state prediction
and cluster idle state prediction algorithms.
The predicted sleep time, latency requirement for the
CPU and the idle state chosen based on the parameters are all
logged in the trace.
config DT_IDLE_STATES
bool

5
drivers/cpuidle/governors/Makefile
View File

@ -3,7 +3,12 @@
# Makefile for cpuidle governors.
#
CFLAGS_qcom_lpm.o := -I$(src)
obj-$(CONFIG_CPU_IDLE_GOV_LADDER) += ladder.o
obj-$(CONFIG_CPU_IDLE_GOV_MENU) += menu.o
obj-$(CONFIG_CPU_IDLE_GOV_QCOM_LPM) += qcom_lpm.o
qcom_lpm-y += qcom-lpm.o
qcom_lpm-y += qcom-cluster-lpm.o
qcom_lpm-y += qcom-lpm-sysfs.o
obj-$(CONFIG_CPU_IDLE_GOV_TEO) += teo.o
obj-$(CONFIG_CPU_IDLE_GOV_HALTPOLL) += haltpoll.o

519
drivers/cpuidle/governors/qcom-cluster-lpm.c Normal file
View File

@ -0,0 +1,519 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
*/
#include <linux/cpu.h>
#include <linux/cpuidle.h>
#include <linux/cpu_pm.h>
#include <linux/kernel.h>
#include <linux/ktime.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/pm_domain.h>
#include <linux/pm_runtime.h>
#include <linux/sched/idle.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/tick.h>
#include <linux/time64.h>
#if defined(_TRACE_HOOK_PM_DOMAIN_H)
#include <trace/hooks/pm_domain.h>
#endif
#define CREATE_TRACE_POINTS
#include "trace-cluster-lpm.h"
#include "qcom-lpm.h"
LIST_HEAD(cluster_dev_list);
static struct lpm_cluster *to_cluster(struct generic_pm_domain *genpd)
{
struct lpm_cluster *cluster_gov;
list_for_each_entry(cluster_gov, &cluster_dev_list, list)
if (cluster_gov->genpd == genpd)
return cluster_gov;
return NULL;
}
/**
* clusttimer_fn() - Will be executed when cluster prediction timer expires
* @h: Cluster prediction timer
*/
static enum hrtimer_restart clusttimer_fn(struct hrtimer *h)
{
struct lpm_cluster *cluster_gov = container_of(h,
struct lpm_cluster, histtimer);
cluster_gov->history_invalid = true;
return HRTIMER_NORESTART;
}
/**
* clusttimer_start() - Programs the hrtimer with given timer value
* @time_ns: Value to be program
*/
static void clusttimer_start(struct lpm_cluster *cluster_gov, uint32_t time_us)
{
struct hrtimer *timer = &cluster_gov->histtimer;
uint64_t time_ns = time_us * NSEC_PER_USEC;
ktime_t clust_ktime = ns_to_ktime(time_ns);
timer->function = clusttimer_fn;
hrtimer_start(timer, clust_ktime, HRTIMER_MODE_REL_PINNED);
}
/**
* clusttimer_cancel() - Cancel the hrtimr after cluster wakeup from sleep
* @cluster_gov: Targeted cluster's lpm data structure
*/
static void clusttimer_cancel(struct lpm_cluster *cluster_gov)
{
ktime_t time_rem;
time_rem = hrtimer_get_remaining(&cluster_gov->histtimer);
if (ktime_to_us(time_rem) > 0)
hrtimer_try_to_cancel(&cluster_gov->histtimer);
}
/**
* cluster_predict() - Predict the cluster's next wakeup.
* @cluster_gov: Targeted cluster's lpm data structure
*/
static void cluster_predict(struct lpm_cluster *cluster_gov)
{
struct generic_pm_domain *genpd = cluster_gov->genpd;
int i, j, idx = genpd->state_idx;
int64_t cur_time = ktime_to_us(cluster_gov->now);
uint64_t avg_residency = 0;
cluster_gov->pred_wakeup = KTIME_MAX;
cluster_gov->predicted = false;
if (prediction_disabled)
return;
/*
* Samples are marked invalid when woken-up due to timer,
* so do not predict.
*/
if (cluster_gov->history_invalid) {
cluster_gov->history_invalid = false;
cluster_gov->htmr_wkup = true;
return;
}
/*
* Cluster wakes up whenever any core of the cluster wakes up.
* Since for the last cluster LPM exit, there could be multiple core(s)
* LPMs. So, consider only recent history for the cluster.
*/
if (cluster_gov->nsamp == MAXSAMPLES) {
for (i = 0; i < MAXSAMPLES; i++) {
if ((cur_time - cluster_gov->history[i].entry_time)
> CLUST_SMPL_INVLD_TIME)
cluster_gov->nsamp--;
}
}
/* Predict only when all the samples are collected. */
if (cluster_gov->nsamp < MAXSAMPLES)
return;
/*
* If cluster's last entered mode is shallower state then calculate
* the next predicted wakeup as avg of previous samples
*/
if (idx < genpd->state_count - 1) {
for (i = 0; i < MAXSAMPLES; i++)
avg_residency += cluster_gov->history[i].residency;
do_div(avg_residency, MAXSAMPLES);
cluster_gov->pred_wakeup = ktime_add_us(avg_residency,
cluster_gov->now);
cluster_gov->predicted = true;
return;
}
/*
* Find the number of premature exits for each of the mode,
* excluding clockgating mode, and they are more than fifty
* percent restrict that and deeper modes.
*/
for (j = 1; j < genpd->state_count; j++) {
uint32_t count = 0;
u32 residency = genpd->states[j].residency_ns;
avg_residency = 0;
for (i = 0; i < MAXSAMPLES; i++) {
if ((cluster_gov->history[i].mode == j) &&
(cluster_gov->history[i].residency <
do_div(residency, NSEC_PER_USEC))) {
count++;
avg_residency +=
cluster_gov->history[i].residency;
}
}
if (count > PRED_PREMATURE_CNT) {
do_div(avg_residency, count);
cluster_gov->pred_wakeup = ktime_add_us(cluster_gov->now,
avg_residency);
cluster_gov->predicted = true;
return;
}
}
}
/**
* clear_cluster_history() - Clears the stored previous samples data.
* It will be called when APSS going to deep sleep.
* @cluster_gov: Targeted cluster's lpm data structure
*/
static void clear_cluster_history(struct lpm_cluster *cluster_gov)
{
int i;
for (i = 0; i < MAXSAMPLES; i++) {
cluster_gov->history[i].residency = 0;
cluster_gov->history[i].mode = -1;
cluster_gov->history[i].entry_time = 0;
}
cluster_gov->samples_idx = 0;
cluster_gov->nsamp = 0;
cluster_gov->history_invalid = false;
cluster_gov->htmr_wkup = false;
}
/**
* update_cluster_history() - Update the smaples history data every time when
* cluster exit from sleep.
* @cluster_gov: Targeted cluster's lpm data structure
*/
static void update_cluster_history(struct lpm_cluster *cluster_gov)
{
bool tmr = false;
uint32_t residency = 0;
struct generic_pm_domain *genpd = cluster_gov->genpd;
int idx = genpd->state_idx, samples_idx = cluster_gov->samples_idx;
if (prediction_disabled)
return;
if ((cluster_gov->entry_idx == -1) || (cluster_gov->entry_idx == idx)) {
residency = ktime_sub(cluster_gov->now, cluster_gov->entry_time);
residency = ktime_to_us(residency);
cluster_gov->history[samples_idx].entry_time =
ktime_to_us(cluster_gov->entry_time);
} else
return;
if (cluster_gov->htmr_wkup) {
if (!samples_idx)
samples_idx = MAXSAMPLES - 1;
else
samples_idx--;
cluster_gov->history[samples_idx].residency += residency;
cluster_gov->htmr_wkup = false;
tmr = true;
} else
cluster_gov->history[samples_idx].residency = residency;
cluster_gov->history[samples_idx].mode = idx;
cluster_gov->entry_idx = INT_MIN;
cluster_gov->entry_time = 0;
if (cluster_gov->nsamp < MAXSAMPLES)
cluster_gov->nsamp++;
trace_cluster_pred_hist(cluster_gov->history[samples_idx].mode,
cluster_gov->history[samples_idx].residency,
samples_idx, tmr);
samples_idx++;
if (samples_idx >= MAXSAMPLES)
samples_idx = 0;
cluster_gov->samples_idx = samples_idx;
}
/**
* cluster_power_down() - Will be called when cluster domain going to power off.
* If this entry's next wakeup was predicted it programs
* the cluster prediction timer and stores the idx entering
* and entry time of this lpm into clusters private data
* structure.
* @cluster_gov: cluster's lpm data structure
*/
static void cluster_power_down(struct lpm_cluster *cluster_gov)
{
struct generic_pm_domain *genpd = cluster_gov->genpd;
int idx = genpd->state_idx;
uint32_t residency;
if (idx < 0)
return;
cluster_gov->entry_time = cluster_gov->now;
cluster_gov->entry_idx = idx;
trace_cluster_pred_select(genpd->state_idx, genpd->next_wakeup,
0, cluster_gov->predicted, cluster_gov->next_wakeup);
if (idx >= genpd->state_count - 1) {
clear_cpu_predict_history();
clear_cluster_history(cluster_gov);
return;
}
if (ktime_compare(cluster_gov->next_wakeup, cluster_gov->pred_wakeup))
return;
residency = genpd->states[idx + 1].residency_ns;
do_div(residency, NSEC_PER_USEC);
clusttimer_start(cluster_gov, residency + PRED_TIMER_ADD);
}
/**
* cluster_power_cb() - It will be called when cluster domain power_off/power_on
* @nb: notifier block of the cluster
* @action: action i.e power_off/power_on
* @data: pointer to private data structure
*
* It returns the NOTIFY_OK/NOTIFY_BAD to notify the notifier call chain
*/
static int cluster_power_cb(struct notifier_block *nb,
unsigned long action, void *data)
{
struct lpm_cluster *cluster_gov = container_of(nb, struct lpm_cluster, genpd_nb);
struct generic_pm_domain *pd = cluster_gov->genpd;
struct genpd_power_state *state = &pd->states[pd->state_idx];
struct lpm_cpu *cpu_gov;
int cpu;
u32 *suspend_param = state->data;
switch (action) {
case GENPD_NOTIFY_ON:
trace_cluster_exit(raw_smp_processor_id(), pd->state_idx, *suspend_param);
if (cluster_gov->genpd->suspended_count != 0)
break;
cluster_gov->now = ktime_get();
clusttimer_cancel(cluster_gov);
update_cluster_history(cluster_gov);
cluster_predict(cluster_gov);
break;
case GENPD_NOTIFY_PRE_OFF:
if (!cluster_gov->state_allowed[pd->state_idx])
return NOTIFY_BAD;
if (cluster_gov->genpd->suspended_count != 0) {
clear_cpu_predict_history();
clear_cluster_history(cluster_gov);
break;
}
for_each_cpu(cpu, cluster_gov->genpd->cpus) {
if (cpu_online(cpu)) {
cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu);
if (cpu_gov->ipi_pending)
return NOTIFY_BAD;
}
}
cluster_gov->now = ktime_get();
cluster_power_down(cluster_gov);
break;
case GENPD_NOTIFY_OFF:
trace_cluster_enter(raw_smp_processor_id(), pd->state_idx, *suspend_param);
break;
default:
break;
}
return NOTIFY_OK;
}
/**
* get_cluster_sleep_time() - It returns the aggregated next_wakeup of all cpus
* which are in online for this cluster domain.
* @cluster_gov: Targeted cluster's lpm data structure
*/
ktime_t get_cluster_sleep_time(struct lpm_cluster *cluster_gov)
{
int cpu;
ktime_t next_wakeup, next_cpu_wakeup;
struct generic_pm_domain *genpd = cluster_gov->genpd;
next_wakeup = KTIME_MAX;
for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) {
next_cpu_wakeup = cluster_gov->cpu_next_wakeup[cpu];
if (ktime_before(next_cpu_wakeup, next_wakeup))
next_wakeup = next_cpu_wakeup;
}
return next_wakeup;
}
/**
* update_cluster_next_wakeup() - Update the this cluster device next wakeup with
* aggregated next_wakeup of all cpus which are in
* lpm for this cluster or this clusters predicted
* next wakeup whichever is earlier.
* @cluster_gov: Targeted cluster's lpm data structure
*/
static void update_cluster_next_wakeup(struct lpm_cluster *cluster_gov)
{
cluster_gov->next_wakeup = get_cluster_sleep_time(cluster_gov);
if (cluster_gov->pred_wakeup) {
if (ktime_before(cluster_gov->pred_wakeup,
cluster_gov->next_wakeup))
cluster_gov->next_wakeup = cluster_gov->pred_wakeup;
}
dev_pm_genpd_set_next_wakeup(cluster_gov->dev,
cluster_gov->next_wakeup);
}
/**
* update_cluster_select() - This will be called when cpu is going to lpm to update
* its next wakeup value to corresponding cluster domain device.
* @cpu_gov: CPU's lpm data structure.
*/
void update_cluster_select(struct lpm_cpu *cpu_gov)
{
struct generic_pm_domain *genpd;
struct lpm_cluster *cluster_gov;
int cpu = cpu_gov->cpu;
list_for_each_entry(cluster_gov, &cluster_dev_list, list) {
if (!cluster_gov->initialized)
continue;
spin_lock(&cluster_gov->lock);
cluster_gov->now = cpu_gov->now;
genpd = cluster_gov->genpd;
if (cpumask_test_cpu(cpu, genpd->cpus)) {
cluster_gov->cpu_next_wakeup[cpu] = cpu_gov->next_wakeup;
update_cluster_next_wakeup(cluster_gov);
}
spin_unlock(&cluster_gov->lock);
}
}
#if defined(_TRACE_HOOK_PM_DOMAIN_H)
static void android_vh_allow_domain_state(void *unused,
struct generic_pm_domain *genpd,
uint32_t idx, bool *allow)
{
struct lpm_cluster *cluster_gov = to_cluster(genpd);
if (!cluster_gov)
return;
*allow = cluster_gov->state_allowed[idx];
}
#endif
static void cluster_gov_disable(void)
{
#if defined(_TRACE_HOOK_PM_DOMAIN_H)
unregister_trace_android_vh_allow_domain_state(android_vh_allow_domain_state, NULL);
#endif
}
static void cluster_gov_enable(void)
{
#if defined(_TRACE_HOOK_PM_DOMAIN_H)
register_trace_android_vh_allow_domain_state(android_vh_allow_domain_state, NULL);
#endif
}
struct cluster_governor gov_ops = {
.select = update_cluster_select,
.enable = cluster_gov_enable,
.disable = cluster_gov_disable,
};
static int lpm_cluster_gov_remove(struct platform_device *pdev)
{
struct generic_pm_domain *genpd = pd_to_genpd(pdev->dev.pm_domain);
struct lpm_cluster *cluster_gov = to_cluster(genpd);
if (!cluster_gov)
return -ENODEV;
pm_runtime_disable(&pdev->dev);
cluster_gov->genpd->flags &= ~GENPD_FLAG_MIN_RESIDENCY;
remove_cluster_sysfs_nodes(cluster_gov);
dev_pm_genpd_remove_notifier(cluster_gov->dev);
return 0;
}
static int lpm_cluster_gov_probe(struct platform_device *pdev)
{
int ret;
int i;
struct lpm_cluster *cluster_gov;
cluster_gov = devm_kzalloc(&pdev->dev,
sizeof(struct lpm_cluster),
GFP_KERNEL);
if (!cluster_gov)
return -ENOMEM;
spin_lock_init(&cluster_gov->lock);
cluster_gov->dev = &pdev->dev;
cluster_gov->pred_wakeup = KTIME_MAX;
pm_runtime_enable(&pdev->dev);
hrtimer_init(&cluster_gov->histtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
cluster_gov->genpd = pd_to_genpd(cluster_gov->dev->pm_domain);
cluster_gov->genpd_nb.notifier_call = cluster_power_cb;
cluster_gov->genpd->flags |= GENPD_FLAG_MIN_RESIDENCY;
ret = dev_pm_genpd_add_notifier(cluster_gov->dev,
&cluster_gov->genpd_nb);
if (ret) {
pm_runtime_disable(&pdev->dev);
return ret;
}
if (create_cluster_sysfs_nodes(cluster_gov)) {
pm_runtime_disable(&pdev->dev);
return ret;
}
list_add_tail(&cluster_gov->list, &cluster_dev_list);
cluster_gov->initialized = true;
for (i = 0; i < cluster_gov->genpd->state_count; i++)
cluster_gov->state_allowed[i] = true;
register_cluster_governor_ops(&gov_ops);
return 0;
}
static const struct of_device_id qcom_cluster_lpm[] = {
{ .compatible = "qcom,lpm-cluster-dev" },
{ }
};
static struct platform_driver qcom_cluster_lpm_driver = {
.probe = lpm_cluster_gov_probe,
.remove = lpm_cluster_gov_remove,
.driver = {
.name = "qcom-cluster-lpm-gov",
.of_match_table = qcom_cluster_lpm,
.suppress_bind_attrs = true,
},
};
void qcom_cluster_lpm_governor_deinit(void)
{
platform_driver_unregister(&qcom_cluster_lpm_driver);
}
int qcom_cluster_lpm_governor_init(void)
{
return platform_driver_register(&qcom_cluster_lpm_driver);
}

212
drivers/cpuidle/governors/qcom-lpm-sysfs.c Normal file
View File

@ -0,0 +1,212 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
*/
#include <linux/cpu.h>
#include <linux/cpuidle.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pm_domain.h>
#include <linux/slab.h>
#include <linux/string.h>
#include "qcom-lpm.h"
static struct kobject *qcom_lpm_kobj;
static ssize_t cluster_idle_set(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t len)
{
struct qcom_cluster_node *d = container_of(attr, struct qcom_cluster_node, disable_attr);
bool disable;
int ret;
ret = strtobool(buf, &disable);
if (ret)
return -EINVAL;
d->cluster->state_allowed[d->state_idx] = !disable;
return len;
}
static ssize_t cluster_idle_get(struct kobject *kobj,
struct kobj_attribute *attr,
char *buf)
{
struct qcom_cluster_node *d = container_of(attr, struct qcom_cluster_node, disable_attr);
return scnprintf(buf, PAGE_SIZE, "%d\n", !d->cluster->state_allowed[d->state_idx]);
}
static int create_cluster_state_node(struct device *dev, struct qcom_cluster_node *d)
{
struct kobj_attribute *attr = &d->disable_attr;
int ret;
d->attr_group = devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
if (!d->attr_group)
return -ENOMEM;
d->attrs = devm_kcalloc(dev, 2, sizeof(struct attribute *), GFP_KERNEL);
if (!d->attrs)
return -ENOMEM;
sysfs_attr_init(&attr->attr);
attr->attr.name = "disable";
attr->attr.mode = 0644;
attr->show = cluster_idle_get;
attr->store = cluster_idle_set;
d->attrs[0] = &attr->attr;
d->attrs[1] = NULL;
d->attr_group->attrs = d->attrs;
ret = sysfs_create_group(d->kobj, d->attr_group);
if (ret)
return -ENOMEM;
return ret;
}
void remove_cluster_sysfs_nodes(struct lpm_cluster *cluster)
{
struct generic_pm_domain *genpd = cluster->genpd;
struct kobject *kobj = cluster->dev_kobj;
int i;
if (!qcom_lpm_kobj)
return;
for (i = 0; i < genpd->state_count; i++) {
struct qcom_cluster_node *d = cluster->dev_node[i];
kobject_put(d->kobj);
}
kobject_put(kobj);
}
int create_cluster_sysfs_nodes(struct lpm_cluster *cluster)
{
char name[10];
int i, ret;
struct generic_pm_domain *genpd = cluster->genpd;
if (!qcom_lpm_kobj)
return -EPROBE_DEFER;
cluster->dev_kobj = kobject_create_and_add(genpd->name, qcom_lpm_kobj);
if (!cluster->dev_kobj)
return -ENOMEM;
for (i = 0; i < genpd->state_count; i++) {
struct qcom_cluster_node *d;
d = devm_kzalloc(cluster->dev, sizeof(*d), GFP_KERNEL);
if (!d)
return -ENOMEM;
d->state_idx = i;
d->cluster = cluster;
scnprintf(name, PAGE_SIZE, "D%u", i);
d->kobj = kobject_create_and_add(name, cluster->dev_kobj);
if (!d->kobj) {
kobject_put(cluster->dev_kobj);
return -ENOMEM;
}
ret = create_cluster_state_node(cluster->dev, d);
if (ret) {
kobject_put(d->kobj);
kobject_put(cluster->dev_kobj);
return ret;
}
cluster->dev_node[i] = d;
}
return 0;
}
static ssize_t sleep_disabled_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *buf)
{
return scnprintf(buf, PAGE_SIZE, "%u\n", sleep_disabled);
}
static ssize_t sleep_disabled_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count)
{
bool val;
int ret;
ret = kstrtobool(buf, &val);
if (ret) {
pr_err("Invalid argument passed\n");
return ret;
}
sleep_disabled = val;
return count;
}
static ssize_t prediction_disabled_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *buf)
{
return scnprintf(buf, PAGE_SIZE, "%u\n", prediction_disabled);
}
static ssize_t prediction_disabled_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count)
{
bool val;
int ret;
ret = kstrtobool(buf, &val);
if (ret) {
pr_err("Invalid argument passed\n");
return ret;
}
prediction_disabled = val;
return count;
}
static struct kobj_attribute attr_sleep_disabled = __ATTR_RW(sleep_disabled);
static struct kobj_attribute attr_prediction_disabled = __ATTR_RW(prediction_disabled);
static struct attribute *lpm_gov_attrs[] = {
&attr_sleep_disabled.attr,
&attr_prediction_disabled.attr,
NULL
};
static struct attribute_group lpm_gov_attr_group = {
.attrs = lpm_gov_attrs,
.name = "parameters",
};
void remove_global_sysfs_nodes(void)
{
kobject_put(qcom_lpm_kobj);
}
int create_global_sysfs_nodes(void)
{
struct kobject *cpuidle_kobj = &cpu_subsys.dev_root->kobj;
qcom_lpm_kobj = kobject_create_and_add(KBUILD_MODNAME, cpuidle_kobj);
if (!qcom_lpm_kobj)
return -ENOMEM;
return sysfs_create_group(qcom_lpm_kobj, &lpm_gov_attr_group);
}

833
drivers/cpuidle/governors/qcom-lpm.c Normal file
View File

@ -0,0 +1,833 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2006-2007 Adam Belay <abelay@novell.com>
* Copyright (C) 2009 Intel Corporation
* Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
*/
#include <linux/cpu.h>
#include <linux/cpuidle.h>
#include <linux/cpu_pm.h>
#include <linux/ktime.h>
#include <linux/module.h>
#include <linux/pm_domain.h>
#include <linux/pm_runtime.h>
#include <linux/pm_qos.h>
#include <linux/sched/idle.h>
#if IS_ENABLED(CONFIG_SCHED_WALT)
#include <linux/sched/walt.h>
#endif
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/suspend.h>
#include <linux/tick.h>
#include <linux/time64.h>
#include <trace/events/ipi.h>
#include <trace/events/power.h>
#include <trace/hooks/cpuidle.h>
#include "qcom-lpm.h"
#define CREATE_TRACE_POINTS
#include "trace-qcom-lpm.h"
#define LPM_SELECT_STATE_DISABLED 0
#define LPM_SELECT_STATE_QOS_UNMET 1
#define LPM_SELECT_STATE_RESIDENCY_UNMET 2
#define LPM_SELECT_STATE_PRED 3
#define LPM_SELECT_STATE_IPI_PENDING 4
#define LPM_SELECT_STATE_SCHED_BIAS 5
#define LPM_SELECT_STATE_MAX 7
#define UPDATE_REASON(i, u) (BIT(u) << (MAX_LPM_CPUS * i))
bool prediction_disabled;
bool sleep_disabled = true;
static bool suspend_in_progress;
static bool traces_registered;
static struct cluster_governor *cluster_gov_ops;
DEFINE_PER_CPU(struct lpm_cpu, lpm_cpu_data);
static inline bool check_cpu_isactive(int cpu)
{
return cpu_active(cpu);
}
static bool lpm_disallowed(s64 sleep_ns, int cpu)
{
#if IS_ENABLED(CONFIG_SCHED_WALT)
struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu);
uint64_t bias_time = 0;
#endif
if (suspend_in_progress)
return true;
if (!check_cpu_isactive(cpu))
return false;
if ((sleep_disabled || sleep_ns < 0))
return true;
#if IS_ENABLED(CONFIG_SCHED_WALT)
if (!sched_lpm_disallowed_time(cpu, &bias_time)) {
cpu_gov->last_idx = 0;
cpu_gov->bias = bias_time;
return true;
}
#endif
return false;
}
/**
* histtimer_fn() - Will be executed when per cpu prediction timer expires
* @h: cpu prediction timer
*/
static enum hrtimer_restart histtimer_fn(struct hrtimer *h)
{
struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data);
cpu_gov->history_invalid = 1;
return HRTIMER_NORESTART;
}
/**
* histtimer_start() - Program the hrtimer with given timer value
* @time_ns: Value to be program
*/
static void histtimer_start(uint32_t time_ns)
{
ktime_t hist_ktime = ns_to_ktime(time_ns * NSEC_PER_USEC);
struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data);
struct hrtimer *cpu_histtimer = &cpu_gov->histtimer;
cpu_histtimer->function = histtimer_fn;
hrtimer_start(cpu_histtimer, hist_ktime, HRTIMER_MODE_REL_PINNED);
}
/**
* histtimer_cancel() - Cancel the histtimer after cpu wakes up from lpm
*/
static void histtimer_cancel(void)
{
struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data);
struct hrtimer *cpu_histtimer = &cpu_gov->histtimer;
ktime_t time_rem;
if (!hrtimer_active(cpu_histtimer))
return;
time_rem = hrtimer_get_remaining(cpu_histtimer);
if (ktime_to_us(time_rem) <= 0)
return;
hrtimer_try_to_cancel(cpu_histtimer);
}
static void biastimer_cancel(void)
{
struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data);
struct hrtimer *cpu_biastimer = &cpu_gov->biastimer;
ktime_t time_rem;
if (!cpu_gov->bias)
return;
cpu_gov->bias = 0;
time_rem = hrtimer_get_remaining(cpu_biastimer);
if (ktime_to_us(time_rem) <= 0)
return;
hrtimer_try_to_cancel(cpu_biastimer);
}
static enum hrtimer_restart biastimer_fn(struct hrtimer *h)
{
return HRTIMER_NORESTART;
}
static void biastimer_start(uint32_t time_ns)
{
ktime_t bias_ktime = ns_to_ktime(time_ns);
struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data);
struct hrtimer *cpu_biastimer = &cpu_gov->biastimer;
cpu_biastimer->function = biastimer_fn;
hrtimer_start(cpu_biastimer, bias_ktime, HRTIMER_MODE_REL_PINNED);
}
/**
* find_deviation() - Try to detect repeat patterns by keeping track of past
* samples and check if the standard deviation of that set
* of previous sample is below a threshold. If it is below
* threshold then use average of these past samples as
* predicted value.
* @cpu_gov: targeted cpu's lpm data structure
* @duration_ns: cpu's scheduler sleep length
*/
static uint64_t find_deviation(struct lpm_cpu *cpu_gov, int *samples_history,
u64 duration_ns)
{
uint64_t max, avg, stddev;
uint64_t thresh = LLONG_MAX;
struct cpuidle_driver *drv = cpu_gov->drv;
int divisor, i, last_level = drv->state_count - 1;
struct cpuidle_state *max_state = &drv->states[last_level];
do {
max = avg = divisor = stddev = 0;
for (i = 0; i < MAXSAMPLES; i++) {
int64_t value = samples_history[i];
if (value <= thresh) {
avg += value;
divisor++;
if (value > max)
max = value;
}
}
do_div(avg, divisor);
for (i = 0; i < MAXSAMPLES; i++) {
int64_t value = samples_history[i];
if (value <= thresh) {
int64_t diff = value - avg;
stddev += diff * diff;
}
}
do_div(stddev, divisor);
stddev = int_sqrt(stddev);
/*
* If the deviation is less, return the average, else
* ignore one maximum sample and retry
*/
if (((avg > stddev * 6) && (divisor >= (MAXSAMPLES - 1)))
|| stddev <= PRED_REF_STDDEV) {
do_div(duration_ns, NSEC_PER_USEC);
if (avg >= duration_ns ||
avg > max_state->target_residency)
return 0;
cpu_gov->next_pred_time = ktime_to_us(cpu_gov->now) + avg;
return avg;
}
thresh = max - 1;
} while (divisor > (MAXSAMPLES - 1));
return 0;
}
/**
* cpu_predict() - Predict the cpus next wakeup.
* @cpu_gov: targeted cpu's lpm data structure
* @duration_ns: cpu's scheduler sleep length
*/
static void cpu_predict(struct lpm_cpu *cpu_gov, u64 duration_ns)
{
int i, j;
struct cpuidle_driver *drv = cpu_gov->drv;
struct cpuidle_state *min_state = &drv->states[0];
struct history_lpm *lpm_history = &cpu_gov->lpm_history;
struct history_ipi *ipi_history = &cpu_gov->ipi_history;
if (prediction_disabled)
return;
/*
* Samples are marked invalid when woken-up due to timer,
* so do not predict.
*/
if (cpu_gov->history_invalid) {
cpu_gov->history_invalid = false;
cpu_gov->htmr_wkup = true;
cpu_gov->next_pred_time = 0;
return;
}
/*
* If the duration_ns itself is not sufficient for deeper
* low power modes than clock gating do not predict
*/
if (min_state->target_residency_ns > duration_ns)
return;
/* Predict only when all the samples are collected */
if (lpm_history->nsamp < MAXSAMPLES) {
cpu_gov->next_pred_time = 0;
return;
}
/*
* Check if the samples are not much deviated, if so use the
* average of those as predicted sleep time. Else if any
* specific mode has more premature exits return the index of
* that mode.
*/
cpu_gov->predicted = find_deviation(cpu_gov, lpm_history->resi, duration_ns);
if (cpu_gov->predicted)
return;
/*
* Find the number of premature exits for each of the mode,
* excluding clockgating mode, and they are more than fifty
* percent restrict that and deeper modes.
*/
for (j = 1; j < drv->state_count; j++) {
struct cpuidle_state *s = &drv->states[j];
uint32_t min_residency = s->target_residency;
uint32_t count = 0;
uint64_t avg_residency = 0;
for (i = 0; i < MAXSAMPLES; i++) {
if ((lpm_history->mode[i] == j) &&
(lpm_history->resi[i] < min_residency)) {
count++;
avg_residency += lpm_history->resi[i];
}
}
if (count >= PRED_PREMATURE_CNT) {
do_div(avg_residency, count);
cpu_gov->predicted = avg_residency;
cpu_gov->next_pred_time = ktime_to_ns(cpu_gov->now)
+ cpu_gov->predicted;
break;
}
}
if (cpu_gov->predicted)
return;
cpu_gov->predicted = find_deviation(cpu_gov, ipi_history->interval,
duration_ns);
}
/**
* clear_cpu_predict_history() - Clears the stored previous samples data.
* It will be called when APSS going to deep sleep.
*/
void clear_cpu_predict_history(void)
{
struct lpm_cpu *cpu_gov;
struct history_lpm *lpm_history;
int i, cpu;
if (prediction_disabled)
return;
for_each_possible_cpu(cpu) {
cpu_gov = this_cpu_ptr(&lpm_cpu_data);
lpm_history = &cpu_gov->lpm_history;
for (i = 0; i < MAXSAMPLES; i++) {
lpm_history->resi[i] = 0;
lpm_history->mode[i] = -1;
lpm_history->samples_idx = 0;
lpm_history->nsamp = 0;
cpu_gov->next_pred_time = 0;
}
}
}
/**
* update_cpu_history() - Update the samples history data every time when
* cpu comes from sleep.
* @cpu_gov: targeted cpu's lpm data structure
*/
static void update_cpu_history(struct lpm_cpu *cpu_gov)
{
bool tmr = false;
int idx = cpu_gov->last_idx;
struct history_lpm *lpm_history = &cpu_gov->lpm_history;
u64 measured_us = ktime_to_us(cpu_gov->dev->last_residency_ns);
struct cpuidle_state *target;
if (prediction_disabled || idx < 0 || idx > cpu_gov->drv->state_count - 1)
return;
target = &cpu_gov->drv->states[idx];
if (measured_us > target->exit_latency)
measured_us -= target->exit_latency;
if (cpu_gov->htmr_wkup) {
if (!lpm_history->samples_idx)
lpm_history->samples_idx = MAXSAMPLES - 1;
else
lpm_history->samples_idx--;
lpm_history->resi[lpm_history->samples_idx] += measured_us;
cpu_gov->htmr_wkup = false;
tmr = true;
} else
lpm_history->resi[lpm_history->samples_idx] = measured_us;
lpm_history->mode[lpm_history->samples_idx] = idx;
trace_gov_pred_hist(idx, lpm_history->resi[lpm_history->samples_idx],
tmr);
if (lpm_history->nsamp < MAXSAMPLES)
lpm_history->nsamp++;
lpm_history->samples_idx++;
if (lpm_history->samples_idx >= MAXSAMPLES)
lpm_history->samples_idx = 0;
}
void update_ipi_history(int cpu)
{
struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu);
struct history_ipi *history = &cpu_gov->ipi_history;
ktime_t now = ktime_get();
history->interval[history->current_ptr] =
ktime_to_us(ktime_sub(now,
history->cpu_idle_resched_ts));
(history->current_ptr)++;
if (history->current_ptr >= MAXSAMPLES)
history->current_ptr = 0;
history->cpu_idle_resched_ts = now;
}
/**
* lpm_cpu_qos_notify() - It will be called when any new request came on PM QoS.
* It wakes up the cpu if it is in idle sleep to honour
* the new PM QoS request.
* @nfb: notifier block of the CPU
* @val: notification value
* @ptr: pointer to private data structure
*/
static int lpm_cpu_qos_notify(struct notifier_block *nfb,
unsigned long val, void *ptr)
{
struct lpm_cpu *cpu_gov = container_of(nfb, struct lpm_cpu, nb);
int cpu = cpu_gov->cpu;
if (!cpu_gov->enable)
return NOTIFY_OK;
preempt_disable();
if (cpu != smp_processor_id() && cpu_online(cpu) &&
check_cpu_isactive(cpu))
wake_up_if_idle(cpu);
preempt_enable();
return NOTIFY_OK;
}
static int lpm_offline_cpu(unsigned int cpu)
{
struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu);
struct device *dev = get_cpu_device(cpu);
if (!dev || !cpu_gov)
return 0;
dev_pm_qos_remove_notifier(dev, &cpu_gov->nb,
DEV_PM_QOS_RESUME_LATENCY);
return 0;
}
static int lpm_online_cpu(unsigned int cpu)
{
struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu);
struct device *dev = get_cpu_device(cpu);
if (!dev || !cpu_gov)
return 0;
cpu_gov->nb.notifier_call = lpm_cpu_qos_notify;
dev_pm_qos_add_notifier(dev, &cpu_gov->nb,
DEV_PM_QOS_RESUME_LATENCY);
return 0;
}
static void ipi_raise(void *ignore, const struct cpumask *mask, const char *unused)
{
int cpu;
if (suspend_in_progress)
return;
for_each_cpu(cpu, mask) {
per_cpu(lpm_cpu_data, cpu).ipi_pending = true;
update_ipi_history(cpu);
}
}
static void ipi_entry(void *ignore, const char *unused)
{
int cpu;
if (suspend_in_progress)
return;
cpu = raw_smp_processor_id();
per_cpu(lpm_cpu_data, cpu).ipi_pending = false;
}
/**
* get_cpus_qos() - Returns the aggrigated PM QoS request.
* @mask: cpumask of the cpus
*/
static inline s64 get_cpus_qos(const struct cpumask *mask)
{
int cpu;
s64 n, latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE * NSEC_PER_USEC;
for_each_cpu(cpu, mask) {
if (!check_cpu_isactive(cpu))
continue;
n = cpuidle_governor_latency_req(cpu);
if (n < latency)
latency = n;
}
return latency;
}
/**
* start_prediction_timer() - Programs the prediction hrtimer and make the timer
* to run. It wakes up the cpus from shallower state in
* misprediction case and saves the power by not letting
* the cpu remains in sollower state.
* @cpu_gov: cpu's lpm data structure
* @duration_us: cpu's scheduled sleep length
*/
static int start_prediction_timer(struct lpm_cpu *cpu_gov, int duration_us)
{
struct cpuidle_state *s;
uint32_t htime = 0, max_residency;
uint32_t last_level = cpu_gov->drv->state_count - 1;
if (!cpu_gov->predicted || cpu_gov->last_idx >= last_level)
return 0;
if (cpu_gov->next_wakeup > cpu_gov->next_pred_time)
cpu_gov->next_wakeup = cpu_gov->next_pred_time;
s = &cpu_gov->drv->states[cpu_gov->last_idx];
max_residency = s[cpu_gov->last_idx + 1].target_residency - 1;
htime = cpu_gov->predicted + PRED_TIMER_ADD;
if (htime > max_residency)
htime = max_residency;
if ((duration_us > htime) && ((duration_us - htime) > max_residency))
histtimer_start(htime);
return htime;
}
void register_cluster_governor_ops(struct cluster_governor *ops)
{
if (!ops)
return;
cluster_gov_ops = ops;
}
/**
* lpm_select() - Find the best idle state for the cpu device
* @dev: Target cpu
* @state: Entered state
* @stop_tick: Is the tick device stopped
*
* Return: Best cpu LPM mode to enter
*/
static int lpm_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
bool *stop_tick)
{
struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data);
s64 latency_req = get_cpus_qos(cpumask_of(dev->cpu));
ktime_t delta_tick;
u64 reason = 0;
uint64_t duration_ns, htime = 0;
int i = 0;
if (!cpu_gov)
return 0;
do_div(latency_req, NSEC_PER_USEC);
cpu_gov->predicted = 0;
cpu_gov->predict_started = false;
cpu_gov->now = ktime_get();
duration_ns = tick_nohz_get_sleep_length(&delta_tick);
update_cpu_history(cpu_gov);
if (lpm_disallowed(duration_ns, dev->cpu))
goto done;
for (i = drv->state_count - 1; i > 0; i--) {
struct cpuidle_state *s = &drv->states[i];
if (i && dev->states_usage[i].disable) {
reason |= UPDATE_REASON(i, LPM_SELECT_STATE_DISABLED);
continue;
}
if (latency_req < s->exit_latency) {
reason |= UPDATE_REASON(i, LPM_SELECT_STATE_QOS_UNMET);
continue;
}
if (s->target_residency_ns > duration_ns) {
reason |= UPDATE_REASON(i,
LPM_SELECT_STATE_RESIDENCY_UNMET);
continue;
}
if (check_cpu_isactive(dev->cpu) && !cpu_gov->predict_started) {
cpu_predict(cpu_gov, duration_ns);
cpu_gov->predict_started = true;
}
if (cpu_gov->predicted)
if (s->target_residency > cpu_gov->predicted) {
reason |= UPDATE_REASON(i,
LPM_SELECT_STATE_PRED);
continue;
}
break;
}
do_div(duration_ns, NSEC_PER_USEC);
cpu_gov->last_idx = i;
cpu_gov->next_wakeup = ktime_add_us(cpu_gov->now, duration_ns);
htime = start_prediction_timer(cpu_gov, duration_ns);
/* update this cpu next_wakeup into its parent power domain device */
if (cpu_gov->last_idx == drv->state_count - 1) {
if (cluster_gov_ops && cluster_gov_ops->select)
cluster_gov_ops->select(cpu_gov);
}
done:
if ((!cpu_gov->last_idx) && cpu_gov->bias) {
biastimer_start(cpu_gov->bias);
reason |= UPDATE_REASON(i, LPM_SELECT_STATE_SCHED_BIAS);
}
trace_lpm_gov_select(i, latency_req, duration_ns, reason);
trace_gov_pred_select(cpu_gov->predicted, cpu_gov->predicted, htime);
return i;
}
/**
* lpm_reflect() - Update the state entered by the cpu device
* @dev: Target CPU
* @state: Entered state
*/
static void lpm_reflect(struct cpuidle_device *dev, int state)
{
}
/**
* lpm_idle_enter() - Notification with cpuidle state during idle entry
* @unused: unused
* @state: selected state by governor's .select
* @dev: cpuidle_device
*/
static void lpm_idle_enter(void *unused, int *state, struct cpuidle_device *dev)
{
struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data);
u64 reason = 0;
/* Restrict to WFI state if there is an IPI pending on current CPU */
if (cpu_gov->ipi_pending) {
reason = UPDATE_REASON(*state, LPM_SELECT_STATE_IPI_PENDING);
*state = 0;
trace_lpm_gov_select(*state, 0xdeaffeed, 0xdeaffeed, reason);
}
}
/**
* lpm_idle_exit() - Notification with cpuidle state during idle exit
* @unused: unused
* @state: actual entered state by cpuidle
* @dev: cpuidle_device
*/
static void lpm_idle_exit(void *unused, int state, struct cpuidle_device *dev)
{
struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, dev->cpu);
if (cpu_gov->enable) {
histtimer_cancel();
biastimer_cancel();
}
}
/**
* lpm_enable_device() - Initialize the governor's data for the CPU
* @drv: cpuidle driver
* @dev: Target CPU
*/
static int lpm_enable_device(struct cpuidle_driver *drv,
struct cpuidle_device *dev)
{
struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, dev->cpu);
struct hrtimer *cpu_histtimer = &cpu_gov->histtimer;
struct hrtimer *cpu_biastimer = &cpu_gov->biastimer;
int ret;
hrtimer_init(cpu_histtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hrtimer_init(cpu_biastimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
if (!traces_registered) {
ret = register_trace_ipi_raise(ipi_raise, NULL);
if (ret)
return ret;
ret = register_trace_ipi_entry(ipi_entry, NULL);
if (ret) {
unregister_trace_ipi_raise(ipi_raise, NULL);
return ret;
}
ret = register_trace_prio_android_vh_cpu_idle_enter(
lpm_idle_enter, NULL, INT_MIN);
if (ret) {
unregister_trace_ipi_raise(ipi_raise, NULL);
unregister_trace_ipi_entry(ipi_entry, NULL);
return ret;
}
ret = register_trace_prio_android_vh_cpu_idle_exit(
lpm_idle_exit, NULL, INT_MIN);
if (ret) {
unregister_trace_ipi_raise(ipi_raise, NULL);
unregister_trace_ipi_entry(ipi_entry, NULL);
unregister_trace_android_vh_cpu_idle_enter(
lpm_idle_enter, NULL);
return ret;
}
if (cluster_gov_ops && cluster_gov_ops->enable)
cluster_gov_ops->enable();
traces_registered = true;
}
cpu_gov->cpu = dev->cpu;
cpu_gov->enable = true;
cpu_gov->drv = drv;
cpu_gov->dev = dev;
cpu_gov->last_idx = -1;
return 0;
}
/**
* lpm_disable_device() - Clean up the governor's data for the CPU
* @drv: cpuidle driver
* @dev: Target CPU
*/
static void lpm_disable_device(struct cpuidle_driver *drv,
struct cpuidle_device *dev)
{
struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, dev->cpu);
int cpu;
cpu_gov->enable = false;
cpu_gov->last_idx = -1;
for_each_possible_cpu(cpu) {
struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu);
if (cpu_gov->enable)
return;
}
if (traces_registered) {
unregister_trace_ipi_raise(ipi_raise, NULL);
unregister_trace_ipi_entry(ipi_entry, NULL);
unregister_trace_android_vh_cpu_idle_enter(
lpm_idle_enter, NULL);
unregister_trace_android_vh_cpu_idle_exit(
lpm_idle_exit, NULL);
if (cluster_gov_ops && cluster_gov_ops->disable)
cluster_gov_ops->disable();
traces_registered = false;
}
}
static void qcom_lpm_suspend_trace(void *unused, const char *action,
int event, bool start)
{
int cpu;
if (start && !strcmp("dpm_suspend_late", action)) {
suspend_in_progress = true;
for_each_online_cpu(cpu)
wake_up_if_idle(cpu);
return;
}
if (!start && !strcmp("dpm_resume_early", action)) {
suspend_in_progress = false;
for_each_online_cpu(cpu)
wake_up_if_idle(cpu);
}
}
static struct cpuidle_governor lpm_governor = {
.name = "qcom-cpu-lpm",
.rating = 50,
.enable = lpm_enable_device,
.disable = lpm_disable_device,
.select = lpm_select,
.reflect = lpm_reflect,
};
static int __init qcom_lpm_governor_init(void)
{
int ret;
ret = create_global_sysfs_nodes();
if (ret)
goto sysfs_fail;
ret = qcom_cluster_lpm_governor_init();
if (ret)
goto cluster_init_fail;
ret = cpuidle_register_governor(&lpm_governor);
if (ret)
goto cpuidle_reg_fail;
ret = register_trace_suspend_resume(qcom_lpm_suspend_trace, NULL);
if (ret)
goto cpuidle_reg_fail;
ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "qcom-cpu-lpm",
lpm_online_cpu, lpm_offline_cpu);
if (ret < 0)
goto cpuhp_setup_fail;
return 0;
cpuhp_setup_fail:
unregister_trace_suspend_resume(qcom_lpm_suspend_trace, NULL);
cpuidle_reg_fail:
qcom_cluster_lpm_governor_deinit();
cluster_init_fail:
remove_global_sysfs_nodes();
sysfs_fail:
return ret;
}
module_init(qcom_lpm_governor_init);
MODULE_DESCRIPTION("Qualcomm Technologies, Inc. cpuidle LPM governor");
MODULE_LICENSE("GPL v2");

115
drivers/cpuidle/governors/qcom-lpm.h Normal file
View File

@ -0,0 +1,115 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
*/
#ifndef __QCOM_LPM_H__
#define __QCOM_LPM_H__
#define MAX_LPM_CPUS 8
#define MAXSAMPLES 5
#define PRED_TIMER_ADD 100
#define PRED_PREMATURE_CNT 3
#define PRED_REF_STDDEV 500
#define CLUST_SMPL_INVLD_TIME 40000
#define MAX_CLUSTER_STATES 4
extern bool sleep_disabled;
extern bool prediction_disabled;
struct qcom_cluster_node {
struct lpm_cluster *cluster;
struct kobject *kobj;
int state_idx;
struct kobj_attribute disable_attr;
struct attribute_group *attr_group;
struct attribute **attrs;
};
struct history_lpm {
int mode[MAXSAMPLES];
uint32_t resi[MAXSAMPLES];
int nsamp;
uint32_t samples_idx;
};
struct history_ipi {
uint32_t interval[MAXSAMPLES];
uint32_t current_ptr;
ktime_t cpu_idle_resched_ts;
};
struct lpm_cpu {
int cpu;
int enable;
int last_idx;
struct notifier_block nb;
struct cpuidle_driver *drv;
struct cpuidle_device *dev;
ktime_t next_wakeup;
uint64_t predicted;
uint32_t history_invalid;
bool predict_started;
bool htmr_wkup;
struct hrtimer histtimer;
struct hrtimer biastimer;
struct history_lpm lpm_history;
struct history_ipi ipi_history;
ktime_t now;
uint64_t bias;
int64_t next_pred_time;
bool ipi_pending;
};
struct cluster_history {
uint64_t residency;
int mode;
uint64_t entry_time;
};
struct lpm_cluster {
struct device *dev;
uint32_t samples_idx;
bool history_invalid;
bool htmr_wkup;
int entry_idx;
int nsamp;
struct cluster_history history[MAXSAMPLES];
struct generic_pm_domain *genpd;
struct qcom_cluster_node *dev_node[MAX_CLUSTER_STATES];
struct kobject *dev_kobj;
struct notifier_block genpd_nb;
struct work_struct work;
struct hrtimer histtimer;
ktime_t entry_time;
ktime_t next_wakeup;
ktime_t pred_wakeup;
ktime_t now;
ktime_t cpu_next_wakeup[MAX_LPM_CPUS];
bool state_allowed[MAX_CLUSTER_STATES];
struct list_head list;
spinlock_t lock;
bool predicted;
bool initialized;
};
struct cluster_governor {
void (*select)(struct lpm_cpu *cpu_gov);
void (*enable)(void);
void (*disable)(void);
void (*reflect)(void);
};
DECLARE_PER_CPU(struct lpm_cpu, lpm_cpu_data);
int qcom_cluster_lpm_governor_init(void);
void qcom_cluster_lpm_governor_deinit(void);
void update_cluster_select(struct lpm_cpu *cpu_gov);
void clear_cpu_predict_history(void);
int create_global_sysfs_nodes(void);
int create_cluster_sysfs_nodes(struct lpm_cluster *cluster_gov);
void register_cluster_governor_ops(struct cluster_governor *ops);
void remove_global_sysfs_nodes(void);
void remove_cluster_sysfs_nodes(struct lpm_cluster *cluster_gov);
#endif /* __QCOM_LPM_H__ */

119
drivers/cpuidle/governors/trace-cluster-lpm.h Normal file
View File

@ -0,0 +1,119 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
*/
#if !defined(_TRACE_CLUSTER_LPM_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_CLUSTER_LPM_H
#undef TRACE_SYSTEM
#define TRACE_SYSTEM cluster_lpm
#include <linux/tracepoint.h>
TRACE_EVENT(cluster_pred_select,
TP_PROTO(int index, u32 sleep_us,
u32 latency, int pred, u32 pred_us),
TP_ARGS(index, sleep_us, latency, pred, pred_us),
TP_STRUCT__entry(
__field(int, index)
__field(u32, sleep_us)
__field(u32, latency)
__field(int, pred)
__field(u32, pred_us)
),
TP_fast_assign(
__entry->index = index;
__entry->sleep_us = sleep_us;
__entry->latency = latency;
__entry->pred = pred;
__entry->pred_us = pred_us;
),
TP_printk("idx:%d sleep_time:%u latency:%u pred:%d pred_us:%u",
__entry->index, __entry->sleep_us,
__entry->latency, __entry->pred, __entry->pred_us)
);
TRACE_EVENT(cluster_pred_hist,
TP_PROTO(int idx, u32 resi, u32 sample, u32 tmr),
TP_ARGS(idx, resi, sample, tmr),
TP_STRUCT__entry(
__field(int, idx)
__field(u32, resi)
__field(u32, sample)
__field(u32, tmr)
),
TP_fast_assign(
__entry->idx = idx;
__entry->resi = resi;
__entry->sample = sample;
__entry->tmr = tmr;
),
TP_printk("idx:%d resi:%u sample:%u tmr:%u",
__entry->idx, __entry->resi,
__entry->sample, __entry->tmr)
);
TRACE_EVENT(cluster_exit,
TP_PROTO(int cpu, u32 idx, u32 suspend_param),
TP_ARGS(cpu, idx, suspend_param),
TP_STRUCT__entry(
__field(int, cpu)
__field(u32, idx)
__field(u32, suspend_param)
),
TP_fast_assign(
__entry->cpu = cpu;
__entry->idx = idx;
__entry->suspend_param = suspend_param;
),
TP_printk("first cpu:%d idx:%u suspend_param:0x%x", __entry->cpu,
__entry->idx, __entry->suspend_param)
);
TRACE_EVENT(cluster_enter,
TP_PROTO(int cpu, u32 idx, u32 suspend_param),
TP_ARGS(cpu, idx, suspend_param),
TP_STRUCT__entry(
__field(int, cpu)
__field(u32, idx)
__field(u32, suspend_param)
),
TP_fast_assign(
__entry->cpu = cpu;
__entry->idx = idx;
__entry->suspend_param = suspend_param;
),
TP_printk("last cpu:%d idx:%u suspend_param:0x%x", __entry->cpu,
__entry->idx, __entry->suspend_param)
);
#endif /* _TRACE_QCOM_LPM_H */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE trace-cluster-lpm
#include <trace/define_trace.h>

89
drivers/cpuidle/governors/trace-qcom-lpm.h Normal file
View File

@ -0,0 +1,89 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2021, The Linux Foundation. All rights reserved.
*/
#if !defined(_TRACE_QCOM_LPM_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_QCOM_LPM_H
#undef TRACE_SYSTEM
#define TRACE_SYSTEM qcom_lpm
#include <linux/tracepoint.h>
TRACE_EVENT(lpm_gov_select,
TP_PROTO(int idx, s64 qos, u64 sleep, u64 reason),
TP_ARGS(idx, qos, sleep, reason),
TP_STRUCT__entry(
__field(int, idx)
__field(s64, qos)
__field(u64, sleep)
__field(u64, reason)
),
TP_fast_assign(
__entry->idx = idx;
__entry->qos = qos;
__entry->sleep = sleep;
__entry->reason = reason;
),
TP_printk("state:%d qos-us:%lld sleep-us:%llu reason:%#x",
__entry->idx, __entry->qos, __entry->sleep, __entry->reason)
);
TRACE_EVENT(gov_pred_select,
TP_PROTO(u32 predtype, u64 predicted, u32 tmr_time),
TP_ARGS(predtype, predicted, tmr_time),
TP_STRUCT__entry(
__field(u32, predtype)
__field(u64, predicted)
__field(u32, tmr_time)
),
TP_fast_assign(
__entry->predtype = predtype;
__entry->predicted = predicted;
__entry->tmr_time = tmr_time;
),
TP_printk("pred:%u time:%lu tmr_time:%u",
__entry->predtype, __entry->predicted, __entry->tmr_time)
);
TRACE_EVENT(gov_pred_hist,
TP_PROTO(int idx, int residency, int tmr),
TP_ARGS(idx, tmr, residency),
TP_STRUCT__entry(
__field(int, idx)
__field(int, residency)
__field(int, tmr)
),
TP_fast_assign(
__entry->idx = idx;
__entry->residency = residency;
__entry->tmr = tmr;
),
TP_printk("idx:%d residency=%d, tmr=%d", __entry->idx, __entry->residency, __entry->tmr)
);
#endif /* _TRACE_QCOM_LPM_H */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE trace-qcom-lpm
#include <trace/define_trace.h>