From c4354bba6e70026c7f89cc8aecea12915afe1db5 Mon Sep 17 00:00:00 2001 From: Rashid Zafar Date: Tue, 7 Jun 2022 12:28:24 -0700 Subject: [PATCH] cpuidle: governors: Add snapshot of qcom-lpm governor Add snapshot of qcom-lpm governor from msm-5.15 'commit <0050a5bbad2c> ("cpuidle: governors: qcom-lpm: Ignore IPI trace when suspend in progress")'. Change-Id: I354b9f34867d42ca0899adc3f593a4a3e33da20d Signed-off-by: Rashid Zafar --- drivers/cpuidle/Kconfig | 13 + drivers/cpuidle/governors/Makefile | 5 + drivers/cpuidle/governors/qcom-cluster-lpm.c | 519 +++++++++++ drivers/cpuidle/governors/qcom-lpm-sysfs.c | 212 +++++ drivers/cpuidle/governors/qcom-lpm.c | 833 ++++++++++++++++++ drivers/cpuidle/governors/qcom-lpm.h | 115 +++ drivers/cpuidle/governors/trace-cluster-lpm.h | 119 +++ drivers/cpuidle/governors/trace-qcom-lpm.h | 89 ++ 8 files changed, 1905 insertions(+) create mode 100644 drivers/cpuidle/governors/qcom-cluster-lpm.c create mode 100644 drivers/cpuidle/governors/qcom-lpm-sysfs.c create mode 100644 drivers/cpuidle/governors/qcom-lpm.c create mode 100644 drivers/cpuidle/governors/qcom-lpm.h create mode 100644 drivers/cpuidle/governors/trace-cluster-lpm.h create mode 100644 drivers/cpuidle/governors/trace-qcom-lpm.h diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig index ff71dd662880..923facc7521b 100644 --- a/drivers/cpuidle/Kconfig +++ b/drivers/cpuidle/Kconfig @@ -44,6 +44,19 @@ config CPU_IDLE_GOV_HALTPOLL Some virtualized workloads benefit from using it. +config CPU_IDLE_GOV_QCOM_LPM + tristate "Qualcomm Technologies, Inc. CPU and Cluster governor" + depends on ARCH_QCOM + depends on ARM_PSCI_CPUIDLE + help + This governor implements effective cpu and cluster idle state + selection with help of scheduler inputs, cpu idle state prediction + and cluster idle state prediction algorithms. + + The predicted sleep time, latency requirement for the + CPU and the idle state chosen based on the parameters are all + logged in the trace. + config DT_IDLE_STATES bool diff --git a/drivers/cpuidle/governors/Makefile b/drivers/cpuidle/governors/Makefile index 63abb5393a4d..c48a0cc953d3 100644 --- a/drivers/cpuidle/governors/Makefile +++ b/drivers/cpuidle/governors/Makefile @@ -3,7 +3,12 @@ # Makefile for cpuidle governors. # +CFLAGS_qcom_lpm.o := -I$(src) obj-$(CONFIG_CPU_IDLE_GOV_LADDER) += ladder.o obj-$(CONFIG_CPU_IDLE_GOV_MENU) += menu.o +obj-$(CONFIG_CPU_IDLE_GOV_QCOM_LPM) += qcom_lpm.o +qcom_lpm-y += qcom-lpm.o +qcom_lpm-y += qcom-cluster-lpm.o +qcom_lpm-y += qcom-lpm-sysfs.o obj-$(CONFIG_CPU_IDLE_GOV_TEO) += teo.o obj-$(CONFIG_CPU_IDLE_GOV_HALTPOLL) += haltpoll.o diff --git a/drivers/cpuidle/governors/qcom-cluster-lpm.c b/drivers/cpuidle/governors/qcom-cluster-lpm.c new file mode 100644 index 000000000000..f329f627556e --- /dev/null +++ b/drivers/cpuidle/governors/qcom-cluster-lpm.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_TRACE_HOOK_PM_DOMAIN_H) +#include +#endif + +#define CREATE_TRACE_POINTS +#include "trace-cluster-lpm.h" +#include "qcom-lpm.h" + +LIST_HEAD(cluster_dev_list); + +static struct lpm_cluster *to_cluster(struct generic_pm_domain *genpd) +{ + struct lpm_cluster *cluster_gov; + + list_for_each_entry(cluster_gov, &cluster_dev_list, list) + if (cluster_gov->genpd == genpd) + return cluster_gov; + + return NULL; +} + +/** + * clusttimer_fn() - Will be executed when cluster prediction timer expires + * @h: Cluster prediction timer + */ +static enum hrtimer_restart clusttimer_fn(struct hrtimer *h) +{ + struct lpm_cluster *cluster_gov = container_of(h, + struct lpm_cluster, histtimer); + + cluster_gov->history_invalid = true; + + return HRTIMER_NORESTART; +} + +/** + * clusttimer_start() - Programs the hrtimer with given timer value + * @time_ns: Value to be program + */ +static void clusttimer_start(struct lpm_cluster *cluster_gov, uint32_t time_us) +{ + struct hrtimer *timer = &cluster_gov->histtimer; + uint64_t time_ns = time_us * NSEC_PER_USEC; + ktime_t clust_ktime = ns_to_ktime(time_ns); + + timer->function = clusttimer_fn; + hrtimer_start(timer, clust_ktime, HRTIMER_MODE_REL_PINNED); +} + +/** + * clusttimer_cancel() - Cancel the hrtimr after cluster wakeup from sleep + * @cluster_gov: Targeted cluster's lpm data structure + */ +static void clusttimer_cancel(struct lpm_cluster *cluster_gov) +{ + ktime_t time_rem; + + time_rem = hrtimer_get_remaining(&cluster_gov->histtimer); + if (ktime_to_us(time_rem) > 0) + hrtimer_try_to_cancel(&cluster_gov->histtimer); +} + +/** + * cluster_predict() - Predict the cluster's next wakeup. + * @cluster_gov: Targeted cluster's lpm data structure + */ +static void cluster_predict(struct lpm_cluster *cluster_gov) +{ + struct generic_pm_domain *genpd = cluster_gov->genpd; + int i, j, idx = genpd->state_idx; + int64_t cur_time = ktime_to_us(cluster_gov->now); + uint64_t avg_residency = 0; + + cluster_gov->pred_wakeup = KTIME_MAX; + cluster_gov->predicted = false; + + if (prediction_disabled) + return; + + /* + * Samples are marked invalid when woken-up due to timer, + * so do not predict. + */ + if (cluster_gov->history_invalid) { + cluster_gov->history_invalid = false; + cluster_gov->htmr_wkup = true; + return; + } + + /* + * Cluster wakes up whenever any core of the cluster wakes up. + * Since for the last cluster LPM exit, there could be multiple core(s) + * LPMs. So, consider only recent history for the cluster. + */ + if (cluster_gov->nsamp == MAXSAMPLES) { + for (i = 0; i < MAXSAMPLES; i++) { + if ((cur_time - cluster_gov->history[i].entry_time) + > CLUST_SMPL_INVLD_TIME) + cluster_gov->nsamp--; + } + } + + /* Predict only when all the samples are collected. */ + if (cluster_gov->nsamp < MAXSAMPLES) + return; + + /* + * If cluster's last entered mode is shallower state then calculate + * the next predicted wakeup as avg of previous samples + */ + if (idx < genpd->state_count - 1) { + for (i = 0; i < MAXSAMPLES; i++) + avg_residency += cluster_gov->history[i].residency; + do_div(avg_residency, MAXSAMPLES); + cluster_gov->pred_wakeup = ktime_add_us(avg_residency, + cluster_gov->now); + cluster_gov->predicted = true; + return; + } + + /* + * Find the number of premature exits for each of the mode, + * excluding clockgating mode, and they are more than fifty + * percent restrict that and deeper modes. + */ + for (j = 1; j < genpd->state_count; j++) { + uint32_t count = 0; + u32 residency = genpd->states[j].residency_ns; + + avg_residency = 0; + for (i = 0; i < MAXSAMPLES; i++) { + + if ((cluster_gov->history[i].mode == j) && + (cluster_gov->history[i].residency < + do_div(residency, NSEC_PER_USEC))) { + count++; + avg_residency += + cluster_gov->history[i].residency; + } + } + + if (count > PRED_PREMATURE_CNT) { + do_div(avg_residency, count); + cluster_gov->pred_wakeup = ktime_add_us(cluster_gov->now, + avg_residency); + cluster_gov->predicted = true; + return; + } + } +} + +/** + * clear_cluster_history() - Clears the stored previous samples data. + * It will be called when APSS going to deep sleep. + * @cluster_gov: Targeted cluster's lpm data structure + */ +static void clear_cluster_history(struct lpm_cluster *cluster_gov) +{ + int i; + + for (i = 0; i < MAXSAMPLES; i++) { + cluster_gov->history[i].residency = 0; + cluster_gov->history[i].mode = -1; + cluster_gov->history[i].entry_time = 0; + } + + cluster_gov->samples_idx = 0; + cluster_gov->nsamp = 0; + cluster_gov->history_invalid = false; + cluster_gov->htmr_wkup = false; +} + +/** + * update_cluster_history() - Update the smaples history data every time when + * cluster exit from sleep. + * @cluster_gov: Targeted cluster's lpm data structure + */ +static void update_cluster_history(struct lpm_cluster *cluster_gov) +{ + bool tmr = false; + uint32_t residency = 0; + struct generic_pm_domain *genpd = cluster_gov->genpd; + int idx = genpd->state_idx, samples_idx = cluster_gov->samples_idx; + + if (prediction_disabled) + return; + + if ((cluster_gov->entry_idx == -1) || (cluster_gov->entry_idx == idx)) { + residency = ktime_sub(cluster_gov->now, cluster_gov->entry_time); + residency = ktime_to_us(residency); + cluster_gov->history[samples_idx].entry_time = + ktime_to_us(cluster_gov->entry_time); + } else + return; + + if (cluster_gov->htmr_wkup) { + if (!samples_idx) + samples_idx = MAXSAMPLES - 1; + else + samples_idx--; + cluster_gov->history[samples_idx].residency += residency; + cluster_gov->htmr_wkup = false; + tmr = true; + } else + cluster_gov->history[samples_idx].residency = residency; + + cluster_gov->history[samples_idx].mode = idx; + cluster_gov->entry_idx = INT_MIN; + cluster_gov->entry_time = 0; + if (cluster_gov->nsamp < MAXSAMPLES) + cluster_gov->nsamp++; + trace_cluster_pred_hist(cluster_gov->history[samples_idx].mode, + cluster_gov->history[samples_idx].residency, + samples_idx, tmr); + samples_idx++; + if (samples_idx >= MAXSAMPLES) + samples_idx = 0; + + cluster_gov->samples_idx = samples_idx; +} + +/** + * cluster_power_down() - Will be called when cluster domain going to power off. + * If this entry's next wakeup was predicted it programs + * the cluster prediction timer and stores the idx entering + * and entry time of this lpm into clusters private data + * structure. + * @cluster_gov: cluster's lpm data structure + */ +static void cluster_power_down(struct lpm_cluster *cluster_gov) +{ + struct generic_pm_domain *genpd = cluster_gov->genpd; + int idx = genpd->state_idx; + uint32_t residency; + + if (idx < 0) + return; + + cluster_gov->entry_time = cluster_gov->now; + cluster_gov->entry_idx = idx; + trace_cluster_pred_select(genpd->state_idx, genpd->next_wakeup, + 0, cluster_gov->predicted, cluster_gov->next_wakeup); + if (idx >= genpd->state_count - 1) { + clear_cpu_predict_history(); + clear_cluster_history(cluster_gov); + return; + } + if (ktime_compare(cluster_gov->next_wakeup, cluster_gov->pred_wakeup)) + return; + + residency = genpd->states[idx + 1].residency_ns; + do_div(residency, NSEC_PER_USEC); + clusttimer_start(cluster_gov, residency + PRED_TIMER_ADD); +} + +/** + * cluster_power_cb() - It will be called when cluster domain power_off/power_on + * @nb: notifier block of the cluster + * @action: action i.e power_off/power_on + * @data: pointer to private data structure + * + * It returns the NOTIFY_OK/NOTIFY_BAD to notify the notifier call chain + */ +static int cluster_power_cb(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct lpm_cluster *cluster_gov = container_of(nb, struct lpm_cluster, genpd_nb); + struct generic_pm_domain *pd = cluster_gov->genpd; + struct genpd_power_state *state = &pd->states[pd->state_idx]; + struct lpm_cpu *cpu_gov; + int cpu; + u32 *suspend_param = state->data; + + switch (action) { + case GENPD_NOTIFY_ON: + trace_cluster_exit(raw_smp_processor_id(), pd->state_idx, *suspend_param); + if (cluster_gov->genpd->suspended_count != 0) + break; + + cluster_gov->now = ktime_get(); + clusttimer_cancel(cluster_gov); + update_cluster_history(cluster_gov); + cluster_predict(cluster_gov); + break; + case GENPD_NOTIFY_PRE_OFF: + if (!cluster_gov->state_allowed[pd->state_idx]) + return NOTIFY_BAD; + + if (cluster_gov->genpd->suspended_count != 0) { + clear_cpu_predict_history(); + clear_cluster_history(cluster_gov); + break; + } + + for_each_cpu(cpu, cluster_gov->genpd->cpus) { + if (cpu_online(cpu)) { + cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu); + if (cpu_gov->ipi_pending) + return NOTIFY_BAD; + } + } + + cluster_gov->now = ktime_get(); + cluster_power_down(cluster_gov); + break; + case GENPD_NOTIFY_OFF: + trace_cluster_enter(raw_smp_processor_id(), pd->state_idx, *suspend_param); + break; + default: + break; + } + + return NOTIFY_OK; +} + +/** + * get_cluster_sleep_time() - It returns the aggregated next_wakeup of all cpus + * which are in online for this cluster domain. + * @cluster_gov: Targeted cluster's lpm data structure + */ +ktime_t get_cluster_sleep_time(struct lpm_cluster *cluster_gov) +{ + int cpu; + ktime_t next_wakeup, next_cpu_wakeup; + struct generic_pm_domain *genpd = cluster_gov->genpd; + + next_wakeup = KTIME_MAX; + for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) { + next_cpu_wakeup = cluster_gov->cpu_next_wakeup[cpu]; + if (ktime_before(next_cpu_wakeup, next_wakeup)) + next_wakeup = next_cpu_wakeup; + } + + return next_wakeup; +} + +/** + * update_cluster_next_wakeup() - Update the this cluster device next wakeup with + * aggregated next_wakeup of all cpus which are in + * lpm for this cluster or this clusters predicted + * next wakeup whichever is earlier. + * @cluster_gov: Targeted cluster's lpm data structure + */ +static void update_cluster_next_wakeup(struct lpm_cluster *cluster_gov) +{ + cluster_gov->next_wakeup = get_cluster_sleep_time(cluster_gov); + if (cluster_gov->pred_wakeup) { + if (ktime_before(cluster_gov->pred_wakeup, + cluster_gov->next_wakeup)) + cluster_gov->next_wakeup = cluster_gov->pred_wakeup; + } + + dev_pm_genpd_set_next_wakeup(cluster_gov->dev, + cluster_gov->next_wakeup); +} + +/** + * update_cluster_select() - This will be called when cpu is going to lpm to update + * its next wakeup value to corresponding cluster domain device. + * @cpu_gov: CPU's lpm data structure. + */ +void update_cluster_select(struct lpm_cpu *cpu_gov) +{ + struct generic_pm_domain *genpd; + struct lpm_cluster *cluster_gov; + int cpu = cpu_gov->cpu; + + list_for_each_entry(cluster_gov, &cluster_dev_list, list) { + if (!cluster_gov->initialized) + continue; + + spin_lock(&cluster_gov->lock); + cluster_gov->now = cpu_gov->now; + genpd = cluster_gov->genpd; + if (cpumask_test_cpu(cpu, genpd->cpus)) { + cluster_gov->cpu_next_wakeup[cpu] = cpu_gov->next_wakeup; + update_cluster_next_wakeup(cluster_gov); + } + spin_unlock(&cluster_gov->lock); + } +} + +#if defined(_TRACE_HOOK_PM_DOMAIN_H) +static void android_vh_allow_domain_state(void *unused, + struct generic_pm_domain *genpd, + uint32_t idx, bool *allow) +{ + struct lpm_cluster *cluster_gov = to_cluster(genpd); + + if (!cluster_gov) + return; + + *allow = cluster_gov->state_allowed[idx]; +} +#endif + +static void cluster_gov_disable(void) +{ +#if defined(_TRACE_HOOK_PM_DOMAIN_H) + unregister_trace_android_vh_allow_domain_state(android_vh_allow_domain_state, NULL); +#endif +} + +static void cluster_gov_enable(void) +{ +#if defined(_TRACE_HOOK_PM_DOMAIN_H) + register_trace_android_vh_allow_domain_state(android_vh_allow_domain_state, NULL); +#endif +} + +struct cluster_governor gov_ops = { + .select = update_cluster_select, + .enable = cluster_gov_enable, + .disable = cluster_gov_disable, +}; + +static int lpm_cluster_gov_remove(struct platform_device *pdev) +{ + struct generic_pm_domain *genpd = pd_to_genpd(pdev->dev.pm_domain); + struct lpm_cluster *cluster_gov = to_cluster(genpd); + + if (!cluster_gov) + return -ENODEV; + + pm_runtime_disable(&pdev->dev); + cluster_gov->genpd->flags &= ~GENPD_FLAG_MIN_RESIDENCY; + remove_cluster_sysfs_nodes(cluster_gov); + dev_pm_genpd_remove_notifier(cluster_gov->dev); + + return 0; +} + +static int lpm_cluster_gov_probe(struct platform_device *pdev) +{ + int ret; + int i; + struct lpm_cluster *cluster_gov; + + cluster_gov = devm_kzalloc(&pdev->dev, + sizeof(struct lpm_cluster), + GFP_KERNEL); + if (!cluster_gov) + return -ENOMEM; + + spin_lock_init(&cluster_gov->lock); + cluster_gov->dev = &pdev->dev; + cluster_gov->pred_wakeup = KTIME_MAX; + pm_runtime_enable(&pdev->dev); + hrtimer_init(&cluster_gov->histtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + cluster_gov->genpd = pd_to_genpd(cluster_gov->dev->pm_domain); + cluster_gov->genpd_nb.notifier_call = cluster_power_cb; + cluster_gov->genpd->flags |= GENPD_FLAG_MIN_RESIDENCY; + ret = dev_pm_genpd_add_notifier(cluster_gov->dev, + &cluster_gov->genpd_nb); + if (ret) { + pm_runtime_disable(&pdev->dev); + return ret; + } + + if (create_cluster_sysfs_nodes(cluster_gov)) { + pm_runtime_disable(&pdev->dev); + return ret; + } + + list_add_tail(&cluster_gov->list, &cluster_dev_list); + cluster_gov->initialized = true; + + for (i = 0; i < cluster_gov->genpd->state_count; i++) + cluster_gov->state_allowed[i] = true; + + register_cluster_governor_ops(&gov_ops); + + return 0; +} + +static const struct of_device_id qcom_cluster_lpm[] = { + { .compatible = "qcom,lpm-cluster-dev" }, + { } +}; + +static struct platform_driver qcom_cluster_lpm_driver = { + .probe = lpm_cluster_gov_probe, + .remove = lpm_cluster_gov_remove, + .driver = { + .name = "qcom-cluster-lpm-gov", + .of_match_table = qcom_cluster_lpm, + .suppress_bind_attrs = true, + }, +}; + +void qcom_cluster_lpm_governor_deinit(void) +{ + platform_driver_unregister(&qcom_cluster_lpm_driver); +} + +int qcom_cluster_lpm_governor_init(void) +{ + return platform_driver_register(&qcom_cluster_lpm_driver); +} diff --git a/drivers/cpuidle/governors/qcom-lpm-sysfs.c b/drivers/cpuidle/governors/qcom-lpm-sysfs.c new file mode 100644 index 000000000000..fff8e0f7dc5f --- /dev/null +++ b/drivers/cpuidle/governors/qcom-lpm-sysfs.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "qcom-lpm.h" + +static struct kobject *qcom_lpm_kobj; + +static ssize_t cluster_idle_set(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t len) +{ + struct qcom_cluster_node *d = container_of(attr, struct qcom_cluster_node, disable_attr); + bool disable; + int ret; + + ret = strtobool(buf, &disable); + if (ret) + return -EINVAL; + + d->cluster->state_allowed[d->state_idx] = !disable; + + return len; +} + +static ssize_t cluster_idle_get(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct qcom_cluster_node *d = container_of(attr, struct qcom_cluster_node, disable_attr); + + return scnprintf(buf, PAGE_SIZE, "%d\n", !d->cluster->state_allowed[d->state_idx]); +} + +static int create_cluster_state_node(struct device *dev, struct qcom_cluster_node *d) +{ + struct kobj_attribute *attr = &d->disable_attr; + int ret; + + d->attr_group = devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL); + if (!d->attr_group) + return -ENOMEM; + + d->attrs = devm_kcalloc(dev, 2, sizeof(struct attribute *), GFP_KERNEL); + if (!d->attrs) + return -ENOMEM; + + sysfs_attr_init(&attr->attr); + attr->attr.name = "disable"; + attr->attr.mode = 0644; + attr->show = cluster_idle_get; + attr->store = cluster_idle_set; + + d->attrs[0] = &attr->attr; + d->attrs[1] = NULL; + d->attr_group->attrs = d->attrs; + + ret = sysfs_create_group(d->kobj, d->attr_group); + if (ret) + return -ENOMEM; + + return ret; +} + +void remove_cluster_sysfs_nodes(struct lpm_cluster *cluster) +{ + struct generic_pm_domain *genpd = cluster->genpd; + struct kobject *kobj = cluster->dev_kobj; + int i; + + if (!qcom_lpm_kobj) + return; + + for (i = 0; i < genpd->state_count; i++) { + struct qcom_cluster_node *d = cluster->dev_node[i]; + + kobject_put(d->kobj); + } + + kobject_put(kobj); +} + +int create_cluster_sysfs_nodes(struct lpm_cluster *cluster) +{ + char name[10]; + int i, ret; + struct generic_pm_domain *genpd = cluster->genpd; + + if (!qcom_lpm_kobj) + return -EPROBE_DEFER; + + cluster->dev_kobj = kobject_create_and_add(genpd->name, qcom_lpm_kobj); + if (!cluster->dev_kobj) + return -ENOMEM; + + for (i = 0; i < genpd->state_count; i++) { + struct qcom_cluster_node *d; + + d = devm_kzalloc(cluster->dev, sizeof(*d), GFP_KERNEL); + if (!d) + return -ENOMEM; + + d->state_idx = i; + d->cluster = cluster; + scnprintf(name, PAGE_SIZE, "D%u", i); + d->kobj = kobject_create_and_add(name, cluster->dev_kobj); + if (!d->kobj) { + kobject_put(cluster->dev_kobj); + return -ENOMEM; + } + + ret = create_cluster_state_node(cluster->dev, d); + if (ret) { + kobject_put(d->kobj); + kobject_put(cluster->dev_kobj); + return ret; + } + + cluster->dev_node[i] = d; + } + + return 0; +} + +static ssize_t sleep_disabled_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", sleep_disabled); +} + +static ssize_t sleep_disabled_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + bool val; + int ret; + + ret = kstrtobool(buf, &val); + if (ret) { + pr_err("Invalid argument passed\n"); + return ret; + } + + sleep_disabled = val; + + return count; +} + +static ssize_t prediction_disabled_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", prediction_disabled); +} + +static ssize_t prediction_disabled_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + bool val; + int ret; + + ret = kstrtobool(buf, &val); + if (ret) { + pr_err("Invalid argument passed\n"); + return ret; + } + + prediction_disabled = val; + + return count; +} + +static struct kobj_attribute attr_sleep_disabled = __ATTR_RW(sleep_disabled); +static struct kobj_attribute attr_prediction_disabled = __ATTR_RW(prediction_disabled); + +static struct attribute *lpm_gov_attrs[] = { + &attr_sleep_disabled.attr, + &attr_prediction_disabled.attr, + NULL +}; + +static struct attribute_group lpm_gov_attr_group = { + .attrs = lpm_gov_attrs, + .name = "parameters", +}; + +void remove_global_sysfs_nodes(void) +{ + kobject_put(qcom_lpm_kobj); +} + +int create_global_sysfs_nodes(void) +{ + struct kobject *cpuidle_kobj = &cpu_subsys.dev_root->kobj; + + qcom_lpm_kobj = kobject_create_and_add(KBUILD_MODNAME, cpuidle_kobj); + if (!qcom_lpm_kobj) + return -ENOMEM; + + return sysfs_create_group(qcom_lpm_kobj, &lpm_gov_attr_group); +} diff --git a/drivers/cpuidle/governors/qcom-lpm.c b/drivers/cpuidle/governors/qcom-lpm.c new file mode 100644 index 000000000000..30f1ce160d76 --- /dev/null +++ b/drivers/cpuidle/governors/qcom-lpm.c @@ -0,0 +1,833 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2006-2007 Adam Belay + * Copyright (C) 2009 Intel Corporation + * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if IS_ENABLED(CONFIG_SCHED_WALT) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qcom-lpm.h" +#define CREATE_TRACE_POINTS +#include "trace-qcom-lpm.h" + +#define LPM_SELECT_STATE_DISABLED 0 +#define LPM_SELECT_STATE_QOS_UNMET 1 +#define LPM_SELECT_STATE_RESIDENCY_UNMET 2 +#define LPM_SELECT_STATE_PRED 3 +#define LPM_SELECT_STATE_IPI_PENDING 4 +#define LPM_SELECT_STATE_SCHED_BIAS 5 +#define LPM_SELECT_STATE_MAX 7 + +#define UPDATE_REASON(i, u) (BIT(u) << (MAX_LPM_CPUS * i)) + +bool prediction_disabled; +bool sleep_disabled = true; +static bool suspend_in_progress; +static bool traces_registered; +static struct cluster_governor *cluster_gov_ops; + +DEFINE_PER_CPU(struct lpm_cpu, lpm_cpu_data); + +static inline bool check_cpu_isactive(int cpu) +{ + return cpu_active(cpu); +} + +static bool lpm_disallowed(s64 sleep_ns, int cpu) +{ +#if IS_ENABLED(CONFIG_SCHED_WALT) + struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu); + uint64_t bias_time = 0; +#endif + + if (suspend_in_progress) + return true; + + if (!check_cpu_isactive(cpu)) + return false; + + if ((sleep_disabled || sleep_ns < 0)) + return true; + +#if IS_ENABLED(CONFIG_SCHED_WALT) + if (!sched_lpm_disallowed_time(cpu, &bias_time)) { + cpu_gov->last_idx = 0; + cpu_gov->bias = bias_time; + return true; + } +#endif + return false; +} + +/** + * histtimer_fn() - Will be executed when per cpu prediction timer expires + * @h: cpu prediction timer + */ +static enum hrtimer_restart histtimer_fn(struct hrtimer *h) +{ + struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data); + + cpu_gov->history_invalid = 1; + + return HRTIMER_NORESTART; +} + +/** + * histtimer_start() - Program the hrtimer with given timer value + * @time_ns: Value to be program + */ +static void histtimer_start(uint32_t time_ns) +{ + ktime_t hist_ktime = ns_to_ktime(time_ns * NSEC_PER_USEC); + struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data); + struct hrtimer *cpu_histtimer = &cpu_gov->histtimer; + + cpu_histtimer->function = histtimer_fn; + hrtimer_start(cpu_histtimer, hist_ktime, HRTIMER_MODE_REL_PINNED); +} + +/** + * histtimer_cancel() - Cancel the histtimer after cpu wakes up from lpm + */ +static void histtimer_cancel(void) +{ + struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data); + struct hrtimer *cpu_histtimer = &cpu_gov->histtimer; + ktime_t time_rem; + + if (!hrtimer_active(cpu_histtimer)) + return; + + time_rem = hrtimer_get_remaining(cpu_histtimer); + if (ktime_to_us(time_rem) <= 0) + return; + + hrtimer_try_to_cancel(cpu_histtimer); +} + +static void biastimer_cancel(void) +{ + struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data); + struct hrtimer *cpu_biastimer = &cpu_gov->biastimer; + ktime_t time_rem; + + if (!cpu_gov->bias) + return; + + cpu_gov->bias = 0; + time_rem = hrtimer_get_remaining(cpu_biastimer); + if (ktime_to_us(time_rem) <= 0) + return; + + hrtimer_try_to_cancel(cpu_biastimer); +} + +static enum hrtimer_restart biastimer_fn(struct hrtimer *h) +{ + return HRTIMER_NORESTART; +} + +static void biastimer_start(uint32_t time_ns) +{ + ktime_t bias_ktime = ns_to_ktime(time_ns); + struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data); + struct hrtimer *cpu_biastimer = &cpu_gov->biastimer; + + cpu_biastimer->function = biastimer_fn; + hrtimer_start(cpu_biastimer, bias_ktime, HRTIMER_MODE_REL_PINNED); +} + +/** + * find_deviation() - Try to detect repeat patterns by keeping track of past + * samples and check if the standard deviation of that set + * of previous sample is below a threshold. If it is below + * threshold then use average of these past samples as + * predicted value. + * @cpu_gov: targeted cpu's lpm data structure + * @duration_ns: cpu's scheduler sleep length + */ +static uint64_t find_deviation(struct lpm_cpu *cpu_gov, int *samples_history, + u64 duration_ns) +{ + uint64_t max, avg, stddev; + uint64_t thresh = LLONG_MAX; + struct cpuidle_driver *drv = cpu_gov->drv; + int divisor, i, last_level = drv->state_count - 1; + struct cpuidle_state *max_state = &drv->states[last_level]; + + do { + max = avg = divisor = stddev = 0; + for (i = 0; i < MAXSAMPLES; i++) { + int64_t value = samples_history[i]; + + if (value <= thresh) { + avg += value; + divisor++; + if (value > max) + max = value; + } + } + do_div(avg, divisor); + + for (i = 0; i < MAXSAMPLES; i++) { + int64_t value = samples_history[i]; + + if (value <= thresh) { + int64_t diff = value - avg; + + stddev += diff * diff; + } + } + do_div(stddev, divisor); + stddev = int_sqrt(stddev); + + /* + * If the deviation is less, return the average, else + * ignore one maximum sample and retry + */ + if (((avg > stddev * 6) && (divisor >= (MAXSAMPLES - 1))) + || stddev <= PRED_REF_STDDEV) { + do_div(duration_ns, NSEC_PER_USEC); + if (avg >= duration_ns || + avg > max_state->target_residency) + return 0; + + cpu_gov->next_pred_time = ktime_to_us(cpu_gov->now) + avg; + return avg; + } + thresh = max - 1; + + } while (divisor > (MAXSAMPLES - 1)); + + return 0; +} + +/** + * cpu_predict() - Predict the cpus next wakeup. + * @cpu_gov: targeted cpu's lpm data structure + * @duration_ns: cpu's scheduler sleep length + */ +static void cpu_predict(struct lpm_cpu *cpu_gov, u64 duration_ns) +{ + int i, j; + struct cpuidle_driver *drv = cpu_gov->drv; + struct cpuidle_state *min_state = &drv->states[0]; + struct history_lpm *lpm_history = &cpu_gov->lpm_history; + struct history_ipi *ipi_history = &cpu_gov->ipi_history; + + if (prediction_disabled) + return; + + /* + * Samples are marked invalid when woken-up due to timer, + * so do not predict. + */ + if (cpu_gov->history_invalid) { + cpu_gov->history_invalid = false; + cpu_gov->htmr_wkup = true; + cpu_gov->next_pred_time = 0; + return; + } + + /* + * If the duration_ns itself is not sufficient for deeper + * low power modes than clock gating do not predict + */ + if (min_state->target_residency_ns > duration_ns) + return; + + /* Predict only when all the samples are collected */ + if (lpm_history->nsamp < MAXSAMPLES) { + cpu_gov->next_pred_time = 0; + return; + } + + /* + * Check if the samples are not much deviated, if so use the + * average of those as predicted sleep time. Else if any + * specific mode has more premature exits return the index of + * that mode. + */ + cpu_gov->predicted = find_deviation(cpu_gov, lpm_history->resi, duration_ns); + if (cpu_gov->predicted) + return; + + /* + * Find the number of premature exits for each of the mode, + * excluding clockgating mode, and they are more than fifty + * percent restrict that and deeper modes. + */ + for (j = 1; j < drv->state_count; j++) { + struct cpuidle_state *s = &drv->states[j]; + uint32_t min_residency = s->target_residency; + uint32_t count = 0; + uint64_t avg_residency = 0; + + for (i = 0; i < MAXSAMPLES; i++) { + if ((lpm_history->mode[i] == j) && + (lpm_history->resi[i] < min_residency)) { + count++; + avg_residency += lpm_history->resi[i]; + } + } + + if (count >= PRED_PREMATURE_CNT) { + do_div(avg_residency, count); + cpu_gov->predicted = avg_residency; + cpu_gov->next_pred_time = ktime_to_ns(cpu_gov->now) + + cpu_gov->predicted; + break; + } + } + + if (cpu_gov->predicted) + return; + + cpu_gov->predicted = find_deviation(cpu_gov, ipi_history->interval, + duration_ns); +} + +/** + * clear_cpu_predict_history() - Clears the stored previous samples data. + * It will be called when APSS going to deep sleep. + */ +void clear_cpu_predict_history(void) +{ + struct lpm_cpu *cpu_gov; + struct history_lpm *lpm_history; + int i, cpu; + + if (prediction_disabled) + return; + + for_each_possible_cpu(cpu) { + cpu_gov = this_cpu_ptr(&lpm_cpu_data); + lpm_history = &cpu_gov->lpm_history; + for (i = 0; i < MAXSAMPLES; i++) { + lpm_history->resi[i] = 0; + lpm_history->mode[i] = -1; + lpm_history->samples_idx = 0; + lpm_history->nsamp = 0; + cpu_gov->next_pred_time = 0; + } + } +} + +/** + * update_cpu_history() - Update the samples history data every time when + * cpu comes from sleep. + * @cpu_gov: targeted cpu's lpm data structure + */ +static void update_cpu_history(struct lpm_cpu *cpu_gov) +{ + bool tmr = false; + int idx = cpu_gov->last_idx; + struct history_lpm *lpm_history = &cpu_gov->lpm_history; + u64 measured_us = ktime_to_us(cpu_gov->dev->last_residency_ns); + struct cpuidle_state *target; + + if (prediction_disabled || idx < 0 || idx > cpu_gov->drv->state_count - 1) + return; + + target = &cpu_gov->drv->states[idx]; + + if (measured_us > target->exit_latency) + measured_us -= target->exit_latency; + + if (cpu_gov->htmr_wkup) { + if (!lpm_history->samples_idx) + lpm_history->samples_idx = MAXSAMPLES - 1; + else + lpm_history->samples_idx--; + + lpm_history->resi[lpm_history->samples_idx] += measured_us; + cpu_gov->htmr_wkup = false; + tmr = true; + } else + lpm_history->resi[lpm_history->samples_idx] = measured_us; + + lpm_history->mode[lpm_history->samples_idx] = idx; + + trace_gov_pred_hist(idx, lpm_history->resi[lpm_history->samples_idx], + tmr); + + if (lpm_history->nsamp < MAXSAMPLES) + lpm_history->nsamp++; + + lpm_history->samples_idx++; + if (lpm_history->samples_idx >= MAXSAMPLES) + lpm_history->samples_idx = 0; +} + +void update_ipi_history(int cpu) +{ + struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu); + struct history_ipi *history = &cpu_gov->ipi_history; + ktime_t now = ktime_get(); + + history->interval[history->current_ptr] = + ktime_to_us(ktime_sub(now, + history->cpu_idle_resched_ts)); + (history->current_ptr)++; + if (history->current_ptr >= MAXSAMPLES) + history->current_ptr = 0; + + history->cpu_idle_resched_ts = now; +} + +/** + * lpm_cpu_qos_notify() - It will be called when any new request came on PM QoS. + * It wakes up the cpu if it is in idle sleep to honour + * the new PM QoS request. + * @nfb: notifier block of the CPU + * @val: notification value + * @ptr: pointer to private data structure + */ +static int lpm_cpu_qos_notify(struct notifier_block *nfb, + unsigned long val, void *ptr) +{ + struct lpm_cpu *cpu_gov = container_of(nfb, struct lpm_cpu, nb); + int cpu = cpu_gov->cpu; + + if (!cpu_gov->enable) + return NOTIFY_OK; + + preempt_disable(); + if (cpu != smp_processor_id() && cpu_online(cpu) && + check_cpu_isactive(cpu)) + wake_up_if_idle(cpu); + preempt_enable(); + + return NOTIFY_OK; +} + +static int lpm_offline_cpu(unsigned int cpu) +{ + struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu); + struct device *dev = get_cpu_device(cpu); + + if (!dev || !cpu_gov) + return 0; + + dev_pm_qos_remove_notifier(dev, &cpu_gov->nb, + DEV_PM_QOS_RESUME_LATENCY); + + return 0; +} + +static int lpm_online_cpu(unsigned int cpu) +{ + struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu); + struct device *dev = get_cpu_device(cpu); + + if (!dev || !cpu_gov) + return 0; + + cpu_gov->nb.notifier_call = lpm_cpu_qos_notify; + dev_pm_qos_add_notifier(dev, &cpu_gov->nb, + DEV_PM_QOS_RESUME_LATENCY); + + return 0; +} + +static void ipi_raise(void *ignore, const struct cpumask *mask, const char *unused) +{ + int cpu; + + if (suspend_in_progress) + return; + + for_each_cpu(cpu, mask) { + per_cpu(lpm_cpu_data, cpu).ipi_pending = true; + update_ipi_history(cpu); + } +} + +static void ipi_entry(void *ignore, const char *unused) +{ + int cpu; + + if (suspend_in_progress) + return; + + cpu = raw_smp_processor_id(); + per_cpu(lpm_cpu_data, cpu).ipi_pending = false; +} + +/** + * get_cpus_qos() - Returns the aggrigated PM QoS request. + * @mask: cpumask of the cpus + */ +static inline s64 get_cpus_qos(const struct cpumask *mask) +{ + int cpu; + s64 n, latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE * NSEC_PER_USEC; + + for_each_cpu(cpu, mask) { + if (!check_cpu_isactive(cpu)) + continue; + n = cpuidle_governor_latency_req(cpu); + if (n < latency) + latency = n; + } + + return latency; +} + +/** + * start_prediction_timer() - Programs the prediction hrtimer and make the timer + * to run. It wakes up the cpus from shallower state in + * misprediction case and saves the power by not letting + * the cpu remains in sollower state. + * @cpu_gov: cpu's lpm data structure + * @duration_us: cpu's scheduled sleep length + */ +static int start_prediction_timer(struct lpm_cpu *cpu_gov, int duration_us) +{ + struct cpuidle_state *s; + uint32_t htime = 0, max_residency; + uint32_t last_level = cpu_gov->drv->state_count - 1; + + if (!cpu_gov->predicted || cpu_gov->last_idx >= last_level) + return 0; + + if (cpu_gov->next_wakeup > cpu_gov->next_pred_time) + cpu_gov->next_wakeup = cpu_gov->next_pred_time; + + s = &cpu_gov->drv->states[cpu_gov->last_idx]; + max_residency = s[cpu_gov->last_idx + 1].target_residency - 1; + htime = cpu_gov->predicted + PRED_TIMER_ADD; + + if (htime > max_residency) + htime = max_residency; + + if ((duration_us > htime) && ((duration_us - htime) > max_residency)) + histtimer_start(htime); + + return htime; +} + +void register_cluster_governor_ops(struct cluster_governor *ops) +{ + if (!ops) + return; + + cluster_gov_ops = ops; +} + +/** + * lpm_select() - Find the best idle state for the cpu device + * @dev: Target cpu + * @state: Entered state + * @stop_tick: Is the tick device stopped + * + * Return: Best cpu LPM mode to enter + */ +static int lpm_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, + bool *stop_tick) +{ + struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data); + s64 latency_req = get_cpus_qos(cpumask_of(dev->cpu)); + ktime_t delta_tick; + u64 reason = 0; + uint64_t duration_ns, htime = 0; + int i = 0; + + if (!cpu_gov) + return 0; + + do_div(latency_req, NSEC_PER_USEC); + cpu_gov->predicted = 0; + cpu_gov->predict_started = false; + cpu_gov->now = ktime_get(); + duration_ns = tick_nohz_get_sleep_length(&delta_tick); + update_cpu_history(cpu_gov); + + if (lpm_disallowed(duration_ns, dev->cpu)) + goto done; + + for (i = drv->state_count - 1; i > 0; i--) { + struct cpuidle_state *s = &drv->states[i]; + + if (i && dev->states_usage[i].disable) { + reason |= UPDATE_REASON(i, LPM_SELECT_STATE_DISABLED); + continue; + } + + if (latency_req < s->exit_latency) { + reason |= UPDATE_REASON(i, LPM_SELECT_STATE_QOS_UNMET); + continue; + } + + if (s->target_residency_ns > duration_ns) { + reason |= UPDATE_REASON(i, + LPM_SELECT_STATE_RESIDENCY_UNMET); + continue; + } + + if (check_cpu_isactive(dev->cpu) && !cpu_gov->predict_started) { + cpu_predict(cpu_gov, duration_ns); + cpu_gov->predict_started = true; + } + + if (cpu_gov->predicted) + if (s->target_residency > cpu_gov->predicted) { + reason |= UPDATE_REASON(i, + LPM_SELECT_STATE_PRED); + continue; + } + break; + } + + do_div(duration_ns, NSEC_PER_USEC); + cpu_gov->last_idx = i; + cpu_gov->next_wakeup = ktime_add_us(cpu_gov->now, duration_ns); + htime = start_prediction_timer(cpu_gov, duration_ns); + + /* update this cpu next_wakeup into its parent power domain device */ + if (cpu_gov->last_idx == drv->state_count - 1) { + if (cluster_gov_ops && cluster_gov_ops->select) + cluster_gov_ops->select(cpu_gov); + } + +done: + if ((!cpu_gov->last_idx) && cpu_gov->bias) { + biastimer_start(cpu_gov->bias); + reason |= UPDATE_REASON(i, LPM_SELECT_STATE_SCHED_BIAS); + } + + trace_lpm_gov_select(i, latency_req, duration_ns, reason); + trace_gov_pred_select(cpu_gov->predicted, cpu_gov->predicted, htime); + + return i; +} + +/** + * lpm_reflect() - Update the state entered by the cpu device + * @dev: Target CPU + * @state: Entered state + */ +static void lpm_reflect(struct cpuidle_device *dev, int state) +{ + +} + +/** + * lpm_idle_enter() - Notification with cpuidle state during idle entry + * @unused: unused + * @state: selected state by governor's .select + * @dev: cpuidle_device + */ +static void lpm_idle_enter(void *unused, int *state, struct cpuidle_device *dev) +{ + struct lpm_cpu *cpu_gov = this_cpu_ptr(&lpm_cpu_data); + u64 reason = 0; + + /* Restrict to WFI state if there is an IPI pending on current CPU */ + if (cpu_gov->ipi_pending) { + reason = UPDATE_REASON(*state, LPM_SELECT_STATE_IPI_PENDING); + *state = 0; + trace_lpm_gov_select(*state, 0xdeaffeed, 0xdeaffeed, reason); + } +} + +/** + * lpm_idle_exit() - Notification with cpuidle state during idle exit + * @unused: unused + * @state: actual entered state by cpuidle + * @dev: cpuidle_device + */ +static void lpm_idle_exit(void *unused, int state, struct cpuidle_device *dev) +{ + struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, dev->cpu); + + if (cpu_gov->enable) { + histtimer_cancel(); + biastimer_cancel(); + } +} + +/** + * lpm_enable_device() - Initialize the governor's data for the CPU + * @drv: cpuidle driver + * @dev: Target CPU + */ +static int lpm_enable_device(struct cpuidle_driver *drv, + struct cpuidle_device *dev) +{ + struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, dev->cpu); + struct hrtimer *cpu_histtimer = &cpu_gov->histtimer; + struct hrtimer *cpu_biastimer = &cpu_gov->biastimer; + int ret; + + hrtimer_init(cpu_histtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer_init(cpu_biastimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + if (!traces_registered) { + ret = register_trace_ipi_raise(ipi_raise, NULL); + if (ret) + return ret; + + ret = register_trace_ipi_entry(ipi_entry, NULL); + if (ret) { + unregister_trace_ipi_raise(ipi_raise, NULL); + return ret; + } + + ret = register_trace_prio_android_vh_cpu_idle_enter( + lpm_idle_enter, NULL, INT_MIN); + if (ret) { + unregister_trace_ipi_raise(ipi_raise, NULL); + unregister_trace_ipi_entry(ipi_entry, NULL); + return ret; + } + + ret = register_trace_prio_android_vh_cpu_idle_exit( + lpm_idle_exit, NULL, INT_MIN); + if (ret) { + unregister_trace_ipi_raise(ipi_raise, NULL); + unregister_trace_ipi_entry(ipi_entry, NULL); + unregister_trace_android_vh_cpu_idle_enter( + lpm_idle_enter, NULL); + return ret; + } + + if (cluster_gov_ops && cluster_gov_ops->enable) + cluster_gov_ops->enable(); + + traces_registered = true; + } + + cpu_gov->cpu = dev->cpu; + cpu_gov->enable = true; + cpu_gov->drv = drv; + cpu_gov->dev = dev; + cpu_gov->last_idx = -1; + + return 0; +} + +/** + * lpm_disable_device() - Clean up the governor's data for the CPU + * @drv: cpuidle driver + * @dev: Target CPU + */ +static void lpm_disable_device(struct cpuidle_driver *drv, + struct cpuidle_device *dev) +{ + struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, dev->cpu); + int cpu; + + cpu_gov->enable = false; + cpu_gov->last_idx = -1; + for_each_possible_cpu(cpu) { + struct lpm_cpu *cpu_gov = per_cpu_ptr(&lpm_cpu_data, cpu); + + if (cpu_gov->enable) + return; + } + + if (traces_registered) { + unregister_trace_ipi_raise(ipi_raise, NULL); + unregister_trace_ipi_entry(ipi_entry, NULL); + unregister_trace_android_vh_cpu_idle_enter( + lpm_idle_enter, NULL); + unregister_trace_android_vh_cpu_idle_exit( + lpm_idle_exit, NULL); + if (cluster_gov_ops && cluster_gov_ops->disable) + cluster_gov_ops->disable(); + + traces_registered = false; + } +} + +static void qcom_lpm_suspend_trace(void *unused, const char *action, + int event, bool start) +{ + int cpu; + + if (start && !strcmp("dpm_suspend_late", action)) { + suspend_in_progress = true; + + for_each_online_cpu(cpu) + wake_up_if_idle(cpu); + return; + } + + if (!start && !strcmp("dpm_resume_early", action)) { + suspend_in_progress = false; + + for_each_online_cpu(cpu) + wake_up_if_idle(cpu); + } +} + +static struct cpuidle_governor lpm_governor = { + .name = "qcom-cpu-lpm", + .rating = 50, + .enable = lpm_enable_device, + .disable = lpm_disable_device, + .select = lpm_select, + .reflect = lpm_reflect, +}; + +static int __init qcom_lpm_governor_init(void) +{ + int ret; + + ret = create_global_sysfs_nodes(); + if (ret) + goto sysfs_fail; + + ret = qcom_cluster_lpm_governor_init(); + if (ret) + goto cluster_init_fail; + + ret = cpuidle_register_governor(&lpm_governor); + if (ret) + goto cpuidle_reg_fail; + + ret = register_trace_suspend_resume(qcom_lpm_suspend_trace, NULL); + if (ret) + goto cpuidle_reg_fail; + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "qcom-cpu-lpm", + lpm_online_cpu, lpm_offline_cpu); + if (ret < 0) + goto cpuhp_setup_fail; + + return 0; + +cpuhp_setup_fail: + unregister_trace_suspend_resume(qcom_lpm_suspend_trace, NULL); +cpuidle_reg_fail: + qcom_cluster_lpm_governor_deinit(); +cluster_init_fail: + remove_global_sysfs_nodes(); +sysfs_fail: + return ret; +} +module_init(qcom_lpm_governor_init); + +MODULE_DESCRIPTION("Qualcomm Technologies, Inc. cpuidle LPM governor"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpuidle/governors/qcom-lpm.h b/drivers/cpuidle/governors/qcom-lpm.h new file mode 100644 index 000000000000..e124aefb7d54 --- /dev/null +++ b/drivers/cpuidle/governors/qcom-lpm.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#ifndef __QCOM_LPM_H__ +#define __QCOM_LPM_H__ + +#define MAX_LPM_CPUS 8 +#define MAXSAMPLES 5 +#define PRED_TIMER_ADD 100 +#define PRED_PREMATURE_CNT 3 +#define PRED_REF_STDDEV 500 +#define CLUST_SMPL_INVLD_TIME 40000 +#define MAX_CLUSTER_STATES 4 + +extern bool sleep_disabled; +extern bool prediction_disabled; + +struct qcom_cluster_node { + struct lpm_cluster *cluster; + struct kobject *kobj; + int state_idx; + struct kobj_attribute disable_attr; + struct attribute_group *attr_group; + struct attribute **attrs; +}; + +struct history_lpm { + int mode[MAXSAMPLES]; + uint32_t resi[MAXSAMPLES]; + int nsamp; + uint32_t samples_idx; +}; + +struct history_ipi { + uint32_t interval[MAXSAMPLES]; + uint32_t current_ptr; + ktime_t cpu_idle_resched_ts; +}; + +struct lpm_cpu { + int cpu; + int enable; + int last_idx; + struct notifier_block nb; + struct cpuidle_driver *drv; + struct cpuidle_device *dev; + ktime_t next_wakeup; + uint64_t predicted; + uint32_t history_invalid; + bool predict_started; + bool htmr_wkup; + struct hrtimer histtimer; + struct hrtimer biastimer; + struct history_lpm lpm_history; + struct history_ipi ipi_history; + ktime_t now; + uint64_t bias; + int64_t next_pred_time; + bool ipi_pending; +}; + +struct cluster_history { + uint64_t residency; + int mode; + uint64_t entry_time; +}; + +struct lpm_cluster { + struct device *dev; + uint32_t samples_idx; + bool history_invalid; + bool htmr_wkup; + int entry_idx; + int nsamp; + struct cluster_history history[MAXSAMPLES]; + struct generic_pm_domain *genpd; + struct qcom_cluster_node *dev_node[MAX_CLUSTER_STATES]; + struct kobject *dev_kobj; + struct notifier_block genpd_nb; + struct work_struct work; + struct hrtimer histtimer; + ktime_t entry_time; + ktime_t next_wakeup; + ktime_t pred_wakeup; + ktime_t now; + ktime_t cpu_next_wakeup[MAX_LPM_CPUS]; + bool state_allowed[MAX_CLUSTER_STATES]; + struct list_head list; + spinlock_t lock; + bool predicted; + bool initialized; +}; + +struct cluster_governor { + void (*select)(struct lpm_cpu *cpu_gov); + void (*enable)(void); + void (*disable)(void); + void (*reflect)(void); +}; + +DECLARE_PER_CPU(struct lpm_cpu, lpm_cpu_data); + +int qcom_cluster_lpm_governor_init(void); +void qcom_cluster_lpm_governor_deinit(void); +void update_cluster_select(struct lpm_cpu *cpu_gov); +void clear_cpu_predict_history(void); +int create_global_sysfs_nodes(void); +int create_cluster_sysfs_nodes(struct lpm_cluster *cluster_gov); +void register_cluster_governor_ops(struct cluster_governor *ops); +void remove_global_sysfs_nodes(void); +void remove_cluster_sysfs_nodes(struct lpm_cluster *cluster_gov); + +#endif /* __QCOM_LPM_H__ */ diff --git a/drivers/cpuidle/governors/trace-cluster-lpm.h b/drivers/cpuidle/governors/trace-cluster-lpm.h new file mode 100644 index 000000000000..a3cd01464e46 --- /dev/null +++ b/drivers/cpuidle/governors/trace-cluster-lpm.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#if !defined(_TRACE_CLUSTER_LPM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_CLUSTER_LPM_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM cluster_lpm + +#include + +TRACE_EVENT(cluster_pred_select, + + TP_PROTO(int index, u32 sleep_us, + u32 latency, int pred, u32 pred_us), + + TP_ARGS(index, sleep_us, latency, pred, pred_us), + + TP_STRUCT__entry( + __field(int, index) + __field(u32, sleep_us) + __field(u32, latency) + __field(int, pred) + __field(u32, pred_us) + ), + + TP_fast_assign( + __entry->index = index; + __entry->sleep_us = sleep_us; + __entry->latency = latency; + __entry->pred = pred; + __entry->pred_us = pred_us; + ), + + TP_printk("idx:%d sleep_time:%u latency:%u pred:%d pred_us:%u", + __entry->index, __entry->sleep_us, + __entry->latency, __entry->pred, __entry->pred_us) +); + +TRACE_EVENT(cluster_pred_hist, + + TP_PROTO(int idx, u32 resi, u32 sample, u32 tmr), + + TP_ARGS(idx, resi, sample, tmr), + + TP_STRUCT__entry( + __field(int, idx) + __field(u32, resi) + __field(u32, sample) + __field(u32, tmr) + ), + + TP_fast_assign( + __entry->idx = idx; + __entry->resi = resi; + __entry->sample = sample; + __entry->tmr = tmr; + ), + + TP_printk("idx:%d resi:%u sample:%u tmr:%u", + __entry->idx, __entry->resi, + __entry->sample, __entry->tmr) +); + +TRACE_EVENT(cluster_exit, + + TP_PROTO(int cpu, u32 idx, u32 suspend_param), + + TP_ARGS(cpu, idx, suspend_param), + + TP_STRUCT__entry( + __field(int, cpu) + __field(u32, idx) + __field(u32, suspend_param) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->idx = idx; + __entry->suspend_param = suspend_param; + ), + + TP_printk("first cpu:%d idx:%u suspend_param:0x%x", __entry->cpu, + __entry->idx, __entry->suspend_param) +); + +TRACE_EVENT(cluster_enter, + + TP_PROTO(int cpu, u32 idx, u32 suspend_param), + + TP_ARGS(cpu, idx, suspend_param), + + TP_STRUCT__entry( + __field(int, cpu) + __field(u32, idx) + __field(u32, suspend_param) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->idx = idx; + __entry->suspend_param = suspend_param; + ), + + TP_printk("last cpu:%d idx:%u suspend_param:0x%x", __entry->cpu, + __entry->idx, __entry->suspend_param) +); + +#endif /* _TRACE_QCOM_LPM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace-cluster-lpm + +#include diff --git a/drivers/cpuidle/governors/trace-qcom-lpm.h b/drivers/cpuidle/governors/trace-qcom-lpm.h new file mode 100644 index 000000000000..dc42d35af677 --- /dev/null +++ b/drivers/cpuidle/governors/trace-qcom-lpm.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#if !defined(_TRACE_QCOM_LPM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_QCOM_LPM_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM qcom_lpm + +#include + +TRACE_EVENT(lpm_gov_select, + + TP_PROTO(int idx, s64 qos, u64 sleep, u64 reason), + + TP_ARGS(idx, qos, sleep, reason), + + TP_STRUCT__entry( + __field(int, idx) + __field(s64, qos) + __field(u64, sleep) + __field(u64, reason) + ), + + TP_fast_assign( + __entry->idx = idx; + __entry->qos = qos; + __entry->sleep = sleep; + __entry->reason = reason; + ), + + TP_printk("state:%d qos-us:%lld sleep-us:%llu reason:%#x", + __entry->idx, __entry->qos, __entry->sleep, __entry->reason) +); + +TRACE_EVENT(gov_pred_select, + + TP_PROTO(u32 predtype, u64 predicted, u32 tmr_time), + + TP_ARGS(predtype, predicted, tmr_time), + + TP_STRUCT__entry( + __field(u32, predtype) + __field(u64, predicted) + __field(u32, tmr_time) + ), + + TP_fast_assign( + __entry->predtype = predtype; + __entry->predicted = predicted; + __entry->tmr_time = tmr_time; + ), + + TP_printk("pred:%u time:%lu tmr_time:%u", + __entry->predtype, __entry->predicted, __entry->tmr_time) +); + +TRACE_EVENT(gov_pred_hist, + + TP_PROTO(int idx, int residency, int tmr), + + TP_ARGS(idx, tmr, residency), + + TP_STRUCT__entry( + __field(int, idx) + __field(int, residency) + __field(int, tmr) + ), + + TP_fast_assign( + __entry->idx = idx; + __entry->residency = residency; + __entry->tmr = tmr; + ), + + TP_printk("idx:%d residency=%d, tmr=%d", __entry->idx, __entry->residency, __entry->tmr) +); + +#endif /* _TRACE_QCOM_LPM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace-qcom-lpm + +#include