From edb19769c065501e555f3aadaacdd3c6fe452e43 Mon Sep 17 00:00:00 2001 From: Amir Vajid Date: Mon, 29 Aug 2022 23:02:55 -0700 Subject: [PATCH 1/3] drivers: soc: dcvs: Add snapshot of dcvs drivers Add snapshot of dcvs drivers from msm-5.15 commit 250869d2bdc2192 ("sched: improve the scheduler")'. Change-Id: I3c38cb8d90deaa140f29f8bbdbad2e1e5989eb31 Signed-off-by: Amir Vajid --- drivers/perf/Kconfig | 9 + drivers/perf/Makefile | 1 + drivers/perf/qcom_llcc_pmu.c | 325 +++++ drivers/soc/qcom/Kconfig | 2 + drivers/soc/qcom/Makefile | 1 + drivers/soc/qcom/dcvs/Kconfig | 91 ++ drivers/soc/qcom/dcvs/Makefile | 11 + drivers/soc/qcom/dcvs/bwmon.c | 1842 +++++++++++++++++++++++++ drivers/soc/qcom/dcvs/bwmon.h | 194 +++ drivers/soc/qcom/dcvs/c1dcvs_scmi.c | 285 ++++ drivers/soc/qcom/dcvs/dcvs.c | 902 ++++++++++++ drivers/soc/qcom/dcvs/dcvs_epss.c | 272 ++++ drivers/soc/qcom/dcvs/dcvs_fp.c | 237 ++++ drivers/soc/qcom/dcvs/dcvs_icc.c | 77 ++ drivers/soc/qcom/dcvs/dcvs_private.h | 65 + drivers/soc/qcom/dcvs/memlat.c | 1901 ++++++++++++++++++++++++++ drivers/soc/qcom/dcvs/memlat_scmi.c | 41 + drivers/soc/qcom/dcvs/pmu_lib.c | 1183 ++++++++++++++++ drivers/soc/qcom/dcvs/pmu_scmi.c | 40 + drivers/soc/qcom/dcvs/trace-dcvs.c | 18 + drivers/soc/qcom/dcvs/trace-dcvs.h | 270 ++++ include/linux/scmi_c1dcvs.h | 48 + include/linux/scmi_memlat.h | 71 + include/linux/scmi_pmu.h | 45 + include/soc/qcom/dcvs.h | 79 ++ include/soc/qcom/pmu_lib.h | 92 ++ include/soc/qcom/qcom_llcc_pmu.h | 22 + 27 files changed, 8124 insertions(+) create mode 100644 drivers/perf/qcom_llcc_pmu.c create mode 100644 drivers/soc/qcom/dcvs/Kconfig create mode 100644 drivers/soc/qcom/dcvs/Makefile create mode 100644 drivers/soc/qcom/dcvs/bwmon.c create mode 100644 drivers/soc/qcom/dcvs/bwmon.h create mode 100644 drivers/soc/qcom/dcvs/c1dcvs_scmi.c create mode 100644 drivers/soc/qcom/dcvs/dcvs.c create mode 100644 drivers/soc/qcom/dcvs/dcvs_epss.c create mode 100644 drivers/soc/qcom/dcvs/dcvs_fp.c create mode 100644 drivers/soc/qcom/dcvs/dcvs_icc.c create mode 100644 drivers/soc/qcom/dcvs/dcvs_private.h create mode 100644 drivers/soc/qcom/dcvs/memlat.c create mode 100644 drivers/soc/qcom/dcvs/memlat_scmi.c create mode 100644 drivers/soc/qcom/dcvs/pmu_lib.c create mode 100644 drivers/soc/qcom/dcvs/pmu_scmi.c create mode 100644 drivers/soc/qcom/dcvs/trace-dcvs.c create mode 100644 drivers/soc/qcom/dcvs/trace-dcvs.h create mode 100644 include/linux/scmi_c1dcvs.h create mode 100644 include/linux/scmi_memlat.h create mode 100644 include/linux/scmi_pmu.h create mode 100644 include/soc/qcom/dcvs.h create mode 100644 include/soc/qcom/pmu_lib.h create mode 100644 include/soc/qcom/qcom_llcc_pmu.h diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index 1e2d69453771..9becc73771aa 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -137,6 +137,15 @@ config QCOM_L3_PMU Adds the L3 cache PMU into the perf events subsystem for monitoring L3 cache events. +config QCOM_LLCC_PMU + tristate "Qualcomm Technologies LLCC PMU" + depends on ARCH_QCOM && ARM64 + help + Provides support for the LLCC performance monitor unit (PMU) in + Qualcomm Technologies processors. + Adds the LLCC PMU into the perf events subsystem for monitoring + LLCC miss events. + config THUNDERX2_PMU tristate "Cavium ThunderX2 SoC PMU UNCORE" depends on ARCH_THUNDER2 || COMPILE_TEST diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile index 57a279c61df5..9ccaaa15e8a9 100644 --- a/drivers/perf/Makefile +++ b/drivers/perf/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o +obj-$(CONFIG_QCOM_LLCC_PMU) += qcom_llcc_pmu.o obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o diff --git a/drivers/perf/qcom_llcc_pmu.c b/drivers/perf/qcom_llcc_pmu.c new file mode 100644 index 000000000000..9fe42115e41a --- /dev/null +++ b/drivers/perf/qcom_llcc_pmu.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum llcc_pmu_version { + LLCC_PMU_VER1 = 1, + LLCC_PMU_VER2, +}; + +struct llcc_pmu { + struct pmu pmu; + struct hlist_node node; + void __iomem *lagg_base; + struct perf_event event; + enum llcc_pmu_version ver; +}; + +#define MON_CFG(m) ((m)->lagg_base + 0x200) +#define MON_CNT(m, cpu) ((m)->lagg_base + 0x220 + 0x4 * cpu) + +#define LLCC_RD_EV QCOM_LLCC_PMU_RD_EV +#define ENABLE 0x1 +#define CLEAR 0x10 +#define CLEAR_POS 16 +#define DISABLE 0x0 +#define SCALING_FACTOR 0x3 +#define NUM_COUNTERS NR_CPUS +#define VALUE_MASK 0xFFFFFF + +static u64 llcc_stats[NUM_COUNTERS]; +static unsigned int users; +static raw_spinlock_t counter_lock; +static raw_spinlock_t users_lock; +static ktime_t last_read; +static DEFINE_PER_CPU(unsigned int, users_alive); +static struct llcc_pmu *llccpmu; + +int qcom_llcc_pmu_hw_type(u32 *type) +{ + if (!llccpmu || !llccpmu->pmu.type) + return -EPROBE_DEFER; + + *type = llccpmu->pmu.type; + + return 0; +} +EXPORT_SYMBOL(qcom_llcc_pmu_hw_type); + +static void mon_disable(int cpu) +{ + u32 reg; + + if (!llccpmu->ver) { + pr_err("LLCCPMU version not correct\n"); + return; + } + + switch (llccpmu->ver) { + case LLCC_PMU_VER1: + writel_relaxed(DISABLE, MON_CFG(llccpmu)); + break; + case LLCC_PMU_VER2: + reg = readl_relaxed(MON_CFG(llccpmu)); + reg &= ~(ENABLE << cpu); + writel_relaxed(reg, MON_CFG(llccpmu)); + break; + } +} + +static void mon_clear(int cpu) +{ + int clear_bit = CLEAR_POS + cpu; + u32 reg; + + if (!llccpmu->ver) { + pr_err("LLCCPMU version not correct\n"); + return; + } + + switch (llccpmu->ver) { + case LLCC_PMU_VER1: + writel_relaxed(CLEAR, MON_CFG(llccpmu)); + break; + case LLCC_PMU_VER2: + reg = readl_relaxed(MON_CFG(llccpmu)); + reg |= (ENABLE << clear_bit); + writel_relaxed(reg, MON_CFG(llccpmu)); + reg &= ~(ENABLE << clear_bit); + writel_relaxed(reg, MON_CFG(llccpmu)); + break; + } +} + +static void mon_enable(int cpu) +{ + u32 reg; + + if (!llccpmu->ver) { + pr_err("LLCCPMU version not correct\n"); + return; + } + + switch (llccpmu->ver) { + case LLCC_PMU_VER1: + writel_relaxed(ENABLE, MON_CFG(llccpmu)); + break; + case LLCC_PMU_VER2: + reg = readl_relaxed(MON_CFG(llccpmu)); + reg |= (ENABLE << cpu); + writel_relaxed(reg, MON_CFG(llccpmu)); + break; + } +} + +static unsigned long read_cnt(int cpu) +{ + unsigned long value; + + if (!llccpmu->ver) { + pr_err("LLCCPMU version not correct\n"); + return -EINVAL; + } + + switch (llccpmu->ver) { + case LLCC_PMU_VER1: + value = readl_relaxed(MON_CNT(llccpmu, cpu)); + break; + case LLCC_PMU_VER2: + value = readl_relaxed(MON_CNT(llccpmu, cpu)); + break; + } + return value; +} + +static int qcom_llcc_event_init(struct perf_event *event) +{ + u64 config = event->attr.config; + + if (config == LLCC_RD_EV) { + event->hw.config_base = event->attr.config; + return 0; + } else + return -ENOENT; +} + +static void qcom_llcc_event_read(struct perf_event *event) +{ + int i = 0, cpu = event->cpu; + unsigned long raw, irq_flags; + ktime_t cur; + + raw_spin_lock_irqsave(&counter_lock, irq_flags); + if (llccpmu->ver == LLCC_PMU_VER1) { + cur = ktime_get(); + if (ktime_ms_delta(cur, last_read) > 1) { + mon_disable(cpu); + for (i = 0; i < NUM_COUNTERS; i++) { + raw = read_cnt(i); + raw &= VALUE_MASK; + llcc_stats[i] += (u64) raw << SCALING_FACTOR; + } + last_read = cur; + mon_clear(cpu); + mon_enable(cpu); + } + } else { + mon_disable(cpu); + raw = read_cnt(cpu); + raw &= VALUE_MASK; + llcc_stats[cpu] += (u64) raw << SCALING_FACTOR; + mon_clear(cpu); + mon_enable(cpu); + } + + if (!(event->hw.state & PERF_HES_STOPPED)) + local64_set(&event->count, llcc_stats[cpu]); + raw_spin_unlock_irqrestore(&counter_lock, irq_flags); +} + +static void qcom_llcc_event_start(struct perf_event *event, int flags) +{ + if (flags & PERF_EF_RELOAD) + WARN_ON(!(event->hw.state & PERF_HES_UPTODATE)); + event->hw.state = 0; +} + +static void qcom_llcc_event_stop(struct perf_event *event, int flags) +{ + qcom_llcc_event_read(event); + event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; +} + +static int qcom_llcc_event_add(struct perf_event *event, int flags) +{ + unsigned int cpu_users; + + raw_spin_lock(&users_lock); + + if (llccpmu->ver == LLCC_PMU_VER1) { + if (!users) + mon_enable(event->cpu); + users++; + } else { + cpu_users = per_cpu(users_alive, event->cpu); + if (!cpu_users) + mon_enable(event->cpu); + cpu_users++; + per_cpu(users_alive, event->cpu) = cpu_users; + } + + raw_spin_unlock(&users_lock); + + event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + + if (flags & PERF_EF_START) + qcom_llcc_event_start(event, PERF_EF_RELOAD); + + return 0; +} + +static void qcom_llcc_event_del(struct perf_event *event, int flags) +{ + unsigned int cpu_users; + + raw_spin_lock(&users_lock); + + if (llccpmu->ver == LLCC_PMU_VER1) { + users--; + if (!users) + mon_disable(event->cpu); + } else { + cpu_users = per_cpu(users_alive, event->cpu); + cpu_users--; + if (!cpu_users) + mon_disable(event->cpu); + per_cpu(users_alive, event->cpu) = cpu_users; + } + + raw_spin_unlock(&users_lock); +} + +static int qcom_llcc_pmu_probe(struct platform_device *pdev) +{ + struct resource *res; + int ret; + + if (llccpmu) { + dev_err(&pdev->dev, "Only one LLCC PMU allowed!\n"); + return -ENODEV; + } + llccpmu = devm_kzalloc(&pdev->dev, sizeof(struct llcc_pmu), GFP_KERNEL); + if (!llccpmu) + return -ENOMEM; + + llccpmu->ver = (enum llcc_pmu_version) + of_device_get_match_data(&pdev->dev); + if (!llccpmu->ver) { + pr_err("Unknown device type!\n"); + return -ENODEV; + } + + llccpmu->pmu = (struct pmu) { + .task_ctx_nr = perf_invalid_context, + .type = 0, + .event_init = qcom_llcc_event_init, + .add = qcom_llcc_event_add, + .del = qcom_llcc_event_del, + .start = qcom_llcc_event_start, + .stop = qcom_llcc_event_stop, + .read = qcom_llcc_event_read, + }; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lagg-base"); + llccpmu->lagg_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(llccpmu->lagg_base)) { + dev_err(&pdev->dev, "Can't map PMU lagg base: @%pa\n", + &res->start); + return PTR_ERR(llccpmu->lagg_base); + } + + raw_spin_lock_init(&counter_lock); + raw_spin_lock_init(&users_lock); + + ret = perf_pmu_register(&llccpmu->pmu, "llcc-pmu", -1); + if (ret < 0) + dev_err(&pdev->dev, "Failed to register LLCC PMU (%d)\n", ret); + + dev_info(&pdev->dev, "Registered llcc_pmu, type: %d\n", + llccpmu->pmu.type); + + return 0; +} + +static const struct of_device_id qcom_llcc_pmu_match_table[] = { + { .compatible = "qcom,llcc-pmu-ver1", .data = (void *) LLCC_PMU_VER1 }, + { .compatible = "qcom,llcc-pmu-ver2", .data = (void *) LLCC_PMU_VER2 }, + {} +}; + +static struct platform_driver qcom_llcc_pmu_driver = { + .driver = { + .name = "qcom-llcc-pmu", + .of_match_table = qcom_llcc_pmu_match_table, + }, + .probe = qcom_llcc_pmu_probe, +}; +module_platform_driver(qcom_llcc_pmu_driver); + +MODULE_DESCRIPTION("QCOM LLCC PMU"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index 80074ad79960..9f8cba1fe784 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -392,6 +392,8 @@ config MINIDUMP_ALL_TASK_INFO collection during panic. This can provide more debug info. +source "drivers/soc/qcom/dcvs/Kconfig" + config QCOM_LOGBUF_VENDOR_HOOKS tristate "QTI Logbuf Vendor Hooks Support" depends on ARCH_QCOM && ANDROID_VENDOR_HOOKS diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index ff0b827052e3..a704d065fae0 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -59,3 +59,4 @@ obj-$(CONFIG_QCOM_HUNG_TASK_ENH) += hung_task_enh.o obj-$(CONFIG_GH_TLMM_VM_MEM_ACCESS) += gh_tlmm_vm_mem_access.o obj-$(CONFIG_QCOM_RAMDUMP) += qcom_ramdump.o obj-$(CONFIG_QCOM_ICC_BWMON) += icc-bwmon.o +obj-$(CONFIG_QCOM_DCVS) += dcvs/ diff --git a/drivers/soc/qcom/dcvs/Kconfig b/drivers/soc/qcom/dcvs/Kconfig new file mode 100644 index 000000000000..8a6044cd5e55 --- /dev/null +++ b/drivers/soc/qcom/dcvs/Kconfig @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# QCOM DCVS drivers +# +config QCOM_PMU_LIB + tristate "QCOM PMU Driver" + depends on ARCH_QCOM + default n + help + This enables the QCOM PMU driver which manages and maintains + per-CPU performance counters that are available on some + Qualcomm Technologies, Inc. (QTI) chipsets. This includes + preserving them across CPU idle and hotplug events. + +config QCOM_DCVS_FP + tristate "Enable QCOM DCVS Fast Path Interface" + depends on ARCH_QCOM && QCOM_RPMH + default n + help + This enables the QCOM DCVS FP (Fast Path) driver which relies on + certain "fast path" APIs from QCOM_RPMH to vote for various DCVS HW + blocks in a faster scheme than standard voting paths. This requires a + Qualcomm Technologies, Inc. (QTI) chipset that supports this feature. + +config QCOM_DCVS + tristate "QCOM DCVS Driver" + select INTERCONNECT + default n + help + This enables the main QCOM DCVS framework which supports various + DCVS HW blocks and voting interfaces that are supported on some + Qualcomm Technologies, Inc. (QTI) chipsets. This includes voting + interfaces from both QCOM_DCVS_FP and INTERCONNECT. + +config QCOM_MEMLAT + tristate "QCOM Memlat Driver" + depends on QCOM_DCVS && QCOM_PMU_LIB + default n + help + This enables the QCOM Memlat driver which monitors CPU performance + counters to identify memory latency bound workloads and vote for + DCVS HW (memory) frequencies through the QCOM DCVS framework. This + driver also configures epss memlat if enabled. + +config QCOM_BWMON + tristate "QCOM BWMON Driver" + depends on QCOM_DCVS + default n + help + This enables the QCOM BWMON driver which monitors bandwidth counters + to identify memory bandwidth bound workloads and vote for DCVS HW + (memory) frequencies through the QCOM DCVS framework. This driver + handles voting for DDR and LLCC. + +config QTI_HW_MEMLAT_SCMI_CLIENT + tristate "Qualcomm Technologies Inc. SCMI client driver for HW MEMLAT" + depends on QCOM_MEMLAT && QTI_SCMI_MEMLAT_PROTOCOL + default n + help + SCMI client driver registers itself with SCMI framework for memlat + vendor protocol, and also registers with the memlat interface + driver. + + This driver delivers the memlat vendor protocol handle to interface + driver, and interface driver will use this handle to communicate with + memlat HW. + + +config QTI_PMU_SCMI_CLIENT + tristate "Qualcomm Technologies Inc. SCMI client driver for PMU" + depends on QTI_SCMI_PMU_PROTOCOL && QCOM_PMU_LIB + default n + help + SCMI client driver registers itself with SCMI framework for pmu + vendor protocol. + + This driver delivers the pmu vendor protocol handle to interface + driver, and interface driver will use this handle to communicate with + rimps. + +config QTI_C1DCVS_SCMI_CLIENT + tristate "Qualcomm Technologies Inc. SCMI client driver for cpudcvs" + depends on QTI_SCMI_C1DCVS_PROTOCOL + default n + help + SCMI client driver registers itself with SCMI framework for c1dcvs + vendor protocol. + + This driver delivers the cpudcvs protocol handle to interface + driver, and interface driver will use this handle to communicate with + cpucp. diff --git a/drivers/soc/qcom/dcvs/Makefile b/drivers/soc/qcom/dcvs/Makefile new file mode 100644 index 000000000000..eb22fd6627c3 --- /dev/null +++ b/drivers/soc/qcom/dcvs/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_QCOM_PMU_LIB) += qcom-pmu-lib.o +qcom-pmu-lib-y := pmu_lib.o +obj-$(CONFIG_QCOM_DCVS_FP) += dcvs_fp.o +obj-$(CONFIG_QCOM_DCVS) += qcom-dcvs.o +qcom-dcvs-y := dcvs.o dcvs_icc.o dcvs_epss.o trace-dcvs.o +obj-$(CONFIG_QCOM_MEMLAT) += memlat.o +obj-$(CONFIG_QCOM_BWMON) += bwmon.o +obj-$(CONFIG_QTI_PMU_SCMI_CLIENT) += pmu_scmi.o +obj-$(CONFIG_QTI_C1DCVS_SCMI_CLIENT) += c1dcvs_scmi.o +obj-$(CONFIG_QTI_HW_MEMLAT_SCMI_CLIENT) += memlat_scmi.o diff --git a/drivers/soc/qcom/dcvs/bwmon.c b/drivers/soc/qcom/dcvs/bwmon.c new file mode 100644 index 000000000000..f9b669ec12e0 --- /dev/null +++ b/drivers/soc/qcom/dcvs/bwmon.c @@ -0,0 +1,1842 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "qcom-bwmon: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "bwmon.h" +#include "trace-dcvs.h" + +static LIST_HEAD(hwmon_list); +static DEFINE_SPINLOCK(list_lock); +static DEFINE_SPINLOCK(sample_irq_lock); +static DEFINE_SPINLOCK(mon_irq_lock); +static DEFINE_MUTEX(bwmon_lock); +static struct workqueue_struct *bwmon_wq; + +struct qcom_bwmon_attr { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct attribute *attr, + char *buf); + ssize_t (*store)(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count); +}; + +#define to_bwmon_attr(_attr) \ + container_of(_attr, struct qcom_bwmon_attr, attr) +#define to_hwmon_node(k) container_of(k, struct hwmon_node, kobj) +#define to_bwmon(ptr) container_of(ptr, struct bwmon, hw) + +#define BWMON_ATTR_RW(_name) \ +struct qcom_bwmon_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) \ + +#define BWMON_ATTR_RO(_name) \ +struct qcom_bwmon_attr _name = \ +__ATTR(_name, 0444, show_##_name, NULL) \ + +#define show_attr(name) \ +static ssize_t show_##name(struct kobject *kobj, \ + struct attribute *attr, char *buf) \ +{ \ + struct hwmon_node *node = to_hwmon_node(kobj); \ + return scnprintf(buf, PAGE_SIZE, "%u\n", node->name); \ +} \ + +#define store_attr(name, _min, _max) \ +static ssize_t store_##name(struct kobject *kobj, \ + struct attribute *attr, const char *buf, \ + size_t count) \ +{ \ + int ret; \ + unsigned int val; \ + struct hwmon_node *node = to_hwmon_node(kobj); \ + ret = kstrtouint(buf, 10, &val); \ + if (ret < 0) \ + return ret; \ + val = max(val, _min); \ + val = min(val, _max); \ + node->name = val; \ + return count; \ +} \ + +#define show_list_attr(name, n) \ +static ssize_t show_##name(struct kobject *kobj, \ + struct attribute *attr, char *buf) \ +{ \ + struct hwmon_node *node = to_hwmon_node(kobj); \ + unsigned int i, cnt = 0; \ + \ + for (i = 0; i < n && node->name[i]; i++) \ + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "%u ", \ + node->name[i]); \ + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); \ + return cnt; \ +} \ + +#define store_list_attr(name, n, _min, _max) \ +static ssize_t store_##name(struct kobject *kobj, \ + struct attribute *attr, const char *buf, \ + size_t count) \ +{ \ + struct hwmon_node *node = to_hwmon_node(kobj); \ + int ret, numvals; \ + unsigned int i = 0, val; \ + char **strlist; \ + \ + strlist = argv_split(GFP_KERNEL, buf, &numvals); \ + if (!strlist) \ + return -ENOMEM; \ + numvals = min(numvals, n - 1); \ + for (i = 0; i < numvals; i++) { \ + ret = kstrtouint(strlist[i], 10, &val); \ + if (ret < 0) \ + goto out; \ + val = max(val, _min); \ + val = min(val, _max); \ + node->name[i] = val; \ + } \ + ret = count; \ +out: \ + argv_free(strlist); \ + node->name[i] = 0; \ + return ret; \ +} \ + +static ssize_t store_min_freq(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret; + unsigned int freq; + struct hwmon_node *node = to_hwmon_node(kobj); + + ret = kstrtouint(buf, 10, &freq); + if (ret < 0) + return ret; + freq = max(freq, node->hw_min_freq); + freq = min(freq, node->max_freq); + + node->min_freq = freq; + + return count; +} + +static ssize_t store_max_freq(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret; + unsigned int freq; + struct hwmon_node *node = to_hwmon_node(kobj); + + ret = kstrtouint(buf, 10, &freq); + if (ret < 0) + return ret; + freq = max(freq, node->min_freq); + freq = min(freq, node->hw_max_freq); + + node->max_freq = freq; + + return count; +} + +static ssize_t store_throttle_adj(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + struct hwmon_node *node = to_hwmon_node(kobj); + int ret; + unsigned int val; + + if (!node->hw->set_throttle_adj) + return -EPERM; + + ret = kstrtouint(buf, 10, &val); + if (ret < 0) + return ret; + + ret = node->hw->set_throttle_adj(node->hw, val); + + if (!ret) + return count; + else + return ret; +} + +static ssize_t show_throttle_adj(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct hwmon_node *node = to_hwmon_node(kobj); + unsigned int val; + + if (!node->hw->get_throttle_adj) + val = 0; + else + val = node->hw->get_throttle_adj(node->hw); + + return scnprintf(buf, PAGE_SIZE, "%u\n", val); +} + +#define SAMPLE_MIN_MS 1U +#define SAMPLE_MAX_MS 50U +static ssize_t store_sample_ms(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + struct hwmon_node *node = to_hwmon_node(kobj); + + int ret; + unsigned int val; + + ret = kstrtoint(buf, 10, &val); + if (ret) + return ret; + + val = max(val, SAMPLE_MIN_MS); + val = min(val, SAMPLE_MAX_MS); + if (val > node->window_ms) + return -EINVAL; + + node->sample_ms = val; + return count; +} + +static ssize_t show_cur_freq(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct hwmon_node *node = to_hwmon_node(kobj); + + return scnprintf(buf, PAGE_SIZE, "%u\n", node->cur_freq.ib); +} + +show_attr(min_freq); +static BWMON_ATTR_RW(min_freq); +show_attr(max_freq); +static BWMON_ATTR_RW(max_freq); +static BWMON_ATTR_RW(throttle_adj); +show_attr(sample_ms); +static BWMON_ATTR_RW(sample_ms); +static BWMON_ATTR_RO(cur_freq); + +show_attr(window_ms); +store_attr(window_ms, 8U, 1000U); +static BWMON_ATTR_RW(window_ms); +show_attr(guard_band_mbps); +store_attr(guard_band_mbps, 0U, 2000U); +static BWMON_ATTR_RW(guard_band_mbps); +show_attr(decay_rate); +store_attr(decay_rate, 0U, 100U); +static BWMON_ATTR_RW(decay_rate); +show_attr(io_percent); +store_attr(io_percent, 1U, 400U); +static BWMON_ATTR_RW(io_percent); +show_attr(bw_step); +store_attr(bw_step, 50U, 1000U); +static BWMON_ATTR_RW(bw_step); +show_attr(up_scale); +store_attr(up_scale, 0U, 500U); +static BWMON_ATTR_RW(up_scale); +show_attr(up_thres); +store_attr(up_thres, 1U, 100U); +static BWMON_ATTR_RW(up_thres); +show_attr(down_thres); +store_attr(down_thres, 0U, 90U); +static BWMON_ATTR_RW(down_thres); +show_attr(down_count); +store_attr(down_count, 0U, 90U); +static BWMON_ATTR_RW(down_count); +show_attr(hist_memory); +store_attr(hist_memory, 0U, 90U); +static BWMON_ATTR_RW(hist_memory); +show_attr(hyst_trigger_count); +store_attr(hyst_trigger_count, 0U, 90U); +static BWMON_ATTR_RW(hyst_trigger_count); +show_attr(hyst_length); +store_attr(hyst_length, 0U, 90U); +static BWMON_ATTR_RW(hyst_length); +show_attr(idle_mbps); +store_attr(idle_mbps, 0U, 2000U); +static BWMON_ATTR_RW(idle_mbps); +show_list_attr(mbps_zones, NUM_MBPS_ZONES); +store_list_attr(mbps_zones, NUM_MBPS_ZONES, 0U, UINT_MAX); +static BWMON_ATTR_RW(mbps_zones); + +static struct attribute *bwmon_attrs[] = { + &min_freq.attr, + &max_freq.attr, + &cur_freq.attr, + &window_ms.attr, + &guard_band_mbps.attr, + &decay_rate.attr, + &io_percent.attr, + &bw_step.attr, + &sample_ms.attr, + &up_scale.attr, + &up_thres.attr, + &down_thres.attr, + &down_count.attr, + &hist_memory.attr, + &hyst_trigger_count.attr, + &hyst_length.attr, + &idle_mbps.attr, + &mbps_zones.attr, + &throttle_adj.attr, + NULL, +}; +ATTRIBUTE_GROUPS(bwmon); + +static ssize_t attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct qcom_bwmon_attr *bwmon_attr = to_bwmon_attr(attr); + ssize_t ret = -EIO; + + if (bwmon_attr->show) + ret = bwmon_attr->show(kobj, attr, buf); + + return ret; +} + +static ssize_t attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct qcom_bwmon_attr *bwmon_attr = to_bwmon_attr(attr); + ssize_t ret = -EIO; + + if (bwmon_attr->store) + ret = bwmon_attr->store(kobj, attr, buf, count); + + return ret; +} + +static const struct sysfs_ops bwmon_sysfs_ops = { + .show = attr_show, + .store = attr_store, +}; + +static struct kobj_type bwmon_ktype = { + .sysfs_ops = &bwmon_sysfs_ops, + .default_groups = bwmon_groups, + +}; + +/* Returns MBps of read/writes for the sampling window. */ +static unsigned long bytes_to_mbps(unsigned long long bytes, unsigned int us) +{ + bytes *= USEC_PER_SEC; + do_div(bytes, us); + bytes = DIV_ROUND_UP_ULL(bytes, SZ_1M); + return bytes; +} + +static unsigned int mbps_to_bytes(unsigned long mbps, unsigned int ms) +{ + mbps *= ms; + mbps = DIV_ROUND_UP(mbps, MSEC_PER_SEC); + mbps *= SZ_1M; + return mbps; +} + +static int __bw_hwmon_sw_sample_end(struct bw_hwmon *hwmon) +{ + struct hwmon_node *node = hwmon->node; + ktime_t ts; + unsigned long bytes, mbps; + unsigned int us; + int wake = 0; + + ts = ktime_get(); + us = ktime_to_us(ktime_sub(ts, node->prev_ts)); + + bytes = hwmon->get_bytes_and_clear(hwmon); + bytes += node->bytes; + node->bytes = 0; + + mbps = bytes_to_mbps(bytes, us); + node->max_mbps = max(node->max_mbps, mbps); + + /* + * If the measured bandwidth in a micro sample is greater than the + * wake up threshold, it indicates an increase in load that's non + * trivial. So, have the governor ignore historical idle time or low + * bandwidth usage and do the bandwidth calculation based on just + * this micro sample. + */ + if (mbps > node->hw->up_wake_mbps) { + wake = UP_WAKE; + } else if (mbps < node->hw->down_wake_mbps) { + if (node->down_cnt) + node->down_cnt--; + if (node->down_cnt <= 0) + wake = DOWN_WAKE; + } + + node->prev_ts = ts; + node->wake = wake; + node->sampled = true; + + trace_bw_hwmon_meas(dev_name(hwmon->dev), + mbps, + us, + wake); + + return wake; +} + +static int __bw_hwmon_hw_sample_end(struct bw_hwmon *hwmon) +{ + struct hwmon_node *node = hwmon->node; + unsigned long bytes, mbps; + int wake = 0; + + /* + * If this read is in response to an IRQ, the HW monitor should + * return the measurement in the micro sample that triggered the IRQ. + * Otherwise, it should return the maximum measured value in any + * micro sample since the last time we called get_bytes_and_clear() + */ + bytes = hwmon->get_bytes_and_clear(hwmon); + mbps = bytes_to_mbps(bytes, node->sample_ms * USEC_PER_MSEC); + node->max_mbps = mbps; + + if (mbps > node->hw->up_wake_mbps) + wake = UP_WAKE; + else if (mbps < node->hw->down_wake_mbps) + wake = DOWN_WAKE; + + node->wake = wake; + node->sampled = true; + + trace_bw_hwmon_meas(dev_name(hwmon->dev), + mbps, + node->sample_ms * USEC_PER_MSEC, + wake); + + return 1; +} + +static int __bw_hwmon_sample_end(struct bw_hwmon *hwmon) +{ + if (hwmon->set_hw_events) + return __bw_hwmon_hw_sample_end(hwmon); + else + return __bw_hwmon_sw_sample_end(hwmon); +} + +static int bw_hwmon_sample_end(struct bw_hwmon *hwmon) +{ + unsigned long flags; + int wake; + + spin_lock_irqsave(&sample_irq_lock, flags); + wake = __bw_hwmon_sample_end(hwmon); + spin_unlock_irqrestore(&sample_irq_lock, flags); + + return wake; +} + +static unsigned long to_mbps_zone(struct hwmon_node *node, unsigned long mbps) +{ + int i; + + for (i = 0; i < NUM_MBPS_ZONES && node->mbps_zones[i]; i++) + if (node->mbps_zones[i] >= mbps) + return node->mbps_zones[i]; + + return KHZ_TO_MBPS(node->max_freq, node->hw->dcvs_width); +} + +#define MIN_MBPS 500UL +#define HIST_PEAK_TOL 75 +static unsigned long get_bw_and_set_irq(struct hwmon_node *node, + struct dcvs_freq *freq_mbps) +{ + unsigned long meas_mbps, thres, flags, req_mbps, adj_mbps; + unsigned long meas_mbps_zone; + unsigned long hist_lo_tol, hyst_lo_tol; + struct bw_hwmon *hw = node->hw; + unsigned int new_bw, io_percent = node->io_percent; + ktime_t ts; + unsigned int ms = 0; + + spin_lock_irqsave(&sample_irq_lock, flags); + + if (!hw->set_hw_events) { + ts = ktime_get(); + ms = ktime_to_ms(ktime_sub(ts, node->prev_ts)); + } + if (!node->sampled || ms >= node->sample_ms) + __bw_hwmon_sample_end(node->hw); + node->sampled = false; + + req_mbps = meas_mbps = node->max_mbps; + node->max_mbps = 0; + + hist_lo_tol = (node->hist_max_mbps * HIST_PEAK_TOL) / 100; + /* Remember historic peak in the past hist_mem decision windows. */ + if (meas_mbps > node->hist_max_mbps || !node->hist_mem) { + /* If new max or no history */ + node->hist_max_mbps = meas_mbps; + node->hist_mem = node->hist_memory; + } else if (meas_mbps >= hist_lo_tol) { + /* + * If subsequent peaks come close (within tolerance) to but + * less than the historic peak, then reset the history start, + * but not the peak value. + */ + node->hist_mem = node->hist_memory; + } else { + /* Count down history expiration. */ + if (node->hist_mem) + node->hist_mem--; + } + + /* + * The AB value that corresponds to the lowest mbps zone greater than + * or equal to the "frequency" the current measurement will pick. + * This upper limit is useful for balancing out any prediction + * mechanisms to be power friendly. + */ + meas_mbps_zone = (meas_mbps * 100) / io_percent; + meas_mbps_zone = to_mbps_zone(node, meas_mbps_zone); + meas_mbps_zone = (meas_mbps_zone * io_percent) / 100; + meas_mbps_zone = max(meas_mbps, meas_mbps_zone); + + /* + * If this is a wake up due to BW increase, vote much higher BW than + * what we measure to stay ahead of increasing traffic and then set + * it up to vote for measured BW if we see down_count short sample + * windows of low traffic. + */ + if (node->wake == UP_WAKE) { + req_mbps += ((meas_mbps - node->prev_req) + * node->up_scale) / 100; + /* + * However if the measured load is less than the historic + * peak, but the over request is higher than the historic + * peak, then we could limit the over requesting to the + * historic peak. + */ + if (req_mbps > node->hist_max_mbps + && meas_mbps < node->hist_max_mbps) + req_mbps = node->hist_max_mbps; + + req_mbps = min(req_mbps, meas_mbps_zone); + } + + hyst_lo_tol = (node->hyst_mbps * HIST_PEAK_TOL) / 100; + if (meas_mbps > node->hyst_mbps && meas_mbps > MIN_MBPS) { + hyst_lo_tol = (meas_mbps * HIST_PEAK_TOL) / 100; + node->hyst_peak = 0; + node->hyst_trig_win = node->hyst_length; + node->hyst_mbps = meas_mbps; + if (node->hyst_en) + node->hyst_en = node->hyst_length; + } + + /* + * Check node->max_mbps to avoid double counting peaks that cause + * early termination of a window. + */ + if (meas_mbps >= hyst_lo_tol && meas_mbps > MIN_MBPS + && !node->max_mbps) { + node->hyst_peak++; + if (node->hyst_peak >= node->hyst_trigger_count) { + node->hyst_peak = 0; + node->hyst_en = node->hyst_length; + } + } + + if (node->hyst_trig_win) + node->hyst_trig_win--; + if (node->hyst_en) + node->hyst_en--; + + if (!node->hyst_trig_win && !node->hyst_en) { + node->hyst_peak = 0; + node->hyst_mbps = 0; + } + + if (node->hyst_en) { + if (meas_mbps > node->idle_mbps) + req_mbps = max(req_mbps, node->hyst_mbps); + } + + /* Stretch the short sample window size, if the traffic is too low */ + if (meas_mbps < MIN_MBPS) { + hw->up_wake_mbps = (max(MIN_MBPS, req_mbps) + * (100 + node->up_thres)) / 100; + hw->down_wake_mbps = 0; + thres = mbps_to_bytes(max(MIN_MBPS, req_mbps / 2), + node->sample_ms); + } else { + /* + * Up wake vs down wake are intentionally a percentage of + * req_mbps vs meas_mbps to make sure the over requesting + * phase is handled properly. We only want to wake up and + * reduce the vote based on the measured mbps being less than + * the previous measurement that caused the "over request". + */ + hw->up_wake_mbps = (req_mbps * (100 + node->up_thres)) / 100; + hw->down_wake_mbps = (meas_mbps * node->down_thres) / 100; + thres = mbps_to_bytes(meas_mbps, node->sample_ms); + } + + if (hw->set_hw_events) { + hw->down_cnt = node->down_count; + hw->set_hw_events(hw, node->sample_ms); + } else { + node->down_cnt = node->down_count; + node->bytes = hw->set_thres(hw, thres); + } + + node->wake = 0; + node->prev_req = req_mbps; + + spin_unlock_irqrestore(&sample_irq_lock, flags); + + adj_mbps = req_mbps + node->guard_band_mbps; + + if (adj_mbps > node->prev_ab) { + new_bw = adj_mbps; + } else { + new_bw = adj_mbps * node->decay_rate + + node->prev_ab * (100 - node->decay_rate); + new_bw /= 100; + } + + node->prev_ab = new_bw; + freq_mbps->ab = roundup(new_bw, node->bw_step); + freq_mbps->ib = (new_bw * 100) / io_percent; + trace_bw_hwmon_update(dev_name(node->hw->dev), + freq_mbps->ab, + freq_mbps->ib, + hw->up_wake_mbps, + hw->down_wake_mbps); + + trace_bw_hwmon_debug(dev_name(node->hw->dev), + req_mbps, + meas_mbps_zone, + node->hist_max_mbps, + node->hist_mem, + node->hyst_mbps, + node->hyst_en); + return req_mbps; +} + +/* + * Governor function that computes new target frequency + * based on bw measurement (mbps) and updates cur_freq (khz). + * Returns true if cur_freq was changed + * Note: must hold node->update_lock before calling + */ +static bool bwmon_update_cur_freq(struct hwmon_node *node) +{ + struct bw_hwmon *hw = node->hw; + struct dcvs_freq new_freq; + + get_bw_and_set_irq(node, &new_freq); + + /* first convert freq from mbps to khz */ + new_freq.ab = MBPS_TO_KHZ(new_freq.ab, hw->dcvs_width); + new_freq.ib = MBPS_TO_KHZ(new_freq.ib, hw->dcvs_width); + new_freq.ib = max(new_freq.ib, node->min_freq); + new_freq.ib = min(new_freq.ib, node->max_freq); + + if (new_freq.ib != node->cur_freq.ib || + new_freq.ab != node->cur_freq.ab) { + node->cur_freq.ib = new_freq.ib; + node->cur_freq.ab = new_freq.ab; + return true; + } + + return false; +} + +static const u64 HALF_TICK_NS = (NSEC_PER_SEC / HZ) >> 1; +static void bwmon_jiffies_update_cb(void *unused, void *extra) +{ + struct bw_hwmon *hw; + struct hwmon_node *node; + unsigned long flags; + ktime_t now = ktime_get(); + s64 delta_ns; + + spin_lock_irqsave(&list_lock, flags); + list_for_each_entry(node, &hwmon_list, list) { + hw = node->hw; + if (!hw->is_active) + continue; + delta_ns = now - hw->last_update_ts + HALF_TICK_NS; + if (delta_ns > ms_to_ktime(hw->node->window_ms)) { + queue_work(bwmon_wq, &hw->work); + hw->last_update_ts = now; + } + } + spin_unlock_irqrestore(&list_lock, flags); +} + +static void bwmon_monitor_work(struct work_struct *work) +{ + int err = 0; + struct bw_hwmon *hw = container_of(work, struct bw_hwmon, work); + struct hwmon_node *node = hw->node; + + /* governor update and commit */ + mutex_lock(&node->update_lock); + if (bwmon_update_cur_freq(node)) + err = qcom_dcvs_update_votes(dev_name(hw->dev), + &node->cur_freq, 1, hw->dcvs_path); + if (err < 0) + dev_err(hw->dev, "bwmon monitor update failed: %d\n", err); + mutex_unlock(&node->update_lock); +} + +static inline void bwmon_monitor_start(struct bw_hwmon *hw) +{ + hw->last_update_ts = ktime_get(); + hw->is_active = true; +} + +static inline void bwmon_monitor_stop(struct bw_hwmon *hw) +{ + hw->is_active = false; + cancel_work_sync(&hw->work); +} + +static int update_bw_hwmon(struct bw_hwmon *hwmon) +{ + struct hwmon_node *node = hwmon->node; + int ret = 0; + + mutex_lock(&node->mon_lock); + if (!node->mon_started) { + mutex_unlock(&node->mon_lock); + return -EBUSY; + } + dev_dbg(hwmon->dev, "Got update request\n"); + bwmon_monitor_stop(hwmon); + + /* governor update and commit */ + mutex_lock(&node->update_lock); + if (bwmon_update_cur_freq(node)) + ret = qcom_dcvs_update_votes(dev_name(hwmon->dev), + &node->cur_freq, 1, hwmon->dcvs_path); + if (ret < 0) + dev_err(hwmon->dev, "bwmon irq update failed: %d\n", ret); + mutex_unlock(&node->update_lock); + + bwmon_monitor_start(hwmon); + mutex_unlock(&node->mon_lock); + + return 0; +} + +static int start_monitor(struct bw_hwmon *hwmon) +{ + struct hwmon_node *node = hwmon->node; + unsigned long mbps; + int ret; + + node->prev_ts = ktime_get(); + node->prev_ab = 0; + mbps = KHZ_TO_MBPS(node->cur_freq.ib, hwmon->dcvs_width) * + node->io_percent / 100; + hwmon->up_wake_mbps = mbps; + hwmon->down_wake_mbps = MIN_MBPS; + ret = hwmon->start_hwmon(hwmon, mbps); + if (ret < 0) { + dev_err(hwmon->dev, "Unable to start HW monitor! (%d)\n", ret); + return ret; + } + + node->mon_started = true; + + return 0; +} + +static void stop_monitor(struct bw_hwmon *hwmon) +{ + struct hwmon_node *node = hwmon->node; + + mutex_lock(&node->mon_lock); + node->mon_started = false; + mutex_unlock(&node->mon_lock); + + hwmon->stop_hwmon(hwmon); + +} + +static int configure_hwmon_node(struct bw_hwmon *hwmon) +{ + struct hwmon_node *node; + unsigned long flags; + + node = devm_kzalloc(hwmon->dev, sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + hwmon->node = node; + + node->guard_band_mbps = 100; + node->decay_rate = 90; + node->io_percent = 16; + node->bw_step = 190; + node->sample_ms = 50; + node->window_ms = 50; + node->up_scale = 0; + node->up_thres = 10; + node->down_thres = 0; + node->down_count = 3; + node->hist_memory = 0; + node->hyst_trigger_count = 3; + node->hyst_length = 0; + node->idle_mbps = 400; + node->mbps_zones[0] = 0; + node->hw = hwmon; + + mutex_init(&node->mon_lock); + mutex_init(&node->update_lock); + spin_lock_irqsave(&list_lock, flags); + list_add_tail(&node->list, &hwmon_list); + spin_unlock_irqrestore(&list_lock, flags); + + return 0; +} + +#define ENABLE_MASK BIT(0) +static __always_inline void mon_enable(struct bwmon *m, enum mon_reg_type type) +{ + switch (type) { + case MON1: + writel_relaxed(ENABLE_MASK | m->throttle_adj, MON_EN(m)); + break; + case MON2: + writel_relaxed(ENABLE_MASK | m->throttle_adj, MON2_EN(m)); + break; + case MON3: + writel_relaxed(ENABLE_MASK | m->throttle_adj, MON3_EN(m)); + break; + } +} + +static __always_inline void mon_disable(struct bwmon *m, enum mon_reg_type type) +{ + switch (type) { + case MON1: + writel_relaxed(m->throttle_adj, MON_EN(m)); + break; + case MON2: + writel_relaxed(m->throttle_adj, MON2_EN(m)); + break; + case MON3: + writel_relaxed(m->throttle_adj, MON3_EN(m)); + break; + } + /* + * mon_disable() and mon_irq_clear(), + * If latter goes first and count happen to trigger irq, we would + * have the irq line high but no one handling it. + */ + mb(); +} + +#define MON_CLEAR_BIT 0x1 +#define MON_CLEAR_ALL_BIT 0x2 +static __always_inline +void mon_clear(struct bwmon *m, bool clear_all, enum mon_reg_type type) +{ + switch (type) { + case MON1: + writel_relaxed(MON_CLEAR_BIT, MON_CLEAR(m)); + break; + case MON2: + if (clear_all) + writel_relaxed(MON_CLEAR_ALL_BIT, MON2_CLEAR(m)); + else + writel_relaxed(MON_CLEAR_BIT, MON2_CLEAR(m)); + break; + case MON3: + if (clear_all) + writel_relaxed(MON_CLEAR_ALL_BIT, MON3_CLEAR(m)); + else + writel_relaxed(MON_CLEAR_BIT, MON3_CLEAR(m)); + /* + * In some hardware versions since MON3_CLEAR(m) register does + * not have self-clearing capability it needs to be cleared + * explicitly. But we also need to ensure the writes to it + * are successful before clearing it. + */ + wmb(); + writel_relaxed(0, MON3_CLEAR(m)); + break; + } + /* + * The counter clear and IRQ clear bits are not in the same 4KB + * region. So, we need to make sure the counter clear is completed + * before we try to clear the IRQ or do any other counter operations. + */ + mb(); +} + +#define SAMPLE_WIN_LIM 0xFFFFFF +static __always_inline +void mon_set_hw_sampling_window(struct bwmon *m, unsigned int sample_ms, + enum mon_reg_type type) +{ + u32 rate; + + if (unlikely(sample_ms != m->sample_size_ms)) { + rate = mult_frac(sample_ms, m->hw_timer_hz, MSEC_PER_SEC); + m->sample_size_ms = sample_ms; + if (unlikely(rate > SAMPLE_WIN_LIM)) { + rate = SAMPLE_WIN_LIM; + pr_warn("Sample window %u larger than hw limit: %u\n", + rate, SAMPLE_WIN_LIM); + } + switch (type) { + case MON1: + WARN(1, "Invalid\n"); + return; + case MON2: + writel_relaxed(rate, MON2_SW(m)); + break; + case MON3: + writel_relaxed(rate, MON3_SW(m)); + break; + } + } +} + +static void mon_glb_irq_enable(struct bwmon *m) +{ + u32 val; + + val = readl_relaxed(GLB_INT_EN(m)); + val |= 1 << m->mport; + writel_relaxed(val, GLB_INT_EN(m)); +} + +static __always_inline +void mon_irq_enable(struct bwmon *m, enum mon_reg_type type) +{ + u32 val; + + spin_lock(&mon_irq_lock); + switch (type) { + case MON1: + mon_glb_irq_enable(m); + val = readl_relaxed(MON_INT_EN(m)); + val |= MON_INT_ENABLE; + writel_relaxed(val, MON_INT_EN(m)); + break; + case MON2: + mon_glb_irq_enable(m); + val = readl_relaxed(MON_INT_EN(m)); + val |= MON2_INT_STATUS_MASK; + writel_relaxed(val, MON_INT_EN(m)); + break; + case MON3: + val = readl_relaxed(MON3_INT_EN(m)); + val |= MON3_INT_STATUS_MASK; + writel_relaxed(val, MON3_INT_EN(m)); + break; + } + spin_unlock(&mon_irq_lock); + /* + * make sure irq enable complete for local and global + * to avoid race with other monitor calls + */ + mb(); +} + +static void mon_glb_irq_disable(struct bwmon *m) +{ + u32 val; + + val = readl_relaxed(GLB_INT_EN(m)); + val &= ~(1 << m->mport); + writel_relaxed(val, GLB_INT_EN(m)); +} + +static __always_inline +void mon_irq_disable(struct bwmon *m, enum mon_reg_type type) +{ + u32 val; + + spin_lock(&mon_irq_lock); + + switch (type) { + case MON1: + mon_glb_irq_disable(m); + val = readl_relaxed(MON_INT_EN(m)); + val &= ~MON_INT_ENABLE; + writel_relaxed(val, MON_INT_EN(m)); + break; + case MON2: + mon_glb_irq_disable(m); + val = readl_relaxed(MON_INT_EN(m)); + val &= ~MON2_INT_STATUS_MASK; + writel_relaxed(val, MON_INT_EN(m)); + break; + case MON3: + val = readl_relaxed(MON3_INT_EN(m)); + val &= ~MON3_INT_STATUS_MASK; + writel_relaxed(val, MON3_INT_EN(m)); + break; + } + spin_unlock(&mon_irq_lock); + /* + * make sure irq disable complete for local and global + * to avoid race with other monitor calls + */ + mb(); +} + +static __always_inline +unsigned int mon_irq_status(struct bwmon *m, enum mon_reg_type type) +{ + u32 mval; + + switch (type) { + case MON1: + mval = readl_relaxed(MON_INT_STATUS(m)); + dev_dbg(m->dev, "IRQ status p:%x, g:%x\n", mval, + readl_relaxed(GLB_INT_STATUS(m))); + mval &= MON_INT_STATUS_MASK; + break; + case MON2: + mval = readl_relaxed(MON_INT_STATUS(m)); + dev_dbg(m->dev, "IRQ status p:%x, g:%x\n", mval, + readl_relaxed(GLB_INT_STATUS(m))); + mval &= MON2_INT_STATUS_MASK; + mval >>= MON2_INT_STATUS_SHIFT; + break; + case MON3: + mval = readl_relaxed(MON3_INT_STATUS(m)); + dev_dbg(m->dev, "IRQ status p:%x\n", mval); + mval &= MON3_INT_STATUS_MASK; + break; + } + + return mval; +} + +static void mon_glb_irq_clear(struct bwmon *m) +{ + /* + * Synchronize the local interrupt clear in mon_irq_clear() + * with the global interrupt clear here. Otherwise, the CPU + * may reorder the two writes and clear the global interrupt + * before the local interrupt, causing the global interrupt + * to be retriggered by the local interrupt still being high. + */ + mb(); + writel_relaxed(1 << m->mport, GLB_INT_CLR(m)); + /* + * Similarly, because the global registers are in a different + * region than the local registers, we need to ensure any register + * writes to enable the monitor after this call are ordered with the + * clearing here so that local writes don't happen before the + * interrupt is cleared. + */ + mb(); +} + +static __always_inline +void mon_irq_clear(struct bwmon *m, enum mon_reg_type type) +{ + switch (type) { + case MON1: + writel_relaxed(MON_INT_STATUS_MASK, MON_INT_CLR(m)); + mon_glb_irq_clear(m); + break; + case MON2: + writel_relaxed(MON2_INT_STATUS_MASK, MON_INT_CLR(m)); + mon_glb_irq_clear(m); + break; + case MON3: + writel_relaxed(MON3_INT_STATUS_MASK, MON3_INT_CLR(m)); + /* + * In some hardware versions since MON3_INT_CLEAR(m) register + * does not have self-clearing capability it needs to be + * cleared explicitly. But we also need to ensure the writes + * to it are successful before clearing it. + */ + wmb(); + writel_relaxed(0, MON3_INT_CLR(m)); + break; + } +} + +#define THROTTLE_MASK 0x1F +#define THROTTLE_SHIFT 16 +static int mon_set_throttle_adj(struct bw_hwmon *hw, uint adj) +{ + struct bwmon *m = to_bwmon(hw); + + if (adj > THROTTLE_MASK) + return -EINVAL; + + adj = (adj & THROTTLE_MASK) << THROTTLE_SHIFT; + m->throttle_adj = adj; + + return 0; +} + +static u32 mon_get_throttle_adj(struct bw_hwmon *hw) +{ + struct bwmon *m = to_bwmon(hw); + + return m->throttle_adj >> THROTTLE_SHIFT; +} + +#define ZONE1_SHIFT 8 +#define ZONE2_SHIFT 16 +#define ZONE3_SHIFT 24 +#define ZONE0_ACTION 0x01 /* Increment zone 0 count */ +#define ZONE1_ACTION 0x09 /* Increment zone 1 & clear lower zones */ +#define ZONE2_ACTION 0x25 /* Increment zone 2 & clear lower zones */ +#define ZONE3_ACTION 0x95 /* Increment zone 3 & clear lower zones */ +static u32 calc_zone_actions(void) +{ + u32 zone_actions; + + zone_actions = ZONE0_ACTION; + zone_actions |= ZONE1_ACTION << ZONE1_SHIFT; + zone_actions |= ZONE2_ACTION << ZONE2_SHIFT; + zone_actions |= ZONE3_ACTION << ZONE3_SHIFT; + + return zone_actions; +} + +#define ZONE_CNT_LIM 0xFFU +#define UP_CNT_1 1 +static u32 calc_zone_counts(struct bw_hwmon *hw) +{ + u32 zone_counts; + + zone_counts = ZONE_CNT_LIM; + zone_counts |= min(hw->down_cnt, ZONE_CNT_LIM) << ZONE1_SHIFT; + zone_counts |= ZONE_CNT_LIM << ZONE2_SHIFT; + zone_counts |= UP_CNT_1 << ZONE3_SHIFT; + + return zone_counts; +} + +#define MB_SHIFT 20 + +static u32 mbps_to_count(unsigned long mbps, unsigned int ms, u8 shift) +{ + mbps *= ms; + + if (shift > MB_SHIFT) + mbps >>= shift - MB_SHIFT; + else + mbps <<= MB_SHIFT - shift; + + return DIV_ROUND_UP(mbps, MSEC_PER_SEC); +} + +/* + * Define the 4 zones using HI, MED & LO thresholds: + * Zone 0: byte count < THRES_LO + * Zone 1: THRES_LO < byte count < THRES_MED + * Zone 2: THRES_MED < byte count < THRES_HI + * Zone 3: THRES_LIM > byte count > THRES_HI + */ +#define THRES_LIM(shift) (0xFFFFFFFF >> shift) + +static __always_inline +void set_zone_thres(struct bwmon *m, unsigned int sample_ms, + enum mon_reg_type type) +{ + struct bw_hwmon *hw = &m->hw; + u32 hi, med, lo; + u32 zone_cnt_thres = calc_zone_counts(hw); + + hi = mbps_to_count(hw->up_wake_mbps, sample_ms, m->count_shift); + med = mbps_to_count(hw->down_wake_mbps, sample_ms, m->count_shift); + lo = 0; + + if (unlikely((hi > m->thres_lim) || (med > hi) || (lo > med))) { + pr_warn("Zone thres larger than hw limit: hi:%u med:%u lo:%u\n", + hi, med, lo); + hi = min(hi, m->thres_lim); + med = min(med, hi - 1); + lo = min(lo, med-1); + } + + switch (type) { + case MON1: + WARN(1, "Invalid\n"); + return; + case MON2: + writel_relaxed(hi, MON2_THRES_HI(m)); + writel_relaxed(med, MON2_THRES_MED(m)); + writel_relaxed(lo, MON2_THRES_LO(m)); + /* Set the zone count thresholds for interrupts */ + writel_relaxed(zone_cnt_thres, MON2_ZONE_CNT_THRES(m)); + break; + case MON3: + writel_relaxed(hi, MON3_THRES_HI(m)); + writel_relaxed(med, MON3_THRES_MED(m)); + writel_relaxed(lo, MON3_THRES_LO(m)); + /* Set the zone count thresholds for interrupts */ + writel_relaxed(zone_cnt_thres, MON3_ZONE_CNT_THRES(m)); + break; + } + + dev_dbg(m->dev, "Thres: hi:%u med:%u lo:%u\n", hi, med, lo); + dev_dbg(m->dev, "Zone Count Thres: %0x\n", zone_cnt_thres); +} + +static __always_inline +void mon_set_zones(struct bwmon *m, unsigned int sample_ms, + enum mon_reg_type type) +{ + mon_set_hw_sampling_window(m, sample_ms, type); + set_zone_thres(m, sample_ms, type); +} + +static void mon_set_limit(struct bwmon *m, u32 count) +{ + writel_relaxed(count, MON_THRES(m)); + dev_dbg(m->dev, "Thres: %08x\n", count); +} + +static u32 mon_get_limit(struct bwmon *m) +{ + return readl_relaxed(MON_THRES(m)); +} + +#define THRES_HIT(status) (status & BIT(0)) +#define OVERFLOW(status) (status & BIT(1)) +static unsigned long mon_get_count1(struct bwmon *m) +{ + unsigned long count, status; + + count = readl_relaxed(MON_CNT(m)); + status = mon_irq_status(m, MON1); + + dev_dbg(m->dev, "Counter: %08lx\n", count); + + if (OVERFLOW(status) && m->spec->overflow) + count += 0xFFFFFFFF; + if (THRES_HIT(status) && m->spec->wrap_on_thres) + count += mon_get_limit(m); + + dev_dbg(m->dev, "Actual Count: %08lx\n", count); + + return count; +} + +static __always_inline +unsigned int get_zone(struct bwmon *m, enum mon_reg_type type) +{ + u32 zone_counts; + u32 zone; + + zone = get_bitmask_order(m->intr_status); + if (zone) { + zone--; + } else { + switch (type) { + case MON1: + WARN(1, "Invalid\n"); + return 0; + case MON2: + zone_counts = readl_relaxed(MON2_ZONE_CNT(m)); + break; + case MON3: + zone_counts = readl_relaxed(MON3_ZONE_CNT(m)); + break; + } + + if (zone_counts) { + zone = get_bitmask_order(zone_counts) - 1; + zone /= 8; + } + } + + m->intr_status = 0; + return zone; +} + +static __always_inline +unsigned long get_zone_count(struct bwmon *m, unsigned int zone, + enum mon_reg_type type) +{ + unsigned long count; + + switch (type) { + case MON1: + WARN(1, "Invalid\n"); + return 0; + case MON2: + count = readl_relaxed(MON2_ZONE_MAX(m, zone)); + break; + case MON3: + count = readl_relaxed(MON3_ZONE_MAX(m, zone)); + break; + } + + if (count) + count++; + + return count; +} + +static __always_inline +unsigned long mon_get_zone_stats(struct bwmon *m, enum mon_reg_type type) +{ + unsigned int zone; + unsigned long count = 0; + + zone = get_zone(m, type); + count = get_zone_count(m, zone, type); + count <<= m->count_shift; + + dev_dbg(m->dev, "Zone%d Max byte count: %08lx\n", zone, count); + + return count; +} + +static __always_inline +unsigned long mon_get_count(struct bwmon *m, enum mon_reg_type type) +{ + unsigned long count; + + switch (type) { + case MON1: + count = mon_get_count1(m); + break; + case MON2: + case MON3: + count = mon_get_zone_stats(m, type); + break; + } + + return count; +} + +/* ********** CPUBW specific code ********** */ + +static __always_inline +unsigned long __get_bytes_and_clear(struct bw_hwmon *hw, enum mon_reg_type type) +{ + struct bwmon *m = to_bwmon(hw); + unsigned long count; + + mon_disable(m, type); + count = mon_get_count(m, type); + mon_clear(m, false, type); + mon_irq_clear(m, type); + mon_enable(m, type); + + return count; +} + +static unsigned long get_bytes_and_clear(struct bw_hwmon *hw) +{ + return __get_bytes_and_clear(hw, MON1); +} + +static unsigned long get_bytes_and_clear2(struct bw_hwmon *hw) +{ + return __get_bytes_and_clear(hw, MON2); +} + +static unsigned long get_bytes_and_clear3(struct bw_hwmon *hw) +{ + return __get_bytes_and_clear(hw, MON3); +} + +static unsigned long set_thres(struct bw_hwmon *hw, unsigned long bytes) +{ + unsigned long count; + u32 limit; + struct bwmon *m = to_bwmon(hw); + + mon_disable(m, MON1); + count = mon_get_count1(m); + mon_clear(m, false, MON1); + mon_irq_clear(m, MON1); + + if (likely(!m->spec->wrap_on_thres)) + limit = bytes; + else + limit = max(bytes, 500000UL); + + mon_set_limit(m, limit); + mon_enable(m, MON1); + + return count; +} + +static unsigned long +__set_hw_events(struct bw_hwmon *hw, unsigned int sample_ms, + enum mon_reg_type type) +{ + struct bwmon *m = to_bwmon(hw); + + mon_disable(m, type); + mon_clear(m, false, type); + mon_irq_clear(m, type); + + mon_set_zones(m, sample_ms, type); + mon_enable(m, type); + + return 0; +} + +static unsigned long set_hw_events(struct bw_hwmon *hw, unsigned int sample_ms) +{ + return __set_hw_events(hw, sample_ms, MON2); +} + +static unsigned long +set_hw_events3(struct bw_hwmon *hw, unsigned int sample_ms) +{ + return __set_hw_events(hw, sample_ms, MON3); +} + +static irqreturn_t +__bwmon_intr_handler(int irq, void *dev, enum mon_reg_type type) +{ + struct bwmon *m = dev; + + m->intr_status = mon_irq_status(m, type); + if (!m->intr_status) + return IRQ_NONE; + + if (bw_hwmon_sample_end(&m->hw) > 0) + return IRQ_WAKE_THREAD; + + return IRQ_HANDLED; +} + +static irqreturn_t bwmon_intr_handler(int irq, void *dev) +{ + return __bwmon_intr_handler(irq, dev, MON1); +} + +static irqreturn_t bwmon_intr_handler2(int irq, void *dev) +{ + return __bwmon_intr_handler(irq, dev, MON2); +} + +static irqreturn_t bwmon_intr_handler3(int irq, void *dev) +{ + return __bwmon_intr_handler(irq, dev, MON3); +} + +static irqreturn_t bwmon_intr_thread(int irq, void *dev) +{ + struct bwmon *m = dev; + + update_bw_hwmon(&m->hw); + return IRQ_HANDLED; +} + +static __always_inline +void mon_set_byte_count_filter(struct bwmon *m, enum mon_reg_type type) +{ + if (!m->byte_mask) + return; + + switch (type) { + case MON1: + case MON2: + writel_relaxed(m->byte_mask, MON_MASK(m)); + writel_relaxed(m->byte_match, MON_MATCH(m)); + break; + case MON3: + writel_relaxed(m->byte_mask, MON3_MASK(m)); + writel_relaxed(m->byte_match, MON3_MATCH(m)); + break; + } +} + +static __always_inline int __start_bw_hwmon(struct bw_hwmon *hw, + unsigned long mbps, enum mon_reg_type type) +{ + struct bwmon *m = to_bwmon(hw); + u32 limit, zone_actions; + int ret; + irq_handler_t handler; + + switch (type) { + case MON1: + handler = bwmon_intr_handler; + limit = mbps_to_bytes(mbps, hw->node->window_ms); + break; + case MON2: + zone_actions = calc_zone_actions(); + handler = bwmon_intr_handler2; + break; + case MON3: + zone_actions = calc_zone_actions(); + handler = bwmon_intr_handler3; + break; + } + + ret = request_threaded_irq(m->irq, handler, bwmon_intr_thread, + IRQF_ONESHOT | IRQF_SHARED, + dev_name(m->dev), m); + if (ret < 0) { + dev_err(m->dev, "Unable to register interrupt handler! (%d)\n", + ret); + return ret; + } + INIT_WORK(&hw->work, &bwmon_monitor_work); + + mon_disable(m, type); + + mon_clear(m, false, type); + + switch (type) { + case MON1: + mon_set_limit(m, limit); + break; + case MON2: + mon_set_zones(m, hw->node->window_ms, type); + /* Set the zone actions to increment appropriate counters */ + writel_relaxed(zone_actions, MON2_ZONE_ACTIONS(m)); + break; + case MON3: + mon_set_zones(m, hw->node->window_ms, type); + /* Set the zone actions to increment appropriate counters */ + writel_relaxed(zone_actions, MON3_ZONE_ACTIONS(m)); + } + + mon_set_byte_count_filter(m, type); + mon_irq_clear(m, type); + mon_irq_enable(m, type); + mon_enable(m, type); + bwmon_monitor_start(hw); + + return 0; +} + +static int start_bw_hwmon(struct bw_hwmon *hw, unsigned long mbps) +{ + return __start_bw_hwmon(hw, mbps, MON1); +} + +static int start_bw_hwmon2(struct bw_hwmon *hw, unsigned long mbps) +{ + return __start_bw_hwmon(hw, mbps, MON2); +} + +static int start_bw_hwmon3(struct bw_hwmon *hw, unsigned long mbps) +{ + return __start_bw_hwmon(hw, mbps, MON3); +} + +static __always_inline +void __stop_bw_hwmon(struct bw_hwmon *hw, enum mon_reg_type type) +{ + struct bwmon *m = to_bwmon(hw); + + bwmon_monitor_stop(hw); + mon_irq_disable(m, type); + free_irq(m->irq, m); + mon_disable(m, type); + mon_clear(m, true, type); + mon_irq_clear(m, type); +} + +static void stop_bw_hwmon(struct bw_hwmon *hw) +{ + return __stop_bw_hwmon(hw, MON1); +} + +static void stop_bw_hwmon2(struct bw_hwmon *hw) +{ + return __stop_bw_hwmon(hw, MON2); +} + +static void stop_bw_hwmon3(struct bw_hwmon *hw) +{ + return __stop_bw_hwmon(hw, MON3); +} + +/*************************************************************************/ + +static const struct bwmon_spec spec[] = { + [0] = { + .wrap_on_thres = true, + .overflow = false, + .throt_adj = false, + .hw_sampling = false, + .has_global_base = true, + .reg_type = MON1, + }, + [1] = { + .wrap_on_thres = false, + .overflow = true, + .throt_adj = false, + .hw_sampling = false, + .has_global_base = true, + .reg_type = MON1, + }, + [2] = { + .wrap_on_thres = false, + .overflow = true, + .throt_adj = true, + .hw_sampling = false, + .has_global_base = true, + .reg_type = MON1, + }, + [3] = { + .wrap_on_thres = false, + .overflow = true, + .throt_adj = true, + .hw_sampling = true, + .has_global_base = true, + .reg_type = MON2, + }, + [4] = { + .wrap_on_thres = false, + .overflow = true, + .throt_adj = false, + .hw_sampling = true, + .reg_type = MON3, + }, +}; + +static const struct of_device_id qcom_bwmon_match_table[] = { + { .compatible = "qcom,bwmon", .data = &spec[0] }, + { .compatible = "qcom,bwmon2", .data = &spec[1] }, + { .compatible = "qcom,bwmon3", .data = &spec[2] }, + { .compatible = "qcom,bwmon4", .data = &spec[3] }, + { .compatible = "qcom,bwmon5", .data = &spec[4] }, + {} +}; + +static int qcom_bwmon_driver_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *res; + struct bwmon *m; + struct hwmon_node *node; + int ret; + u32 data, count_unit; + u32 dcvs_hw = NUM_DCVS_PATHS; + struct kobject *dcvs_kobj; + struct device_node *of_node; + + m = devm_kzalloc(dev, sizeof(*m), GFP_KERNEL); + if (!m) + return -ENOMEM; + m->dev = dev; + m->hw.dev = dev; + + m->spec = of_device_get_match_data(dev); + if (!m->spec) { + dev_err(dev, "Unknown device type!\n"); + return -ENODEV; + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base"); + if (!res) { + dev_err(dev, "base not found!\n"); + return -EINVAL; + } + m->base = devm_ioremap(dev, res->start, resource_size(res)); + if (!m->base) { + dev_err(dev, "Unable map base!\n"); + return -ENOMEM; + } + + if (m->spec->has_global_base) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "global_base"); + if (!res) { + dev_err(dev, "global_base not found!\n"); + return -EINVAL; + } + m->global_base = devm_ioremap(dev, res->start, + resource_size(res)); + if (!m->global_base) { + dev_err(dev, "Unable map global_base!\n"); + return -ENOMEM; + } + + ret = of_property_read_u32(dev->of_node, "qcom,mport", &data); + if (ret < 0) { + dev_err(dev, "mport not found! (%d)\n", ret); + return ret; + } + m->mport = data; + } + + m->irq = platform_get_irq(pdev, 0); + if (m->irq < 0) { + dev_err(dev, "Unable to get IRQ number\n"); + return m->irq; + } + + if (m->spec->hw_sampling) { + ret = of_property_read_u32(dev->of_node, "qcom,hw-timer-hz", + &m->hw_timer_hz); + if (ret < 0) { + dev_err(dev, "HW sampling rate not specified!\n"); + return ret; + } + } + + if (of_property_read_u32(dev->of_node, "qcom,count-unit", &count_unit)) + count_unit = SZ_1M; + m->count_shift = order_base_2(count_unit); + m->thres_lim = THRES_LIM(m->count_shift); + + switch (m->spec->reg_type) { + case MON3: + m->hw.start_hwmon = start_bw_hwmon3; + m->hw.stop_hwmon = stop_bw_hwmon3; + m->hw.get_bytes_and_clear = get_bytes_and_clear3; + m->hw.set_hw_events = set_hw_events3; + break; + case MON2: + m->hw.start_hwmon = start_bw_hwmon2; + m->hw.stop_hwmon = stop_bw_hwmon2; + m->hw.get_bytes_and_clear = get_bytes_and_clear2; + m->hw.set_hw_events = set_hw_events; + break; + case MON1: + m->hw.start_hwmon = start_bw_hwmon; + m->hw.stop_hwmon = stop_bw_hwmon; + m->hw.get_bytes_and_clear = get_bytes_and_clear; + m->hw.set_thres = set_thres; + break; + } + + of_property_read_u32(dev->of_node, "qcom,byte-mid-match", + &m->byte_match); + of_property_read_u32(dev->of_node, "qcom,byte-mid-mask", + &m->byte_mask); + + if (m->spec->throt_adj) { + m->hw.set_throttle_adj = mon_set_throttle_adj; + m->hw.get_throttle_adj = mon_get_throttle_adj; + } + + of_node = of_parse_phandle(dev->of_node, "qcom,target-dev", 0); + if (!of_node) { + dev_err(dev, "Unable to find target-dev for bwmon device\n"); + return -EINVAL; + } + ret = of_property_read_u32(of_node, "qcom,dcvs-hw-type", &dcvs_hw); + if (ret < 0 || dcvs_hw >= NUM_DCVS_HW_TYPES) { + dev_err(dev, "invalid dcvs_hw=%d, ret=%d\n", dcvs_hw, ret); + return -EINVAL; + } + m->hw.dcvs_hw = dcvs_hw; + ret = of_property_read_u32(of_node, "qcom,bus-width", + &m->hw.dcvs_width); + if (ret < 0 || !m->hw.dcvs_width) { + dev_err(dev, "invalid hw width=%d, ret=%d\n", + m->hw.dcvs_width, ret); + return -EINVAL; + } + m->hw.dcvs_path = DCVS_SLOW_PATH; + + ret = qcom_dcvs_register_voter(dev_name(dev), dcvs_hw, m->hw.dcvs_path); + if (ret < 0) { + if (ret != -EPROBE_DEFER) + dev_err(dev, "qcom dcvs registration error: %d\n", ret); + return ret; + } + + ret = configure_hwmon_node(&m->hw); + if (ret < 0) { + dev_err(dev, "bwmon node configuration failed: %d\n", ret); + return ret; + } + node = m->hw.node; + + ret = qcom_dcvs_hw_minmax_get(dcvs_hw, &node->hw_min_freq, + &node->hw_max_freq); + if (ret < 0) { + dev_err(dev, "error getting minmax from qcom dcvs: %d\n", ret); + return ret; + } + node->min_freq = node->hw_min_freq; + node->max_freq = node->hw_max_freq; + node->cur_freq.ib = node->min_freq; + node->cur_freq.ab = 0; + node->cur_freq.hw_type = dcvs_hw; + + m->hw.is_active = false; + mutex_lock(&bwmon_lock); + if (!bwmon_wq) { + bwmon_wq = create_freezable_workqueue("bwmon_wq"); + if (!bwmon_wq) { + dev_err(dev, "Couldn't create bwmon workqueue.\n"); + mutex_unlock(&bwmon_lock); + return -ENOMEM; + } + register_trace_android_vh_jiffies_update( + bwmon_jiffies_update_cb, NULL); + } + mutex_unlock(&bwmon_lock); + ret = start_monitor(&m->hw); + if (ret < 0) { + dev_err(dev, "Error starting BWMON monitor: %d\n", ret); + return ret; + } + + dcvs_kobj = qcom_dcvs_kobject_get(dcvs_hw); + if (IS_ERR(dcvs_kobj)) { + ret = PTR_ERR(dcvs_kobj); + dev_err(dev, "error getting kobj from qcom_dcvs: %d\n", ret); + goto err_sysfs; + } + ret = kobject_init_and_add(&node->kobj, &bwmon_ktype, dcvs_kobj, + dev_name(dev)); + if (ret < 0) { + dev_err(dev, "failed to init bwmon kobj: %d\n", ret); + kobject_put(&node->kobj); + goto err_sysfs; + } + + return 0; + +err_sysfs: + stop_monitor(&m->hw); + return ret; +} + +static struct platform_driver qcom_bwmon_driver = { + .probe = qcom_bwmon_driver_probe, + .driver = { + .name = "qcom-bwmon", + .of_match_table = qcom_bwmon_match_table, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(qcom_bwmon_driver); + +MODULE_DESCRIPTION("QCOM BWMON driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/dcvs/bwmon.h b/drivers/soc/qcom/dcvs/bwmon.h new file mode 100644 index 000000000000..5579ae8811bc --- /dev/null +++ b/drivers/soc/qcom/dcvs/bwmon.h @@ -0,0 +1,194 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2014-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _QCOM_BWMON_H +#define _QCOM_BWMON_H + +#include +#include + +#define NUM_MBPS_ZONES 10 +#define UP_WAKE 1 +#define DOWN_WAKE 2 +#define MBYTE (1ULL << 20) +#define MBPS_TO_KHZ(mbps, w) (mult_frac(mbps, MBYTE, w * 1000ULL)) +#define KHZ_TO_MBPS(khz, w) (mult_frac(w * 1000ULL, khz, MBYTE)) +#define to_bwmon(ptr) container_of(ptr, struct bwmon, hw) + +enum mon_reg_type { + MON1, + MON2, + MON3, +}; + +struct bwmon_spec { + bool wrap_on_thres; + bool overflow; + bool throt_adj; + bool hw_sampling; + bool has_global_base; + enum mon_reg_type reg_type; +}; + +/** + * struct bw_hwmon - dev BW HW monitor info + * @start_hwmon: Start the HW monitoring of the dev BW + * @stop_hwmon: Stop the HW monitoring of dev BW + * @set_thres: Set the count threshold to generate an IRQ + * @set_hw_events: Set hw settings for up/down wake events + * @get_bytes_and_clear: Get the bytes transferred since the last call + * and reset the counter to start over. + * @set_throttle_adj: Set throttle adjust field to the given value + * @get_throttle_adj: Get the value written to throttle adjust field + * @dev: Pointer to device tied to this HW monitor + * @dcvs_hw: DCVS HW type that this HW is monitoring for + * @dcvs_path: DCVS Path type that this monitor votes on + * @node: Pointer to hwmon node that contains tunables + * @last_update_ts: Time that the last bwmon work was queued + * @work: bwmon monitor work + * @is_active: Toggled when HW monitor is started/stopped + * @up_wake_mbps: Setting for HW monitor to send IRQ for up wake + * @down_wake_mbps: Setting for HW monitor to send IRQ fow down wake + * @down_cnt: Setting for down sample count needed for wake + */ +struct bw_hwmon { + int (*start_hwmon)(struct bw_hwmon *hw, + unsigned long mbps); + void (*stop_hwmon)(struct bw_hwmon *hw); + unsigned long (*set_thres)(struct bw_hwmon *hw, + unsigned long bytes); + unsigned long (*set_hw_events)(struct bw_hwmon *hw, + unsigned int sample_ms); + unsigned long (*get_bytes_and_clear)(struct bw_hwmon *hw); + int (*set_throttle_adj)(struct bw_hwmon *hw, + uint adj); + u32 (*get_throttle_adj)(struct bw_hwmon *hw); + struct device *dev; + enum dcvs_hw_type dcvs_hw; + enum dcvs_path_type dcvs_path; + u32 dcvs_width; + struct hwmon_node *node; + ktime_t last_update_ts; + struct work_struct work; + bool is_active; + unsigned long up_wake_mbps; + unsigned long down_wake_mbps; + unsigned int down_cnt; +}; + +struct bwmon { + void __iomem *base; + void __iomem *global_base; + unsigned int mport; + int irq; + const struct bwmon_spec *spec; + struct device *dev; + struct bw_hwmon hw; + u32 hw_timer_hz; + u32 throttle_adj; + u32 sample_size_ms; + u32 intr_status; + u8 count_shift; + u32 thres_lim; + u32 byte_mask; + u32 byte_match; +}; + +struct hwmon_node { + u32 hw_min_freq; + u32 hw_max_freq; + u32 min_freq; + u32 max_freq; + struct dcvs_freq cur_freq; + u32 window_ms; + unsigned int guard_band_mbps; + unsigned int decay_rate; + unsigned int io_percent; + unsigned int bw_step; + unsigned int sample_ms; + unsigned int up_scale; + unsigned int up_thres; + unsigned int down_thres; + unsigned int down_count; + unsigned int hist_memory; + unsigned int hyst_trigger_count; + unsigned int hyst_length; + unsigned int idle_mbps; + unsigned int mbps_zones[NUM_MBPS_ZONES]; + unsigned long prev_ab; + unsigned long bytes; + unsigned long max_mbps; + unsigned long hist_max_mbps; + unsigned long hist_mem; + unsigned long hyst_peak; + unsigned long hyst_mbps; + unsigned long hyst_trig_win; + unsigned long hyst_en; + unsigned long prev_req; + unsigned int wake; + unsigned int down_cnt; + ktime_t prev_ts; + ktime_t hist_max_ts; + bool sampled; + bool mon_started; + struct list_head list; + struct bw_hwmon *hw; + struct kobject kobj; + struct mutex mon_lock; + struct mutex update_lock; +}; + +/* BWMON register offsets */ +#define GLB_INT_STATUS(m) ((m)->global_base + 0x100) +#define GLB_INT_CLR(m) ((m)->global_base + 0x108) +#define GLB_INT_EN(m) ((m)->global_base + 0x10C) +#define MON_INT_STATUS(m) ((m)->base + 0x100) +#define MON_INT_STATUS_MASK 0x03 +#define MON2_INT_STATUS_MASK 0xF0 +#define MON2_INT_STATUS_SHIFT 4 +#define MON_INT_CLR(m) ((m)->base + 0x108) +#define MON_INT_EN(m) ((m)->base + 0x10C) +#define MON_INT_ENABLE 0x1 +#define MON_EN(m) ((m)->base + 0x280) +#define MON_CLEAR(m) ((m)->base + 0x284) +#define MON_CNT(m) ((m)->base + 0x288) +#define MON_THRES(m) ((m)->base + 0x290) +#define MON_MASK(m) ((m)->base + 0x298) +#define MON_MATCH(m) ((m)->base + 0x29C) + +#define MON2_EN(m) ((m)->base + 0x2A0) +#define MON2_CLEAR(m) ((m)->base + 0x2A4) +#define MON2_SW(m) ((m)->base + 0x2A8) +#define MON2_THRES_HI(m) ((m)->base + 0x2AC) +#define MON2_THRES_MED(m) ((m)->base + 0x2B0) +#define MON2_THRES_LO(m) ((m)->base + 0x2B4) +#define MON2_ZONE_ACTIONS(m) ((m)->base + 0x2B8) +#define MON2_ZONE_CNT_THRES(m) ((m)->base + 0x2BC) +#define MON2_BYTE_CNT(m) ((m)->base + 0x2D0) +#define MON2_WIN_TIMER(m) ((m)->base + 0x2D4) +#define MON2_ZONE_CNT(m) ((m)->base + 0x2D8) +#define MON2_ZONE_MAX(m, zone) ((m)->base + 0x2E0 + 0x4 * zone) + +#define MON3_INT_STATUS(m) ((m)->base + 0x00) +#define MON3_INT_CLR(m) ((m)->base + 0x08) +#define MON3_INT_EN(m) ((m)->base + 0x0C) +#define MON3_INT_STATUS_MASK 0x0F +#define MON3_EN(m) ((m)->base + 0x10) +#define MON3_CLEAR(m) ((m)->base + 0x14) +#define MON3_MASK(m) ((m)->base + 0x18) +#define MON3_MATCH(m) ((m)->base + 0x1C) +#define MON3_SW(m) ((m)->base + 0x20) +#define MON3_THRES_HI(m) ((m)->base + 0x24) +#define MON3_THRES_MED(m) ((m)->base + 0x28) +#define MON3_THRES_LO(m) ((m)->base + 0x2C) +#define MON3_ZONE_ACTIONS(m) ((m)->base + 0x30) +#define MON3_ZONE_CNT_THRES(m) ((m)->base + 0x34) +#define MON3_BYTE_CNT(m) ((m)->base + 0x38) +#define MON3_WIN_TIMER(m) ((m)->base + 0x3C) +#define MON3_ZONE_CNT(m) ((m)->base + 0x40) +#define MON3_ZONE_MAX(m, zone) ((m)->base + 0x44 + 0x4 * zone) + +#endif /* _QCOM_BWMON_H */ diff --git a/drivers/soc/qcom/dcvs/c1dcvs_scmi.c b/drivers/soc/qcom/dcvs/c1dcvs_scmi.c new file mode 100644 index 000000000000..2751a6fe84b9 --- /dev/null +++ b/drivers/soc/qcom/dcvs/c1dcvs_scmi.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct kobject c1dcvs_kobj; +static struct scmi_protocol_handle *ph; +static const struct scmi_c1dcvs_vendor_ops *ops; +static unsigned int user_c1dcvs_en; +static unsigned int kernel_c1dcvs_en; +static DEFINE_MUTEX(c1dcvs_lock); + +struct qcom_c1dcvs_attr { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct attribute *attr, + char *buf); + ssize_t (*store)(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count); +}; + +#define to_c1dcvs_attr(_attr) \ + container_of(_attr, struct qcom_c1dcvs_attr, attr) +#define C1DCVS_ATTR_RW(_name) \ +static struct qcom_c1dcvs_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) \ + +#define store_c1dcvs_attr(name) \ +static ssize_t store_##name(struct kobject *kobj, \ + struct attribute *attr, const char *buf,\ + size_t count) \ +{ \ + unsigned int var; \ + int ret; \ + \ + if (!ops) \ + return -ENODEV; \ + \ + ret = kstrtouint(buf, 10, &var); \ + if (ret < 0) \ + return ret; \ + \ + ret = ops->set_##name(ph, &var); \ + return ((ret < 0) ? ret : count); \ +} \ + +#define show_c1dcvs_attr(name) \ +static ssize_t show_##name(struct kobject *kobj, \ + struct attribute *attr, char *buf) \ +{ \ + unsigned int var; \ + int ret; \ + \ + if (!ops) \ + return -ENODEV; \ + \ + ret = ops->get_##name(ph, &var); \ + if (ret < 0) \ + return ret; \ + \ + return scnprintf(buf, PAGE_SIZE, "%lu\n", le32_to_cpu(var)); \ +} \ + +/* + * Must hold c1dcvs_lock before calling this function + */ +static int update_enable_c1dcvs(void) +{ + unsigned int enable = min(user_c1dcvs_en, kernel_c1dcvs_en); + + if (!ops) + return -ENODEV; + + return ops->set_enable_c1dcvs(ph, &enable); +} + +static ssize_t store_enable_c1dcvs(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + unsigned int var; + int ret; + + if (!ops) + return -ENODEV; + + ret = kstrtouint(buf, 10, &var); + if (ret < 0) + return ret; + + mutex_lock(&c1dcvs_lock); + user_c1dcvs_en = var; + ret = update_enable_c1dcvs(); + mutex_unlock(&c1dcvs_lock); + + return ((ret < 0) ? ret : count); +} + +int c1dcvs_enable(bool enable) +{ + unsigned int data = enable ? 1 : 0; + int ret; + + if (!ops) + return -EPROBE_DEFER; + + mutex_lock(&c1dcvs_lock); + kernel_c1dcvs_en = data; + ret = update_enable_c1dcvs(); + mutex_unlock(&c1dcvs_lock); + + return ret; +} +EXPORT_SYMBOL(c1dcvs_enable); + +store_c1dcvs_attr(enable_trace); +show_c1dcvs_attr(enable_trace); +C1DCVS_ATTR_RW(enable_trace); +store_c1dcvs_attr(hysteresis); +show_c1dcvs_attr(hysteresis); +C1DCVS_ATTR_RW(hysteresis); +show_c1dcvs_attr(enable_c1dcvs); +C1DCVS_ATTR_RW(enable_c1dcvs); + +#define store_c1dcvs_thresh(name) \ +static ssize_t store_##name(struct kobject *kobj, \ + struct attribute *attr, const char *buf,\ + size_t count) \ +{ \ + int ret, i = 0; \ + char *s = kstrdup(buf, GFP_KERNEL); \ + unsigned int msg[2]; \ + char *str; \ + \ + if (!ops) \ + return -ENODEV; \ + \ + while (((str = strsep(&s, " ")) != NULL) && i < 2) { \ + ret = kstrtouint(str, 10, &msg[i]); \ + if (ret < 0) { \ + pr_err("Invalid value :%d\n", ret); \ + return -EINVAL; \ + } \ + i++; \ + } \ + \ + pr_info("Input threshold :%lu for cluster :%lu\n", msg[1], msg[0]);\ + ret = ops->set_##name(ph, msg); \ + return ((ret < 0) ? ret : count); \ +} \ + +#define show_c1dcvs_thresh(name) \ +static ssize_t show_##name(struct kobject *kobj, \ + struct attribute *attr, char *buf) \ +{ \ + unsigned int *vars = NULL; \ + int i, ret, tot = 0; \ + \ + if (!ops) \ + return -ENODEV; \ + \ + vars = kcalloc(num_possible_cpus(), sizeof(unsigned int), GFP_KERNEL);\ + if (!vars) \ + return -ENOMEM; \ + \ + ret = ops->get_##name(ph, vars); \ + if (ret < 0) { \ + kfree(vars); \ + return ret; \ + } \ + \ + for (i = 0; i < num_possible_cpus(); i++) { \ + vars[i] = le32_to_cpu(vars[i]); \ + if (!vars[i]) \ + break; \ + tot += scnprintf(buf + tot, PAGE_SIZE - tot, "%lu\t", vars[i]);\ + } \ + tot += scnprintf(buf + tot, PAGE_SIZE - tot, "\n"); \ + \ + kfree(vars); \ + return tot; \ +} \ + +store_c1dcvs_thresh(ipc_thresh); +show_c1dcvs_thresh(ipc_thresh); +C1DCVS_ATTR_RW(ipc_thresh); +store_c1dcvs_thresh(efreq_thresh); +show_c1dcvs_thresh(efreq_thresh); +C1DCVS_ATTR_RW(efreq_thresh); + +static struct attribute *c1dcvs_settings_attrs[] = { + &enable_c1dcvs.attr, + &enable_trace.attr, + &ipc_thresh.attr, + &efreq_thresh.attr, + &hysteresis.attr, + NULL, +}; +ATTRIBUTE_GROUPS(c1dcvs_settings); + +static ssize_t attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct qcom_c1dcvs_attr *c1dcvs_attr = to_c1dcvs_attr(attr); + ssize_t ret = -EIO; + + if (c1dcvs_attr->show) + ret = c1dcvs_attr->show(kobj, attr, buf); + + return ret; +} + +static ssize_t attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct qcom_c1dcvs_attr *c1dcvs_attr = to_c1dcvs_attr(attr); + ssize_t ret = -EIO; + + if (c1dcvs_attr->store) + ret = c1dcvs_attr->store(kobj, attr, buf, count); + + return ret; +} + +static const struct sysfs_ops c1dcvs_sysfs_ops = { + .show = attr_show, + .store = attr_store, +}; +static struct kobj_type c1dcvs_settings_ktype = { + .sysfs_ops = &c1dcvs_sysfs_ops, + .default_groups = c1dcvs_settings_groups, +}; + +static int scmi_c1dcvs_probe(struct scmi_device *sdev) +{ + int ret; + + if (!sdev) + return -ENODEV; + + ops = sdev->handle->devm_protocol_get(sdev, SCMI_C1DCVS_PROTOCOL, &ph); + if (!ops) + return -ENODEV; + + ret = kobject_init_and_add(&c1dcvs_kobj, &c1dcvs_settings_ktype, + &cpu_subsys.dev_root->kobj, "c1dcvs"); + if (ret < 0) { + pr_err("failed to init c1 dcvs kobj: %d\n", ret); + kobject_put(&c1dcvs_kobj); + } + + user_c1dcvs_en = kernel_c1dcvs_en = 1; + + return 0; +} + +static const struct scmi_device_id scmi_id_table[] = { + { .protocol_id = SCMI_C1DCVS_PROTOCOL, .name = "scmi_c1dcvs_protocol" }, + { }, +}; +MODULE_DEVICE_TABLE(scmi, scmi_id_table); + +static struct scmi_driver scmi_c1dcvs_drv = { + .name = "scmi-c1dcvs-driver", + .probe = scmi_c1dcvs_probe, + .id_table = scmi_id_table, +}; +module_scmi_driver(scmi_c1dcvs_drv); + +MODULE_SOFTDEP("pre: c1dcvs_vendor"); +MODULE_DESCRIPTION("ARM SCMI C1DCVS driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/dcvs/dcvs.c b/drivers/soc/qcom/dcvs/dcvs.c new file mode 100644 index 000000000000..6ef370f954d6 --- /dev/null +++ b/drivers/soc/qcom/dcvs/dcvs.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "qcom-dcvs: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dcvs_private.h" +#include "trace-dcvs.h" + +static const char * const dcvs_hw_names[NUM_DCVS_HW_TYPES] = { + [DCVS_DDR] = "DDR", + [DCVS_LLCC] = "LLCC", + [DCVS_L3] = "L3", + [DCVS_DDRQOS] = "DDRQOS", +}; + +enum dcvs_type { + QCOM_DCVS_DEV, + QCOM_DCVS_HW, + QCOM_DCVS_PATH, + NUM_DCVS_TYPES +}; + +struct qcom_dcvs_spec { + enum dcvs_type type; +}; + +struct dcvs_voter { + const char *name; + struct dcvs_freq freq; + struct list_head node; +}; + +struct qcom_dcvs_data { + struct kobject kobj; + struct dcvs_hw *hw_devs[NUM_DCVS_HW_TYPES]; + u32 num_hw; + u32 num_inited_hw; + bool inited; + +}; +static struct qcom_dcvs_data *dcvs_data; + +static u32 get_target_freq(struct dcvs_path *path, u32 freq); + +struct qcom_dcvs_attr { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct attribute *attr, + char *buf); + ssize_t (*store)(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count); +}; + +#define to_qcom_dcvs_attr(_attr) \ + container_of(_attr, struct qcom_dcvs_attr, attr) +#define to_dcvs_hw(k) container_of(k, struct dcvs_hw, kobj) + +#define DCVS_ATTR_RW(_name) \ +static struct qcom_dcvs_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) \ + +#define DCVS_ATTR_RO(_name) \ +static struct qcom_dcvs_attr _name = \ +__ATTR(_name, 0444, show_##_name, NULL) \ + + +#define show_attr(name) \ +static ssize_t show_##name(struct kobject *kobj, \ + struct attribute *attr, char *buf) \ +{ \ + struct dcvs_hw *hw = to_dcvs_hw(kobj); \ + return scnprintf(buf, PAGE_SIZE, "%u\n", hw->name); \ +} \ + +#define store_attr(name, _min, _max) \ +static ssize_t store_##name(struct kobject *kobj, \ + struct attribute *attr, const char *buf, \ + size_t count) \ +{ \ + int ret; \ + unsigned int val; \ + struct dcvs_hw *hw = to_dcvs_hw(kobj); \ + ret = kstrtouint(buf, 10, &val); \ + if (ret < 0) \ + return ret; \ + val = max(val, _min); \ + val = min(val, _max); \ + hw->name = val; \ + return count; \ +} \ + +static ssize_t store_boost_freq(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret; + unsigned int val; + struct dcvs_hw *hw = to_dcvs_hw(kobj); + struct dcvs_path *path; + struct dcvs_voter *voter; + struct dcvs_freq new_freq; + + ret = kstrtouint(buf, 10, &val); + if (ret < 0) + return ret; + if (val > hw->hw_max_freq) + return -EINVAL; + /* boost_freq only supported on hw with slow path */ + path = hw->dcvs_paths[DCVS_SLOW_PATH]; + if (!path) + return -EPERM; + + val = max(val, hw->hw_min_freq); + hw->boost_freq = val; + + /* must re-aggregate votes to get new freq after boost update */ + mutex_lock(&path->voter_lock); + new_freq.ib = new_freq.ab = 0; + new_freq.hw_type = hw->type; + list_for_each_entry(voter, &path->voter_list, node) { + new_freq.ib = max(voter->freq.ib, new_freq.ib); + new_freq.ab += voter->freq.ab; + } + new_freq.ib = get_target_freq(path, new_freq.ib); + if (new_freq.ib != path->cur_freq.ib) { + ret = path->commit_dcvs_freqs(path, &new_freq, 1); + if (ret < 0) + pr_err("Error setting boost freq: %d\n", ret); + } + mutex_unlock(&path->voter_lock); + + trace_qcom_dcvs_boost(hw->type, path->type, hw->boost_freq, + new_freq.ib, new_freq.ab); + + return count; +} + +static ssize_t show_cur_freq(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + int i, cpu; + struct dcvs_hw *hw = to_dcvs_hw(kobj); + struct dcvs_path *path; + u32 cur_freq = 0; + + for (i = 0; i < NUM_DCVS_PATHS; i++) { + path = hw->dcvs_paths[i]; + if (!path) + continue; + if (path->type != DCVS_PERCPU_PATH) { + cur_freq = max(cur_freq, path->cur_freq.ib); + continue; + } + for_each_possible_cpu(cpu) + cur_freq = max(cur_freq, path->percpu_cur_freqs[cpu]); + } + + return scnprintf(buf, PAGE_SIZE, "%lu\n", cur_freq); +} + +static ssize_t show_available_frequencies(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct dcvs_hw *hw = to_dcvs_hw(kobj); + int i, cnt = 0; + + for (i = 0; i < hw->table_len; i++) + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "%lu ", + hw->freq_table[i]); + + if (cnt) + cnt--; + + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); + + return cnt; +} + +show_attr(hw_min_freq); +show_attr(hw_max_freq); +show_attr(boost_freq); + +DCVS_ATTR_RO(hw_min_freq); +DCVS_ATTR_RO(hw_max_freq); +DCVS_ATTR_RW(boost_freq); +DCVS_ATTR_RO(cur_freq); +DCVS_ATTR_RO(available_frequencies); + +static struct attribute *dcvs_hw_attrs[] = { + &hw_min_freq.attr, + &hw_max_freq.attr, + &boost_freq.attr, + &cur_freq.attr, + &available_frequencies.attr, + NULL, +}; +ATTRIBUTE_GROUPS(dcvs_hw); + +static ssize_t attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct qcom_dcvs_attr *dcvs_attr = to_qcom_dcvs_attr(attr); + ssize_t ret = -EIO; + + if (dcvs_attr->show) + ret = dcvs_attr->show(kobj, attr, buf); + + return ret; +} + +static ssize_t attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct qcom_dcvs_attr *dcvs_attr = to_qcom_dcvs_attr(attr); + ssize_t ret = -EIO; + + if (dcvs_attr->store) + ret = dcvs_attr->store(kobj, attr, buf, count); + + return ret; +} + +static const struct sysfs_ops qcom_dcvs_sysfs_ops = { + .show = attr_show, + .store = attr_store, +}; + +static struct kobj_type qcom_dcvs_ktype = { + .sysfs_ops = &qcom_dcvs_sysfs_ops, +}; + +static struct kobj_type dcvs_hw_ktype = { + .sysfs_ops = &qcom_dcvs_sysfs_ops, + .default_groups = dcvs_hw_groups, +}; + +static inline struct dcvs_path *get_dcvs_path(enum dcvs_hw_type hw, + enum dcvs_path_type path) +{ + if (dcvs_data && dcvs_data->hw_devs[hw] && + dcvs_data->hw_devs[hw]->dcvs_paths[path]) + return dcvs_data->hw_devs[hw]->dcvs_paths[path]; + + return NULL; +} + +static u32 get_target_freq(struct dcvs_path *path, u32 freq) +{ + struct dcvs_hw *hw = path->hw; + u32 *freq_table = hw->freq_table; + u32 len = hw->table_len; + u32 target_freq = 0; + int i; + + if (path->type == DCVS_SLOW_PATH) + freq = max(freq, hw->boost_freq); + + for (i = 0; i < len; i++) { + if (freq <= freq_table[i]) { + target_freq = freq_table[i]; + break; + } + } + + if (i == len) + target_freq = freq_table[len-1]; + + return target_freq; +} + +/* slow path update: does aggregation across multiple clients */ +static int qcom_dcvs_sp_update(const char *name, struct dcvs_freq *votes, + u32 update_mask) +{ + int i, ret = 0; + bool found = false; + struct dcvs_voter *voter; + struct dcvs_path *path; + struct dcvs_freq new_freq; + enum dcvs_hw_type hw_type; + + if (!name || !votes || !update_mask) + return -EINVAL; + + for (i = 0; i < NUM_DCVS_HW_TYPES; i++) { + if (!(update_mask & BIT(i))) + continue; + hw_type = votes[i].hw_type; + path = get_dcvs_path(hw_type, DCVS_SLOW_PATH); + if (!path) + return -EINVAL; + + mutex_lock(&path->voter_lock); + new_freq.ib = new_freq.ab = 0; + new_freq.hw_type = hw_type; + found = false; + list_for_each_entry(voter, &path->voter_list, node) { + if (!strcmp(voter->name, name)) { + found = true; + voter->freq.ib = votes[i].ib; + voter->freq.ab = votes[i].ab; + } + new_freq.ib = max(voter->freq.ib, new_freq.ib); + new_freq.ab += voter->freq.ab; + } + if (!found) { + mutex_unlock(&path->voter_lock); + return -EINVAL; + } + new_freq.ib = get_target_freq(path, new_freq.ib); + if (new_freq.ib != path->cur_freq.ib || + new_freq.ab != path->cur_freq.ab) + ret = path->commit_dcvs_freqs(path, &new_freq, 1); + mutex_unlock(&path->voter_lock); + if (ret < 0) + return ret; + trace_qcom_dcvs_update(name, hw_type, path->type, votes[i].ib, + new_freq.ib, votes[i].ab, new_freq.ab, + path->hw->boost_freq); + } + + return ret; +} + +/* fast path update: only single client allowed so lockless */ +static int qcom_dcvs_fp_update(const char *name, struct dcvs_freq *votes, + u32 update_mask) +{ + int i, ret = 0; + u32 commit_mask = 0; + struct dcvs_voter *voter; + struct dcvs_path *path; + struct dcvs_freq new_freqs[NUM_DCVS_HW_TYPES]; + enum dcvs_hw_type hw_type; + + if (!name || !votes || !update_mask) + return -EINVAL; + + for (i = 0; i < NUM_DCVS_HW_TYPES; i++) { + if (!(update_mask & BIT(i))) + continue; + hw_type = votes[i].hw_type; + path = get_dcvs_path(hw_type, DCVS_FAST_PATH); + /* fast path requires votes be passed in correct order */ + if (!path || i != hw_type) + return -EINVAL; + + /* should match one and only voter in list */ + voter = list_first_entry(&path->voter_list, struct dcvs_voter, + node); + if (!voter || strcmp(voter->name, name)) + return -EINVAL; + + /* no aggregation required since only single client allowed */ + new_freqs[i].ib = get_target_freq(path, votes[i].ib); + if (new_freqs[i].ib != path->cur_freq.ib) + commit_mask |= BIT(i); + trace_qcom_dcvs_update(name, hw_type, path->type, votes[i].ib, + new_freqs[i].ib, 0, 0, 0); + } + + if (commit_mask) + ret = path->commit_dcvs_freqs(path, new_freqs, commit_mask); + + return ret; +} + +/* + * percpu path update: only single client per cpu allowed so lockless. + * Also note that client is responsible for disabling preemption + */ +static int qcom_dcvs_percpu_update(const char *name, struct dcvs_freq *votes, + u32 update_mask) +{ + int i, ret = 0; + u32 cpu = smp_processor_id(); + struct dcvs_voter *voter; + struct dcvs_path *path; + struct dcvs_freq new_freq; + enum dcvs_hw_type hw_type; + + if (!name || !votes || !update_mask) + return -EINVAL; + + for (i = 0; i < NUM_DCVS_HW_TYPES; i++) { + if (!(update_mask & BIT(i))) + continue; + hw_type = votes[i].hw_type; + path = get_dcvs_path(hw_type, DCVS_PERCPU_PATH); + if (!path) + return -EINVAL; + + /* should match one and only voter in list */ + voter = list_first_entry(&path->voter_list, struct dcvs_voter, + node); + if (!voter || strcmp(voter->name, name)) + return -EINVAL; + + /* no aggregation required since only single client per cpu */ + new_freq.ib = get_target_freq(path, votes[i].ib); + new_freq.hw_type = hw_type; + if (new_freq.ib != path->percpu_cur_freqs[cpu]) { + ret = path->commit_dcvs_freqs(path, &new_freq, 1); + if (ret < 0) + return ret; + } + trace_qcom_dcvs_update(name, hw_type, path->type, votes[i].ib, + new_freq.ib, 0, 0, 0); + } + + return ret; +} + +int qcom_dcvs_update_votes(const char *name, struct dcvs_freq *votes, + u32 update_mask, enum dcvs_path_type path) +{ + switch (path) { + case DCVS_SLOW_PATH: + return qcom_dcvs_sp_update(name, votes, update_mask); + case DCVS_FAST_PATH: + return qcom_dcvs_fp_update(name, votes, update_mask); + case DCVS_PERCPU_PATH: + return qcom_dcvs_percpu_update(name, votes, update_mask); + default: + break; + } + + return -EINVAL; +} +EXPORT_SYMBOL(qcom_dcvs_update_votes); + +int qcom_dcvs_register_voter(const char *name, enum dcvs_hw_type hw_type, + enum dcvs_path_type path_type) +{ + int ret = 0; + struct dcvs_voter *voter; + struct dcvs_path *path; + + if (!name || hw_type >= NUM_DCVS_HW_TYPES || path_type >= NUM_DCVS_PATHS) + return -EINVAL; + + if (!dcvs_data->inited) + return -EPROBE_DEFER; + + path = get_dcvs_path(hw_type, path_type); + if (!path) + return -ENODEV; + + mutex_lock(&path->voter_lock); + if (path_type == DCVS_FAST_PATH && path->num_voters >= 1) { + ret = -EINVAL; + goto unlock_out; + } + list_for_each_entry(voter, &path->voter_list, node) + if (!strcmp(voter->name, name)) { + ret = -EINVAL; + goto unlock_out; + } + + voter = kzalloc(sizeof(*voter), GFP_KERNEL); + if (!voter) { + ret = -ENOMEM; + goto unlock_out; + } + voter->name = name; + list_add_tail(&voter->node, &path->voter_list); + path->num_voters++; + +unlock_out: + mutex_unlock(&path->voter_lock); + return ret; +} +EXPORT_SYMBOL(qcom_dcvs_register_voter); + +int qcom_dcvs_unregister_voter(const char *name, enum dcvs_hw_type hw_type, + enum dcvs_path_type path_type) +{ + int ret = 0; + struct dcvs_voter *voter; + struct dcvs_path *path; + bool found = false; + + if (!name || hw_type >= NUM_DCVS_HW_TYPES || path_type >= NUM_DCVS_PATHS) + return -EINVAL; + + path = get_dcvs_path(hw_type, path_type); + if (!path) + return -ENODEV; + + mutex_lock(&path->voter_lock); + list_for_each_entry(voter, &path->voter_list, node) + if (!strcmp(voter->name, name)) { + found = true; + break; + } + + if (!found) { + ret = -EINVAL; + goto unlock_out; + } + + path->num_voters--; + list_del(&voter->node); + kfree(voter); + +unlock_out: + mutex_unlock(&path->voter_lock); + return ret; +} +EXPORT_SYMBOL(qcom_dcvs_unregister_voter); + +struct kobject *qcom_dcvs_kobject_get(enum dcvs_hw_type type) +{ + struct kobject *kobj; + + if (type > NUM_DCVS_HW_TYPES) + return ERR_PTR(-EINVAL); + + if (!dcvs_data->inited) + return ERR_PTR(-EPROBE_DEFER); + + if (type == NUM_DCVS_HW_TYPES) + kobj = &dcvs_data->kobj; + else if (dcvs_data->hw_devs[type]) + kobj = &dcvs_data->hw_devs[type]->kobj; + else + return ERR_PTR(-ENODEV); + + if (!kobj->state_initialized) + return ERR_PTR(-ENODEV); + + return kobj; +} +EXPORT_SYMBOL(qcom_dcvs_kobject_get); + +int qcom_dcvs_hw_minmax_get(enum dcvs_hw_type hw_type, u32 *min, u32 *max) +{ + struct dcvs_hw *hw; + + if (hw_type >= NUM_DCVS_HW_TYPES) + return -EINVAL; + + if (!dcvs_data->inited) + return -EPROBE_DEFER; + + hw = dcvs_data->hw_devs[hw_type]; + if (!hw) + return -ENODEV; + + *min = hw->hw_min_freq; + *max = hw->hw_max_freq; + + return 0; +} +EXPORT_SYMBOL(qcom_dcvs_hw_minmax_get); + +struct device_node *qcom_dcvs_get_ddr_child_node( + struct device_node *of_parent) +{ + struct device_node *of_child; + int dcvs_ddr_type = -1; + int of_ddr_type = of_fdt_get_ddrtype(); + int ret; + + for_each_child_of_node(of_parent, of_child) { + ret = of_property_read_u32(of_child, "qcom,ddr-type", + &dcvs_ddr_type); + if (!ret && (dcvs_ddr_type == of_ddr_type)) + return of_child; + } + + return NULL; +} +EXPORT_SYMBOL(qcom_dcvs_get_ddr_child_node); + +static bool qcom_dcvs_hw_and_paths_inited(void) +{ + int i; + struct dcvs_hw *hw; + + if (dcvs_data->num_inited_hw < dcvs_data->num_hw) + return false; + + for (i = 0; i < NUM_DCVS_HW_TYPES; i++) { + hw = dcvs_data->hw_devs[i]; + if (!hw) + continue; + if (hw->num_inited_paths < hw->num_paths) + return false; + } + + return true; +} + +#define FTBL_PROP "qcom,freq-tbl" +static int populate_freq_table(struct device *dev, u32 **freq_table) +{ + int ret, len; + struct device_node *of_node = dev->of_node; + + if (of_parse_phandle(of_node, FTBL_PROP, 0)) + of_node = of_parse_phandle(of_node, FTBL_PROP, 0); + if (of_get_child_count(of_node)) + of_node = qcom_dcvs_get_ddr_child_node(of_node); + + if (!of_find_property(of_node, FTBL_PROP, &len)) { + dev_err(dev, "Unable to find freq tbl prop\n"); + return -EINVAL; + } + len /= sizeof(**freq_table); + if (!len) { + dev_err(dev, "Error: empty freq table\n"); + return -EINVAL; + } + + *freq_table = devm_kzalloc(dev, len * sizeof(**freq_table), GFP_KERNEL); + if (!*freq_table) + return -ENOMEM; + + ret = of_property_read_u32_array(of_node, FTBL_PROP, *freq_table, len); + if (ret < 0) { + dev_err(dev, "Error reading freq table from DT: %d\n", ret); + return ret; + } + + return len; +} + +static int qcom_dcvs_dev_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret; + + dcvs_data = devm_kzalloc(dev, sizeof(*dcvs_data), GFP_KERNEL); + if (!dcvs_data) + return -ENOMEM; + + dcvs_data->num_hw = of_get_available_child_count(dev->of_node); + if (!dcvs_data->num_hw) { + dev_err(dev, "No dcvs hw nodes provided!\n"); + return -ENODEV; + } + + ret = kobject_init_and_add(&dcvs_data->kobj, &qcom_dcvs_ktype, + &cpu_subsys.dev_root->kobj, "bus_dcvs"); + if (ret < 0) { + dev_err(dev, "failed to init qcom-dcvs kobj: %d\n", ret); + kobject_put(&dcvs_data->kobj); + return ret; + } + dev_dbg(dev, "Created kobj: %s\n", kobject_name(&dcvs_data->kobj)); + + return 0; +} + +static int qcom_dcvs_hw_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + enum dcvs_hw_type hw_type = NUM_DCVS_HW_TYPES; + struct dcvs_hw *hw = NULL; + int ret = 0; + + if (!dcvs_data) { + dev_err(dev, "Missing QCOM DCVS dev data\n"); + return -ENODEV; + } + + ret = of_property_read_u32(dev->of_node, QCOM_DCVS_HW_PROP, &hw_type); + if (ret < 0 || hw_type >= NUM_DCVS_HW_TYPES) { + dev_err(dev, "Invalid dcvs hw type:%d, ret:%d\n", hw_type, ret); + return -ENODEV; + } + + hw = devm_kzalloc(dev, sizeof(*hw), GFP_KERNEL); + if (!hw) + return -ENOMEM; + hw->dev = dev; + hw->type = hw_type; + + hw->num_paths = of_get_available_child_count(dev->of_node); + if (!hw->num_paths) { + dev_err(dev, "No dcvs paths provided!\n"); + return -ENODEV; + } + + if (hw_type == DCVS_L3) + ret = populate_l3_table(dev, &hw->freq_table); + else + ret = populate_freq_table(dev, &hw->freq_table); + + if (ret <= 0) { + dev_err(dev, "Error reading freq table: %d\n", ret); + return ret; + } + hw->table_len = ret; + + hw->hw_max_freq = hw->freq_table[hw->table_len-1]; + hw->hw_min_freq = hw->freq_table[0]; + + ret = of_property_read_u32(dev->of_node, QCOM_DCVS_WIDTH_PROP, + &hw->width); + if (ret < 0 || !hw->width) { + dev_err(dev, "Missing or invalid bus-width: %d\n", ret); + return -EINVAL; + } + + ret = kobject_init_and_add(&hw->kobj, &dcvs_hw_ktype, + &dcvs_data->kobj, dcvs_hw_names[hw_type]); + if (ret < 0) { + dev_err(dev, "failed to init dcvs hw kobj: %d\n", ret); + kobject_put(&hw->kobj); + return ret; + } + dev_dbg(dev, "Created hw kobj: %s\n", kobject_name(&hw->kobj)); + + dev_set_drvdata(dev, hw); + dcvs_data->hw_devs[hw_type] = hw; + dcvs_data->num_inited_hw++; + + return ret; +} + +static int qcom_dcvs_path_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret = 0; + enum dcvs_path_type path_type = NUM_DCVS_PATHS; + struct dcvs_hw *hw = dev_get_drvdata(dev->parent); + struct dcvs_path *path = NULL; + struct dcvs_freq new_freqs[NUM_DCVS_HW_TYPES]; + + if (!hw) { + dev_err(dev, "QCOM DCVS HW not configured\n"); + return -ENODEV; + } + + ret = of_property_read_u32(dev->of_node, QCOM_DCVS_PATH_PROP, + &path_type); + if (ret < 0 || path_type >= NUM_DCVS_PATHS) { + dev_err(dev, "Invalid path type:%d, ret:%d\n", path_type, ret); + return -ENODEV; + } + + path = devm_kzalloc(dev, sizeof(*path), GFP_KERNEL); + if (!path) + return -ENOMEM; + path->dev = dev; + path->type = path_type; + path->hw = hw; + + switch (path_type) { + case DCVS_SLOW_PATH: + if (hw->type == DCVS_DDR || hw->type == DCVS_LLCC + || hw->type == DCVS_DDRQOS) + ret = setup_icc_sp_device(dev, hw, path); + else if (hw->type == DCVS_L3) + ret = setup_epss_l3_sp_device(dev, hw, path); + if (ret < 0) { + dev_err(dev, "Error setting up sp dev: %d\n", ret); + return ret; + } + break; + case DCVS_FAST_PATH: + if (hw->type != DCVS_DDR && hw->type != DCVS_LLCC) { + dev_err(dev, "Unsupported HW for dcvs fp: %d\n", ret); + return -EINVAL; + } + ret = setup_ddrllcc_fp_device(dev, hw, path); + if (ret < 0) { + dev_err(dev, "Error setting up fp dev: %d\n", ret); + return ret; + } + break; + case DCVS_PERCPU_PATH: + if (hw->type != DCVS_L3) { + dev_err(dev, "Unsupported HW for path: %d\n", ret); + return -EINVAL; + } + path->percpu_cur_freqs = devm_kzalloc(dev, num_possible_cpus() * + sizeof(*path->percpu_cur_freqs), + GFP_KERNEL); + if (!path->percpu_cur_freqs) + return -ENOMEM; + ret = setup_epss_l3_percpu_device(dev, hw, path); + if (ret < 0) { + dev_err(dev, "Error setting up percpu dev: %d\n", ret); + return ret; + } + break; + default: + /* this should never happen */ + return -EINVAL; + } + + INIT_LIST_HEAD(&path->voter_list); + mutex_init(&path->voter_lock); + + /* start slow paths with boost_freq = max_freq for better boot perf */ + if (path->type == DCVS_SLOW_PATH) { + hw->boost_freq = hw->hw_max_freq; + new_freqs[hw->type].ib = hw->hw_max_freq; + new_freqs[hw->type].ab = 0; + new_freqs[hw->type].hw_type = hw->type; + ret = path->commit_dcvs_freqs(path, &new_freqs[hw->type], 1); + if (ret < 0) + dev_err(dev, "Err committing freq for path=%d\n", ret); + } + + hw->dcvs_paths[path_type] = path; + hw->num_inited_paths++; + + if (qcom_dcvs_hw_and_paths_inited()) + dcvs_data->inited = true; + + return ret; +} + +static int qcom_dcvs_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret = 0; + const struct qcom_dcvs_spec *spec = of_device_get_match_data(dev); + enum dcvs_type type = NUM_DCVS_TYPES; + + if (spec) + type = spec->type; + + switch (type) { + case QCOM_DCVS_DEV: + if (dcvs_data) { + dev_err(dev, "Only one qcom-dcvs device allowed\n"); + ret = -ENODEV; + break; + } + ret = qcom_dcvs_dev_probe(pdev); + if (!ret && of_get_available_child_count(dev->of_node)) + of_platform_populate(dev->of_node, NULL, NULL, dev); + break; + case QCOM_DCVS_HW: + ret = qcom_dcvs_hw_probe(pdev); + if (!ret && of_get_available_child_count(dev->of_node)) + of_platform_populate(dev->of_node, NULL, NULL, dev); + break; + case QCOM_DCVS_PATH: + ret = qcom_dcvs_path_probe(pdev); + break; + default: + dev_err(dev, "Invalid qcom-dcvs type: %u\n", type); + return -EINVAL; + } + + if (ret < 0) { + dev_err(dev, "Failed to probe qcom-dcvs device: %d\n", ret); + return ret; + } + + return 0; +} + +static const struct qcom_dcvs_spec spec[] = { + [0] = { QCOM_DCVS_DEV }, + [1] = { QCOM_DCVS_HW }, + [2] = { QCOM_DCVS_PATH }, +}; + +static const struct of_device_id qcom_dcvs_match_table[] = { + { .compatible = "qcom,dcvs", .data = &spec[0] }, + { .compatible = "qcom,dcvs-hw", .data = &spec[1] }, + { .compatible = "qcom,dcvs-path", .data = &spec[2] }, + {} +}; + +static struct platform_driver qcom_dcvs_driver = { + .probe = qcom_dcvs_probe, + .driver = { + .name = "qcom-dcvs", + .of_match_table = qcom_dcvs_match_table, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(qcom_dcvs_driver); + +MODULE_DESCRIPTION("QCOM DCVS Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/dcvs/dcvs_epss.c b/drivers/soc/qcom/dcvs/dcvs_epss.c new file mode 100644 index 000000000000..7e609b10a5a7 --- /dev/null +++ b/drivers/soc/qcom/dcvs/dcvs_epss.c @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "qcom-dcvs-epss: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dcvs_private.h" + +struct epss_dev_data { + void __iomem *l3_base; + u32 l3_shared_offset; + u32 *l3_percpu_offsets; +}; + +struct epss_dev_data *epss_data; +static DEFINE_MUTEX(epss_lock); + +#define L3_VOTING_OFFSET 0x90 +#define L3_DOMAIN_OFFSET 0x1000 + +#define MAX_L3_ENTRIES 40U +#define INIT_HZ 300000000UL +#define XO_HZ 19200000UL +#define FTBL_ROW_SIZE 4 +#define SRC_MASK GENMASK(31, 30) +#define SRC_SHIFT 30 +#define MULT_MASK GENMASK(7, 0) +int populate_l3_table(struct device *dev, u32 **freq_table) +{ + int idx, ret, len; + u32 data, src, mult; + unsigned long freq, prev_freq = 0; + struct resource res; + void __iomem *ftbl_base; + unsigned int ftbl_row_size; + u32 *tmp_l3_table; + + idx = of_property_match_string(dev->of_node, "reg-names", "l3tbl-base"); + if (idx < 0) { + dev_err(dev, "Unable to find l3tbl-base: %d\n", idx); + return -EINVAL; + } + + ret = of_address_to_resource(dev->of_node, idx, &res); + if (ret < 0) { + dev_err(dev, "Unable to get resource from address: %d\n", ret); + return -EINVAL; + } + + ftbl_base = ioremap(res.start, resource_size(&res)); + if (!ftbl_base) { + dev_err(dev, "Unable to map l3tbl-base!\n"); + return -ENOMEM; + } + + ret = of_property_read_u32(dev->of_node, "qcom,ftbl-row-size", + &ftbl_row_size); + if (ret < 0) + ftbl_row_size = FTBL_ROW_SIZE; + + tmp_l3_table = kcalloc(MAX_L3_ENTRIES, sizeof(*tmp_l3_table), GFP_KERNEL); + if (!tmp_l3_table) { + iounmap(ftbl_base); + return -ENOMEM; + } + + for (idx = 0; idx < MAX_L3_ENTRIES; idx++) { + data = readl_relaxed(ftbl_base + idx * ftbl_row_size); + src = ((data & SRC_MASK) >> SRC_SHIFT); + mult = (data & MULT_MASK); + freq = src ? XO_HZ * mult : INIT_HZ; + + /* Two of the same frequencies means end of table */ + if (idx > 0 && prev_freq == freq) + break; + + tmp_l3_table[idx] = freq / 1000UL; + prev_freq = freq; + } + len = idx; + + *freq_table = devm_kzalloc(dev, len * sizeof(**freq_table), GFP_KERNEL); + if (!*freq_table) { + iounmap(ftbl_base); + return -ENOMEM; + } + + for (idx = 0; idx < len; idx++) + (*freq_table)[idx] = tmp_l3_table[idx]; + + iounmap(ftbl_base); + kfree(tmp_l3_table); + + return len; +} + +static int commit_epss_l3(struct dcvs_path *path, struct dcvs_freq *freqs, + u32 update_mask, bool shared) +{ + struct dcvs_hw *hw = path->hw; + struct epss_dev_data *d = path->data; + int cpu; + u32 idx, offset; + + for (idx = 0; idx < hw->table_len; idx++) + if (freqs->ib <= hw->freq_table[idx]) + break; + + if (hw->type == DCVS_L3) { + if (shared) + offset = d->l3_shared_offset; + else { + cpu = smp_processor_id(); + offset = d->l3_percpu_offsets[cpu]; + } + writel_relaxed(idx, d->l3_base + offset); + } + + path->cur_freq.ib = freqs->ib; + + return 0; +} + +static int commit_epss_l3_shared(struct dcvs_path *path, + struct dcvs_freq *freqs, + u32 update_mask) +{ + return commit_epss_l3(path, freqs, update_mask, true); +} + +static int commit_epss_l3_percpu(struct dcvs_path *path, + struct dcvs_freq *freqs, + u32 update_mask) +{ + return commit_epss_l3(path, freqs, update_mask, false); +} + +static int init_epss_data(struct device *dev) +{ + int idx, ret = 0; + struct resource res; + + epss_data = devm_kzalloc(dev, sizeof(*epss_data), GFP_KERNEL); + if (!epss_data) + return -ENOMEM; + + idx = of_property_match_string(dev->parent->of_node, "reg-names", + "l3-base"); + if (idx < 0) { + dev_err(dev, "%s: Unable to find l3-base: %d\n", __func__, idx); + return -EINVAL; + } + + ret = of_address_to_resource(dev->parent->of_node, idx, &res); + if (ret < 0) { + dev_err(dev, "Unable to get resource from address: %d\n", ret); + return ret; + } + + epss_data->l3_base = devm_ioremap(dev->parent, res.start, + resource_size(&res)); + if (!epss_data->l3_base) { + dev_err(dev, "Unable to map l3-base!\n"); + return -ENOMEM; + } + + return ret; + +} + +static int populate_shared_offset(struct device *dev, u32 *offset) +{ + int ret; + + ret = of_property_read_u32(dev->of_node, "qcom,shared-offset", offset); + if (ret < 0) { + dev_err(dev, "Error reading shared offset: %d\n", ret); + return ret; + } + + return ret; +} + +#define PERCPU_OFFSETS "qcom,percpu-offsets" +static int populate_percpu_offsets(struct device *dev, u32 **cpu_offsets) +{ + int ret, len; + struct device_node *of_node = dev->of_node; + + if (of_parse_phandle(of_node, PERCPU_OFFSETS, 0)) + of_node = of_parse_phandle(of_node, PERCPU_OFFSETS, 0); + + if (!of_find_property(of_node, PERCPU_OFFSETS, &len)) { + dev_err(dev, "Unable to find percpu offsets prop!\n"); + return -EINVAL; + } + len /= sizeof(**cpu_offsets); + if (len != num_possible_cpus()) { + dev_err(dev, "Invalid percpu offsets table len=%d\n", len); + return -EINVAL; + } + + *cpu_offsets = devm_kzalloc(dev, len * sizeof(**cpu_offsets), + GFP_KERNEL); + if (!*cpu_offsets) + return -ENOMEM; + + ret = of_property_read_u32_array(of_node, PERCPU_OFFSETS, *cpu_offsets, + len); + if (ret < 0) { + dev_err(dev, "Error reading percpu offsets from DT: %d\n", ret); + return ret; + } + + return ret; +} + +static int setup_epss_l3_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path, bool shared) +{ + int ret = 0; + + mutex_lock(&epss_lock); + if (!epss_data) + ret = init_epss_data(dev); + mutex_unlock(&epss_lock); + + if (ret < 0) + return ret; + + if (shared) { + ret = populate_shared_offset(dev, &epss_data->l3_shared_offset); + path->commit_dcvs_freqs = commit_epss_l3_shared; + } else { + ret = populate_percpu_offsets(dev, &epss_data->l3_percpu_offsets); + path->commit_dcvs_freqs = commit_epss_l3_percpu; + } + if (ret < 0) + return ret; + + path->data = epss_data; + + return ret; +} + +int setup_epss_l3_sp_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path) +{ + return setup_epss_l3_device(dev, hw, path, true); +} + +int setup_epss_l3_percpu_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path) +{ + return setup_epss_l3_device(dev, hw, path, false); +} diff --git a/drivers/soc/qcom/dcvs/dcvs_fp.c b/drivers/soc/qcom/dcvs/dcvs_fp.c new file mode 100644 index 000000000000..a5d78e521432 --- /dev/null +++ b/drivers/soc/qcom/dcvs/dcvs_fp.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "qcom-dcvs-fp: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dcvs_private.h" + +struct bcm_db { + __le32 unit; + __le16 width; + u8 vcd; + u8 reserved; +}; + +struct bcm_data { + u32 addr; + u32 unit; + u32 width; + u32 vcd; +}; + +enum ddrllcc_fp_idx { + DDR_IDX, + LLCC_IDX, + NUM_FP_CMDS +}; + +struct ddrllcc_fp_data { + struct device *dev; + struct dcvs_path *paths[NUM_FP_CMDS]; + struct bcm_data bcms[NUM_FP_CMDS]; + struct tcs_cmd tcs_cmds[NUM_FP_CMDS]; +}; + +struct ddrllcc_fp_data *ddrllcc_data; +static DEFINE_MUTEX(ddrllcc_lock); + +static int ddrllcc_fp_commit(struct dcvs_path *path, struct dcvs_freq *freqs, + u32 update_mask) +{ + struct ddrllcc_fp_data *fp_data = path->data; + struct tcs_cmd *tcs_cmds = fp_data->tcs_cmds; + struct bcm_data *bcms = fp_data->bcms; + struct dcvs_path **paths = fp_data->paths; + struct device *dev = fp_data->dev; + u32 bcm_vote; + int i, ret = 0; + + for (i = 0; i < NUM_FP_CMDS; i++) { + if (!(update_mask & BIT(i))) + continue; + bcm_vote = freqs[i].ib * bcms[i].width / bcms[i].unit; + tcs_cmds[i].data = BCM_TCS_CMD(1, 1, 0, bcm_vote); + } + + ret = rpmh_update_fast_path(dev, tcs_cmds, NUM_FP_CMDS, update_mask); + if (ret < 0) { + dev_err(dev, "Error updating RPMH fast path: %d\n", ret); + return ret; + } + + for (i = 0; i < NUM_FP_CMDS; i++) { + if (!(update_mask & BIT(i))) + continue; + paths[i]->cur_freq.ib = freqs[i].ib; + } + + return ret; +} + +static int populate_bcm_data(struct device *dev, struct bcm_data *bcm, + const char *of_prop) +{ + const char *bcm_name; + const struct bcm_db *data; + size_t data_len = 0; + int ret; + + ret = of_property_read_string(dev->of_node, of_prop, &bcm_name); + if (ret < 0) { + dev_err(dev, "Error reading %s: %d\n", of_prop, ret); + return ret; + } + bcm->addr = cmd_db_read_addr(bcm_name); + if (!bcm->addr) { + dev_err(dev, "Error getting addr for: %s\n", bcm_name); + return -EINVAL; + } + + data = cmd_db_read_aux_data(bcm_name, &data_len); + if (IS_ERR(data)) { + ret = PTR_ERR(data); + dev_err(dev, "Error reading %s aux data: %d\n", bcm_name, ret); + return ret; + } + if (data_len != sizeof(*data)) { + dev_err(dev, "Bad data len for %s: %d\n", bcm_name, data_len); + return -EINVAL; + } + bcm->unit = le32_to_cpu(data->unit) / 1000UL; + bcm->width = le16_to_cpu(data->width); + bcm->vcd = data->vcd; + dev_dbg(dev, "Got BCM %s: addr=%lu, unit=%lu, width=%lu, vcd=%lu\n", + bcm_name, bcm->addr, bcm->unit, bcm->width, bcm->vcd); + + return 0; +} + +int setup_ddrllcc_fp_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path) +{ + int ret = 0; + + if (hw->type != DCVS_DDR && hw->type != DCVS_LLCC) + return -EINVAL; + + mutex_lock(&ddrllcc_lock); + if (!ddrllcc_data) { + dev_dbg(dev, "Probe deferred since FP not init yet\n"); + mutex_unlock(&ddrllcc_lock); + return -EPROBE_DEFER; + } + path->data = ddrllcc_data; + path->commit_dcvs_freqs = ddrllcc_fp_commit; + if (hw->type == DCVS_DDR) + ddrllcc_data->paths[DDR_IDX] = path; + else + ddrllcc_data->paths[LLCC_IDX] = path; + mutex_unlock(&ddrllcc_lock); + + return ret; +} +EXPORT_SYMBOL(setup_ddrllcc_fp_device); + +#define DDR_BCM_PROP "qcom,ddr-bcm-name" +#define LLCC_BCM_PROP "qcom,llcc-bcm-name" +static int qcom_dcvs_fp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int i, ret = 0; + struct ddrllcc_fp_data *fp_data; + + mutex_lock(&ddrllcc_lock); + if (ddrllcc_data) { + dev_err(dev, "Only one fast path client allowed\n"); + ret = -EINVAL; + goto out; + } + fp_data = devm_kzalloc(dev, sizeof(*fp_data), GFP_KERNEL); + if (!fp_data) { + ret = -ENOMEM; + goto out; + } + + fp_data->dev = dev; + ret = populate_bcm_data(dev, &fp_data->bcms[DDR_IDX], DDR_BCM_PROP); + if (ret < 0) { + dev_err(dev, "Error importing %s bcm data: %d\n", + DDR_BCM_PROP, ret); + goto out; + } + ret = populate_bcm_data(dev, &fp_data->bcms[LLCC_IDX], LLCC_BCM_PROP); + if (ret < 0) { + dev_err(dev, "Error importing %s bcm data: %d\n", + LLCC_BCM_PROP, ret); + goto out; + } + + for (i = 0; i < NUM_FP_CMDS; i++) { + fp_data->tcs_cmds[i].addr = fp_data->bcms[i].addr; + fp_data->tcs_cmds[i].data = BCM_TCS_CMD(1, 1, 0, 0); + fp_data->tcs_cmds[i].wait = 0; + } + + ret = rpmh_init_fast_path(dev, fp_data->tcs_cmds, NUM_FP_CMDS); + if (ret < 0) { + dev_err(dev, "Error initializing rpmh fast path: %d\n", ret); + goto out; + } + ret = rpmh_write_async(dev, RPMH_SLEEP_STATE, fp_data->tcs_cmds, + NUM_FP_CMDS); + if (ret < 0) { + dev_err(dev, "Error initing dcvs_fp sleep vote: %d\n", ret); + goto out; + } + for (i = 0; i < NUM_FP_CMDS; i++) + fp_data->tcs_cmds[i].data = BCM_TCS_CMD(1, 1, 0, 1); + ret = rpmh_write_async(dev, RPMH_WAKE_ONLY_STATE, fp_data->tcs_cmds, + NUM_FP_CMDS); + if (ret < 0) { + dev_err(dev, "Error initing dcvs_fp wake vote: %d\n", ret); + goto out; + } + ddrllcc_data = fp_data; +out: + mutex_unlock(&ddrllcc_lock); + + return ret; +} + +static const struct of_device_id qcom_dcvs_fp_match_table[] = { + { .compatible = "qcom,dcvs-fp" }, + {} +}; + +static struct platform_driver qcom_dcvs_fp_driver = { + .probe = qcom_dcvs_fp_probe, + .driver = { + .name = "qcom-dcvs-fp", + .of_match_table = qcom_dcvs_fp_match_table, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(qcom_dcvs_fp_driver); + +MODULE_DESCRIPTION("QCOM DCVS FP Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/dcvs/dcvs_icc.c b/drivers/soc/qcom/dcvs/dcvs_icc.c new file mode 100644 index 000000000000..df97dd182cb2 --- /dev/null +++ b/drivers/soc/qcom/dcvs/dcvs_icc.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "qcom-dcvs-icc: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dcvs_private.h" + +struct icc_sp_data { + struct icc_path *icc_path; +}; + +static int commit_icc_freq(struct dcvs_path *path, struct dcvs_freq *freqs, + u32 update_mask) +{ + int ret; + struct dcvs_hw *hw = path->hw; + struct icc_sp_data *sp_data = path->data; + struct icc_path *icc_path = sp_data->icc_path; + u32 icc_ib = freqs->ib * hw->width; + u32 icc_ab = freqs->ab * hw->width; + + ret = icc_set_bw(icc_path, icc_ab, icc_ib); + if (ret < 0) { + dev_err(path->dev, "icc set bw request failed: %d\n", ret); + return ret; + } + path->cur_freq.ib = freqs->ib; + path->cur_freq.ab = freqs->ab; + + return ret; +} + +#define ACTIVE_ONLY_TAG 0x3 +#define PERF_MODE_TAG 0x8 +int setup_icc_sp_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path) +{ + struct icc_sp_data *sp_data = NULL; + int ret = 0; + + if (hw->type != DCVS_DDR && hw->type != DCVS_LLCC + && hw->type != DCVS_DDRQOS) + return -EINVAL; + + sp_data = devm_kzalloc(dev, sizeof(*sp_data), GFP_KERNEL); + if (!sp_data) + return -ENOMEM; + sp_data->icc_path = of_icc_get(dev, NULL); + if (IS_ERR(sp_data->icc_path)) { + ret = PTR_ERR(sp_data->icc_path); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Unable to register icc path: %d\n", ret); + return ret; + } + if (hw->type == DCVS_DDR || hw->type == DCVS_LLCC) + icc_set_tag(sp_data->icc_path, ACTIVE_ONLY_TAG); + else if (hw->type == DCVS_DDRQOS) + icc_set_tag(sp_data->icc_path, ACTIVE_ONLY_TAG | PERF_MODE_TAG); + path->data = sp_data; + path->commit_dcvs_freqs = commit_icc_freq; + + return ret; +} diff --git a/drivers/soc/qcom/dcvs/dcvs_private.h b/drivers/soc/qcom/dcvs/dcvs_private.h new file mode 100644 index 000000000000..215aa74e7591 --- /dev/null +++ b/drivers/soc/qcom/dcvs/dcvs_private.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _QCOM_DCVS_PRIVATE_H +#define _QCOM_DCVS_PRIVATE_H + +#include +#include + +/* + * NOTE: cur_freq is maintained by SLOW and FAST paths only + * and percpu_cur_freqs is maintained by PERCPU paths only + */ +struct dcvs_path { + struct device *dev; + enum dcvs_path_type type; + struct dcvs_hw *hw; + struct dcvs_freq cur_freq; + u32 *percpu_cur_freqs; + struct list_head voter_list; + u32 num_voters; + struct mutex voter_lock; + void *data; /* node-specific private data */ + int (*commit_dcvs_freqs)(struct dcvs_path *path, + struct dcvs_freq *freqs, + u32 update_mask); +}; + +struct dcvs_hw { + struct device *dev; + struct kobject kobj; + enum dcvs_hw_type type; + u32 *freq_table; + u32 table_len; + u32 width; + u32 hw_min_freq; + u32 hw_max_freq; + u32 boost_freq; + struct dcvs_path *dcvs_paths[NUM_DCVS_PATHS]; + u32 num_paths; + u32 num_inited_paths; +}; + + +int populate_l3_table(struct device *dev, u32 **freq_table); +int setup_epss_l3_sp_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path); +int setup_epss_l3_percpu_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path); +#if IS_ENABLED(CONFIG_QCOM_DCVS_FP) +int setup_ddrllcc_fp_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path); +#else +static inline int setup_ddrllcc_fp_device(struct device *dev, + struct dcvs_hw *hw, struct dcvs_path *path) +{ + return -ENODEV; +} +#endif +int setup_icc_sp_device(struct device *dev, struct dcvs_hw *hw, + struct dcvs_path *path); +#endif /* _QCOM_DCVS_PRIVATE_H */ diff --git a/drivers/soc/qcom/dcvs/memlat.c b/drivers/soc/qcom/dcvs/memlat.c new file mode 100644 index 000000000000..9b4b60d1d99b --- /dev/null +++ b/drivers/soc/qcom/dcvs/memlat.c @@ -0,0 +1,1901 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "qcom-memlat: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "trace-dcvs.h" + +#define MAX_MEMLAT_GRPS NUM_DCVS_HW_TYPES +#define FP_NAME "memlat_fp" + +enum common_ev_idx { + INST_IDX, + CYC_IDX, + FE_STALL_IDX, + BE_STALL_IDX, + NUM_COMMON_EVS +}; + +enum grp_ev_idx { + MISS_IDX, + WB_IDX, + ACC_IDX, + NUM_GRP_EVS +}; + +enum mon_type { + SAMPLING_MON = BIT(0), + THREADLAT_MON = BIT(1), + CPUCP_MON = BIT(2), + NUM_MON_TYPES +}; + +#define SAMPLING_VOTER (num_possible_cpus()) +#define NUM_FP_VOTERS (SAMPLING_VOTER + 1) + +enum memlat_type { + MEMLAT_DEV, + MEMLAT_GRP, + MEMLAT_MON, + NUM_MEMLAT_TYPES +}; + +struct memlat_spec { + enum memlat_type type; +}; + +struct cpu_ctrs { + u64 common_ctrs[NUM_COMMON_EVS]; + u64 grp_ctrs[MAX_MEMLAT_GRPS][NUM_GRP_EVS]; +}; + +struct cpu_stats { + struct cpu_ctrs prev; + struct cpu_ctrs curr; + struct cpu_ctrs delta; + struct qcom_pmu_data raw_ctrs; + bool idle_sample; + ktime_t sample_ts; + ktime_t last_sample_ts; + spinlock_t ctrs_lock; + u32 freq_mhz; + u32 fe_stall_pct; + u32 be_stall_pct; + u32 ipm[MAX_MEMLAT_GRPS]; + u32 wb_pct[MAX_MEMLAT_GRPS]; +}; + +struct cpufreq_memfreq_map { + unsigned int cpufreq_mhz; + unsigned int memfreq_khz; +}; + +/* cur_freq is maintained by sampling algorithm only */ +struct memlat_mon { + struct device *dev; + struct memlat_group *memlat_grp; + enum mon_type type; + u32 cpus_mpidr; + cpumask_t cpus; + struct cpufreq_memfreq_map *freq_map; + u32 freq_map_len; + u32 ipm_ceil; + u32 fe_stall_floor; + u32 be_stall_floor; + u32 freq_scale_pct; + u32 freq_scale_limit_mhz; + u32 wb_pct_thres; + u32 wb_filter_ipm; + u32 min_freq; + u32 max_freq; + u32 mon_min_freq; + u32 mon_max_freq; + u32 cur_freq; + struct kobject kobj; + bool is_compute; + u32 index; +}; + +struct memlat_group { + struct device *dev; + struct kobject *dcvs_kobj; + enum dcvs_hw_type hw_type; + enum dcvs_path_type sampling_path_type; + enum dcvs_path_type threadlat_path_type; + bool cpucp_enabled; + u32 sampling_cur_freq; + u32 adaptive_cur_freq; + u32 adaptive_high_freq; + u32 adaptive_low_freq; + bool fp_voting_enabled; + u32 fp_freq; + u32 *fp_votes; + u32 grp_ev_ids[NUM_GRP_EVS]; + struct memlat_mon *mons; + u32 num_mons; + u32 num_inited_mons; + struct mutex mons_lock; + struct kobject kobj; +}; + +struct memlat_dev_data { + struct device *dev; + struct kobject kobj; + u32 common_ev_ids[NUM_COMMON_EVS]; + struct work_struct work; + struct workqueue_struct *memlat_wq; + u32 sample_ms; + struct hrtimer timer; + ktime_t last_update_ts; + ktime_t last_jiffy_ts; + struct memlat_group *groups[MAX_MEMLAT_GRPS]; + int num_grps; + int num_inited_grps; + spinlock_t fp_agg_lock; + spinlock_t fp_commit_lock; + bool fp_enabled; + bool sampling_enabled; + bool inited; +/* CPUCP related struct fields */ + const struct scmi_memlat_vendor_ops *memlat_ops; + struct scmi_protocol_handle *ph; + u32 cpucp_sample_ms; + u32 cpucp_log_level; +}; + +static struct memlat_dev_data *memlat_data; +static DEFINE_PER_CPU(struct cpu_stats *, sampling_stats); +static DEFINE_MUTEX(memlat_lock); + +struct qcom_memlat_attr { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct attribute *attr, + char *buf); + ssize_t (*store)(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count); +}; + +#define to_memlat_attr(_attr) \ + container_of(_attr, struct qcom_memlat_attr, attr) +#define to_memlat_mon(k) container_of(k, struct memlat_mon, kobj) +#define to_memlat_grp(k) container_of(k, struct memlat_group, kobj) + +#define MEMLAT_ATTR_RW(_name) \ +static struct qcom_memlat_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) \ + +#define MEMLAT_ATTR_RO(_name) \ +static struct qcom_memlat_attr _name = \ +__ATTR(_name, 0444, show_##_name, NULL) \ + +#define show_attr(name) \ +static ssize_t show_##name(struct kobject *kobj, \ + struct attribute *attr, char *buf) \ +{ \ + struct memlat_mon *mon = to_memlat_mon(kobj); \ + return scnprintf(buf, PAGE_SIZE, "%u\n", mon->name); \ +} \ + +#define store_attr(name, _min, _max) \ +static ssize_t store_##name(struct kobject *kobj, \ + struct attribute *attr, const char *buf, \ + size_t count) \ +{ \ + int ret; \ + unsigned int val; \ + struct memlat_mon *mon = to_memlat_mon(kobj); \ + struct memlat_group *grp = mon->memlat_grp; \ + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; \ + ret = kstrtouint(buf, 10, &val); \ + if (ret < 0) \ + return ret; \ + val = max(val, _min); \ + val = min(val, _max); \ + mon->name = val; \ + if (mon->type == CPUCP_MON && ops) { \ + ret = ops->name(memlat_data->ph, grp->hw_type, \ + mon->index, mon->name); \ + if (ret < 0) { \ + pr_err("failed to set mon tunable :%d\n", ret); \ + return ret; \ + } \ + } \ + return count; \ +} \ + +#define show_grp_attr(name) \ +static ssize_t show_##name(struct kobject *kobj, \ + struct attribute *attr, char *buf) \ +{ \ + struct memlat_group *grp = to_memlat_grp(kobj); \ + return scnprintf(buf, PAGE_SIZE, "%u\n", grp->name); \ +} \ + +#define store_grp_attr(name, _min, _max) \ +static ssize_t store_##name(struct kobject *kobj, \ + struct attribute *attr, const char *buf, \ + size_t count) \ +{ \ + int ret; \ + unsigned int val; \ + struct memlat_group *grp = to_memlat_grp(kobj); \ + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; \ + ret = kstrtouint(buf, 10, &val); \ + if (ret < 0) \ + return ret; \ + val = max(val, _min); \ + val = min(val, _max); \ + grp->name = val; \ + if (grp->cpucp_enabled && ops) { \ + ret = ops->name(memlat_data->ph, grp->hw_type, \ + 0, grp->name); \ + if (ret < 0) { \ + pr_err("failed to set grp tunable :%d\n", ret); \ + return ret; \ + } \ + } \ + return count; \ +} \ + +static ssize_t store_min_freq(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret; + unsigned int freq; + struct memlat_mon *mon = to_memlat_mon(kobj); + struct memlat_group *grp = mon->memlat_grp; + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + + ret = kstrtouint(buf, 10, &freq); + if (ret < 0) + return ret; + freq = max(freq, mon->mon_min_freq); + freq = min(freq, mon->max_freq); + mon->min_freq = freq; + + if (mon->type == CPUCP_MON && ops) { + ret = ops->min_freq(memlat_data->ph, grp->hw_type, + mon->index, mon->min_freq); + if (ret < 0) { + pr_err("failed to set min_freq :%d\n", ret); + return ret; + } + } + + return count; +} + +static ssize_t store_max_freq(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret; + unsigned int freq; + struct memlat_mon *mon = to_memlat_mon(kobj); + struct memlat_group *grp = mon->memlat_grp; + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + + ret = kstrtouint(buf, 10, &freq); + if (ret < 0) + return ret; + freq = max(freq, mon->min_freq); + freq = min(freq, mon->mon_max_freq); + mon->max_freq = freq; + + if (mon->type == CPUCP_MON && ops) { + ret = ops->max_freq(memlat_data->ph, grp->hw_type, + mon->index, mon->max_freq); + if (ret < 0) { + pr_err("failed to set max_freq :%d\n", ret); + return ret; + } + } + + return count; +} + +static ssize_t show_freq_map(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct memlat_mon *mon = to_memlat_mon(kobj); + struct cpufreq_memfreq_map *map = mon->freq_map; + unsigned int cnt = 0; + + cnt += scnprintf(buf, PAGE_SIZE, "CPU freq (MHz)\tMem freq (kHz)\n"); + + while (map->cpufreq_mhz && cnt < PAGE_SIZE) { + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "%14u\t%14u\n", + map->cpufreq_mhz, map->memfreq_khz); + map++; + } + if (cnt < PAGE_SIZE) + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); + + return cnt; +} + +#define MIN_SAMPLE_MS 4U +#define MAX_SAMPLE_MS 1000U +static ssize_t store_sample_ms(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret; + unsigned int val; + + ret = kstrtouint(buf, 10, &val); + if (ret < 0) + return ret; + val = max(val, MIN_SAMPLE_MS); + val = min(val, MAX_SAMPLE_MS); + + memlat_data->sample_ms = val; + + return count; +} + +static ssize_t show_sample_ms(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", memlat_data->sample_ms); +} + +static ssize_t store_cpucp_sample_ms(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret, i; + unsigned int val; + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + struct memlat_group *grp; + + if (!ops) + return -ENODEV; + + for (i = 0; i < MAX_MEMLAT_GRPS; i++) { + grp = memlat_data->groups[i]; + if (grp->cpucp_enabled) + break; + } + if (i == MAX_MEMLAT_GRPS) + return count; + + ret = kstrtouint(buf, 10, &val); + if (ret < 0) + return ret; + val = max(val, MIN_SAMPLE_MS); + val = min(val, MAX_SAMPLE_MS); + + ret = ops->sample_ms(memlat_data->ph, val); + if (ret < 0) { + pr_err("Failed to set cpucp sample ms :%d\n", ret); + return ret; + } + + memlat_data->cpucp_sample_ms = val; + return count; +} + +static ssize_t show_cpucp_sample_ms(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%lu\n", memlat_data->cpucp_sample_ms); +} + +static ssize_t store_cpucp_log_level(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret, i; + unsigned int val; + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + struct memlat_group *grp; + + if (!ops) + return -ENODEV; + + for (i = 0; i < MAX_MEMLAT_GRPS; i++) { + grp = memlat_data->groups[i]; + if (grp->cpucp_enabled) + break; + } + if (i == MAX_MEMLAT_GRPS) + return count; + + ret = kstrtouint(buf, 10, &val); + if (ret < 0) + return ret; + + ret = ops->set_log_level(memlat_data->ph, val); + if (ret < 0) { + pr_err("failed to configure log_level, ret = %d\n", ret); + return ret; + } + + memlat_data->cpucp_log_level = val; + return count; +} + +static ssize_t show_cpucp_log_level(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%lu\n", memlat_data->cpucp_log_level); +} + +static ssize_t store_flush_cpucp_log(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + int ret; + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + + if (!ops) + return -ENODEV; + + ret = ops->flush_cpucp_log(memlat_data->ph); + if (ret < 0) { + pr_err("failed to flush cpucp log, ret = %d\n", ret); + return ret; + } + + return count; +} + +static ssize_t show_flush_cpucp_log(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "Echo here to flush cpucp logs\n"); +} + +static ssize_t show_hlos_cpucp_offset(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + int ret; + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + uint64_t cpucp_ts, hlos_ts; + + if (!ops) + return -ENODEV; + + ret = ops->get_timestamp(memlat_data->ph, &cpucp_ts); + if (ret < 0) { + pr_err("failed to get cpucp timestamp\n"); + return ret; + } + + hlos_ts = ktime_get()/1000; + + return scnprintf(buf, PAGE_SIZE, "%ld\n", le64_to_cpu(cpucp_ts) - hlos_ts); +} + +static ssize_t show_cur_freq(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + struct memlat_mon *mon = to_memlat_mon(kobj); + struct memlat_group *grp = mon->memlat_grp; + uint32_t cur_freq; + int ret; + + if (mon->type != CPUCP_MON) + return scnprintf(buf, PAGE_SIZE, "%lu\n", mon->cur_freq); + + if (!ops) + return -ENODEV; + + ret = ops->get_cur_freq(memlat_data->ph, grp->hw_type, mon->index, &cur_freq); + if (ret < 0) { + pr_err("failed to get mon current frequency\n"); + return ret; + } + return scnprintf(buf, PAGE_SIZE, "%lu\n", le32_to_cpu(cur_freq)); +} + +static ssize_t show_adaptive_cur_freq(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + uint32_t adaptive_cur_freq; + struct memlat_group *grp = to_memlat_grp(kobj); + int ret; + + if (!grp->cpucp_enabled) + return scnprintf(buf, PAGE_SIZE, "%lu\n", grp->adaptive_cur_freq); + + if (!ops) + return -ENODEV; + + ret = ops->get_adaptive_cur_freq(memlat_data->ph, grp->hw_type, 0, &adaptive_cur_freq); + if (ret < 0) { + pr_err("failed to get grp adaptive current frequency\n"); + return ret; + } + return scnprintf(buf, PAGE_SIZE, "%lu\n", le32_to_cpu(adaptive_cur_freq)); +} + +show_grp_attr(sampling_cur_freq); +show_grp_attr(adaptive_high_freq); +store_grp_attr(adaptive_high_freq, 0U, 8000000U); +show_grp_attr(adaptive_low_freq); +store_grp_attr(adaptive_low_freq, 0U, 8000000U); + +show_attr(min_freq); +show_attr(max_freq); +show_attr(ipm_ceil); +store_attr(ipm_ceil, 1U, 50000U); +show_attr(fe_stall_floor); +store_attr(fe_stall_floor, 0U, 100U); +show_attr(be_stall_floor); +store_attr(be_stall_floor, 0U, 100U); +show_attr(freq_scale_pct); +store_attr(freq_scale_pct, 0U, 1000U); +show_attr(wb_pct_thres); +store_attr(wb_pct_thres, 0U, 100U); +show_attr(wb_filter_ipm); +store_attr(wb_filter_ipm, 0U, 50000U); +store_attr(freq_scale_limit_mhz, 0U, 5000U); +show_attr(freq_scale_limit_mhz); + +MEMLAT_ATTR_RW(sample_ms); +MEMLAT_ATTR_RW(cpucp_sample_ms); +MEMLAT_ATTR_RW(cpucp_log_level); +MEMLAT_ATTR_RW(flush_cpucp_log); +MEMLAT_ATTR_RO(hlos_cpucp_offset); + +MEMLAT_ATTR_RO(sampling_cur_freq); +MEMLAT_ATTR_RO(adaptive_cur_freq); +MEMLAT_ATTR_RW(adaptive_low_freq); +MEMLAT_ATTR_RW(adaptive_high_freq); + +MEMLAT_ATTR_RW(min_freq); +MEMLAT_ATTR_RW(max_freq); +MEMLAT_ATTR_RO(freq_map); +MEMLAT_ATTR_RO(cur_freq); +MEMLAT_ATTR_RW(ipm_ceil); +MEMLAT_ATTR_RW(fe_stall_floor); +MEMLAT_ATTR_RW(be_stall_floor); +MEMLAT_ATTR_RW(freq_scale_pct); +MEMLAT_ATTR_RW(wb_pct_thres); +MEMLAT_ATTR_RW(wb_filter_ipm); +MEMLAT_ATTR_RW(freq_scale_limit_mhz); + +static struct attribute *memlat_settings_attrs[] = { + &sample_ms.attr, + &cpucp_sample_ms.attr, + &cpucp_log_level.attr, + &flush_cpucp_log.attr, + &hlos_cpucp_offset.attr, + NULL, +}; +ATTRIBUTE_GROUPS(memlat_settings); + +static struct attribute *memlat_grp_attrs[] = { + &sampling_cur_freq.attr, + &adaptive_cur_freq.attr, + &adaptive_high_freq.attr, + &adaptive_low_freq.attr, + NULL, +}; +ATTRIBUTE_GROUPS(memlat_grp); + +static struct attribute *memlat_mon_attrs[] = { + &min_freq.attr, + &max_freq.attr, + &freq_map.attr, + &cur_freq.attr, + &ipm_ceil.attr, + &fe_stall_floor.attr, + &be_stall_floor.attr, + &freq_scale_pct.attr, + &wb_pct_thres.attr, + &wb_filter_ipm.attr, + &freq_scale_limit_mhz.attr, + NULL, +}; +ATTRIBUTE_GROUPS(memlat_mon); + +static struct attribute *compute_mon_attrs[] = { + &min_freq.attr, + &max_freq.attr, + &freq_map.attr, + &cur_freq.attr, + NULL, +}; +ATTRIBUTE_GROUPS(compute_mon); + +static ssize_t attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct qcom_memlat_attr *memlat_attr = to_memlat_attr(attr); + ssize_t ret = -EIO; + + if (memlat_attr->show) + ret = memlat_attr->show(kobj, attr, buf); + + return ret; +} + +static ssize_t attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct qcom_memlat_attr *memlat_attr = to_memlat_attr(attr); + ssize_t ret = -EIO; + + if (memlat_attr->store) + ret = memlat_attr->store(kobj, attr, buf, count); + + return ret; +} + +static const struct sysfs_ops memlat_sysfs_ops = { + .show = attr_show, + .store = attr_store, +}; + +static struct kobj_type memlat_settings_ktype = { + .sysfs_ops = &memlat_sysfs_ops, + .default_groups = memlat_settings_groups, + +}; + +static struct kobj_type memlat_mon_ktype = { + .sysfs_ops = &memlat_sysfs_ops, + .default_groups = memlat_mon_groups, + +}; + +static struct kobj_type compute_mon_ktype = { + .sysfs_ops = &memlat_sysfs_ops, + .default_groups = compute_mon_groups, + +}; + +static struct kobj_type memlat_grp_ktype = { + .sysfs_ops = &memlat_sysfs_ops, + .default_groups = memlat_grp_groups, + +}; + +static u32 cpufreq_to_memfreq(struct memlat_mon *mon, u32 cpu_mhz) +{ + struct cpufreq_memfreq_map *map = mon->freq_map; + u32 mem_khz = 0; + + if (!map) + goto out; + + while (map->cpufreq_mhz && map->cpufreq_mhz < cpu_mhz) + map++; + if (!map->cpufreq_mhz) + map--; + mem_khz = map->memfreq_khz; + +out: + return mem_khz; +} + +static void calculate_sampling_stats(void) +{ + int i, grp, cpu, level = 0; + unsigned long flags; + struct cpu_stats *stats; + struct cpu_ctrs *delta; + struct memlat_group *memlat_grp; + ktime_t now = ktime_get(); + s64 delta_us, update_us; + + update_us = ktime_us_delta(now, memlat_data->last_update_ts); + memlat_data->last_update_ts = now; + + local_irq_save(flags); + for_each_possible_cpu(cpu) { + stats = per_cpu(sampling_stats, cpu); + if (level == 0) + spin_lock(&stats->ctrs_lock); + else + spin_lock_nested(&stats->ctrs_lock, level); + level++; + } + + for_each_possible_cpu(cpu) { + stats = per_cpu(sampling_stats, cpu); + delta = &stats->delta; + /* use update_us and now to synchronize idle cpus */ + if (stats->idle_sample) { + delta_us = update_us; + stats->last_sample_ts = now; + } else { + delta_us = ktime_us_delta(stats->sample_ts, + stats->last_sample_ts); + stats->last_sample_ts = stats->sample_ts; + } + for (i = 0; i < NUM_COMMON_EVS; i++) { + if (!memlat_data->common_ev_ids[i]) + continue; + delta->common_ctrs[i] = stats->curr.common_ctrs[i] - + stats->prev.common_ctrs[i]; + } + + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + memlat_grp = memlat_data->groups[grp]; + if (!memlat_grp) + continue; + for (i = 0; i < NUM_GRP_EVS; i++) { + if (!memlat_grp->grp_ev_ids[i]) + continue; + delta->grp_ctrs[grp][i] = + stats->curr.grp_ctrs[grp][i] - + stats->prev.grp_ctrs[grp][i]; + } + } + + stats->freq_mhz = delta->common_ctrs[CYC_IDX] / delta_us; + if (!memlat_data->common_ev_ids[FE_STALL_IDX]) + stats->fe_stall_pct = 100; + else + stats->fe_stall_pct = mult_frac(100, + delta->common_ctrs[FE_STALL_IDX], + delta->common_ctrs[CYC_IDX]); + if (!memlat_data->common_ev_ids[BE_STALL_IDX]) + stats->be_stall_pct = 100; + else + stats->be_stall_pct = mult_frac(100, + delta->common_ctrs[BE_STALL_IDX], + delta->common_ctrs[CYC_IDX]); + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + memlat_grp = memlat_data->groups[grp]; + if (!memlat_grp) { + stats->ipm[grp] = 0; + stats->wb_pct[grp] = 0; + continue; + } + stats->ipm[grp] = delta->common_ctrs[INST_IDX]; + if (delta->grp_ctrs[grp][MISS_IDX]) + stats->ipm[grp] /= + delta->grp_ctrs[grp][MISS_IDX]; + + if (!memlat_grp->grp_ev_ids[WB_IDX] + || !memlat_grp->grp_ev_ids[ACC_IDX]) + stats->wb_pct[grp] = 0; + else + stats->wb_pct[grp] = mult_frac(100, + delta->grp_ctrs[grp][WB_IDX], + delta->grp_ctrs[grp][ACC_IDX]); + + /* one meas event per memlat_group with group name */ + trace_memlat_dev_meas(dev_name(memlat_grp->dev), cpu, + delta->common_ctrs[INST_IDX], + delta->grp_ctrs[grp][MISS_IDX], + stats->freq_mhz, stats->be_stall_pct, + stats->wb_pct[grp], stats->ipm[grp], + stats->fe_stall_pct); + + } + memcpy(&stats->prev, &stats->curr, sizeof(stats->curr)); + } + + for_each_possible_cpu(cpu) { + stats = per_cpu(sampling_stats, cpu); + spin_unlock(&stats->ctrs_lock); + } + local_irq_restore(flags); +} + +static inline void set_higher_freq(int *max_cpu, int cpu, u32 *max_cpufreq, + u32 cpufreq) +{ + if (cpufreq > *max_cpufreq) { + *max_cpu = cpu; + *max_cpufreq = cpufreq; + } +} + +static inline void apply_adaptive_freq(struct memlat_group *memlat_grp, + u32 *max_freq) +{ + u32 prev_freq = memlat_grp->adaptive_cur_freq; + + if (*max_freq < memlat_grp->adaptive_low_freq) { + *max_freq = memlat_grp->adaptive_low_freq; + memlat_grp->adaptive_cur_freq = memlat_grp->adaptive_low_freq; + } else if (*max_freq < memlat_grp->adaptive_high_freq) { + *max_freq = memlat_grp->adaptive_high_freq; + memlat_grp->adaptive_cur_freq = memlat_grp->adaptive_high_freq; + } else + memlat_grp->adaptive_cur_freq = memlat_grp->adaptive_high_freq; + + if (prev_freq != memlat_grp->adaptive_cur_freq) + trace_memlat_dev_update(dev_name(memlat_grp->dev), + 0, 0, 0, 0, memlat_grp->adaptive_cur_freq); +} + +static void calculate_mon_sampling_freq(struct memlat_mon *mon) +{ + struct cpu_stats *stats; + int cpu, max_cpu = cpumask_first(&mon->cpus); + u32 max_memfreq, max_cpufreq = 0; + u32 max_cpufreq_scaled = 0, ipm_diff; + u32 hw = mon->memlat_grp->hw_type; + + if (hw >= NUM_DCVS_HW_TYPES) + return; + + for_each_cpu(cpu, &mon->cpus) { + stats = per_cpu(sampling_stats, cpu); + if (mon->is_compute || (stats->wb_pct[hw] >= mon->wb_pct_thres + && stats->ipm[hw] <= mon->wb_filter_ipm)) + set_higher_freq(&max_cpu, cpu, &max_cpufreq, + stats->freq_mhz); + else if (stats->ipm[hw] <= mon->ipm_ceil) { + ipm_diff = mon->ipm_ceil - stats->ipm[hw]; + max_cpufreq_scaled = stats->freq_mhz; + if (mon->freq_scale_pct && stats->freq_mhz && + (stats->freq_mhz < mon->freq_scale_limit_mhz) && + (stats->fe_stall_pct >= mon->fe_stall_floor || + stats->be_stall_pct >= mon->be_stall_floor)) { + max_cpufreq_scaled += (stats->freq_mhz * ipm_diff * + mon->freq_scale_pct) / (mon->ipm_ceil * 100); + max_cpufreq_scaled = min(mon->freq_scale_limit_mhz, + max_cpufreq_scaled); + } + set_higher_freq(&max_cpu, cpu, &max_cpufreq, + max_cpufreq_scaled); + } + } + + max_memfreq = cpufreq_to_memfreq(mon, max_cpufreq); + max_memfreq = max(max_memfreq, mon->min_freq); + max_memfreq = min(max_memfreq, mon->max_freq); + + if (max_cpufreq || mon->cur_freq != mon->min_freq) { + stats = per_cpu(sampling_stats, max_cpu); + trace_memlat_dev_update(dev_name(mon->dev), max_cpu, + stats->delta.common_ctrs[INST_IDX], + stats->delta.grp_ctrs[hw][MISS_IDX], + max_cpufreq, max_memfreq); + } + + mon->cur_freq = max_memfreq; +} + +/* + * updates fast path votes for "cpu" (i.e. fp_votes index) + * fp_freqs array length MAX_MEMLAT_GRPS (i.e. one freq per grp) + */ +static void update_memlat_fp_vote(int cpu, u32 *fp_freqs) +{ + struct dcvs_freq voted_freqs[MAX_MEMLAT_GRPS]; + u32 max_freqs[MAX_MEMLAT_GRPS] = { 0 }; + int grp, i, ret; + unsigned long flags; + struct memlat_group *memlat_grp; + u32 commit_mask = 0; + + if (!memlat_data->fp_enabled || cpu > NUM_FP_VOTERS) + return; + + local_irq_save(flags); + spin_lock(&memlat_data->fp_agg_lock); + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + memlat_grp = memlat_data->groups[grp]; + if (!memlat_grp || !memlat_grp->fp_voting_enabled) + continue; + memlat_grp->fp_votes[cpu] = fp_freqs[grp]; + + /* aggregate across all "cpus" */ + for (i = 0; i < NUM_FP_VOTERS; i++) + max_freqs[grp] = max(memlat_grp->fp_votes[i], + max_freqs[grp]); + if (max_freqs[grp] != memlat_grp->fp_freq) + commit_mask |= BIT(grp); + } + + if (!commit_mask) { + spin_unlock(&memlat_data->fp_agg_lock); + local_irq_restore(flags); + return; + } + + spin_lock_nested(&memlat_data->fp_commit_lock, SINGLE_DEPTH_NESTING); + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + if (!(commit_mask & BIT(grp))) + continue; + memlat_grp = memlat_data->groups[grp]; + memlat_grp->fp_freq = max_freqs[grp]; + } + spin_unlock(&memlat_data->fp_agg_lock); + + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + if (!(commit_mask & BIT(grp))) + continue; + voted_freqs[grp].ib = max_freqs[grp]; + voted_freqs[grp].hw_type = grp; + } + ret = qcom_dcvs_update_votes(FP_NAME, voted_freqs, commit_mask, + DCVS_FAST_PATH); + if (ret < 0) + pr_err("error updating qcom dcvs fp: %d\n", ret); + spin_unlock(&memlat_data->fp_commit_lock); + local_irq_restore(flags); +} + +/* sampling path update work */ +static void memlat_update_work(struct work_struct *work) +{ + int i, grp, ret; + struct memlat_group *memlat_grp; + struct memlat_mon *mon; + struct dcvs_freq new_freq; + u32 max_freqs[MAX_MEMLAT_GRPS] = { 0 }; + + calculate_sampling_stats(); + + /* aggregate mons to calculate max freq per memlat_group */ + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + memlat_grp = memlat_data->groups[grp]; + if (!memlat_grp) + continue; + for (i = 0; i < memlat_grp->num_inited_mons; i++) { + mon = &memlat_grp->mons[i]; + if (!mon || !(mon->type & SAMPLING_MON)) + continue; + calculate_mon_sampling_freq(mon); + max_freqs[grp] = max(mon->cur_freq, max_freqs[grp]); + } + if (memlat_grp->adaptive_high_freq || + memlat_grp->adaptive_low_freq || + memlat_grp->adaptive_cur_freq) + apply_adaptive_freq(memlat_grp, &max_freqs[grp]); + } + + update_memlat_fp_vote(SAMPLING_VOTER, max_freqs); + + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + memlat_grp = memlat_data->groups[grp]; + if (!memlat_grp || memlat_grp->sampling_cur_freq == max_freqs[grp] || + memlat_grp->sampling_path_type == NUM_DCVS_PATHS) + continue; + memlat_grp->sampling_cur_freq = max_freqs[grp]; + if (memlat_grp->fp_voting_enabled) + continue; + new_freq.ib = max_freqs[grp]; + new_freq.ab = 0; + new_freq.hw_type = grp; + ret = qcom_dcvs_update_votes(dev_name(memlat_grp->dev), + &new_freq, 1, memlat_grp->sampling_path_type); + if (ret < 0) + dev_err(memlat_grp->dev, "qcom dcvs err: %d\n", ret); + } +} + +static enum hrtimer_restart memlat_hrtimer_handler(struct hrtimer *timer) +{ + queue_work(memlat_data->memlat_wq, &memlat_data->work); + + return HRTIMER_NORESTART; +} + +static const u64 HALF_TICK_NS = (NSEC_PER_SEC / HZ) >> 1; +#define MEMLAT_UPDATE_DELAY (100 * NSEC_PER_USEC) +static void memlat_jiffies_update_cb(void *unused, void *extra) +{ + ktime_t now = ktime_get(); + s64 delta_ns = now - memlat_data->last_jiffy_ts + HALF_TICK_NS; + + if (unlikely(!memlat_data->inited)) + return; + + if (delta_ns > ms_to_ktime(memlat_data->sample_ms)) { + hrtimer_start(&memlat_data->timer, MEMLAT_UPDATE_DELAY, + HRTIMER_MODE_REL_PINNED); + memlat_data->last_jiffy_ts = now; + } +} + +/* + * Note: must hold stats->ctrs_lock and populate stats->raw_ctrs + * before calling this API. + */ +static void process_raw_ctrs(struct cpu_stats *stats) +{ + int i, grp, idx; + struct cpu_ctrs *curr_ctrs = &stats->curr; + struct qcom_pmu_data *raw_ctrs = &stats->raw_ctrs; + struct memlat_group *memlat_grp; + u32 event_id; + u64 ev_data; + + for (i = 0; i < raw_ctrs->num_evs; i++) { + event_id = raw_ctrs->event_ids[i]; + ev_data = raw_ctrs->ev_data[i]; + if (!event_id) + break; + + for (idx = 0; idx < NUM_COMMON_EVS; idx++) { + if (event_id != memlat_data->common_ev_ids[idx]) + continue; + curr_ctrs->common_ctrs[idx] = ev_data; + break; + } + if (idx < NUM_COMMON_EVS) + continue; + + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + memlat_grp = memlat_data->groups[grp]; + if (!memlat_grp) + continue; + for (idx = 0; idx < NUM_GRP_EVS; idx++) { + if (event_id != memlat_grp->grp_ev_ids[idx]) + continue; + curr_ctrs->grp_ctrs[grp][idx] = ev_data; + break; + } + } + } +} + +static void memlat_pmu_idle_cb(struct qcom_pmu_data *data, int cpu, int state) +{ + struct cpu_stats *stats = per_cpu(sampling_stats, cpu); + unsigned long flags; + + if (unlikely(!memlat_data->inited)) + return; + + spin_lock_irqsave(&stats->ctrs_lock, flags); + memcpy(&stats->raw_ctrs, data, sizeof(*data)); + process_raw_ctrs(stats); + stats->idle_sample = true; + spin_unlock_irqrestore(&stats->ctrs_lock, flags); +} + +static struct qcom_pmu_notif_node memlat_idle_notif = { + .idle_cb = memlat_pmu_idle_cb, +}; + +static void memlat_sched_tick_cb(void *unused, struct rq *rq) +{ + int ret, cpu = smp_processor_id(); + struct cpu_stats *stats = per_cpu(sampling_stats, cpu); + ktime_t now = ktime_get(); + s64 delta_ns; + unsigned long flags; + + if (unlikely(!memlat_data->inited)) + return; + + spin_lock_irqsave(&stats->ctrs_lock, flags); + delta_ns = now - stats->last_sample_ts + HALF_TICK_NS; + if (delta_ns < ms_to_ktime(memlat_data->sample_ms)) + goto out; + stats->sample_ts = now; + stats->idle_sample = false; + stats->raw_ctrs.num_evs = 0; + ret = qcom_pmu_read_all_local(&stats->raw_ctrs); + if (ret < 0 || stats->raw_ctrs.num_evs == 0) { + pr_err("error reading pmu counters on cpu%d: %d\n", cpu, ret); + goto out; + } + process_raw_ctrs(stats); + +out: + spin_unlock_irqrestore(&stats->ctrs_lock, flags); +} + +static void get_mpidr_cpu(void *cpu) +{ + u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; + + *((uint32_t *)cpu) = MPIDR_AFFINITY_LEVEL(mpidr, 1); +} + +static int get_mask_and_mpidr_from_pdev(struct platform_device *pdev, + cpumask_t *mask, u32 *cpus_mpidr) +{ + struct device *dev = &pdev->dev; + struct device_node *dev_phandle; + struct device *cpu_dev; + int cpu, i = 0; + uint32_t physical_cpu; + int ret = -ENODEV; + + dev_phandle = of_parse_phandle(dev->of_node, "qcom,cpulist", i++); + while (dev_phandle) { + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (cpu_dev && cpu_dev->of_node == dev_phandle) { + cpumask_set_cpu(cpu, mask); + smp_call_function_single(cpu, get_mpidr_cpu, + &physical_cpu, true); + *cpus_mpidr |= BIT(physical_cpu); + ret = 0; + break; + } + } + dev_phandle = of_parse_phandle(dev->of_node, "qcom,cpulist", i++); + } + + return ret; +} + +#define COREDEV_TBL_PROP "qcom,cpufreq-memfreq-tbl" +#define NUM_COLS 2 +static struct cpufreq_memfreq_map *init_cpufreq_memfreq_map(struct device *dev, + struct device_node *of_node, + u32 *cnt) +{ + int len, nf, i, j; + u32 data; + struct cpufreq_memfreq_map *tbl; + int ret; + + if (!of_find_property(of_node, COREDEV_TBL_PROP, &len)) + return NULL; + len /= sizeof(data); + + if (len % NUM_COLS || len == 0) + return NULL; + nf = len / NUM_COLS; + + tbl = devm_kzalloc(dev, (nf + 1) * sizeof(struct cpufreq_memfreq_map), + GFP_KERNEL); + if (!tbl) + return NULL; + + for (i = 0, j = 0; i < nf; i++, j += 2) { + ret = of_property_read_u32_index(of_node, COREDEV_TBL_PROP, + j, &data); + if (ret < 0) + return NULL; + tbl[i].cpufreq_mhz = data / 1000; + + ret = of_property_read_u32_index(of_node, COREDEV_TBL_PROP, + j + 1, &data); + if (ret < 0) + return NULL; + tbl[i].memfreq_khz = data; + pr_debug("Entry%d CPU:%u, Mem:%u\n", i, tbl[i].cpufreq_mhz, + tbl[i].memfreq_khz); + } + *cnt = nf; + tbl[i].cpufreq_mhz = 0; + + return tbl; +} + +static bool memlat_grps_and_mons_inited(void) +{ + struct memlat_group *memlat_grp; + int grp; + + if (memlat_data->num_inited_grps < memlat_data->num_grps) + return false; + + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + memlat_grp = memlat_data->groups[grp]; + if (!memlat_grp) + continue; + if (memlat_grp->num_inited_mons < memlat_grp->num_mons) + return false; + } + + return true; +} + +static int memlat_sampling_init(void) +{ + int cpu; + struct device *dev = memlat_data->dev; + struct cpu_stats *stats; + + for_each_possible_cpu(cpu) { + stats = devm_kzalloc(dev, sizeof(*stats), GFP_KERNEL); + if (!stats) + return -ENOMEM; + per_cpu(sampling_stats, cpu) = stats; + spin_lock_init(&stats->ctrs_lock); + } + + memlat_data->memlat_wq = create_freezable_workqueue("memlat_wq"); + if (!memlat_data->memlat_wq) { + dev_err(dev, "Couldn't create memlat workqueue.\n"); + return -ENOMEM; + } + INIT_WORK(&memlat_data->work, &memlat_update_work); + + hrtimer_init(&memlat_data->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + memlat_data->timer.function = memlat_hrtimer_handler; + + register_trace_android_vh_scheduler_tick(memlat_sched_tick_cb, NULL); + register_trace_android_vh_jiffies_update(memlat_jiffies_update_cb, NULL); + qcom_pmu_idle_register(&memlat_idle_notif); + + return 0; +} + +static inline bool should_enable_memlat_fp(void) +{ + int grp; + struct memlat_group *memlat_grp; + + /* wait until all groups have inited before enabling */ + if (memlat_data->num_inited_grps < memlat_data->num_grps) + return false; + + for (grp = 0; grp < MAX_MEMLAT_GRPS; grp++) { + memlat_grp = memlat_data->groups[grp]; + if (memlat_grp && memlat_grp->fp_voting_enabled) + return true; + } + + return false; +} + +static int configure_cpucp_common_events(void) +{ + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + int ret = 0, i, j = 0; + u8 ev_map[NUM_COMMON_EVS]; + + memset(ev_map, 0xFF, NUM_COMMON_EVS); + for (i = 0; i < NUM_COMMON_EVS; i++, j++) { + if (!memlat_data->common_ev_ids[i]) + continue; + + ret = qcom_get_cpucp_id(memlat_data->common_ev_ids[i], 0); + if (ret >= 0 && ret < MAX_CPUCP_EVT) + ev_map[j] = ret; + } + + ret = ops->set_common_ev_map(memlat_data->ph, ev_map, NUM_COMMON_EVS); + return ret; +} + +static int configure_cpucp_grp(struct memlat_group *grp) +{ + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + int ret = 0, i, j = 0; + struct device_node *of_node = grp->dev->of_node; + u8 ev_map[NUM_GRP_EVS]; + + ret = ops->set_mem_grp(memlat_data->ph, *cpumask_bits(cpu_possible_mask), + grp->hw_type); + if (ret < 0) { + pr_err("Failed to configure mem grp %s\n", of_node->name); + return ret; + } + + memset(ev_map, 0xFF, NUM_GRP_EVS); + for (i = 0; i < NUM_GRP_EVS; i++, j++) { + if (!grp->grp_ev_ids[i]) + continue; + + ret = qcom_get_cpucp_id(grp->grp_ev_ids[i], 0); + if (ret >= 0 && ret < MAX_CPUCP_EVT) + ev_map[j] = ret; + } + + ret = ops->set_grp_ev_map(memlat_data->ph, grp->hw_type, ev_map, NUM_GRP_EVS); + if (ret < 0) { + pr_err("Failed to configure event map for mem grp %s\n", + of_node->name); + return ret; + } + + ret = ops->adaptive_low_freq(memlat_data->ph, grp->hw_type, 0, grp->adaptive_low_freq); + if (ret < 0) { + pr_err("Failed to configure grp adaptive low freq for mem grp %s\n", + of_node->name); + return ret; + } + + ret = ops->adaptive_high_freq(memlat_data->ph, grp->hw_type, 0, grp->adaptive_high_freq); + if (ret < 0) + pr_err("Failed to configure grp adaptive high freq for mem grp %s\n", + of_node->name); + return ret; +} + +static int configure_cpucp_mon(struct memlat_mon *mon) +{ + struct memlat_group *grp = mon->memlat_grp; + const struct scmi_memlat_vendor_ops *ops = memlat_data->memlat_ops; + struct device_node *of_node = mon->dev->of_node; + int ret; + const char c = ':'; + + ret = ops->set_mon(memlat_data->ph, mon->cpus_mpidr, grp->hw_type, + mon->is_compute, mon->index, (strrchr(dev_name(mon->dev), c) + 1)); + if (ret < 0) { + pr_err("failed to configure monitor %s\n", of_node->name); + return ret; + } + + ret = ops->ipm_ceil(memlat_data->ph, grp->hw_type, mon->index, + mon->ipm_ceil); + if (ret < 0) { + pr_err("failed to set ipm ceil for %s\n", of_node->name); + return ret; + } + + ret = ops->fe_stall_floor(memlat_data->ph, grp->hw_type, mon->index, + mon->fe_stall_floor); + if (ret < 0) { + pr_err("failed to set fe stall floor for %s\n", of_node->name); + return ret; + } + + ret = ops->be_stall_floor(memlat_data->ph, grp->hw_type, mon->index, + mon->be_stall_floor); + if (ret < 0) { + pr_err("failed to set be stall floor for %s\n", of_node->name); + return ret; + } + + ret = ops->wb_pct_thres(memlat_data->ph, grp->hw_type, mon->index, + mon->wb_pct_thres); + if (ret < 0) { + pr_err("failed to set wb pct for %s\n", of_node->name); + return ret; + } + + ret = ops->wb_filter_ipm(memlat_data->ph, grp->hw_type, mon->index, + mon->wb_filter_ipm); + if (ret < 0) { + pr_err("failed to set wb filter ipm for %s\n", of_node->name); + return ret; + } + + ret = ops->freq_scale_pct(memlat_data->ph, grp->hw_type, mon->index, + mon->freq_scale_pct); + if (ret < 0) { + pr_err("failed to set freq_scale_pct for %s\n", of_node->name); + return ret; + } + + ret = ops->freq_scale_limit_mhz(memlat_data->ph, grp->hw_type, mon->index, + mon->freq_scale_limit_mhz); + if (ret < 0) { + pr_err("failed to set wb filter ipm for %s\n", of_node->name); + return ret; + } + + ret = ops->freq_map(memlat_data->ph, grp->hw_type, mon->index, + mon->freq_map_len, mon->freq_map); + if (ret < 0) { + pr_err("failed to configure freq_map for %s\n", of_node->name); + return ret; + } + + ret = ops->min_freq(memlat_data->ph, grp->hw_type, mon->index, + mon->min_freq); + if (ret < 0) { + pr_err("failed to set min_freq for %s\n", of_node->name); + return ret; + } + + ret = ops->max_freq(memlat_data->ph, grp->hw_type, mon->index, + mon->max_freq); + if (ret < 0) + pr_err("failed to set max_freq for %s\n", of_node->name); + + return ret; +} + +int cpucp_memlat_init(struct scmi_device *sdev) +{ + int ret = 0, i, j; + struct scmi_protocol_handle *ph; + const struct scmi_memlat_vendor_ops *ops; + struct memlat_group *grp; + bool start_cpucp_timer = false; + + if (!memlat_data || !memlat_data->inited) + return -EPROBE_DEFER; + + if (!sdev || !sdev->handle) + return -EINVAL; + + ops = sdev->handle->devm_protocol_get(sdev, SCMI_PROTOCOL_MEMLAT, &ph); + if (!ops) + return -ENODEV; + + mutex_lock(&memlat_lock); + memlat_data->ph = ph; + memlat_data->memlat_ops = ops; + + /* Configure common events */ + ret = configure_cpucp_common_events(); + if (ret < 0) { + pr_err("Failed to configure common events: %d\n", ret); + goto memlat_unlock; + } + + /* Configure group/mon parameters */ + for (i = 0; i < MAX_MEMLAT_GRPS; i++) { + grp = memlat_data->groups[i]; + if (!grp || !grp->cpucp_enabled) + continue; + ret = configure_cpucp_grp(grp); + if (ret < 0) { + pr_err("Failed to configure mem group: %d\n", ret); + ops = NULL; + goto memlat_unlock; + } + + mutex_lock(&grp->mons_lock); + for (j = 0; j < grp->num_inited_mons; j++) { + if (grp->mons[j].type != CPUCP_MON) + continue; + /* Configure per monitor parameters */ + ret = configure_cpucp_mon(&grp->mons[j]); + if (ret < 0) { + pr_err("failed to configure mon: %d\n", ret); + ops = NULL; + goto mons_unlock; + } + start_cpucp_timer = true; + } + mutex_unlock(&grp->mons_lock); + } + + ret = ops->sample_ms(memlat_data->ph, memlat_data->cpucp_sample_ms); + if (ret < 0) { + pr_err("failed to set cpucp sample_ms\n"); + goto memlat_unlock; + } + + /* Start sampling and voting timer */ + if (!start_cpucp_timer) + goto memlat_unlock; + + ret = ops->start_timer(memlat_data->ph); + if (ret < 0) + pr_err("Error in starting the mem group timer %d\n", ret); + + goto memlat_unlock; + +mons_unlock: + mutex_unlock(&grp->mons_lock); +memlat_unlock: + mutex_unlock(&memlat_lock); + return ret; +} +EXPORT_SYMBOL(cpucp_memlat_init); + +#define INST_EV 0x08 +#define CYC_EV 0x11 +static int memlat_dev_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct kobject *dcvs_kobj; + struct memlat_dev_data *dev_data; + int i, cpu, ret; + u32 event_id; + + dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return -ENOMEM; + + dev_data->dev = dev; + dev_data->sample_ms = 8; + dev_data->cpucp_sample_ms = 8; + + dev_data->num_grps = of_get_available_child_count(dev->of_node); + if (!dev_data->num_grps) { + dev_err(dev, "No memlat grps provided!\n"); + return -ENODEV; + } + + ret = of_property_read_u32(dev->of_node, "qcom,inst-ev", &event_id); + if (ret < 0) { + dev_dbg(dev, "Inst event not specified. Using def:0x%x\n", + INST_EV); + event_id = INST_EV; + } + dev_data->common_ev_ids[INST_IDX] = event_id; + + ret = of_property_read_u32(dev->of_node, "qcom,cyc-ev", &event_id); + if (ret < 0) { + dev_dbg(dev, "Cyc event not specified. Using def:0x%x\n", + CYC_EV); + event_id = CYC_EV; + } + dev_data->common_ev_ids[CYC_IDX] = event_id; + + ret = of_property_read_u32(dev->of_node, "qcom,fe-stall-ev", &event_id); + if (ret < 0) + dev_dbg(dev, "FE Stall event not specified. Skipping.\n"); + else + dev_data->common_ev_ids[FE_STALL_IDX] = event_id; + + ret = of_property_read_u32(dev->of_node, "qcom,be-stall-ev", &event_id); + if (ret < 0) + dev_dbg(dev, "BE Stall event not specified. Skipping.\n"); + else + dev_data->common_ev_ids[BE_STALL_IDX] = event_id; + + for_each_possible_cpu(cpu) { + for (i = 0; i < NUM_COMMON_EVS; i++) { + event_id = dev_data->common_ev_ids[i]; + if (!event_id) + continue; + ret = qcom_pmu_event_supported(event_id, cpu); + if (!ret) + continue; + if (ret != -EPROBE_DEFER) + dev_err(dev, "ev=%lu not found on cpu%d: %d\n", + event_id, cpu, ret); + return ret; + } + } + + dcvs_kobj = qcom_dcvs_kobject_get(NUM_DCVS_HW_TYPES); + if (IS_ERR(dcvs_kobj)) { + ret = PTR_ERR(dcvs_kobj); + dev_err(dev, "error getting kobj from qcom_dcvs: %d\n", ret); + return ret; + } + ret = kobject_init_and_add(&dev_data->kobj, &memlat_settings_ktype, + dcvs_kobj, "memlat_settings"); + if (ret < 0) { + dev_err(dev, "failed to init memlat settings kobj: %d\n", ret); + kobject_put(&dev_data->kobj); + return ret; + } + + memlat_data = dev_data; + + return 0; +} + +static int memlat_grp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct memlat_group *memlat_grp; + int i, cpu, ret; + u32 event_id, num_mons; + u32 hw_type = NUM_DCVS_PATHS, path_type = NUM_DCVS_PATHS; + struct device_node *of_node; + + of_node = of_parse_phandle(dev->of_node, "qcom,target-dev", 0); + if (!of_node) { + dev_err(dev, "Unable to find target-dev for grp\n"); + return -EINVAL; + } + + ret = of_property_read_u32(of_node, "qcom,dcvs-hw-type", &hw_type); + if (ret < 0 || hw_type >= NUM_DCVS_HW_TYPES) { + dev_err(dev, "invalid dcvs hw_type=%d, ret=%d\n", hw_type, ret); + return -EINVAL; + } + + memlat_grp = devm_kzalloc(dev, sizeof(*memlat_grp), GFP_KERNEL); + if (!memlat_grp) + return -ENOMEM; + + memlat_grp->hw_type = hw_type; + memlat_grp->dev = dev; + + memlat_grp->dcvs_kobj = qcom_dcvs_kobject_get(hw_type); + if (IS_ERR(memlat_grp->dcvs_kobj)) { + ret = PTR_ERR(memlat_grp->dcvs_kobj); + dev_err(dev, "error getting kobj from qcom_dcvs: %d\n", ret); + return ret; + } + + of_node = of_parse_phandle(dev->of_node, "qcom,sampling-path", 0); + if (of_node) { + ret = of_property_read_u32(of_node, "qcom,dcvs-path-type", + &path_type); + if (ret < 0 || path_type >= NUM_DCVS_PATHS) { + dev_err(dev, "invalid dcvs path: %d, ret=%d\n", + path_type, ret); + return -EINVAL; + } + if (path_type == DCVS_FAST_PATH) + ret = qcom_dcvs_register_voter(FP_NAME, hw_type, + path_type); + else + ret = qcom_dcvs_register_voter(dev_name(dev), hw_type, + path_type); + if (ret < 0) { + dev_err(dev, "qcom dcvs registration error: %d\n", ret); + return ret; + } + memlat_grp->sampling_path_type = path_type; + } else + memlat_grp->sampling_path_type = NUM_DCVS_PATHS; + + of_node = of_parse_phandle(dev->of_node, "qcom,threadlat-path", 0); + if (of_node) { + ret = of_property_read_u32(of_node, "qcom,dcvs-path-type", + &path_type); + if (ret < 0 || path_type >= NUM_DCVS_PATHS) { + dev_err(dev, "invalid dcvs path: %d, ret=%d\n", + path_type, ret); + return -EINVAL; + } + memlat_grp->threadlat_path_type = path_type; + } else + memlat_grp->threadlat_path_type = NUM_DCVS_PATHS; + + if (path_type >= NUM_DCVS_PATHS) { + dev_err(dev, "error: no dcvs voting paths\n"); + return -ENODEV; + } + if (memlat_grp->sampling_path_type == DCVS_FAST_PATH || + memlat_grp->threadlat_path_type == DCVS_FAST_PATH) { + memlat_grp->fp_voting_enabled = true; + memlat_grp->fp_votes = devm_kzalloc(dev, NUM_FP_VOTERS * + sizeof(*memlat_grp->fp_votes), + GFP_KERNEL); + } + + num_mons = of_get_available_child_count(dev->of_node); + if (!num_mons) { + dev_err(dev, "No mons provided!\n"); + return -ENODEV; + } + memlat_grp->mons = + devm_kzalloc(dev, num_mons * sizeof(*memlat_grp->mons), + GFP_KERNEL); + if (!memlat_grp->mons) + return -ENOMEM; + memlat_grp->num_mons = num_mons; + memlat_grp->num_inited_mons = 0; + mutex_init(&memlat_grp->mons_lock); + + ret = of_property_read_u32(dev->of_node, "qcom,miss-ev", &event_id); + if (ret < 0) { + dev_err(dev, "Cache miss event missing for grp: %d\n", ret); + return -EINVAL; + } + memlat_grp->grp_ev_ids[MISS_IDX] = event_id; + + ret = of_property_read_u32(dev->of_node, "qcom,access-ev", + &event_id); + if (ret < 0) + dev_dbg(dev, "Access event not specified. Skipping.\n"); + else + memlat_grp->grp_ev_ids[ACC_IDX] = event_id; + + ret = of_property_read_u32(dev->of_node, "qcom,wb-ev", &event_id); + if (ret < 0) + dev_dbg(dev, "WB event not specified. Skipping.\n"); + else + memlat_grp->grp_ev_ids[WB_IDX] = event_id; + + for_each_possible_cpu(cpu) { + for (i = 0; i < NUM_GRP_EVS; i++) { + event_id = memlat_grp->grp_ev_ids[i]; + if (!event_id) + continue; + ret = qcom_pmu_event_supported(event_id, cpu); + if (!ret) + continue; + if (ret != -EPROBE_DEFER) + dev_err(dev, "ev=%lu not found on cpu%d: %d\n", + event_id, cpu, ret); + return ret; + } + } + + ret = kobject_init_and_add(&memlat_grp->kobj, &memlat_grp_ktype, + memlat_grp->dcvs_kobj, "memlat"); + + mutex_lock(&memlat_lock); + memlat_data->num_inited_grps++; + memlat_data->groups[hw_type] = memlat_grp; + if (!memlat_data->fp_enabled && should_enable_memlat_fp()) { + spin_lock_init(&memlat_data->fp_agg_lock); + spin_lock_init(&memlat_data->fp_commit_lock); + memlat_data->fp_enabled = true; + } + mutex_unlock(&memlat_lock); + + dev_set_drvdata(dev, memlat_grp); + + return 0; +} + +static int memlat_mon_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret = 0; + struct memlat_group *memlat_grp; + struct memlat_mon *mon; + struct device_node *of_node = dev->of_node; + u32 num_cpus; + + memlat_grp = dev_get_drvdata(dev->parent); + if (!memlat_grp) { + dev_err(dev, "Mon probe called without memlat_grp inited.\n"); + return -ENODEV; + } + + mutex_lock(&memlat_grp->mons_lock); + mon = &memlat_grp->mons[memlat_grp->num_inited_mons]; + mon->memlat_grp = memlat_grp; + mon->dev = dev; + + if (get_mask_and_mpidr_from_pdev(pdev, &mon->cpus, &mon->cpus_mpidr)) { + dev_err(dev, "Mon missing cpulist\n"); + ret = -ENODEV; + goto unlock_out; + } + + num_cpus = cpumask_weight(&mon->cpus); + + if (of_property_read_bool(dev->of_node, "qcom,sampling-enabled")) { + mutex_lock(&memlat_lock); + if (!memlat_data->sampling_enabled) { + ret = memlat_sampling_init(); + memlat_data->sampling_enabled = true; + } + mutex_unlock(&memlat_lock); + mon->type |= SAMPLING_MON; + } + + if (of_property_read_bool(dev->of_node, "qcom,threadlat-enabled")) + mon->type |= THREADLAT_MON; + + if (of_property_read_bool(dev->of_node, "qcom,cpucp-enabled")) { + mon->type |= CPUCP_MON; + memlat_grp->cpucp_enabled = true; + } + + if (!mon->type) { + dev_err(dev, "No types configured for mon!\n"); + ret = -ENODEV; + goto unlock_out; + } + + if (of_property_read_bool(dev->of_node, "qcom,compute-mon")) + mon->is_compute = true; + + mon->ipm_ceil = 400; + mon->fe_stall_floor = 0; + mon->be_stall_floor = 0; + mon->freq_scale_pct = 0; + mon->wb_pct_thres = 100; + mon->wb_filter_ipm = 25000; + mon->freq_scale_limit_mhz = 5000; + + if (of_parse_phandle(of_node, COREDEV_TBL_PROP, 0)) + of_node = of_parse_phandle(of_node, COREDEV_TBL_PROP, 0); + if (of_get_child_count(of_node)) + of_node = qcom_dcvs_get_ddr_child_node(of_node); + + mon->freq_map = init_cpufreq_memfreq_map(dev, of_node, + &mon->freq_map_len); + if (!mon->freq_map) { + dev_err(dev, "error importing cpufreq-memfreq table!\n"); + ret = -EINVAL; + goto unlock_out; + } + + mon->mon_min_freq = mon->min_freq = cpufreq_to_memfreq(mon, 0); + mon->mon_max_freq = mon->max_freq = cpufreq_to_memfreq(mon, U32_MAX); + mon->cur_freq = mon->min_freq; + + if (mon->is_compute) + ret = kobject_init_and_add(&mon->kobj, &compute_mon_ktype, + memlat_grp->dcvs_kobj, dev_name(dev)); + else + ret = kobject_init_and_add(&mon->kobj, &memlat_mon_ktype, + memlat_grp->dcvs_kobj, dev_name(dev)); + if (ret < 0) { + dev_err(dev, "failed to init memlat mon kobj: %d\n", ret); + kobject_put(&mon->kobj); + goto unlock_out; + } + + mon->index = memlat_grp->num_inited_mons++; + if (memlat_grps_and_mons_inited()) + memlat_data->inited = true; + +unlock_out: + mutex_unlock(&memlat_grp->mons_lock); + return ret; +} + +static int qcom_memlat_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret = 0; + const struct memlat_spec *spec = of_device_get_match_data(dev); + enum memlat_type type = NUM_MEMLAT_TYPES; + + if (spec) + type = spec->type; + + switch (type) { + case MEMLAT_DEV: + if (memlat_data) { + dev_err(dev, "only one memlat device allowed\n"); + ret = -ENODEV; + } + ret = memlat_dev_probe(pdev); + if (!ret && of_get_available_child_count(dev->of_node)) + of_platform_populate(dev->of_node, NULL, NULL, dev); + break; + case MEMLAT_GRP: + ret = memlat_grp_probe(pdev); + if (!ret && of_get_available_child_count(dev->of_node)) + of_platform_populate(dev->of_node, NULL, NULL, dev); + break; + case MEMLAT_MON: + ret = memlat_mon_probe(pdev); + break; + default: + /* + * This should never happen. + */ + dev_err(dev, "Invalid memlat mon type specified: %u\n", type); + return -EINVAL; + } + + if (ret < 0) { + dev_err(dev, "Failure to probe memlat device: %d\n", ret); + return ret; + } + + return 0; +} + +static const struct memlat_spec spec[] = { + [0] = { MEMLAT_DEV }, + [1] = { MEMLAT_GRP }, + [2] = { MEMLAT_MON }, +}; + +static const struct of_device_id memlat_match_table[] = { + { .compatible = "qcom,memlat", .data = &spec[0] }, + { .compatible = "qcom,memlat-grp", .data = &spec[1] }, + { .compatible = "qcom,memlat-mon", .data = &spec[2] }, + {} +}; + +static struct platform_driver qcom_memlat_driver = { + .probe = qcom_memlat_probe, + .driver = { + .name = "qcom-memlat", + .of_match_table = memlat_match_table, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(qcom_memlat_driver); + +MODULE_DESCRIPTION("QCOM MEMLAT Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/dcvs/memlat_scmi.c b/drivers/soc/qcom/dcvs/memlat_scmi.c new file mode 100644 index 000000000000..b837a6088ee0 --- /dev/null +++ b/drivers/soc/qcom/dcvs/memlat_scmi.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +extern int cpucp_memlat_init(struct scmi_device *sdev); + +static int scmi_memlat_probe(struct scmi_device *sdev) +{ + if (!sdev) + return -ENODEV; + + return cpucp_memlat_init(sdev); +} + +static const struct scmi_device_id scmi_id_table[] = { + { .protocol_id = SCMI_PROTOCOL_MEMLAT, .name = "scmi_protocol_memlat" }, + { }, +}; +MODULE_DEVICE_TABLE(scmi, scmi_id_table); + +static struct scmi_driver scmi_memlat_drv = { + .name = "scmi-memlat-driver", + .probe = scmi_memlat_probe, + .id_table = scmi_id_table, +}; +module_scmi_driver(scmi_memlat_drv); + +MODULE_SOFTDEP("pre: memlat_vendor"); +MODULE_DESCRIPTION("ARM SCMI Memlat driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/dcvs/pmu_lib.c b/drivers/soc/qcom/dcvs/pmu_lib.c new file mode 100644 index 000000000000..acf7f5917e54 --- /dev/null +++ b/drivers/soc/qcom/dcvs/pmu_lib.c @@ -0,0 +1,1183 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "qcom-pmu: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_PMU_EVS QCOM_PMU_MAX_EVS +#define INVALID_ID 0xFF +static void __iomem *pmu_base; + +struct cpucp_pmu_ctrs { + u32 evctrs[MAX_CPUCP_EVT]; + u32 valid; +}; + +struct event_data { + u32 event_id; + struct perf_event *pevent; + int cpu; + u64 cached_count; + enum amu_counters amu_id; + enum cpucp_ev_idx cid; +}; + +struct amu_data { + enum amu_counters amu_id; + u64 count; +}; + +struct cpu_data { + bool is_idle; + bool is_hp; + bool is_pc; + struct event_data events[MAX_PMU_EVS]; + u32 num_evs; + atomic_t read_cnt; + spinlock_t read_lock; +}; + +static DEFINE_PER_CPU(struct cpu_data *, cpu_ev_data); +static bool qcom_pmu_inited; +static bool pmu_long_counter; +static int cpuhp_state; +static struct scmi_protocol_handle *ph; +static const struct scmi_pmu_vendor_ops *ops; +static LIST_HEAD(idle_notif_list); +static DEFINE_SPINLOCK(idle_list_lock); +static struct cpucp_hlos_map cpucp_map[MAX_CPUCP_EVT]; +static struct kobject pmu_kobj; +static bool pmu_counters_enabled = true; +static unsigned int pmu_enable_trace; +static bool llcc_ignore_setup; + +/* + * is_amu_valid: Check if AMUs are supported and if the id corresponds to the + * four supported AMU counters i.e. SYS_AMEVCNTR0_CONST_EL0, + * SYS_AMEVCNTR0_CORE_EL0, SYS_AMEVCNTR0_INST_RET_EL0, SYS_AMEVCNTR0_MEM_STALL + */ +static inline bool is_amu_valid(enum amu_counters amu_id) +{ + return (amu_id >= SYS_AMU_CONST_CYC && amu_id < SYS_AMU_MAX && + IS_ENABLED(CONFIG_ARM64_AMU_EXTN)); +} + +/* + * is_cid_valid: Check if events are supported and if the id corresponds to the + * supported CPUCP events i.e. enum cpucp_ev_idx, + */ +static inline bool is_cid_valid(enum cpucp_ev_idx cid) +{ + return (cid >= CPU_CYC_EVT && cid < MAX_CPUCP_EVT); +} + +/* + * is_event_valid: Check if event has an id and a corresponding pevent or + * valid amu id. + */ +static inline bool is_event_valid(struct event_data *ev) +{ + return (ev->event_id && (ev->pevent || is_amu_valid(ev->amu_id))); +} + +/* + * is_event_shared: Check if event is supposed to be shared with cpucp. + */ +static inline bool is_event_shared(struct event_data *ev) +{ + return (is_cid_valid(ev->cid) && cpucp_map[ev->cid].shared); +} + +#define CYCLE_COUNTER_ID 0x11 +static inline u64 cached_count_value(struct event_data *ev, u64 event_cached_count, bool amu) +{ + struct arm_pmu *cpu_pmu = container_of(ev->pevent->pmu, struct arm_pmu, pmu); + + if (amu) + return event_cached_count; + + if (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5) + event_cached_count |= (pmu_long_counter ? BIT(63) : GENMASK(63, 31)); + else { + if (ev->event_id == CYCLE_COUNTER_ID) + event_cached_count |= GENMASK(63, 31); + else + event_cached_count = ((event_cached_count & GENMASK(31, 0)) | + BIT(31)); + } + + return event_cached_count; +} + +static struct perf_event_attr *alloc_attr(void) +{ + struct perf_event_attr *attr; + + attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL); + if (!attr) + return attr; + + attr->size = sizeof(struct perf_event_attr); + attr->pinned = 1; + + return attr; +} + +static int set_event(struct event_data *ev, int cpu, + struct perf_event_attr *attr) +{ + struct perf_event *pevent; + u32 type = PERF_TYPE_RAW; + int ret; + + /* Set the cpu and exit if amu is supported */ + if (is_amu_valid(ev->amu_id)) + goto set_cpu; + else + ev->amu_id = SYS_AMU_MAX; + + if (!is_cid_valid(ev->cid)) + ev->cid = MAX_CPUCP_EVT; + + if (!ev->event_id) + return 0; + + attr->config = ev->event_id; + /* enable 64-bit counter */ + if (pmu_long_counter) + attr->config1 = 1; + + if (ev->event_id == QCOM_LLCC_PMU_RD_EV) { + /* Ignore setting up the event if property set. This will avoid + * reading of event as well since ev->pevent will be NULL. + */ + if (llcc_ignore_setup) + goto set_cpu; + + ret = qcom_llcc_pmu_hw_type(&type); + if (ret < 0) + return ret; + } + attr->type = type; + pevent = perf_event_create_kernel_counter(attr, cpu, NULL, NULL, NULL); + if (IS_ERR(pevent)) + return PTR_ERR(pevent); + + perf_event_enable(pevent); + ev->pevent = pevent; +set_cpu: + ev->cpu = cpu; + + return 0; +} + +static inline void delete_event(struct event_data *event) +{ + if (event->pevent) { + perf_event_release_kernel(event->pevent); + event->pevent = NULL; + } +} + +static void read_amu_reg(void *amu_data) +{ + struct amu_data *data = amu_data; + + switch (data->amu_id) { + case SYS_AMU_CONST_CYC: + data->count = read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0); + break; + case SYS_AMU_CORE_CYC: + data->count = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0); + break; + case SYS_AMU_INST_RET: + data->count = read_sysreg_s(SYS_AMEVCNTR0_INST_RET_EL0); + break; + case SYS_AMU_STALL_MEM: + data->count = read_sysreg_s(SYS_AMEVCNTR0_MEM_STALL); + break; + default: + pr_err("AMU counter %d not supported!\n", data->amu_id); + } +} + +static inline u64 read_event(struct event_data *event, bool local) +{ + u64 enabled, running, total = 0; + struct amu_data data; + int ret = 0; + + if (is_amu_valid(event->amu_id)) { + data.amu_id = event->amu_id; + if (local) + read_amu_reg(&data); + else { + ret = smp_call_function_single(event->cpu, read_amu_reg, + &data, true); + if (ret < 0) + return event->cached_count; + } + total = data.count; + } else { + if (!event->pevent) + return event->cached_count; + if (local) + perf_event_read_local(event->pevent, &total, NULL, NULL); + else + total = perf_event_read_value(event->pevent, &enabled, + &running); + } + event->cached_count = total; + + return total; +} + +static int __qcom_pmu_read(int cpu, u32 event_id, u64 *pmu_data, bool local) +{ + struct cpu_data *cpu_data; + struct event_data *event; + int i; + unsigned long flags; + + if (!qcom_pmu_inited) + return -ENODEV; + + if (!event_id || !pmu_data || cpu >= num_possible_cpus()) + return -EINVAL; + + cpu_data = per_cpu(cpu_ev_data, cpu); + for (i = 0; i < cpu_data->num_evs; i++) { + event = &cpu_data->events[i]; + if (event->event_id == event_id) + break; + } + if (i == cpu_data->num_evs) + return -ENOENT; + + spin_lock_irqsave(&cpu_data->read_lock, flags); + if (cpu_data->is_hp || cpu_data->is_idle || cpu_data->is_pc) { + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + *pmu_data = event->cached_count; + return 0; + } + atomic_inc(&cpu_data->read_cnt); + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + *pmu_data = read_event(event, local); + atomic_dec(&cpu_data->read_cnt); + + return 0; +} + +int __qcom_pmu_read_all(int cpu, struct qcom_pmu_data *data, bool local) +{ + struct cpu_data *cpu_data; + struct event_data *event; + int i, cnt = 0; + bool use_cache = false; + unsigned long flags; + + if (!qcom_pmu_inited) + return -ENODEV; + + if (!data || cpu >= num_possible_cpus()) + return -EINVAL; + + cpu_data = per_cpu(cpu_ev_data, cpu); + spin_lock_irqsave(&cpu_data->read_lock, flags); + if (cpu_data->is_hp || cpu_data->is_idle || cpu_data->is_pc) + use_cache = true; + else + atomic_inc(&cpu_data->read_cnt); + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + + for (i = 0; i < cpu_data->num_evs; i++) { + event = &cpu_data->events[i]; + if (!event->event_id) + continue; + data->event_ids[cnt] = event->event_id; + if (use_cache) + data->ev_data[cnt] = event->cached_count; + else + data->ev_data[cnt] = read_event(event, local); + cnt++; + } + data->num_evs = cnt; + + if (!use_cache) + atomic_dec(&cpu_data->read_cnt); + + return 0; +} + +static struct event_data *get_event(u32 event_id, int cpu) +{ + struct cpu_data *cpu_data; + struct event_data *event; + int i; + + if (!qcom_pmu_inited) + return ERR_PTR(-EPROBE_DEFER); + + if (!event_id || cpu >= num_possible_cpus()) + return ERR_PTR(-EINVAL); + + cpu_data = per_cpu(cpu_ev_data, cpu); + for (i = 0; i < cpu_data->num_evs; i++) { + event = &cpu_data->events[i]; + if (event->event_id == event_id) + return event; + } + return ERR_PTR(-ENOENT); +} + +int qcom_get_cpucp_id(u32 event_id, int cpu) +{ + struct event_data *event; + + event = get_event(event_id, cpu); + if (IS_ERR(event)) { + pr_err("error getting event %d\n", PTR_ERR(event)); + return PTR_ERR(event); + } + + return event->cid; +} +EXPORT_SYMBOL(qcom_get_cpucp_id); + +int qcom_pmu_event_supported(u32 event_id, int cpu) +{ + struct event_data *event; + + event = get_event(event_id, cpu); + + return PTR_ERR_OR_ZERO(event); +} +EXPORT_SYMBOL(qcom_pmu_event_supported); + +int qcom_pmu_read(int cpu, u32 event_id, u64 *pmu_data) +{ + return __qcom_pmu_read(cpu, event_id, pmu_data, false); +} +EXPORT_SYMBOL(qcom_pmu_read); + +int qcom_pmu_read_local(u32 event_id, u64 *pmu_data) +{ + int this_cpu = smp_processor_id(); + + return __qcom_pmu_read(this_cpu, event_id, pmu_data, true); +} +EXPORT_SYMBOL(qcom_pmu_read_local); + +int qcom_pmu_read_all(int cpu, struct qcom_pmu_data *data) +{ + return __qcom_pmu_read_all(cpu, data, false); +} +EXPORT_SYMBOL(qcom_pmu_read_all); + +int qcom_pmu_read_all_local(struct qcom_pmu_data *data) +{ + int this_cpu = smp_processor_id(); + + return __qcom_pmu_read_all(this_cpu, data, true); +} +EXPORT_SYMBOL(qcom_pmu_read_all_local); + +int qcom_pmu_idle_register(struct qcom_pmu_notif_node *idle_node) +{ + struct qcom_pmu_notif_node *tmp_node; + + if (!idle_node || !idle_node->idle_cb) + return -EINVAL; + + spin_lock(&idle_list_lock); + list_for_each_entry(tmp_node, &idle_notif_list, node) + if (tmp_node->idle_cb == idle_node->idle_cb) + goto out; + list_add_tail(&idle_node->node, &idle_notif_list); +out: + spin_unlock(&idle_list_lock); + return 0; +} +EXPORT_SYMBOL(qcom_pmu_idle_register); + +int qcom_pmu_idle_unregister(struct qcom_pmu_notif_node *idle_node) +{ + struct qcom_pmu_notif_node *tmp_node; + int ret = -EINVAL; + + if (!idle_node || !idle_node->idle_cb) + return ret; + + spin_lock(&idle_list_lock); + list_for_each_entry(tmp_node, &idle_notif_list, node) { + if (tmp_node->idle_cb == idle_node->idle_cb) { + list_del(&tmp_node->node); + ret = 0; + break; + } + } + spin_unlock(&idle_list_lock); + return ret; +} +EXPORT_SYMBOL(qcom_pmu_idle_unregister); + +static int events_caching_enable(void) +{ + int ret = 0; + unsigned int enable = 1; + + if (!qcom_pmu_inited) + return -EPROBE_DEFER; + + if (!ops || !pmu_base) + return ret; + + ret = ops->set_cache_enable(ph, &enable); + + return ret; +} + +static int configure_cpucp_map(cpumask_t mask) +{ + struct event_data *event; + int i, cpu, ret = 0, cid; + u8 pmu_map[MAX_NUM_CPUS][MAX_CPUCP_EVT]; + struct cpu_data *cpu_data; + + if (!qcom_pmu_inited) + return -EPROBE_DEFER; + + if (!ops) + return ret; + + /* + * Only set the hw cntrs for cpus that are part of the cpumask passed + * in argument and cpucp_map events mask. Set rest of the memory with + * INVALID_ID which is ignored on cpucp side. + */ + memset(pmu_map, INVALID_ID, MAX_NUM_CPUS * MAX_CPUCP_EVT); + for_each_cpu(cpu, &mask) { + cpu_data = per_cpu(cpu_ev_data, cpu); + for (i = 0; i < cpu_data->num_evs; i++) { + event = &cpu_data->events[i]; + cid = event->cid; + if (!is_event_shared(event) || + is_amu_valid(event->amu_id) || !event->pevent || + !cpumask_test_cpu(cpu, to_cpumask(&cpucp_map[cid].cpus))) + continue; + pmu_map[cpu][cid] = event->pevent->hw.idx; + } + } + + ret = ops->set_pmu_map(ph, pmu_map); + + return ret; +} + +static void qcom_pmu_idle_enter_notif(void *unused, int *state, + struct cpuidle_device *dev) +{ + struct cpu_data *cpu_data = per_cpu(cpu_ev_data, dev->cpu); + struct qcom_pmu_data pmu_data; + struct event_data *ev; + struct qcom_pmu_notif_node *idle_node; + int i, cnt = 0; + unsigned long flags; + + spin_lock_irqsave(&cpu_data->read_lock, flags); + if (cpu_data->is_idle || cpu_data->is_hp || cpu_data->is_pc) { + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + return; + } + cpu_data->is_idle = true; + atomic_inc(&cpu_data->read_cnt); + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + for (i = 0; i < cpu_data->num_evs; i++) { + ev = &cpu_data->events[i]; + if (!is_event_valid(ev)) + continue; + ev->cached_count = read_event(ev, true); + pmu_data.event_ids[cnt] = ev->event_id; + pmu_data.ev_data[cnt] = ev->cached_count; + cnt++; + } + atomic_dec(&cpu_data->read_cnt); + pmu_data.num_evs = cnt; + + /* send snapshot of pmu data to all registered idle clients */ + list_for_each_entry(idle_node, &idle_notif_list, node) + idle_node->idle_cb(&pmu_data, dev->cpu, *state); +} + +static void qcom_pmu_idle_exit_notif(void *unused, int state, + struct cpuidle_device *dev) +{ + struct cpu_data *cpu_data = per_cpu(cpu_ev_data, dev->cpu); + + cpu_data->is_idle = false; +} + +static int memlat_pm_notif(struct notifier_block *nb, unsigned long action, + void *data) +{ + int cpu = smp_processor_id(); + struct cpu_data *cpu_data = per_cpu(cpu_ev_data, cpu); + struct event_data *ev; + int i, cid, aid; + u32 count; + bool pmu_valid = false; + bool read_ev = true; + struct cpucp_pmu_ctrs *base = pmu_base + (sizeof(struct cpucp_pmu_ctrs) * cpu); + unsigned long flags; + + /* Exit if cpu is in hotplug */ + spin_lock_irqsave(&cpu_data->read_lock, flags); + if (cpu_data->is_hp) { + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + return NOTIFY_OK; + } + + if (action == CPU_PM_EXIT) { + if (pmu_base) + writel_relaxed(0, &base->valid); + cpu_data->is_pc = false; + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + return NOTIFY_OK; + } + + if (cpu_data->is_idle || cpu_data->is_pc) + read_ev = false; + else + atomic_inc(&cpu_data->read_cnt); + cpu_data->is_pc = true; + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + + if (!pmu_base) + goto dec_read_cnt; + + for (i = 0; i < cpu_data->num_evs; i++) { + ev = &cpu_data->events[i]; + cid = ev->cid; + aid = ev->amu_id; + if (!is_event_valid(ev) || !is_event_shared(ev)) + continue; + if (read_ev) + ev->cached_count = read_event(ev, true); + /* Store pmu values in allocated cpucp pmu region */ + pmu_valid = true; + count = cached_count_value(ev, ev->cached_count, is_amu_valid(aid)); + writel_relaxed(count, &base->evctrs[cid]); + } + /* Set valid cache flag to allow cpucp to read from this memory location */ + if (pmu_valid) + writel_relaxed(1, &base->valid); + +dec_read_cnt: + if (read_ev) + atomic_dec(&cpu_data->read_cnt); + + return NOTIFY_OK; +} + +static struct notifier_block memlat_event_pm_nb = { + .notifier_call = memlat_pm_notif, +}; + +#if IS_ENABLED(CONFIG_HOTPLUG_CPU) +static int qcom_pmu_hotplug_coming_up(unsigned int cpu) +{ + struct perf_event_attr *attr = alloc_attr(); + struct cpu_data *cpu_data = per_cpu(cpu_ev_data, cpu); + int i, ret = 0; + unsigned long flags; + struct event_data *ev; + struct cpucp_pmu_ctrs *base = pmu_base + (sizeof(struct cpucp_pmu_ctrs) * cpu); + cpumask_t mask; + + if (!attr) + return -ENOMEM; + + if (!qcom_pmu_inited) + goto out; + + for (i = 0; i < cpu_data->num_evs; i++) { + ev = &cpu_data->events[i]; + ret = set_event(ev, cpu, attr); + if (ret < 0) { + pr_err("event %d not set for cpu %d ret %d\n", + ev->event_id, cpu, ret); + break; + } + } + cpumask_clear(&mask); + cpumask_set_cpu(cpu, &mask); + configure_cpucp_map(mask); + /* Set valid as 0 as exiting hotplug */ + if (pmu_base) + writel_relaxed(0, &base->valid); + + spin_lock_irqsave(&cpu_data->read_lock, flags); + cpu_data->is_hp = false; + spin_unlock_irqrestore(&cpu_data->read_lock, flags); +out: + kfree(attr); + return 0; +} + +static int qcom_pmu_hotplug_going_down(unsigned int cpu) +{ + struct cpu_data *cpu_data = per_cpu(cpu_ev_data, cpu); + struct event_data *ev; + int i, cid, aid; + unsigned long flags; + bool pmu_valid = false; + u32 count; + struct cpucp_pmu_ctrs *base = pmu_base + (sizeof(struct cpucp_pmu_ctrs) * cpu); + + if (!qcom_pmu_inited) + return 0; + + spin_lock_irqsave(&cpu_data->read_lock, flags); + cpu_data->is_hp = true; + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + while (atomic_read(&cpu_data->read_cnt) > 0) + udelay(10); + for (i = 0; i < cpu_data->num_evs; i++) { + ev = &cpu_data->events[i]; + cid = ev->cid; + aid = ev->amu_id; + if (!is_event_valid(ev)) + continue; + ev->cached_count = read_event(ev, false); + /* Store pmu values in allocated cpucp pmu region */ + if (pmu_base && is_event_shared(ev)) { + pmu_valid = true; + count = cached_count_value(ev, ev->cached_count, is_amu_valid(aid)); + writel_relaxed(count, &base->evctrs[cid]); + } + delete_event(ev); + } + + if (pmu_valid) + writel_relaxed(1, &base->valid); + return 0; +} + +static int qcom_pmu_cpu_hp_init(void) +{ + int ret; + + ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, + "QCOM_PMU", + qcom_pmu_hotplug_coming_up, + qcom_pmu_hotplug_going_down); + if (ret < 0) + pr_err("qcom_pmu: CPU hotplug notifier error: %d\n", + ret); + + return ret; +} +#else +static int qcom_pmu_cpu_hp_init(void) { return 0; } +#endif + +static void cache_counters(void) +{ + struct cpu_data *cpu_data; + int i, cid, aid; + unsigned int cpu; + struct event_data *event; + struct cpucp_pmu_ctrs *base; + bool pmu_valid; + u32 count; + + for_each_possible_cpu(cpu) { + cpu_data = per_cpu(cpu_ev_data, cpu); + base = pmu_base + (sizeof(struct cpucp_pmu_ctrs) * cpu); + pmu_valid = false; + for (i = 0; i < cpu_data->num_evs; i++) { + event = &cpu_data->events[i]; + cid = event->cid; + aid = event->amu_id; + if (!is_event_valid(event)) + continue; + read_event(event, false); + /* Store pmu values in allocated cpucp pmu region */ + if (pmu_base && is_event_shared(event)) { + pmu_valid = true; + count = cached_count_value(event, event->cached_count, + is_amu_valid(aid)); + writel_relaxed(count, &base->evctrs[cid]); + } + } + if (pmu_valid) + writel_relaxed(1, &base->valid); + } +} + +static void delete_events(void) +{ + int i; + unsigned int cpu; + struct cpu_data *cpu_data; + struct event_data *event; + unsigned long flags; + + if (cpuhp_state > 0) + cpuhp_remove_state_nocalls(cpuhp_state); + unregister_trace_android_vh_cpu_idle_enter(qcom_pmu_idle_enter_notif, NULL); + unregister_trace_android_vh_cpu_idle_exit(qcom_pmu_idle_exit_notif, NULL); + cpu_pm_unregister_notifier(&memlat_event_pm_nb); + for_each_possible_cpu(cpu) { + cpu_data = per_cpu(cpu_ev_data, cpu); + spin_lock_irqsave(&cpu_data->read_lock, flags); + cpu_data->is_hp = true; + cpu_data->is_idle = true; + cpu_data->is_pc = true; + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + } + + for_each_possible_cpu(cpu) { + cpu_data = per_cpu(cpu_ev_data, cpu); + while (atomic_read(&cpu_data->read_cnt) > 0) + udelay(10); + for (i = 0; i < cpu_data->num_evs; i++) { + event = &cpu_data->events[i]; + if (!is_event_valid(event)) + continue; + delete_event(event); + } + } +} + +static void unload_pmu_counters(void) +{ + if (!qcom_pmu_inited || !pmu_counters_enabled) + return; + + cache_counters(); + delete_events(); + pr_info("Disabled all perf counters\n"); + pmu_counters_enabled = false; +} + +static int setup_events(void) +{ + struct perf_event_attr *attr = alloc_attr(); + struct cpu_data *cpu_data; + int i, ret = 0; + unsigned int cpu; + struct event_data *event; + unsigned long flags; + + if (!attr) + return -ENOMEM; + + cpus_read_lock(); + for_each_possible_cpu(cpu) { + cpu_data = per_cpu(cpu_ev_data, cpu); + for (i = 0; i < cpu_data->num_evs; i++) { + event = &cpu_data->events[i]; + ret = set_event(event, cpu, attr); + if (ret < 0) { + pr_err("event %d not set for cpu %d ret %d\n", + event->event_id, cpu, ret); + event->event_id = 0; + /* + * Only return error for -EPROBE_DEFER. Clear + * ret for all other cases as it is okay for + * some events to fail. + */ + if (ret == -EPROBE_DEFER) + goto cleanup_events; + else + ret = 0; + } + } + spin_lock_irqsave(&cpu_data->read_lock, flags); + cpu_data->is_hp = !cpumask_test_cpu(cpu, cpu_online_mask); + cpu_data->is_idle = false; + cpu_data->is_pc = false; + spin_unlock_irqrestore(&cpu_data->read_lock, flags); + } + + cpuhp_state = qcom_pmu_cpu_hp_init(); + if (cpuhp_state < 0) { + ret = cpuhp_state; + pr_err("qcom pmu driver failed to initialize hotplug: %d\n", ret); + goto out; + } + register_trace_android_vh_cpu_idle_enter(qcom_pmu_idle_enter_notif, NULL); + register_trace_android_vh_cpu_idle_exit(qcom_pmu_idle_exit_notif, NULL); + cpu_pm_register_notifier(&memlat_event_pm_nb); + goto out; + +cleanup_events: + for_each_possible_cpu(cpu) { + cpu_data = per_cpu(cpu_ev_data, cpu); + for (i = 0; i < cpu_data->num_evs; i++) { + event = &cpu_data->events[i]; + delete_event(event); + } + } +out: + cpus_read_unlock(); + kfree(attr); + return ret; +} + +static void load_pmu_counters(void) +{ + int ret; + + if (pmu_counters_enabled) + return; + + ret = setup_events(); + if (ret < 0) { + pr_err("Error setting up counters %d\n", ret); + return; + } + configure_cpucp_map(*cpu_possible_mask); + pmu_counters_enabled = true; + pr_info("Enabled all perf counters\n"); +} + +int rimps_pmu_init(struct scmi_device *sdev) +{ + int ret = 0; + + if (!sdev || !sdev->handle) + return -EINVAL; + + ops = sdev->handle->devm_protocol_get(sdev, SCMI_PMU_PROTOCOL, &ph); + if (!ops) + return -EINVAL; + + /* + * If communication with cpucp doesn't succeed here the device memory + * will be de-allocated. Make ops NULL to avoid further scmi calls. + */ + ret = configure_cpucp_map(*cpu_possible_mask); + if (ret < 0) { + ops = NULL; + return ret; + } + + ret = events_caching_enable(); + if (ret < 0) + ops = NULL; + + return ret; +} +EXPORT_SYMBOL(rimps_pmu_init); + +static int configure_pmu_event(u32 event_id, int amu_id, int cid, int cpu) +{ + struct cpu_data *cpu_data; + struct event_data *event; + + if (!event_id || cpu >= num_possible_cpus()) + return -EINVAL; + + cpu_data = per_cpu(cpu_ev_data, cpu); + if (cpu_data->num_evs >= MAX_PMU_EVS) + return -ENOSPC; + + event = &cpu_data->events[cpu_data->num_evs]; + event->event_id = event_id; + event->amu_id = amu_id; + event->cid = cid; + cpu_data->num_evs++; + + return 0; +} + +#define PMU_TBL_PROP "qcom,pmu-events-tbl" +#define NUM_COL 4 +static int init_pmu_events(struct device *dev) +{ + struct device_node *of_node = dev->of_node; + int ret, len, i, j, cpu; + u32 data = 0, event_id, cid; + unsigned long cpus; + int amu_id; + + if (of_find_property(of_node, "qcom,long-counter", &len)) + pmu_long_counter = true; + + if (!of_find_property(of_node, PMU_TBL_PROP, &len)) + return -ENODEV; + len /= sizeof(data); + if (len % NUM_COL || len == 0) + return -EINVAL; + len /= NUM_COL; + if (len >= MAX_PMU_EVS) + return -ENOSPC; + + for (i = 0, j = 0; i < len; i++, j += NUM_COL) { + ret = of_property_read_u32_index(of_node, PMU_TBL_PROP, j, + &event_id); + if (ret < 0 || !event_id) + return -EINVAL; + + ret = of_property_read_u32_index(of_node, PMU_TBL_PROP, j + 1, + &data); + if (ret < 0 || !data) + return -EINVAL; + cpus = (unsigned long)data; + + ret = of_property_read_u32_index(of_node, PMU_TBL_PROP, j + 2, + &amu_id); + if (ret < 0) + return -EINVAL; + + ret = of_property_read_u32_index(of_node, PMU_TBL_PROP, j + 3, + &cid); + if (ret < 0) + return -EINVAL; + + for_each_cpu(cpu, to_cpumask(&cpus)) { + ret = configure_pmu_event(event_id, amu_id, cid, cpu); + if (ret < 0) + return ret; + } + + if (is_cid_valid(cid)) { + cpucp_map[cid].shared = true; + cpucp_map[cid].cpus = cpus; + } + dev_dbg(dev, "entry=%d: ev=%lu, cpus=%lu cpucp id=%lu amu_id=%d\n", + i, event_id, cpus, cid, amu_id); + } + + return 0; +} + +struct qcom_pmu_attr { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct attribute *attr, + char *buf); + ssize_t (*store)(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count); +}; + +#define to_pmu_attr(_attr) \ + container_of(_attr, struct qcom_pmu_attr, attr) +#define PMU_ATTR_RW(_name) \ +static struct qcom_pmu_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name)\ + +static ssize_t show_enable_counters(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", pmu_counters_enabled); +} + +#define DISABLE_MAGIC "DEADBEEF" +#define ENABLE_MAGIC "BEEFDEAD" +static ssize_t store_enable_counters(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + if (sysfs_streq(buf, ENABLE_MAGIC)) + load_pmu_counters(); + else if (sysfs_streq(buf, DISABLE_MAGIC)) + unload_pmu_counters(); + + return count; +} + +static ssize_t show_enable_trace(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", pmu_enable_trace); +} + +static ssize_t store_enable_trace(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + unsigned int var; + int ret; + + if (!ops) + return -ENODEV; + + ret = kstrtouint(buf, 10, &var); + if (ret < 0) + return ret; + + ret = ops->set_enable_trace(ph, &var); + if (ret < 0) + return ret; + + pmu_enable_trace = var; + + return count; +} + +PMU_ATTR_RW(enable_counters); +PMU_ATTR_RW(enable_trace); +static struct attribute *pmu_settings_attrs[] = { + &enable_counters.attr, + &enable_trace.attr, + NULL, +}; +ATTRIBUTE_GROUPS(pmu_settings); + +static ssize_t attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct qcom_pmu_attr *pmu_attr = to_pmu_attr(attr); + ssize_t ret = -EIO; + + if (pmu_attr->show) + ret = pmu_attr->show(kobj, attr, buf); + + return ret; +} + +static ssize_t attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct qcom_pmu_attr *pmu_attr = to_pmu_attr(attr); + ssize_t ret = -EIO; + + if (pmu_attr->store) + ret = pmu_attr->store(kobj, attr, buf, count); + + return ret; +} + +static const struct sysfs_ops pmu_sysfs_ops = { + .show = attr_show, + .store = attr_store, +}; + +static struct kobj_type pmu_settings_ktype = { + .sysfs_ops = &pmu_sysfs_ops, + .default_groups = pmu_settings_groups, + +}; + +static int qcom_pmu_driver_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret = 0, idx, len; + unsigned int cpu; + struct cpu_data *cpu_data; + struct resource res; + + if (!pmu_base) { + idx = of_property_match_string(dev->of_node, "reg-names", "pmu-base"); + if (idx < 0) { + dev_dbg(dev, "pmu base not found\n"); + goto skip_pmu; + } + ret = of_address_to_resource(dev->of_node, idx, &res); + if (ret < 0) { + dev_err(dev, "failed to get resource ret %d\n", ret); + goto skip_pmu; + } + pmu_base = devm_ioremap(dev, res.start, resource_size(&res)); + if (!pmu_base) + goto skip_pmu; + /* Zero out the pmu memory region */ + memset_io(pmu_base, 0, resource_size(&res)); + } +skip_pmu: + if (of_find_property(dev->of_node, "qcom,ignore-llcc-setup", &len)) { + dev_dbg(dev, "Ignoring llcc setup\n"); + llcc_ignore_setup = true; + } + + for_each_possible_cpu(cpu) { + cpu_data = devm_kzalloc(dev, sizeof(*cpu_data), GFP_KERNEL); + if (!cpu_data) + return -ENOMEM; + + spin_lock_init(&cpu_data->read_lock); + atomic_set(&cpu_data->read_cnt, 0); + per_cpu(cpu_ev_data, cpu) = cpu_data; + } + + ret = init_pmu_events(dev); + if (ret < 0) { + dev_err(dev, "failed to initialize pmu events: %d\n", ret); + return ret; + } + + ret = kobject_init_and_add(&pmu_kobj, &pmu_settings_ktype, + &cpu_subsys.dev_root->kobj, "pmu_lib"); + if (ret < 0) { + dev_err(dev, "failed to init pmu counters kobj: %d\n", ret); + kobject_put(&pmu_kobj); + return ret; + } + + ret = setup_events(); + if (ret < 0) { + dev_err(dev, "failed to setup all pmu/amu events: %d\n", ret); + kobject_put(&pmu_kobj); + return ret; + } + + qcom_pmu_inited = true; + return ret; +} + +static int qcom_pmu_driver_remove(struct platform_device *pdev) +{ + qcom_pmu_inited = false; + delete_events(); + + return 0; +} + +static const struct of_device_id pmu_match_table[] = { + { .compatible = "qcom,pmu" }, + {} +}; + +static struct platform_driver qcom_pmu_driver = { + .probe = qcom_pmu_driver_probe, + .remove = qcom_pmu_driver_remove, + .driver = { + .name = "qcom-pmu", + .of_match_table = pmu_match_table, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(qcom_pmu_driver); + +MODULE_DESCRIPTION("QCOM PMU Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/dcvs/pmu_scmi.c b/drivers/soc/qcom/dcvs/pmu_scmi.c new file mode 100644 index 000000000000..fd856041cc57 --- /dev/null +++ b/drivers/soc/qcom/dcvs/pmu_scmi.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int scmi_pmu_probe(struct scmi_device *sdev) +{ + if (!sdev) + return -ENODEV; + + return rimps_pmu_init(sdev); +} + +static const struct scmi_device_id scmi_id_table[] = { + { .protocol_id = SCMI_PMU_PROTOCOL, .name = "scmi_pmu_protocol" }, + { }, +}; +MODULE_DEVICE_TABLE(scmi, scmi_id_table); + +static struct scmi_driver scmi_pmu_drv = { + .name = "scmi-pmu-driver", + .probe = scmi_pmu_probe, + .id_table = scmi_id_table, +}; +module_scmi_driver(scmi_pmu_drv); + +MODULE_SOFTDEP("pre: pmu_vendor"); +MODULE_DESCRIPTION("ARM SCMI PMU driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/dcvs/trace-dcvs.c b/drivers/soc/qcom/dcvs/trace-dcvs.c new file mode 100644 index 000000000000..69d173e3bf0b --- /dev/null +++ b/drivers/soc/qcom/dcvs/trace-dcvs.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include + +#define CREATE_TRACE_POINTS +#include "trace-dcvs.h" + +EXPORT_TRACEPOINT_SYMBOL(qcom_dcvs_update); +EXPORT_TRACEPOINT_SYMBOL(qcom_dcvs_boost); +EXPORT_TRACEPOINT_SYMBOL(memlat_dev_meas); +EXPORT_TRACEPOINT_SYMBOL(memlat_dev_update); +EXPORT_TRACEPOINT_SYMBOL(bw_hwmon_meas); +EXPORT_TRACEPOINT_SYMBOL(bw_hwmon_update); +EXPORT_TRACEPOINT_SYMBOL(bw_hwmon_debug); diff --git a/drivers/soc/qcom/dcvs/trace-dcvs.h b/drivers/soc/qcom/dcvs/trace-dcvs.h new file mode 100644 index 000000000000..40b591e6589b --- /dev/null +++ b/drivers/soc/qcom/dcvs/trace-dcvs.h @@ -0,0 +1,270 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#if !defined(_TRACE_DCVS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_DCVS_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM dcvs + +#include +#include + +TRACE_EVENT(qcom_dcvs_update, + + TP_PROTO(const char *name, int hw, int path, unsigned long ib, + unsigned long path_ib, unsigned long ab, unsigned long path_ab, + unsigned long boost_freq), + + TP_ARGS(name, hw, path, ib, path_ib, ab, path_ab, boost_freq), + + TP_STRUCT__entry( + __string(name, name) + __field(int, hw) + __field(int, path) + __field(unsigned long, ib) + __field(unsigned long, path_ib) + __field(unsigned long, ab) + __field(unsigned long, path_ab) + __field(unsigned long, boost_freq) + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->hw = hw; + __entry->path = path; + __entry->ib = ib; + __entry->path_ib = path_ib; + __entry->ab = ab; + __entry->path_ab = path_ab; + __entry->boost_freq = boost_freq; + ), + + TP_printk("name=%s hw=%d path=%d ib=%lu path_ib=%lu ab=%lu path_ab=%lu boost=%lu", + __get_str(name), + __entry->hw, + __entry->path, + __entry->ib, + __entry->path_ib, + __entry->ab, + __entry->path_ab, + __entry->boost_freq) +); + +TRACE_EVENT(qcom_dcvs_boost, + + TP_PROTO(int hw, int path, unsigned long boost, unsigned long path_ib, + unsigned long path_ab), + + TP_ARGS(hw, path, boost, path_ib, path_ab), + + TP_STRUCT__entry( + __field(int, hw) + __field(int, path) + __field(unsigned long, boost) + __field(unsigned long, path_ib) + __field(unsigned long, path_ab) + ), + + TP_fast_assign( + __entry->hw = hw; + __entry->path = path; + __entry->boost = boost; + __entry->path_ib = path_ib; + __entry->path_ab = path_ab; + ), + + TP_printk("hw=%d path=%d boost=%lu path_ib=%lu path_ab=%lu", + __entry->hw, + __entry->path, + __entry->boost, + __entry->path_ib, + __entry->path_ab) +); + +TRACE_EVENT(memlat_dev_meas, + + TP_PROTO(const char *name, unsigned int dev_id, unsigned long inst, + unsigned long mem, unsigned long freq, unsigned int stall, + unsigned int wb, unsigned int ratio, unsigned int fe_stall), + + TP_ARGS(name, dev_id, inst, mem, freq, stall, wb, ratio, fe_stall), + + TP_STRUCT__entry( + __string(name, name) + __field(unsigned int, dev_id) + __field(unsigned long, inst) + __field(unsigned long, mem) + __field(unsigned long, freq) + __field(unsigned int, stall) + __field(unsigned int, wb) + __field(unsigned int, ratio) + __field(unsigned int, fe_stall) + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->dev_id = dev_id; + __entry->inst = inst; + __entry->mem = mem; + __entry->freq = freq; + __entry->stall = stall; + __entry->wb = wb; + __entry->ratio = ratio; + __entry->fe_stall = fe_stall; + ), + + TP_printk("dev: %s, id=%u, inst=%lu, mem=%lu, freq=%lu, stall=%u, wb=%u, ratio=%u, fe_stall=%u", + __get_str(name), + __entry->dev_id, + __entry->inst, + __entry->mem, + __entry->freq, + __entry->stall, + __entry->wb, + __entry->ratio, + __entry->fe_stall) +); + +TRACE_EVENT(memlat_dev_update, + + TP_PROTO(const char *name, unsigned int dev_id, unsigned long inst, + unsigned long mem, unsigned long freq, unsigned long vote), + + TP_ARGS(name, dev_id, inst, mem, freq, vote), + + TP_STRUCT__entry( + __string(name, name) + __field(unsigned int, dev_id) + __field(unsigned long, inst) + __field(unsigned long, mem) + __field(unsigned long, freq) + __field(unsigned long, vote) + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->dev_id = dev_id; + __entry->inst = inst; + __entry->mem = mem; + __entry->freq = freq; + __entry->vote = vote; + ), + + TP_printk("dev: %s, id=%u, inst=%lu, mem=%lu, freq=%lu, vote=%lu", + __get_str(name), + __entry->dev_id, + __entry->inst, + __entry->mem, + __entry->freq, + __entry->vote) +); + +TRACE_EVENT(bw_hwmon_meas, + + TP_PROTO(const char *name, unsigned long mbps, + unsigned long us, int wake), + + TP_ARGS(name, mbps, us, wake), + + TP_STRUCT__entry( + __string(name, name) + __field(unsigned long, mbps) + __field(unsigned long, us) + __field(int, wake) + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->mbps = mbps; + __entry->us = us; + __entry->wake = wake; + ), + + TP_printk("dev: %s, mbps = %lu, us = %lu, wake = %d", + __get_str(name), + __entry->mbps, + __entry->us, + __entry->wake) +); + +TRACE_EVENT(bw_hwmon_update, + + TP_PROTO(const char *name, unsigned long mbps, unsigned long freq, + unsigned long up_thres, unsigned long down_thres), + + TP_ARGS(name, mbps, freq, up_thres, down_thres), + + TP_STRUCT__entry( + __string(name, name) + __field(unsigned long, mbps) + __field(unsigned long, freq) + __field(unsigned long, up_thres) + __field(unsigned long, down_thres) + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->mbps = mbps; + __entry->freq = freq; + __entry->up_thres = up_thres; + __entry->down_thres = down_thres; + ), + + TP_printk("dev: %s, mbps = %lu, freq = %lu, up = %lu, down = %lu", + __get_str(name), + __entry->mbps, + __entry->freq, + __entry->up_thres, + __entry->down_thres) +); + +TRACE_EVENT(bw_hwmon_debug, + + TP_PROTO(const char *name, unsigned long mbps, unsigned long zone, + unsigned long hist_max, unsigned long hist_mem, + unsigned long hyst_mbps, unsigned long hyst_len), + + TP_ARGS(name, mbps, zone, hist_max, hist_mem, hyst_mbps, hyst_len), + + TP_STRUCT__entry( + __string(name, name) + __field(unsigned long, mbps) + __field(unsigned long, zone) + __field(unsigned long, hist_max) + __field(unsigned long, hist_mem) + __field(unsigned long, hyst_mbps) + __field(unsigned long, hyst_len) + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->mbps = mbps; + __entry->zone = zone; + __entry->hist_max = hist_max; + __entry->hist_mem = hist_mem; + __entry->hyst_mbps = hyst_mbps; + __entry->hyst_len = hyst_len; + ), + + TP_printk("dev=%s mbps=%lu zone=%lu hist_max=%lu hist_mem=%lu hyst_mbps=%lu hyst_len=%lu", + __get_str(name), + __entry->mbps, + __entry->zone, + __entry->hist_max, + __entry->hist_mem, + __entry->hyst_mbps, + __entry->hyst_len) +); + +#endif /* _TRACE_DCVS_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace-dcvs + +#include diff --git a/include/linux/scmi_c1dcvs.h b/include/linux/scmi_c1dcvs.h new file mode 100644 index 000000000000..07bfe8145706 --- /dev/null +++ b/include/linux/scmi_c1dcvs.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * scmi c1dcvs protocols header + * + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _SCMI_C1DCVS_H +#define _SCMI_C1DCVS_H + +#include +#include +#include + + +#define SCMI_C1DCVS_PROTOCOL 0x87 + +struct scmi_protocol_handle; + +/** + * struct scmi_c1dcvs_vendor_ops - represents the various operations provided + * by scmi c1dcvs protocol + */ +struct scmi_c1dcvs_vendor_ops { + int (*set_enable_c1dcvs)(const struct scmi_protocol_handle *ph, void *buf); + int (*get_enable_c1dcvs)(const struct scmi_protocol_handle *ph, void *buf); + int (*set_enable_trace)(const struct scmi_protocol_handle *ph, void *buf); + int (*get_enable_trace)(const struct scmi_protocol_handle *ph, void *buf); + int (*set_ipc_thresh)(const struct scmi_protocol_handle *ph, void *buf); + int (*get_ipc_thresh)(const struct scmi_protocol_handle *ph, void *buf); + int (*set_efreq_thresh)(const struct scmi_protocol_handle *ph, void *buf); + int (*get_efreq_thresh)(const struct scmi_protocol_handle *ph, void *buf); + int (*set_hysteresis)(const struct scmi_protocol_handle *ph, void *buf); + int (*get_hysteresis)(const struct scmi_protocol_handle *ph, void *buf); +}; + +#if IS_ENABLED(CONFIG_QTI_C1DCVS_SCMI_CLIENT) +int c1dcvs_enable(bool enable); +#else +static inline int c1dcvs_enable(bool enable) +{ + return -ENODEV; +} +#endif + +#endif /* _SCMI_C1DCVS_H */ + diff --git a/include/linux/scmi_memlat.h b/include/linux/scmi_memlat.h new file mode 100644 index 000000000000..d92b0ac9fb56 --- /dev/null +++ b/include/linux/scmi_memlat.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * SCMI Vendor Protocols header + * + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _SCMI_MEMLAT_H +#define _SCMI_MEMLAT_H + +#include +#include +#include + + +#define SCMI_PROTOCOL_MEMLAT 0x80 +#define MAX_EV_CNTRS 4 /* Maximum number of grp or common events */ +#define MAX_NAME_LEN 20 + +struct scmi_protocol_handle; +/** + * struct scmi_memlat_vendor_ops - represents the various operations provided + * by SCMI HW Memlat Protocol + */ +struct scmi_memlat_vendor_ops { + int (*set_mem_grp)(const struct scmi_protocol_handle *ph, + u32 cpus_mpidr, u32 hw_type); + int (*set_mon)(const struct scmi_protocol_handle *ph, u32 cpus_mpidr, + u32 hw_type, u32 mon_type, u32 index, const char *mon_name); + int (*set_grp_ev_map)(const struct scmi_protocol_handle *ph, u32 hw_type, + void *buf, u32 num_evs); + int (*adaptive_low_freq)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*adaptive_high_freq)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*get_adaptive_cur_freq)(const struct scmi_protocol_handle *ph, u32 hw_type, + u32 mon_idx, void *buf); + int (*set_common_ev_map)(const struct scmi_protocol_handle *ph, void *buf, + u32 num_evs); + int (*ipm_ceil)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*fe_stall_floor)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*be_stall_floor)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*wb_pct_thres)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*wb_filter_ipm)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*freq_scale_pct)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*freq_scale_limit_mhz)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*sample_ms)(const struct scmi_protocol_handle *ph, u32 val); + int (*freq_map)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 nr_rows, void *buf); + int (*min_freq)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*max_freq)(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 index, u32 val); + int (*get_cur_freq)(const struct scmi_protocol_handle *ph, u32 hw_type, + u32 mon_idx, void *buf); + int (*start_timer)(const struct scmi_protocol_handle *ph); + int (*stop_timer)(const struct scmi_protocol_handle *ph); + int (*set_log_level)(const struct scmi_protocol_handle *ph, u32 val); + int (*flush_cpucp_log)(const struct scmi_protocol_handle *ph); + int (*get_timestamp)(const struct scmi_protocol_handle *ph, void *buf); +}; + +#endif /* _SCMI_MEMLAT_H */ diff --git a/include/linux/scmi_pmu.h b/include/linux/scmi_pmu.h new file mode 100644 index 000000000000..c281a150e86b --- /dev/null +++ b/include/linux/scmi_pmu.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * SCMI PMU Protocols header + * + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _SCMI_PMU_H +#define _SCMI_PMU_H + +#include +#include +#include + + +#define SCMI_PMU_PROTOCOL 0x86 +#define MAX_NUM_CPUS 8 + +enum cpucp_ev_idx { + CPU_CYC_EVT = 0, + CNT_CYC_EVT, + INST_RETIRED_EVT, + STALL_BACKEND_EVT, + L2D_CACHE_REFILL_EVT, + L2D_WB_EVT, + L3_CACHE_REFILL_EVT, + L3_ACCESS_EVT, + LLCC_CACHE_REFILL_EVT, + MAX_CPUCP_EVT, +}; + +struct scmi_protocol_handle; + +/** + * struct scmi_pmu_vendor_ops - represents the various operations provided + * by SCMI PMU Protocol + */ +struct scmi_pmu_vendor_ops { + int (*set_pmu_map)(const struct scmi_protocol_handle *ph, void *buf); + int (*set_enable_trace)(const struct scmi_protocol_handle *ph, void *buf); + int (*set_cache_enable)(const struct scmi_protocol_handle *ph, void *buf); +}; + +#endif /* _SCMI_PMU_H */ diff --git a/include/soc/qcom/dcvs.h b/include/soc/qcom/dcvs.h new file mode 100644 index 000000000000..963a1f408af1 --- /dev/null +++ b/include/soc/qcom/dcvs.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_DCVS_H +#define _QCOM_DCVS_H + +#include + +#define QCOM_DCVS_WIDTH_PROP "qcom,bus-width" +#define QCOM_DCVS_HW_PROP "qcom,dcvs-hw-type" +#define QCOM_DCVS_PATH_PROP "qcom,dcvs-path-type" + +enum dcvs_hw_type { + DCVS_DDR, + DCVS_LLCC, + DCVS_L3, + DCVS_DDRQOS, + NUM_DCVS_HW_TYPES +}; + +enum dcvs_path_type { + DCVS_SLOW_PATH, + DCVS_FAST_PATH, + DCVS_PERCPU_PATH, + NUM_DCVS_PATHS +}; + +struct dcvs_freq { + u32 ib; + u32 ab; + enum dcvs_hw_type hw_type; +}; + +#if IS_ENABLED(CONFIG_QCOM_DCVS) +int qcom_dcvs_register_voter(const char *name, enum dcvs_hw_type hw_type, + enum dcvs_path_type path_type); +int qcom_dcvs_unregister_voter(const char *name, enum dcvs_hw_type hw_type, + enum dcvs_path_type path_type); +int qcom_dcvs_update_votes(const char *name, struct dcvs_freq *votes, + u32 update_mask, enum dcvs_path_type path); +struct kobject *qcom_dcvs_kobject_get(enum dcvs_hw_type type); +int qcom_dcvs_hw_minmax_get(enum dcvs_hw_type hw_type, u32 *min, u32 *max); +struct device_node *qcom_dcvs_get_ddr_child_node(struct device_node *of_parent); +#else +static inline int qcom_dcvs_register_voter(const char *name, + enum dcvs_hw_type hw_type, enum dcvs_path_type path_type) +{ + return -ENODEV; +} +static inline int qcom_dcvs_unregister_voter(const char *name, + enum dcvs_hw_type hw_type, enum dcvs_path_type path_type) +{ + return -ENODEV; +} +static inline int qcom_dcvs_update_votes(const char *name, + struct dcvs_freq *votes, u32 update_mask, + enum dcvs_path_type path) +{ + return -ENODEV; +} +static inline struct kobject *qcom_dcvs_kobject_get(enum dcvs_hw_type type) +{ + return NULL; +} +static inline int qcom_dcvs_hw_minmax_get(enum dcvs_hw_type hw_type, u32 *min, + u32 *max) +{ + return -ENODEV; +} +static inline struct device_node *qcom_dcvs_get_ddr_child_node( + struct device_node *of_parent) +{ + return NULL; +} +#endif + +#endif /* _QCOM_DCVS_H */ diff --git a/include/soc/qcom/pmu_lib.h b/include/soc/qcom/pmu_lib.h new file mode 100644 index 000000000000..0b04cd3de80b --- /dev/null +++ b/include/soc/qcom/pmu_lib.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_PMU_H +#define _QCOM_PMU_H + +#include +#include +#include + +/* (1) ccntr + (6) evcntr + (1) llcc */ +#define QCOM_PMU_MAX_EVS 8 +#define INVALID_PMU_HW_IDX 0xFF + +struct cpucp_hlos_map { + bool shared; + unsigned long cpus; +}; + +struct qcom_pmu_data { + u32 event_ids[QCOM_PMU_MAX_EVS]; + u64 ev_data[QCOM_PMU_MAX_EVS]; + u32 num_evs; +}; + +typedef void (*idle_fn_t)(struct qcom_pmu_data *data, int cpu, int state); +struct qcom_pmu_notif_node { + idle_fn_t idle_cb; + struct list_head node; +}; + +enum amu_counters { + SYS_AMU_CONST_CYC, + SYS_AMU_CORE_CYC, + SYS_AMU_INST_RET, + SYS_AMU_STALL_MEM, + SYS_AMU_MAX, +}; + +#if IS_ENABLED(CONFIG_QCOM_PMU_LIB) +int qcom_pmu_event_supported(u32 event_id, int cpu); +int qcom_get_cpucp_id(u32 event_id, int cpu); +int qcom_pmu_read(int cpu, u32 event_id, u64 *pmu_data); +int qcom_pmu_read_local(u32 event_id, u64 *pmu_data); +int qcom_pmu_read_all(int cpu, struct qcom_pmu_data *data); +int qcom_pmu_read_all_local(struct qcom_pmu_data *data); +int qcom_pmu_idle_register(struct qcom_pmu_notif_node *idle_node); +int qcom_pmu_idle_unregister(struct qcom_pmu_notif_node *idle_node); +int rimps_pmu_init(struct scmi_device *sdev); +#else +static inline int qcom_pmu_event_supported(u32 event_id, int cpu) +{ + return -ENODEV; +} +static inline int qcom_get_cpucp_id(u32 event_id, int cpu) +{ + return -ENODEV; +} +static inline int qcom_pmu_read(int cpu, u32 event_id, u64 *pmu_data) +{ + return -ENODEV; +} +static inline int qcom_pmu_read_local(u32 event_id, u64 *pmu_data) +{ + return -ENODEV; +} +static inline int qcom_pmu_read_all(int cpu, struct qcom_pmu_data *data) +{ + return -ENODEV; +} +static inline int qcom_pmu_read_all_local(struct qcom_pmu_data *data) +{ + return -ENODEV; +} +static inline int qcom_pmu_idle_register(struct qcom_pmu_notif_node *idle_node) +{ + return -ENODEV; +} +static inline int qcom_pmu_idle_unregister( + struct qcom_pmu_notif_node *idle_node) +{ + return -ENODEV; +} +static inline int rimps_pmu_init(struct scmi_device *sdev) +{ + return -ENODEV; +} +#endif + +#endif /* _QCOM_PMU_H */ diff --git a/include/soc/qcom/qcom_llcc_pmu.h b/include/soc/qcom/qcom_llcc_pmu.h new file mode 100644 index 000000000000..9506422db0df --- /dev/null +++ b/include/soc/qcom/qcom_llcc_pmu.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_LLCC_PMU_H +#define _QCOM_LLCC_PMU_H + +#include + +#define QCOM_LLCC_PMU_RD_EV 0x1000 + +#if IS_ENABLED(CONFIG_QCOM_LLCC_PMU) +int qcom_llcc_pmu_hw_type(u32 *type); +#else +static inline int qcom_llcc_pmu_hw_type(u32 *type) +{ + return 0; +} +#endif + +#endif /* _QCOM_LLCC_PMU_H */ From 0076ed22a390794f36cc739de0adae050ff48bb7 Mon Sep 17 00:00:00 2001 From: Amir Vajid Date: Mon, 29 Aug 2022 23:03:01 -0700 Subject: [PATCH 2/3] drivers: firmware: arm_scmi: Add snapshot of dcvs scmi drivers Add dcvs scmi drivers. This snapshot is taken as of msm-5.15 commit 250869d2bdc2192 ("sched: improve the scheduler"). Change-Id: Id8408e23a11c60de4ac7fa7e3205f907af68ea08 Signed-off-by: Amir Vajid --- drivers/firmware/arm_scmi/Kconfig | 35 ++ drivers/firmware/arm_scmi/Makefile | 3 + drivers/firmware/arm_scmi/c1dcvs_vendor.c | 164 +++++++++ drivers/firmware/arm_scmi/memlat_vendor.c | 387 ++++++++++++++++++++++ drivers/firmware/arm_scmi/pmu_vendor.c | 106 ++++++ 5 files changed, 695 insertions(+) create mode 100644 drivers/firmware/arm_scmi/c1dcvs_vendor.c create mode 100644 drivers/firmware/arm_scmi/memlat_vendor.c create mode 100644 drivers/firmware/arm_scmi/pmu_vendor.c diff --git a/drivers/firmware/arm_scmi/Kconfig b/drivers/firmware/arm_scmi/Kconfig index a14f65444b35..c5871a57630e 100644 --- a/drivers/firmware/arm_scmi/Kconfig +++ b/drivers/firmware/arm_scmi/Kconfig @@ -134,6 +134,41 @@ config ARM_SCMI_TRANSPORT_VIRTIO_ATOMIC_ENABLE in atomic context too, at the price of using a number of busy-waiting primitives all over instead. If unsure say N. +config QTI_SCMI_PMU_PROTOCOL + tristate "Qualcomm Technologies, Inc. SCMI PMU vendor Protocol" + depends on ARM || ARM64 || COMPILE_TEST + depends on ARM_SCMI_PROTOCOL && QCOM_CPUCP + help + System Control and Management Interface (SCMI) pmu vendor protocol. + This protocol provides interface to communicate with micro controller + which maintains the PMU configuration for multiple clients. + + This driver defines the comands or message ID's used for this + communication and also exposes the ops used by clients. + +config QTI_SCMI_C1DCVS_PROTOCOL + tristate "Qualcomm Technologies, Inc. SCMI C1DCVS vendor Protocol" + depends on ARM || ARM64 || COMPILE_TEST + depends on ARM_SCMI_PROTOCOL && QCOM_CPUCP + help + System Control and Management Interface (SCMI) c1dcvs vendor protocol. + This protocol provides interface to communicate with micro controller + which maintains the c1dcvs algorithm. + + This driver defines the comands or message ID's used for this + communication and also exposes the ops used by clients. + +config QTI_SCMI_MEMLAT_PROTOCOL + tristate "Qualcomm Technologies, Inc. SCMI MEMLAT vendor Protocol" + depends on ARM || ARM64 || COMPILE_TEST + depends on ARM_SCMI_PROTOCOL && QCOM_CPUCP + help + System Control and Management Interface (SCMI) memlat vendor protocol. + This protocol provides interface to communicate with micro controller + which is executing the hw memlat governor. This driver defines the + commands or message ID's used for this communication and also exposes + the ops used by clients. + endif #ARM_SCMI_PROTOCOL config ARM_SCMI_POWER_DOMAIN diff --git a/drivers/firmware/arm_scmi/Makefile b/drivers/firmware/arm_scmi/Makefile index 9ea86f8cc8f7..d59800b17943 100644 --- a/drivers/firmware/arm_scmi/Makefile +++ b/drivers/firmware/arm_scmi/Makefile @@ -13,6 +13,9 @@ scmi-module-objs := $(scmi-bus-y) $(scmi-driver-y) $(scmi-protocols-y) \ obj-$(CONFIG_ARM_SCMI_PROTOCOL) += scmi-module.o obj-$(CONFIG_ARM_SCMI_POWER_DOMAIN) += scmi_pm_domain.o obj-$(CONFIG_ARM_SCMI_POWER_CONTROL) += scmi_power_control.o +obj-$(CONFIG_QTI_SCMI_PMU_PROTOCOL) += pmu_vendor.o +obj-$(CONFIG_QTI_SCMI_C1DCVS_PROTOCOL) += c1dcvs_vendor.o +obj-$(CONFIG_QTI_SCMI_MEMLAT_PROTOCOL) += memlat_vendor.o ifeq ($(CONFIG_THUMB2_KERNEL)$(CONFIG_CC_IS_CLANG),yy) # The use of R7 in the SMCCC conflicts with the compiler's use of R7 as a frame diff --git a/drivers/firmware/arm_scmi/c1dcvs_vendor.c b/drivers/firmware/arm_scmi/c1dcvs_vendor.c new file mode 100644 index 000000000000..738b890fe001 --- /dev/null +++ b/drivers/firmware/arm_scmi/c1dcvs_vendor.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include "common.h" + +#define SCMI_MAX_RX_SIZE 128 + +enum scmi_c1dcvs_protocol_cmd { + SET_ENABLE_C1DCVS = 11, + GET_ENABLE_C1DCVS, + SET_ENABLE_TRACE, + GET_ENABLE_TRACE, + SET_IPC_THRESH, + GET_IPC_THRESH, + SET_EFREQ_THRESH, + GET_EFREQ_THRESH, + SET_HYSTERESIS, + GET_HYSTERESIS, +}; + +struct c1dcvs_thresh { + unsigned int cluster; + unsigned int thresh; +}; + +static int scmi_send_tunable_c1dcvs(const struct scmi_protocol_handle *ph, + void *buf, u32 msg_id) +{ + int ret; + struct scmi_xfer *t; + unsigned int *msg; + unsigned int *src = buf; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), sizeof(*msg), + &t); + if (ret) + return ret; + + msg = t->tx.buf; + *msg = cpu_to_le32(*src); + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_get_tunable_c1dcvs(const struct scmi_protocol_handle *ph, + void *buf, u32 msg_id) +{ + int ret; + struct scmi_xfer *t; + struct c1dcvs_thresh *msg; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), SCMI_MAX_RX_SIZE, + &t); + if (ret) + return ret; + + ret = ph->xops->do_xfer(ph, t); + memcpy(buf, t->rx.buf, t->rx.len); + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_send_thresh_c1dcvs(const struct scmi_protocol_handle *ph, + void *buf, u32 msg_id) +{ + int ret; + struct scmi_xfer *t; + struct c1dcvs_thresh *msg; + unsigned int *src = buf; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), sizeof(*msg), + &t); + if (ret) + return ret; + + msg = t->tx.buf; + msg->cluster = cpu_to_le32(src[0]); + msg->thresh = cpu_to_le32(src[1]); + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_set_enable_c1dcvs(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_send_tunable_c1dcvs(ph, buf, SET_ENABLE_C1DCVS); +} +static int scmi_get_enable_c1dcvs(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_get_tunable_c1dcvs(ph, buf, GET_ENABLE_C1DCVS); +} +static int scmi_set_enable_trace(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_send_tunable_c1dcvs(ph, buf, SET_ENABLE_TRACE); +} +static int scmi_get_enable_trace(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_get_tunable_c1dcvs(ph, buf, GET_ENABLE_TRACE); +} +static int scmi_set_ipc_thresh(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_send_thresh_c1dcvs(ph, buf, SET_IPC_THRESH); +} +static int scmi_get_ipc_thresh(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_get_tunable_c1dcvs(ph, buf, GET_IPC_THRESH); +} +static int scmi_set_efreq_thresh(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_send_thresh_c1dcvs(ph, buf, SET_EFREQ_THRESH); +} +static int scmi_get_efreq_thresh(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_get_tunable_c1dcvs(ph, buf, GET_EFREQ_THRESH); +} +static int scmi_set_hysteresis(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_send_tunable_c1dcvs(ph, buf, SET_HYSTERESIS); +} +static int scmi_get_hysteresis(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_get_tunable_c1dcvs(ph, buf, GET_HYSTERESIS); +} + +static struct scmi_c1dcvs_vendor_ops c1dcvs_config_ops = { + .set_enable_c1dcvs = scmi_set_enable_c1dcvs, + .get_enable_c1dcvs = scmi_get_enable_c1dcvs, + .set_enable_trace = scmi_set_enable_trace, + .get_enable_trace = scmi_get_enable_trace, + .set_ipc_thresh = scmi_set_ipc_thresh, + .get_ipc_thresh = scmi_get_ipc_thresh, + .set_efreq_thresh = scmi_set_efreq_thresh, + .get_efreq_thresh = scmi_get_efreq_thresh, + .set_hysteresis = scmi_set_hysteresis, + .get_hysteresis = scmi_get_hysteresis, +}; + +static int scmi_c1dcvs_protocol_init(const struct scmi_protocol_handle *ph) +{ + u32 version; + + ph->xops->version_get(ph, &version); + + dev_dbg(ph->dev, "version %d.%d\n", + PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version)); + + return 0; +} + +static const struct scmi_protocol scmi_c1dcvs = { + .id = SCMI_C1DCVS_PROTOCOL, + .owner = THIS_MODULE, + .instance_init = &scmi_c1dcvs_protocol_init, + .ops = &c1dcvs_config_ops, +}; +module_scmi_protocol(scmi_c1dcvs); + +MODULE_DESCRIPTION("SCMI C1DCVS vendor Protocol"); +MODULE_LICENSE("GPL"); diff --git a/drivers/firmware/arm_scmi/memlat_vendor.c b/drivers/firmware/arm_scmi/memlat_vendor.c new file mode 100644 index 000000000000..15abf1cf65f7 --- /dev/null +++ b/drivers/firmware/arm_scmi/memlat_vendor.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include "common.h" + +#define MAX_MAP_ENTRIES 14 + +#define SCMI_VENDOR_MSG_START (3) +#define SCMI_VENDOR_MSG_MODULE_START (16) +#define SCMI_MAX_RX_SIZE 128 +#define SCMI_MAX_GET_DATA_SIZE 124 +#define INVALID_IDX 0xFF + +enum scmi_memlat_protocol_cmd { + MEMLAT_SET_LOG_LEVEL = SCMI_VENDOR_MSG_START, + MEMLAT_FLUSH_LOGBUF, + MEMLAT_SET_MEM_GROUP = SCMI_VENDOR_MSG_MODULE_START, + MEMLAT_SET_MONITOR, + MEMLAT_SET_COMMON_EV_MAP, + MEMLAT_SET_GRP_EV_MAP, + MEMLAT_ADAPTIVE_LOW_FREQ, + MEMLAT_ADAPTIVE_HIGH_FREQ, + MEMLAT_GET_ADAPTIVE_CUR_FREQ, + MEMLAT_IPM_CEIL, + MEMLAT_FE_STALL_FLOOR, + MEMLAT_BE_STALL_FLOOR, + MEMLAT_WB_PCT, + MEMLAT_IPM_FILTER, + MEMLAT_FREQ_SCALE_PCT, + MEMLAT_FREQ_SCALE_LIMIT_MHZ, + MEMLAT_SAMPLE_MS, + MEMLAT_MON_FREQ_MAP, + MEMLAT_SET_MIN_FREQ, + MEMLAT_SET_MAX_FREQ, + MEMLAT_GET_CUR_FREQ, + MEMLAT_START_TIMER, + MEMLAT_STOP_TIMER, + MEMLAT_GET_TIMESTAMP, + MEMLAT_MAX_MSG +}; + +struct node_msg { + uint32_t cpumask; + uint32_t hw_type; + uint32_t mon_type; + uint32_t mon_idx; + char mon_name[MAX_NAME_LEN]; +}; + +struct scalar_param_msg { + uint32_t hw_type; + uint32_t mon_idx; + uint32_t val; +}; + +struct map_table { + uint32_t v1; + uint32_t v2; +}; + +struct map_param_msg { + uint32_t hw_type; + uint32_t mon_idx; + uint32_t nr_rows; + struct map_table tbl[MAX_MAP_ENTRIES]; +}; + +struct ev_map_msg { + uint32_t num_evs; + uint32_t hw_type; + uint8_t cid[MAX_EV_CNTRS]; +}; + +static int scmi_set_ev_map(const struct scmi_protocol_handle *ph, u32 hw_type, + void *buf, u32 msg_id, u32 num_evs) +{ + int ret, i = 0; + struct scmi_xfer *t; + struct ev_map_msg *msg; + uint8_t *src = buf; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), sizeof(*msg), + &t); + if (ret) + return ret; + + msg = t->tx.buf; + msg->num_evs = cpu_to_le32(num_evs); + msg->hw_type = cpu_to_le32(hw_type); + + for (i = 0; i < num_evs; i++) + msg->cid[i] = src[i]; + + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_set_grp_map(const struct scmi_protocol_handle *ph, u32 hw_type, + void *buf, u32 num_evs) +{ + return scmi_set_ev_map(ph, hw_type, buf, MEMLAT_SET_GRP_EV_MAP, num_evs); +} + +static int scmi_set_common_map(const struct scmi_protocol_handle *ph, void *buf, u32 num_evs) +{ + return scmi_set_ev_map(ph, INVALID_IDX, buf, MEMLAT_SET_COMMON_EV_MAP, num_evs); +} + +static int scmi_set_memgrp_mon(const struct scmi_protocol_handle *ph, + u32 cpus_mpidr, u32 hw_type, u32 mon_type, + u32 mon_idx, const char *mon_name, u32 msg_id) +{ + int ret = 0; + struct scmi_xfer *t; + struct node_msg *msg; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), sizeof(*msg), + &t); + if (ret) + return ret; + + msg = t->tx.buf; + msg->cpumask = cpu_to_le32(cpus_mpidr); + msg->hw_type = cpu_to_le32(hw_type); + msg->mon_type = cpu_to_le32(mon_type); + msg->mon_idx = cpu_to_le32(mon_idx); + if (mon_name) + snprintf(msg->mon_name, MAX_NAME_LEN, mon_name); + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_set_mon(const struct scmi_protocol_handle *ph, u32 cpus_mpidr, + u32 hw_type, u32 mon_type, u32 mon_idx, const char *mon_name) +{ + return scmi_set_memgrp_mon(ph, cpus_mpidr, hw_type, mon_type, + mon_idx, mon_name, MEMLAT_SET_MONITOR); +} + +static int scmi_set_mem_grp(const struct scmi_protocol_handle *ph, + u32 cpus_mpidr, u32 hw_type) +{ + return scmi_set_memgrp_mon(ph, cpus_mpidr, hw_type, 0, + 0, NULL, MEMLAT_SET_MEM_GROUP); +} + +static int scmi_freq_map(const struct scmi_protocol_handle *ph, u32 hw_type, + u32 mon_idx, u32 nr_rows, void *buf) +{ + int ret, i = 0; + struct scmi_xfer *t; + struct map_param_msg *msg; + struct map_table *tbl, *src = buf; + + if (nr_rows > MAX_MAP_ENTRIES) + return -EINVAL; + + ret = ph->xops->xfer_get_init(ph, MEMLAT_MON_FREQ_MAP, sizeof(*msg), + sizeof(*msg), &t); + if (ret) + return ret; + + msg = t->tx.buf; + msg->hw_type = cpu_to_le32(hw_type); + msg->mon_idx = cpu_to_le32(mon_idx); + msg->nr_rows = cpu_to_le32(nr_rows); + tbl = msg->tbl; + + for (i = 0; i < nr_rows; i++) { + tbl[i].v1 = cpu_to_le32(src[i].v1); + tbl[i].v2 = cpu_to_le32(src[i].v2); + } + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_set_tunable(const struct scmi_protocol_handle *ph, + u32 hw_type, u32 msg_id, u32 mon_idx, u32 val) +{ + int ret = 0; + struct scmi_xfer *t; + struct scalar_param_msg *msg; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), + sizeof(*msg), &t); + if (ret) + return ret; + msg = t->tx.buf; + msg->hw_type = cpu_to_le32(hw_type); + msg->mon_idx = cpu_to_le32(mon_idx); + msg->val = cpu_to_le32(val); + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + + return ret; +} + +#define scmi_send_cmd(name, _msg_id) \ +static int scmi_##name(const struct scmi_protocol_handle *ph, \ + u32 hw_type, u32 mon_idx, u32 val) \ +{ \ + return scmi_set_tunable(ph, hw_type, _msg_id, mon_idx, val); \ +} \ + +scmi_send_cmd(ipm_ceil, MEMLAT_IPM_CEIL); +scmi_send_cmd(fe_stall_floor, MEMLAT_FE_STALL_FLOOR); +scmi_send_cmd(be_stall_floor, MEMLAT_BE_STALL_FLOOR); +scmi_send_cmd(wb_pct_thres, MEMLAT_WB_PCT); +scmi_send_cmd(wb_filter_ipm, MEMLAT_IPM_FILTER); +scmi_send_cmd(freq_scale_pct, MEMLAT_FREQ_SCALE_PCT); +scmi_send_cmd(freq_scale_limit_mhz, MEMLAT_FREQ_SCALE_LIMIT_MHZ); +scmi_send_cmd(min_freq, MEMLAT_SET_MIN_FREQ); +scmi_send_cmd(max_freq, MEMLAT_SET_MAX_FREQ); +scmi_send_cmd(adaptive_low_freq, MEMLAT_ADAPTIVE_LOW_FREQ); +scmi_send_cmd(adaptive_high_freq, MEMLAT_ADAPTIVE_HIGH_FREQ); + +static int scmi_send_start_stop(const struct scmi_protocol_handle *ph, u32 msg_id) +{ + int ret = 0; + struct scmi_xfer *t; + + ret = ph->xops->xfer_get_init(ph, msg_id, 0, 0, &t); + if (ret) + return ret; + + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_stop_timer(const struct scmi_protocol_handle *ph) +{ + return scmi_send_start_stop(ph, MEMLAT_STOP_TIMER); +} + +static int scmi_start_timer(const struct scmi_protocol_handle *ph) +{ + return scmi_send_start_stop(ph, MEMLAT_START_TIMER); +} + +static int scmi_flush_cpucp_log(const struct scmi_protocol_handle *ph) +{ + return scmi_send_start_stop(ph, MEMLAT_FLUSH_LOGBUF); +} + +static int scmi_set_global_var(const struct scmi_protocol_handle *ph, u32 val, u32 msg_id) +{ + int ret = 0; + struct scmi_xfer *t; + u32 *ptr; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(u32), sizeof(u32), &t); + if (ret) + return ret; + ptr = (u32 *)t->tx.buf; + *ptr = cpu_to_le32(val); + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + + return ret; +} + +static int scmi_set_log_level(const struct scmi_protocol_handle *ph, u32 val) +{ + return scmi_set_global_var(ph, val, MEMLAT_SET_LOG_LEVEL); +} + +static int scmi_set_sample_ms(const struct scmi_protocol_handle *ph, u32 val) +{ + return scmi_set_global_var(ph, val, MEMLAT_SAMPLE_MS); +} + +static int scmi_get_timestamp(const struct scmi_protocol_handle *ph, void *buf) +{ + int ret = 0; + struct scmi_xfer *t; + + ret = ph->xops->xfer_get_init(ph, MEMLAT_GET_TIMESTAMP, sizeof(u32), + SCMI_MAX_RX_SIZE, &t); + if (ret) + return ret; + + ret = ph->xops->do_xfer(ph, t); + if (t->rx.len != sizeof(u64)) + return -EMSGSIZE; + + memcpy(buf, t->rx.buf, t->rx.len); + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_get_freq(const struct scmi_protocol_handle *ph, uint32_t hw_type, + uint32_t mon_idx, void *buf, uint32_t msg_id) +{ + int ret = 0; + struct scmi_xfer *t; + struct scalar_param_msg *msg; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), + SCMI_MAX_RX_SIZE, &t); + if (ret) + return ret; + + msg = t->tx.buf; + msg->hw_type = cpu_to_le32(hw_type); + msg->mon_idx = cpu_to_le32(mon_idx); + ret = ph->xops->do_xfer(ph, t); + if (t->rx.len != sizeof(u32)) + return -EMSGSIZE; + + memcpy(buf, t->rx.buf, t->rx.len); + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_get_cur_freq(const struct scmi_protocol_handle *ph, uint32_t hw_type, + uint32_t mon_idx, void *buf) +{ + return scmi_get_freq(ph, hw_type, mon_idx, buf, MEMLAT_GET_CUR_FREQ); +} + +static int scmi_get_adaptive_cur_freq(const struct scmi_protocol_handle *ph, uint32_t hw_type, + uint32_t mon_idx, void *buf) +{ + return scmi_get_freq(ph, hw_type, mon_idx, buf, MEMLAT_GET_ADAPTIVE_CUR_FREQ); +} + +static struct scmi_memlat_vendor_ops memlat_proto_ops = { + .set_mem_grp = scmi_set_mem_grp, + .freq_map = scmi_freq_map, + .set_mon = scmi_set_mon, + .set_common_ev_map = scmi_set_common_map, + .set_grp_ev_map = scmi_set_grp_map, + .adaptive_low_freq = scmi_adaptive_low_freq, + .adaptive_high_freq = scmi_adaptive_high_freq, + .get_adaptive_cur_freq = scmi_get_adaptive_cur_freq, + .ipm_ceil = scmi_ipm_ceil, + .fe_stall_floor = scmi_fe_stall_floor, + .be_stall_floor = scmi_be_stall_floor, + .sample_ms = scmi_set_sample_ms, + .wb_filter_ipm = scmi_wb_filter_ipm, + .wb_pct_thres = scmi_wb_pct_thres, + .freq_scale_pct = scmi_freq_scale_pct, + .freq_scale_limit_mhz = scmi_freq_scale_limit_mhz, + .min_freq = scmi_min_freq, + .max_freq = scmi_max_freq, + .get_cur_freq = scmi_get_cur_freq, + .start_timer = scmi_start_timer, + .stop_timer = scmi_stop_timer, + .set_log_level = scmi_set_log_level, + .flush_cpucp_log = scmi_flush_cpucp_log, + .get_timestamp = scmi_get_timestamp, +}; + +static int scmi_memlat_vendor_protocol_init(const struct scmi_protocol_handle *ph) +{ + u32 version; + + ph->xops->version_get(ph, &version); + + dev_dbg(ph->dev, "memlat version %d.%d\n", + PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version)); + + return 0; +} + +static const struct scmi_protocol scmi_memlat_vendor = { + .id = SCMI_PROTOCOL_MEMLAT, + .owner = THIS_MODULE, + .instance_init = &scmi_memlat_vendor_protocol_init, + .ops = &memlat_proto_ops, +}; +module_scmi_protocol(scmi_memlat_vendor); + +MODULE_DESCRIPTION("SCMI memlat vendor Protocol"); +MODULE_LICENSE("GPL"); diff --git a/drivers/firmware/arm_scmi/pmu_vendor.c b/drivers/firmware/arm_scmi/pmu_vendor.c new file mode 100644 index 000000000000..0becebf89d1d --- /dev/null +++ b/drivers/firmware/arm_scmi/pmu_vendor.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include "common.h" + +enum scmi_c1dcvs_protocol_cmd { + SET_PMU_MAP = 11, + SET_ENABLE_TRACE, + SET_ENABLE_CACHING, +}; + +struct pmu_map_msg { + uint8_t hw_cntrs[MAX_NUM_CPUS][MAX_CPUCP_EVT]; +}; + +static int scmi_send_pmu_map(const struct scmi_protocol_handle *ph, + void *buf, u32 msg_id) +{ + int ret, i, j; + struct scmi_xfer *t; + struct pmu_map_msg *msg; + uint8_t *src = buf; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), sizeof(*msg), + &t); + if (ret) + return ret; + + msg = t->tx.buf; + + for (i = 0; i < MAX_NUM_CPUS; i++) + for (j = 0; j < MAX_CPUCP_EVT; j++) + msg->hw_cntrs[i][j] = *((src + i * MAX_CPUCP_EVT) + j); + + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_send_tunable_pmu(const struct scmi_protocol_handle *ph, + void *buf, u32 msg_id) +{ + int ret; + struct scmi_xfer *t; + unsigned int *msg; + unsigned int *src = buf; + + ret = ph->xops->xfer_get_init(ph, msg_id, sizeof(*msg), sizeof(*msg), + &t); + if (ret) + return ret; + + msg = t->tx.buf; + *msg = cpu_to_le32(*src); + ret = ph->xops->do_xfer(ph, t); + ph->xops->xfer_put(ph, t); + return ret; +} + +static int scmi_pmu_map(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_send_pmu_map(ph, buf, SET_PMU_MAP); +} + +static int scmi_set_enable_trace(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_send_tunable_pmu(ph, buf, SET_ENABLE_TRACE); +} + +static int scmi_set_caching_enable(const struct scmi_protocol_handle *ph, void *buf) +{ + return scmi_send_tunable_pmu(ph, buf, SET_ENABLE_CACHING); +} + +static struct scmi_pmu_vendor_ops pmu_config_ops = { + .set_pmu_map = scmi_pmu_map, + .set_enable_trace = scmi_set_enable_trace, + .set_cache_enable = scmi_set_caching_enable, +}; + +static int scmi_pmu_protocol_init(const struct scmi_protocol_handle *ph) +{ + u32 version; + + ph->xops->version_get(ph, &version); + + dev_dbg(ph->dev, "version %d.%d\n", + PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version)); + + return 0; +} + +static const struct scmi_protocol scmi_pmu = { + .id = SCMI_PMU_PROTOCOL, + .owner = THIS_MODULE, + .instance_init = &scmi_pmu_protocol_init, + .ops = &pmu_config_ops, +}; +module_scmi_protocol(scmi_pmu); + +MODULE_DESCRIPTION("SCMI PMU vendor Protocol"); +MODULE_LICENSE("GPL"); From f59e8ca5a0c7715d07eea2388c514c08a53d0b52 Mon Sep 17 00:00:00 2001 From: Amir Vajid Date: Mon, 29 Aug 2022 23:03:05 -0700 Subject: [PATCH 3/3] drivers: soc: qcom: Add snapshot of cpucp communication drivers Add snapshot of cpucp mailbox and logging drivers from msm-5.15 commit 250869d2bdc2192 ("sched: improve the scheduler")'. Change-Id: Ibe7497e4c27b0d031b6f7857f0a24ff0035e85fa Signed-off-by: Amir Vajid --- drivers/soc/qcom/Kconfig | 21 ++ drivers/soc/qcom/Makefile | 2 + drivers/soc/qcom/cpucp_log.c | 351 +++++++++++++++++++++++++++++++++ drivers/soc/qcom/qcom_cpucp.c | 230 +++++++++++++++++++++ drivers/soc/qcom/trace_cpucp.h | 40 ++++ 5 files changed, 644 insertions(+) create mode 100644 drivers/soc/qcom/cpucp_log.c create mode 100644 drivers/soc/qcom/qcom_cpucp.c create mode 100644 drivers/soc/qcom/trace_cpucp.h diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index 9f8cba1fe784..81f33acdb3f2 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -87,6 +87,27 @@ config QCOM_MSM_IPCC Say Y here to compile the driver as a part of kernel or M to compile as a module. +config QCOM_CPUCP + tristate "Qualcomm Technologies, Inc. CPUCP driver" + depends on MAILBOX + help + Qualcomm Technologies, Inc. CPUCP driver for MSM devices. This driver + acts as a mailbox controller to do doorbell between APSS and CPUCP + subsystem. Say yes here to enable rx and tx channel between both + the subsystems. + If unsure, say n. + +config QTI_CPUCP_LOG + tristate "Qualcomm Technologies Inc. HW CPUCP Logging" + depends on QCOM_CPUCP + default n + help + CPUCP logging driver, this driver has the infra to collect logs + generated in CPUCP HW and log the buffers. + + This driver register with IPC_Logging framework, to have dedicated + buffer for cpucp hw device. + config QCOM_LLCC tristate "Qualcomm Technologies, Inc. LLCC driver" depends on ARCH_QCOM || COMPILE_TEST diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index a704d065fae0..0b730aae1b0a 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -60,3 +60,5 @@ obj-$(CONFIG_GH_TLMM_VM_MEM_ACCESS) += gh_tlmm_vm_mem_access.o obj-$(CONFIG_QCOM_RAMDUMP) += qcom_ramdump.o obj-$(CONFIG_QCOM_ICC_BWMON) += icc-bwmon.o obj-$(CONFIG_QCOM_DCVS) += dcvs/ +obj-$(CONFIG_QCOM_CPUCP) += qcom_cpucp.o +obj-$(CONFIG_QTI_CPUCP_LOG) += cpucp_log.o diff --git a/drivers/soc/qcom/cpucp_log.c b/drivers/soc/qcom/cpucp_log.c new file mode 100644 index 000000000000..d033db6a4797 --- /dev/null +++ b/drivers/soc/qcom/cpucp_log.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_PRINT_SIZE 1024 +#define MAX_BUF_NUM 4 +#define MAX_RESIDUAL_SIZE MAX_PRINT_SIZE +#define SIZE_ADJUST 4 +#define SRC_OFFSET 4 + +#define CREATE_TRACE_POINTS +#include "trace_cpucp.h" + +struct remote_mem { + void __iomem *start; + unsigned long long size; +}; + +struct cpucp_buf { + struct list_head node; + char *buf; + u32 size; + u32 cpy_idx; +}; + +struct cpucp_log_info { + struct remote_mem *rmem; + struct mbox_client cl; + struct mbox_chan *ch; + struct delayed_work work; + struct device *dev; + void __iomem *base; + unsigned int rmem_idx; + unsigned int num_bufs; + unsigned int total_buf_size; + char *rem_buf; + char *glb_buf; + int rem_len; + spinlock_t free_list_lock; + spinlock_t full_list_lock; +}; + +static LIST_HEAD(full_buffers_list); +static LIST_HEAD(free_buffers_list); +static struct workqueue_struct *cpucp_wq; + +static inline bool get_last_newline(char *buf, int size, int *cnt) +{ + int i; + + for (i = (size - 1); i >= 0 ; i--) { + if (buf[i] == '\n') { + buf[i] = '\0'; + *cnt = i + 1; + return true; + } + } + + *cnt = size; + return false; +} + +static void cpucp_log_work(struct work_struct *work) +{ + struct cpucp_log_info *info = container_of(work, + struct cpucp_log_info, + work.work); + char *src; + int buf_start = 0; + int cnt = 0, print_size = 0, buf_size = 0; + bool ret; + char tmp_buf[MAX_PRINT_SIZE + 1]; + struct cpucp_buf *buf_node; + unsigned long flags; + + while (1) { + spin_lock_irqsave(&info->full_list_lock, flags); + if (list_empty(&full_buffers_list)) { + spin_unlock_irqrestore(&info->full_list_lock, flags); + return; + } + buf_node = list_first_entry(&full_buffers_list, + struct cpucp_buf, node); + list_del(&buf_node->node); + spin_unlock_irqrestore(&info->full_list_lock, flags); + buf_start = buf_node->cpy_idx - info->rem_len; + src = &buf_node->buf[buf_start]; + buf_size = buf_node->size + info->rem_len; + if (info->rem_len) { + memcpy(&buf_node->buf[buf_start], + info->rem_buf, info->rem_len); + info->rem_len = 0; + } + do { + print_size = (buf_size >= MAX_PRINT_SIZE) ? + MAX_PRINT_SIZE : buf_size; + ret = get_last_newline(src, print_size, &cnt); + if (cnt == print_size) { + if (!ret && buf_size < MAX_PRINT_SIZE) { + info->rem_len = buf_size; + memcpy(info->rem_buf, src, buf_size); + goto out; + } else { + snprintf(tmp_buf, print_size + 1, "%s", src); + trace_cpucp_log(tmp_buf); + } + } else + trace_cpucp_log(src); + + buf_start += cnt; + buf_size -= cnt; + src = &buf_node->buf[buf_start]; + } while (buf_size > 0); + +out: + spin_lock_irqsave(&info->free_list_lock, flags); + list_add_tail(&buf_node->node, &free_buffers_list); + spin_unlock_irqrestore(&info->free_list_lock, flags); + } +} + +static struct cpucp_buf *get_free_buffer(struct cpucp_log_info *info) +{ + struct cpucp_buf *buf_node; + unsigned long flags; + + spin_lock_irqsave(&info->free_list_lock, flags); + if (list_empty(&free_buffers_list)) { + spin_unlock_irqrestore(&info->free_list_lock, flags); + return NULL; + } + + buf_node = list_first_entry(&free_buffers_list, + struct cpucp_buf, node); + list_del(&buf_node->node); + spin_unlock_irqrestore(&info->free_list_lock, flags); + return buf_node; +} + +static void cpucp_log_rx(struct mbox_client *client, void *msg) +{ + struct cpucp_log_info *info = dev_get_drvdata(client->dev); + struct device *dev = info->dev; + struct cpucp_buf *buf_node; + struct remote_mem *rmem; + void __iomem *src; + u32 marker; + unsigned long long rmem_size; + unsigned long flags; + int src_offset = 0; + int size_adj = 0; + + buf_node = get_free_buffer(info); + if (!buf_node) { + dev_err(dev, "global buffer full dropping buffers\n"); + return; + } + + marker = *(u32 *)(info->rmem)->start; + if (marker <= info->rmem->size) { + info->rmem_idx = 0; + rmem_size = marker; + } else if (marker <= info->total_buf_size) { + info->rmem_idx = 1; + rmem_size = marker - info->rmem->size; + } else { + pr_err("%s: Log marker incorrect: %u\n", __func__, marker); + return; + } + + if (info->rmem_idx == 0) { + size_adj = SIZE_ADJUST; + src_offset = SRC_OFFSET; + } + + rmem = info->rmem + info->rmem_idx; + rmem_size -= size_adj; + src = rmem->start + src_offset; + memcpy_fromio(&buf_node->buf[buf_node->cpy_idx], src, rmem_size); + buf_node->size = rmem_size; + spin_lock_irqsave(&info->full_list_lock, flags); + list_add_tail(&buf_node->node, &full_buffers_list); + spin_unlock_irqrestore(&info->full_list_lock, flags); + + if (!delayed_work_pending(&info->work)) + queue_delayed_work(cpucp_wq, &info->work, 0); +} + +static int populate_free_buffers(struct cpucp_log_info *info, + int rmem_size) +{ + int i = 0; + struct cpucp_buf *buf_nodes; + + buf_nodes = devm_kzalloc(info->dev, + MAX_BUF_NUM * sizeof(struct cpucp_buf), + GFP_KERNEL); + if (!buf_nodes) + return -ENOMEM; + + for (i = 0; i < MAX_BUF_NUM; i++) { + buf_nodes[i].buf = &info->glb_buf[i * (rmem_size + MAX_PRINT_SIZE)]; + buf_nodes[i].size = rmem_size; + buf_nodes[i].cpy_idx = MAX_PRINT_SIZE; + list_add_tail(&buf_nodes[i].node, &free_buffers_list); + } + return 0; +} + +static int cpucp_log_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct cpucp_log_info *info; + struct mbox_client *cl; + int ret, i = 0; + struct resource *res; + void __iomem *mem_base; + struct remote_mem *rmem; + int prev_size = 0; + + info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->dev = dev; + + rmem = kcalloc(pdev->num_resources, sizeof(struct remote_mem), + GFP_KERNEL); + if (!rmem) + return -ENOMEM; + + info->rmem = rmem; + + for (i = 0; i < pdev->num_resources; i++) { + struct remote_mem *rmem = &info->rmem[i]; + + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + if (!res) { + dev_err(dev, + "Failed to get the device base address\n"); + ret = -ENODEV; + goto exit; + } + + mem_base = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (IS_ERR(mem_base)) { + ret = PTR_ERR(mem_base); + dev_err(dev, "Failed to io remap the region err: %d\n", ret); + goto exit; + } + rmem->start = mem_base; + rmem->size = resource_size(res); + if (prev_size && (rmem->size != prev_size)) { + ret = -EINVAL; + goto exit; + } else if (!prev_size) { + prev_size = rmem->size; + } + + info->total_buf_size += rmem->size; + info->num_bufs++; + } + info->glb_buf = devm_kzalloc(dev, MAX_BUF_NUM * + (rmem->size + MAX_PRINT_SIZE), + GFP_KERNEL); + if (!info->glb_buf) { + ret = -ENOMEM; + goto exit; + } + + info->rem_buf = devm_kzalloc(dev, MAX_RESIDUAL_SIZE, GFP_KERNEL); + if (!info->rem_buf) { + ret = -ENOMEM; + goto exit; + } + + ret = populate_free_buffers(info, rmem->size); + if (ret < 0) + goto exit; + + cl = &info->cl; + cl->dev = dev; + cl->tx_block = false; + cl->knows_txdone = true; + cl->rx_callback = cpucp_log_rx; + + dev_set_drvdata(dev, info); + INIT_DEFERRABLE_WORK(&info->work, &cpucp_log_work); + spin_lock_init(&info->free_list_lock); + spin_lock_init(&info->full_list_lock); + cpucp_wq = create_freezable_workqueue("cpucp_wq"); + + info->ch = mbox_request_channel(cl, 0); + if (IS_ERR(info->ch)) { + ret = PTR_ERR(info->ch); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to request mbox info: %d\n", ret); + goto exit; + } + + dev_dbg(dev, "CPUCP logging initialized\n"); + + return 0; + +exit: + kfree(info->rmem); + return ret; +} + +static int cpucp_log_remove(struct platform_device *pdev) +{ + struct cpucp_log_info *info; + + info = dev_get_drvdata(&pdev->dev); + + mbox_free_channel(info->ch); + + return 0; +} + +static const struct of_device_id cpucp_log[] = { + {.compatible = "qcom,cpucp-log"}, + {}, +}; + +static struct platform_driver cpucp_log_driver = { + .driver = { + .name = "cpucp-log", + .of_match_table = cpucp_log, + }, + .probe = cpucp_log_probe, + .remove = cpucp_log_remove, +}; +builtin_platform_driver(cpucp_log_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/qcom_cpucp.c b/drivers/soc/qcom/qcom_cpucp.c new file mode 100644 index 000000000000..c3217d35bf2a --- /dev/null +++ b/drivers/soc/qcom/qcom_cpucp.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +/* CPUCP Register offsets */ +#define CPUCP_IPC_CHAN_SUPPORTED 2 +#define CPUCP_SEND_IRQ_OFFSET 0xC +#define CPUCP_SEND_IRQ_VAL BIT(28) +#define CPUCP_CLEAR_IRQ_OFFSET 0x308 +#define CPUCP_STATUS_IRQ_OFFSET 0x30C +#define CPUCP_CLEAR_IRQ_VAL BIT(3) +#define CPUCP_STATUS_IRQ_VAL BIT(3) +#define CPUCP_CLOCK_DOMAIN_OFFSET 0x1000 + +/** + * struct cpucp_ipc ipc per channel + * @mbox: mailbox-controller interface + * @chans: The mailbox clients' channel array + * @tx_irq_base: Memory address for sending irq + * @rx_irq_base: Memory address for receiving irq + * @dev: Device associated with this instance + * @irq: CPUCP to HLOS irq + * @num_chan: Number of ipc channels supported + */ +struct qcom_cpucp_ipc { + struct mbox_controller mbox; + struct mbox_chan chans[CPUCP_IPC_CHAN_SUPPORTED]; + void __iomem *tx_irq_base; + void __iomem *rx_irq_base; + struct device *dev; + int irq; + int num_chan; +}; + +static irqreturn_t qcom_cpucp_rx_interrupt(int irq, void *p) +{ + struct qcom_cpucp_ipc *cpucp_ipc; + u32 val; + int i; + + cpucp_ipc = p; + + for (i = 0; i < cpucp_ipc->num_chan; i++) { + + val = readl(cpucp_ipc->rx_irq_base + + CPUCP_STATUS_IRQ_OFFSET + (i * CPUCP_CLOCK_DOMAIN_OFFSET)); + if (val & CPUCP_STATUS_IRQ_VAL) { + + val = CPUCP_CLEAR_IRQ_VAL; + writel(val, cpucp_ipc->rx_irq_base + + CPUCP_CLEAR_IRQ_OFFSET + + (i * CPUCP_CLOCK_DOMAIN_OFFSET)); + /* Make sure reg write is complete before proceeding */ + mb(); + + if (cpucp_ipc->chans[i].con_priv) + mbox_chan_received_data(&cpucp_ipc->chans[i] + , NULL); + } + } + + return IRQ_HANDLED; +} + +static void qcom_cpucp_mbox_shutdown(struct mbox_chan *chan) +{ + chan->con_priv = NULL; +} + +static int qcom_cpucp_mbox_send_data(struct mbox_chan *chan, void *data) +{ + struct qcom_cpucp_ipc *cpucp_ipc = container_of(chan->mbox, + struct qcom_cpucp_ipc, mbox); + + writel(CPUCP_SEND_IRQ_VAL, + cpucp_ipc->tx_irq_base + CPUCP_SEND_IRQ_OFFSET); + return 0; +} + +static struct mbox_chan *qcom_cpucp_mbox_xlate(struct mbox_controller *mbox, + const struct of_phandle_args *sp) +{ + struct qcom_cpucp_ipc *cpucp_ipc = container_of(mbox, + struct qcom_cpucp_ipc, mbox); + unsigned long ind = sp->args[0]; + + if (sp->args_count != 1) + return ERR_PTR(-EINVAL); + + if (ind >= mbox->num_chans) + return ERR_PTR(-EINVAL); + + if (mbox->chans[ind].con_priv) + return ERR_PTR(-EBUSY); + + mbox->chans[ind].con_priv = cpucp_ipc; + return &mbox->chans[ind]; +} + +static const struct mbox_chan_ops cpucp_mbox_chan_ops = { + .send_data = qcom_cpucp_mbox_send_data, + .shutdown = qcom_cpucp_mbox_shutdown +}; + +static int qcom_cpucp_ipc_setup_mbox(struct qcom_cpucp_ipc *cpucp_ipc) +{ + struct mbox_controller *mbox; + struct device *dev = cpucp_ipc->dev; + unsigned long i; + + /* Initialize channel identifiers */ + for (i = 0; i < ARRAY_SIZE(cpucp_ipc->chans); i++) + cpucp_ipc->chans[i].con_priv = NULL; + + mbox = &cpucp_ipc->mbox; + mbox->dev = dev; + mbox->num_chans = cpucp_ipc->num_chan; + mbox->chans = cpucp_ipc->chans; + mbox->ops = &cpucp_mbox_chan_ops; + mbox->of_xlate = qcom_cpucp_mbox_xlate; + mbox->txdone_irq = false; + mbox->txdone_poll = false; + + return mbox_controller_register(mbox); +} + +static int qcom_cpucp_probe(struct platform_device *pdev) +{ + struct qcom_cpucp_ipc *cpucp_ipc; + struct resource *res; + int ret; + + cpucp_ipc = devm_kzalloc(&pdev->dev, sizeof(*cpucp_ipc), GFP_KERNEL); + if (!cpucp_ipc) + return -ENOMEM; + + cpucp_ipc->dev = &pdev->dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&pdev->dev, "Failed to get the device base address\n"); + return -ENODEV; + } + + cpucp_ipc->tx_irq_base = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!cpucp_ipc->tx_irq_base) { + dev_err(&pdev->dev, "Failed to ioremap cpucp tx irq addr\n"); + return -ENOMEM; + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!res) { + dev_err(&pdev->dev, "Failed to get the device base address\n"); + return -ENODEV; + } + + cpucp_ipc->rx_irq_base = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!cpucp_ipc->rx_irq_base) { + dev_err(&pdev->dev, "Failed to ioremap cpucp rx irq addr\n"); + return -ENOMEM; + } + + cpucp_ipc->irq = platform_get_irq(pdev, 0); + if (cpucp_ipc->irq < 0) { + dev_err(&pdev->dev, "Failed to get the IRQ\n"); + return cpucp_ipc->irq; + } + + cpucp_ipc->num_chan = CPUCP_IPC_CHAN_SUPPORTED; + ret = qcom_cpucp_ipc_setup_mbox(cpucp_ipc); + if (ret) { + dev_err(&pdev->dev, "Failed to create mailbox\n"); + return ret; + } + + ret = devm_request_irq(&pdev->dev, cpucp_ipc->irq, + qcom_cpucp_rx_interrupt, IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, + "qcom_cpucp", cpucp_ipc); + + if (ret < 0) { + dev_err(&pdev->dev, "Failed to register the irq: %d\n", ret); + goto err_mbox; + } + platform_set_drvdata(pdev, cpucp_ipc); + + return 0; + +err_mbox: + mbox_controller_unregister(&cpucp_ipc->mbox); + return ret; +} + +static int qcom_cpucp_remove(struct platform_device *pdev) +{ + struct qcom_cpucp_ipc *cpucp_ipc = platform_get_drvdata(pdev); + + mbox_controller_unregister(&cpucp_ipc->mbox); + + return 0; +} + +static const struct of_device_id qcom_cpucp_of_match[] = { + { .compatible = "qcom,cpucp"}, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_cpucp_of_match); + +static struct platform_driver qcom_cpucp_driver = { + .probe = qcom_cpucp_probe, + .remove = qcom_cpucp_remove, + .driver = { + .name = "qcom_cpucp", + .of_match_table = qcom_cpucp_of_match, + }, +}; +module_platform_driver(qcom_cpucp_driver); + +MODULE_DESCRIPTION("QTI CPUCP Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/trace_cpucp.h b/drivers/soc/qcom/trace_cpucp.h new file mode 100644 index 000000000000..e7ebd2c7732f --- /dev/null +++ b/drivers/soc/qcom/trace_cpucp.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + * + */ + +#if !defined(_TRACE_CPUCP_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_CPUCP_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM cpucp + +#include + +TRACE_EVENT(cpucp_log, + + TP_PROTO(char *str), + + TP_ARGS(str), + + TP_STRUCT__entry( + __string(str, str) + ), + + TP_fast_assign( + __assign_str(str, str); + ), + + TP_printk("%s\n", __get_str(str)) +); + +#endif /* _TRACE_CPUCP_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace_cpucp + +#include