From 106bcadd4a1a53ab0d987e96a957a9bb1e5c124e Mon Sep 17 00:00:00 2001 From: Lina Iyer Date: Mon, 5 Oct 2020 11:39:15 -0600 Subject: [PATCH 1/8] soc: qcom: rpmh: Add fast-path interface Add fast-path interface for making a lock-less RPMH request. The interface provides APIs that can be called from a interrupt/scheduler context. The _init_ call shall be used to initialize the dedicated TCS with {addr, data} and subsequent calls shall _update_ the data and trigger the request immediately. Only active state requests shall be sent through these interfaces. Requests to remove the vote during sleep and re-apply the vote during wake must use the regular rpmh_write/_async variants APIs (locking). Change-Id: I2505fc487ddca1a32185b116c0d8680818dfd2b8 Signed-off-by: Lina Iyer Signed-off-by: Maulik Shah --- drivers/soc/qcom/rpmh.c | 77 +++++++++++++++++++++++++++++++++++++++++ include/soc/qcom/rpmh.h | 16 ++++++++- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/drivers/soc/qcom/rpmh.c b/drivers/soc/qcom/rpmh.c index 43a06d93cd56..77d7ef0035c0 100644 --- a/drivers/soc/qcom/rpmh.c +++ b/drivers/soc/qcom/rpmh.c @@ -621,3 +621,80 @@ int rpmh_mode_solver_set(const struct device *dev, bool enable) return ret; } EXPORT_SYMBOL(rpmh_mode_solver_set); + + +/** + * RPMH Fast Path mode + * + * - Fast path mode is a lock-less request transmission to AOSS. + * - Requests may be sent from interrupt/scheduler context. + * - Dedicated fast-path TCS must be available in the RSC. + * - Error returned, when dedicated TCS is not available. + * - Only one client is expected to call RPMH fast path. + * - Serialization of requests is a responsibility of the client. + * - Only active state requests may be made through this mode. + * - Sleep/Wake requests for the nodes must be made through + * the standard RPMH APIs (locking modes). + * - Fast path requests are not cached and sent immediately. + * - Fast path requests are all blocking requests. + * - Fast path requests do not use AMC completion interrupts, + * therefore will not receive a tx_done callback. + */ + +/** + * rpmh_init_fast_path: Initialize TCS request in fast-path TCS + * + * @dev: The device making the request + * @cmd: The {addr, data} to be initialized with. + * @n: The number of @cmd + * + * Return: + * * 0 - Success + * * Error code - Otherwise + */ +int rpmh_init_fast_path(const struct device *dev, + struct tcs_cmd *cmd, int n) +{ + struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); + struct tcs_request req; + + if (rpmh_standalone) + return 0; + + req.cmds = cmd; + req.num_cmds = n; + req.wait_for_compl = 0; + + return rpmh_rsc_init_fast_path(ctrlr_to_drv(ctrlr), &req); +} +EXPORT_SYMBOL(rpmh_init_fast_path); + +/** + * rpmh_update_fast_path: Initialize TCS request in fast-path TCS + * + * @dev: The device making the request + * @cmd: The data to be updated. + * @n: The number of @cmd + * @update_mask: The elements in @cmd that only need to be updated + * + * Return: + * * 0 - Success + * * Error code - Otherwise + */ +int rpmh_update_fast_path(const struct device *dev, + struct tcs_cmd *cmd, int n, u32 update_mask) +{ + struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); + struct tcs_request req; + + if (rpmh_standalone) + return 0; + + req.cmds = cmd; + req.num_cmds = n; + req.wait_for_compl = 0; + + return rpmh_rsc_update_fast_path(ctrlr_to_drv(ctrlr), &req, + update_mask); +} +EXPORT_SYMBOL(rpmh_update_fast_path); diff --git a/include/soc/qcom/rpmh.h b/include/soc/qcom/rpmh.h index dd4fada3b7e2..ff1887452c64 100644 --- a/include/soc/qcom/rpmh.h +++ b/include/soc/qcom/rpmh.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved. */ #ifndef __SOC_QCOM_RPMH_H__ @@ -26,6 +26,11 @@ int rpmh_write_sleep_and_wake(const struct device *dev); void rpmh_invalidate(const struct device *dev); +int rpmh_init_fast_path(const struct device *dev, + struct tcs_cmd *cmd, int n); + +int rpmh_update_fast_path(const struct device *dev, + struct tcs_cmd *cmd, int n, u32 update_mask); #else static inline int rpmh_write(const struct device *dev, enum rpmh_state state, @@ -52,6 +57,15 @@ static inline void rpmh_invalidate(const struct device *dev) { } +static inline int rpmh_init_fast_path(const struct device *dev, + struct tcs_cmd *msg, int n) +{ return -ENODEV; } + +static inline int rpmh_update_fast_path(const struct device *dev, + struct tcs_cmd *msg, int n, + u32 update_mask) +{ return -ENODEV; } + #endif /* CONFIG_QCOM_RPMH */ #endif /* __SOC_QCOM_RPMH_H__ */ From 8ff0d62cc3f151fdd2b8ef1e5aa673191c4d5fea Mon Sep 17 00:00:00 2001 From: Maulik Shah Date: Mon, 9 May 2022 13:12:40 +0530 Subject: [PATCH 2/8] soc: qcom: rpmh-rsc: Add fast-path TCS to debug dump When we crash the system and display the TCS controller state, let's also dump the fast-path TCS information if there was an active transaction going on at that time. Change-Id: Iabc7b8af22dd148fca617e571f1392ba3d9a93ab Signed-off-by: Maulik Shah --- drivers/soc/qcom/rpmh-rsc.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index bc54853e4b4b..cbcfcb0ed4c9 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -760,14 +760,14 @@ static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, bool in_use = test_bit(tcs_id, drv->tcs_in_use); int i; - if (!tcs_grp || !req) - return; - sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); cmds_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); if (!cmds_enabled) return; + if (!tcs_grp || !req) + goto print_tcs_data; + data = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); irq_sts = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); pr_warn("Request: tcs-in-use:%s active_tcs=%s(%d) state=%d wait_for_compl=%u]\n", @@ -780,6 +780,7 @@ static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, *aoss_irq_sts = (irq_sts & BIT(tcs_id)) ? true : false; +print_tcs_data: for_each_set_bit(i, &cmds_enabled, MAX_CMDS_PER_TCS) { addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, i); data = read_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, i); @@ -834,6 +835,17 @@ void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl) else if (gic_irq_sts) pr_warn("ERROR:Possible lockup in Linux\n"); + /* Show fast path status, if the TCS is busy */ + if (drv->tcs[FAST_PATH_TCS].num_tcs) { + int tcs_id = drv->tcs[FAST_PATH_TCS].offset; + bool sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); + + if (!sts) { + pr_err("Fast-path TCS information:\n"); + print_tcs_info(drv, tcs_id, &accl, &aoss_irq_sts); + } + } + /* * The TCS(s) are busy waiting, we have no way to recover from this. * If this debug function is called, we assume it's because timeout From 406c64307dd3b3bceede0a80b01bf1ba4c2b7ea8 Mon Sep 17 00:00:00 2001 From: Lina Iyer Date: Tue, 26 Jan 2021 14:39:22 -0700 Subject: [PATCH 3/8] drivers: qcom: rpmh: Avoid unnecessary checks on irq-done response The RSC interrupt is issued only after the request is complete. For fire-n-forget requests, the irq-done interrupt is sent after issuing the RPMH request and for response-required request, the interrupt is triggered only after all the requests are complete. These unnecessary checks in the interrupt handler issues AHB reads from a critical path. Let's remove them and clean up error handling in rpmh_request data structures. Change-Id: Ie3a3569603f001d6cdb59310c3f2752cdcd7a84d Signed-off-by: Lina Iyer Signed-off-by: Maulik Shah --- drivers/soc/qcom/rpmh-internal.h | 6 ++---- drivers/soc/qcom/rpmh-rsc.c | 24 ++++-------------------- drivers/soc/qcom/rpmh.c | 12 +++--------- drivers/soc/qcom/trace-rpmh.h | 13 +++++-------- 4 files changed, 14 insertions(+), 41 deletions(-) diff --git a/drivers/soc/qcom/rpmh-internal.h b/drivers/soc/qcom/rpmh-internal.h index 95e641702f4b..a9d50afb48a8 100644 --- a/drivers/soc/qcom/rpmh-internal.h +++ b/drivers/soc/qcom/rpmh-internal.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2018, 2020-2021, The Linux Foundation. All rights reserved. */ @@ -62,7 +62,6 @@ struct tcs_group { * @cmd: the payload that will be part of the @msg * @completion: triggered when request is done * @dev: the device making the request - * @err: err return from the controller * @needs_free: check to free dynamically allocated request object */ struct rpmh_request { @@ -70,7 +69,6 @@ struct rpmh_request { struct tcs_cmd cmd[MAX_RPMH_PAYLOAD]; struct completion *completion; const struct device *dev; - int err; bool needs_free; }; @@ -149,7 +147,7 @@ void rpmh_rsc_invalidate(struct rsc_drv *drv); void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl); int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable); -void rpmh_tx_done(const struct tcs_request *msg, int r); +void rpmh_tx_done(const struct tcs_request *msg); int rpmh_flush(struct rpmh_ctrlr *ctrlr); int _rpmh_flush(struct rpmh_ctrlr *ctrlr); diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index cbcfcb0ed4c9..ea70b5b79d71 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2018, 2020-2021, The Linux Foundation. All rights reserved. */ #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME @@ -400,10 +400,9 @@ static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) static irqreturn_t tcs_tx_done(int irq, void *p) { struct rsc_drv *drv = p; - int i, j, err = 0; + int i; unsigned long irq_status; const struct tcs_request *req; - struct tcs_cmd *cmd; irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); @@ -412,22 +411,7 @@ static irqreturn_t tcs_tx_done(int irq, void *p) if (WARN_ON(!req)) goto skip; - err = 0; - for (j = 0; j < req->num_cmds; j++) { - u32 sts; - - cmd = &req->cmds[j]; - sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j); - if (!(sts & CMD_STATUS_ISSUED) || - ((req->wait_for_compl || cmd->wait) && - !(sts & CMD_STATUS_COMPL))) { - pr_err("Incomplete request: %s: addr=%#x data=%#x", - drv->name, cmd->addr, cmd->data); - err = -EIO; - } - } - - trace_rpmh_tx_done(drv, i, req, err); + trace_rpmh_tx_done(drv, i, req); /* * If wake tcs was re-purposed for sending active @@ -452,7 +436,7 @@ static irqreturn_t tcs_tx_done(int irq, void *p) spin_unlock(&drv->lock); wake_up(&drv->tcs_wait); if (req) - rpmh_tx_done(req, err); + rpmh_tx_done(req); } return IRQ_HANDLED; diff --git a/drivers/soc/qcom/rpmh.c b/drivers/soc/qcom/rpmh.c index 77d7ef0035c0..92815cdb7054 100644 --- a/drivers/soc/qcom/rpmh.c +++ b/drivers/soc/qcom/rpmh.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2018, 2020-2021, The Linux Foundation. All rights reserved. */ #include @@ -95,19 +95,13 @@ static int check_ctrlr_state(struct rpmh_ctrlr *ctrlr, enum rpmh_state state) return ret; } -void rpmh_tx_done(const struct tcs_request *msg, int r) +void rpmh_tx_done(const struct tcs_request *msg) { struct rpmh_request *rpm_msg = container_of(msg, struct rpmh_request, msg); struct completion *compl = rpm_msg->completion; bool free = rpm_msg->needs_free; - rpm_msg->err = r; - - if (r) - dev_err(rpm_msg->dev, "RPMH TX fail in msg addr=%#x, err=%d\n", - rpm_msg->msg.cmds[0].addr, r); - if (!compl) goto exit; @@ -213,7 +207,7 @@ static int __rpmh_write(const struct device *dev, enum rpmh_state state, } else { /* Clean up our call by spoofing tx_done */ ret = 0; - rpmh_tx_done(&rpm_msg->msg, ret); + rpmh_tx_done(&rpm_msg->msg); } return ret; diff --git a/drivers/soc/qcom/trace-rpmh.h b/drivers/soc/qcom/trace-rpmh.h index fb7285185f96..efaf50a4b67d 100644 --- a/drivers/soc/qcom/trace-rpmh.h +++ b/drivers/soc/qcom/trace-rpmh.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2019, 2021, The Linux Foundation. All rights reserved. */ #if !defined(_TRACE_RPMH_H) || defined(TRACE_HEADER_MULTI_READ) @@ -14,16 +14,15 @@ TRACE_EVENT(rpmh_tx_done, - TP_PROTO(struct rsc_drv *d, int m, const struct tcs_request *r, int e), + TP_PROTO(struct rsc_drv *d, int m, const struct tcs_request *r), - TP_ARGS(d, m, r, e), + TP_ARGS(d, m, r), TP_STRUCT__entry( __string(name, d->name) __field(int, m) __field(u32, addr) __field(u32, data) - __field(int, err) ), TP_fast_assign( @@ -31,12 +30,10 @@ TRACE_EVENT(rpmh_tx_done, __entry->m = m; __entry->addr = r->cmds[0].addr; __entry->data = r->cmds[0].data; - __entry->err = e; ), - TP_printk("%s: ack: tcs-m: %d addr: %#x data: %#x errno: %d", - __get_str(name), __entry->m, __entry->addr, __entry->data, - __entry->err) + TP_printk("%s: ack: tcs-m: %d addr: %#x data: %#x", + __get_str(name), __entry->m, __entry->addr, __entry->data) ); TRACE_EVENT(rpmh_send_msg, From dac9bfe822338665cf95bec33e5eacf827fe7586 Mon Sep 17 00:00:00 2001 From: Lina Iyer Date: Wed, 6 Mar 2019 12:58:44 -0700 Subject: [PATCH 4/8] drivers: qcom: rpmh-rsc: Add IPC logging support for RSC driver In addition to logging using FTRACE. let's also log in the IPC logging for all RPMH transactions. Change-Id: I2b19006bbc776053f63eee55b447428fdb023ba4 Signed-off-by: Lina Iyer Signed-off-by: Maulik Shah --- drivers/soc/qcom/rpmh-internal.h | 2 ++ drivers/soc/qcom/rpmh-rsc.c | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/drivers/soc/qcom/rpmh-internal.h b/drivers/soc/qcom/rpmh-internal.h index a9d50afb48a8..7c1a13db3ce0 100644 --- a/drivers/soc/qcom/rpmh-internal.h +++ b/drivers/soc/qcom/rpmh-internal.h @@ -119,6 +119,7 @@ struct rpmh_ctrlr { * @client: Handle to the DRV's client. * @genpd_nb: PM Domain notifier * @dev: RSC device + * @ipc_log_ctx: IPC logger handle */ struct rsc_drv { const char *name; @@ -136,6 +137,7 @@ struct rsc_drv { struct rpmh_ctrlr client; struct notifier_block genpd_nb; struct device *dev; + void *ipc_log_ctx; }; extern bool rpmh_standalone; diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index ea70b5b79d71..27ed00539bae 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -34,6 +34,10 @@ #define CREATE_TRACE_POINTS #include "trace-rpmh.h" +#include + +#define RSC_DRV_IPC_LOG_SIZE 2 + #define RSC_DRV_TCS_OFFSET 672 #define RSC_DRV_CMD_OFFSET 20 @@ -412,6 +416,7 @@ static irqreturn_t tcs_tx_done(int irq, void *p) goto skip; trace_rpmh_tx_done(drv, i, req); + ipc_log_string(drv->ipc_log_ctx, "IRQ response: m=%d", i); /* * If wake tcs was re-purposed for sending active @@ -478,6 +483,10 @@ static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd); + ipc_log_string(drv->ipc_log_ctx, + "TCS write: m=%d n=%d msgid=%#x addr=%#x data=%#x wait=%d", + tcs_id, j, msgid, cmd->addr, + cmd->data, cmd->wait); } cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); @@ -648,6 +657,7 @@ int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) */ __tcs_buffer_write(drv, tcs_id, 0, msg); __tcs_set_trigger(drv, tcs_id, true); + ipc_log_string(drv->ipc_log_ctx, "TCS trigger: m=%d", tcs_id); return 0; } @@ -978,6 +988,8 @@ int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable) if (!enable || !rpmh_rsc_ctrlr_is_busy(drv)) { drv->in_solver_mode = enable; trace_rpmh_solver_set(drv, enable); + ipc_log_string(drv->ipc_log_ctx, + "solver mode set: %d", enable); ret = 0; } spin_unlock(&drv->lock); @@ -1262,6 +1274,9 @@ static int rpmh_rsc_probe(struct platform_device *pdev) INIT_LIST_HEAD(&drv->client.cache); INIT_LIST_HEAD(&drv->client.batch_cache); + drv->ipc_log_ctx = ipc_log_context_create(RSC_DRV_IPC_LOG_SIZE, + drv->name, 0); + dev_set_drvdata(&pdev->dev, drv); if (__rsc_count < MAX_RSC_COUNT) From c02c62a522cbd34a4767e56fa5eb4b2d22335e83 Mon Sep 17 00:00:00 2001 From: Maulik Shah Date: Sun, 15 Aug 2021 19:15:10 +0530 Subject: [PATCH 5/8] soc: qcom: rpmh-rsc: Add RSC version 3 support RSC major version 3.0 has different register offsets than previous versions. Add support to read the current version during probe and use register offsets accordingly. Change-Id: I8174516ecd2ab7a370e1bb6592d4329c4ff1f81d Signed-off-by: Maulik Shah --- drivers/soc/qcom/rpmh-internal.h | 4 + drivers/soc/qcom/rpmh-rsc.c | 215 +++++++++++++++++++------------ 2 files changed, 139 insertions(+), 80 deletions(-) diff --git a/drivers/soc/qcom/rpmh-internal.h b/drivers/soc/qcom/rpmh-internal.h index 7c1a13db3ce0..6fb7e0618f7d 100644 --- a/drivers/soc/qcom/rpmh-internal.h +++ b/drivers/soc/qcom/rpmh-internal.h @@ -96,7 +96,9 @@ struct rpmh_ctrlr { * Resource State Coordinator controller (RSC) * * @name: Controller identifier. + * @base: Base address of the RSC controller. * @tcs_base: Start address of the TCS registers in this controller. + * @reg: Register offsets for RSC controller. * @id: Instance id in the controller (Direct Resource Voter). * @num_tcs: Number of TCSes in this DRV. * @irq: IRQ at gic. @@ -123,7 +125,9 @@ struct rpmh_ctrlr { */ struct rsc_drv { const char *name; + void __iomem *base; void __iomem *tcs_base; + u32 *regs; int id; int num_tcs; int irq; diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index 27ed00539bae..917cb7e216a1 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -38,50 +38,23 @@ #define RSC_DRV_IPC_LOG_SIZE 2 -#define RSC_DRV_TCS_OFFSET 672 -#define RSC_DRV_CMD_OFFSET 20 +/* DRV ID Register */ +#define RSC_DRV_ID 0 +#define MAJOR_VER_MASK 0xFF +#define MAJOR_VER_SHIFT 16 +#define MINOR_VER_MASK 0xFF +#define MINOR_VER_SHIFT 8 -/* DRV HW Solver Configuration Information Register */ -#define DRV_SOLVER_CONFIG 0x04 +/* DRV HW Solver Configuration Register Mask */ #define DRV_HW_SOLVER_MASK 1 #define DRV_HW_SOLVER_SHIFT 24 -/* DRV TCS Configuration Information Register */ -#define DRV_PRNT_CHLD_CONFIG 0x0C +/* DRV TCS Configuration Information Register Mask */ #define DRV_NUM_TCS_MASK 0x3F #define DRV_NUM_TCS_SHIFT 6 #define DRV_NCPT_MASK 0x1F #define DRV_NCPT_SHIFT 27 -/* Offsets for common TCS Registers, one bit per TCS */ -#define RSC_DRV_IRQ_ENABLE 0x00 -#define RSC_DRV_IRQ_STATUS 0x04 -#define RSC_DRV_IRQ_CLEAR 0x08 /* w/o; write 1 to clear */ - -/* - * Offsets for per TCS Registers. - * - * TCSes start at 0x10 from tcs_base and are stored one after another. - * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one - * of the below to find a register. - */ -#define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 /* 1 bit per command */ -#define RSC_DRV_CONTROL 0x14 -#define RSC_DRV_STATUS 0x18 /* zero if tcs is busy */ -#define RSC_DRV_CMD_ENABLE 0x1C /* 1 bit per command */ - -/* - * Offsets for per command in a TCS. - * - * Commands (up to 16) start at 0x30 in a TCS; multiply command index - * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. - */ -#define RSC_DRV_CMD_MSGID 0x30 -#define RSC_DRV_CMD_ADDR 0x34 -#define RSC_DRV_CMD_DATA 0x38 -#define RSC_DRV_CMD_STATUS 0x3C -#define RSC_DRV_CMD_RESP_DATA 0x40 - #define TCS_AMC_MODE_ENABLE BIT(16) #define TCS_AMC_MODE_TRIGGER BIT(24) @@ -158,16 +131,89 @@ bool rpmh_standalone; * +---------------------------------------------------+ */ +enum { + RSC_DRV_TCS_OFFSET, + RSC_DRV_CMD_OFFSET, +/* DRV HW Solver Configuration Information Register */ + DRV_SOLVER_CONFIG, +/* DRV TCS Configuration Information Register */ + DRV_PRNT_CHLD_CONFIG, +/* Offsets for common TCS Registers, one bit per TCS */ + RSC_DRV_IRQ_ENABLE, + RSC_DRV_IRQ_STATUS, + RSC_DRV_IRQ_CLEAR, /* w/o; write 1 to clear */ +/* + * Offsets for per TCS Registers. + * + * TCSes start at 0x10 from tcs_base and are stored one after another. + * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one + * of the below to find a register. + */ + RSC_DRV_CMD_WAIT_FOR_CMPL, /* 1 bit per command */ + RSC_DRV_CONTROL, + RSC_DRV_STATUS, /* zero if tcs is busy */ + RSC_DRV_CMD_ENABLE, /* 1 bit per command */ +/* + * Offsets for per command in a TCS. + * + * Commands (up to 16) start at 0x30 in a TCS; multiply command index + * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. + */ + RSC_DRV_CMD_MSGID, + RSC_DRV_CMD_ADDR, + RSC_DRV_CMD_DATA, + RSC_DRV_CMD_STATUS, + RSC_DRV_CMD_RESP_DATA, +}; + +static u32 rpmh_rsc_reg_offsets_ver_2_7[] = { + [RSC_DRV_TCS_OFFSET] = 672, + [RSC_DRV_CMD_OFFSET] = 20, + [DRV_SOLVER_CONFIG] = 0x04, + [DRV_PRNT_CHLD_CONFIG] = 0x0C, + [RSC_DRV_IRQ_ENABLE] = 0x00, + [RSC_DRV_IRQ_STATUS] = 0x04, + [RSC_DRV_IRQ_CLEAR] = 0x08, + [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10, + [RSC_DRV_CONTROL] = 0x14, + [RSC_DRV_STATUS] = 0x18, + [RSC_DRV_CMD_ENABLE] = 0x1C, + [RSC_DRV_CMD_MSGID] = 0x30, + [RSC_DRV_CMD_ADDR] = 0x34, + [RSC_DRV_CMD_DATA] = 0x38, + [RSC_DRV_CMD_STATUS] = 0x3C, + [RSC_DRV_CMD_RESP_DATA] = 0x40, +}; + +static u32 rpmh_rsc_reg_offsets_ver_3_0[] = { + [RSC_DRV_TCS_OFFSET] = 672, + [RSC_DRV_CMD_OFFSET] = 24, + [DRV_SOLVER_CONFIG] = 0x04, + [DRV_PRNT_CHLD_CONFIG] = 0x0C, + [RSC_DRV_IRQ_ENABLE] = 0x00, + [RSC_DRV_IRQ_STATUS] = 0x04, + [RSC_DRV_IRQ_CLEAR] = 0x08, + [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20, + [RSC_DRV_CONTROL] = 0x24, + [RSC_DRV_STATUS] = 0x28, + [RSC_DRV_CMD_ENABLE] = 0x2C, + [RSC_DRV_CMD_MSGID] = 0x34, + [RSC_DRV_CMD_ADDR] = 0x38, + [RSC_DRV_CMD_DATA] = 0x3C, + [RSC_DRV_CMD_STATUS] = 0x40, + [RSC_DRV_CMD_RESP_DATA] = 0x44, +}; + static inline void __iomem * tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) { - return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg; + return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg; } static inline void __iomem * tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) { - return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id; + return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id; } static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, @@ -235,8 +281,7 @@ static void tcs_invalidate(struct rsc_drv *drv, int type) return; for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) - write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0); - + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0); bitmap_zero(tcs->slots, MAX_TCS_SLOTS); } @@ -355,18 +400,18 @@ static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) * While clearing ensure that the AMC mode trigger is cleared * and then the mode enable is cleared. */ - enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); + enable = read_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id); enable &= ~TCS_AMC_MODE_TRIGGER; - write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CONTROL], tcs_id, enable); enable &= ~TCS_AMC_MODE_ENABLE; - write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CONTROL], tcs_id, enable); if (trigger) { /* Enable the AMC mode on the TCS and then trigger the TCS */ enable = TCS_AMC_MODE_ENABLE; - write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CONTROL], tcs_id, enable); enable |= TCS_AMC_MODE_TRIGGER; - write_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id, enable); + write_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id, enable); } } @@ -383,12 +428,12 @@ static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) { u32 data; - data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE); + data = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); if (enable) data |= BIT(tcs_id); else data &= ~BIT(tcs_id); - writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE); + writel_relaxed(data, drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); } /** @@ -408,7 +453,7 @@ static irqreturn_t tcs_tx_done(int irq, void *p) unsigned long irq_status; const struct tcs_request *req; - irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); + irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) { req = get_req_from_tcs(drv, i); @@ -427,8 +472,8 @@ static irqreturn_t tcs_tx_done(int irq, void *p) __tcs_set_trigger(drv, i, false); skip: /* Reclaim the TCS */ - write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0); - writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR); + write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0); + writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]); spin_lock(&drv->lock); clear_bit(i, drv->tcs_in_use); /* @@ -479,9 +524,9 @@ static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, */ msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; - write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid); - write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); - write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); + write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid); + write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr); + write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data); trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd); ipc_log_string(drv->ipc_log_ctx, "TCS write: m=%d n=%d msgid=%#x addr=%#x data=%#x wait=%d", @@ -489,8 +534,8 @@ static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, cmd->data, cmd->wait); } - cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); - write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable); + cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); + write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable); } /** @@ -522,10 +567,10 @@ static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, int i = tcs->offset; for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) { - curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i); + curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i); for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { - addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, i, j); + addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j); for (k = 0; k < msg->num_cmds; k++) { if (addr == msg->cmds[k].addr) return -EBUSY; @@ -642,7 +687,7 @@ int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) * repurposed TCS to avoid triggering them. tcs->slots will be * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() */ - write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0); enable_tcs_irq(drv, tcs_id, true); } spin_unlock_irqrestore(&drv->lock, flags); @@ -754,16 +799,16 @@ static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, bool in_use = test_bit(tcs_id, drv->tcs_in_use); int i; - sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); - cmds_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); + sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); + cmds_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); if (!cmds_enabled) return; if (!tcs_grp || !req) goto print_tcs_data; - data = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); - irq_sts = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); + data = read_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id); + irq_sts = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); pr_warn("Request: tcs-in-use:%s active_tcs=%s(%d) state=%d wait_for_compl=%u]\n", (in_use ? "YES" : "NO"), ((tcs_grp->type == ACTIVE_TCS) ? "YES" : "NO"), @@ -776,10 +821,10 @@ static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, print_tcs_data: for_each_set_bit(i, &cmds_enabled, MAX_CMDS_PER_TCS) { - addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, i); - data = read_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, i); - msgid = read_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, i); - sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, tcs_id, i); + addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, i); + data = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i); + msgid = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, i); + sts = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_STATUS], tcs_id, i); pr_warn("\tCMD=%d [addr=0x%x data=0x%x hdr=0x%x sts=0x%x enabled=1]\n", i, addr, data, msgid, sts); if (!(sts & CMD_STATUS_ISSUED)) @@ -832,7 +877,7 @@ void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl) /* Show fast path status, if the TCS is busy */ if (drv->tcs[FAST_PATH_TCS].num_tcs) { int tcs_id = drv->tcs[FAST_PATH_TCS].offset; - bool sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); + bool sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); if (!sts) { pr_err("Fast-path TCS information:\n"); @@ -884,7 +929,7 @@ static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) /* Check if there is pending fastpath transaction */ if (drv->tcs[FAST_PATH_TCS].num_tcs && - !read_tcs_reg(drv, RSC_DRV_STATUS, drv->tcs[FAST_PATH_TCS].offset)) + !read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], drv->tcs[FAST_PATH_TCS].offset)) return true; return set < max; @@ -1052,7 +1097,7 @@ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, /* Ensure the TCS is free before writing to the TCS */ do { - sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); + sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); if (!sts) { retry--; /* Report and bail, if it took too many attempts */ @@ -1073,7 +1118,7 @@ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, if (!(mask & BIT(i))) continue; cmd = &msg->cmds[i]; - write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, i, cmd->data); + write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i, cmd->data); } /* Trigger the TCS to send the request */ @@ -1083,7 +1128,7 @@ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, } static int rpmh_probe_tcs_config(struct platform_device *pdev, - struct rsc_drv *drv, void __iomem *base) + struct rsc_drv *drv) { struct tcs_type_config { u32 type; @@ -1097,9 +1142,9 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev, ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); if (ret) return ret; - drv->tcs_base = base + offset; + drv->tcs_base = drv->base + offset; - config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG); + config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]); max_tcs = config; max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); @@ -1186,8 +1231,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev) struct rsc_drv *drv; char drv_id[10] = {0}; int ret, irq; - u32 solver_config; - void __iomem *base; + u32 rsc_id, major_ver, minor_ver, solver_config; /* * Even though RPMh doesn't directly use cmd-db, all of its children @@ -1219,11 +1263,22 @@ static int rpmh_rsc_probe(struct platform_device *pdev) drv->name = dev_name(&pdev->dev); snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); - base = devm_platform_ioremap_resource_byname(pdev, drv_id); - if (IS_ERR(base)) - return PTR_ERR(base); + drv->base = devm_platform_ioremap_resource_byname(pdev, drv_id); + if (IS_ERR(drv->base)) + return PTR_ERR(drv->base); - ret = rpmh_probe_tcs_config(pdev, drv, base); + rsc_id = readl_relaxed(drv->base + RSC_DRV_ID); + major_ver = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT); + major_ver >>= MAJOR_VER_SHIFT; + minor_ver = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); + minor_ver >>= MINOR_VER_SHIFT; + + if (major_ver >= 3 && minor_ver >= 0) + drv->regs = rpmh_rsc_reg_offsets_ver_3_0; + else + drv->regs = rpmh_rsc_reg_offsets_ver_2_7; + + ret = rpmh_probe_tcs_config(pdev, drv); if (ret) return ret; @@ -1249,7 +1304,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev) * 'HW solver' mode where they can be in autonomous mode executing low * power mode to power down. */ - solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG); + solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]); solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; if (of_find_property(dn, "power-domains", NULL)) { @@ -1268,7 +1323,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev) /* Enable the active TCS to send requests immediately */ writel_relaxed(drv->tcs[ACTIVE_TCS].mask, - drv->tcs_base + RSC_DRV_IRQ_ENABLE); + drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); spin_lock_init(&drv->client.cache_lock); INIT_LIST_HEAD(&drv->client.cache); From 8d44aff55cce580fc1ffa03a17c953aa9d8375b7 Mon Sep 17 00:00:00 2001 From: Maulik Shah Date: Mon, 18 Nov 2019 18:15:15 +0530 Subject: [PATCH 6/8] soc: qcom: rpmh: Correct rpm_msg pointer offset and add list_del rpm_msgs address is calculated from memory allocated during write_batch. Correct the offset from where rpm_msgs actually starts and assign req pointer to point to correct address. Also add list_del before freeing memory in rpmh_invalidate. Change-Id: Idca43fe73bd238e90911158b08b3889c001e1123 Signed-off-by: Maulik Shah --- drivers/soc/qcom/rpmh.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/soc/qcom/rpmh.c b/drivers/soc/qcom/rpmh.c index 92815cdb7054..8b0cdbd3ffa1 100644 --- a/drivers/soc/qcom/rpmh.c +++ b/drivers/soc/qcom/rpmh.c @@ -66,7 +66,7 @@ struct cache_req { struct batch_cache_req { struct list_head list; int count; - struct rpmh_request rpm_msgs[]; + struct rpmh_request *rpm_msgs; }; static struct rpmh_ctrlr *get_rpmh_ctrlr(const struct device *dev) @@ -393,10 +393,11 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state, return -ENOMEM; req = ptr; + rpm_msgs = ptr + sizeof(*req); compls = ptr + sizeof(*req) + count * sizeof(*rpm_msgs); req->count = count; - rpm_msgs = req->rpm_msgs; + req->rpm_msgs = rpm_msgs; for (i = 0; i < count; i++) { __fill_rpmh_msg(rpm_msgs + i, state, cmd, n[i]); @@ -577,8 +578,11 @@ void rpmh_invalidate(const struct device *dev) return; spin_lock_irqsave(&ctrlr->cache_lock, flags); - list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list) + list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list) { + list_del(&req->list); kfree(req); + } + INIT_LIST_HEAD(&ctrlr->batch_cache); ctrlr->dirty = true; spin_unlock_irqrestore(&ctrlr->cache_lock, flags); From 7926741d4ed07dab1ebf315ca0fb8a226d5ec342 Mon Sep 17 00:00:00 2001 From: Maulik Shah Date: Sun, 4 Jul 2021 12:51:45 +0530 Subject: [PATCH 7/8] drivers: rpmh: Always bug_on() upon timeout in rpmh_write_batch() rpmh_rsc_debug() can race with tcs_tx_done() interrupt handler and may not see any tcs busy and hence won't do bug_on(). Later it may go ahead and free() the request which interrupt handler is processing. Lets always bug_on() upon timeout. Change-Id: I238a6a3639077850df158cb1f0190656e014bb57 Signed-off-by: Maulik Shah --- drivers/soc/qcom/rpmh.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/soc/qcom/rpmh.c b/drivers/soc/qcom/rpmh.c index 8b0cdbd3ffa1..b6fc1fe9c59d 100644 --- a/drivers/soc/qcom/rpmh.c +++ b/drivers/soc/qcom/rpmh.c @@ -432,12 +432,10 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state, * we've returned from this function. */ rpmh_rsc_debug(ctrlr_to_drv(ctrlr), &compls[i]); - ret = -ETIMEDOUT; - goto exit; + BUG_ON(1); } } -exit: kfree(ptr); return ret; From b6a55a9af0f24ea7c6dd6cbe1e79a887396e98a3 Mon Sep 17 00:00:00 2001 From: Maulik Shah Date: Mon, 9 May 2022 13:13:36 +0530 Subject: [PATCH 8/8] soc: qcom: rpmh-rsc: Add support for multiple DRVs Add support to probe multiple DRVs in a single RSC device. Each DRV can have TCSes spread in channels for use by a hardware directly communicating with the RSC. A 'channel' is a collection of TCS groups, that may independently be used by the RSC sequencer. The channel number will be derrived from the device that is making an RPMH request and to support same add channel parameter to all RPMH requests. Each channel can cache the sleep and wake requests independently and when the rpmh_flush() is requested by a device the respective channel's cached data will be written to the TCSes. The existing RSCes can be considered running with a single channel. Change-Id: Id0956136f37c28f2039e139d21f63ade5044ae1f Signed-off-by: Maulik Shah --- drivers/soc/qcom/rpmh-internal.h | 47 +++- drivers/soc/qcom/rpmh-rsc.c | 466 +++++++++++++++++++------------ drivers/soc/qcom/rpmh.c | 65 +++-- 3 files changed, 379 insertions(+), 199 deletions(-) diff --git a/drivers/soc/qcom/rpmh-internal.h b/drivers/soc/qcom/rpmh-internal.h index 6fb7e0618f7d..0997ce9bdffc 100644 --- a/drivers/soc/qcom/rpmh-internal.h +++ b/drivers/soc/qcom/rpmh-internal.h @@ -11,6 +11,12 @@ #include #include +#define MAX_NAME_LENGTH 20 + +#define CH0 0 +#define CH1 1 +#define MAX_CHANNEL 2 + #define TCS_TYPE_NR 5 #define MAX_CMDS_PER_TCS 16 #define MAX_TCS_PER_TYPE 3 @@ -91,6 +97,19 @@ struct rpmh_ctrlr { struct list_head batch_cache; }; +/** + * struct drv_channel: our representation of the drv channels + * + * @tcs: TCS groups. + * @drv: DRV containing the channel + * @initialized: Whether channel is initialized + */ +struct drv_channel { + struct tcs_group tcs[TCS_TYPE_NR]; + struct rsc_drv *drv; + bool initialized; +}; + /** * struct rsc_drv: the Direct Resource Voter (DRV) of the * Resource State Coordinator controller (RSC) @@ -101,13 +120,15 @@ struct rpmh_ctrlr { * @reg: Register offsets for RSC controller. * @id: Instance id in the controller (Direct Resource Voter). * @num_tcs: Number of TCSes in this DRV. + * @num_channels: Number of channels in this DRV. * @irq: IRQ at gic. * @in_solver_mode: Controller is busy in solver mode + * @initialized: Whether DRV is initialized * @rsc_pm: CPU PM notifier for controller. * Used when solver mode is not present. * @cpus_in_pm: Number of CPUs not in idle power collapse. * Used when solver mode is not present. - * @tcs: TCS groups. + * @ch: DRV channels. * @tcs_in_use: S/W state of the TCS; only set for ACTIVE_ONLY * transfers, but might show a sleep/wake TCS in use if * it was borrowed for an active_only transfer. You @@ -122,19 +143,22 @@ struct rpmh_ctrlr { * @genpd_nb: PM Domain notifier * @dev: RSC device * @ipc_log_ctx: IPC logger handle + * @pdev: platform device */ struct rsc_drv { - const char *name; + char name[MAX_NAME_LENGTH]; void __iomem *base; void __iomem *tcs_base; u32 *regs; int id; int num_tcs; + int num_channels; int irq; bool in_solver_mode; + bool initialized; struct notifier_block rsc_pm; atomic_t cpus_in_pm; - struct tcs_group tcs[TCS_TYPE_NR]; + struct drv_channel ch[MAX_CHANNEL]; DECLARE_BITMAP(tcs_in_use, MAX_TCS_NR); spinlock_t lock; wait_queue_head_t tcs_wait; @@ -142,24 +166,27 @@ struct rsc_drv { struct notifier_block genpd_nb; struct device *dev; void *ipc_log_ctx; + struct platform_device *pdev; }; extern bool rpmh_standalone; -int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg); +int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch); int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, - const struct tcs_request *msg); -void rpmh_rsc_invalidate(struct rsc_drv *drv); + const struct tcs_request *msg, + int ch); +void rpmh_rsc_invalidate(struct rsc_drv *drv, int ch); void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl); int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable); +int rpmh_rsc_get_channel(struct rsc_drv *drv); void rpmh_tx_done(const struct tcs_request *msg); -int rpmh_flush(struct rpmh_ctrlr *ctrlr); -int _rpmh_flush(struct rpmh_ctrlr *ctrlr); +int rpmh_flush(struct rpmh_ctrlr *ctrlr, int ch); +int _rpmh_flush(struct rpmh_ctrlr *ctrlr, int ch); -int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg); +int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, int ch); int rpmh_rsc_update_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, - u32 update_mask); + u32 update_mask, int ch); #endif /* __RPM_INTERNAL_H__ */ diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index 917cb7e216a1..011bcb0825da 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -65,9 +65,18 @@ #define CMD_STATUS_ISSUED BIT(8) #define CMD_STATUS_COMPL BIT(16) +/* Offsets for DRV channel status register */ +#define CH0_CHN_BUSY BIT(0) +#define CH1_CHN_BUSY BIT(1) +#define CH0_WAKE_TCS_STATUS BIT(0) +#define CH0_SLEEP_TCS_STATUS BIT(1) +#define CH1_WAKE_TCS_STATUS BIT(2) +#define CH1_SLEEP_TCS_STATUS BIT(3) +#define CH_CLEAR_STATUS BIT(31) + #define ACCL_TYPE(addr) ((addr >> 16) & 0xF) #define NR_ACCL_TYPES 3 -#define MAX_RSC_COUNT 2 +#define MAX_RSC_COUNT 5 static const char * const accl_str[] = { "", "", "", "CLK", "VREG", "BUS", @@ -263,6 +272,7 @@ static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). * @drv: The RSC controller. * @type: SLEEP_TCS or WAKE_TCS + * @ch: Channel number * * This will clear the "slots" variable of the given tcs_group and also * tell the hardware to forget about all entries. @@ -271,32 +281,47 @@ static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, * function is called, since otherwise the device may immediately become * used again even before this function exits. */ -static void tcs_invalidate(struct rsc_drv *drv, int type) +static void tcs_invalidate(struct rsc_drv *drv, int type, int ch) { int m; - struct tcs_group *tcs = &drv->tcs[type]; + struct tcs_group *tcs = &drv->ch[ch].tcs[type]; /* Caller ensures nobody else is running so no lock */ - if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) + if (bitmap_empty(tcs->slots, tcs->ncpt * tcs->num_tcs)) return; for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0); - bitmap_zero(tcs->slots, MAX_TCS_SLOTS); + bitmap_zero(tcs->slots, tcs->ncpt * tcs->num_tcs); +} + +/** + * rpmh_rsc_get_channel() - Get the Unused channel to send data on. + * @drv: The RSC controller. + * + * Return: 0 on success, else -error. + */ +int rpmh_rsc_get_channel(struct rsc_drv *drv) +{ + if (drv->num_channels == 1) + return CH0; + else + return -EBUSY; } /** * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. * @drv: The RSC controller. + * @ch: Channel number * - * The caller must ensure that no other RPMH actions are happening when this + * The caller must ensure that no other RPMH actions are happening when thi]s * function is called, since otherwise the device may immediately become * used again even before this function exits. */ -void rpmh_rsc_invalidate(struct rsc_drv *drv) +void rpmh_rsc_invalidate(struct rsc_drv *drv, int ch) { - tcs_invalidate(drv, SLEEP_TCS); - tcs_invalidate(drv, WAKE_TCS); + tcs_invalidate(drv, SLEEP_TCS, ch); + tcs_invalidate(drv, WAKE_TCS, ch); } /** @@ -310,12 +335,13 @@ void rpmh_rsc_invalidate(struct rsc_drv *drv) * Return: A pointer to a tcs_group or an ERR_PTR. */ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, - const struct tcs_request *msg) + enum rpmh_state state, + int ch) { int type; struct tcs_group *tcs; - switch (msg->state) { + switch (state) { case RPMH_ACTIVE_ONLY_STATE: type = ACTIVE_TCS; break; @@ -336,9 +362,9 @@ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, * transfers have finished before we use it (maybe by running from * the last CPU in PM code). */ - tcs = &drv->tcs[type]; - if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) - tcs = &drv->tcs[WAKE_TCS]; + tcs = &drv->ch[ch].tcs[type]; + if (state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) + tcs = &drv->ch[ch].tcs[WAKE_TCS]; return tcs; } @@ -360,15 +386,21 @@ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, * Return: The stashed request. */ static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, - int tcs_id) + int tcs_id, + int *ch) { struct tcs_group *tcs; int i; - for (i = 0; i < TCS_TYPE_NR; i++) { - tcs = &drv->tcs[i]; - if (tcs->mask & BIT(tcs_id)) + for (i = 0; i < MAX_CHANNEL; i++) { + if (!drv->ch[i].initialized) + continue; + + tcs = get_tcs_for_msg(drv, RPMH_ACTIVE_ONLY_STATE, i); + if (tcs->mask & BIT(tcs_id)) { + *ch = i; return tcs->req[tcs_id - tcs->offset]; + } } return NULL; @@ -449,14 +481,14 @@ static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) static irqreturn_t tcs_tx_done(int irq, void *p) { struct rsc_drv *drv = p; - int i; + int i, ch; unsigned long irq_status; const struct tcs_request *req; irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) { - req = get_req_from_tcs(drv, i); + req = get_req_from_tcs(drv, i, &ch); if (WARN_ON(!req)) goto skip; @@ -468,7 +500,7 @@ static irqreturn_t tcs_tx_done(int irq, void *p) * votes, clear AMC trigger & enable modes and * disable interrupt for this TCS */ - if (!drv->tcs[ACTIVE_TCS].num_tcs) + if (!drv->ch[ch].tcs[ACTIVE_TCS].num_tcs) __tcs_set_trigger(drv, i, false); skip: /* Reclaim the TCS */ @@ -481,7 +513,7 @@ static irqreturn_t tcs_tx_done(int irq, void *p) * spammed with interrupts coming when the solver * sends its wake votes. */ - if (!drv->tcs[ACTIVE_TCS].num_tcs) + if (!drv->ch[ch].tcs[ACTIVE_TCS].num_tcs) enable_tcs_irq(drv, i, false); spin_unlock(&drv->lock); wake_up(&drv->tcs_wait); @@ -569,7 +601,7 @@ static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) { curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i); - for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { + for_each_set_bit(j, &curr_enabled, tcs->ncpt) { addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j); for (k = 0; k < msg->num_cmds; k++) { if (addr == msg->cmds[k].addr) @@ -638,6 +670,7 @@ static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, * rpmh_rsc_send_data() - Write / trigger active-only message. * @drv: The controller. * @msg: The data to be sent. + * @ch: Channel number * * NOTES: * - This is only used for "ACTIVE_ONLY" since the limitations of this @@ -656,13 +689,13 @@ static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, * * Return: 0 on success, -EINVAL on error. */ -int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) +int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { struct tcs_group *tcs; int tcs_id; unsigned long flags; - tcs = get_tcs_for_msg(drv, msg); + tcs = get_tcs_for_msg(drv, msg->state, ch); if (IS_ERR(tcs)) return PTR_ERR(tcs); @@ -730,7 +763,8 @@ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, /* Do over, until we can fit the full payload in a single TCS */ do { - slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, + slot = bitmap_find_next_zero_area(tcs->slots, + tcs->ncpt * tcs->num_tcs, i, msg->num_cmds, 0); if (slot >= tcs->num_tcs * tcs->ncpt) return -ENOMEM; @@ -750,6 +784,7 @@ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. * @drv: The controller. * @msg: The data to be written to the controller. + * @ch: Channel number * * This should only be called for sleep/wake state, never active-only * state. @@ -759,13 +794,13 @@ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, * * Return: 0 if no error; else -error. */ -int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) +int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { struct tcs_group *tcs; int tcs_id = 0, cmd_id = 0; int ret; - tcs = get_tcs_for_msg(drv, msg); + tcs = get_tcs_for_msg(drv, msg->state, ch); if (IS_ERR(tcs)) return PTR_ERR(tcs); @@ -779,11 +814,16 @@ int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) static struct tcs_group *get_tcs_from_index(struct rsc_drv *drv, int tcs_id) { - unsigned int i; + unsigned int i, j; for (i = 0; i < TCS_TYPE_NR; i++) { - if (drv->tcs[i].mask & BIT(tcs_id)) - return &drv->tcs[i]; + for (j = 0; j < MAX_CHANNEL; j++) { + if (!drv->ch[j].initialized) + continue; + + if (drv->ch[j].tcs[i].mask & BIT(tcs_id)) + return &drv->ch[j].tcs[i]; + } } return NULL; @@ -792,8 +832,9 @@ static struct tcs_group *get_tcs_from_index(struct rsc_drv *drv, int tcs_id) static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, bool *aoss_irq_sts) { + int ch = 0; struct tcs_group *tcs_grp = get_tcs_from_index(drv, tcs_id); - const struct tcs_request *req = get_req_from_tcs(drv, tcs_id); + const struct tcs_request *req = get_req_from_tcs(drv, tcs_id, &ch); unsigned long cmds_enabled; u32 addr, data, msgid, sts, irq_sts; bool in_use = test_bit(tcs_id, drv->tcs_in_use); @@ -801,10 +842,10 @@ static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); cmds_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); - if (!cmds_enabled) + if (!cmds_enabled || !tcs_grp) return; - if (!tcs_grp || !req) + if (!req) goto print_tcs_data; data = read_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id); @@ -820,7 +861,7 @@ static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, *aoss_irq_sts = (irq_sts & BIT(tcs_id)) ? true : false; print_tcs_data: - for_each_set_bit(i, &cmds_enabled, MAX_CMDS_PER_TCS) { + for_each_set_bit(i, &cmds_enabled, tcs_grp->ncpt) { addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, i); data = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i); msgid = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, i); @@ -875,13 +916,19 @@ void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl) pr_warn("ERROR:Possible lockup in Linux\n"); /* Show fast path status, if the TCS is busy */ - if (drv->tcs[FAST_PATH_TCS].num_tcs) { - int tcs_id = drv->tcs[FAST_PATH_TCS].offset; - bool sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); + for (i = 0; i < MAX_CHANNEL; i++) { + if (!drv->ch[i].initialized) + continue; - if (!sts) { - pr_err("Fast-path TCS information:\n"); - print_tcs_info(drv, tcs_id, &accl, &aoss_irq_sts); + /* Show fast path status, if the TCS is busy */ + if (drv->ch[i].tcs[FAST_PATH_TCS].num_tcs) { + int tcs_id = drv->ch[i].tcs[FAST_PATH_TCS].offset; + bool sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); + + if (!sts) { + pr_err("Fast-path TCS information:\n"); + print_tcs_info(drv, tcs_id, &accl, &aoss_irq_sts); + } } } @@ -911,28 +958,38 @@ void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl) */ static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) { + int i; + struct tcs_group *tcs; unsigned long set; - const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; unsigned long max; - /* - * If we made an active request on a RSC that does not have a - * dedicated TCS for active state use, then re-purposed wake TCSes - * should be checked for not busy, because we used wake TCSes for - * active requests in this case. - */ - if (!tcs->num_tcs) - tcs = &drv->tcs[WAKE_TCS]; + for (i = 0; i < MAX_CHANNEL; i++) { + if (!drv->ch[i].initialized) + continue; - max = tcs->offset + tcs->num_tcs; - set = find_next_bit(drv->tcs_in_use, max, tcs->offset); + tcs = &drv->ch[i].tcs[ACTIVE_TCS]; + /* + * If we made an active request on a RSC that does not have a + * dedicated TCS for active state use, then re-purposed wake TCSes + * should be checked for not busy, because we used wake TCSes for + * active requests in this case. + */ + if (!tcs->num_tcs) + tcs = &drv->ch[i].tcs[WAKE_TCS]; - /* Check if there is pending fastpath transaction */ - if (drv->tcs[FAST_PATH_TCS].num_tcs && - !read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], drv->tcs[FAST_PATH_TCS].offset)) - return true; + max = tcs->offset + tcs->num_tcs; + set = find_next_bit(drv->tcs_in_use, max, tcs->offset); + if (set < max) + return true; - return set < max; + /* Check if there is pending fastpath transaction */ + tcs = &drv->ch[i].tcs[FAST_PATH_TCS]; + if (tcs->num_tcs && + !read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs->offset)) + return true; + } + + return false; } /** @@ -957,7 +1014,7 @@ static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, { struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); int ret = NOTIFY_OK; - int cpus_in_pm; + int cpus_in_pm, ch; switch (action) { case CPU_PM_ENTER: @@ -996,7 +1053,8 @@ static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, * CPU. */ if (spin_trylock(&drv->lock)) { - if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) + ch = rpmh_rsc_get_channel(drv); + if (ch < 0 || rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client, ch)) ret = NOTIFY_BAD; spin_unlock(&drv->lock); } else { @@ -1047,19 +1105,20 @@ int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable) * rpmh_rsc_init_fast_path() - Initialize the fast-path TCS contents * @drv: The controller. * @msg: The TCS request to populate. + * @ch: Channel number * * Return: * * 0 - success * * -ENODEV - no fast-path TCS available */ -int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg) +int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { int tcs_id; - if (!drv->tcs[FAST_PATH_TCS].num_tcs) + if (!drv->ch[ch].tcs[FAST_PATH_TCS].num_tcs) return -ENODEV; - tcs_id = drv->tcs[FAST_PATH_TCS].offset; + tcs_id = drv->ch[ch].tcs[FAST_PATH_TCS].offset; /* We won't use the AMC IRQ to confirm if the TCS is free */ enable_tcs_irq(drv, tcs_id, false); @@ -1074,6 +1133,7 @@ int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg) * @drv: The controller. * @msg: The TCS request data to be updated. * @mask: The update mask for elements in @msg to be sent + * @ch: Channel number * * NOTE: Caller should ensure serialization before making this call. * Return: @@ -1082,7 +1142,7 @@ int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg) */ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, - u32 mask) + u32 mask, int ch) { int i; u32 sts; @@ -1090,10 +1150,10 @@ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, struct tcs_cmd *cmd; int retry = 5; - if (!drv->tcs[FAST_PATH_TCS].num_tcs) + if (!drv->ch[ch].tcs[FAST_PATH_TCS].num_tcs) return -ENODEV; - tcs_id = drv->tcs[FAST_PATH_TCS].offset; + tcs_id = drv->ch[ch].tcs[FAST_PATH_TCS].offset; /* Ensure the TCS is free before writing to the TCS */ do { @@ -1127,45 +1187,32 @@ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, return 0; } -static int rpmh_probe_tcs_config(struct platform_device *pdev, - struct rsc_drv *drv) +static int rpmh_probe_channel_tcs_config(struct device_node *np, + struct rsc_drv *drv, + u32 max_tcs, u32 ncpt, int ch) { struct tcs_type_config { u32 type; u32 n; } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; - struct device_node *dn = pdev->dev.of_node; - u32 config, max_tcs, ncpt, offset; - int i, ret, n, st = 0; struct tcs_group *tcs; + struct drv_channel *channel = &drv->ch[ch]; + int i, ret, n, st = 0; + u32 tcs_mask; - ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); - if (ret) - return ret; - drv->tcs_base = drv->base + offset; - - config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]); - - max_tcs = config; - max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); - max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); - - ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); - ncpt = ncpt >> DRV_NCPT_SHIFT; - - n = of_property_count_u32_elems(dn, "qcom,tcs-config"); + n = of_property_count_u32_elems(np, "qcom,tcs-config"); if (n != 2 * TCS_TYPE_NR) return -EINVAL; for (i = 0; i < TCS_TYPE_NR; i++) { - ret = of_property_read_u32_index(dn, "qcom,tcs-config", + ret = of_property_read_u32_index(np, "qcom,tcs-config", i * 2, &tcs_cfg[i].type); if (ret) return ret; if (tcs_cfg[i].type >= TCS_TYPE_NR) return -EINVAL; - ret = of_property_read_u32_index(dn, "qcom,tcs-config", + ret = of_property_read_u32_index(np, "qcom,tcs-config", i * 2 + 1, &tcs_cfg[i].n); if (ret) return ret; @@ -1174,7 +1221,7 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev, } for (i = 0; i < TCS_TYPE_NR; i++) { - tcs = &drv->tcs[tcs_cfg[i].type]; + tcs = &channel->tcs[tcs_cfg[i].type]; if (tcs->drv) return -EINVAL; tcs->drv = drv; @@ -1189,12 +1236,54 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev, st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) return -EINVAL; - tcs->mask = ((1 << tcs->num_tcs) - 1) << st; - tcs->offset = st; + tcs->mask = ((1 << tcs->num_tcs) - 1) << (st + drv->num_tcs); + tcs->offset = st + drv->num_tcs; st += tcs->num_tcs; } - drv->num_tcs = st; + /* Enable the active TCS to send requests immediately */ + tcs_mask = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); + tcs_mask |= drv->ch[ch].tcs[ACTIVE_TCS].mask; + writel_relaxed(tcs_mask, drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); + + channel->drv = drv; + channel->initialized = true; + drv->num_tcs += st; + + return 0; +} + +static int rpmh_probe_tcs_config(struct rsc_drv *drv) +{ + struct device_node *cn, *np = drv->dev->of_node; + int ch = 0, ret; + u32 offset, config; + u32 max_tcs, ncpt; + + ret = of_property_read_u32(np, "qcom,tcs-offset", &offset); + if (ret) + return ret; + drv->tcs_base = drv->base + offset; + + config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]); + + max_tcs = config; + max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); + max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); + + ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); + ncpt = ncpt >> DRV_NCPT_SHIFT; + for_each_child_of_node(np, cn) { + + if (!of_node_name_eq(cn, "channel")) + continue; + + ret = rpmh_probe_channel_tcs_config(cn, drv, max_tcs, ncpt, ch); + if (ret) + return ret; + ch++; + } + drv->num_channels = ch; return 0; } @@ -1203,10 +1292,15 @@ static int rpmh_rsc_pd_cb(struct notifier_block *nb, unsigned long action, void *data) { struct rsc_drv *drv = container_of(nb, struct rsc_drv, genpd_nb); + int ch; + + if (action != GENPD_NOTIFY_PRE_OFF) + return NOTIFY_OK; + + ch = rpmh_rsc_get_channel(drv); /* We don't need to lock as domin on/off are serialized */ - if ((action == GENPD_NOTIFY_PRE_OFF) && - (rpmh_rsc_ctrlr_is_busy(drv) || _rpmh_flush(&drv->client))) + if (ch < 0 || rpmh_rsc_ctrlr_is_busy(drv) || _rpmh_flush(&drv->client, ch)) return NOTIFY_BAD; return NOTIFY_OK; @@ -1216,22 +1310,23 @@ static int rpmh_rsc_pd_attach(struct rsc_drv *drv) { int ret; - pm_runtime_enable(drv->dev); - ret = dev_pm_domain_attach(drv->dev, false); + pm_runtime_enable(&drv->pdev->dev); + ret = dev_pm_domain_attach(&drv->pdev->dev, false); if (ret) return ret; drv->genpd_nb.notifier_call = rpmh_rsc_pd_cb; - return dev_pm_genpd_add_notifier(drv->dev, &drv->genpd_nb); + return dev_pm_genpd_add_notifier(&drv->pdev->dev, &drv->genpd_nb); } static int rpmh_rsc_probe(struct platform_device *pdev) { - struct device_node *dn = pdev->dev.of_node; + struct device_node *np, *dn = pdev->dev.of_node; struct rsc_drv *drv; - char drv_id[10] = {0}; int ret, irq; u32 rsc_id, major_ver, minor_ver, solver_config; + int i, drv_count; + const char *name; /* * Even though RPMh doesn't directly use cmd-db, all of its children @@ -1250,92 +1345,121 @@ static int rpmh_rsc_probe(struct platform_device *pdev) dev_info(&pdev->dev, "RPMH is running in standalone mode.\n"); - drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); + ret = of_property_read_u32(dn, "qcom,drv-count", &drv_count); + if (ret) + return ret; + + drv = devm_kcalloc(&pdev->dev, drv_count, sizeof(*drv), GFP_KERNEL); if (!drv) return -ENOMEM; - ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); - if (ret) - return ret; + name = of_get_property(dn, "label", NULL); + if (!name) + name = dev_name(&pdev->dev); - drv->name = of_get_property(dn, "label", NULL); - if (!drv->name) - drv->name = dev_name(&pdev->dev); + for_each_child_of_node(dn, np) { + struct device *drv_dev; - snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); - drv->base = devm_platform_ioremap_resource_byname(pdev, drv_id); - if (IS_ERR(drv->base)) - return PTR_ERR(drv->base); + if (!of_node_name_eq(np, "drv")) + continue; - rsc_id = readl_relaxed(drv->base + RSC_DRV_ID); - major_ver = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT); - major_ver >>= MAJOR_VER_SHIFT; - minor_ver = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); - minor_ver >>= MINOR_VER_SHIFT; - - if (major_ver >= 3 && minor_ver >= 0) - drv->regs = rpmh_rsc_reg_offsets_ver_3_0; - else - drv->regs = rpmh_rsc_reg_offsets_ver_2_7; - - ret = rpmh_probe_tcs_config(pdev, drv); - if (ret) - return ret; - - spin_lock_init(&drv->lock); - init_waitqueue_head(&drv->tcs_wait); - bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); - - irq = platform_get_irq(pdev, drv->id); - if (irq < 0) - return irq; - - drv->irq = irq; - - ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, - IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, - drv->name, drv); - if (ret) - return ret; - - drv->dev = &pdev->dev; - /* - * CPU PM notification are not required for controllers that support - * 'HW solver' mode where they can be in autonomous mode executing low - * power mode to power down. - */ - solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]); - solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; - solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; - if (of_find_property(dn, "power-domains", NULL)) { - ret = rpmh_rsc_pd_attach(drv); - if (ret == -EPROBE_DEFER) { - pr_err("Failed to attach RSC %s to domain ret=%d\n", drv->name, ret); + ret = of_property_read_u32(np, "qcom,drv-id", &i); + if (ret) return ret; - } - } else if (!solver_config) { - drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; - cpu_pm_register_notifier(&drv->rsc_pm); - drv->client.flags &= ~SOLVER_PRESENT; - } else { - drv->client.flags |= SOLVER_PRESENT; + + scnprintf(drv[i].name, sizeof(drv[i].name), "%s-drv-%d", name, i); + + drv[i].base = devm_platform_ioremap_resource(pdev, i); + if (IS_ERR(drv[i].base)) + return PTR_ERR(drv[i].base); + + drv_dev = kzalloc(sizeof(*drv_dev), GFP_KERNEL); + if (!drv_dev) + return -ENOMEM; + + drv[i].id = i; + drv[i].pdev = pdev; + drv[i].dev = drv_dev; + drv_dev->parent = &pdev->dev; + drv_dev->of_node = np; + dev_set_name(drv_dev, "%s:%pOFn%d", dev_name(drv_dev->parent), np, i); + ret = device_register(drv_dev); + if (ret) + return ret; + + rsc_id = readl_relaxed(drv[i].base + RSC_DRV_ID); + major_ver = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT); + major_ver >>= MAJOR_VER_SHIFT; + minor_ver = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); + minor_ver >>= MINOR_VER_SHIFT; + + if (major_ver >= 3 && minor_ver >= 0) + drv[i].regs = rpmh_rsc_reg_offsets_ver_3_0; + else + drv[i].regs = rpmh_rsc_reg_offsets_ver_2_7; + + ret = rpmh_probe_tcs_config(&drv[i]); + if (ret) + return ret; + + dev_set_drvdata(drv_dev, &drv[i]); + drv[i].initialized = true; } - /* Enable the active TCS to send requests immediately */ - writel_relaxed(drv->tcs[ACTIVE_TCS].mask, - drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); + for (i = 0; i < drv_count; i++) { + if (!drv[i].initialized) + continue; - spin_lock_init(&drv->client.cache_lock); - INIT_LIST_HEAD(&drv->client.cache); - INIT_LIST_HEAD(&drv->client.batch_cache); + /* + * CPU PM notification are not required for controllers that support + * 'HW solver' mode where they can be in autonomous mode executing low + * power mode to power down. + */ + solver_config = readl_relaxed(drv[i].base + + drv[i].regs[DRV_SOLVER_CONFIG]); + solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; + solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; + if (of_find_property(dn, "power-domains", NULL)) { + ret = rpmh_rsc_pd_attach(&drv[i]); + if (ret) + return ret; + } else if (!solver_config) { + drv[i].rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; + cpu_pm_register_notifier(&drv[i].rsc_pm); + } else + drv[i].client.flags = SOLVER_PRESENT; - drv->ipc_log_ctx = ipc_log_context_create(RSC_DRV_IPC_LOG_SIZE, - drv->name, 0); + spin_lock_init(&drv[i].lock); + init_waitqueue_head(&drv[i].tcs_wait); + bitmap_zero(drv[i].tcs_in_use, MAX_TCS_NR); - dev_set_drvdata(&pdev->dev, drv); + irq = platform_get_irq(pdev, drv[i].id); + if (irq < 0) + return irq; - if (__rsc_count < MAX_RSC_COUNT) - __rsc_drv[__rsc_count++] = drv; + drv[i].irq = irq; + + ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, + IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, + drv[i].name, &drv[i]); + if (ret) + return ret; + + spin_lock_init(&drv[i].client.cache_lock); + INIT_LIST_HEAD(&drv[i].client.cache); + INIT_LIST_HEAD(&drv[i].client.batch_cache); + + drv[i].ipc_log_ctx = ipc_log_context_create( + RSC_DRV_IPC_LOG_SIZE, + drv[i].name, 0); + + if (__rsc_count < MAX_RSC_COUNT) + __rsc_drv[__rsc_count++] = &drv[i]; + + ret = devm_of_platform_populate(drv[i].dev); + if (ret) + return ret; + } return devm_of_platform_populate(&pdev->dev); } diff --git a/drivers/soc/qcom/rpmh.c b/drivers/soc/qcom/rpmh.c index b6fc1fe9c59d..d0bc02fc59c2 100644 --- a/drivers/soc/qcom/rpmh.c +++ b/drivers/soc/qcom/rpmh.c @@ -192,7 +192,7 @@ static int __rpmh_write(const struct device *dev, enum rpmh_state state, struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); int ret = -EINVAL; struct cache_req *req; - int i; + int i, ch; /* Cache the request in our store and link the payload */ for (i = 0; i < rpm_msg->msg.num_cmds; i++) { @@ -203,7 +203,12 @@ static int __rpmh_write(const struct device *dev, enum rpmh_state state, if (state == RPMH_ACTIVE_ONLY_STATE) { WARN_ON(irqs_disabled()); - ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg); + + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) + return ch; + + ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg, ch); } else { /* Clean up our call by spoofing tx_done */ ret = 0; @@ -321,7 +326,7 @@ static void cache_batch(struct rpmh_ctrlr *ctrlr, struct batch_cache_req *req) spin_unlock_irqrestore(&ctrlr->cache_lock, flags); } -static int flush_batch(struct rpmh_ctrlr *ctrlr) +static int flush_batch(struct rpmh_ctrlr *ctrlr, int ch) { struct batch_cache_req *req; const struct rpmh_request *rpm_msg; @@ -333,7 +338,7 @@ static int flush_batch(struct rpmh_ctrlr *ctrlr) for (i = 0; i < req->count; i++) { rpm_msg = req->rpm_msgs + i; ret = rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), - &rpm_msg->msg); + &rpm_msg->msg, ch); if (ret) break; } @@ -368,7 +373,7 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state, struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); unsigned long time_left; int count = 0; - int ret, i; + int ret, i, ch; void *ptr; if (rpmh_standalone) @@ -409,12 +414,18 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state, return 0; } + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) { + kfree(ptr); + return ch; + } + for (i = 0; i < count; i++) { struct completion *compl = &compls[i]; init_completion(compl); rpm_msgs[i].completion = compl; - ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg); + ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg, ch); if (ret) { pr_err("Error(%d) sending RPMH message addr=%#x\n", ret, rpm_msgs[i].msg.cmds[0].addr); @@ -450,7 +461,7 @@ static int is_req_valid(struct cache_req *req) } static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state, - u32 addr, u32 data) + u32 addr, u32 data, int ch) { DEFINE_RPMH_MSG_ONSTACK(NULL, state, NULL, rpm_msg); @@ -460,10 +471,10 @@ static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state, rpm_msg.cmd[0].data = data; rpm_msg.msg.num_cmds = 1; - return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg); + return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg, ch); } -int _rpmh_flush(struct rpmh_ctrlr *ctrlr) +int _rpmh_flush(struct rpmh_ctrlr *ctrlr, int ch) { struct cache_req *p; int ret = 0; @@ -474,10 +485,10 @@ int _rpmh_flush(struct rpmh_ctrlr *ctrlr) } /* Invalidate the TCSes first to avoid stale data */ - rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr)); + rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr), ch); /* First flush the cached batch requests */ - ret = flush_batch(ctrlr); + ret = flush_batch(ctrlr, ch); if (ret) return ret; @@ -488,11 +499,11 @@ int _rpmh_flush(struct rpmh_ctrlr *ctrlr) continue; } ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr, - p->sleep_val); + p->sleep_val, ch); if (ret) return ret; ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr, - p->wake_val); + p->wake_val, ch); if (ret) return ret; } @@ -506,12 +517,13 @@ int _rpmh_flush(struct rpmh_ctrlr *ctrlr) * rpmh_flush() - Flushes the buffered sleep and wake sets to TCSes * * @ctrlr: Controller making request to flush cached data + * @ch: Channel number * * Return: * * 0 - Success * * Error code - Otherwise */ -int rpmh_flush(struct rpmh_ctrlr *ctrlr) +int rpmh_flush(struct rpmh_ctrlr *ctrlr, int ch) { int ret; @@ -538,7 +550,7 @@ int rpmh_flush(struct rpmh_ctrlr *ctrlr) */ if (!spin_trylock(&ctrlr->cache_lock)) return -EBUSY; - ret = _rpmh_flush(ctrlr); + ret = _rpmh_flush(ctrlr, ch); spin_unlock(&ctrlr->cache_lock); return ret; @@ -555,7 +567,14 @@ int rpmh_flush(struct rpmh_ctrlr *ctrlr) */ int rpmh_write_sleep_and_wake(const struct device *dev) { - return rpmh_flush(get_rpmh_ctrlr(dev)); + struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); + int ch; + + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) + return ch; + + return rpmh_flush(ctrlr, ch); } EXPORT_SYMBOL(rpmh_write_sleep_and_wake); @@ -653,15 +672,20 @@ int rpmh_init_fast_path(const struct device *dev, { struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); struct tcs_request req; + int ch; if (rpmh_standalone) return 0; + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) + return ch; + req.cmds = cmd; req.num_cmds = n; req.wait_for_compl = 0; - return rpmh_rsc_init_fast_path(ctrlr_to_drv(ctrlr), &req); + return rpmh_rsc_init_fast_path(ctrlr_to_drv(ctrlr), &req, ch); } EXPORT_SYMBOL(rpmh_init_fast_path); @@ -682,15 +706,20 @@ int rpmh_update_fast_path(const struct device *dev, { struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); struct tcs_request req; + int ch; if (rpmh_standalone) return 0; + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) + return ch; + req.cmds = cmd; req.num_cmds = n; req.wait_for_compl = 0; return rpmh_rsc_update_fast_path(ctrlr_to_drv(ctrlr), &req, - update_mask); + update_mask, ch); } EXPORT_SYMBOL(rpmh_update_fast_path);