diff --git a/drivers/soc/qcom/rpmh-internal.h b/drivers/soc/qcom/rpmh-internal.h index 95e641702f4b..0997ce9bdffc 100644 --- a/drivers/soc/qcom/rpmh-internal.h +++ b/drivers/soc/qcom/rpmh-internal.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2018, 2020-2021, The Linux Foundation. All rights reserved. */ @@ -11,6 +11,12 @@ #include #include +#define MAX_NAME_LENGTH 20 + +#define CH0 0 +#define CH1 1 +#define MAX_CHANNEL 2 + #define TCS_TYPE_NR 5 #define MAX_CMDS_PER_TCS 16 #define MAX_TCS_PER_TYPE 3 @@ -62,7 +68,6 @@ struct tcs_group { * @cmd: the payload that will be part of the @msg * @completion: triggered when request is done * @dev: the device making the request - * @err: err return from the controller * @needs_free: check to free dynamically allocated request object */ struct rpmh_request { @@ -70,7 +75,6 @@ struct rpmh_request { struct tcs_cmd cmd[MAX_RPMH_PAYLOAD]; struct completion *completion; const struct device *dev; - int err; bool needs_free; }; @@ -93,21 +97,38 @@ struct rpmh_ctrlr { struct list_head batch_cache; }; +/** + * struct drv_channel: our representation of the drv channels + * + * @tcs: TCS groups. + * @drv: DRV containing the channel + * @initialized: Whether channel is initialized + */ +struct drv_channel { + struct tcs_group tcs[TCS_TYPE_NR]; + struct rsc_drv *drv; + bool initialized; +}; + /** * struct rsc_drv: the Direct Resource Voter (DRV) of the * Resource State Coordinator controller (RSC) * * @name: Controller identifier. + * @base: Base address of the RSC controller. * @tcs_base: Start address of the TCS registers in this controller. + * @reg: Register offsets for RSC controller. * @id: Instance id in the controller (Direct Resource Voter). * @num_tcs: Number of TCSes in this DRV. + * @num_channels: Number of channels in this DRV. * @irq: IRQ at gic. * @in_solver_mode: Controller is busy in solver mode + * @initialized: Whether DRV is initialized * @rsc_pm: CPU PM notifier for controller. * Used when solver mode is not present. * @cpus_in_pm: Number of CPUs not in idle power collapse. * Used when solver mode is not present. - * @tcs: TCS groups. + * @ch: DRV channels. * @tcs_in_use: S/W state of the TCS; only set for ACTIVE_ONLY * transfers, but might show a sleep/wake TCS in use if * it was borrowed for an active_only transfer. You @@ -121,41 +142,51 @@ struct rpmh_ctrlr { * @client: Handle to the DRV's client. * @genpd_nb: PM Domain notifier * @dev: RSC device + * @ipc_log_ctx: IPC logger handle + * @pdev: platform device */ struct rsc_drv { - const char *name; + char name[MAX_NAME_LENGTH]; + void __iomem *base; void __iomem *tcs_base; + u32 *regs; int id; int num_tcs; + int num_channels; int irq; bool in_solver_mode; + bool initialized; struct notifier_block rsc_pm; atomic_t cpus_in_pm; - struct tcs_group tcs[TCS_TYPE_NR]; + struct drv_channel ch[MAX_CHANNEL]; DECLARE_BITMAP(tcs_in_use, MAX_TCS_NR); spinlock_t lock; wait_queue_head_t tcs_wait; struct rpmh_ctrlr client; struct notifier_block genpd_nb; struct device *dev; + void *ipc_log_ctx; + struct platform_device *pdev; }; extern bool rpmh_standalone; -int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg); +int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch); int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, - const struct tcs_request *msg); -void rpmh_rsc_invalidate(struct rsc_drv *drv); + const struct tcs_request *msg, + int ch); +void rpmh_rsc_invalidate(struct rsc_drv *drv, int ch); void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl); int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable); +int rpmh_rsc_get_channel(struct rsc_drv *drv); -void rpmh_tx_done(const struct tcs_request *msg, int r); -int rpmh_flush(struct rpmh_ctrlr *ctrlr); -int _rpmh_flush(struct rpmh_ctrlr *ctrlr); +void rpmh_tx_done(const struct tcs_request *msg); +int rpmh_flush(struct rpmh_ctrlr *ctrlr, int ch); +int _rpmh_flush(struct rpmh_ctrlr *ctrlr, int ch); -int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg); +int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, int ch); int rpmh_rsc_update_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, - u32 update_mask); + u32 update_mask, int ch); #endif /* __RPM_INTERNAL_H__ */ diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index bc54853e4b4b..011bcb0825da 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2018, 2020-2021, The Linux Foundation. All rights reserved. */ #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME @@ -34,50 +34,27 @@ #define CREATE_TRACE_POINTS #include "trace-rpmh.h" -#define RSC_DRV_TCS_OFFSET 672 -#define RSC_DRV_CMD_OFFSET 20 +#include -/* DRV HW Solver Configuration Information Register */ -#define DRV_SOLVER_CONFIG 0x04 +#define RSC_DRV_IPC_LOG_SIZE 2 + +/* DRV ID Register */ +#define RSC_DRV_ID 0 +#define MAJOR_VER_MASK 0xFF +#define MAJOR_VER_SHIFT 16 +#define MINOR_VER_MASK 0xFF +#define MINOR_VER_SHIFT 8 + +/* DRV HW Solver Configuration Register Mask */ #define DRV_HW_SOLVER_MASK 1 #define DRV_HW_SOLVER_SHIFT 24 -/* DRV TCS Configuration Information Register */ -#define DRV_PRNT_CHLD_CONFIG 0x0C +/* DRV TCS Configuration Information Register Mask */ #define DRV_NUM_TCS_MASK 0x3F #define DRV_NUM_TCS_SHIFT 6 #define DRV_NCPT_MASK 0x1F #define DRV_NCPT_SHIFT 27 -/* Offsets for common TCS Registers, one bit per TCS */ -#define RSC_DRV_IRQ_ENABLE 0x00 -#define RSC_DRV_IRQ_STATUS 0x04 -#define RSC_DRV_IRQ_CLEAR 0x08 /* w/o; write 1 to clear */ - -/* - * Offsets for per TCS Registers. - * - * TCSes start at 0x10 from tcs_base and are stored one after another. - * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one - * of the below to find a register. - */ -#define RSC_DRV_CMD_WAIT_FOR_CMPL 0x10 /* 1 bit per command */ -#define RSC_DRV_CONTROL 0x14 -#define RSC_DRV_STATUS 0x18 /* zero if tcs is busy */ -#define RSC_DRV_CMD_ENABLE 0x1C /* 1 bit per command */ - -/* - * Offsets for per command in a TCS. - * - * Commands (up to 16) start at 0x30 in a TCS; multiply command index - * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. - */ -#define RSC_DRV_CMD_MSGID 0x30 -#define RSC_DRV_CMD_ADDR 0x34 -#define RSC_DRV_CMD_DATA 0x38 -#define RSC_DRV_CMD_STATUS 0x3C -#define RSC_DRV_CMD_RESP_DATA 0x40 - #define TCS_AMC_MODE_ENABLE BIT(16) #define TCS_AMC_MODE_TRIGGER BIT(24) @@ -88,9 +65,18 @@ #define CMD_STATUS_ISSUED BIT(8) #define CMD_STATUS_COMPL BIT(16) +/* Offsets for DRV channel status register */ +#define CH0_CHN_BUSY BIT(0) +#define CH1_CHN_BUSY BIT(1) +#define CH0_WAKE_TCS_STATUS BIT(0) +#define CH0_SLEEP_TCS_STATUS BIT(1) +#define CH1_WAKE_TCS_STATUS BIT(2) +#define CH1_SLEEP_TCS_STATUS BIT(3) +#define CH_CLEAR_STATUS BIT(31) + #define ACCL_TYPE(addr) ((addr >> 16) & 0xF) #define NR_ACCL_TYPES 3 -#define MAX_RSC_COUNT 2 +#define MAX_RSC_COUNT 5 static const char * const accl_str[] = { "", "", "", "CLK", "VREG", "BUS", @@ -154,16 +140,89 @@ bool rpmh_standalone; * +---------------------------------------------------+ */ +enum { + RSC_DRV_TCS_OFFSET, + RSC_DRV_CMD_OFFSET, +/* DRV HW Solver Configuration Information Register */ + DRV_SOLVER_CONFIG, +/* DRV TCS Configuration Information Register */ + DRV_PRNT_CHLD_CONFIG, +/* Offsets for common TCS Registers, one bit per TCS */ + RSC_DRV_IRQ_ENABLE, + RSC_DRV_IRQ_STATUS, + RSC_DRV_IRQ_CLEAR, /* w/o; write 1 to clear */ +/* + * Offsets for per TCS Registers. + * + * TCSes start at 0x10 from tcs_base and are stored one after another. + * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one + * of the below to find a register. + */ + RSC_DRV_CMD_WAIT_FOR_CMPL, /* 1 bit per command */ + RSC_DRV_CONTROL, + RSC_DRV_STATUS, /* zero if tcs is busy */ + RSC_DRV_CMD_ENABLE, /* 1 bit per command */ +/* + * Offsets for per command in a TCS. + * + * Commands (up to 16) start at 0x30 in a TCS; multiply command index + * by RSC_DRV_CMD_OFFSET and add one of the below to find a register. + */ + RSC_DRV_CMD_MSGID, + RSC_DRV_CMD_ADDR, + RSC_DRV_CMD_DATA, + RSC_DRV_CMD_STATUS, + RSC_DRV_CMD_RESP_DATA, +}; + +static u32 rpmh_rsc_reg_offsets_ver_2_7[] = { + [RSC_DRV_TCS_OFFSET] = 672, + [RSC_DRV_CMD_OFFSET] = 20, + [DRV_SOLVER_CONFIG] = 0x04, + [DRV_PRNT_CHLD_CONFIG] = 0x0C, + [RSC_DRV_IRQ_ENABLE] = 0x00, + [RSC_DRV_IRQ_STATUS] = 0x04, + [RSC_DRV_IRQ_CLEAR] = 0x08, + [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10, + [RSC_DRV_CONTROL] = 0x14, + [RSC_DRV_STATUS] = 0x18, + [RSC_DRV_CMD_ENABLE] = 0x1C, + [RSC_DRV_CMD_MSGID] = 0x30, + [RSC_DRV_CMD_ADDR] = 0x34, + [RSC_DRV_CMD_DATA] = 0x38, + [RSC_DRV_CMD_STATUS] = 0x3C, + [RSC_DRV_CMD_RESP_DATA] = 0x40, +}; + +static u32 rpmh_rsc_reg_offsets_ver_3_0[] = { + [RSC_DRV_TCS_OFFSET] = 672, + [RSC_DRV_CMD_OFFSET] = 24, + [DRV_SOLVER_CONFIG] = 0x04, + [DRV_PRNT_CHLD_CONFIG] = 0x0C, + [RSC_DRV_IRQ_ENABLE] = 0x00, + [RSC_DRV_IRQ_STATUS] = 0x04, + [RSC_DRV_IRQ_CLEAR] = 0x08, + [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20, + [RSC_DRV_CONTROL] = 0x24, + [RSC_DRV_STATUS] = 0x28, + [RSC_DRV_CMD_ENABLE] = 0x2C, + [RSC_DRV_CMD_MSGID] = 0x34, + [RSC_DRV_CMD_ADDR] = 0x38, + [RSC_DRV_CMD_DATA] = 0x3C, + [RSC_DRV_CMD_STATUS] = 0x40, + [RSC_DRV_CMD_RESP_DATA] = 0x44, +}; + static inline void __iomem * tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) { - return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg; + return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg; } static inline void __iomem * tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) { - return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id; + return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id; } static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, @@ -213,6 +272,7 @@ static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). * @drv: The RSC controller. * @type: SLEEP_TCS or WAKE_TCS + * @ch: Channel number * * This will clear the "slots" variable of the given tcs_group and also * tell the hardware to forget about all entries. @@ -221,33 +281,47 @@ static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, * function is called, since otherwise the device may immediately become * used again even before this function exits. */ -static void tcs_invalidate(struct rsc_drv *drv, int type) +static void tcs_invalidate(struct rsc_drv *drv, int type, int ch) { int m; - struct tcs_group *tcs = &drv->tcs[type]; + struct tcs_group *tcs = &drv->ch[ch].tcs[type]; /* Caller ensures nobody else is running so no lock */ - if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) + if (bitmap_empty(tcs->slots, tcs->ncpt * tcs->num_tcs)) return; for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) - write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0); + bitmap_zero(tcs->slots, tcs->ncpt * tcs->num_tcs); +} - bitmap_zero(tcs->slots, MAX_TCS_SLOTS); +/** + * rpmh_rsc_get_channel() - Get the Unused channel to send data on. + * @drv: The RSC controller. + * + * Return: 0 on success, else -error. + */ +int rpmh_rsc_get_channel(struct rsc_drv *drv) +{ + if (drv->num_channels == 1) + return CH0; + else + return -EBUSY; } /** * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. * @drv: The RSC controller. + * @ch: Channel number * - * The caller must ensure that no other RPMH actions are happening when this + * The caller must ensure that no other RPMH actions are happening when thi]s * function is called, since otherwise the device may immediately become * used again even before this function exits. */ -void rpmh_rsc_invalidate(struct rsc_drv *drv) +void rpmh_rsc_invalidate(struct rsc_drv *drv, int ch) { - tcs_invalidate(drv, SLEEP_TCS); - tcs_invalidate(drv, WAKE_TCS); + tcs_invalidate(drv, SLEEP_TCS, ch); + tcs_invalidate(drv, WAKE_TCS, ch); } /** @@ -261,12 +335,13 @@ void rpmh_rsc_invalidate(struct rsc_drv *drv) * Return: A pointer to a tcs_group or an ERR_PTR. */ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, - const struct tcs_request *msg) + enum rpmh_state state, + int ch) { int type; struct tcs_group *tcs; - switch (msg->state) { + switch (state) { case RPMH_ACTIVE_ONLY_STATE: type = ACTIVE_TCS; break; @@ -287,9 +362,9 @@ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, * transfers have finished before we use it (maybe by running from * the last CPU in PM code). */ - tcs = &drv->tcs[type]; - if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) - tcs = &drv->tcs[WAKE_TCS]; + tcs = &drv->ch[ch].tcs[type]; + if (state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) + tcs = &drv->ch[ch].tcs[WAKE_TCS]; return tcs; } @@ -311,15 +386,21 @@ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, * Return: The stashed request. */ static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, - int tcs_id) + int tcs_id, + int *ch) { struct tcs_group *tcs; int i; - for (i = 0; i < TCS_TYPE_NR; i++) { - tcs = &drv->tcs[i]; - if (tcs->mask & BIT(tcs_id)) + for (i = 0; i < MAX_CHANNEL; i++) { + if (!drv->ch[i].initialized) + continue; + + tcs = get_tcs_for_msg(drv, RPMH_ACTIVE_ONLY_STATE, i); + if (tcs->mask & BIT(tcs_id)) { + *ch = i; return tcs->req[tcs_id - tcs->offset]; + } } return NULL; @@ -351,18 +432,18 @@ static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) * While clearing ensure that the AMC mode trigger is cleared * and then the mode enable is cleared. */ - enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); + enable = read_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id); enable &= ~TCS_AMC_MODE_TRIGGER; - write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CONTROL], tcs_id, enable); enable &= ~TCS_AMC_MODE_ENABLE; - write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CONTROL], tcs_id, enable); if (trigger) { /* Enable the AMC mode on the TCS and then trigger the TCS */ enable = TCS_AMC_MODE_ENABLE; - write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CONTROL], tcs_id, enable); enable |= TCS_AMC_MODE_TRIGGER; - write_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id, enable); + write_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id, enable); } } @@ -379,12 +460,12 @@ static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) { u32 data; - data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE); + data = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); if (enable) data |= BIT(tcs_id); else data &= ~BIT(tcs_id); - writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE); + writel_relaxed(data, drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); } /** @@ -400,46 +481,31 @@ static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) static irqreturn_t tcs_tx_done(int irq, void *p) { struct rsc_drv *drv = p; - int i, j, err = 0; + int i, ch; unsigned long irq_status; const struct tcs_request *req; - struct tcs_cmd *cmd; - irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); + irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) { - req = get_req_from_tcs(drv, i); + req = get_req_from_tcs(drv, i, &ch); if (WARN_ON(!req)) goto skip; - err = 0; - for (j = 0; j < req->num_cmds; j++) { - u32 sts; - - cmd = &req->cmds[j]; - sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j); - if (!(sts & CMD_STATUS_ISSUED) || - ((req->wait_for_compl || cmd->wait) && - !(sts & CMD_STATUS_COMPL))) { - pr_err("Incomplete request: %s: addr=%#x data=%#x", - drv->name, cmd->addr, cmd->data); - err = -EIO; - } - } - - trace_rpmh_tx_done(drv, i, req, err); + trace_rpmh_tx_done(drv, i, req); + ipc_log_string(drv->ipc_log_ctx, "IRQ response: m=%d", i); /* * If wake tcs was re-purposed for sending active * votes, clear AMC trigger & enable modes and * disable interrupt for this TCS */ - if (!drv->tcs[ACTIVE_TCS].num_tcs) + if (!drv->ch[ch].tcs[ACTIVE_TCS].num_tcs) __tcs_set_trigger(drv, i, false); skip: /* Reclaim the TCS */ - write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0); - writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR); + write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0); + writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]); spin_lock(&drv->lock); clear_bit(i, drv->tcs_in_use); /* @@ -447,12 +513,12 @@ static irqreturn_t tcs_tx_done(int irq, void *p) * spammed with interrupts coming when the solver * sends its wake votes. */ - if (!drv->tcs[ACTIVE_TCS].num_tcs) + if (!drv->ch[ch].tcs[ACTIVE_TCS].num_tcs) enable_tcs_irq(drv, i, false); spin_unlock(&drv->lock); wake_up(&drv->tcs_wait); if (req) - rpmh_tx_done(req, err); + rpmh_tx_done(req); } return IRQ_HANDLED; @@ -490,14 +556,18 @@ static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, */ msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; - write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid); - write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr); - write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data); + write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid); + write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr); + write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data); trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd); + ipc_log_string(drv->ipc_log_ctx, + "TCS write: m=%d n=%d msgid=%#x addr=%#x data=%#x wait=%d", + tcs_id, j, msgid, cmd->addr, + cmd->data, cmd->wait); } - cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); - write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable); + cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); + write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable); } /** @@ -529,10 +599,10 @@ static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, int i = tcs->offset; for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) { - curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i); + curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i); - for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { - addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, i, j); + for_each_set_bit(j, &curr_enabled, tcs->ncpt) { + addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j); for (k = 0; k < msg->num_cmds; k++) { if (addr == msg->cmds[k].addr) return -EBUSY; @@ -600,6 +670,7 @@ static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, * rpmh_rsc_send_data() - Write / trigger active-only message. * @drv: The controller. * @msg: The data to be sent. + * @ch: Channel number * * NOTES: * - This is only used for "ACTIVE_ONLY" since the limitations of this @@ -618,13 +689,13 @@ static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, * * Return: 0 on success, -EINVAL on error. */ -int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) +int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { struct tcs_group *tcs; int tcs_id; unsigned long flags; - tcs = get_tcs_for_msg(drv, msg); + tcs = get_tcs_for_msg(drv, msg->state, ch); if (IS_ERR(tcs)) return PTR_ERR(tcs); @@ -649,7 +720,7 @@ int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) * repurposed TCS to avoid triggering them. tcs->slots will be * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() */ - write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0); + write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0); enable_tcs_irq(drv, tcs_id, true); } spin_unlock_irqrestore(&drv->lock, flags); @@ -664,6 +735,7 @@ int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) */ __tcs_buffer_write(drv, tcs_id, 0, msg); __tcs_set_trigger(drv, tcs_id, true); + ipc_log_string(drv->ipc_log_ctx, "TCS trigger: m=%d", tcs_id); return 0; } @@ -691,7 +763,8 @@ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, /* Do over, until we can fit the full payload in a single TCS */ do { - slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS, + slot = bitmap_find_next_zero_area(tcs->slots, + tcs->ncpt * tcs->num_tcs, i, msg->num_cmds, 0); if (slot >= tcs->num_tcs * tcs->ncpt) return -ENOMEM; @@ -711,6 +784,7 @@ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. * @drv: The controller. * @msg: The data to be written to the controller. + * @ch: Channel number * * This should only be called for sleep/wake state, never active-only * state. @@ -720,13 +794,13 @@ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, * * Return: 0 if no error; else -error. */ -int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) +int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { struct tcs_group *tcs; int tcs_id = 0, cmd_id = 0; int ret; - tcs = get_tcs_for_msg(drv, msg); + tcs = get_tcs_for_msg(drv, msg->state, ch); if (IS_ERR(tcs)) return PTR_ERR(tcs); @@ -740,11 +814,16 @@ int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) static struct tcs_group *get_tcs_from_index(struct rsc_drv *drv, int tcs_id) { - unsigned int i; + unsigned int i, j; for (i = 0; i < TCS_TYPE_NR; i++) { - if (drv->tcs[i].mask & BIT(tcs_id)) - return &drv->tcs[i]; + for (j = 0; j < MAX_CHANNEL; j++) { + if (!drv->ch[j].initialized) + continue; + + if (drv->ch[j].tcs[i].mask & BIT(tcs_id)) + return &drv->ch[j].tcs[i]; + } } return NULL; @@ -753,23 +832,24 @@ static struct tcs_group *get_tcs_from_index(struct rsc_drv *drv, int tcs_id) static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, bool *aoss_irq_sts) { + int ch = 0; struct tcs_group *tcs_grp = get_tcs_from_index(drv, tcs_id); - const struct tcs_request *req = get_req_from_tcs(drv, tcs_id); + const struct tcs_request *req = get_req_from_tcs(drv, tcs_id, &ch); unsigned long cmds_enabled; u32 addr, data, msgid, sts, irq_sts; bool in_use = test_bit(tcs_id, drv->tcs_in_use); int i; - if (!tcs_grp || !req) + sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); + cmds_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); + if (!cmds_enabled || !tcs_grp) return; - sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); - cmds_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id); - if (!cmds_enabled) - return; + if (!req) + goto print_tcs_data; - data = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id); - irq_sts = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS); + data = read_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id); + irq_sts = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); pr_warn("Request: tcs-in-use:%s active_tcs=%s(%d) state=%d wait_for_compl=%u]\n", (in_use ? "YES" : "NO"), ((tcs_grp->type == ACTIVE_TCS) ? "YES" : "NO"), @@ -780,11 +860,12 @@ static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, *aoss_irq_sts = (irq_sts & BIT(tcs_id)) ? true : false; - for_each_set_bit(i, &cmds_enabled, MAX_CMDS_PER_TCS) { - addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, i); - data = read_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, i); - msgid = read_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, i); - sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, tcs_id, i); +print_tcs_data: + for_each_set_bit(i, &cmds_enabled, tcs_grp->ncpt) { + addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, i); + data = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i); + msgid = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, i); + sts = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_STATUS], tcs_id, i); pr_warn("\tCMD=%d [addr=0x%x data=0x%x hdr=0x%x sts=0x%x enabled=1]\n", i, addr, data, msgid, sts); if (!(sts & CMD_STATUS_ISSUED)) @@ -834,6 +915,23 @@ void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl) else if (gic_irq_sts) pr_warn("ERROR:Possible lockup in Linux\n"); + /* Show fast path status, if the TCS is busy */ + for (i = 0; i < MAX_CHANNEL; i++) { + if (!drv->ch[i].initialized) + continue; + + /* Show fast path status, if the TCS is busy */ + if (drv->ch[i].tcs[FAST_PATH_TCS].num_tcs) { + int tcs_id = drv->ch[i].tcs[FAST_PATH_TCS].offset; + bool sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); + + if (!sts) { + pr_err("Fast-path TCS information:\n"); + print_tcs_info(drv, tcs_id, &accl, &aoss_irq_sts); + } + } + } + /* * The TCS(s) are busy waiting, we have no way to recover from this. * If this debug function is called, we assume it's because timeout @@ -860,28 +958,38 @@ void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl) */ static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) { + int i; + struct tcs_group *tcs; unsigned long set; - const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; unsigned long max; - /* - * If we made an active request on a RSC that does not have a - * dedicated TCS for active state use, then re-purposed wake TCSes - * should be checked for not busy, because we used wake TCSes for - * active requests in this case. - */ - if (!tcs->num_tcs) - tcs = &drv->tcs[WAKE_TCS]; + for (i = 0; i < MAX_CHANNEL; i++) { + if (!drv->ch[i].initialized) + continue; - max = tcs->offset + tcs->num_tcs; - set = find_next_bit(drv->tcs_in_use, max, tcs->offset); + tcs = &drv->ch[i].tcs[ACTIVE_TCS]; + /* + * If we made an active request on a RSC that does not have a + * dedicated TCS for active state use, then re-purposed wake TCSes + * should be checked for not busy, because we used wake TCSes for + * active requests in this case. + */ + if (!tcs->num_tcs) + tcs = &drv->ch[i].tcs[WAKE_TCS]; - /* Check if there is pending fastpath transaction */ - if (drv->tcs[FAST_PATH_TCS].num_tcs && - !read_tcs_reg(drv, RSC_DRV_STATUS, drv->tcs[FAST_PATH_TCS].offset)) - return true; + max = tcs->offset + tcs->num_tcs; + set = find_next_bit(drv->tcs_in_use, max, tcs->offset); + if (set < max) + return true; - return set < max; + /* Check if there is pending fastpath transaction */ + tcs = &drv->ch[i].tcs[FAST_PATH_TCS]; + if (tcs->num_tcs && + !read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs->offset)) + return true; + } + + return false; } /** @@ -906,7 +1014,7 @@ static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, { struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); int ret = NOTIFY_OK; - int cpus_in_pm; + int cpus_in_pm, ch; switch (action) { case CPU_PM_ENTER: @@ -945,7 +1053,8 @@ static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, * CPU. */ if (spin_trylock(&drv->lock)) { - if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)) + ch = rpmh_rsc_get_channel(drv); + if (ch < 0 || rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client, ch)) ret = NOTIFY_BAD; spin_unlock(&drv->lock); } else { @@ -982,6 +1091,8 @@ int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable) if (!enable || !rpmh_rsc_ctrlr_is_busy(drv)) { drv->in_solver_mode = enable; trace_rpmh_solver_set(drv, enable); + ipc_log_string(drv->ipc_log_ctx, + "solver mode set: %d", enable); ret = 0; } spin_unlock(&drv->lock); @@ -994,19 +1105,20 @@ int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable) * rpmh_rsc_init_fast_path() - Initialize the fast-path TCS contents * @drv: The controller. * @msg: The TCS request to populate. + * @ch: Channel number * * Return: * * 0 - success * * -ENODEV - no fast-path TCS available */ -int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg) +int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { int tcs_id; - if (!drv->tcs[FAST_PATH_TCS].num_tcs) + if (!drv->ch[ch].tcs[FAST_PATH_TCS].num_tcs) return -ENODEV; - tcs_id = drv->tcs[FAST_PATH_TCS].offset; + tcs_id = drv->ch[ch].tcs[FAST_PATH_TCS].offset; /* We won't use the AMC IRQ to confirm if the TCS is free */ enable_tcs_irq(drv, tcs_id, false); @@ -1021,6 +1133,7 @@ int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg) * @drv: The controller. * @msg: The TCS request data to be updated. * @mask: The update mask for elements in @msg to be sent + * @ch: Channel number * * NOTE: Caller should ensure serialization before making this call. * Return: @@ -1029,7 +1142,7 @@ int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg) */ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, - u32 mask) + u32 mask, int ch) { int i; u32 sts; @@ -1037,14 +1150,14 @@ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, struct tcs_cmd *cmd; int retry = 5; - if (!drv->tcs[FAST_PATH_TCS].num_tcs) + if (!drv->ch[ch].tcs[FAST_PATH_TCS].num_tcs) return -ENODEV; - tcs_id = drv->tcs[FAST_PATH_TCS].offset; + tcs_id = drv->ch[ch].tcs[FAST_PATH_TCS].offset; /* Ensure the TCS is free before writing to the TCS */ do { - sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id); + sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); if (!sts) { retry--; /* Report and bail, if it took too many attempts */ @@ -1065,7 +1178,7 @@ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, if (!(mask & BIT(i))) continue; cmd = &msg->cmds[i]; - write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, i, cmd->data); + write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i, cmd->data); } /* Trigger the TCS to send the request */ @@ -1074,45 +1187,32 @@ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, return 0; } -static int rpmh_probe_tcs_config(struct platform_device *pdev, - struct rsc_drv *drv, void __iomem *base) +static int rpmh_probe_channel_tcs_config(struct device_node *np, + struct rsc_drv *drv, + u32 max_tcs, u32 ncpt, int ch) { struct tcs_type_config { u32 type; u32 n; } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; - struct device_node *dn = pdev->dev.of_node; - u32 config, max_tcs, ncpt, offset; - int i, ret, n, st = 0; struct tcs_group *tcs; + struct drv_channel *channel = &drv->ch[ch]; + int i, ret, n, st = 0; + u32 tcs_mask; - ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset); - if (ret) - return ret; - drv->tcs_base = base + offset; - - config = readl_relaxed(base + DRV_PRNT_CHLD_CONFIG); - - max_tcs = config; - max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); - max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); - - ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); - ncpt = ncpt >> DRV_NCPT_SHIFT; - - n = of_property_count_u32_elems(dn, "qcom,tcs-config"); + n = of_property_count_u32_elems(np, "qcom,tcs-config"); if (n != 2 * TCS_TYPE_NR) return -EINVAL; for (i = 0; i < TCS_TYPE_NR; i++) { - ret = of_property_read_u32_index(dn, "qcom,tcs-config", + ret = of_property_read_u32_index(np, "qcom,tcs-config", i * 2, &tcs_cfg[i].type); if (ret) return ret; if (tcs_cfg[i].type >= TCS_TYPE_NR) return -EINVAL; - ret = of_property_read_u32_index(dn, "qcom,tcs-config", + ret = of_property_read_u32_index(np, "qcom,tcs-config", i * 2 + 1, &tcs_cfg[i].n); if (ret) return ret; @@ -1121,7 +1221,7 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev, } for (i = 0; i < TCS_TYPE_NR; i++) { - tcs = &drv->tcs[tcs_cfg[i].type]; + tcs = &channel->tcs[tcs_cfg[i].type]; if (tcs->drv) return -EINVAL; tcs->drv = drv; @@ -1136,12 +1236,54 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev, st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) return -EINVAL; - tcs->mask = ((1 << tcs->num_tcs) - 1) << st; - tcs->offset = st; + tcs->mask = ((1 << tcs->num_tcs) - 1) << (st + drv->num_tcs); + tcs->offset = st + drv->num_tcs; st += tcs->num_tcs; } - drv->num_tcs = st; + /* Enable the active TCS to send requests immediately */ + tcs_mask = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); + tcs_mask |= drv->ch[ch].tcs[ACTIVE_TCS].mask; + writel_relaxed(tcs_mask, drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); + + channel->drv = drv; + channel->initialized = true; + drv->num_tcs += st; + + return 0; +} + +static int rpmh_probe_tcs_config(struct rsc_drv *drv) +{ + struct device_node *cn, *np = drv->dev->of_node; + int ch = 0, ret; + u32 offset, config; + u32 max_tcs, ncpt; + + ret = of_property_read_u32(np, "qcom,tcs-offset", &offset); + if (ret) + return ret; + drv->tcs_base = drv->base + offset; + + config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]); + + max_tcs = config; + max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); + max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); + + ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); + ncpt = ncpt >> DRV_NCPT_SHIFT; + for_each_child_of_node(np, cn) { + + if (!of_node_name_eq(cn, "channel")) + continue; + + ret = rpmh_probe_channel_tcs_config(cn, drv, max_tcs, ncpt, ch); + if (ret) + return ret; + ch++; + } + drv->num_channels = ch; return 0; } @@ -1150,10 +1292,15 @@ static int rpmh_rsc_pd_cb(struct notifier_block *nb, unsigned long action, void *data) { struct rsc_drv *drv = container_of(nb, struct rsc_drv, genpd_nb); + int ch; + + if (action != GENPD_NOTIFY_PRE_OFF) + return NOTIFY_OK; + + ch = rpmh_rsc_get_channel(drv); /* We don't need to lock as domin on/off are serialized */ - if ((action == GENPD_NOTIFY_PRE_OFF) && - (rpmh_rsc_ctrlr_is_busy(drv) || _rpmh_flush(&drv->client))) + if (ch < 0 || rpmh_rsc_ctrlr_is_busy(drv) || _rpmh_flush(&drv->client, ch)) return NOTIFY_BAD; return NOTIFY_OK; @@ -1163,23 +1310,23 @@ static int rpmh_rsc_pd_attach(struct rsc_drv *drv) { int ret; - pm_runtime_enable(drv->dev); - ret = dev_pm_domain_attach(drv->dev, false); + pm_runtime_enable(&drv->pdev->dev); + ret = dev_pm_domain_attach(&drv->pdev->dev, false); if (ret) return ret; drv->genpd_nb.notifier_call = rpmh_rsc_pd_cb; - return dev_pm_genpd_add_notifier(drv->dev, &drv->genpd_nb); + return dev_pm_genpd_add_notifier(&drv->pdev->dev, &drv->genpd_nb); } static int rpmh_rsc_probe(struct platform_device *pdev) { - struct device_node *dn = pdev->dev.of_node; + struct device_node *np, *dn = pdev->dev.of_node; struct rsc_drv *drv; - char drv_id[10] = {0}; int ret, irq; - u32 solver_config; - void __iomem *base; + u32 rsc_id, major_ver, minor_ver, solver_config; + int i, drv_count; + const char *name; /* * Even though RPMh doesn't directly use cmd-db, all of its children @@ -1198,78 +1345,121 @@ static int rpmh_rsc_probe(struct platform_device *pdev) dev_info(&pdev->dev, "RPMH is running in standalone mode.\n"); - drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL); + ret = of_property_read_u32(dn, "qcom,drv-count", &drv_count); + if (ret) + return ret; + + drv = devm_kcalloc(&pdev->dev, drv_count, sizeof(*drv), GFP_KERNEL); if (!drv) return -ENOMEM; - ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id); - if (ret) - return ret; + name = of_get_property(dn, "label", NULL); + if (!name) + name = dev_name(&pdev->dev); - drv->name = of_get_property(dn, "label", NULL); - if (!drv->name) - drv->name = dev_name(&pdev->dev); + for_each_child_of_node(dn, np) { + struct device *drv_dev; - snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id); - base = devm_platform_ioremap_resource_byname(pdev, drv_id); - if (IS_ERR(base)) - return PTR_ERR(base); + if (!of_node_name_eq(np, "drv")) + continue; - ret = rpmh_probe_tcs_config(pdev, drv, base); - if (ret) - return ret; - - spin_lock_init(&drv->lock); - init_waitqueue_head(&drv->tcs_wait); - bitmap_zero(drv->tcs_in_use, MAX_TCS_NR); - - irq = platform_get_irq(pdev, drv->id); - if (irq < 0) - return irq; - - drv->irq = irq; - - ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, - IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, - drv->name, drv); - if (ret) - return ret; - - drv->dev = &pdev->dev; - /* - * CPU PM notification are not required for controllers that support - * 'HW solver' mode where they can be in autonomous mode executing low - * power mode to power down. - */ - solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG); - solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; - solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; - if (of_find_property(dn, "power-domains", NULL)) { - ret = rpmh_rsc_pd_attach(drv); - if (ret == -EPROBE_DEFER) { - pr_err("Failed to attach RSC %s to domain ret=%d\n", drv->name, ret); + ret = of_property_read_u32(np, "qcom,drv-id", &i); + if (ret) return ret; - } - } else if (!solver_config) { - drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; - cpu_pm_register_notifier(&drv->rsc_pm); - drv->client.flags &= ~SOLVER_PRESENT; - } else { - drv->client.flags |= SOLVER_PRESENT; + + scnprintf(drv[i].name, sizeof(drv[i].name), "%s-drv-%d", name, i); + + drv[i].base = devm_platform_ioremap_resource(pdev, i); + if (IS_ERR(drv[i].base)) + return PTR_ERR(drv[i].base); + + drv_dev = kzalloc(sizeof(*drv_dev), GFP_KERNEL); + if (!drv_dev) + return -ENOMEM; + + drv[i].id = i; + drv[i].pdev = pdev; + drv[i].dev = drv_dev; + drv_dev->parent = &pdev->dev; + drv_dev->of_node = np; + dev_set_name(drv_dev, "%s:%pOFn%d", dev_name(drv_dev->parent), np, i); + ret = device_register(drv_dev); + if (ret) + return ret; + + rsc_id = readl_relaxed(drv[i].base + RSC_DRV_ID); + major_ver = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT); + major_ver >>= MAJOR_VER_SHIFT; + minor_ver = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); + minor_ver >>= MINOR_VER_SHIFT; + + if (major_ver >= 3 && minor_ver >= 0) + drv[i].regs = rpmh_rsc_reg_offsets_ver_3_0; + else + drv[i].regs = rpmh_rsc_reg_offsets_ver_2_7; + + ret = rpmh_probe_tcs_config(&drv[i]); + if (ret) + return ret; + + dev_set_drvdata(drv_dev, &drv[i]); + drv[i].initialized = true; } - /* Enable the active TCS to send requests immediately */ - writel_relaxed(drv->tcs[ACTIVE_TCS].mask, - drv->tcs_base + RSC_DRV_IRQ_ENABLE); + for (i = 0; i < drv_count; i++) { + if (!drv[i].initialized) + continue; - spin_lock_init(&drv->client.cache_lock); - INIT_LIST_HEAD(&drv->client.cache); - INIT_LIST_HEAD(&drv->client.batch_cache); + /* + * CPU PM notification are not required for controllers that support + * 'HW solver' mode where they can be in autonomous mode executing low + * power mode to power down. + */ + solver_config = readl_relaxed(drv[i].base + + drv[i].regs[DRV_SOLVER_CONFIG]); + solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; + solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; + if (of_find_property(dn, "power-domains", NULL)) { + ret = rpmh_rsc_pd_attach(&drv[i]); + if (ret) + return ret; + } else if (!solver_config) { + drv[i].rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; + cpu_pm_register_notifier(&drv[i].rsc_pm); + } else + drv[i].client.flags = SOLVER_PRESENT; - dev_set_drvdata(&pdev->dev, drv); + spin_lock_init(&drv[i].lock); + init_waitqueue_head(&drv[i].tcs_wait); + bitmap_zero(drv[i].tcs_in_use, MAX_TCS_NR); - if (__rsc_count < MAX_RSC_COUNT) - __rsc_drv[__rsc_count++] = drv; + irq = platform_get_irq(pdev, drv[i].id); + if (irq < 0) + return irq; + + drv[i].irq = irq; + + ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, + IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, + drv[i].name, &drv[i]); + if (ret) + return ret; + + spin_lock_init(&drv[i].client.cache_lock); + INIT_LIST_HEAD(&drv[i].client.cache); + INIT_LIST_HEAD(&drv[i].client.batch_cache); + + drv[i].ipc_log_ctx = ipc_log_context_create( + RSC_DRV_IPC_LOG_SIZE, + drv[i].name, 0); + + if (__rsc_count < MAX_RSC_COUNT) + __rsc_drv[__rsc_count++] = &drv[i]; + + ret = devm_of_platform_populate(drv[i].dev); + if (ret) + return ret; + } return devm_of_platform_populate(&pdev->dev); } diff --git a/drivers/soc/qcom/rpmh.c b/drivers/soc/qcom/rpmh.c index 43a06d93cd56..d0bc02fc59c2 100644 --- a/drivers/soc/qcom/rpmh.c +++ b/drivers/soc/qcom/rpmh.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2018, 2020-2021, The Linux Foundation. All rights reserved. */ #include @@ -66,7 +66,7 @@ struct cache_req { struct batch_cache_req { struct list_head list; int count; - struct rpmh_request rpm_msgs[]; + struct rpmh_request *rpm_msgs; }; static struct rpmh_ctrlr *get_rpmh_ctrlr(const struct device *dev) @@ -95,19 +95,13 @@ static int check_ctrlr_state(struct rpmh_ctrlr *ctrlr, enum rpmh_state state) return ret; } -void rpmh_tx_done(const struct tcs_request *msg, int r) +void rpmh_tx_done(const struct tcs_request *msg) { struct rpmh_request *rpm_msg = container_of(msg, struct rpmh_request, msg); struct completion *compl = rpm_msg->completion; bool free = rpm_msg->needs_free; - rpm_msg->err = r; - - if (r) - dev_err(rpm_msg->dev, "RPMH TX fail in msg addr=%#x, err=%d\n", - rpm_msg->msg.cmds[0].addr, r); - if (!compl) goto exit; @@ -198,7 +192,7 @@ static int __rpmh_write(const struct device *dev, enum rpmh_state state, struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); int ret = -EINVAL; struct cache_req *req; - int i; + int i, ch; /* Cache the request in our store and link the payload */ for (i = 0; i < rpm_msg->msg.num_cmds; i++) { @@ -209,11 +203,16 @@ static int __rpmh_write(const struct device *dev, enum rpmh_state state, if (state == RPMH_ACTIVE_ONLY_STATE) { WARN_ON(irqs_disabled()); - ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg); + + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) + return ch; + + ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msg->msg, ch); } else { /* Clean up our call by spoofing tx_done */ ret = 0; - rpmh_tx_done(&rpm_msg->msg, ret); + rpmh_tx_done(&rpm_msg->msg); } return ret; @@ -327,7 +326,7 @@ static void cache_batch(struct rpmh_ctrlr *ctrlr, struct batch_cache_req *req) spin_unlock_irqrestore(&ctrlr->cache_lock, flags); } -static int flush_batch(struct rpmh_ctrlr *ctrlr) +static int flush_batch(struct rpmh_ctrlr *ctrlr, int ch) { struct batch_cache_req *req; const struct rpmh_request *rpm_msg; @@ -339,7 +338,7 @@ static int flush_batch(struct rpmh_ctrlr *ctrlr) for (i = 0; i < req->count; i++) { rpm_msg = req->rpm_msgs + i; ret = rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), - &rpm_msg->msg); + &rpm_msg->msg, ch); if (ret) break; } @@ -374,7 +373,7 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state, struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); unsigned long time_left; int count = 0; - int ret, i; + int ret, i, ch; void *ptr; if (rpmh_standalone) @@ -399,10 +398,11 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state, return -ENOMEM; req = ptr; + rpm_msgs = ptr + sizeof(*req); compls = ptr + sizeof(*req) + count * sizeof(*rpm_msgs); req->count = count; - rpm_msgs = req->rpm_msgs; + req->rpm_msgs = rpm_msgs; for (i = 0; i < count; i++) { __fill_rpmh_msg(rpm_msgs + i, state, cmd, n[i]); @@ -414,12 +414,18 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state, return 0; } + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) { + kfree(ptr); + return ch; + } + for (i = 0; i < count; i++) { struct completion *compl = &compls[i]; init_completion(compl); rpm_msgs[i].completion = compl; - ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg); + ret = rpmh_rsc_send_data(ctrlr_to_drv(ctrlr), &rpm_msgs[i].msg, ch); if (ret) { pr_err("Error(%d) sending RPMH message addr=%#x\n", ret, rpm_msgs[i].msg.cmds[0].addr); @@ -437,12 +443,10 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state, * we've returned from this function. */ rpmh_rsc_debug(ctrlr_to_drv(ctrlr), &compls[i]); - ret = -ETIMEDOUT; - goto exit; + BUG_ON(1); } } -exit: kfree(ptr); return ret; @@ -457,7 +461,7 @@ static int is_req_valid(struct cache_req *req) } static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state, - u32 addr, u32 data) + u32 addr, u32 data, int ch) { DEFINE_RPMH_MSG_ONSTACK(NULL, state, NULL, rpm_msg); @@ -467,10 +471,10 @@ static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state, rpm_msg.cmd[0].data = data; rpm_msg.msg.num_cmds = 1; - return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg); + return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg, ch); } -int _rpmh_flush(struct rpmh_ctrlr *ctrlr) +int _rpmh_flush(struct rpmh_ctrlr *ctrlr, int ch) { struct cache_req *p; int ret = 0; @@ -481,10 +485,10 @@ int _rpmh_flush(struct rpmh_ctrlr *ctrlr) } /* Invalidate the TCSes first to avoid stale data */ - rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr)); + rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr), ch); /* First flush the cached batch requests */ - ret = flush_batch(ctrlr); + ret = flush_batch(ctrlr, ch); if (ret) return ret; @@ -495,11 +499,11 @@ int _rpmh_flush(struct rpmh_ctrlr *ctrlr) continue; } ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr, - p->sleep_val); + p->sleep_val, ch); if (ret) return ret; ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr, - p->wake_val); + p->wake_val, ch); if (ret) return ret; } @@ -513,12 +517,13 @@ int _rpmh_flush(struct rpmh_ctrlr *ctrlr) * rpmh_flush() - Flushes the buffered sleep and wake sets to TCSes * * @ctrlr: Controller making request to flush cached data + * @ch: Channel number * * Return: * * 0 - Success * * Error code - Otherwise */ -int rpmh_flush(struct rpmh_ctrlr *ctrlr) +int rpmh_flush(struct rpmh_ctrlr *ctrlr, int ch) { int ret; @@ -545,7 +550,7 @@ int rpmh_flush(struct rpmh_ctrlr *ctrlr) */ if (!spin_trylock(&ctrlr->cache_lock)) return -EBUSY; - ret = _rpmh_flush(ctrlr); + ret = _rpmh_flush(ctrlr, ch); spin_unlock(&ctrlr->cache_lock); return ret; @@ -562,7 +567,14 @@ int rpmh_flush(struct rpmh_ctrlr *ctrlr) */ int rpmh_write_sleep_and_wake(const struct device *dev) { - return rpmh_flush(get_rpmh_ctrlr(dev)); + struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); + int ch; + + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) + return ch; + + return rpmh_flush(ctrlr, ch); } EXPORT_SYMBOL(rpmh_write_sleep_and_wake); @@ -583,8 +595,11 @@ void rpmh_invalidate(const struct device *dev) return; spin_lock_irqsave(&ctrlr->cache_lock, flags); - list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list) + list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list) { + list_del(&req->list); kfree(req); + } + INIT_LIST_HEAD(&ctrlr->batch_cache); ctrlr->dirty = true; spin_unlock_irqrestore(&ctrlr->cache_lock, flags); @@ -621,3 +636,90 @@ int rpmh_mode_solver_set(const struct device *dev, bool enable) return ret; } EXPORT_SYMBOL(rpmh_mode_solver_set); + + +/** + * RPMH Fast Path mode + * + * - Fast path mode is a lock-less request transmission to AOSS. + * - Requests may be sent from interrupt/scheduler context. + * - Dedicated fast-path TCS must be available in the RSC. + * - Error returned, when dedicated TCS is not available. + * - Only one client is expected to call RPMH fast path. + * - Serialization of requests is a responsibility of the client. + * - Only active state requests may be made through this mode. + * - Sleep/Wake requests for the nodes must be made through + * the standard RPMH APIs (locking modes). + * - Fast path requests are not cached and sent immediately. + * - Fast path requests are all blocking requests. + * - Fast path requests do not use AMC completion interrupts, + * therefore will not receive a tx_done callback. + */ + +/** + * rpmh_init_fast_path: Initialize TCS request in fast-path TCS + * + * @dev: The device making the request + * @cmd: The {addr, data} to be initialized with. + * @n: The number of @cmd + * + * Return: + * * 0 - Success + * * Error code - Otherwise + */ +int rpmh_init_fast_path(const struct device *dev, + struct tcs_cmd *cmd, int n) +{ + struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); + struct tcs_request req; + int ch; + + if (rpmh_standalone) + return 0; + + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) + return ch; + + req.cmds = cmd; + req.num_cmds = n; + req.wait_for_compl = 0; + + return rpmh_rsc_init_fast_path(ctrlr_to_drv(ctrlr), &req, ch); +} +EXPORT_SYMBOL(rpmh_init_fast_path); + +/** + * rpmh_update_fast_path: Initialize TCS request in fast-path TCS + * + * @dev: The device making the request + * @cmd: The data to be updated. + * @n: The number of @cmd + * @update_mask: The elements in @cmd that only need to be updated + * + * Return: + * * 0 - Success + * * Error code - Otherwise + */ +int rpmh_update_fast_path(const struct device *dev, + struct tcs_cmd *cmd, int n, u32 update_mask) +{ + struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev); + struct tcs_request req; + int ch; + + if (rpmh_standalone) + return 0; + + ch = rpmh_rsc_get_channel(ctrlr_to_drv(ctrlr)); + if (ch < 0) + return ch; + + req.cmds = cmd; + req.num_cmds = n; + req.wait_for_compl = 0; + + return rpmh_rsc_update_fast_path(ctrlr_to_drv(ctrlr), &req, + update_mask, ch); +} +EXPORT_SYMBOL(rpmh_update_fast_path); diff --git a/drivers/soc/qcom/trace-rpmh.h b/drivers/soc/qcom/trace-rpmh.h index fb7285185f96..efaf50a4b67d 100644 --- a/drivers/soc/qcom/trace-rpmh.h +++ b/drivers/soc/qcom/trace-rpmh.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2019, 2021, The Linux Foundation. All rights reserved. */ #if !defined(_TRACE_RPMH_H) || defined(TRACE_HEADER_MULTI_READ) @@ -14,16 +14,15 @@ TRACE_EVENT(rpmh_tx_done, - TP_PROTO(struct rsc_drv *d, int m, const struct tcs_request *r, int e), + TP_PROTO(struct rsc_drv *d, int m, const struct tcs_request *r), - TP_ARGS(d, m, r, e), + TP_ARGS(d, m, r), TP_STRUCT__entry( __string(name, d->name) __field(int, m) __field(u32, addr) __field(u32, data) - __field(int, err) ), TP_fast_assign( @@ -31,12 +30,10 @@ TRACE_EVENT(rpmh_tx_done, __entry->m = m; __entry->addr = r->cmds[0].addr; __entry->data = r->cmds[0].data; - __entry->err = e; ), - TP_printk("%s: ack: tcs-m: %d addr: %#x data: %#x errno: %d", - __get_str(name), __entry->m, __entry->addr, __entry->data, - __entry->err) + TP_printk("%s: ack: tcs-m: %d addr: %#x data: %#x", + __get_str(name), __entry->m, __entry->addr, __entry->data) ); TRACE_EVENT(rpmh_send_msg, diff --git a/include/soc/qcom/rpmh.h b/include/soc/qcom/rpmh.h index dd4fada3b7e2..ff1887452c64 100644 --- a/include/soc/qcom/rpmh.h +++ b/include/soc/qcom/rpmh.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved. */ #ifndef __SOC_QCOM_RPMH_H__ @@ -26,6 +26,11 @@ int rpmh_write_sleep_and_wake(const struct device *dev); void rpmh_invalidate(const struct device *dev); +int rpmh_init_fast_path(const struct device *dev, + struct tcs_cmd *cmd, int n); + +int rpmh_update_fast_path(const struct device *dev, + struct tcs_cmd *cmd, int n, u32 update_mask); #else static inline int rpmh_write(const struct device *dev, enum rpmh_state state, @@ -52,6 +57,15 @@ static inline void rpmh_invalidate(const struct device *dev) { } +static inline int rpmh_init_fast_path(const struct device *dev, + struct tcs_cmd *msg, int n) +{ return -ENODEV; } + +static inline int rpmh_update_fast_path(const struct device *dev, + struct tcs_cmd *msg, int n, + u32 update_mask) +{ return -ENODEV; } + #endif /* CONFIG_QCOM_RPMH */ #endif /* __SOC_QCOM_RPMH_H__ */