Merge "soc: qcom: rpmh-rsc: Add fast-path request support"

drivers/soc/qcom/rpmh-internal.h

@@ -11,12 +11,15 @@
 #include <linux/wait.h>
 #include <soc/qcom/tcs.h>
 
-#define TCS_TYPE_NR			4
+#define TCS_TYPE_NR			5
 #define MAX_CMDS_PER_TCS		16
 #define MAX_TCS_PER_TYPE		3
 #define MAX_TCS_NR			(MAX_TCS_PER_TYPE * TCS_TYPE_NR)
 #define MAX_TCS_SLOTS			(MAX_CMDS_PER_TCS * MAX_TCS_PER_TYPE)
 
+/* CTRLR specific flags */
+#define SOLVER_PRESENT			1
+
 struct rsc_drv;
 
 /**
@@ -78,6 +81,7 @@ struct rpmh_request {
  * @cache_lock: synchronize access to the cache data
  * @dirty: was the cache updated since flush
  * @in_solver_mode: Controller is busy in solver mode
+ * @flags: Controller specific flags
  * @batch_cache: Cache sleep and wake requests sent as batch
  */
 struct rpmh_ctrlr {
@@ -85,6 +89,7 @@ struct rpmh_ctrlr {
 	spinlock_t cache_lock;
 	bool dirty;
 	bool in_solver_mode;
+	u32 flags;
 	struct list_head batch_cache;
 };
 
@@ -114,6 +119,8 @@ struct rpmh_ctrlr {
  * @tcs_wait:           Wait queue used to wait for @tcs_in_use to free up a
  *                      slot
  * @client:             Handle to the DRV's client.
+ * @genpd_nb:           PM Domain notifier
+ * @dev:                RSC device
  */
 struct rsc_drv {
 	const char *name;
@@ -129,8 +136,12 @@ struct rsc_drv {
 	spinlock_t lock;
 	wait_queue_head_t tcs_wait;
 	struct rpmh_ctrlr client;
+	struct notifier_block genpd_nb;
+	struct device *dev;
 };
 
+extern bool rpmh_standalone;
+
 int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg);
 int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv,
 			     const struct tcs_request *msg);
@@ -140,5 +151,11 @@ int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable);
 
 void rpmh_tx_done(const struct tcs_request *msg, int r);
 int rpmh_flush(struct rpmh_ctrlr *ctrlr);
+int _rpmh_flush(struct rpmh_ctrlr *ctrlr);
+
+int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg);
+int rpmh_rsc_update_fast_path(struct rsc_drv *drv,
+			      const struct tcs_request *msg,
+			      u32 update_mask);
 
 #endif /* __RPM_INTERNAL_H__ */

drivers/soc/qcom/rpmh-rsc.c

@@ -14,10 +14,13 @@
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/module.h>
+#include <linux/notifier.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/wait.h>
@@ -95,6 +98,7 @@ static const char * const accl_str[] = {
 
 static struct rsc_drv *__rsc_drv[MAX_RSC_COUNT];
 static int __rsc_count;
+bool rpmh_standalone;
 
 /*
  * Here's a high level overview of how all the registers in RPMH work
@@ -872,6 +876,11 @@ static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
 	max = tcs->offset + tcs->num_tcs;
 	set = find_next_bit(drv->tcs_in_use, max, tcs->offset);
 
+	/* Check if there is pending fastpath transaction */
+	if (drv->tcs[FAST_PATH_TCS].num_tcs &&
+	    !read_tcs_reg(drv, RSC_DRV_STATUS, drv->tcs[FAST_PATH_TCS].offset))
+		return true;
+
 	return set < max;
 }
 
@@ -981,6 +990,90 @@ int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable)
 	return ret;
 }
 
+/**
+ * rpmh_rsc_init_fast_path() - Initialize the fast-path TCS contents
+ * @drv:    The controller.
+ * @msg:    The TCS request to populate.
+ *
+ * Return:
+ * * 0			- success
+ * * -ENODEV            - no fast-path TCS available
+ */
+int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg)
+{
+	int tcs_id;
+
+	if (!drv->tcs[FAST_PATH_TCS].num_tcs)
+		return -ENODEV;
+
+	tcs_id = drv->tcs[FAST_PATH_TCS].offset;
+
+	/* We won't use the AMC IRQ to confirm if the TCS is free */
+	enable_tcs_irq(drv, tcs_id, false);
+
+	__tcs_buffer_write(drv, tcs_id, 0, msg);
+
+	return 0;
+}
+
+/**
+ * rpmh_rsc_update_fast_path() - Update the fast-path TCS data and trigger
+ * @drv:     The controller.
+ * @msg:     The TCS request data to be updated.
+ * @mask:    The update mask for elements in @msg to be sent
+ *
+ * NOTE: Caller should ensure serialization before making this call.
+ * Return:
+ * * 0			- success
+ * * -ENODEV            - no fast-path TCS available
+ */
+int rpmh_rsc_update_fast_path(struct rsc_drv *drv,
+			      const struct tcs_request *msg,
+			      u32 mask)
+{
+	int i;
+	u32 sts;
+	int tcs_id;
+	struct tcs_cmd *cmd;
+	int retry = 5;
+
+	if (!drv->tcs[FAST_PATH_TCS].num_tcs)
+		return -ENODEV;
+
+	tcs_id = drv->tcs[FAST_PATH_TCS].offset;
+
+	/* Ensure the TCS is free before writing to the TCS */
+	do {
+		sts = read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id);
+		if (!sts) {
+			retry--;
+			/* Report and bail, if it took too many attempts */
+			if (!retry) {
+				pr_err("Fast-path TCS is too busy\n");
+				return -EBUSY;
+			}
+			udelay(1);
+		}
+	} while (!sts);
+
+	/*
+	 * We only update the data, everything else remains the same.
+	 * The number of commands and the addresses do not change with
+	 * updates.
+	 */
+	for (i = 0; i < msg->num_cmds; i++) {
+		if (!(mask & BIT(i)))
+			continue;
+		cmd = &msg->cmds[i];
+		write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, i, cmd->data);
+	}
+
+	/* Trigger the TCS to send the request */
+	__tcs_set_trigger(drv, tcs_id, true);
+
+	return 0;
+}
+
 static int rpmh_probe_tcs_config(struct platform_device *pdev,
 				 struct rsc_drv *drv, void __iomem *base)
 {
@@ -1053,6 +1146,32 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev,
 	return 0;
 }
 
+static int rpmh_rsc_pd_cb(struct notifier_block *nb,
+			  unsigned long action, void *data)
+{
+	struct rsc_drv *drv = container_of(nb, struct rsc_drv, genpd_nb);
+
+	/* We don't need to lock as domin on/off are serialized */
+	if ((action == GENPD_NOTIFY_PRE_OFF) &&
+	    (rpmh_rsc_ctrlr_is_busy(drv) || _rpmh_flush(&drv->client)))
+		return NOTIFY_BAD;
+
+	return NOTIFY_OK;
+}
+
+static int rpmh_rsc_pd_attach(struct rsc_drv *drv)
+{
+	int ret;
+
+	pm_runtime_enable(drv->dev);
+	ret = dev_pm_domain_attach(drv->dev, false);
+	if (ret)
+		return ret;
+
+	drv->genpd_nb.notifier_call = rpmh_rsc_pd_cb;
+	return dev_pm_genpd_add_notifier(drv->dev, &drv->genpd_nb);
+}
+
 static int rpmh_rsc_probe(struct platform_device *pdev)
 {
 	struct device_node *dn = pdev->dev.of_node;
@@ -1074,6 +1193,11 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	rpmh_standalone = cmd_db_is_standalone();
+	if (rpmh_standalone)
+		dev_info(&pdev->dev, "RPMH is running in standalone mode.\n");
+
+
 	drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);
 	if (!drv)
 		return -ENOMEM;
@@ -1111,6 +1235,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	drv->dev = &pdev->dev;
 	/*
 	 * CPU PM notification are not required for controllers that support
 	 * 'HW solver' mode where they can be in autonomous mode executing low
@@ -1119,9 +1244,18 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 	solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG);
 	solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
 	solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
-	if (!solver_config) {
+	if (of_find_property(dn, "power-domains", NULL)) {
+		ret = rpmh_rsc_pd_attach(drv);
+		if (ret == -EPROBE_DEFER) {
+			pr_err("Failed to attach RSC %s to domain ret=%d\n", drv->name, ret);
+			return ret;
+		}
+	} else if (!solver_config) {
 		drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
 		cpu_pm_register_notifier(&drv->rsc_pm);
+		drv->client.flags &= ~SOLVER_PRESENT;
+	} else {
+		drv->client.flags |= SOLVER_PRESENT;
 	}
 
 	/* Enable the active TCS to send requests immediately */

drivers/soc/qcom/rpmh.c

@@ -83,6 +83,9 @@ static int check_ctrlr_state(struct rpmh_ctrlr *ctrlr, enum rpmh_state state)
 	if (state != RPMH_ACTIVE_ONLY_STATE)
 		return ret;
 
+	if (!(ctrlr->flags & SOLVER_PRESENT))
+		return ret;
+
 	/* Do not allow sending active votes when in solver mode */
 	spin_lock(&ctrlr->cache_lock);
 	if (ctrlr->in_solver_mode)
@@ -249,6 +252,9 @@ int rpmh_write_async(const struct device *dev, enum rpmh_state state,
 	struct rpmh_request *rpm_msg;
 	int ret;
 
+	if (rpmh_standalone)
+		return 0;
+
 	ret = check_ctrlr_state(ctrlr, state);
 	if (ret)
 		return ret;
@@ -286,6 +292,9 @@ int rpmh_write(const struct device *dev, enum rpmh_state state,
 	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
 	int ret;
 
+	if (rpmh_standalone)
+		return 0;
+
 	ret = check_ctrlr_state(ctrlr, state);
 	if (ret)
 		return ret;
@@ -368,6 +377,9 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state,
 	int ret, i;
 	void *ptr;
 
+	if (rpmh_standalone)
+		return 0;
+
 	ret = check_ctrlr_state(ctrlr, state);
 	if (ret)
 		return ret;
@@ -458,6 +470,45 @@ static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
 	return rpmh_rsc_write_ctrl_data(ctrlr_to_drv(ctrlr), &rpm_msg.msg);
 }
 
+int _rpmh_flush(struct rpmh_ctrlr *ctrlr)
+{
+	struct cache_req *p;
+	int ret = 0;
+
+	if (!ctrlr->dirty) {
+		pr_debug("Skipping flush, TCS has latest data.\n");
+		return ret;
+	}
+
+	/* Invalidate the TCSes first to avoid stale data */
+	rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));
+
+	/* First flush the cached batch requests */
+	ret = flush_batch(ctrlr);
+	if (ret)
+		return ret;
+
+	list_for_each_entry(p, &ctrlr->cache, list) {
+		if (!is_req_valid(p)) {
+			pr_debug("%s: skipping RPMH req: a:%#x s:%#x w:%#x\n",
+				 __func__, p->addr, p->sleep_val, p->wake_val);
+			continue;
+		}
+		ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
+				  p->sleep_val);
+		if (ret)
+			return ret;
+		ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
+				  p->wake_val);
+		if (ret)
+			return ret;
+	}
+
+	ctrlr->dirty = false;
+
+	return ret;
+}
+
 /**
  * rpmh_flush() - Flushes the buffered sleep and wake sets to TCSes
  *
@@ -469,55 +520,52 @@ static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
  */
 int rpmh_flush(struct rpmh_ctrlr *ctrlr)
 {
-	struct cache_req *p;
-	int ret = 0;
+	int ret;
 
-	lockdep_assert_irqs_disabled();
+	if (rpmh_standalone)
+		return 0;
 
 	/*
-	 * Currently rpmh_flush() is only called when we think we're running
-	 * on the last processor.  If the lock is busy it means another
-	 * processor is up and it's better to abort than spin.
+	 * For RSC that don't have solver mode,
+	 * rpmh_flush() is only called when we think we're running
+	 * on the last CPU with irqs_disabled.
+	 *
+	 * For RSC that have solver mode,
+	 * rpmh_flush() can be invoked with irqs enabled by any CPU.
+	 *
+	 * Conditionally check for irqs_disabled only when solver mode
+	 * is not available.
+	 */
+	if (!(ctrlr->flags & SOLVER_PRESENT))
+		lockdep_assert_irqs_disabled();
+
+	/*
+	 * If the lock is busy it means another transaction is on going,
+	 * in such case it's better to abort than spin.
 	 */
 	if (!spin_trylock(&ctrlr->cache_lock))
 		return -EBUSY;
-
-	if (!ctrlr->dirty) {
-		pr_debug("Skipping flush, TCS has latest data.\n");
-		goto exit;
-	}
-
-	/* Invalidate the TCSes first to avoid stale data */
-	rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));
-
-	/* First flush the cached batch requests */
-	ret = flush_batch(ctrlr);
-	if (ret)
-		goto exit;
-
-	list_for_each_entry(p, &ctrlr->cache, list) {
-		if (!is_req_valid(p)) {
-			pr_debug("%s: skipping RPMH req: a:%#x s:%#x w:%#x",
-				 __func__, p->addr, p->sleep_val, p->wake_val);
-			continue;
-		}
-		ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
-				  p->sleep_val);
-		if (ret)
-			goto exit;
-		ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
-				  p->wake_val);
-		if (ret)
-			goto exit;
-	}
-
-	ctrlr->dirty = false;
-
-exit:
+	ret = _rpmh_flush(ctrlr);
 	spin_unlock(&ctrlr->cache_lock);
+
 	return ret;
 }
 
+/**
+ * rpmh_write_sleep_and_wake: Writes the buffered wake and sleep sets to TCSes
+ *
+ * @dev: The device making the request
+ *
+ * Return:
+ * * 0          - Success
+ * * Error code - Otherwise
+ */
+int rpmh_write_sleep_and_wake(const struct device *dev)
+{
+	return rpmh_flush(get_rpmh_ctrlr(dev));
+}
+EXPORT_SYMBOL(rpmh_write_sleep_and_wake);
+
 /**
  * rpmh_invalidate: Invalidate sleep and wake sets in batch_cache
  *
@@ -531,6 +579,9 @@ void rpmh_invalidate(const struct device *dev)
 	struct batch_cache_req *req, *tmp;
 	unsigned long flags;
 
+	if (rpmh_standalone)
+		return;
+
 	spin_lock_irqsave(&ctrlr->cache_lock, flags);
 	list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list)
 		kfree(req);
@@ -555,6 +606,12 @@ int rpmh_mode_solver_set(const struct device *dev, bool enable)
 	int ret;
 	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
 
+	if (rpmh_standalone)
+		return 0;
+
+	if (!(ctrlr->flags & SOLVER_PRESENT))
+		return -EINVAL;
+
 	spin_lock(&ctrlr->cache_lock);
 	ret = rpmh_rsc_mode_solver_set(ctrlr_to_drv(ctrlr), enable);
 	if (!ret)

include/dt-bindings/soc/qcom,rpmh-rsc.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved.
  */
 
 #ifndef __DT_QCOM_RPMH_RSC_H__
@@ -10,5 +10,6 @@
 #define WAKE_TCS	1
 #define ACTIVE_TCS	2
 #define CONTROL_TCS	3
+#define FAST_PATH_TCS	4
 
 #endif /* __DT_QCOM_RPMH_RSC_H__ */

include/soc/qcom/rpmh.h

@@ -22,6 +22,8 @@ int rpmh_write_batch(const struct device *dev, enum rpmh_state state,
 
 int rpmh_mode_solver_set(const struct device *dev, bool enable);
 
+int rpmh_write_sleep_and_wake(const struct device *dev);
+
 void rpmh_invalidate(const struct device *dev);
 
 #else
@@ -43,6 +45,9 @@ static inline int rpmh_write_batch(const struct device *dev,
 static int rpmh_mode_solver_set(const struct device *dev, bool enable)
 { return -ENODEV; }
 
+static int rpmh_write_sleep_and_wake(const struct device *dev)
+{ return -ENODEV; }
+
 static inline void rpmh_invalidate(const struct device *dev)
 {
 }