sched/pause: support refcounting on pause cpus operation

It is possible that a cpu paused with thermal might be
unpaused by core-ctl, as well as the reverse. This means
that a cpu that has been paused for thermal reasons
can be unpaused when there is need.

Ensure that cpus paused by more than one entity are sticky,
such that only the last unpause request will actually
perform the operation.

Change-Id: Ieb2f836af7348d48a43ae1b22e11d97046ef9dea
Signed-off-by: Stephen Dickey <dickey@codeaurora.org>
Signed-off-by: Sai Harshini Nimmala <snimmala@codeaurora.org>

drivers/thermal/qcom/thermal_pause.c

@@ -0,0 +1,441 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
+ */
+
+#define pr_fmt(fmt) "%s:%s " fmt, KBUILD_MODNAME, __func__
+
+#include <linux/module.h>
+#include <linux/thermal.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/cpu.h>
+#include <linux/of_device.h>
+#include <linux/suspend.h>
+#include <linux/cpumask.h>
+#include <linux/sched/walt.h>
+
+enum thermal_pause_levels {
+	THERMAL_NO_CPU_PAUSE,
+	THERMAL_GROUP_CPU_PAUSE,
+
+	/* define new pause levels above this line */
+	MAX_THERMAL_PAUSE_LEVEL
+};
+
+struct thermal_pause_cdev {
+	struct list_head		node;
+	cpumask_t			cpu_mask;
+	bool				thermal_pause_level;
+	struct thermal_cooling_device	*cdev;
+	struct device_node		*np;
+	char				cdev_name[THERMAL_NAME_LENGTH];
+	struct work_struct		reg_work;
+};
+
+static DEFINE_MUTEX(cpus_pause_lock);
+static LIST_HEAD(thermal_pause_cdev_list);
+static struct cpumask cpus_paused_by_thermal;
+static struct cpumask cpus_in_max_cooling_level;
+static struct cpumask cpus_pending_online;
+static enum cpuhp_state cpu_hp_online;
+
+static BLOCKING_NOTIFIER_HEAD(multi_max_cooling_level_notifer);
+
+void cpu_cooling_multi_max_level_notifier_register(struct notifier_block *n)
+{
+	blocking_notifier_chain_register(&multi_max_cooling_level_notifer, n);
+}
+
+void cpu_cooling_multi_max_level_notifier_unregister(struct notifier_block *n)
+{
+	blocking_notifier_chain_unregister(&multi_max_cooling_level_notifer, n);
+}
+
+const struct cpumask *cpu_cooling_multi_get_max_level_cpumask(void)
+{
+	return &cpus_in_max_cooling_level;
+}
+
+static int thermal_pause_hp_online(unsigned int online_cpu)
+{
+	struct thermal_pause_cdev *thermal_pause_cdev;
+	int ret = 0;
+
+	pr_debug("online entry CPU:%d. mask:%*pbl pend:%*pbl\n", online_cpu,
+			cpumask_pr_args(&cpus_paused_by_thermal),
+			cpumask_pr_args(&cpus_pending_online));
+
+	mutex_lock(&cpus_pause_lock);
+
+	if (cpumask_test_cpu(online_cpu, &cpus_paused_by_thermal)) {
+		ret = NOTIFY_BAD;
+		cpumask_set_cpu(online_cpu, &cpus_pending_online);
+	}
+
+	list_for_each_entry(thermal_pause_cdev, &thermal_pause_cdev_list, node) {
+		if (cpumask_test_cpu(online_cpu, &thermal_pause_cdev->cpu_mask)
+			&& !thermal_pause_cdev->cdev)
+			queue_work(system_highpri_wq,
+				   &thermal_pause_cdev->reg_work);
+	}
+
+	mutex_unlock(&cpus_pause_lock);
+	pr_debug("online exit CPU:%d. mask:%*pbl pend:%*pbl\n", online_cpu,
+			cpumask_pr_args(&cpus_paused_by_thermal),
+			cpumask_pr_args(&cpus_pending_online));
+
+	return ret;
+}
+
+/**
+ * thermal_pause_cpus_pause - function to pause a group of cpus at
+ *                         the specified level.
+ *
+ * @thermal_pause_cdev: the pause device
+ *
+ * function to handle setting the current cpus paused by
+ * this driver for the mask specified in the device.
+ * it assumes the mutex is locked.
+ *
+ * Returns 0 if CPUs were paused, error otherwise
+ */
+static int thermal_pause_cpus_pause(struct thermal_pause_cdev *thermal_pause_cdev)
+{
+	int cpu = 0;
+	int ret = -ENODEV;
+	cpumask_t cpus_to_pause;
+
+	cpumask_clear(&cpus_to_pause);
+	cpumask_andnot(&cpus_to_pause, &thermal_pause_cdev->cpu_mask,
+			&cpus_paused_by_thermal);
+
+	if (!cpumask_empty(&cpus_to_pause)) {
+
+		cpumask_or(&cpus_paused_by_thermal,
+			   &cpus_paused_by_thermal, &cpus_to_pause);
+		pr_debug("Active req:%*pbl pause:%*pbl mask:%*pbl\n",
+			cpumask_pr_args(&cpus_paused_by_thermal),
+			cpumask_pr_args(&cpus_to_pause),
+			cpumask_pr_args(&thermal_pause_cdev->cpu_mask));
+		mutex_unlock(&cpus_pause_lock);
+
+		ret = walt_pause_cpus(&cpus_to_pause);
+
+		mutex_lock(&cpus_pause_lock);
+
+		for_each_cpu(cpu, &cpus_to_pause)
+			blocking_notifier_call_chain(
+				&multi_max_cooling_level_notifer,
+				1, (void *)(long)cpu);
+	}
+
+	cpumask_copy(&cpus_in_max_cooling_level, &cpus_paused_by_thermal);
+
+	return ret;
+}
+
+/**
+ * thermal_pause_cpus_unpause - function to unpause a
+ *       group of cpus in the mask for this cdev
+ *
+ * @thermal_pause_cdev: the pause device
+ *
+ * function to handle enabling the group of cpus in the cdev
+ *
+ * Returns 0 if CPUs were unpaused,
+ */
+static int thermal_pause_cpus_unpause(struct thermal_pause_cdev *thermal_pause_cdev)
+{
+	int cpu = 0;
+	int ret = -ENODEV;
+	struct thermal_pause_cdev *cdev;
+	cpumask_t cpus_offlined, new_cpu_pause, cpus_to_unpause;
+
+	cpumask_clear(&new_cpu_pause);
+	list_for_each_entry(cdev, &thermal_pause_cdev_list, node) {
+		if (!cdev->thermal_pause_level)
+			continue;
+		cpumask_or(&new_cpu_pause, &new_cpu_pause,
+				&cdev->cpu_mask);
+	}
+	cpumask_andnot(&cpus_to_unpause, &cpus_paused_by_thermal,
+		    &new_cpu_pause);
+
+	cpumask_and(&cpus_offlined, &cpus_pending_online,
+			&cpus_to_unpause);
+
+	pr_debug("Old req:%*pbl New req:%*pbl Unpause:%*pbl Online:%*pbl\n",
+			cpumask_pr_args(&cpus_paused_by_thermal),
+			cpumask_pr_args(&new_cpu_pause),
+			cpumask_pr_args(&cpus_to_unpause),
+			cpumask_pr_args(&cpus_offlined));
+
+	cpumask_copy(&cpus_in_max_cooling_level, &new_cpu_pause);
+	cpumask_copy(&cpus_paused_by_thermal, &new_cpu_pause);
+
+	if (!cpumask_empty(&cpus_to_unpause)) {
+		mutex_unlock(&cpus_pause_lock);
+		ret = walt_resume_cpus(&cpus_to_unpause);
+		if (ret)
+			pr_err("Error resuming CPU:%*pbl. err:%d\n",
+					cpumask_pr_args(&cpus_to_unpause), ret);
+		mutex_lock(&cpus_pause_lock);
+	}
+
+	for_each_cpu(cpu, &cpus_offlined) {
+		cpumask_clear_cpu(cpu, &cpus_pending_online);
+		mutex_unlock(&cpus_pause_lock);
+		ret = add_cpu(cpu);
+		if (ret)
+			pr_err("Error Adding CPU:%d. err:%d\n", cpu, ret);
+		mutex_lock(&cpus_pause_lock);
+	}
+
+	for_each_cpu(cpu, &cpus_to_unpause)
+		blocking_notifier_call_chain(
+			&multi_max_cooling_level_notifer,
+			0, (void *)(long)cpu);
+
+	return ret;
+}
+
+/**
+ * thermal_pause_set_cur_state - callback function to set the current cooling
+ *				level.
+ * @cdev: thermal cooling device pointer.
+ * @level: set this variable to the current cooling level.
+ *
+ * Callback for the thermal cooling device to change the cpu pause
+ * current cooling level.
+ *
+ * Return: 0 on success, an error code otherwise.
+ */
+static int thermal_pause_set_cur_state(struct thermal_cooling_device *cdev,
+				   unsigned long level)
+{
+	struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata;
+	int ret = 0;
+
+	if (level >= MAX_THERMAL_PAUSE_LEVEL)
+		return -EINVAL;
+
+	if (thermal_pause_cdev->thermal_pause_level == level)
+		return 0;
+
+	mutex_lock(&cpus_pause_lock);
+
+	thermal_pause_cdev->thermal_pause_level = level;
+
+	if (level == THERMAL_GROUP_CPU_PAUSE)
+		ret = thermal_pause_cpus_pause(thermal_pause_cdev);
+	else
+		ret = thermal_pause_cpus_unpause(thermal_pause_cdev);
+
+	mutex_unlock(&cpus_pause_lock);
+
+	return ret;
+}
+
+/**
+ * thermal_pause_get_cur_state - callback function to get the current cooling
+ *				state.
+ * @cdev: thermal cooling device pointer.
+ * @state: fill this variable with the current cooling state.
+ *
+ * Callback for the thermal cooling device to return the cpu pause
+ * current cooling level
+ *
+ * Return: 0 on success, an error code otherwise.
+ */
+static int thermal_pause_get_cur_state(struct thermal_cooling_device *cdev,
+				   unsigned long *level)
+{
+	struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata;
+
+	*level = thermal_pause_cdev->thermal_pause_level;
+
+	return 0;
+}
+
+/**
+ * thermal_pause_get_max_state - callback function to get the max cooling state.
+ * @cdev: thermal cooling device pointer.
+ * @level: fill this variable with the max cooling level
+ *
+ * Callback for the thermal cooling device to return the cpu
+ * pause max cooling state.
+ *
+ * Return: 0 on success, an error code otherwise.
+ */
+static int thermal_pause_get_max_state(struct thermal_cooling_device *cdev,
+				   unsigned long *level)
+{
+	*level = MAX_THERMAL_PAUSE_LEVEL - 1;
+	return 0;
+}
+
+static struct thermal_cooling_device_ops thermal_pause_cooling_ops = {
+	.get_max_state = thermal_pause_get_max_state,
+	.get_cur_state = thermal_pause_get_cur_state,
+	.set_cur_state = thermal_pause_set_cur_state,
+};
+
+static void thermal_pause_register_cdev(struct work_struct *work)
+{
+	struct thermal_pause_cdev *thermal_pause_cdev =
+			container_of(work, struct thermal_pause_cdev, reg_work);
+	int ret = 0;
+	cpumask_t cpus_online;
+
+	cpumask_and(&cpus_online,
+			&thermal_pause_cdev->cpu_mask,
+			cpu_online_mask);
+	if (!cpumask_equal(&thermal_pause_cdev->cpu_mask, &cpus_online))
+		return;
+
+	thermal_pause_cdev->cdev = thermal_of_cooling_device_register(
+					thermal_pause_cdev->np,
+					thermal_pause_cdev->cdev_name,
+					thermal_pause_cdev,
+					&thermal_pause_cooling_ops);
+
+	if (IS_ERR(thermal_pause_cdev->cdev)) {
+		ret = PTR_ERR(thermal_pause_cdev->cdev);
+		pr_err("Cooling register failed for %s, ret:%d\n",
+			thermal_pause_cdev->cdev_name, ret);
+		thermal_pause_cdev->cdev = NULL;
+		return;
+	}
+
+	pr_debug("Cooling device [%s] registered.\n",
+			thermal_pause_cdev->cdev_name);
+}
+
+static int thermal_pause_probe(struct platform_device *pdev)
+{
+	int ret = 0, cpu = 0;
+	struct device_node *subsys_np = NULL, *cpu_phandle = NULL;
+	struct device *cpu_dev;
+	struct thermal_pause_cdev *thermal_pause_cdev = NULL;
+	struct device_node *np = pdev->dev.of_node;
+	struct of_phandle_iterator it;
+	cpumask_t cpu_mask;
+	unsigned long mask = 0;
+
+	INIT_LIST_HEAD(&thermal_pause_cdev_list);
+	cpumask_clear(&cpus_in_max_cooling_level);
+	cpumask_clear(&cpus_paused_by_thermal);
+	cpumask_clear(&cpus_pending_online);
+
+	for_each_available_child_of_node(np, subsys_np) {
+
+		cpumask_clear(&cpu_mask);
+		mask = 0;
+		of_phandle_iterator_init(&it, subsys_np, "qcom,cpus", NULL, 0);
+		while (of_phandle_iterator_next(&it) == 0) {
+			cpu_phandle = it.node;
+			for_each_possible_cpu(cpu) {
+				cpu_dev = get_cpu_device(cpu);
+				if (cpu_dev && cpu_dev->of_node
+						== cpu_phandle) {
+					cpumask_set_cpu(cpu, &cpu_mask);
+					mask = mask | BIT(cpu);
+					break;
+				}
+			}
+		}
+
+		if (cpumask_empty(&cpu_mask))
+			continue;
+
+		thermal_pause_cdev = devm_kzalloc(&pdev->dev,
+				sizeof(*thermal_pause_cdev), GFP_KERNEL);
+
+		if (!thermal_pause_cdev) {
+			of_node_put(subsys_np);
+			return -ENOMEM;
+		}
+
+		snprintf(thermal_pause_cdev->cdev_name, THERMAL_NAME_LENGTH,
+				"thermal-pause-%X", mask);
+
+		thermal_pause_cdev->thermal_pause_level = false;
+		thermal_pause_cdev->cdev = NULL;
+		thermal_pause_cdev->np = subsys_np;
+		cpumask_copy(&thermal_pause_cdev->cpu_mask, &cpu_mask);
+
+		INIT_WORK(&thermal_pause_cdev->reg_work,
+				thermal_pause_register_cdev);
+		list_add(&thermal_pause_cdev->node, &thermal_pause_cdev_list);
+	}
+
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal-pause/cdev:online",
+				thermal_pause_hp_online, NULL);
+	if (ret < 0)
+		return ret;
+	cpu_hp_online = ret;
+
+	return 0;
+}
+
+static int thermal_pause_remove(struct platform_device *pdev)
+{
+	struct thermal_pause_cdev *thermal_pause_cdev = NULL, *next = NULL;
+	int ret = 0, cpu = 0;
+
+	if (cpu_hp_online) {
+		cpuhp_remove_state_nocalls(cpu_hp_online);
+		cpu_hp_online = 0;
+	}
+
+	mutex_lock(&cpus_pause_lock);
+	list_for_each_entry_safe(thermal_pause_cdev, next,
+			&thermal_pause_cdev_list, node) {
+		if (thermal_pause_cdev->cdev)
+			thermal_cooling_device_unregister(
+					thermal_pause_cdev->cdev);
+		list_del(&thermal_pause_cdev->node);
+	}
+
+	cpumask_andnot(&cpus_paused_by_thermal, &cpus_paused_by_thermal,
+			&cpus_pending_online);
+
+	if (!cpumask_empty(&cpus_paused_by_thermal)) {
+		mutex_unlock(&cpus_pause_lock);
+		ret = walt_resume_cpus(&cpus_paused_by_thermal);
+		if (ret)
+			pr_err("Error resuming CPU:%*pbl. err:%d\n",
+				cpumask_pr_args(&cpus_paused_by_thermal), ret);
+		mutex_lock(&cpus_pause_lock);
+	}
+
+	for_each_cpu(cpu, &cpus_pending_online) {
+		mutex_unlock(&cpus_pause_lock);
+		ret = add_cpu(cpu);
+		if (ret)
+			pr_err("Error Adding CPU:%d. err:%d\n", cpu, ret);
+		mutex_lock(&cpus_pause_lock);
+	}
+
+	mutex_unlock(&cpus_pause_lock);
+
+	return 0;
+}
+
+static const struct of_device_id thermal_pause_match[] = {
+	{ .compatible = "qcom,thermal-pause", },
+	{},
+};
+
+static struct platform_driver thermal_pause_driver = {
+	.probe		= thermal_pause_probe,
+	.remove         = thermal_pause_remove,
+	.driver		= {
+		.name		= KBUILD_MODNAME,
+		.of_match_table = thermal_pause_match,
+	},
+};
+
+module_platform_driver(thermal_pause_driver);
+MODULE_LICENSE("GPL v2");

include/linux/sched/walt.h

@@ -133,6 +133,8 @@ struct notifier_block;
 extern void core_ctl_notifier_register(struct notifier_block *n);
 extern void core_ctl_notifier_unregister(struct notifier_block *n);
 extern int core_ctl_set_boost(bool boost);
+extern int walt_pause_cpus(struct cpumask *cpus);
+extern int walt_resume_cpus(struct cpumask *cpus);
 #else
 static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout)
 {
@@ -170,6 +172,14 @@ static inline void core_ctl_notifier_unregister(struct notifier_block *n)
 {
 }
 
+inline int walt_pause_cpus(struct cpumask *cpus)
+{
+	return 0;
+}
+inline int walt_resume_cpus(struct cpumask *cpus)
+{
+	return 0;
+}
 #endif
 
 #endif /* _LINUX_SCHED_WALT_H */

kernel/sched/walt/Makefile

@@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n
 KCSAN_SANITIZE := n
 
 obj-$(CONFIG_SCHED_WALT) += sched-walt.o
-sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o
+sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o
 
 obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o
 sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o

kernel/sched/walt/core_ctl.c

@@ -14,6 +14,7 @@
 #include <linux/sched/rt.h>
 #include <linux/syscore_ops.h>
 #include <uapi/linux/sched/types.h>
+#include <linux/sched/walt.h>
 
 #include "walt.h"
 #include "trace.h"
@@ -1157,10 +1158,10 @@ static void __ref do_core_ctl(void)
 	}
 
 	if (cpumask_any(&cpus_to_pause) < nr_cpu_ids)
-		pause_cpus(&cpus_to_pause);
+		walt_pause_cpus(&cpus_to_pause);
 
 	if (cpumask_any(&cpus_to_unpause) < nr_cpu_ids)
-		resume_cpus(&cpus_to_unpause);
+		walt_resume_cpus(&cpus_to_unpause);
 }
 
 static int __ref try_core_ctl(void *data)

kernel/sched/walt/walt_pause.c

@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+DEFINE_SPINLOCK(pause_lock);
+
+struct pause_cpu_state {
+	int		ref_count;
+};
+
+static DEFINE_PER_CPU(struct pause_cpu_state, pause_state);
+
+/* increment ref count for cpus in passed mask */
+static void inc_ref_counts(struct cpumask *cpus)
+{
+	int cpu;
+	struct pause_cpu_state *pause_cpu_state;
+
+	spin_lock(&pause_lock);
+	for_each_cpu(cpu, cpus) {
+		pause_cpu_state = per_cpu_ptr(&pause_state, cpu);
+		if (pause_cpu_state->ref_count)
+			cpumask_clear_cpu(cpu, cpus);
+		pause_cpu_state->ref_count++;
+	}
+	spin_unlock(&pause_lock);
+}
+
+/*
+ * decrement ref count for cpus in passed mask
+ * updates the cpus to include only cpus ready to be unpaused
+ */
+static void dec_test_ref_counts(struct cpumask *cpus)
+{
+	int cpu;
+	struct pause_cpu_state *pause_cpu_state;
+
+	spin_lock(&pause_lock);
+	for_each_cpu(cpu, cpus) {
+		pause_cpu_state = per_cpu_ptr(&pause_state, cpu);
+		WARN_ON_ONCE(pause_cpu_state->ref_count == 0);
+		pause_cpu_state->ref_count--;
+		if (pause_cpu_state->ref_count)
+			cpumask_clear_cpu(cpu, cpus);
+	}
+	spin_unlock(&pause_lock);
+}
+
+/* cpus will be modified */
+int walt_pause_cpus(struct cpumask *cpus)
+{
+	int ret;
+	cpumask_t saved_cpus;
+
+	cpumask_copy(&saved_cpus, cpus);
+
+	/* prior to operation, track cpus requested to be paused */
+	inc_ref_counts(cpus);
+	ret = pause_cpus(cpus);
+	if (ret < 0) {
+		dec_test_ref_counts(&saved_cpus);
+		pr_err("pause_cpus failure ret=%d cpus=%*pbl\n", ret,
+		       cpumask_pr_args(&saved_cpus));
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(walt_pause_cpus);
+
+/* cpus will be modified */
+int walt_resume_cpus(struct cpumask *cpus)
+{
+	int ret;
+	cpumask_t saved_cpus;
+
+	cpumask_copy(&saved_cpus, cpus);
+
+	dec_test_ref_counts(cpus);
+	ret = resume_cpus(cpus);
+	if (ret < 0) {
+		inc_ref_counts(&saved_cpus);
+		pr_err("resume_cpus failure ret=%d cpus=%*pbl\n", ret,
+		       cpumask_pr_args(&saved_cpus));
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(walt_resume_cpus);
+
+#endif /* CONFIG_HOTPLUG_CPU */