sched/walt/halt: walt halt functionality

Provide apis to the system in support of halting
cpus, as a replacement for pause/isolation.

Track the requests to halt a cpu, and only
start the cpu again when all requests have been
recanted.

Perform a migration of tasks off of a halted cpu,
if that cpu is online and not idle.

Change-Id: I99340b12e01ff53499b4107d843b7d48e6d6dff1
Signed-off-by: Stephen Dickey <quic_dickey@quicinc.com>

include/linux/sched/walt.h

@@ -157,6 +157,8 @@ extern void core_ctl_notifier_unregister(struct notifier_block *n);
 extern int core_ctl_set_boost(bool boost);
 extern int walt_pause_cpus(struct cpumask *cpus);
 extern int walt_resume_cpus(struct cpumask *cpus);
+extern int walt_halt_cpus(struct cpumask *cpus);
+extern int walt_start_cpus(struct cpumask *cpus);
 #else
 static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout)
 {

kernel/sched/walt/Makefile

@@ -4,7 +4,7 @@ KCOV_INSTRUMENT := n
 KCSAN_SANITIZE := n
 
 obj-$(CONFIG_SCHED_WALT) += sched-walt.o
-sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o
+sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_pause.o walt_halt.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o
 
 obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o
 sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o

kernel/sched/walt/walt.c

@@ -4324,6 +4324,7 @@ static void walt_init(struct work_struct *work)
 	walt_rt_init();
 	walt_cfs_init();
 	walt_pause_init();
+	walt_halt_init();
 
 	stop_machine(walt_init_stop_handler, NULL, NULL);
 

kernel/sched/walt/walt.h

@@ -757,6 +757,7 @@ extern void sched_update_hyst_times(void);
 extern void walt_rt_init(void);
 extern void walt_cfs_init(void);
 extern void walt_pause_init(void);
+extern void walt_halt_init(void);
 extern void walt_fixup_init(void);
 extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu,
 					int sync, int sibling_count_hint);
@@ -901,11 +902,22 @@ struct compute_energy_output {
 	unsigned int	cluster_first_cpu[MAX_CLUSTERS];
 };
 
+extern struct cpumask __cpu_halt_mask;
+#define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask)
+#define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask)
+
+extern struct cpumask __cpu_can_halt_mask;
+#define cpu_can_halt_mask ((struct cpumask *)&__cpu_can_halt_mask)
+#define cpu_can_halt(cpu) cpumask_test_cpu((cpu), cpu_can_halt_mask)
+
 extern void walt_task_dump(struct task_struct *p);
 extern void walt_rq_dump(int cpu);
 extern void walt_dump(void);
 extern int in_sched_bug;
 
+extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
+				 struct task_struct *p, int dest_cpu);
+
 #define WALT_PANIC(condition)				\
 ({							\
 	if (unlikely(!!(condition)) && !in_sched_bug) {	\
@@ -931,4 +943,5 @@ extern int in_sched_bug;
 	}								\
 })
 
+
 #endif /* _WALT_H */

kernel/sched/walt/walt_halt.c

@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <walt.h>
+#include "trace.h"
+
+#ifdef CONFIG_HOTPLUG_CPU
+
+/* if a cpu is halting */
+struct cpumask __cpu_halt_mask;
+struct cpumask __cpu_can_halt_mask;
+
+static DEFINE_MUTEX(halt_lock);
+
+struct halt_cpu_state {
+	int		ref_count;
+};
+
+static DEFINE_PER_CPU(struct halt_cpu_state, halt_state);
+
+/*
+ * Remove a task from the runqueue and pretend that it's migrating. This
+ * should prevent migrations for the detached task and disallow further
+ * changes to tsk_cpus_allowed.
+ */
+void
+detach_one_task_core(struct task_struct *p, struct rq *rq,
+		     struct list_head *tasks)
+{
+	lockdep_assert_held(&rq->__lock);
+
+	p->on_rq = TASK_ON_RQ_MIGRATING;
+	deactivate_task(rq, p, 0);
+	list_add(&p->se.group_node, tasks);
+}
+
+void attach_tasks_core(struct list_head *tasks, struct rq *rq)
+{
+	struct task_struct *p;
+
+	lockdep_assert_held(&rq->__lock);
+
+	while (!list_empty(tasks)) {
+		p = list_first_entry(tasks, struct task_struct, se.group_node);
+		list_del_init(&p->se.group_node);
+
+		BUG_ON(task_rq(p) != rq);
+		activate_task(rq, p, 0);
+		p->on_rq = TASK_ON_RQ_QUEUED;
+	}
+}
+
+/*
+ * Migrate all tasks from the rq, sleeping tasks will be migrated by
+ * try_to_wake_up()->select_task_rq().
+ *
+ * Called with rq->__lock held even though we'er in stop_machine() and
+ * there's no concurrency possible, we hold the required locks anyway
+ * because of lock validation efforts.
+ *
+ * force: if false, the function will skip CPU pinned kthreads.
+ */
+static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, bool force)
+{
+	struct rq *rq = dead_rq;
+	struct task_struct *next, *stop = rq->stop;
+	LIST_HEAD(percpu_kthreads);
+	unsigned int num_pinned_kthreads = 1;
+	struct rq_flags orf = *rf;
+	int dest_cpu;
+
+	/*
+	 * Fudge the rq selection such that the below task selection loop
+	 * doesn't get stuck on the currently eligible stop task.
+	 *
+	 * We're currently inside stop_machine() and the rq is either stuck
+	 * in the stop_machine_cpu_stop() loop, or we're executing this code,
+	 * either way we should never end up calling schedule() until we're
+	 * done here.
+	 */
+	rq->stop = NULL;
+
+	/*
+	 * put_prev_task() and pick_next_task() sched
+	 * class method both need to have an up-to-date
+	 * value of rq->clock[_task]
+	 */
+	update_rq_clock(rq);
+
+#ifdef CONFIG_SCHED_DEBUG
+	/* note the clock update in orf */
+	orf.clock_update_flags |= RQCF_UPDATED;
+#endif
+
+	for (;;) {
+		/*
+		 * There's this thread running, bail when that's the only
+		 * remaining thread:
+		 */
+		if (rq->nr_running == 1)
+			break;
+
+		next = pick_migrate_task(rq);
+
+		/*
+		 * Argh ... no iterator for tasks, we need to remove the
+		 * kthread from the run-queue to continue.
+		 */
+
+		if (!force && is_per_cpu_kthread(next)) {
+			detach_one_task_core(next, rq, &percpu_kthreads);
+			num_pinned_kthreads += 1;
+			continue;
+		}
+
+		/*
+		 * Rules for changing task_struct::cpus_mask are holding
+		 * both pi_lock and rq->__lock, such that holding either
+		 * stabilizes the mask.
+		 *
+		 * Drop rq->__lock is not quite as disastrous as it usually is
+		 * because !cpu_active at this point, which means load-balance
+		 * will not interfere. Also, stop-machine.
+		 */
+		rq_unlock(rq, rf);
+		raw_spin_lock(&next->pi_lock);
+		rq_relock(rq, rf);
+
+		/*
+		 * Since we're inside stop-machine, _nothing_ should have
+		 * changed the task, WARN if weird stuff happened, because in
+		 * that case the above rq->__lock drop is a fail too.
+		 */
+		if (task_rq(next) != rq || !task_on_rq_queued(next)) {
+			/*
+			 * In the !force case, there is a hole between
+			 * rq_unlock() and rq_relock(), where another CPU might
+			 * not observe an up to date cpu_active_mask and try to
+			 * move tasks around.
+			 */
+			WARN_ON(force);
+			raw_spin_unlock(&next->pi_lock);
+			continue;
+		}
+
+		/* Find suitable destination for @next, with force if needed. */
+		dest_cpu = select_fallback_rq(dead_rq->cpu, next);
+		rq = __migrate_task(rq, rf, next, dest_cpu);
+		if (rq != dead_rq) {
+			rq_unlock(rq, rf);
+			rq = dead_rq;
+			*rf = orf;
+			rq_relock(rq, rf);
+		}
+		raw_spin_unlock(&next->pi_lock);
+	}
+
+	if (num_pinned_kthreads > 1)
+		attach_tasks_core(&percpu_kthreads, rq);
+
+	rq->stop = stop;
+}
+
+static int drain_rq_cpu_stop(void *data)
+{
+	struct rq *rq = this_rq();
+	struct rq_flags rf;
+
+	rq_lock_irqsave(rq, &rf);
+	migrate_tasks(rq, &rf, false);
+	rq_unlock_irqrestore(rq, &rf);
+
+	return 0;
+}
+
+static int sched_cpu_drain_rq(unsigned int cpu)
+{
+	if (available_idle_cpu(cpu))
+		return 0;
+
+	return stop_one_cpu(cpu, drain_rq_cpu_stop, NULL);
+}
+
+/*
+ * 1) add the cpus to the halt mask
+ * 2) migrate tasks off the cpu
+ *
+ */
+static int halt_cpus(struct cpumask *cpus)
+{
+	int cpu;
+	int ret = 0;
+
+	for_each_cpu(cpu, cpus) {
+
+		/* set the cpu as halted */
+		cpumask_set_cpu(cpu, cpu_halt_mask);
+
+		/* only drain online cpus */
+		if (cpu_online(cpu)) {
+			/* drain the online CPU */
+			ret = sched_cpu_drain_rq(cpu);
+		}
+
+		if (ret < 0) {
+			/* cpu failed to drain, do not mark as halted */
+			cpumask_clear_cpu(cpu, cpu_halt_mask);
+			break;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * 1) remove the cpus from the halt mask
+ *
+ */
+static int start_cpus(struct cpumask *cpus)
+{
+	if (!cpumask_empty(cpus)) {
+
+		/* remove the cpus from the halt mask */
+		cpumask_andnot(cpu_halt_mask, cpu_halt_mask, cpus);
+	}
+
+	return 0;
+}
+
+/* increment/decrement ref count for cpus in yield/halt mask */
+static void update_ref_counts(struct cpumask *cpus, bool halt)
+{
+	int cpu;
+	struct halt_cpu_state *halt_cpu_state;
+
+	for_each_cpu(cpu, cpus) {
+		halt_cpu_state = per_cpu_ptr(&halt_state, cpu);
+		if (halt)
+			halt_cpu_state->ref_count++;
+		else {
+			WARN_ON_ONCE(halt_cpu_state->ref_count == 0);
+			halt_cpu_state->ref_count--;
+		}
+	}
+}
+
+static void update_halt_cpus(struct cpumask *cpus)
+{
+	int cpu;
+	struct halt_cpu_state *halt_cpu_state;
+
+	for_each_cpu(cpu, cpus) {
+		halt_cpu_state = per_cpu_ptr(&halt_state, cpu);
+		if (halt_cpu_state->ref_count)
+			cpumask_clear_cpu(cpu, cpus);
+	}
+}
+
+/* cpus will be modified */
+int walt_halt_cpus(struct cpumask *cpus)
+{
+	int ret = 0;
+	cpumask_t requested_cpus;
+
+	mutex_lock(&halt_lock);
+
+	cpumask_copy(&requested_cpus, cpus);
+	update_halt_cpus(cpus);
+
+	if (cpumask_empty(cpus)) {
+		update_ref_counts(&requested_cpus, true);
+		goto unlock;
+	}
+
+	ret = halt_cpus(cpus);
+
+	if (ret < 0)
+		pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret,
+			 cpumask_pr_args(&requested_cpus));
+	else
+		update_ref_counts(&requested_cpus, true);
+unlock:
+	mutex_unlock(&halt_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(walt_halt_cpus);
+
+/* cpus will be modified */
+int walt_start_cpus(struct cpumask *cpus)
+{
+	int ret = 0;
+	cpumask_t requested_cpus;
+
+	mutex_lock(&halt_lock);
+	cpumask_copy(&requested_cpus, cpus);
+	update_ref_counts(&requested_cpus, false);
+	update_halt_cpus(cpus);
+
+	ret = start_cpus(cpus);
+
+	if (ret < 0) {
+		pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret,
+			 cpumask_pr_args(&requested_cpus));
+		/* restore/increment ref counts in case of error */
+		update_ref_counts(&requested_cpus, true);
+	}
+
+	mutex_unlock(&halt_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(walt_start_cpus);
+
+void walt_halt_init(void)
+{
+	cpumask_set_cpu(5, cpu_can_halt_mask);
+	cpumask_set_cpu(6, cpu_can_halt_mask);
+	cpumask_set_cpu(7, cpu_can_halt_mask);
+}
+
+#endif /* CONFIG_HOTPLUG_CPU */