diff --git a/Kconfig b/Kconfig index 57a142d8d8b4..2066ad68dd4f 100644 --- a/Kconfig +++ b/Kconfig @@ -33,3 +33,5 @@ source "Documentation/Kconfig" # ANDROID: Set KCONFIG_EXT_PREFIX to decend into an external project. source "$(KCONFIG_EXT_PREFIX)Kconfig.ext" + +source "kernel/sched/walt/Kconfig" diff --git a/drivers/soc/qcom/hyp_core_ctl.c b/drivers/soc/qcom/hyp_core_ctl.c new file mode 100644 index 000000000000..2e490c09b561 --- /dev/null +++ b/drivers/soc/qcom/hyp_core_ctl.c @@ -0,0 +1,1101 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) "hyp_core_ctl: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#define MAX_RESERVE_CPUS (num_possible_cpus()/2) + +static DEFINE_PER_CPU(struct freq_qos_request, qos_min_req); +static DEFINE_PER_CPU(unsigned int, qos_min_freq); + +/** + * struct hyp_core_ctl_cpumap - vcpu to pcpu mapping for the other guest + * @cap_id: System call id to be used while referring to this vcpu + * @pcpu: The physical CPU number corresponding to this vcpu + * @curr_pcpu: The current physical CPU number corresponding to this vcpu. + * The curr_pcu is set to another CPU when the original assigned + * CPU i.e pcpu can't be used due to thermal condition. + * + */ +struct hyp_core_ctl_cpu_map { + hh_capid_t cap_id; + hh_label_t pcpu; + hh_label_t curr_pcpu; +}; + +/** + * struct hyp_core_ctl_data - The private data structure of this driver + * @lock: spinlock to serialize task wakeup and enable/reserve_cpus + * @task: task_struct pointer to the thread running the state machine + * @pending: state machine work pending status + * @reservation_enabled: status of the reservation + * @reservation_mutex: synchronization between thermal handling and + * reservation. The physical CPUs are re-assigned + * during thermal conditions while reservation is + * not enabled. So this synchronization is needed. + * @reserve_cpus: The CPUs to be reserved. input. + * @our_paused_cpus: The CPUs paused by hyp_core_ctl driver. output. + * @final_reserved_cpus: The CPUs reserved for the Hypervisor. output. + * @cpumap: The vcpu to pcpu mapping table + */ +struct hyp_core_ctl_data { + spinlock_t lock; + struct task_struct *task; + bool pending; + bool reservation_enabled; + struct mutex reservation_mutex; + cpumask_t reserve_cpus; + cpumask_t our_paused_cpus; + cpumask_t final_reserved_cpus; + struct hyp_core_ctl_cpu_map cpumap[NR_CPUS]; +}; + +#define CREATE_TRACE_POINTS +#include "hyp_core_ctl_trace.h" + +static struct hyp_core_ctl_data *the_hcd; +static struct hyp_core_ctl_cpu_map hh_cpumap[NR_CPUS]; +static bool is_vcpu_info_populated; +static bool init_done; +static int nr_vcpus; +static bool freq_qos_init_done; + +static inline void hyp_core_ctl_print_status(char *msg) +{ + trace_hyp_core_ctl_status(the_hcd, msg); + + pr_debug("%s: reserve=%*pbl reserved=%*pbl our_paused=%*pbl online=%*pbl active=%*pbl thermal=%*pbl\n", + msg, cpumask_pr_args(&the_hcd->reserve_cpus), + cpumask_pr_args(&the_hcd->final_reserved_cpus), + cpumask_pr_args(&the_hcd->our_paused_cpus), + cpumask_pr_args(cpu_online_mask), + cpumask_pr_args(cpu_active_mask), + cpumask_pr_args(cpu_cooling_multi_get_max_level_cpumask())); +} + +static inline int pause_cpu(int cpu) +{ + cpumask_t cpus_to_pause; + int ret; + + cpumask_clear(&cpus_to_pause); + cpumask_set_cpu(cpu, &cpus_to_pause); + + ret = walt_pause_cpus(&cpus_to_pause, PAUSE_HYP); + + return ret; +} + +static inline int resume_cpu(int cpu) +{ + cpumask_t cpus_to_resume; + int ret; + + cpumask_clear(&cpus_to_resume); + cpumask_set_cpu(cpu, &cpus_to_resume); + + ret = walt_resume_cpus(&cpus_to_resume, PAUSE_HYP); + + return ret; +} + +static void hyp_core_ctl_undo_reservation(struct hyp_core_ctl_data *hcd) +{ + int cpu, ret; + struct freq_qos_request *qos_req; + + hyp_core_ctl_print_status("undo_reservation_start"); + + for_each_cpu(cpu, &hcd->our_paused_cpus) { + ret = resume_cpu(cpu); + if (ret < 0) { + pr_err("fail to un-pause CPU%d. ret=%d\n", cpu, ret); + continue; + } + + cpumask_clear_cpu(cpu, &hcd->our_paused_cpus); + + if (freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, cpu); + ret = freq_qos_update_request(qos_req, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + cpu, ret); + } + } + + hyp_core_ctl_print_status("undo_reservation_end"); +} + +static void finalize_reservation(struct hyp_core_ctl_data *hcd, cpumask_t *temp) +{ + cpumask_t vcpu_adjust_mask; + int i, orig_cpu, curr_cpu, replacement_cpu; + int err; + + /* + * When thermal conditions are not present, we return + * from here. + */ + if (cpumask_equal(temp, &hcd->final_reserved_cpus)) + return; + + /* + * When we can't match with the original reserve CPUs request, + * don't change the existing scheme. We can't assign the + * same physical CPU to multiple virtual CPUs. + * + * This may only happen when thermal pause more CPUs. + */ + if (cpumask_weight(temp) < cpumask_weight(&hcd->reserve_cpus)) { + pr_debug("Fail to reserve some CPUs\n"); + return; + } + + cpumask_copy(&hcd->final_reserved_cpus, temp); + cpumask_clear(&vcpu_adjust_mask); + + /* + * In the first pass, we traverse all virtual CPUs and try + * to assign their original physical CPUs if they are + * reserved. if the original physical CPU is not reserved, + * then check the current physical CPU is reserved or not. + * so that we continue to use the current physical CPU. + * + * If both original CPU and the current CPU are not reserved, + * we have to find a replacement. These virtual CPUs are + * maintained in vcpu_adjust_mask and processed in the 2nd pass. + */ + for (i = 0; i < MAX_RESERVE_CPUS; i++) { + if (hcd->cpumap[i].cap_id == 0) + break; + + orig_cpu = hcd->cpumap[i].pcpu; + curr_cpu = hcd->cpumap[i].curr_pcpu; + + if (cpumask_test_cpu(orig_cpu, &hcd->final_reserved_cpus)) { + cpumask_clear_cpu(orig_cpu, temp); + + if (orig_cpu == curr_cpu) + continue; + + /* + * The original pcpu corresponding to this vcpu i.e i + * is available in final_reserved_cpus. so restore + * the assignment. + */ + err = hh_hcall_vcpu_affinity_set(hcd->cpumap[i].cap_id, + orig_cpu); + if (err != HH_ERROR_OK) { + pr_err("restore: fail to assign pcpu for vcpu#%d err=%d cap_id=%d cpu=%d\n", + i, err, hcd->cpumap[i].cap_id, orig_cpu); + continue; + } + + hcd->cpumap[i].curr_pcpu = orig_cpu; + pr_debug("err=%u vcpu=%d pcpu=%u curr_cpu=%u\n", + err, i, hcd->cpumap[i].pcpu, + hcd->cpumap[i].curr_pcpu); + continue; + } + + /* + * The original CPU is not available but the previously + * assigned CPU i.e curr_cpu is still available. so keep + * using it. + */ + if (cpumask_test_cpu(curr_cpu, &hcd->final_reserved_cpus)) { + cpumask_clear_cpu(curr_cpu, temp); + continue; + } + + /* + * A replacement CPU is found in the 2nd pass below. Make + * a note of this virtual CPU for which both original and + * current physical CPUs are not available in the + * final_reserved_cpus. + */ + cpumask_set_cpu(i, &vcpu_adjust_mask); + } + + /* + * The vcpu_adjust_mask contain the virtual CPUs that needs + * re-assignment. The temp CPU mask contains the remaining + * reserved CPUs. so we pick one by one from the remaining + * reserved CPUs and assign them to the pending virtual + * CPUs. + */ + for_each_cpu(i, &vcpu_adjust_mask) { + replacement_cpu = cpumask_any(temp); + cpumask_clear_cpu(replacement_cpu, temp); + + err = hh_hcall_vcpu_affinity_set(hcd->cpumap[i].cap_id, + replacement_cpu); + if (err != HH_ERROR_OK) { + pr_err("adjust: fail to assign pcpu for vcpu#%d err=%d cap_id=%d cpu=%d\n", + i, err, hcd->cpumap[i].cap_id, replacement_cpu); + continue; + } + + hcd->cpumap[i].curr_pcpu = replacement_cpu; + pr_debug("adjust err=%u vcpu=%d pcpu=%u curr_cpu=%u\n", + err, i, hcd->cpumap[i].pcpu, + hcd->cpumap[i].curr_pcpu); + + } + + /* Did we reserve more CPUs than needed? */ + WARN_ON(!cpumask_empty(temp)); +} + +static void hyp_core_ctl_do_reservation(struct hyp_core_ctl_data *hcd) +{ + cpumask_t offline_cpus, iter_cpus, temp_reserved_cpus; + int i, ret, pause_req, pause_done; + const cpumask_t *thermal_cpus = cpu_cooling_multi_get_max_level_cpumask(); + struct freq_qos_request *qos_req; + unsigned int min_freq; + + cpumask_clear(&offline_cpus); + cpumask_clear(&temp_reserved_cpus); + + hyp_core_ctl_print_status("reservation_start"); + + /* + * Iterate all reserve CPUs and pause them if not done already. + * The offline CPUs can't be paused but they are considered + * reserved. When an offline and reserved CPU comes online, it + * will be paused to honor the reservation. + */ + cpumask_andnot(&iter_cpus, &hcd->reserve_cpus, &hcd->our_paused_cpus); + cpumask_andnot(&iter_cpus, &iter_cpus, thermal_cpus); + + for_each_cpu(i, &iter_cpus) { + if (!cpu_online(i)) { + cpumask_set_cpu(i, &offline_cpus); + continue; + } + + ret = pause_cpu(i); + if (ret < 0) { + pr_debug("fail to pause CPU%d. ret=%d\n", i, ret); + continue; + } + + cpumask_set_cpu(i, &hcd->our_paused_cpus); + + min_freq = per_cpu(qos_min_freq, i); + if (min_freq && freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, i); + ret = freq_qos_update_request(qos_req, min_freq); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + i, ret); + } + } + + cpumask_andnot(&iter_cpus, &hcd->reserve_cpus, &offline_cpus); + pause_req = cpumask_weight(&iter_cpus); + pause_done = cpumask_weight(&hcd->our_paused_cpus); + + if (pause_done < pause_req) { + int pause_need; + + /* + * We have paused fewer CPUs than required. This happens + * when some of the CPUs from the reserved_cpus mask + * are managed by thermal. Find the replacement CPUs and + * pause them. + */ + pause_need = pause_req - pause_done; + + /* + * Create a cpumask from which replacement CPUs can be + * picked. Exclude our paused CPUs, thermal managed + * CPUs and offline CPUs, which are already considered + * as reserved. + */ + cpumask_andnot(&iter_cpus, cpu_possible_mask, + &hcd->our_paused_cpus); + cpumask_andnot(&iter_cpus, &iter_cpus, thermal_cpus); + cpumask_andnot(&iter_cpus, &iter_cpus, &offline_cpus); + + /* + * Keep the replacement policy simple. The offline CPUs + * comes for free. so pick them first. + */ + for_each_cpu(i, &iter_cpus) { + if (!cpu_online(i)) { + cpumask_set_cpu(i, &offline_cpus); + if (--pause_need == 0) + goto done; + } + } + + cpumask_andnot(&iter_cpus, &iter_cpus, &offline_cpus); + + for_each_cpu(i, &iter_cpus) { + ret = pause_cpu(i); + if (ret < 0) { + pr_debug("fail to pause CPU%d. ret=%d\n", + i, ret); + continue; + } + + cpumask_set_cpu(i, &hcd->our_paused_cpus); + + min_freq = per_cpu(qos_min_freq, i); + if (min_freq && freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, i); + ret = freq_qos_update_request(qos_req, + min_freq); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + i, ret); + } + + if (--pause_need == 0) + break; + } + } else if (pause_done > pause_req) { + int unpause_need; + + /* + * We have paused more CPUs than required. Un-pause + * the additional CPUs which are not part of the + * reserve_cpus mask. + * + * This happens in the following scenario. + * + * - Lets say reserve CPUs are CPU4 and CPU5. They are + * paused. + * - CPU4 is paused by thermal. We found CPU0 as the + * replacement CPU. Now CPU0 and CPU5 are paused by + * us. + * - CPU4 is un-paused by thermal. We first pause CPU4 + * since it is part of our reserve CPUs. Now CPU0, CPU4 + * and CPU5 are paused by us. + * - Since pause_done (3) > pause_req (2), un-pause + * a CPU which is not part of the reserve CPU. i.e CPU0. + */ + unpause_need = pause_done - pause_req; + cpumask_andnot(&iter_cpus, &hcd->our_paused_cpus, + &hcd->reserve_cpus); + for_each_cpu(i, &iter_cpus) { + ret = resume_cpu(i); + if (ret < 0) { + pr_err("fail to unpause CPU%d. ret=%d\n", + i, ret); + continue; + } + + cpumask_clear_cpu(i, &hcd->our_paused_cpus); + + if (freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, i); + ret = freq_qos_update_request(qos_req, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + i, ret); + } + + if (--unpause_need == 0) + break; + } + } + +done: + cpumask_or(&temp_reserved_cpus, &hcd->our_paused_cpus, &offline_cpus); + finalize_reservation(hcd, &temp_reserved_cpus); + + hyp_core_ctl_print_status("reservation_end"); +} + +static int hyp_core_ctl_thread(void *data) +{ + struct hyp_core_ctl_data *hcd = data; + + while (1) { + spin_lock(&hcd->lock); + if (!hcd->pending) { + set_current_state(TASK_INTERRUPTIBLE); + spin_unlock(&hcd->lock); + + schedule(); + + spin_lock(&hcd->lock); + set_current_state(TASK_RUNNING); + } + hcd->pending = false; + spin_unlock(&hcd->lock); + + if (kthread_should_stop()) + break; + + /* + * The reservation mutex synchronize the reservation + * happens in this thread against the thermal handling. + * The CPU re-assignment happens directly from the + * thermal callback context when the reservation is + * not enabled, since there is no need for isolating. + */ + mutex_lock(&hcd->reservation_mutex); + if (hcd->reservation_enabled) + hyp_core_ctl_do_reservation(hcd); + else + hyp_core_ctl_undo_reservation(hcd); + mutex_unlock(&hcd->reservation_mutex); + } + + return 0; +} + +static void hyp_core_ctl_handle_thermal(struct hyp_core_ctl_data *hcd, + int cpu, bool throttled) +{ + cpumask_t temp_mask, iter_cpus; + const cpumask_t *thermal_cpus = cpu_cooling_multi_get_max_level_cpumask(); + bool notify = false; + int replacement_cpu; + + hyp_core_ctl_print_status("handle_thermal_start"); + + /* + * Take a copy of the final_reserved_cpus and adjust the mask + * based on the notified CPU's thermal state. + */ + cpumask_copy(&temp_mask, &hcd->final_reserved_cpus); + + if (throttled) { + /* + * Find a replacement CPU for this throttled CPU. Select + * any CPU that is not managed by thermal and not already + * part of the assigned CPUs. + */ + cpumask_andnot(&iter_cpus, cpu_possible_mask, thermal_cpus); + cpumask_andnot(&iter_cpus, &iter_cpus, + &hcd->final_reserved_cpus); + replacement_cpu = cpumask_any(&iter_cpus); + + if (replacement_cpu < nr_cpu_ids) { + cpumask_clear_cpu(cpu, &temp_mask); + cpumask_set_cpu(replacement_cpu, &temp_mask); + notify = true; + } + } else { + /* + * One of the original assigned CPU is unthrottled by thermal. + * Swap this CPU with any one of the replacement CPUs. + */ + cpumask_andnot(&iter_cpus, &hcd->final_reserved_cpus, + &hcd->reserve_cpus); + replacement_cpu = cpumask_any(&iter_cpus); + + if (replacement_cpu < nr_cpu_ids) { + cpumask_clear_cpu(replacement_cpu, &temp_mask); + cpumask_set_cpu(cpu, &temp_mask); + notify = true; + } + } + + if (notify) + finalize_reservation(hcd, &temp_mask); + + hyp_core_ctl_print_status("handle_thermal_end"); +} + +static int hyp_core_ctl_cpu_cooling_cb(struct notifier_block *nb, + unsigned long val, void *data) +{ + int cpu = (long) data; + const cpumask_t *thermal_cpus = cpu_cooling_multi_get_max_level_cpumask(); + struct freq_qos_request *qos_req; + int ret; + + if (!the_hcd) + return NOTIFY_DONE; + + mutex_lock(&the_hcd->reservation_mutex); + + pr_debug("CPU%d is %s by thermal\n", cpu, + val ? "throttled" : "unthrottled"); + + if (val) { + /* + * The thermal mitigated CPU is not part of our reserved + * CPUs. So nothing to do. + */ + if (!cpumask_test_cpu(cpu, &the_hcd->final_reserved_cpus)) + goto out; + + /* + * The thermal mitigated CPU is part of our reserved CPUs. + * + * If it is paused by us, unpause it. If it is not + * paused, probably it is offline. In both cases, kick + * the state machine to find a replacement CPU. + */ + if (cpumask_test_cpu(cpu, &the_hcd->our_paused_cpus)) { + resume_cpu(cpu); + cpumask_clear_cpu(cpu, &the_hcd->our_paused_cpus); + if (freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, cpu); + ret = freq_qos_update_request(qos_req, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + cpu, ret); + } + } + } else { + /* + * A CPU is unblocked by thermal. We are interested if + * + * (1) This CPU is part of the original reservation request + * In this case, this CPU should be swapped with one of + * the replacement CPU that is currently reserved. + * (2) When some of the thermal mitigated CPUs are currently + * reserved due to unavailability of CPUs. Now that + * thermal unblocked a CPU, swap this with one of the + * thermal mitigated CPU that is currently reserved. + */ + if (!cpumask_test_cpu(cpu, &the_hcd->reserve_cpus) && + !cpumask_intersects(&the_hcd->final_reserved_cpus, + thermal_cpus)) + goto out; + } + + if (the_hcd->reservation_enabled) { + spin_lock(&the_hcd->lock); + the_hcd->pending = true; + wake_up_process(the_hcd->task); + spin_unlock(&the_hcd->lock); + } else { + /* + * When the reservation is enabled, the state machine + * takes care of finding the new replacement CPU or + * isolating the unthrottled CPU. However when the + * reservation is not enabled, we still want to + * re-assign another CPU for a throttled CPU. + */ + hyp_core_ctl_handle_thermal(the_hcd, cpu, val); + } +out: + mutex_unlock(&the_hcd->reservation_mutex); + return NOTIFY_OK; +} + +static struct notifier_block hyp_core_ctl_nb = { + .notifier_call = hyp_core_ctl_cpu_cooling_cb, +}; + +static int hyp_core_ctl_hp_offline(unsigned int cpu) +{ + struct freq_qos_request *qos_req; + int ret; + + if (!the_hcd || !the_hcd->reservation_enabled) + return 0; + + if (cpumask_test_and_clear_cpu(cpu, &the_hcd->our_paused_cpus)) { + if (freq_qos_init_done) { + qos_req = &per_cpu(qos_min_req, cpu); + ret = freq_qos_update_request(qos_req, + FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + pr_err("fail to update min freq for CPU%d ret=%d\n", + cpu, ret); + } + } + + return 0; +} + +static int hyp_core_ctl_hp_online(unsigned int cpu) +{ + if (!the_hcd || !the_hcd->reservation_enabled) + return 0; + + /* + * A reserved CPU is coming online. It should be paused + * to honor the reservation. So kick the state machine. + */ + spin_lock(&the_hcd->lock); + if (cpumask_test_cpu(cpu, &the_hcd->final_reserved_cpus)) { + the_hcd->pending = true; + wake_up_process(the_hcd->task); + } + spin_unlock(&the_hcd->lock); + + return 0; +} + +static void hyp_core_ctl_init_reserve_cpus(struct hyp_core_ctl_data *hcd) +{ + int i; + + spin_lock(&hcd->lock); + cpumask_clear(&hcd->reserve_cpus); + + for (i = 0; i < MAX_RESERVE_CPUS; i++) { + if (hh_cpumap[i].cap_id == 0) + break; + + hcd->cpumap[i].cap_id = hh_cpumap[i].cap_id; + hcd->cpumap[i].pcpu = hh_cpumap[i].pcpu; + hcd->cpumap[i].curr_pcpu = hh_cpumap[i].curr_pcpu; + cpumask_set_cpu(hcd->cpumap[i].pcpu, &hcd->reserve_cpus); + pr_debug("vcpu%u map to pcpu%u\n", i, hcd->cpumap[i].pcpu); + } + + cpumask_copy(&hcd->final_reserved_cpus, &hcd->reserve_cpus); + spin_unlock(&hcd->lock); + pr_info("reserve_cpus=%*pbl\n", cpumask_pr_args(&hcd->reserve_cpus)); +} + +/* + * Called when vm_status is STATUS_READY, multiple times before status + * moves to STATUS_RUNNING + */ +int hh_vcpu_populate_affinity_info(u32 cpu_idx, u64 cap_id) +{ + if (!init_done) { + pr_err("Driver probe failed\n"); + return -ENXIO; + } + + if (!is_vcpu_info_populated) { + hh_cpumap[nr_vcpus].cap_id = cap_id; + hh_cpumap[nr_vcpus].pcpu = cpu_idx; + hh_cpumap[nr_vcpus].curr_pcpu = cpu_idx; + + nr_vcpus++; + pr_debug("cpu_index:%u vcpu_cap_id:%llu nr_vcpus:%d\n", + cpu_idx, cap_id, nr_vcpus); + } + + return 0; +} + +static int hh_vcpu_done_populate_affinity_info(struct notifier_block *nb, + unsigned long cmd, void *data) +{ + struct hh_rm_notif_vm_status_payload *vm_status_payload = data; + u8 vm_status = vm_status_payload->vm_status; + + if (cmd == HH_RM_NOTIF_VM_STATUS && + vm_status == HH_RM_VM_STATUS_RUNNING && + !is_vcpu_info_populated) { + mutex_lock(&the_hcd->reservation_mutex); + hyp_core_ctl_init_reserve_cpus(the_hcd); + is_vcpu_info_populated = true; + mutex_unlock(&the_hcd->reservation_mutex); + } + + return NOTIFY_DONE; +} + +static struct notifier_block hh_vcpu_nb = { + .notifier_call = hh_vcpu_done_populate_affinity_info, +}; + +static void hyp_core_ctl_enable(bool enable) +{ + mutex_lock(&the_hcd->reservation_mutex); + if (!is_vcpu_info_populated) { + pr_err("VCPU info isn't populated\n"); + goto err_out; + } + + spin_lock(&the_hcd->lock); + if (enable == the_hcd->reservation_enabled) + goto out; + + trace_hyp_core_ctl_enable(enable); + pr_debug("reservation %s\n", enable ? "enabled" : "disabled"); + + the_hcd->reservation_enabled = enable; + the_hcd->pending = true; + wake_up_process(the_hcd->task); +out: + spin_unlock(&the_hcd->lock); +err_out: + mutex_unlock(&the_hcd->reservation_mutex); +} + +static ssize_t enable_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + bool enable; + int ret; + + ret = kstrtobool(buf, &enable); + if (ret < 0) + return -EINVAL; + + hyp_core_ctl_enable(enable); + + return count; +} + +static ssize_t enable_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", the_hcd->reservation_enabled); +} + +static DEVICE_ATTR_RW(enable); + +static ssize_t status_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct hyp_core_ctl_data *hcd = the_hcd; + ssize_t count; + int i; + + mutex_lock(&hcd->reservation_mutex); + + count = scnprintf(buf, PAGE_SIZE, "enabled=%d\n", + hcd->reservation_enabled); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "reserve_cpus=%*pbl\n", + cpumask_pr_args(&hcd->reserve_cpus)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "reserved_cpus=%*pbl\n", + cpumask_pr_args(&hcd->final_reserved_cpus)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "our_paused_cpus=%*pbl\n", + cpumask_pr_args(&hcd->our_paused_cpus)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "online_cpus=%*pbl\n", + cpumask_pr_args(cpu_online_mask)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "active_cpus=%*pbl\n", + cpumask_pr_args(cpu_active_mask)); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "thermal_cpus=%*pbl\n", + cpumask_pr_args(cpu_cooling_multi_get_max_level_cpumask())); + + count += scnprintf(buf + count, PAGE_SIZE - count, + "Vcpu to Pcpu mappings:\n"); + + for (i = 0; i < MAX_RESERVE_CPUS; i++) { + if (hcd->cpumap[i].cap_id == 0) + break; + + count += scnprintf(buf + count, PAGE_SIZE - count, + "vcpu=%d pcpu=%u curr_pcpu=%u\n", + i, hcd->cpumap[i].pcpu, hcd->cpumap[i].curr_pcpu); + + } + + mutex_unlock(&hcd->reservation_mutex); + + return count; +} + +static DEVICE_ATTR_RO(status); + +static int init_freq_qos_req(void) +{ + int cpu, ret; + struct cpufreq_policy *policy; + struct freq_qos_request *qos_req; + + for_each_possible_cpu(cpu) { + policy = cpufreq_cpu_get(cpu); + if (!policy) { + pr_err("cpufreq policy not found for cpu%d\n", cpu); + ret = -ESRCH; + goto remove_qos_req; + } + + qos_req = &per_cpu(qos_min_req, cpu); + ret = freq_qos_add_request(&policy->constraints, qos_req, + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) { + pr_err("Failed to add min freq constraint (%d)\n", ret); + cpufreq_cpu_put(policy); + goto remove_qos_req; + } + cpufreq_cpu_put(policy); + } + + return 0; + +remove_qos_req: + for_each_possible_cpu(cpu) { + qos_req = &per_cpu(qos_min_req, cpu); + if (freq_qos_request_active(qos_req)) + freq_qos_remove_request(qos_req); + } + + return ret; +} + +static ssize_t hcc_min_freq_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int i, ret, ntokens = 0; + unsigned int val, cpu; + const char *cp = buf; + + mutex_lock(&the_hcd->reservation_mutex); + if (!is_vcpu_info_populated) { + pr_err("VCPU info isn't populated\n"); + goto err_out; + } + + if (!freq_qos_init_done) { + if (init_freq_qos_req()) + goto err_out; + freq_qos_init_done = true; + } + + while ((cp = strpbrk(cp + 1, " :"))) + ntokens++; + + /* CPU:value pair */ + if (!(ntokens % 2)) + goto err_out; + + cp = buf; + for (i = 0; i < ntokens; i += 2) { + if (sscanf(cp, "%u:%u", &cpu, &val) != 2) + goto err_out; + if (cpu >= num_possible_cpus()) + goto err_out; + + per_cpu(qos_min_freq, cpu) = val; + cp = strnchr(cp, strlen(cp), ' '); + cp++; + } + + mutex_unlock(&the_hcd->reservation_mutex); + return count; + +err_out: + ret = -EINVAL; + mutex_unlock(&the_hcd->reservation_mutex); + return ret; +} + +static ssize_t hcc_min_freq_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int cnt = 0, cpu; + + for_each_possible_cpu(cpu) { + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, + "%d:%u ", cpu, + per_cpu(qos_min_freq, cpu)); + } + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); + return cnt; +} + +static DEVICE_ATTR_RW(hcc_min_freq); + +static struct attribute *hyp_core_ctl_attrs[] = { + &dev_attr_enable.attr, + &dev_attr_status.attr, + &dev_attr_hcc_min_freq.attr, + NULL +}; + +static struct attribute_group hyp_core_ctl_attr_group = { + .attrs = hyp_core_ctl_attrs, + .name = "hyp_core_ctl", +}; + +#define CPULIST_SZ 32 +static ssize_t read_reserve_cpus(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + char kbuf[CPULIST_SZ]; + int ret; + + ret = scnprintf(kbuf, CPULIST_SZ, "%*pbl\n", + cpumask_pr_args(&the_hcd->reserve_cpus)); + + return simple_read_from_buffer(ubuf, count, ppos, kbuf, ret); +} + +static ssize_t write_reserve_cpus(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + char kbuf[CPULIST_SZ]; + int ret; + cpumask_t temp_mask; + + mutex_lock(&the_hcd->reservation_mutex); + if (!is_vcpu_info_populated) { + pr_err("VCPU info isn't populated\n"); + ret = -EPERM; + goto err_out; + } + + ret = simple_write_to_buffer(kbuf, CPULIST_SZ - 1, ppos, ubuf, count); + if (ret < 0) + goto err_out; + + kbuf[ret] = '\0'; + ret = cpulist_parse(kbuf, &temp_mask); + if (ret < 0) + goto err_out; + + if (cpumask_weight(&temp_mask) != + cpumask_weight(&the_hcd->reserve_cpus)) { + pr_err("incorrect reserve CPU count. expected=%u\n", + cpumask_weight(&the_hcd->reserve_cpus)); + ret = -EINVAL; + goto err_out; + } + + spin_lock(&the_hcd->lock); + if (the_hcd->reservation_enabled) { + count = -EPERM; + pr_err("reservation is enabled, can't change reserve_cpus\n"); + } else { + cpumask_copy(&the_hcd->reserve_cpus, &temp_mask); + } + spin_unlock(&the_hcd->lock); + mutex_unlock(&the_hcd->reservation_mutex); + + return count; +err_out: + mutex_unlock(&the_hcd->reservation_mutex); + return ret; +} + +static const struct file_operations debugfs_reserve_cpus_ops = { + .read = read_reserve_cpus, + .write = write_reserve_cpus, +}; + +static void hyp_core_ctl_debugfs_init(void) +{ + struct dentry *dir, *file; + + dir = debugfs_create_dir("hyp_core_ctl", NULL); + if (IS_ERR_OR_NULL(dir)) + return; + + file = debugfs_create_file("reserve_cpus", 0644, dir, NULL, + &debugfs_reserve_cpus_ops); + if (!file) + debugfs_remove(dir); +} + +static int hyp_core_ctl_probe(struct platform_device *pdev) +{ + int ret; + struct hyp_core_ctl_data *hcd; + struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; + + ret = hh_rm_register_notifier(&hh_vcpu_nb); + if (ret) + return ret; + + hcd = kzalloc(sizeof(*hcd), GFP_KERNEL); + if (!hcd) { + ret = -ENOMEM; + goto unregister_rm_notifier; + } + + spin_lock_init(&hcd->lock); + mutex_init(&hcd->reservation_mutex); + hcd->task = kthread_run(hyp_core_ctl_thread, (void *) hcd, + "hyp_core_ctl"); + + if (IS_ERR(hcd->task)) { + ret = PTR_ERR(hcd->task); + goto free_hcd; + } + + sched_setscheduler_nocheck(hcd->task, SCHED_FIFO, ¶m); + + ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, + &hyp_core_ctl_attr_group); + if (ret < 0) { + pr_err("Fail to create sysfs files. ret=%d\n", ret); + goto stop_task; + } + + cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN_END, + "qcom/hyp_core_ctl:online", + hyp_core_ctl_hp_online, + hyp_core_ctl_hp_offline); + + cpu_cooling_multi_max_level_notifier_register(&hyp_core_ctl_nb); + hyp_core_ctl_debugfs_init(); + + the_hcd = hcd; + init_done = true; + return 0; + +stop_task: + kthread_stop(hcd->task); +free_hcd: + kfree(hcd); +unregister_rm_notifier: + hh_rm_unregister_notifier(&hh_vcpu_nb); + + return ret; +} + +static const struct of_device_id hyp_core_ctl_match_table[] = { + { .compatible = "qcom,hyp-core-ctl" }, + {}, +}; + +static struct platform_driver hyp_core_ctl_driver = { + .probe = hyp_core_ctl_probe, + .driver = { + .name = "hyp_core_ctl", + .owner = THIS_MODULE, + .of_match_table = hyp_core_ctl_match_table, + }, +}; + +builtin_platform_driver(hyp_core_ctl_driver); +MODULE_DESCRIPTION("Core Control for Hypervisor"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/soc/qcom/msm_performance.c b/drivers/soc/qcom/msm_performance.c new file mode 100644 index 000000000000..a2dcfac2f267 --- /dev/null +++ b/drivers/soc/qcom/msm_performance.c @@ -0,0 +1,1231 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define POLL_INT 25 +#define NODE_NAME_MAX_CHARS 16 + +#define QUEUE_POOL_SIZE 512 /*2^8 always keep in 2^x */ +#define INST_EV 0x08 /* 0th event*/ +#define CYC_EV 0x11 /* 1st event*/ +#define INIT "Init" +#define CPU_CYCLE_THRESHOLD 650000 + + +static DEFINE_PER_CPU(bool, cpu_is_idle); +static DEFINE_PER_CPU(bool, cpu_is_hp); +static DEFINE_MUTEX(perfevent_lock); + +enum event_idx { + INST_EVENT, + CYC_EVENT, + NO_OF_EVENT +}; + +enum cpu_clusters { + MIN = 0, + MID = 1, + MAX = 2, + CLUSTER_MAX +}; + +static struct kset *msm_perf_kset; +static struct kobject *param_kobj; + +static ssize_t get_cpu_min_freq(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_cpu_min_freq(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +static ssize_t get_cpu_max_freq(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_cpu_max_freq(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +static ssize_t get_cpu_total_instruction(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t get_core_ctl_register(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_core_ctl_register(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +static ssize_t get_game_start_pid(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_game_start_pid(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +#ifdef CONFIG_QTI_PLH +static ssize_t get_plh_log_level(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_plh_log_level(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +static ssize_t get_splh_notif(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +static ssize_t set_splh_notif(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count); +#endif +static struct kobj_attribute cpu_min_freq_attr = + __ATTR(cpu_min_freq, 0644, get_cpu_min_freq, set_cpu_min_freq); +static struct kobj_attribute cpu_max_freq_attr = + __ATTR(cpu_max_freq, 0644, get_cpu_max_freq, set_cpu_max_freq); +static struct kobj_attribute inst_attr = + __ATTR(inst, 0444, get_cpu_total_instruction, NULL); +static struct kobj_attribute core_ctl_register_attr = + __ATTR(core_ctl_register, 0644, get_core_ctl_register, + set_core_ctl_register); +static struct kobj_attribute evnt_gplaf_pid_attr = + __ATTR(evnt_gplaf_pid, 0644, get_game_start_pid, set_game_start_pid); +#ifdef CONFIG_QTI_PLH +static struct kobj_attribute plh_log_level_attr = + __ATTR(plh_log_level, 0644, get_plh_log_level, set_plh_log_level); +static struct kobj_attribute splh_notify_attr = + __ATTR(splh_notify, 0644, get_splh_notif, set_splh_notif); +#endif + +static struct attribute *param_attrs[] = { + &cpu_min_freq_attr.attr, + &cpu_max_freq_attr.attr, + &inst_attr.attr, + &core_ctl_register_attr.attr, + &evnt_gplaf_pid_attr.attr, +#ifdef CONFIG_QTI_PLH + &plh_log_level_attr.attr, + &splh_notify_attr.attr, +#endif + NULL, +}; + +static struct attribute_group param_attr_group = { + .attrs = param_attrs, +}; + +static int add_module_params(void) +{ + int ret; + struct kobject *module_kobj; + + module_kobj = &msm_perf_kset->kobj; + + param_kobj = kobject_create_and_add("parameters", module_kobj); + if (!param_kobj) { + pr_err("msm_perf: Failed to add param_kobj\n"); + return -ENOMEM; + } + + ret = sysfs_create_group(param_kobj, ¶m_attr_group); + if (ret) { + pr_err("msm_perf: Failed to create sysfs\n"); + return ret; + } + return 0; +} + +/* To handle cpufreq min/max request */ +struct cpu_status { + unsigned int min; + unsigned int max; +}; +static DEFINE_PER_CPU(struct cpu_status, msm_perf_cpu_stats); +static DEFINE_PER_CPU(struct freq_qos_request, qos_req_min); +static DEFINE_PER_CPU(struct freq_qos_request, qos_req_max); + +static cpumask_var_t limit_mask_min; +static cpumask_var_t limit_mask_max; + +static DECLARE_COMPLETION(gfx_evt_arrival); + +struct gpu_data { + pid_t pid; + int ctx_id; + unsigned int timestamp; + ktime_t arrive_ts; + int evt_typ; +}; + +static struct gpu_data gpu_circ_buff[QUEUE_POOL_SIZE]; + +struct queue_indicies { + int head; + int tail; +}; +static struct queue_indicies curr_pos; + +static DEFINE_SPINLOCK(gfx_circ_buff_lock); + +struct event_data { + struct perf_event *pevent; + u64 prev_count; + u64 cur_delta; + u64 cached_total_count; +}; +static struct event_data **pmu_events; +static unsigned long min_cpu_capacity = ULONG_MAX; + +struct events { + spinlock_t cpu_hotplug_lock; + bool cpu_hotplug; + bool init_success; +}; +static struct events events_group; +static struct task_struct *events_notify_thread; + +static unsigned int aggr_big_nr; +static unsigned int aggr_top_load; +static unsigned int top_load[CLUSTER_MAX]; +static unsigned int curr_cap[CLUSTER_MAX]; +static atomic_t game_status_pid; +static bool ready_for_freq_updates; + +static int freq_qos_request_init(void) +{ + unsigned int cpu; + int ret; + + struct cpufreq_policy *policy; + struct freq_qos_request *req; + + for_each_present_cpu(cpu) { + policy = cpufreq_cpu_get(cpu); + if (!policy) { + pr_err("%s: Failed to get cpufreq policy for cpu%d\n", + __func__, cpu); + ret = -EAGAIN; + goto cleanup; + } + per_cpu(msm_perf_cpu_stats, cpu).min = 0; + req = &per_cpu(qos_req_min, cpu); + ret = freq_qos_add_request(&policy->constraints, req, + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) { + pr_err("%s: Failed to add min freq constraint (%d)\n", + __func__, ret); + cpufreq_cpu_put(policy); + goto cleanup; + } + + per_cpu(msm_perf_cpu_stats, cpu).max = UINT_MAX; + req = &per_cpu(qos_req_max, cpu); + ret = freq_qos_add_request(&policy->constraints, req, + FREQ_QOS_MAX, FREQ_QOS_MAX_DEFAULT_VALUE); + if (ret < 0) { + pr_err("%s: Failed to add max freq constraint (%d)\n", + __func__, ret); + cpufreq_cpu_put(policy); + goto cleanup; + } + + cpufreq_cpu_put(policy); + } + return 0; + +cleanup: + for_each_present_cpu(cpu) { + req = &per_cpu(qos_req_min, cpu); + if (req && freq_qos_request_active(req)) + freq_qos_remove_request(req); + + + req = &per_cpu(qos_req_max, cpu); + if (req && freq_qos_request_active(req)) + freq_qos_remove_request(req); + + per_cpu(msm_perf_cpu_stats, cpu).min = 0; + per_cpu(msm_perf_cpu_stats, cpu).max = UINT_MAX; + } + return ret; +} + +/*******************************sysfs start************************************/ +static ssize_t set_cpu_min_freq(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + int i, j, ntokens = 0; + unsigned int val, cpu; + const char *cp = buf; + struct cpu_status *i_cpu_stats; + struct cpufreq_policy policy; + struct freq_qos_request *req; + int ret = 0; + + if (!ready_for_freq_updates) { + ret = freq_qos_request_init(); + if (ret) { + pr_err("%s: Failed to init qos requests policy for ret=%d\n", + __func__, ret); + return ret; + } + ready_for_freq_updates = true; + } + + while ((cp = strpbrk(cp + 1, " :"))) + ntokens++; + + /* CPU:value pair */ + if (!(ntokens % 2)) + return -EINVAL; + + cp = buf; + cpumask_clear(limit_mask_min); + for (i = 0; i < ntokens; i += 2) { + if (sscanf(cp, "%u:%u", &cpu, &val) != 2) + return -EINVAL; + if (cpu > (num_present_cpus() - 1)) + return -EINVAL; + + i_cpu_stats = &per_cpu(msm_perf_cpu_stats, cpu); + + i_cpu_stats->min = val; + cpumask_set_cpu(cpu, limit_mask_min); + + cp = strnchr(cp, strlen(cp), ' '); + cp++; + } + + /* + * Since on synchronous systems policy is shared amongst multiple + * CPUs only one CPU needs to be updated for the limit to be + * reflected for the entire cluster. We can avoid updating the policy + * of other CPUs in the cluster once it is done for at least one CPU + * in the cluster + */ + cpus_read_lock(); + for_each_cpu(i, limit_mask_min) { + i_cpu_stats = &per_cpu(msm_perf_cpu_stats, i); + + if (cpufreq_get_policy(&policy, i)) + continue; + + if (cpu_online(i)) { + req = &per_cpu(qos_req_min, i); + if (freq_qos_update_request(req, i_cpu_stats->min) < 0) + break; + } + + for_each_cpu(j, policy.related_cpus) + cpumask_clear_cpu(j, limit_mask_min); + } + cpus_read_unlock(); + + return count; +} + +static ssize_t get_cpu_min_freq(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int cnt = 0, cpu; + + for_each_present_cpu(cpu) { + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, + "%d:%u ", cpu, + per_cpu(msm_perf_cpu_stats, cpu).min); + } + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); + return cnt; +} + +static ssize_t set_cpu_max_freq(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + int i, j, ntokens = 0; + unsigned int val, cpu; + const char *cp = buf; + struct cpu_status *i_cpu_stats; + struct cpufreq_policy policy; + struct freq_qos_request *req; + int ret = 0; + + if (!ready_for_freq_updates) { + ret = freq_qos_request_init(); + if (ret) { + pr_err("%s: Failed to init qos requests policy for ret=%d\n", + __func__, ret); + return ret; + } + ready_for_freq_updates = true; + } + + while ((cp = strpbrk(cp + 1, " :"))) + ntokens++; + + /* CPU:value pair */ + if (!(ntokens % 2)) + return -EINVAL; + + cp = buf; + cpumask_clear(limit_mask_max); + for (i = 0; i < ntokens; i += 2) { + if (sscanf(cp, "%u:%u", &cpu, &val) != 2) + return -EINVAL; + if (cpu > (num_present_cpus() - 1)) + return -EINVAL; + + i_cpu_stats = &per_cpu(msm_perf_cpu_stats, cpu); + + i_cpu_stats->max = val; + cpumask_set_cpu(cpu, limit_mask_max); + + cp = strnchr(cp, strlen(cp), ' '); + cp++; + } + + cpus_read_lock(); + for_each_cpu(i, limit_mask_max) { + i_cpu_stats = &per_cpu(msm_perf_cpu_stats, i); + if (cpufreq_get_policy(&policy, i)) + continue; + + if (cpu_online(i)) { + req = &per_cpu(qos_req_max, i); + if (freq_qos_update_request(req, i_cpu_stats->max) < 0) + break; + } + + for_each_cpu(j, policy.related_cpus) + cpumask_clear_cpu(j, limit_mask_max); + } + cpus_read_unlock(); + + return count; +} + +static ssize_t get_cpu_max_freq(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int cnt = 0, cpu; + + for_each_present_cpu(cpu) { + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, + "%d:%u ", cpu, + per_cpu(msm_perf_cpu_stats, cpu).max); + } + cnt += scnprintf(buf + cnt, PAGE_SIZE - cnt, "\n"); + return cnt; +} + +static struct kobject *events_kobj; + +static ssize_t show_cpu_hotplug(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "\n"); +} +static struct kobj_attribute cpu_hotplug_attr = +__ATTR(cpu_hotplug, 0444, show_cpu_hotplug, NULL); + +static struct attribute *events_attrs[] = { + &cpu_hotplug_attr.attr, + NULL, +}; + +static struct attribute_group events_attr_group = { + .attrs = events_attrs, +}; + +static ssize_t show_perf_gfx_evts(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + struct queue_indicies updated_pos; + unsigned long flags; + ssize_t retval = 0; + int idx = 0, size, act_idx, ret = -1; + + ret = wait_for_completion_interruptible(&gfx_evt_arrival); + if (ret) + return 0; + spin_lock_irqsave(&gfx_circ_buff_lock, flags); + updated_pos.head = curr_pos.head; + updated_pos.tail = curr_pos.tail; + size = CIRC_CNT(updated_pos.head, updated_pos.tail, QUEUE_POOL_SIZE); + curr_pos.tail = (curr_pos.tail + size) % QUEUE_POOL_SIZE; + spin_unlock_irqrestore(&gfx_circ_buff_lock, flags); + + for (idx = 0; idx < size; idx++) { + act_idx = (updated_pos.tail + idx) % QUEUE_POOL_SIZE; + retval += scnprintf(buf + retval, PAGE_SIZE - retval, + "%d %d %u %d %lu :", + gpu_circ_buff[act_idx].pid, + gpu_circ_buff[act_idx].ctx_id, + gpu_circ_buff[act_idx].timestamp, + gpu_circ_buff[act_idx].evt_typ, + ktime_to_us(gpu_circ_buff[act_idx].arrive_ts)); + if (retval >= PAGE_SIZE) { + pr_err("msm_perf:data limit exceed\n"); + break; + } + } + return retval; +} + +static struct kobj_attribute gfx_event_info_attr = +__ATTR(gfx_evt, 0444, show_perf_gfx_evts, NULL); + +static ssize_t show_big_nr(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", aggr_big_nr); +} + +static struct kobj_attribute big_nr_attr = +__ATTR(aggr_big_nr, 0444, show_big_nr, NULL); + +static ssize_t show_top_load(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", aggr_top_load); +} + +static struct kobj_attribute top_load_attr = +__ATTR(aggr_top_load, 0444, show_top_load, NULL); + + +static ssize_t show_top_load_cluster(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u %u %u\n", + top_load[MIN], top_load[MID], + top_load[MAX]); +} + +static struct kobj_attribute cluster_top_load_attr = +__ATTR(top_load_cluster, 0444, show_top_load_cluster, NULL); + +static ssize_t show_curr_cap_cluster(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u %u %u\n", + curr_cap[MIN], curr_cap[MID], + curr_cap[MAX]); +} + +static struct kobj_attribute cluster_curr_cap_attr = +__ATTR(curr_cap_cluster, 0444, show_curr_cap_cluster, NULL); + +static struct attribute *notify_attrs[] = { + &big_nr_attr.attr, + &top_load_attr.attr, + &cluster_top_load_attr.attr, + &cluster_curr_cap_attr.attr, + &gfx_event_info_attr.attr, + NULL, +}; + +static struct attribute_group notify_attr_group = { + .attrs = notify_attrs, +}; +static struct kobject *notify_kobj; + +/*******************************sysfs ends************************************/ + +/*****************PMU Data Collection*****************/ +static struct perf_event_attr attr; +static void msm_perf_init_attr(void) +{ + memset(&attr, 0, sizeof(struct perf_event_attr)); + + attr.type = PERF_TYPE_RAW; + attr.size = sizeof(struct perf_event_attr); + attr.pinned = 1; +} + +static int set_event(struct event_data *ev, int cpu) +{ + struct perf_event *pevent; + + pevent = perf_event_create_kernel_counter(&attr, + cpu, NULL, NULL, NULL); + if (IS_ERR(pevent)) { + pr_err("msm_perf: %s failed, eventId:0x%x, cpu:%d, error code:%ld\n", + __func__, attr.config, cpu, PTR_ERR(pevent)); + return PTR_ERR(pevent); + } + ev->pevent = pevent; + perf_event_enable(pevent); + + return 0; +} + +static void free_pmu_counters(unsigned int cpu) +{ + int i = 0; + + for (i = 0; i < NO_OF_EVENT; i++) { + pmu_events[i][cpu].prev_count = 0; + pmu_events[i][cpu].cur_delta = 0; + pmu_events[i][cpu].cached_total_count = 0; + if (pmu_events[i][cpu].pevent) { + perf_event_disable(pmu_events[i][cpu].pevent); + perf_event_release_kernel(pmu_events[i][cpu].pevent); + pmu_events[i][cpu].pevent = NULL; + } + } +} + +static int init_pmu_counter(void) +{ + int cpu; + unsigned long cpu_capacity; + int ret = 0; + + int i = 0, j = 0; + int no_of_cpus = 0; + + msm_perf_init_attr(); + for_each_possible_cpu(cpu) + no_of_cpus++; + + pmu_events = kcalloc(NO_OF_EVENT, sizeof(struct event_data *), GFP_KERNEL); + if (!pmu_events) + return -ENOMEM; + for (i = 0; i < NO_OF_EVENT; i++) { + pmu_events[i] = kcalloc(no_of_cpus, sizeof(struct event_data), GFP_KERNEL); + if (!pmu_events[i]) { + for (j = i; j >= 0; j--) { + kfree(pmu_events[j]); + pmu_events[j] = NULL; + } + kfree(pmu_events); + pmu_events = NULL; + return -ENOMEM; + } + } + + /* Create events per CPU */ + for_each_possible_cpu(cpu) { + /* create Instruction event */ + attr.config = INST_EV; + ret = set_event(&pmu_events[INST_EVENT][cpu], cpu); + if (ret < 0) + return ret; + /* create cycle event */ + attr.config = CYC_EV; + ret = set_event(&pmu_events[CYC_EVENT][cpu], cpu); + if (ret < 0) { + free_pmu_counters(cpu); + return ret; + } + /* find capacity per cpu */ + cpu_capacity = arch_scale_cpu_capacity(cpu); + if (cpu_capacity < min_cpu_capacity) + min_cpu_capacity = cpu_capacity; + } + return 0; +} + +static inline void msm_perf_read_event(struct event_data *event) +{ + u64 ev_count = 0; + u64 total, enabled, running; + + mutex_lock(&perfevent_lock); + if (!event->pevent) { + mutex_unlock(&perfevent_lock); + return; + } + + if (!per_cpu(cpu_is_idle, event->pevent->cpu) && + !per_cpu(cpu_is_hp, event->pevent->cpu)) + total = perf_event_read_value(event->pevent, &enabled, &running); + else + total = event->cached_total_count; + + ev_count = total - event->prev_count; + event->prev_count = total; + event->cur_delta = ev_count; + mutex_unlock(&perfevent_lock); +} + +static ssize_t get_cpu_total_instruction(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + u64 instruction = 0; + u64 cycles = 0; + u64 total_inst_big = 0; + u64 total_inst_little = 0; + u64 ipc_big = 0; + u64 ipc_little = 0; + int cnt = 0, cpu; + + for_each_possible_cpu(cpu) { + /* Read Instruction event */ + msm_perf_read_event(&pmu_events[INST_EVENT][cpu]); + /* Read Cycle event */ + msm_perf_read_event(&pmu_events[CYC_EVENT][cpu]); + instruction = pmu_events[INST_EVENT][cpu].cur_delta; + cycles = pmu_events[CYC_EVENT][cpu].cur_delta; + /* collecting max inst and ipc for max cap and min cap cpus */ + if (arch_scale_cpu_capacity(cpu) > min_cpu_capacity) { + if (cycles && cycles >= CPU_CYCLE_THRESHOLD) + ipc_big = max(ipc_big, + ((instruction*100)/cycles)); + total_inst_big += instruction; + } else { + if (cycles) + ipc_little = max(ipc_little, + ((instruction*100)/cycles)); + total_inst_little += instruction; + } + } + + cnt += scnprintf(buf, PAGE_SIZE, "%llu:%llu:%llu:%llu\n", + total_inst_big, ipc_big, + total_inst_little, ipc_little); + + return cnt; +} + +static int restart_events(unsigned int cpu, bool cpu_up) +{ + int ret = 0; + + msm_perf_init_attr(); + + if (cpu_up) { + /* create Instruction event */ + attr.config = INST_EV; + ret = set_event(&pmu_events[INST_EVENT][cpu], cpu); + if (ret < 0) + return ret; + /* create cycle event */ + attr.config = CYC_EV; + ret = set_event(&pmu_events[CYC_EVENT][cpu], cpu); + if (ret < 0) { + free_pmu_counters(cpu); + return ret; + } + } else { + free_pmu_counters(cpu); + } + + return 0; +} + +static int hotplug_notify_down(unsigned int cpu) +{ + mutex_lock(&perfevent_lock); + per_cpu(cpu_is_hp, cpu) = true; + restart_events(cpu, false); + mutex_unlock(&perfevent_lock); + + return 0; +} + +static int hotplug_notify_up(unsigned int cpu) +{ + unsigned long flags; + + mutex_lock(&perfevent_lock); + restart_events(cpu, true); + per_cpu(cpu_is_hp, cpu) = false; + mutex_unlock(&perfevent_lock); + + if (events_group.init_success) { + spin_lock_irqsave(&(events_group.cpu_hotplug_lock), flags); + events_group.cpu_hotplug = true; + spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), flags); + wake_up_process(events_notify_thread); + } + + return 0; +} + +static int msm_perf_idle_read_events(unsigned int cpu) +{ + int ret = 0, i; + + for (i = 0; i < NO_OF_EVENT; i++) { + if (pmu_events[i][cpu].pevent) + ret = perf_event_read_local(pmu_events[i][cpu].pevent, + &pmu_events[i][cpu].cached_total_count, NULL, NULL); + } + + return ret; +} + +static void msm_perf_idle_notif(void *unused, unsigned int state, + unsigned int cpu) +{ + if (state == PWR_EVENT_EXIT) { + __this_cpu_write(cpu_is_idle, false); + } else { + __this_cpu_write(cpu_is_idle, true); + if (!per_cpu(cpu_is_hp, cpu)) + msm_perf_idle_read_events(cpu); + } +} + +static int events_notify_userspace(void *data) +{ + unsigned long flags; + bool notify_change; + + while (1) { + + set_current_state(TASK_INTERRUPTIBLE); + spin_lock_irqsave(&(events_group.cpu_hotplug_lock), flags); + + if (!events_group.cpu_hotplug) { + spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), + flags); + + schedule(); + if (kthread_should_stop()) + break; + spin_lock_irqsave(&(events_group.cpu_hotplug_lock), + flags); + } + + set_current_state(TASK_RUNNING); + notify_change = events_group.cpu_hotplug; + events_group.cpu_hotplug = false; + spin_unlock_irqrestore(&(events_group.cpu_hotplug_lock), flags); + + if (notify_change) + sysfs_notify(events_kobj, NULL, "cpu_hotplug"); + } + + return 0; +} + +static int init_notify_group(void) +{ + int ret; + struct kobject *module_kobj = &msm_perf_kset->kobj; + + notify_kobj = kobject_create_and_add("notify", module_kobj); + if (!notify_kobj) { + pr_err("msm_perf: Failed to add notify_kobj\n"); + return -ENOMEM; + } + + ret = sysfs_create_group(notify_kobj, ¬ify_attr_group); + if (ret) { + kobject_put(notify_kobj); + pr_err("msm_perf: Failed to create sysfs\n"); + return ret; + } + return 0; +} + +static int init_events_group(void) +{ + int ret; + struct kobject *module_kobj = &msm_perf_kset->kobj; + + events_kobj = kobject_create_and_add("events", module_kobj); + if (!events_kobj) { + pr_err("msm_perf: Failed to add events_kobj\n"); + return -ENOMEM; + } + + ret = sysfs_create_group(events_kobj, &events_attr_group); + if (ret) { + pr_err("msm_perf: Failed to create sysfs\n"); + return ret; + } + + spin_lock_init(&(events_group.cpu_hotplug_lock)); + events_notify_thread = kthread_run(events_notify_userspace, + NULL, "msm_perf:events_notify"); + if (IS_ERR(events_notify_thread)) + return PTR_ERR(events_notify_thread); + + events_group.init_success = true; + + return 0; +} + +static void nr_notify_userspace(struct work_struct *work) +{ + sysfs_notify(notify_kobj, NULL, "aggr_top_load"); + sysfs_notify(notify_kobj, NULL, "aggr_big_nr"); + sysfs_notify(notify_kobj, NULL, "top_load_cluster"); + sysfs_notify(notify_kobj, NULL, "curr_cap_cluster"); +} + +static int msm_perf_core_ctl_notify(struct notifier_block *nb, + unsigned long unused, + void *data) +{ + static unsigned int tld, nrb, i; + static unsigned int top_ld[CLUSTER_MAX], curr_cp[CLUSTER_MAX]; + static DECLARE_WORK(sysfs_notify_work, nr_notify_userspace); + struct core_ctl_notif_data *d = data; + int cluster = 0; + + nrb += d->nr_big; + tld += d->coloc_load_pct; + for (cluster = 0; cluster < CLUSTER_MAX; cluster++) { + top_ld[cluster] += d->ta_util_pct[cluster]; + curr_cp[cluster] += d->cur_cap_pct[cluster]; + } + i++; + if (i == POLL_INT) { + aggr_big_nr = ((nrb%POLL_INT) ? 1 : 0) + nrb/POLL_INT; + aggr_top_load = tld/POLL_INT; + for (cluster = 0; cluster < CLUSTER_MAX; cluster++) { + top_load[cluster] = top_ld[cluster]/POLL_INT; + curr_cap[cluster] = curr_cp[cluster]/POLL_INT; + top_ld[cluster] = 0; + curr_cp[cluster] = 0; + } + tld = 0; + nrb = 0; + i = 0; + schedule_work(&sysfs_notify_work); + } + return NOTIFY_OK; +} + +static struct notifier_block msm_perf_nb = { + .notifier_call = msm_perf_core_ctl_notify +}; + +static bool core_ctl_register; +static ssize_t get_core_ctl_register(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%c\n", core_ctl_register ? 'Y' : 'N'); +} + +static ssize_t set_core_ctl_register(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + bool old_val = core_ctl_register; + int ret; + + ret = sscanf(buf, "%du", &core_ctl_register); + if (ret < 0) { + pr_err("msm_perf: getting new core_ctl_register failed, ret=%d\n", ret); + return ret; + } + + if (core_ctl_register == old_val) + return count; + + if (core_ctl_register) + core_ctl_notifier_register(&msm_perf_nb); + else + core_ctl_notifier_unregister(&msm_perf_nb); + + return count; +} + +void msm_perf_events_update(enum evt_update_t update_typ, + enum gfx_evt_t evt_typ, pid_t pid, + uint32_t ctx_id, uint32_t timestamp, bool end_of_frame) +{ + unsigned long flags; + int idx = 0; + + if (update_typ != MSM_PERF_GFX) + return; + + if (pid != atomic_read(&game_status_pid) || (timestamp == 0) + || !(end_of_frame)) + return; + + spin_lock_irqsave(&gfx_circ_buff_lock, flags); + idx = curr_pos.head; + curr_pos.head = ((curr_pos.head + 1) % QUEUE_POOL_SIZE); + spin_unlock_irqrestore(&gfx_circ_buff_lock, flags); + gpu_circ_buff[idx].pid = pid; + gpu_circ_buff[idx].ctx_id = ctx_id; + gpu_circ_buff[idx].timestamp = timestamp; + gpu_circ_buff[idx].evt_typ = evt_typ; + gpu_circ_buff[idx].arrive_ts = ktime_get(); + + if (evt_typ == MSM_PERF_QUEUE || evt_typ == MSM_PERF_RETIRED) + complete(&gfx_evt_arrival); +} +EXPORT_SYMBOL(msm_perf_events_update); + +static ssize_t set_game_start_pid(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + long usr_val = 0; + kstrtol(buf, 0, &usr_val); + atomic_set(&game_status_pid, usr_val); + return count; +} +static ssize_t get_game_start_pid(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + long usr_val = atomic_read(&game_status_pid); + + return scnprintf(buf, PAGE_SIZE, "%ld\n", usr_val); +} + +/*******************************GFX Call************************************/ +#ifdef CONFIG_QTI_PLH +static struct scmi_handle *plh_handle; +void rimps_plh_init(struct scmi_handle *handle) +{ + if (handle) + plh_handle = handle; +} +EXPORT_SYMBOL(rimps_plh_init); + +static int splh_notif, splh_init_done, plh_log_level; + +#define PLH_MIN_LOG_LEVEL 0 +#define PLH_MAX_LOG_LEVEL 0xF +#define SPLH_FPS_MAX_CNT 8 +#define SPLH_IPC_FREQ_VTBL_MAX_CNT 5 /* ipc freq pair */ +#define SPLH_INIT_IPC_FREQ_TBL_PARAMS \ + (2 + SPLH_FPS_MAX_CNT * (1 + (2 * SPLH_IPC_FREQ_VTBL_MAX_CNT))) + +static ssize_t get_plh_log_level(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%d\n", plh_log_level); +} + +static ssize_t set_plh_log_level(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret, log_val_backup; + struct scmi_plh_vendor_ops *ops; + + if (!plh_handle || !plh_handle->plh_ops) { + pr_err("msm_perf: plh scmi handle or vendor ops null\n"); + return -EINVAL; + } + + ops = plh_handle->plh_ops; + + log_val_backup = plh_log_level; + + ret = sscanf(buf, "%du", &plh_log_level); + + if (ret < 0) { + pr_err("msm_perf: getting new plh_log_level failed, ret=%d\n", ret); + return ret; + } + + plh_log_level = clamp(plh_log_level, PLH_MIN_LOG_LEVEL, PLH_MAX_LOG_LEVEL); + ret = ops->set_plh_log_level(plh_handle, plh_log_level); + if (ret < 0) { + plh_log_level = log_val_backup; + pr_err("msm_perf: setting new plh_log_level failed, ret=%d\n", ret); + return ret; + } + return count; +} + +static int init_splh_notif(const char *buf) +{ + int i, j, ret; + u16 tmp[SPLH_INIT_IPC_FREQ_TBL_PARAMS]; + u16 *ptmp = tmp, ntokens, nfps, n_ipc_freq_pair, tmp_valid_len = 0; + const char *cp, *cp1; + struct scmi_plh_vendor_ops *ops; + + /* buf contains the init info from user */ + if (buf == NULL || !plh_handle || !plh_handle->plh_ops) + return -EINVAL; + + cp = buf; + ntokens = 0; + while ((cp = strpbrk(cp + 1, ":"))) + ntokens++; + + /* format of cmd nfps, n_ipc_freq_pair, ,...>,... */ + cp = buf; + if (sscanf(cp, INIT ":%hu", &nfps)) { + if ((nfps != ntokens-1) || (nfps == 0) || (nfps > SPLH_FPS_MAX_CNT)) + return -EINVAL; + + cp = strnchr(cp, strlen(cp), ':'); /* skip INIT */ + cp++; + cp = strnchr(cp, strlen(cp), ':'); /* skip nfps */ + + *ptmp++ = nfps; /* nfps is first cmd param */ + tmp_valid_len++; + cp1 = cp; + ntokens = 0; + /* get count of nfps * n_ipc_freq_pair * */ + while ((cp1 = strpbrk(cp1 + 1, ","))) + ntokens++; + + if (ntokens % (2 * nfps)) /* ipc freq pair values should be multiple of nfps */ + return -EINVAL; + + n_ipc_freq_pair = ntokens / (2 * nfps); /* ipc_freq pair values for each FPS */ + if ((n_ipc_freq_pair == 0) || (n_ipc_freq_pair > SPLH_IPC_FREQ_VTBL_MAX_CNT)) + return -EINVAL; + + *ptmp++ = n_ipc_freq_pair; /* n_ipc_freq_pair is second cmd param */ + tmp_valid_len++; + cp1 = cp; + for (i = 0; i < nfps; i++) { + if (sscanf(cp1, ":%hu", ptmp) != 1) + return -EINVAL; + + ptmp++; /* increment after storing FPS val */ + tmp_valid_len++; + cp1 = strnchr(cp1, strlen(cp1), ','); /* move to ,ipc */ + for (j = 0; j < 2 * n_ipc_freq_pair; j++) { + if (sscanf(cp1, ",%hu", ptmp) != 1) + return -EINVAL; + + ptmp++; /* increment after storing ipc or freq */ + tmp_valid_len++; + cp1++; + if (j != (2 * n_ipc_freq_pair - 1)) + cp1 = strnchr(cp1, strlen(cp1), ','); /* move to next */ + } + + if (i != (nfps - 1)) + cp1 = strnchr(cp1, strlen(cp1), ':'); /* move to next FPS val */ + + } + } else { + return -EINVAL; + } + + ops = plh_handle->plh_ops; + ret = ops->init_splh_ipc_freq_tbl(plh_handle, tmp, tmp_valid_len); + if (ret < 0) + return -EINVAL; + + pr_info("msm_perf: nfps=%hu n_ipc_freq_pair=%hu last_freq_val=%hu len=%hu\n", + nfps, n_ipc_freq_pair, *--ptmp, tmp_valid_len); + splh_init_done = 1; + return 0; +} + +static void activate_splh_notif(void) +{ + int ret; + struct scmi_plh_vendor_ops *ops; + /* received event notification here */ + if (!plh_handle || !plh_handle->plh_ops) { + pr_err("msm_perf: splh not supported\n"); + return; + } + ops = plh_handle->plh_ops; + + if (splh_notif) + ret = ops->start_splh(plh_handle, splh_notif); /* splh_notif is fps */ + else + ret = ops->stop_splh(plh_handle); + + if (ret < 0) { + pr_err("msm_perf: splh start or stop failed, ret=%d\n", ret); + return; + } +} + +static ssize_t get_splh_notif(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%d\n", splh_notif); +} + +static ssize_t set_splh_notif(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int ret; + + if (strnstr(buf, INIT, sizeof(INIT)) != NULL) { + splh_init_done = 0; + ret = init_splh_notif(buf); + if (ret < 0) + pr_err("msm_perf: splh ipc freq tbl init failed, ret=%d\n", ret); + + return ret; + } + + if (!splh_init_done) { + pr_err("msm_perf: splh ipc freq tbl not initialized\n"); + return -EINVAL; + } + + ret = sscanf(buf, "%du", &splh_notif); + if (ret < 0) + return ret; + + activate_splh_notif(); + + return count; +} +#endif /* CONFIG_QTI_PLH */ + +static int __init msm_performance_init(void) +{ + unsigned int cpu; + int ret; + if (!alloc_cpumask_var(&limit_mask_min, GFP_KERNEL)) + return -ENOMEM; + + if (!alloc_cpumask_var(&limit_mask_max, GFP_KERNEL)) { + free_cpumask_var(limit_mask_min); + return -ENOMEM; + } + cpus_read_lock(); + for_each_possible_cpu(cpu) { + if (!cpumask_test_cpu(cpu, cpu_online_mask)) + per_cpu(cpu_is_hp, cpu) = true; + } + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "msm_performance_cpu_hotplug", + hotplug_notify_up, + hotplug_notify_down); + + cpus_read_unlock(); + + msm_perf_kset = kset_create_and_add("msm_performance", NULL, kernel_kobj); + if (!msm_perf_kset) { + free_cpumask_var(limit_mask_min); + free_cpumask_var(limit_mask_max); + return -ENOMEM; + } + + add_module_params(); + + init_events_group(); + init_notify_group(); + init_pmu_counter(); + + register_trace_cpu_idle(msm_perf_idle_notif, NULL); + return 0; +} +MODULE_LICENSE("GPL v2"); +late_initcall(msm_performance_init); diff --git a/drivers/thermal/qcom/Kconfig b/drivers/thermal/qcom/Kconfig index bfd889422dd3..53641c3470a1 100644 --- a/drivers/thermal/qcom/Kconfig +++ b/drivers/thermal/qcom/Kconfig @@ -41,3 +41,14 @@ config QCOM_LMH input from temperature and current sensors. On many newer Qualcomm SoCs LMh is configured in the firmware and this feature need not be enabled. However, on certain SoCs like sdm845 LMh has to be configured from kernel. + +config QTI_CPU_PAUSE_COOLING_DEVICE + tristate "QTI CPU Pause cooling devices" + depends on THERMAL_OF && HOTPLUG_CPU + depends on ARCH_QCOM || COMPILE_TEST + help + This enables the QTI CPU Pause cooling device. These cooling + devices will be used by QTI chipset to pause a CPU from being + scheduled and hence will let the CPU to power collapse. Pausing + a CPU will be used when the CPU frequency mitigation + is not good enough to achieve the necessary cooling. \ No newline at end of file diff --git a/drivers/thermal/qcom/Makefile b/drivers/thermal/qcom/Makefile index 0fa2512042e7..99a16fd8a22c 100644 --- a/drivers/thermal/qcom/Makefile +++ b/drivers/thermal/qcom/Makefile @@ -6,3 +6,4 @@ qcom_tsens-y += tsens.o tsens-v2.o tsens-v1.o tsens-v0_1.o \ obj-$(CONFIG_QCOM_SPMI_ADC_TM5) += qcom-spmi-adc-tm5.o obj-$(CONFIG_QCOM_SPMI_TEMP_ALARM) += qcom-spmi-temp-alarm.o obj-$(CONFIG_QCOM_LMH) += lmh.o +obj-$(CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE) += thermal_pause.o diff --git a/drivers/thermal/qcom/thermal_pause.c b/drivers/thermal/qcom/thermal_pause.c new file mode 100644 index 000000000000..5e367cb59cd4 --- /dev/null +++ b/drivers/thermal/qcom/thermal_pause.c @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) "%s:%s " fmt, KBUILD_MODNAME, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum thermal_pause_levels { + THERMAL_NO_CPU_PAUSE, + THERMAL_GROUP_CPU_PAUSE, + + /* define new pause levels above this line */ + MAX_THERMAL_PAUSE_LEVEL +}; + +#define THERMAL_PAUSE_RETRY_COUNT 5 + +struct thermal_pause_cdev { + struct list_head node; + cpumask_t cpu_mask; + enum thermal_pause_levels thermal_pause_level; + enum thermal_pause_levels thermal_pause_req; + struct thermal_cooling_device *cdev; + struct device_node *np; + char cdev_name[THERMAL_NAME_LENGTH]; + struct work_struct reg_work; + struct work_struct pause_update_work; +}; + +static DEFINE_MUTEX(cpus_pause_lock); +static LIST_HEAD(thermal_pause_cdev_list); + +static struct cpumask cpus_in_max_cooling_level; +static enum cpuhp_state cpu_hp_online; + +static BLOCKING_NOTIFIER_HEAD(thermal_pause_notifier); + +void thermal_pause_notifier_register(struct notifier_block *n) +{ + blocking_notifier_chain_register(&thermal_pause_notifier, n); +} +EXPORT_SYMBOL(thermal_pause_notifier_register); + +void thermal_pause_notifier_unregister(struct notifier_block *n) +{ + blocking_notifier_chain_unregister(&thermal_pause_notifier, n); +} +EXPORT_SYMBOL(thermal_pause_notifier_unregister); + +const struct cpumask *thermal_paused_cpumask(void) +{ + return &cpus_in_max_cooling_level; +} +EXPORT_SYMBOL(thermal_paused_cpumask); + +static int thermal_pause_hp_online(unsigned int online_cpu) +{ + struct thermal_pause_cdev *thermal_pause_cdev; + int ret = 0; + + pr_debug("online entry CPU:%d\n", online_cpu); + + mutex_lock(&cpus_pause_lock); + list_for_each_entry(thermal_pause_cdev, &thermal_pause_cdev_list, node) { + if (cpumask_test_cpu(online_cpu, &thermal_pause_cdev->cpu_mask) + && !thermal_pause_cdev->cdev) + queue_work(system_highpri_wq, + &thermal_pause_cdev->reg_work); + } + mutex_unlock(&cpus_pause_lock); + pr_debug("online exit CPU:%d\n", online_cpu); + return ret; +} + +/** + * thermal_pause_work - work function to pause a group of cpus at + * the specified level. + * + * @thermal_pasue_cdev: the cdev currently being processed + * + * Function to handle setting the current cpus paused by + * this driver for the mask specified in the device. + * it assumes the mutex is locked upon entrance. + */ +static int thermal_pause_work(struct thermal_pause_cdev *thermal_pause_cdev) +{ + int cpu = 0; + int ret = -EBUSY; + cpumask_t cpus_to_pause, cpus_to_notify; + + cpumask_copy(&cpus_to_pause, &thermal_pause_cdev->cpu_mask); + pr_debug("Pause:%*pbl\n", cpumask_pr_args(&thermal_pause_cdev->cpu_mask)); + + mutex_unlock(&cpus_pause_lock); + ret = walt_pause_cpus(&cpus_to_pause, PAUSE_THERMAL); + mutex_lock(&cpus_pause_lock); + + if (ret == 0) { + /* remove CPUs that have already been notified */ + cpumask_andnot(&cpus_to_notify, &thermal_pause_cdev->cpu_mask, + &cpus_in_max_cooling_level); + + for_each_cpu(cpu, &cpus_to_notify) + blocking_notifier_call_chain(&thermal_pause_notifier, 1, + (void *)(long)cpu); + + /* track CPUs currently in cooling level */ + cpumask_or(&cpus_in_max_cooling_level, + &cpus_in_max_cooling_level, + &thermal_pause_cdev->cpu_mask); + } else { + /* Failure. These cpus not paused by thermal */ + pr_err("Error pausing CPU:%*pbl. err:%d\n", + cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret); + } + + return ret; +} + +/** + * thermal_resume_work - work function to unpause a + * group of cpus in the mask for this cdev + * + * @thermal_pasue_cdev: the cdev currently being processed + * + * Function to handle enabling the group of cpus in the cdev. + * This is performed as a work function to avoid conflicts + * between a hotplug event invoking a pause cooling device, + * and a thermal event invoking a pause cooling device. + */ +static int thermal_resume_work(struct thermal_pause_cdev *thermal_pause_cdev) +{ + int cpu = 0; + int ret = -ENODEV; + cpumask_t cpus_to_unpause, new_paused_cpus, cpus_to_notify; + struct thermal_pause_cdev *cdev; + + cpumask_copy(&cpus_to_unpause, &thermal_pause_cdev->cpu_mask); + pr_debug("Unpause:%*pbl\n", cpumask_pr_args(&cpus_to_unpause)); + + mutex_unlock(&cpus_pause_lock); + ret = walt_resume_cpus(&cpus_to_unpause, PAUSE_THERMAL); + mutex_lock(&cpus_pause_lock); + + if (ret == 0) { + /* gather up the cpus paused state from all the cdevs */ + cpumask_clear(&new_paused_cpus); + list_for_each_entry(cdev, &thermal_pause_cdev_list, node) { + if (!cdev->thermal_pause_level || cdev == thermal_pause_cdev) + continue; + cpumask_or(&new_paused_cpus, &new_paused_cpus, &cdev->cpu_mask); + } + + /* remove CPUs that will remain paused */ + cpumask_andnot(&cpus_to_notify, &cpus_in_max_cooling_level, &new_paused_cpus); + + /* Notify for each CPU that we intended to resume */ + for_each_cpu(cpu, &cpus_to_notify) + blocking_notifier_call_chain(&thermal_pause_notifier, 0, + (void *)(long)cpu); + + /* update the cpus cooling mask */ + cpumask_copy(&cpus_in_max_cooling_level, &new_paused_cpus); + } else { + /* Failure. Ref-count for cpus controlled by thermal still set */ + pr_err("Error resuming CPU:%*pbl. err:%d\n", + cpumask_pr_args(&thermal_pause_cdev->cpu_mask), ret); + } + + return ret; +} + +/** + * thermal_pause_set_update_work: Enforce Requested State + * + * @work: the work structure for this work + * + * Enforce the most recent requested cooling state, if + * it is a mismatch with the current state. Since the + * request is made in a different context from the + * enforcement, it is possible to have an updated request + * after completing the resume/pause request. + * + * Handle a post-operation mismatch between the cooling state + * and the requested state. + * + * This is performed as a work function to avoid conflicts + * between a hotplug event invoking a pause cooling device, + * and a thermal event invoking a pause cooling device. + */ +static void thermal_pause_update_work(struct work_struct *work) +{ + int ret = 0; + int retcnt = THERMAL_PAUSE_RETRY_COUNT; + struct thermal_pause_cdev *thermal_pause_cdev = + container_of(work, struct thermal_pause_cdev, pause_update_work); + + mutex_lock(&cpus_pause_lock); + +retry: + if (thermal_pause_cdev->thermal_pause_req != thermal_pause_cdev->thermal_pause_level) { + if (thermal_pause_cdev->thermal_pause_req == THERMAL_NO_CPU_PAUSE) { + ret = thermal_resume_work(thermal_pause_cdev); + if (ret >= 0) + thermal_pause_cdev->thermal_pause_level = THERMAL_NO_CPU_PAUSE; + } else { + ret = thermal_pause_work(thermal_pause_cdev); + if (ret >= 0) + thermal_pause_cdev->thermal_pause_level = THERMAL_GROUP_CPU_PAUSE; + } + if (ret < 0 && retcnt > 0) { + retcnt--; + goto retry; + } + } + + /* + * if the pause/resume operation itself failed (and failed THERMAL_PAUSE_RETRY_COUNT + * times) then ret will be negative here. Do not repeatedly retry if pause itself + * failed, because this can happen indefinitely. + * + * If instead the pause request has been toggled back and forth many times by + * the thermal framework, this can be handled here. + */ + if (ret >= 0 && + thermal_pause_cdev->thermal_pause_req != thermal_pause_cdev->thermal_pause_level) { + pr_debug("Pause: requested state changed while workfn running\n"); + retcnt = THERMAL_PAUSE_RETRY_COUNT; + goto retry; + } + + mutex_unlock(&cpus_pause_lock); +} + +/** + * thermal_pause_set_cur_state - callback function to set the current cooling + * level. + * @cdev: thermal cooling device pointer. + * @level: set this variable to the current cooling level. + * + * Callback for the thermal cooling device to change the cpu pause + * current cooling level, by making a requested and queueing the + * work to be done. + * + * Return: 0 on success, an error code otherwise. + */ +static int thermal_pause_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long level) +{ + struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata; + + if (level >= MAX_THERMAL_PAUSE_LEVEL) + return -EINVAL; + + mutex_lock(&cpus_pause_lock); + + if (level) + thermal_pause_cdev->thermal_pause_req = THERMAL_GROUP_CPU_PAUSE; + else + thermal_pause_cdev->thermal_pause_req = THERMAL_NO_CPU_PAUSE; + + queue_work(system_highpri_wq, &thermal_pause_cdev->pause_update_work); + + mutex_unlock(&cpus_pause_lock); + + return 0; +} + +/** + * thermal_pause_get_cur_state - callback function to get the current cooling + * state. + * @cdev: thermal cooling device pointer. + * @state: fill this variable with the current cooling state. + * + * Callback for the thermal cooling device to return the cpu pause + * current cooling level + * + * Return: 0 on success, an error code otherwise. + */ +static int thermal_pause_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *level) +{ + struct thermal_pause_cdev *thermal_pause_cdev = cdev->devdata; + + *level = thermal_pause_cdev->thermal_pause_level; + + return 0; +} + +/** + * thermal_pause_get_max_state - callback function to get the max cooling state. + * @cdev: thermal cooling device pointer. + * @level: fill this variable with the max cooling level + * + * Callback for the thermal cooling device to return the cpu + * pause max cooling state. + * + * Return: 0 on success, an error code otherwise. + */ +static int thermal_pause_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *level) +{ + *level = MAX_THERMAL_PAUSE_LEVEL - 1; + return 0; +} + +static struct thermal_cooling_device_ops thermal_pause_cooling_ops = { + .get_max_state = thermal_pause_get_max_state, + .get_cur_state = thermal_pause_get_cur_state, + .set_cur_state = thermal_pause_set_cur_state, +}; + +static void thermal_pause_register_cdev(struct work_struct *work) +{ + struct thermal_pause_cdev *thermal_pause_cdev = + container_of(work, struct thermal_pause_cdev, reg_work); + int ret = 0; + cpumask_t cpus_online; + + cpumask_and(&cpus_online, + &thermal_pause_cdev->cpu_mask, + cpu_online_mask); + if (!cpumask_equal(&thermal_pause_cdev->cpu_mask, &cpus_online)) + return; + + thermal_pause_cdev->cdev = thermal_of_cooling_device_register( + thermal_pause_cdev->np, + thermal_pause_cdev->cdev_name, + thermal_pause_cdev, + &thermal_pause_cooling_ops); + + if (IS_ERR(thermal_pause_cdev->cdev)) { + ret = PTR_ERR(thermal_pause_cdev->cdev); + pr_err("Cooling register failed for %s, ret:%d\n", + thermal_pause_cdev->cdev_name, ret); + thermal_pause_cdev->cdev = NULL; + return; + } + pr_debug("Cooling device [%s] registered.\n", + thermal_pause_cdev->cdev_name); +} + +static int thermal_pause_probe(struct platform_device *pdev) +{ + int ret = 0, cpu = 0; + struct device_node *subsys_np = NULL, *cpu_phandle = NULL; + struct device *cpu_dev; + struct thermal_pause_cdev *thermal_pause_cdev = NULL; + struct device_node *np = pdev->dev.of_node; + struct of_phandle_iterator it; + cpumask_t cpu_mask; + unsigned long mask = 0; + const char *alias; + + INIT_LIST_HEAD(&thermal_pause_cdev_list); + cpumask_clear(&cpus_in_max_cooling_level); + + for_each_available_child_of_node(np, subsys_np) { + + cpumask_clear(&cpu_mask); + mask = 0; + of_phandle_iterator_init(&it, subsys_np, "qcom,cpus", NULL, 0); + while (of_phandle_iterator_next(&it) == 0) { + cpu_phandle = it.node; + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (cpu_dev && cpu_dev->of_node + == cpu_phandle) { + cpumask_set_cpu(cpu, &cpu_mask); + mask = mask | BIT(cpu); + break; + } + } + } + + if (cpumask_empty(&cpu_mask)) + continue; + + thermal_pause_cdev = devm_kzalloc(&pdev->dev, + sizeof(*thermal_pause_cdev), GFP_KERNEL); + + if (!thermal_pause_cdev) { + of_node_put(subsys_np); + return -ENOMEM; + } + ret = of_property_read_string(subsys_np, + "qcom,cdev-alias", &alias); + if (ret) + snprintf(thermal_pause_cdev->cdev_name, THERMAL_NAME_LENGTH, + "thermal-pause-%X", mask); + else + strscpy(thermal_pause_cdev->cdev_name, alias, + THERMAL_NAME_LENGTH); + + thermal_pause_cdev->thermal_pause_level = false; + thermal_pause_cdev->cdev = NULL; + thermal_pause_cdev->np = subsys_np; + cpumask_copy(&thermal_pause_cdev->cpu_mask, &cpu_mask); + + INIT_WORK(&thermal_pause_cdev->reg_work, thermal_pause_register_cdev); + INIT_WORK(&thermal_pause_cdev->pause_update_work, thermal_pause_update_work); + list_add(&thermal_pause_cdev->node, &thermal_pause_cdev_list); + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal-pause/cdev:online", + thermal_pause_hp_online, NULL); + if (ret < 0) + return ret; + cpu_hp_online = ret; + + return 0; +} + +static int thermal_pause_remove(struct platform_device *pdev) +{ + struct thermal_pause_cdev *thermal_pause_cdev = NULL, *next = NULL; + int ret = 0; + + if (cpu_hp_online) { + cpuhp_remove_state_nocalls(cpu_hp_online); + cpu_hp_online = 0; + } + + mutex_lock(&cpus_pause_lock); + list_for_each_entry_safe(thermal_pause_cdev, next, + &thermal_pause_cdev_list, node) { + + /* for each asserted cooling device, resume the CPUs */ + if (thermal_pause_cdev->thermal_pause_level) { + thermal_pause_cdev->thermal_pause_req = THERMAL_NO_CPU_PAUSE; + queue_work(system_highpri_wq, &thermal_pause_cdev->pause_update_work); + } + + if (thermal_pause_cdev->cdev) + thermal_cooling_device_unregister( + thermal_pause_cdev->cdev); + list_del(&thermal_pause_cdev->node); + } + + mutex_unlock(&cpus_pause_lock); + + /* if the resume failed, thermal still controls the CPUs. + * ensure that the error is passed to the caller. + */ + return ret; +} + +static const struct of_device_id thermal_pause_match[] = { + { .compatible = "qcom,thermal-pause", }, + {}, +}; + +static struct platform_driver thermal_pause_driver = { + .probe = thermal_pause_probe, + .remove = thermal_pause_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = thermal_pause_match, + }, +}; + +module_platform_driver(thermal_pause_driver); +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/sched/walt.h b/include/linux/sched/walt.h new file mode 100644 index 000000000000..85eda2491683 --- /dev/null +++ b/include/linux/sched/walt.h @@ -0,0 +1,229 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _LINUX_SCHED_WALT_H +#define _LINUX_SCHED_WALT_H + +#include +#include +#include + +enum pause_reason { + PAUSE_CORE_CTL = 0x01, + PAUSE_THERMAL = 0x02, + PAUSE_HYP = 0x04, +}; + +#if IS_ENABLED(CONFIG_SCHED_WALT) + +#define MAX_CPUS_PER_CLUSTER 6 +#define MAX_CLUSTERS 3 + +struct core_ctl_notif_data { + unsigned int nr_big; + unsigned int coloc_load_pct; + unsigned int ta_util_pct[MAX_CLUSTERS]; + unsigned int cur_cap_pct[MAX_CLUSTERS]; +}; + +enum task_boost_type { + TASK_BOOST_NONE = 0, + TASK_BOOST_ON_MID, + TASK_BOOST_ON_MAX, + TASK_BOOST_STRICT_MAX, + TASK_BOOST_END, +}; + +#define WALT_NR_CPUS 8 +#define RAVG_HIST_SIZE 8 +/* wts->bucket_bitmask needs to be updated if NUM_BUSY_BUCKETS > 16 */ +#define NUM_BUSY_BUCKETS 16 +#define NUM_BUSY_BUCKETS_SHIFT 4 + +struct walt_related_thread_group { + int id; + raw_spinlock_t lock; + struct list_head tasks; + struct list_head list; + bool skip_min; + struct rcu_head rcu; + u64 last_update; + u64 downmigrate_ts; + u64 start_ktime_ts; +}; + +struct walt_task_struct { + /* + * 'mark_start' marks the beginning of an event (task waking up, task + * starting to execute, task being preempted) within a window + * + * 'sum' represents how runnable a task has been within current + * window. It incorporates both running time and wait time and is + * frequency scaled. + * + * 'sum_history' keeps track of history of 'sum' seen over previous + * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are + * ignored. + * + * 'demand' represents maximum sum seen over previous + * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency + * demand for tasks. + * + * 'curr_window_cpu' represents task's contribution to cpu busy time on + * various CPUs in the current window + * + * 'prev_window_cpu' represents task's contribution to cpu busy time on + * various CPUs in the previous window + * + * 'curr_window' represents the sum of all entries in curr_window_cpu + * + * 'prev_window' represents the sum of all entries in prev_window_cpu + * + * 'pred_demand_scaled' represents task's current predicted cpu busy time + * in terms of 1024 units + * + * 'busy_buckets' groups historical busy time into different buckets + * used for prediction + * + * 'demand_scaled' represents task's demand scaled to 1024 + * + * 'prev_on_rq' tracks enqueue/dequeue of a task for error conditions + * 0 = nothing, 1 = enqueued, 2 = dequeued + */ + u64 mark_start; + u64 window_start; + u32 sum, demand; + u32 coloc_demand; + u32 sum_history[RAVG_HIST_SIZE]; + u16 sum_history_util[RAVG_HIST_SIZE]; + u32 curr_window_cpu[WALT_NR_CPUS]; + u32 prev_window_cpu[WALT_NR_CPUS]; + u32 curr_window, prev_window; + u8 busy_buckets[NUM_BUSY_BUCKETS]; + u16 bucket_bitmask; + u16 demand_scaled; + u16 pred_demand_scaled; + u64 active_time; + u64 last_win_size; + int boost; + bool wake_up_idle; + bool misfit; + bool rtg_high_prio; + u8 low_latency; + u64 boost_period; + u64 boost_expires; + u64 last_sleep_ts; + u32 init_load_pct; + u32 unfilter; + u64 last_wake_ts; + u64 last_enqueued_ts; + struct walt_related_thread_group __rcu *grp; + struct list_head grp_list; + u64 cpu_cycles; + cpumask_t cpus_requested; + bool iowaited; + int prev_on_rq; + int prev_on_rq_cpu; + struct list_head mvp_list; + u64 sum_exec_snapshot_for_slice; + u64 sum_exec_snapshot_for_total; + u64 total_exec; + int mvp_prio; + int cidx; + int load_boost; + int64_t boosted_task_load; + int prev_cpu; + int new_cpu; + u8 enqueue_after_migration; +}; + +#define wts_to_ts(wts) ({ \ + void *__mptr = (void *)(wts); \ + ((struct task_struct *)(__mptr - \ + offsetof(struct task_struct, android_vendor_data1))); }) + +static inline bool sched_get_wake_up_idle(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->wake_up_idle; +} + +static inline int sched_set_wake_up_idle(struct task_struct *p, bool wake_up_idle) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->wake_up_idle = wake_up_idle; + return 0; +} + +static inline void set_wake_up_idle(bool wake_up_idle) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + + wts->wake_up_idle = wake_up_idle; +} + +extern int sched_lpm_disallowed_time(int cpu, u64 *timeout); +extern int set_task_boost(int boost, u64 period); + +struct notifier_block; +extern void core_ctl_notifier_register(struct notifier_block *n); +extern void core_ctl_notifier_unregister(struct notifier_block *n); +extern int core_ctl_set_boost(bool boost); + +extern int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason); +extern int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason); +extern int walt_halt_cpus(struct cpumask *cpus, enum pause_reason reason); +extern int walt_start_cpus(struct cpumask *cpus, enum pause_reason reason); +#else +static inline int sched_lpm_disallowed_time(int cpu, u64 *timeout) +{ + return INT_MAX; +} +static inline int set_task_boost(int boost, u64 period) +{ + return 0; +} + +static inline bool sched_get_wake_up_idle(struct task_struct *p) +{ + return false; +} + +static inline int sched_set_wake_up_idle(struct task_struct *p, bool wake_up_idle) +{ + return 0; +} + +static inline void set_wake_up_idle(bool wake_up_idle) +{ +} + +static inline int core_ctl_set_boost(bool boost) +{ + return 0; +} + +static inline void core_ctl_notifier_register(struct notifier_block *n) +{ +} + +static inline void core_ctl_notifier_unregister(struct notifier_block *n) +{ +} + +inline int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason) +{ + return 0; +} +inline int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason) +{ + return 0; +} +#endif + +#endif /* _LINUX_SCHED_WALT_H */ diff --git a/include/linux/thermal_pause.h b/include/linux/thermal_pause.h new file mode 100644 index 000000000000..0769cdd8d460 --- /dev/null +++ b/include/linux/thermal_pause.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _LINUX_THERMAL_PAUSE_H +#define _LINUX_THERMAL_PAUSE_H + +#include +#include + +#if IS_ENABLED(CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE) +extern void thermal_pause_notifier_register(struct notifier_block *n); +extern void thermal_pause_notifier_unregister(struct notifier_block *n); +extern const struct cpumask *thermal_paused_cpumask(void); +#else +static inline +void thermal_pause_notifier_register(struct notifier_block *n) +{ +} + +static inline +void thermal_pause_notifier_unregister(struct notifier_block *n) +{ +} + +static inline const struct cpumask *thermal_paused_cpumask(void) +{ + return cpu_none_mask; +} +#endif /* CONFIG_QTI_CPU_PAUSE_COOLING_DEVICE */ + +#endif /* _LINUX_THERMAL_PAUSE_H */ diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 976092b7bd45..4b73828ecee6 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -32,3 +32,5 @@ obj-y += core.o obj-y += fair.o obj-y += build_policy.o obj-y += build_utility.o + +obj-$(CONFIG_SCHED_WALT) += walt/ diff --git a/kernel/sched/walt/Kconfig b/kernel/sched/walt/Kconfig new file mode 100644 index 000000000000..4157dffa013c --- /dev/null +++ b/kernel/sched/walt/Kconfig @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# QTI WALT based scheduler +# +menu "QTI WALT based scheduler features" + +config SCHED_WALT + tristate "Support window based load tracking" + depends on SMP + help + This feature will allow the scheduler to maintain a tunable window + based set of metrics for tasks and runqueues. These metrics can be + used to guide task placement as well as task frequency requirements + for cpufreq governors. + +config SCHED_WALT_DEBUG + tristate "WALT debug module" + depends on SCHED_WALT + select TRACE_PREEMPT_TOGGLE + select TRACE_IRQFLAGS + help + This module provides the means of debugging long preempt and + irq disable code. This helps in identifying the scheduling + latencies. The module rely on preemptirq trace hooks and + print the stacktrace to the ftrace upon long preempt and irq + events. Sysctl knobs are available for the user to configure + the thresholds. + + This module also used to crash the system to catch issues + in scenarios like RT throttling and sleeping while in atomic + context etc. + +config SCHED_CONSERVATIVE_BOOST_LPM_BIAS + bool "Enable LPM bias if conservative boost is enabled" + default n + help + This feature will allow the scheduler to disable low power + modes on a cpu if conservative boost is active. The cpu + will not enter low power mode for a hysteresis time period, + which can be configured from userspace. +endmenu diff --git a/kernel/sched/walt/Makefile b/kernel/sched/walt/Makefile new file mode 100644 index 000000000000..63f3f1b50e95 --- /dev/null +++ b/kernel/sched/walt/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only + +KCOV_INSTRUMENT := n +KCSAN_SANITIZE := n + +obj-$(CONFIG_SCHED_WALT) += sched-walt.o +sched-walt-$(CONFIG_SCHED_WALT) := walt.o boost.o sched_avg.o walt_halt.o core_ctl.o trace.o input-boost.o sysctl.o cpufreq_walt.o fixup.o walt_lb.o walt_rt.o walt_cfs.o walt_tp.o + +obj-$(CONFIG_SCHED_WALT_DEBUG) += sched-walt-debug.o +sched-walt-debug-$(CONFIG_SCHED_WALT_DEBUG) := walt_debug.o preemptirq_long.o diff --git a/kernel/sched/walt/boost.c b/kernel/sched/walt/boost.c new file mode 100644 index 000000000000..ddfe83366013 --- /dev/null +++ b/kernel/sched/walt/boost.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. + */ + +#include + +#include "walt.h" +#include "trace.h" + +/* + * Scheduler boost is a mechanism to temporarily place tasks on CPUs + * with higher capacity than those where a task would have normally + * ended up with their load characteristics. Any entity enabling + * boost is responsible for disabling it as well. + */ +unsigned int sched_boost_type; +enum sched_boost_policy boost_policy; + +static DEFINE_MUTEX(boost_mutex); + +void walt_init_tg(struct task_group *tg) +{ + struct walt_task_group *wtg; + + wtg = (struct walt_task_group *) tg->android_vendor_data1; + + wtg->colocate = false; + wtg->sched_boost_enable[NO_BOOST] = false; + wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true; + wtg->sched_boost_enable[CONSERVATIVE_BOOST] = false; + wtg->sched_boost_enable[RESTRAINED_BOOST] = false; +} + +void walt_init_topapp_tg(struct task_group *tg) +{ + struct walt_task_group *wtg; + + wtg = (struct walt_task_group *) tg->android_vendor_data1; + + wtg->colocate = true; + wtg->sched_boost_enable[NO_BOOST] = false; + wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true; + wtg->sched_boost_enable[CONSERVATIVE_BOOST] = true; + wtg->sched_boost_enable[RESTRAINED_BOOST] = false; +} + +void walt_init_foreground_tg(struct task_group *tg) +{ + struct walt_task_group *wtg; + + wtg = (struct walt_task_group *) tg->android_vendor_data1; + + wtg->colocate = false; + wtg->sched_boost_enable[NO_BOOST] = false; + wtg->sched_boost_enable[FULL_THROTTLE_BOOST] = true; + wtg->sched_boost_enable[CONSERVATIVE_BOOST] = true; + wtg->sched_boost_enable[RESTRAINED_BOOST] = false; +} + +/* + * Scheduler boost type and boost policy might at first seem unrelated, + * however, there exists a connection between them that will allow us + * to use them interchangeably during placement decisions. We'll explain + * the connection here in one possible way so that the implications are + * clear when looking at placement policies. + * + * When policy = SCHED_BOOST_NONE, type is either none or RESTRAINED + * When policy = SCHED_BOOST_ON_ALL or SCHED_BOOST_ON_BIG, type can + * neither be none nor RESTRAINED. + */ +static void set_boost_policy(int type) +{ + if (type == NO_BOOST || type == RESTRAINED_BOOST) { + boost_policy = SCHED_BOOST_NONE; + return; + } + + if (hmp_capable()) { + boost_policy = SCHED_BOOST_ON_BIG; + return; + } + + boost_policy = SCHED_BOOST_ON_ALL; +} + +static bool verify_boost_params(int type) +{ + return type >= RESTRAINED_BOOST_DISABLE && type <= RESTRAINED_BOOST; +} + +static void sched_no_boost_nop(void) +{ +} + +static void sched_full_throttle_boost_enter(void) +{ + core_ctl_set_boost(true); + walt_enable_frequency_aggregation(true); +} + +static void sched_full_throttle_boost_exit(void) +{ + core_ctl_set_boost(false); + walt_enable_frequency_aggregation(false); +} + +static void sched_conservative_boost_enter(void) +{ +} + +static void sched_conservative_boost_exit(void) +{ +} + +static void sched_restrained_boost_enter(void) +{ + walt_enable_frequency_aggregation(true); +} + +static void sched_restrained_boost_exit(void) +{ + walt_enable_frequency_aggregation(false); +} + +struct sched_boost_data { + int refcount; + void (*enter)(void); + void (*exit)(void); +}; + +static struct sched_boost_data sched_boosts[] = { + [NO_BOOST] = { + .refcount = 0, + .enter = sched_no_boost_nop, + .exit = sched_no_boost_nop, + }, + [FULL_THROTTLE_BOOST] = { + .refcount = 0, + .enter = sched_full_throttle_boost_enter, + .exit = sched_full_throttle_boost_exit, + }, + [CONSERVATIVE_BOOST] = { + .refcount = 0, + .enter = sched_conservative_boost_enter, + .exit = sched_conservative_boost_exit, + }, + [RESTRAINED_BOOST] = { + .refcount = 0, + .enter = sched_restrained_boost_enter, + .exit = sched_restrained_boost_exit, + }, +}; + +#define SCHED_BOOST_START FULL_THROTTLE_BOOST +#define SCHED_BOOST_END (RESTRAINED_BOOST + 1) + +static int sched_effective_boost(void) +{ + int i; + + /* + * The boosts are sorted in descending order by + * priority. + */ + for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) { + if (sched_boosts[i].refcount >= 1) + return i; + } + + return NO_BOOST; +} + +static void sched_boost_disable(int type) +{ + struct sched_boost_data *sb = &sched_boosts[type]; + int next_boost, prev_boost = sched_boost_type; + + if (sb->refcount <= 0) + return; + + sb->refcount--; + + if (sb->refcount) + return; + + next_boost = sched_effective_boost(); + if (next_boost == prev_boost) + return; + /* + * This boost's refcount becomes zero, so it must + * be disabled. Disable it first and then apply + * the next boost. + */ + sched_boosts[prev_boost].exit(); + sched_boosts[next_boost].enter(); +} + +static void sched_boost_enable(int type) +{ + struct sched_boost_data *sb = &sched_boosts[type]; + int next_boost, prev_boost = sched_boost_type; + + sb->refcount++; + + if (sb->refcount != 1) + return; + + /* + * This boost enable request did not come before. + * Take this new request and find the next boost + * by aggregating all the enabled boosts. If there + * is a change, disable the previous boost and enable + * the next boost. + */ + + next_boost = sched_effective_boost(); + if (next_boost == prev_boost) + return; + + sched_boosts[prev_boost].exit(); + sched_boosts[next_boost].enter(); +} + +static void sched_boost_disable_all(void) +{ + int i; + int prev_boost = sched_boost_type; + + if (prev_boost != NO_BOOST) { + sched_boosts[prev_boost].exit(); + for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) + sched_boosts[i].refcount = 0; + } +} + +static void _sched_set_boost(int type) +{ + if (type == 0) + sched_boost_disable_all(); + else if (type > 0) + sched_boost_enable(type); + else + sched_boost_disable(-type); + + /* + * sysctl_sched_boost holds the boost request from + * user space which could be different from the + * effectively enabled boost. Update the effective + * boost here. + */ + + sched_boost_type = sched_effective_boost(); + sysctl_sched_boost = sched_boost_type; + set_boost_policy(sysctl_sched_boost); + trace_sched_set_boost(sysctl_sched_boost); +} + +int sched_set_boost(int type) +{ + int ret = 0; + + mutex_lock(&boost_mutex); + if (verify_boost_params(type)) + _sched_set_boost(type); + else + ret = -EINVAL; + mutex_unlock(&boost_mutex); + return ret; +} + +int sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + unsigned int *data = (unsigned int *)table->data; + + mutex_lock(&boost_mutex); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (ret || !write) + goto done; + + if (verify_boost_params(*data)) + _sched_set_boost(*data); + else + ret = -EINVAL; + +done: + mutex_unlock(&boost_mutex); + return ret; +} + +void walt_boost_init(void) +{ + /* force call the callbacks for default boost */ + sched_set_boost(FULL_THROTTLE_BOOST); +} diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c new file mode 100644 index 000000000000..cdb123d90b8f --- /dev/null +++ b/kernel/sched/walt/core_ctl.c @@ -0,0 +1,1360 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "core_ctl: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "walt.h" +#include "trace.h" + +/* mask of CPUs on which there is an outstanding pause claim */ +static cpumask_t cpus_paused_by_us = { CPU_BITS_NONE }; + +struct cluster_data { + bool inited; + unsigned int min_cpus; + unsigned int max_cpus; + unsigned int offline_delay_ms; + unsigned int busy_up_thres[MAX_CPUS_PER_CLUSTER]; + unsigned int busy_down_thres[MAX_CPUS_PER_CLUSTER]; + unsigned int active_cpus; + unsigned int num_cpus; + unsigned int nr_not_preferred_cpus; + cpumask_t cpu_mask; + unsigned int need_cpus; + unsigned int task_thres; + unsigned int max_nr; + unsigned int nr_prev_assist; + unsigned int nr_prev_assist_thresh; + s64 need_ts; + struct list_head lru; + bool enable; + int nrrun; + unsigned int first_cpu; + unsigned int boost; + struct kobject kobj; + unsigned int strict_nrrun; +}; + +struct cpu_data { + bool is_busy; + unsigned int busy_pct; + unsigned int cpu; + bool not_preferred; + struct cluster_data *cluster; + struct list_head sib; + bool disabled; +}; + +static DEFINE_PER_CPU(struct cpu_data, cpu_state); +static struct cluster_data cluster_state[MAX_CLUSTERS]; +static unsigned int num_clusters; + +#define for_each_cluster(cluster, idx) \ + for (; (idx) < num_clusters && ((cluster) = &cluster_state[idx]);\ + idx++) + +/* single core_ctl thread for all pause/unpause core_ctl operations */ +struct task_struct *core_ctl_thread; + +/* single lock per single thread for core_ctl + * protects core_ctl_pending flag + */ +spinlock_t core_ctl_pending_lock; +bool core_ctl_pending; + +static DEFINE_SPINLOCK(state_lock); +static void apply_need(struct cluster_data *state); +static void wake_up_core_ctl_thread(void); +static bool initialized; + +ATOMIC_NOTIFIER_HEAD(core_ctl_notifier); +static unsigned int last_nr_big; + +static unsigned int get_active_cpu_count(const struct cluster_data *cluster); +static void __ref do_core_ctl(void); + +/* ========================= sysfs interface =========================== */ + +static ssize_t store_min_cpus(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + state->min_cpus = min(val, state->num_cpus); + apply_need(state); + + return count; +} + +static ssize_t show_min_cpus(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->min_cpus); +} + +static ssize_t store_max_cpus(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + state->max_cpus = min(val, state->num_cpus); + apply_need(state); + + return count; +} + +static ssize_t show_max_cpus(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->max_cpus); +} + +static ssize_t store_offline_delay_ms(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + state->offline_delay_ms = val; + apply_need(state); + + return count; +} + +static ssize_t show_task_thres(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->task_thres); +} + +static ssize_t store_task_thres(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + if (val < state->num_cpus) + return -EINVAL; + + state->task_thres = val; + apply_need(state); + + return count; +} + +static ssize_t show_nr_prev_assist_thresh(const struct cluster_data *state, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->nr_prev_assist_thresh); +} + +static ssize_t store_nr_prev_assist_thresh(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + state->nr_prev_assist_thresh = val; + apply_need(state); + + return count; +} + +static ssize_t show_offline_delay_ms(const struct cluster_data *state, + char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->offline_delay_ms); +} + +static ssize_t store_busy_up_thres(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val[MAX_CPUS_PER_CLUSTER]; + int ret, i; + + ret = sscanf(buf, "%u %u %u %u %u %u\n", + &val[0], &val[1], &val[2], &val[3], + &val[4], &val[5]); + if (ret != 1 && ret != state->num_cpus) + return -EINVAL; + + if (ret == 1) { + for (i = 0; i < state->num_cpus; i++) + state->busy_up_thres[i] = val[0]; + } else { + for (i = 0; i < state->num_cpus; i++) + state->busy_up_thres[i] = val[i]; + } + apply_need(state); + return count; +} + +static ssize_t show_busy_up_thres(const struct cluster_data *state, char *buf) +{ + int i, count = 0; + + for (i = 0; i < state->num_cpus; i++) + count += scnprintf(buf + count, PAGE_SIZE - count, "%u ", + state->busy_up_thres[i]); + + count += scnprintf(buf + count, PAGE_SIZE - count, "\n"); + return count; +} + +static ssize_t store_busy_down_thres(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val[MAX_CPUS_PER_CLUSTER]; + int ret, i; + + ret = sscanf(buf, "%u %u %u %u %u %u\n", + &val[0], &val[1], &val[2], &val[3], + &val[4], &val[5]); + if (ret != 1 && ret != state->num_cpus) + return -EINVAL; + + if (ret == 1) { + for (i = 0; i < state->num_cpus; i++) + state->busy_down_thres[i] = val[0]; + } else { + for (i = 0; i < state->num_cpus; i++) + state->busy_down_thres[i] = val[i]; + } + apply_need(state); + return count; +} + +static ssize_t show_busy_down_thres(const struct cluster_data *state, char *buf) +{ + int i, count = 0; + + for (i = 0; i < state->num_cpus; i++) + count += scnprintf(buf + count, PAGE_SIZE - count, "%u ", + state->busy_down_thres[i]); + + count += scnprintf(buf + count, PAGE_SIZE - count, "\n"); + return count; +} + +static ssize_t store_enable(struct cluster_data *state, + const char *buf, size_t count) +{ + unsigned int val; + bool bval; + + if (sscanf(buf, "%u\n", &val) != 1) + return -EINVAL; + + bval = !!val; + if (bval != state->enable) { + state->enable = bval; + apply_need(state); + } + + return count; +} + +static ssize_t show_enable(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->enable); +} + +static ssize_t show_need_cpus(const struct cluster_data *state, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%u\n", state->need_cpus); +} + +static ssize_t show_active_cpus(const struct cluster_data *state, char *buf) +{ + int active_cpus = get_active_cpu_count(state); + + return scnprintf(buf, PAGE_SIZE, "%u\n", active_cpus); +} + +static unsigned int cluster_paused_cpus(const struct cluster_data *cluster) +{ + cpumask_t cluster_paused_cpus; + + cpumask_and(&cluster_paused_cpus, &cluster->cpu_mask, &cpus_paused_by_us); + return cpumask_weight(&cluster_paused_cpus); +} + +static ssize_t show_global_state(const struct cluster_data *state, char *buf) +{ + struct cpu_data *c; + struct cluster_data *cluster; + ssize_t count = 0; + unsigned int cpu; + + spin_lock_irq(&state_lock); + for_each_possible_cpu(cpu) { + c = &per_cpu(cpu_state, cpu); + cluster = c->cluster; + if (!cluster || !cluster->inited) + continue; + + count += scnprintf(buf + count, PAGE_SIZE - count, + "CPU%u\n", cpu); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tCPU: %u\n", c->cpu); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tOnline: %u\n", + cpu_online(c->cpu)); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tPaused: %u\n", + cpu_halted(c->cpu)); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tFirst CPU: %u\n", + cluster->first_cpu); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tBusy%%: %u\n", c->busy_pct); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tIs busy: %u\n", c->is_busy); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tNot preferred: %u\n", + c->not_preferred); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tNr running: %u\n", cluster->nrrun); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tActive CPUs: %u\n", get_active_cpu_count(cluster)); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tNeed CPUs: %u\n", cluster->need_cpus); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tCluster paused CPUs: %u\n", + cluster_paused_cpus(cluster)); + count += scnprintf(buf + count, PAGE_SIZE - count, + "\tBoost: %u\n", (unsigned int) cluster->boost); + } + spin_unlock_irq(&state_lock); + + return count; +} + +static ssize_t store_not_preferred(struct cluster_data *state, + const char *buf, size_t count) +{ + struct cpu_data *c; + unsigned int i; + unsigned int val[MAX_CPUS_PER_CLUSTER]; + unsigned long flags; + int ret; + int not_preferred_count = 0; + + ret = sscanf(buf, "%u %u %u %u %u %u\n", + &val[0], &val[1], &val[2], &val[3], + &val[4], &val[5]); + if (ret != state->num_cpus) + return -EINVAL; + + spin_lock_irqsave(&state_lock, flags); + for (i = 0; i < state->num_cpus; i++) { + c = &per_cpu(cpu_state, i + state->first_cpu); + c->not_preferred = val[i]; + not_preferred_count += !!val[i]; + } + state->nr_not_preferred_cpus = not_preferred_count; + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +static ssize_t show_not_preferred(const struct cluster_data *state, char *buf) +{ + struct cpu_data *c; + ssize_t count = 0; + unsigned long flags; + int i; + + spin_lock_irqsave(&state_lock, flags); + for (i = 0; i < state->num_cpus; i++) { + c = &per_cpu(cpu_state, i + state->first_cpu); + count += scnprintf(buf + count, PAGE_SIZE - count, + "CPU#%d: %u\n", c->cpu, c->not_preferred); + } + spin_unlock_irqrestore(&state_lock, flags); + + return count; +} + +struct core_ctl_attr { + struct attribute attr; + ssize_t (*show)(const struct cluster_data *cd, char *c); + ssize_t (*store)(struct cluster_data *cd, const char *c, + size_t count); +}; + +#define core_ctl_attr_ro(_name) \ +static struct core_ctl_attr _name = \ +__ATTR(_name, 0444, show_##_name, NULL) + +#define core_ctl_attr_rw(_name) \ +static struct core_ctl_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) + +core_ctl_attr_rw(min_cpus); +core_ctl_attr_rw(max_cpus); +core_ctl_attr_rw(offline_delay_ms); +core_ctl_attr_rw(busy_up_thres); +core_ctl_attr_rw(busy_down_thres); +core_ctl_attr_rw(task_thres); +core_ctl_attr_rw(nr_prev_assist_thresh); +core_ctl_attr_ro(need_cpus); +core_ctl_attr_ro(active_cpus); +core_ctl_attr_ro(global_state); +core_ctl_attr_rw(not_preferred); +core_ctl_attr_rw(enable); + +static struct attribute *default_attrs[] = { + &min_cpus.attr, + &max_cpus.attr, + &offline_delay_ms.attr, + &busy_up_thres.attr, + &busy_down_thres.attr, + &task_thres.attr, + &nr_prev_assist_thresh.attr, + &enable.attr, + &need_cpus.attr, + &active_cpus.attr, + &global_state.attr, + ¬_preferred.attr, + NULL +}; + +#define to_cluster_data(k) container_of(k, struct cluster_data, kobj) +#define to_attr(a) container_of(a, struct core_ctl_attr, attr) +static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct cluster_data *data = to_cluster_data(kobj); + struct core_ctl_attr *cattr = to_attr(attr); + ssize_t ret = -EIO; + + if (cattr->show) + ret = cattr->show(data, buf); + + return ret; +} + +static ssize_t store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct cluster_data *data = to_cluster_data(kobj); + struct core_ctl_attr *cattr = to_attr(attr); + ssize_t ret = -EIO; + + if (cattr->store) + ret = cattr->store(data, buf, count); + + return ret; +} + +static const struct sysfs_ops sysfs_ops = { + .show = show, + .store = store, +}; + +static struct kobj_type ktype_core_ctl = { + .sysfs_ops = &sysfs_ops, + .default_attrs = default_attrs, +}; + +/* ==================== runqueue based core count =================== */ + +static struct sched_avg_stats *nr_stats; + +/* + * nr_need: + * Number of tasks running on this cluster plus + * tasks running on higher capacity clusters. + * To find out CPUs needed from this cluster. + * + * For example: + * On dual cluster system with 4 min capacity + * CPUs and 4 max capacity CPUs, if there are + * 4 small tasks running on min capacity CPUs + * and 2 big tasks running on 2 max capacity + * CPUs, nr_need has to be 6 for min capacity + * cluster and 2 for max capacity cluster. + * This is because, min capacity cluster has to + * account for tasks running on max capacity + * cluster, so that, the min capacity cluster + * can be ready to accommodate tasks running on max + * capacity CPUs if the demand of tasks goes down. + */ +static int compute_cluster_nr_need(int index) +{ + int cpu; + struct cluster_data *cluster; + int nr_need = 0; + + for_each_cluster(cluster, index) { + for_each_cpu(cpu, &cluster->cpu_mask) + nr_need += nr_stats[cpu].nr; + } + + return nr_need; +} + +/* + * prev_misfit_need: + * Tasks running on smaller capacity cluster which + * needs to be migrated to higher capacity cluster. + * To find out how many tasks need higher capacity CPUs. + * + * For example: + * On dual cluster system with 4 min capacity + * CPUs and 4 max capacity CPUs, if there are + * 2 small tasks and 2 big tasks running on + * min capacity CPUs and no tasks running on + * max cpacity, prev_misfit_need of min capacity + * cluster will be 0 and prev_misfit_need of + * max capacity cluster will be 2. + */ +static int compute_prev_cluster_misfit_need(int index) +{ + int cpu; + struct cluster_data *prev_cluster; + int prev_misfit_need = 0; + + /* + * Lowest capacity cluster does not have to + * accommodate any misfit tasks. + */ + if (index == 0) + return 0; + + prev_cluster = &cluster_state[index - 1]; + + for_each_cpu(cpu, &prev_cluster->cpu_mask) + prev_misfit_need += nr_stats[cpu].nr_misfit; + + return prev_misfit_need; +} + +static int compute_cluster_max_nr(int index) +{ + int cpu; + struct cluster_data *cluster = &cluster_state[index]; + int max_nr = 0; + + for_each_cpu(cpu, &cluster->cpu_mask) + max_nr = max(max_nr, nr_stats[cpu].nr_max); + + return max_nr; +} + +static int cluster_real_big_tasks(int index) +{ + int nr_big = 0; + int cpu; + struct cluster_data *cluster = &cluster_state[index]; + + if (index == 0) { + for_each_cpu(cpu, &cluster->cpu_mask) + nr_big += nr_stats[cpu].nr_misfit; + } else { + for_each_cpu(cpu, &cluster->cpu_mask) + nr_big += nr_stats[cpu].nr; + } + + return nr_big; +} + +/* + * prev_nr_need_assist: + * Tasks that are eligible to run on the previous + * cluster but cannot run because of insufficient + * CPUs there. prev_nr_need_assist is indicative + * of number of CPUs in this cluster that should + * assist its previous cluster to makeup for + * insufficient CPUs there. + * + * For example: + * On tri-cluster system with 4 min capacity + * CPUs, 3 intermediate capacity CPUs and 1 + * max capacity CPU, if there are 4 small + * tasks running on min capacity CPUs, 4 big + * tasks running on intermediate capacity CPUs + * and no tasks running on max capacity CPU, + * prev_nr_need_assist for min & max capacity + * clusters will be 0, but, for intermediate + * capacity cluster prev_nr_need_assist will + * be 1 as it has 3 CPUs, but, there are 4 big + * tasks to be served. + */ +static int prev_cluster_nr_need_assist(int index) +{ + int need = 0; + int cpu; + struct cluster_data *prev_cluster; + + if (index == 0) + return 0; + + index--; + prev_cluster = &cluster_state[index]; + + /* + * Next cluster should not assist, while there are paused cpus + * in this cluster. + */ + if (cluster_paused_cpus(prev_cluster)) + return 0; + + for_each_cpu(cpu, &prev_cluster->cpu_mask) + need += nr_stats[cpu].nr; + + need += compute_prev_cluster_misfit_need(index); + + if (need > prev_cluster->active_cpus) + need = need - prev_cluster->active_cpus; + else + need = 0; + + return need; +} + +/* + * This is only implemented for min capacity cluster. + * + * Bringing a little CPU out of pause and using it + * more does not hurt power as much as bringing big CPUs. + * + * little cluster provides help needed for the other clusters. + * we take nr_scaled (which gives better resolution) and find + * the total nr in the system. Then take out the active higher + * capacity CPUs from the nr and consider the remaining nr as + * strict and consider that many little CPUs are needed. + */ +static int compute_cluster_nr_strict_need(int index) +{ + int cpu; + struct cluster_data *cluster; + int nr_strict_need = 0; + + if (index != 0) + return 0; + + for_each_cluster(cluster, index) { + int nr_scaled = 0; + int active_cpus = cluster->active_cpus; + + for_each_cpu(cpu, &cluster->cpu_mask) + nr_scaled += nr_stats[cpu].nr_scaled; + + nr_scaled /= 100; + + /* + * For little cluster, nr_scaled becomes the nr_strict, + * for other cluster, overflow is counted towards + * the little cluster need. + */ + if (index == 0) + nr_strict_need += nr_scaled; + else + nr_strict_need += max(0, nr_scaled - active_cpus); + } + + return nr_strict_need; +} +static void update_running_avg(void) +{ + struct cluster_data *cluster; + unsigned int index = 0; + unsigned long flags; + int big_avg = 0; + + nr_stats = sched_get_nr_running_avg(); + + spin_lock_irqsave(&state_lock, flags); + for_each_cluster(cluster, index) { + int nr_need, prev_misfit_need; + + if (!cluster->inited) + continue; + + nr_need = compute_cluster_nr_need(index); + prev_misfit_need = compute_prev_cluster_misfit_need(index); + + cluster->nrrun = nr_need + prev_misfit_need; + cluster->max_nr = compute_cluster_max_nr(index); + cluster->nr_prev_assist = prev_cluster_nr_need_assist(index); + + cluster->strict_nrrun = compute_cluster_nr_strict_need(index); + + trace_core_ctl_update_nr_need(cluster->first_cpu, nr_need, + prev_misfit_need, + cluster->nrrun, cluster->max_nr, + cluster->nr_prev_assist); + + big_avg += cluster_real_big_tasks(index); + } + spin_unlock_irqrestore(&state_lock, flags); + + last_nr_big = big_avg; + walt_rotation_checkpoint(big_avg); +} + +#define MAX_NR_THRESHOLD 4 +/* adjust needed CPUs based on current runqueue information */ +static unsigned int apply_task_need(const struct cluster_data *cluster, + unsigned int new_need) +{ + /* resume all cores if there are enough tasks */ + if (cluster->nrrun >= cluster->task_thres) + return cluster->num_cpus; + + /* + * resume as many cores as the previous cluster + * needs assistance with. + */ + if (cluster->nr_prev_assist >= cluster->nr_prev_assist_thresh) + new_need = new_need + cluster->nr_prev_assist; + + /* only resume more cores if there are tasks to run */ + if (cluster->nrrun > new_need) + new_need = new_need + 1; + + /* + * We don't want tasks to be overcrowded in a cluster. + * If any CPU has more than MAX_NR_THRESHOLD in the last + * window, bring another CPU to help out. + */ + if (cluster->max_nr > MAX_NR_THRESHOLD) + new_need = new_need + 1; + + /* + * For little cluster, we use a bit more relaxed approach + * and impose the strict nr condition. Because all tasks can + * spill onto little if big cluster is crowded. + */ + if (new_need < cluster->strict_nrrun) + new_need = cluster->strict_nrrun; + + return new_need; +} + +/* ======================= load based core count ====================== */ + +static unsigned int apply_limits(const struct cluster_data *cluster, + unsigned int need_cpus) +{ + return min(max(cluster->min_cpus, need_cpus), cluster->max_cpus); +} + +static unsigned int get_active_cpu_count(const struct cluster_data *cluster) +{ + cpumask_t cpus; + + cpumask_andnot(&cpus, &cluster->cpu_mask, cpu_halt_mask); + return cpumask_weight(&cpus); +} + +static bool is_active(const struct cpu_data *state) +{ + return cpu_active(state->cpu) && !cpu_halted(state->cpu); +} + +static bool adjustment_possible(const struct cluster_data *cluster, + unsigned int need) +{ + return (need < cluster->active_cpus || (need > cluster->active_cpus && + cluster_paused_cpus(cluster))); +} + +static bool eval_need(struct cluster_data *cluster) +{ + unsigned long flags; + struct cpu_data *c; + unsigned int need_cpus = 0, last_need, thres_idx; + bool adj_now = false; + bool adj_possible = false; + unsigned int new_need; + s64 now, elapsed; + + if (unlikely(!cluster->inited)) + return false; + + spin_lock_irqsave(&state_lock, flags); + + if (cluster->boost || !cluster->enable) { + need_cpus = cluster->max_cpus; + } else { + cluster->active_cpus = get_active_cpu_count(cluster); + thres_idx = cluster->active_cpus ? cluster->active_cpus - 1 : 0; + list_for_each_entry(c, &cluster->lru, sib) { + bool old_is_busy = c->is_busy; + + if (c->busy_pct >= cluster->busy_up_thres[thres_idx] || + sched_cpu_high_irqload(c->cpu)) + c->is_busy = true; + else if (c->busy_pct < cluster->busy_down_thres[thres_idx]) + c->is_busy = false; + + trace_core_ctl_set_busy(c->cpu, c->busy_pct, old_is_busy, + c->is_busy); + need_cpus += c->is_busy; + } + need_cpus = apply_task_need(cluster, need_cpus); + } + new_need = apply_limits(cluster, need_cpus); + + last_need = cluster->need_cpus; + now = ktime_to_ms(ktime_get()); + + if (new_need > cluster->active_cpus) { + adj_now = true; + } else { + /* + * When there is no change in need and there are no more + * active CPUs than currently needed, just update the + * need time stamp and return. + */ + if (new_need == last_need && new_need == cluster->active_cpus) { + cluster->need_ts = now; + adj_now = false; + goto unlock; + } + + elapsed = now - cluster->need_ts; + adj_now = elapsed >= cluster->offline_delay_ms; + } + + if (adj_now) { + adj_possible = adjustment_possible(cluster, new_need); + cluster->need_ts = now; + cluster->need_cpus = new_need; + } + +unlock: + trace_core_ctl_eval_need(cluster->first_cpu, last_need, new_need, + cluster->active_cpus, adj_now, adj_possible, + adj_now && adj_possible, cluster->need_ts); + spin_unlock_irqrestore(&state_lock, flags); + + return adj_now && adj_possible; +} + +static void apply_need(struct cluster_data *cluster) +{ + if (eval_need(cluster)) + wake_up_core_ctl_thread(); +} + +/* ========================= core count enforcement ==================== */ + +static void wake_up_core_ctl_thread(void) +{ + unsigned long flags; + + spin_lock_irqsave(&core_ctl_pending_lock, flags); + core_ctl_pending = true; + spin_unlock_irqrestore(&core_ctl_pending_lock, flags); + + wake_up_process(core_ctl_thread); +} + +static u64 core_ctl_check_timestamp; + +int core_ctl_set_boost(bool boost) +{ + unsigned int index = 0; + struct cluster_data *cluster = NULL; + unsigned long flags; + int ret = 0; + bool boost_state_changed = false; + + if (unlikely(!initialized)) + return 0; + + spin_lock_irqsave(&state_lock, flags); + for_each_cluster(cluster, index) { + if (boost) { + boost_state_changed = !cluster->boost; + ++cluster->boost; + } else { + if (!cluster->boost) { + ret = -EINVAL; + break; + } + --cluster->boost; + boost_state_changed = !cluster->boost; + } + } + spin_unlock_irqrestore(&state_lock, flags); + + if (boost_state_changed) { + index = 0; + for_each_cluster(cluster, index) + apply_need(cluster); + } + + if (cluster) + trace_core_ctl_set_boost(cluster->boost, ret); + + return ret; +} +EXPORT_SYMBOL(core_ctl_set_boost); + +void core_ctl_notifier_register(struct notifier_block *n) +{ + atomic_notifier_chain_register(&core_ctl_notifier, n); +} +EXPORT_SYMBOL(core_ctl_notifier_register); + +void core_ctl_notifier_unregister(struct notifier_block *n) +{ + atomic_notifier_chain_unregister(&core_ctl_notifier, n); +} +EXPORT_SYMBOL(core_ctl_notifier_unregister); + +static void core_ctl_call_notifier(void) +{ + struct core_ctl_notif_data ndata = {0}; + struct notifier_block *nb; + + /* + * Don't bother querying the stats when the notifier + * chain is empty. + */ + rcu_read_lock(); + nb = rcu_dereference_raw(core_ctl_notifier.head); + rcu_read_unlock(); + + if (!nb) + return; + + ndata.nr_big = last_nr_big; + walt_fill_ta_data(&ndata); + trace_core_ctl_notif_data(ndata.nr_big, ndata.coloc_load_pct, + ndata.ta_util_pct, ndata.cur_cap_pct); + + atomic_notifier_call_chain(&core_ctl_notifier, 0, &ndata); +} + +/* + * sched_get_nr_running_avg will wipe out previous statistics and + * update it to the values computed since the last call. + * + * core_ctl_check assumes that the statistics are stable, hence + * window based. Therefore core_ctl_check must only be called from + * window rollover, or walt_irq_work for not migration. + */ +void core_ctl_check(u64 window_start) +{ + int cpu; + struct cpu_data *c; + struct cluster_data *cluster; + unsigned int index = 0; + unsigned long flags; + unsigned int wakeup = 0; + + if (unlikely(!initialized)) + return; + + if (window_start == core_ctl_check_timestamp) + return; + + core_ctl_check_timestamp = window_start; + + spin_lock_irqsave(&state_lock, flags); + for_each_possible_cpu(cpu) { + + c = &per_cpu(cpu_state, cpu); + cluster = c->cluster; + + if (!cluster || !cluster->inited) + continue; + + c->busy_pct = sched_get_cpu_util_pct(cpu); + } + spin_unlock_irqrestore(&state_lock, flags); + + update_running_avg(); + + for_each_cluster(cluster, index) + wakeup |= eval_need(cluster); + + if (wakeup) + do_core_ctl(); + core_ctl_call_notifier(); +} + +/* must be called with state_lock held */ +static void move_cpu_lru(struct cpu_data *cpu_data) +{ + list_del(&cpu_data->sib); + list_add_tail(&cpu_data->sib, &cpu_data->cluster->lru); +} + +static void try_to_pause(struct cluster_data *cluster, unsigned int need, + struct cpumask *pause_cpus) +{ + struct cpu_data *c, *tmp; + unsigned long flags; + unsigned int num_cpus = cluster->num_cpus; + unsigned int nr_pending = 0, active_cpus = cluster->active_cpus; + bool first_pass = cluster->nr_not_preferred_cpus; + + /* + * Protect against entry being removed (and added at tail) by other + * thread (hotplug). + */ + spin_lock_irqsave(&state_lock, flags); + list_for_each_entry_safe(c, tmp, &cluster->lru, sib) { + if (!num_cpus--) + break; + + if (c->disabled) + continue; + if (!is_active(c)) + continue; + if (active_cpus - nr_pending == need) + break; + /* Don't pause busy CPUs. */ + if (c->is_busy) + continue; + /* + * We pause only the not_preferred CPUs. If none + * of the CPUs are selected as not_preferred, then + * all CPUs are eligible for pausing. + */ + if (cluster->nr_not_preferred_cpus && !c->not_preferred) + continue; + + pr_debug("Trying to pause CPU%u\n", c->cpu); + cpumask_set_cpu(c->cpu, pause_cpus); + nr_pending++; + move_cpu_lru(c); + } + +again: + /* + * If the number of active CPUs is within the limits, then + * don't force pause of any busy CPUs. + */ + if (active_cpus - nr_pending <= cluster->max_cpus) + goto unlock; + + num_cpus = cluster->num_cpus; + list_for_each_entry_safe(c, tmp, &cluster->lru, sib) { + if (!num_cpus--) + break; + + if (c->disabled) + continue; + if (!is_active(c)) + continue; + if (active_cpus - nr_pending <= cluster->max_cpus) + break; + + if (first_pass && !c->not_preferred) + continue; + + cpumask_set_cpu(c->cpu, pause_cpus); + nr_pending++; + move_cpu_lru(c); + } + + if (first_pass && active_cpus - nr_pending > cluster->max_cpus) { + first_pass = false; + goto again; + } +unlock: + spin_unlock_irqrestore(&state_lock, flags); +} + +static int __try_to_resume(struct cluster_data *cluster, unsigned int need, + bool force, struct cpumask *unpause_cpus) +{ + struct cpu_data *c, *tmp; + unsigned long flags; + unsigned int num_cpus = cluster->num_cpus; + unsigned int nr_pending = 0, active_cpus = cluster->active_cpus; + + /* + * Protect against entry being removed (and added at tail) by other + * thread (hotplug). + */ + spin_lock_irqsave(&state_lock, flags); + list_for_each_entry_safe(c, tmp, &cluster->lru, sib) { + if (!num_cpus--) + break; + + if (!cpumask_test_cpu(c->cpu, &cpus_paused_by_us)) + continue; + if ((cpu_active(c->cpu) && !cpu_halted(c->cpu)) || + (!force && c->not_preferred)) + continue; + if (active_cpus + nr_pending == need) + break; + + pr_debug("Trying to resume CPU%u\n", c->cpu); + + cpumask_set_cpu(c->cpu, unpause_cpus); + nr_pending++; + move_cpu_lru(c); + } + + spin_unlock_irqrestore(&state_lock, flags); + + return nr_pending; +} + +static void try_to_resume(struct cluster_data *cluster, unsigned int need, + struct cpumask *unpause_cpus) +{ + bool force_use_non_preferred = false; + unsigned int nr_pending; + + /* + * __try_to_resume() marks the CPUs to be resumed but active_cpus + * won't be reflected yet. So use the nr_pending to adjust active + * count. + */ + nr_pending = __try_to_resume(cluster, need, force_use_non_preferred, unpause_cpus); + + if (cluster->active_cpus + nr_pending == need) + return; + + force_use_non_preferred = true; + __try_to_resume(cluster, need, force_use_non_preferred, unpause_cpus); +} + +/* + * core_ctl_pause_cpus: pause a set of CPUs as requested by core_ctl, handling errors. + * + * In order to handle errors properly, and properly track success, the cpus being + * passed to walt_pause_cpus needs to be saved off. It needs to be saved because + * walt_pause_cpus will modify the value (through pause_cpus()). Pause_cpus modifies + * the value because it updates the variable to eliminate CPUs that are already paused. + * THIS code, however, must be very careful to track what cpus were requested, rather + * than what cpus actually were paused in this action. Otherwise, the ref-counts in + * walt_pause.c will get out of sync with this code. + */ +static void core_ctl_pause_cpus(struct cpumask *cpus_to_pause) +{ + cpumask_t saved_cpus; + + /* be careful to only pause CPUs not paused before to ensure ref-count sync */ + cpumask_andnot(cpus_to_pause, cpus_to_pause, &cpus_paused_by_us); + cpumask_copy(&saved_cpus, cpus_to_pause); + + if (cpumask_any(cpus_to_pause) < nr_cpu_ids) { + if (walt_halt_cpus(cpus_to_pause, PAUSE_CORE_CTL) < 0) + pr_debug("core_ctl pause operation failed cpus=%*pbl paused_by_us=%*pbl\n", + cpumask_pr_args(cpus_to_pause), + cpumask_pr_args(&cpus_paused_by_us)); + else + cpumask_or(&cpus_paused_by_us, &cpus_paused_by_us, &saved_cpus); + } +} + +/* + * core_ctl_resume_cpus: resume a set of CPUs as requested by core_ctl, handling errors. + * + * In order to handle errors properly, and properly track success, the cpus being + * passed to walt_resume_cpus needs to be saved off. It needs to be saved because + * walt_resume_cpus will modify the value (through resume_cpus()). Resume_cpus modifies + * the value because it updates the variable to eliminate CPUs that are already resumed. + * THIS code, however, must be very careful to track what cpus were requested, rather + * than what cpus actually were resumed in this action. Otherwise, the ref-counts in + * walt_pause.c will get out of sync with this code. + */ +static void core_ctl_resume_cpus(struct cpumask *cpus_to_unpause) +{ + cpumask_t saved_cpus; + + /* be careful to only unpause CPUs paused before to ensure ref-count sync */ + cpumask_and(cpus_to_unpause, cpus_to_unpause, &cpus_paused_by_us); + cpumask_copy(&saved_cpus, cpus_to_unpause); + + if (cpumask_any(cpus_to_unpause) < nr_cpu_ids) { + if (walt_start_cpus(cpus_to_unpause, PAUSE_CORE_CTL) < 0) + pr_debug("core_ctl resume operation failed cpus=%*pbl paused_by_us=%*pbl\n", + cpumask_pr_args(cpus_to_unpause), + cpumask_pr_args(&cpus_paused_by_us)); + else + cpumask_andnot(&cpus_paused_by_us, &cpus_paused_by_us, &saved_cpus); + } +} + +static void __ref do_core_ctl(void) +{ + struct cluster_data *cluster; + unsigned int index = 0; + unsigned int need; + cpumask_t cpus_to_pause = { CPU_BITS_NONE }; + cpumask_t cpus_to_unpause = { CPU_BITS_NONE }; + + for_each_cluster(cluster, index) { + + cluster->active_cpus = get_active_cpu_count(cluster); + need = apply_limits(cluster, cluster->need_cpus); + + if (adjustment_possible(cluster, need)) { + pr_debug("Trying to adjust group %u from %u to %u\n", + cluster->first_cpu, cluster->active_cpus, need); + + if (cluster->active_cpus > need) + try_to_pause(cluster, need, &cpus_to_pause); + + else if (cluster->active_cpus < need) + try_to_resume(cluster, need, &cpus_to_unpause); + } + } + + core_ctl_pause_cpus(&cpus_to_pause); + core_ctl_resume_cpus(&cpus_to_unpause); +} + +static int __ref try_core_ctl(void *data) +{ + unsigned long flags; + + while (1) { + set_current_state(TASK_INTERRUPTIBLE); + spin_lock_irqsave(&core_ctl_pending_lock, flags); + if (!core_ctl_pending) { + spin_unlock_irqrestore(&core_ctl_pending_lock, flags); + schedule(); + if (kthread_should_stop()) + break; + spin_lock_irqsave(&core_ctl_pending_lock, flags); + } + set_current_state(TASK_RUNNING); + core_ctl_pending = false; + spin_unlock_irqrestore(&core_ctl_pending_lock, flags); + + do_core_ctl(); + } + + return 0; +} + +/* ============================ init code ============================== */ + +static struct cluster_data *find_cluster_by_first_cpu(unsigned int first_cpu) +{ + unsigned int i; + + for (i = 0; i < num_clusters; ++i) { + if (cluster_state[i].first_cpu == first_cpu) + return &cluster_state[i]; + } + + return NULL; +} + +static int cluster_init(const struct cpumask *mask) +{ + struct device *dev; + unsigned int first_cpu = cpumask_first(mask); + struct cluster_data *cluster; + struct cpu_data *state; + unsigned int cpu; + + if (find_cluster_by_first_cpu(first_cpu)) + return 0; + + dev = get_cpu_device(first_cpu); + if (!dev) + return -ENODEV; + + pr_info("Creating CPU group %d\n", first_cpu); + + if (num_clusters == MAX_CLUSTERS) { + pr_err("Unsupported number of clusters. Only %u supported\n", + MAX_CLUSTERS); + return -EINVAL; + } + cluster = &cluster_state[num_clusters]; + ++num_clusters; + + cpumask_copy(&cluster->cpu_mask, mask); + cluster->num_cpus = cpumask_weight(mask); + if (cluster->num_cpus > MAX_CPUS_PER_CLUSTER) { + pr_err("HW configuration not supported\n"); + return -EINVAL; + } + cluster->first_cpu = first_cpu; + cluster->min_cpus = 1; + cluster->max_cpus = cluster->num_cpus; + cluster->need_cpus = cluster->num_cpus; + cluster->offline_delay_ms = 100; + cluster->task_thres = UINT_MAX; + cluster->nr_prev_assist_thresh = UINT_MAX; + cluster->nrrun = cluster->num_cpus; + cluster->enable = true; + cluster->nr_not_preferred_cpus = 0; + cluster->strict_nrrun = 0; + INIT_LIST_HEAD(&cluster->lru); + + for_each_cpu(cpu, mask) { + pr_info("Init CPU%u state\n", cpu); + + state = &per_cpu(cpu_state, cpu); + state->cluster = cluster; + state->cpu = cpu; + state->disabled = get_cpu_device(cpu) && + get_cpu_device(cpu)->offline_disabled; + list_add_tail(&state->sib, &cluster->lru); + } + cluster->active_cpus = get_active_cpu_count(cluster); + + cluster->inited = true; + + kobject_init(&cluster->kobj, &ktype_core_ctl); + return kobject_add(&cluster->kobj, &dev->kobj, "core_ctl"); +} + +int core_ctl_init(void) +{ + struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; + struct walt_sched_cluster *cluster; + int ret; + + spin_lock_init(&core_ctl_pending_lock); + + nr_stats = sched_get_nr_running_avg(); + + /* initialize our single kthread, after spin lock init */ + core_ctl_thread = kthread_run(try_core_ctl, NULL, "core_ctl"); + + if (IS_ERR(core_ctl_thread)) + return PTR_ERR(core_ctl_thread); + + sched_setscheduler_nocheck(core_ctl_thread, SCHED_FIFO, ¶m); + + for_each_sched_cluster(cluster) { + ret = cluster_init(&cluster->cpus); + if (ret) + pr_warn("unable to create core ctl group: %d\n", ret); + } + + initialized = true; + + return 0; +} diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c new file mode 100644 index 000000000000..7de85593f288 --- /dev/null +++ b/kernel/sched/walt/cpufreq_walt.c @@ -0,0 +1,1027 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This is based on schedutil governor but modified to work with + * WALT. + * + * Copyright (C) 2016, Intel Corporation + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +#include "walt.h" +#include "trace.h" + +struct waltgov_tunables { + struct gov_attr_set attr_set; + unsigned int up_rate_limit_us; + unsigned int down_rate_limit_us; + unsigned int hispeed_load; + unsigned int hispeed_freq; + unsigned int rtg_boost_freq; + unsigned int adaptive_low_freq; + unsigned int adaptive_high_freq; + unsigned int target_load_thresh; + unsigned int target_load_shift; + bool pl; + int boost; +}; + +struct waltgov_policy { + struct cpufreq_policy *policy; + u64 last_ws; + u64 curr_cycles; + u64 last_cyc_update_time; + unsigned long avg_cap; + struct waltgov_tunables *tunables; + struct list_head tunables_hook; + unsigned long hispeed_util; + unsigned long rtg_boost_util; + unsigned long max; + + raw_spinlock_t update_lock; + u64 last_freq_update_time; + s64 min_rate_limit_ns; + s64 up_rate_delay_ns; + s64 down_rate_delay_ns; + unsigned int next_freq; + unsigned int cached_raw_freq; + unsigned int driving_cpu; + + /* The next fields are only needed if fast switch cannot be used: */ + struct irq_work irq_work; + struct kthread_work work; + struct mutex work_lock; + struct kthread_worker worker; + struct task_struct *thread; + + bool limits_changed; + bool need_freq_update; +}; + +struct waltgov_cpu { + struct waltgov_callback cb; + struct waltgov_policy *wg_policy; + unsigned int cpu; + struct walt_cpu_load walt_load; + unsigned long util; + unsigned long max; + unsigned int flags; + unsigned int reasons; +}; + +DEFINE_PER_CPU(struct waltgov_callback *, waltgov_cb_data); +static DEFINE_PER_CPU(struct waltgov_cpu, waltgov_cpu); +static DEFINE_PER_CPU(struct waltgov_tunables *, cached_tunables); + +/************************ Governor internals ***********************/ + +static bool waltgov_should_update_freq(struct waltgov_policy *wg_policy, u64 time) +{ + s64 delta_ns; + + if (unlikely(wg_policy->limits_changed)) { + wg_policy->limits_changed = false; + wg_policy->need_freq_update = true; + return true; + } + + /* + * No need to recalculate next freq for min_rate_limit_us + * at least. However we might still decide to further rate + * limit once frequency change direction is decided, according + * to the separate rate limits. + */ + + delta_ns = time - wg_policy->last_freq_update_time; + return delta_ns >= wg_policy->min_rate_limit_ns; +} + +static bool waltgov_up_down_rate_limit(struct waltgov_policy *wg_policy, u64 time, + unsigned int next_freq) +{ + s64 delta_ns; + + delta_ns = time - wg_policy->last_freq_update_time; + + if (next_freq > wg_policy->next_freq && + delta_ns < wg_policy->up_rate_delay_ns) + return true; + + if (next_freq < wg_policy->next_freq && + delta_ns < wg_policy->down_rate_delay_ns) + return true; + + return false; +} + +static bool waltgov_update_next_freq(struct waltgov_policy *wg_policy, u64 time, + unsigned int next_freq, + unsigned int raw_freq) +{ + if (wg_policy->next_freq == next_freq) + return false; + + if (waltgov_up_down_rate_limit(wg_policy, time, next_freq)) { + wg_policy->cached_raw_freq = 0; + return false; + } + + wg_policy->cached_raw_freq = raw_freq; + wg_policy->next_freq = next_freq; + wg_policy->last_freq_update_time = time; + + return true; +} + +static unsigned long freq_to_util(struct waltgov_policy *wg_policy, + unsigned int freq) +{ + return mult_frac(wg_policy->max, freq, + wg_policy->policy->cpuinfo.max_freq); +} + +#define KHZ 1000 +static void waltgov_track_cycles(struct waltgov_policy *wg_policy, + unsigned int prev_freq, + u64 upto) +{ + u64 delta_ns, cycles; + u64 next_ws = wg_policy->last_ws + sched_ravg_window; + + upto = min(upto, next_ws); + /* Track cycles in current window */ + delta_ns = upto - wg_policy->last_cyc_update_time; + delta_ns *= prev_freq; + do_div(delta_ns, (NSEC_PER_SEC / KHZ)); + cycles = delta_ns; + wg_policy->curr_cycles += cycles; + wg_policy->last_cyc_update_time = upto; +} + +static void waltgov_calc_avg_cap(struct waltgov_policy *wg_policy, u64 curr_ws, + unsigned int prev_freq) +{ + u64 last_ws = wg_policy->last_ws; + unsigned int avg_freq; + + BUG_ON(curr_ws < last_ws); + if (curr_ws <= last_ws) + return; + + /* If we skipped some windows */ + if (curr_ws > (last_ws + sched_ravg_window)) { + avg_freq = prev_freq; + /* Reset tracking history */ + wg_policy->last_cyc_update_time = curr_ws; + } else { + waltgov_track_cycles(wg_policy, prev_freq, curr_ws); + avg_freq = wg_policy->curr_cycles; + avg_freq /= sched_ravg_window / (NSEC_PER_SEC / KHZ); + } + wg_policy->avg_cap = freq_to_util(wg_policy, avg_freq); + wg_policy->curr_cycles = 0; + wg_policy->last_ws = curr_ws; +} + +static void waltgov_fast_switch(struct waltgov_policy *wg_policy, u64 time, + unsigned int next_freq) +{ + struct cpufreq_policy *policy = wg_policy->policy; + + waltgov_track_cycles(wg_policy, wg_policy->policy->cur, time); + cpufreq_driver_fast_switch(policy, next_freq); +} + +static void waltgov_deferred_update(struct waltgov_policy *wg_policy, u64 time, + unsigned int next_freq) +{ + walt_irq_work_queue(&wg_policy->irq_work); +} + +#define TARGET_LOAD 80 +static inline unsigned long walt_map_util_freq(unsigned long util, + struct waltgov_policy *wg_policy, + unsigned long cap, int cpu) +{ + unsigned long fmax = wg_policy->policy->cpuinfo.max_freq; + unsigned int shift = wg_policy->tunables->target_load_shift; + + if (util >= wg_policy->tunables->target_load_thresh && + cpu_util_rt(cpu_rq(cpu)) < (cap >> 2)) + return max( + (fmax + (fmax >> shift)) * util, + (fmax + (fmax >> 2)) * wg_policy->tunables->target_load_thresh + )/cap; + return (fmax + (fmax >> 2)) * util / cap; +} + +static unsigned int get_next_freq(struct waltgov_policy *wg_policy, + unsigned long util, unsigned long max, + struct waltgov_cpu *wg_cpu, u64 time) +{ + struct cpufreq_policy *policy = wg_policy->policy; + unsigned int freq, raw_freq, final_freq; + struct waltgov_cpu *wg_driv_cpu = &per_cpu(waltgov_cpu, wg_policy->driving_cpu); + + raw_freq = walt_map_util_freq(util, wg_policy, max, wg_driv_cpu->cpu); + freq = raw_freq; + + if (wg_policy->tunables->adaptive_high_freq) { + if (raw_freq < wg_policy->tunables->adaptive_low_freq) { + freq = wg_policy->tunables->adaptive_low_freq; + wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_LOW; + } else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) { + freq = wg_policy->tunables->adaptive_high_freq; + wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_HIGH; + } + } + + trace_waltgov_next_freq(policy->cpu, util, max, raw_freq, freq, policy->min, policy->max, + wg_policy->cached_raw_freq, wg_policy->need_freq_update, + wg_driv_cpu->cpu, wg_driv_cpu->reasons); + + if (wg_policy->cached_raw_freq && freq == wg_policy->cached_raw_freq && + !wg_policy->need_freq_update) + return 0; + + wg_policy->need_freq_update = false; + + final_freq = cpufreq_driver_resolve_freq(policy, freq); + + if (!waltgov_update_next_freq(wg_policy, time, final_freq, freq)) + return 0; + + return final_freq; +} + +static unsigned long waltgov_get_util(struct waltgov_cpu *wg_cpu) +{ + struct rq *rq = cpu_rq(wg_cpu->cpu); + unsigned long max = arch_scale_cpu_capacity(wg_cpu->cpu); + unsigned long util; + + wg_cpu->max = max; + wg_cpu->reasons = 0; + util = cpu_util_freq_walt(wg_cpu->cpu, &wg_cpu->walt_load, &wg_cpu->reasons); + return uclamp_rq_util_with(rq, util, NULL); +} + +#define NL_RATIO 75 +#define DEFAULT_HISPEED_LOAD 90 +#define DEFAULT_SILVER_RTG_BOOST_FREQ 1000000 +#define DEFAULT_GOLD_RTG_BOOST_FREQ 768000 +#define DEFAULT_PRIME_RTG_BOOST_FREQ 0 +#define DEFAULT_TARGET_LOAD_THRESH 1024 +#define DEFAULT_TARGET_LOAD_SHIFT 4 +static inline void max_and_reason(unsigned long *cur_util, unsigned long boost_util, + struct waltgov_cpu *wg_cpu, unsigned int reason) +{ + if (boost_util && boost_util >= *cur_util) { + *cur_util = boost_util; + wg_cpu->reasons = reason; + wg_cpu->wg_policy->driving_cpu = wg_cpu->cpu; + } +} + +static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_util, + unsigned long nl, unsigned long *util, + unsigned long *max) +{ + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + bool is_migration = wg_cpu->flags & WALT_CPUFREQ_IC_MIGRATION; + bool is_rtg_boost = wg_cpu->walt_load.rtgb_active; + bool is_hiload; + unsigned long pl = wg_cpu->walt_load.pl; + + if (is_rtg_boost) + max_and_reason(util, wg_policy->rtg_boost_util, wg_cpu, CPUFREQ_REASON_RTG_BOOST); + + is_hiload = (cpu_util >= mult_frac(wg_policy->avg_cap, + wg_policy->tunables->hispeed_load, + 100)); + + if (is_hiload && !is_migration) + max_and_reason(util, wg_policy->hispeed_util, wg_cpu, CPUFREQ_REASON_HISPEED); + + if (is_hiload && nl >= mult_frac(cpu_util, NL_RATIO, 100)) + max_and_reason(util, *max, wg_cpu, CPUFREQ_REASON_NWD); + + if (wg_policy->tunables->pl) { + if (sysctl_sched_conservative_pl) + pl = mult_frac(pl, TARGET_LOAD, 100); + max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL); + } +} + +static inline unsigned long target_util(struct waltgov_policy *wg_policy, + unsigned int freq) +{ + unsigned long util; + + util = freq_to_util(wg_policy, freq); + + if (is_min_cluster_cpu(wg_policy->policy->cpu) && + util >= wg_policy->tunables->target_load_thresh) + util = mult_frac(util, 94, 100); + else + util = mult_frac(util, TARGET_LOAD, 100); + + return util; +} + +static unsigned int waltgov_next_freq_shared(struct waltgov_cpu *wg_cpu, u64 time) +{ + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + struct cpufreq_policy *policy = wg_policy->policy; + unsigned long util = 0, max = 1; + unsigned int j; + int boost = wg_policy->tunables->boost; + + for_each_cpu(j, policy->cpus) { + struct waltgov_cpu *j_wg_cpu = &per_cpu(waltgov_cpu, j); + unsigned long j_util, j_max, j_nl; + + /* + * If the util value for all CPUs in a policy is 0, just using > + * will result in a max value of 1. WALT stats can later update + * the aggregated util value, causing get_next_freq() to compute + * freq = max_freq * 1.25 * (util / max) for nonzero util, + * leading to spurious jumps to fmax. + */ + j_util = j_wg_cpu->util; + j_nl = j_wg_cpu->walt_load.nl; + j_max = j_wg_cpu->max; + if (boost) { + j_util = mult_frac(j_util, boost + 100, 100); + j_nl = mult_frac(j_nl, boost + 100, 100); + } + + if (j_util * max >= j_max * util) { + util = j_util; + max = j_max; + wg_policy->driving_cpu = j; + } + + waltgov_walt_adjust(j_wg_cpu, j_util, j_nl, &util, &max); + } + + return get_next_freq(wg_policy, util, max, wg_cpu, time); +} + +static void waltgov_update_freq(struct waltgov_callback *cb, u64 time, + unsigned int flags) +{ + struct waltgov_cpu *wg_cpu = container_of(cb, struct waltgov_cpu, cb); + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + unsigned long hs_util, rtg_boost_util; + unsigned int next_f; + + if (!wg_policy->tunables->pl && flags & WALT_CPUFREQ_PL) + return; + + wg_cpu->util = waltgov_get_util(wg_cpu); + wg_cpu->flags = flags; + raw_spin_lock(&wg_policy->update_lock); + + if (wg_policy->max != wg_cpu->max) { + wg_policy->max = wg_cpu->max; + hs_util = target_util(wg_policy, + wg_policy->tunables->hispeed_freq); + wg_policy->hispeed_util = hs_util; + + rtg_boost_util = target_util(wg_policy, + wg_policy->tunables->rtg_boost_freq); + wg_policy->rtg_boost_util = rtg_boost_util; + } + + waltgov_calc_avg_cap(wg_policy, wg_cpu->walt_load.ws, + wg_policy->policy->cur); + + trace_waltgov_util_update(wg_cpu->cpu, wg_cpu->util, wg_policy->avg_cap, + wg_cpu->max, wg_cpu->walt_load.nl, + wg_cpu->walt_load.pl, + wg_cpu->walt_load.rtgb_active, flags); + + if (waltgov_should_update_freq(wg_policy, time) && + !(flags & WALT_CPUFREQ_CONTINUE)) { + next_f = waltgov_next_freq_shared(wg_cpu, time); + + if (!next_f) + goto out; + + if (wg_policy->policy->fast_switch_enabled) + waltgov_fast_switch(wg_policy, time, next_f); + else + waltgov_deferred_update(wg_policy, time, next_f); + } + +out: + raw_spin_unlock(&wg_policy->update_lock); +} + +static void waltgov_work(struct kthread_work *work) +{ + struct waltgov_policy *wg_policy = container_of(work, struct waltgov_policy, work); + unsigned int freq; + unsigned long flags; + + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + freq = wg_policy->next_freq; + waltgov_track_cycles(wg_policy, wg_policy->policy->cur, + walt_sched_clock()); + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + + mutex_lock(&wg_policy->work_lock); + __cpufreq_driver_target(wg_policy->policy, freq, CPUFREQ_RELATION_L); + mutex_unlock(&wg_policy->work_lock); +} + +static void waltgov_irq_work(struct irq_work *irq_work) +{ + struct waltgov_policy *wg_policy; + + wg_policy = container_of(irq_work, struct waltgov_policy, irq_work); + + kthread_queue_work(&wg_policy->worker, &wg_policy->work); +} + +/************************** sysfs interface ************************/ + +static inline struct waltgov_tunables *to_waltgov_tunables(struct gov_attr_set *attr_set) +{ + return container_of(attr_set, struct waltgov_tunables, attr_set); +} + +static DEFINE_MUTEX(min_rate_lock); + +static void update_min_rate_limit_ns(struct waltgov_policy *wg_policy) +{ + mutex_lock(&min_rate_lock); + wg_policy->min_rate_limit_ns = min(wg_policy->up_rate_delay_ns, + wg_policy->down_rate_delay_ns); + mutex_unlock(&min_rate_lock); +} + +static ssize_t up_rate_limit_us_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->up_rate_limit_us); +} + +static ssize_t down_rate_limit_us_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->down_rate_limit_us); +} + +static ssize_t up_rate_limit_us_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + struct waltgov_policy *wg_policy; + unsigned int rate_limit_us; + + if (kstrtouint(buf, 10, &rate_limit_us)) + return -EINVAL; + + tunables->up_rate_limit_us = rate_limit_us; + + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + wg_policy->up_rate_delay_ns = rate_limit_us * NSEC_PER_USEC; + update_min_rate_limit_ns(wg_policy); + } + + return count; +} + +static ssize_t down_rate_limit_us_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + struct waltgov_policy *wg_policy; + unsigned int rate_limit_us; + + if (kstrtouint(buf, 10, &rate_limit_us)) + return -EINVAL; + + tunables->down_rate_limit_us = rate_limit_us; + + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + wg_policy->down_rate_delay_ns = rate_limit_us * NSEC_PER_USEC; + update_min_rate_limit_ns(wg_policy); + } + + return count; +} + +static struct governor_attr up_rate_limit_us = __ATTR_RW(up_rate_limit_us); +static struct governor_attr down_rate_limit_us = __ATTR_RW(down_rate_limit_us); + +static ssize_t hispeed_load_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->hispeed_load); +} + +static ssize_t hispeed_load_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + if (kstrtouint(buf, 10, &tunables->hispeed_load)) + return -EINVAL; + + tunables->hispeed_load = min(100U, tunables->hispeed_load); + + return count; +} + +static ssize_t hispeed_freq_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->hispeed_freq); +} + +static ssize_t hispeed_freq_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + unsigned int val; + struct waltgov_policy *wg_policy; + unsigned long hs_util; + unsigned long flags; + + if (kstrtouint(buf, 10, &val)) + return -EINVAL; + + tunables->hispeed_freq = val; + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + hs_util = target_util(wg_policy, + wg_policy->tunables->hispeed_freq); + wg_policy->hispeed_util = hs_util; + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + } + + return count; +} + +static ssize_t rtg_boost_freq_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->rtg_boost_freq); +} + +static ssize_t rtg_boost_freq_store(struct gov_attr_set *attr_set, + const char *buf, size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + unsigned int val; + struct waltgov_policy *wg_policy; + unsigned long rtg_boost_util; + unsigned long flags; + + if (kstrtouint(buf, 10, &val)) + return -EINVAL; + + tunables->rtg_boost_freq = val; + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + rtg_boost_util = target_util(wg_policy, + wg_policy->tunables->rtg_boost_freq); + wg_policy->rtg_boost_util = rtg_boost_util; + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + } + + return count; +} + +static ssize_t pl_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->pl); +} + +static ssize_t pl_store(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + if (kstrtobool(buf, &tunables->pl)) + return -EINVAL; + + return count; +} + +static ssize_t boost_show(struct gov_attr_set *attr_set, char *buf) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + + return scnprintf(buf, PAGE_SIZE, "%d\n", tunables->boost); +} + +static ssize_t boost_store(struct gov_attr_set *attr_set, const char *buf, + size_t count) +{ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); + struct waltgov_policy *wg_policy; + int val; + + if (kstrtoint(buf, 10, &val)) + return -EINVAL; + + if (val < -100 || val > 1000) + return -EINVAL; + + tunables->boost = val; + list_for_each_entry(wg_policy, &attr_set->policy_list, tunables_hook) { + struct rq *rq = cpu_rq(wg_policy->policy->cpu); + unsigned long flags; + + raw_spin_lock_irqsave(&rq->__lock, flags); + waltgov_run_callback(rq, WALT_CPUFREQ_BOOST_UPDATE); + raw_spin_unlock_irqrestore(&rq->__lock, flags); + } + return count; +} + +#define WALTGOV_ATTR_RW(_name) \ +static struct governor_attr _name = \ +__ATTR(_name, 0644, show_##_name, store_##_name) \ + +#define show_attr(name) \ +static ssize_t show_##name(struct gov_attr_set *attr_set, char *buf) \ +{ \ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); \ + return scnprintf(buf, PAGE_SIZE, "%lu\n", tunables->name); \ +} \ + +#define store_attr(name) \ +static ssize_t store_##name(struct gov_attr_set *attr_set, \ + const char *buf, size_t count) \ +{ \ + struct waltgov_tunables *tunables = to_waltgov_tunables(attr_set); \ + \ + if (kstrtouint(buf, 10, &tunables->name)) \ + return -EINVAL; \ + \ + return count; \ +} \ + +show_attr(adaptive_low_freq); +store_attr(adaptive_low_freq); +show_attr(adaptive_high_freq); +store_attr(adaptive_high_freq); +show_attr(target_load_thresh); +store_attr(target_load_thresh); +show_attr(target_load_shift); +store_attr(target_load_shift); + +static struct governor_attr hispeed_load = __ATTR_RW(hispeed_load); +static struct governor_attr hispeed_freq = __ATTR_RW(hispeed_freq); +static struct governor_attr rtg_boost_freq = __ATTR_RW(rtg_boost_freq); +static struct governor_attr pl = __ATTR_RW(pl); +static struct governor_attr boost = __ATTR_RW(boost); +WALTGOV_ATTR_RW(adaptive_low_freq); +WALTGOV_ATTR_RW(adaptive_high_freq); +WALTGOV_ATTR_RW(target_load_thresh); +WALTGOV_ATTR_RW(target_load_shift); + +static struct attribute *waltgov_attributes[] = { + &up_rate_limit_us.attr, + &down_rate_limit_us.attr, + &hispeed_load.attr, + &hispeed_freq.attr, + &rtg_boost_freq.attr, + &pl.attr, + &boost.attr, + &adaptive_low_freq.attr, + &adaptive_high_freq.attr, + &target_load_thresh.attr, + &target_load_shift.attr, + NULL +}; + +static struct kobj_type waltgov_tunables_ktype = { + .default_attrs = waltgov_attributes, + .sysfs_ops = &governor_sysfs_ops, +}; + +/********************** cpufreq governor interface *********************/ + +static struct cpufreq_governor walt_gov; + +static struct waltgov_policy *waltgov_policy_alloc(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy; + + wg_policy = kzalloc(sizeof(*wg_policy), GFP_KERNEL); + if (!wg_policy) + return NULL; + + wg_policy->policy = policy; + raw_spin_lock_init(&wg_policy->update_lock); + return wg_policy; +} + +static void waltgov_policy_free(struct waltgov_policy *wg_policy) +{ + kfree(wg_policy); +} + +static int waltgov_kthread_create(struct waltgov_policy *wg_policy) +{ + struct task_struct *thread; + struct sched_param param = { .sched_priority = MAX_RT_PRIO / 2 }; + struct cpufreq_policy *policy = wg_policy->policy; + int ret; + + /* kthread only required for slow path */ + if (policy->fast_switch_enabled) + return 0; + + kthread_init_work(&wg_policy->work, waltgov_work); + kthread_init_worker(&wg_policy->worker); + thread = kthread_create(kthread_worker_fn, &wg_policy->worker, + "waltgov:%d", + cpumask_first(policy->related_cpus)); + if (IS_ERR(thread)) { + pr_err("failed to create waltgov thread: %ld\n", PTR_ERR(thread)); + return PTR_ERR(thread); + } + + ret = sched_setscheduler_nocheck(thread, SCHED_FIFO, ¶m); + if (ret) { + kthread_stop(thread); + pr_warn("%s: failed to set SCHED_FIFO\n", __func__); + return ret; + } + + wg_policy->thread = thread; + kthread_bind_mask(thread, policy->related_cpus); + init_irq_work(&wg_policy->irq_work, waltgov_irq_work); + mutex_init(&wg_policy->work_lock); + + wake_up_process(thread); + + return 0; +} + +static void waltgov_kthread_stop(struct waltgov_policy *wg_policy) +{ + /* kthread only required for slow path */ + if (wg_policy->policy->fast_switch_enabled) + return; + + kthread_flush_worker(&wg_policy->worker); + kthread_stop(wg_policy->thread); + mutex_destroy(&wg_policy->work_lock); +} + +static void waltgov_tunables_save(struct cpufreq_policy *policy, + struct waltgov_tunables *tunables) +{ + int cpu; + struct waltgov_tunables *cached = per_cpu(cached_tunables, policy->cpu); + + if (!cached) { + cached = kzalloc(sizeof(*tunables), GFP_KERNEL); + if (!cached) + return; + + for_each_cpu(cpu, policy->related_cpus) + per_cpu(cached_tunables, cpu) = cached; + } + + cached->pl = tunables->pl; + cached->hispeed_load = tunables->hispeed_load; + cached->rtg_boost_freq = tunables->rtg_boost_freq; + cached->hispeed_freq = tunables->hispeed_freq; + cached->up_rate_limit_us = tunables->up_rate_limit_us; + cached->down_rate_limit_us = tunables->down_rate_limit_us; + cached->boost = tunables->boost; + cached->adaptive_low_freq = tunables->adaptive_low_freq; + cached->adaptive_high_freq = tunables->adaptive_high_freq; + cached->target_load_thresh = tunables->target_load_thresh; + cached->target_load_shift = tunables->target_load_shift; +} + +static void waltgov_tunables_restore(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + struct waltgov_tunables *tunables = wg_policy->tunables; + struct waltgov_tunables *cached = per_cpu(cached_tunables, policy->cpu); + + if (!cached) + return; + + tunables->pl = cached->pl; + tunables->hispeed_load = cached->hispeed_load; + tunables->rtg_boost_freq = cached->rtg_boost_freq; + tunables->hispeed_freq = cached->hispeed_freq; + tunables->up_rate_limit_us = cached->up_rate_limit_us; + tunables->down_rate_limit_us = cached->down_rate_limit_us; + tunables->boost = cached->boost; + tunables->adaptive_low_freq = cached->adaptive_low_freq; + tunables->adaptive_high_freq = cached->adaptive_high_freq; + tunables->target_load_thresh = cached->target_load_thresh; + tunables->target_load_shift = cached->target_load_shift; +} + +static int waltgov_init(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy; + struct waltgov_tunables *tunables; + int ret = 0; + + /* State should be equivalent to EXIT */ + if (policy->governor_data) + return -EBUSY; + + cpufreq_enable_fast_switch(policy); + + if (policy->fast_switch_possible && !policy->fast_switch_enabled) + BUG_ON(1); + + wg_policy = waltgov_policy_alloc(policy); + if (!wg_policy) { + ret = -ENOMEM; + goto disable_fast_switch; + } + + ret = waltgov_kthread_create(wg_policy); + if (ret) + goto free_wg_policy; + + tunables = kzalloc(sizeof(*tunables), GFP_KERNEL); + if (!tunables) { + ret = -ENOMEM; + goto stop_kthread; + } + + gov_attr_set_init(&tunables->attr_set, &wg_policy->tunables_hook); + tunables->hispeed_load = DEFAULT_HISPEED_LOAD; + tunables->target_load_thresh = DEFAULT_TARGET_LOAD_THRESH; + tunables->target_load_shift = DEFAULT_TARGET_LOAD_SHIFT; + + if (is_min_cluster_cpu(policy->cpu)) + tunables->rtg_boost_freq = DEFAULT_SILVER_RTG_BOOST_FREQ; + else if (is_max_cluster_cpu(policy->cpu)) + tunables->rtg_boost_freq = DEFAULT_PRIME_RTG_BOOST_FREQ; + else + tunables->rtg_boost_freq = DEFAULT_GOLD_RTG_BOOST_FREQ; + + policy->governor_data = wg_policy; + wg_policy->tunables = tunables; + waltgov_tunables_restore(policy); + + ret = kobject_init_and_add(&tunables->attr_set.kobj, &waltgov_tunables_ktype, + get_governor_parent_kobj(policy), "%s", + walt_gov.name); + if (ret) + goto fail; + + return 0; + +fail: + kobject_put(&tunables->attr_set.kobj); + policy->governor_data = NULL; + kfree(tunables); +stop_kthread: + waltgov_kthread_stop(wg_policy); +free_wg_policy: + waltgov_policy_free(wg_policy); +disable_fast_switch: + cpufreq_disable_fast_switch(policy); + + pr_err("initialization failed (error %d)\n", ret); + return ret; +} + +static void waltgov_exit(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + struct waltgov_tunables *tunables = wg_policy->tunables; + unsigned int count; + + count = gov_attr_set_put(&tunables->attr_set, &wg_policy->tunables_hook); + policy->governor_data = NULL; + if (!count) { + waltgov_tunables_save(policy, tunables); + kfree(tunables); + } + + waltgov_kthread_stop(wg_policy); + waltgov_policy_free(wg_policy); + cpufreq_disable_fast_switch(policy); +} + +static int waltgov_start(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + unsigned int cpu; + + wg_policy->up_rate_delay_ns = + wg_policy->tunables->up_rate_limit_us * NSEC_PER_USEC; + wg_policy->down_rate_delay_ns = + wg_policy->tunables->down_rate_limit_us * NSEC_PER_USEC; + update_min_rate_limit_ns(wg_policy); + wg_policy->last_freq_update_time = 0; + wg_policy->next_freq = 0; + wg_policy->limits_changed = false; + wg_policy->need_freq_update = false; + wg_policy->cached_raw_freq = 0; + + for_each_cpu(cpu, policy->cpus) { + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + + memset(wg_cpu, 0, sizeof(*wg_cpu)); + wg_cpu->cpu = cpu; + wg_cpu->wg_policy = wg_policy; + } + + for_each_cpu(cpu, policy->cpus) { + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + + waltgov_add_callback(cpu, &wg_cpu->cb, waltgov_update_freq); + } + + return 0; +} + +static void waltgov_stop(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + unsigned int cpu; + + for_each_cpu(cpu, policy->cpus) + waltgov_remove_callback(cpu); + + synchronize_rcu(); + + if (!policy->fast_switch_enabled) { + irq_work_sync(&wg_policy->irq_work); + kthread_cancel_work_sync(&wg_policy->work); + } +} + +static void waltgov_limits(struct cpufreq_policy *policy) +{ + struct waltgov_policy *wg_policy = policy->governor_data; + unsigned long flags, now; + unsigned int freq, final_freq; + + if (!policy->fast_switch_enabled) { + mutex_lock(&wg_policy->work_lock); + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + waltgov_track_cycles(wg_policy, wg_policy->policy->cur, + walt_sched_clock()); + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + cpufreq_policy_apply_limits(policy); + mutex_unlock(&wg_policy->work_lock); + } else { + raw_spin_lock_irqsave(&wg_policy->update_lock, flags); + freq = policy->cur; + now = walt_sched_clock(); + + /* + * cpufreq_driver_resolve_freq() has a clamp, so we do not need + * to do any sort of additional validation here. + */ + final_freq = cpufreq_driver_resolve_freq(policy, freq); + + if (waltgov_update_next_freq(wg_policy, now, final_freq, + final_freq)) { + waltgov_fast_switch(wg_policy, now, final_freq); + } + raw_spin_unlock_irqrestore(&wg_policy->update_lock, flags); + } + + wg_policy->limits_changed = true; +} + +static struct cpufreq_governor walt_gov = { + .name = "walt", + .init = waltgov_init, + .exit = waltgov_exit, + .start = waltgov_start, + .stop = waltgov_stop, + .limits = waltgov_limits, + .owner = THIS_MODULE, +}; + +int waltgov_register(void) +{ + return cpufreq_register_governor(&walt_gov); +} diff --git a/kernel/sched/walt/fixup.c b/kernel/sched/walt/fixup.c new file mode 100644 index 000000000000..c0fc19acb347 --- /dev/null +++ b/kernel/sched/walt/fixup.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include + +#include "walt.h" + +unsigned int cpuinfo_max_freq_cached; + +char sched_lib_name[LIB_PATH_LENGTH]; +unsigned int sched_lib_mask_force; + +static bool is_sched_lib_based_app(pid_t pid) +{ + const char *name = NULL; + char *libname, *lib_list; + struct vm_area_struct *vma; + char path_buf[LIB_PATH_LENGTH]; + char *tmp_lib_name; + bool found = false; + struct task_struct *p; + struct mm_struct *mm; + + if (strnlen(sched_lib_name, LIB_PATH_LENGTH) == 0) + return false; + + tmp_lib_name = kmalloc(LIB_PATH_LENGTH, GFP_KERNEL); + if (!tmp_lib_name) + return false; + + rcu_read_lock(); + p = pid ? get_pid_task(find_vpid(pid), PIDTYPE_PID) : get_task_struct(current); + rcu_read_unlock(); + if (!p) { + kfree(tmp_lib_name); + return false; + } + + mm = get_task_mm(p); + if (!mm) + goto put_task_struct; + + down_read(&mm->mmap_lock); + for (vma = mm->mmap; vma ; vma = vma->vm_next) { + if (vma->vm_file && vma->vm_flags & VM_EXEC) { + name = d_path(&vma->vm_file->f_path, + path_buf, LIB_PATH_LENGTH); + if (IS_ERR(name)) + goto release_sem; + + strlcpy(tmp_lib_name, sched_lib_name, LIB_PATH_LENGTH); + lib_list = tmp_lib_name; + while ((libname = strsep(&lib_list, ","))) { + libname = skip_spaces(libname); + if (strnstr(name, libname, + strnlen(name, LIB_PATH_LENGTH))) { + found = true; + goto release_sem; + } + } + } + } + +release_sem: + up_read(&mm->mmap_lock); + mmput(mm); +put_task_struct: + put_task_struct(p); + kfree(tmp_lib_name); + return found; +} + +static void android_rvh_show_max_freq(void *unused, struct cpufreq_policy *policy, + unsigned int *max_freq) +{ + if (!cpuinfo_max_freq_cached) + return; + + if (!(BIT(policy->cpu) & sched_lib_mask_force)) + return; + + if (is_sched_lib_based_app(current->pid)) + *max_freq = cpuinfo_max_freq_cached << 1; +} + +void walt_fixup_init(void) +{ + register_trace_android_rvh_show_max_freq(android_rvh_show_max_freq, NULL); +} diff --git a/kernel/sched/walt/input-boost.c b/kernel/sched/walt/input-boost.c new file mode 100644 index 000000000000..5372d899bd45 --- /dev/null +++ b/kernel/sched/walt/input-boost.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013-2015,2017,2019-2021, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) "input-boost: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "walt.h" + +#define input_boost_attr_rw(_name) \ +static struct kobj_attribute _name##_attr = \ +__ATTR(_name, 0644, show_##_name, store_##_name) + +#define show_one(file_name) \ +static ssize_t show_##file_name \ +(struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ +{ \ + return scnprintf(buf, PAGE_SIZE, "%u\n", file_name); \ +} + +#define store_one(file_name) \ +static ssize_t store_##file_name \ +(struct kobject *kobj, struct kobj_attribute *attr, \ +const char *buf, size_t count) \ +{ \ + \ + sscanf(buf, "%u", &file_name); \ + return count; \ +} + +struct cpu_sync { + int cpu; + unsigned int input_boost_min; + unsigned int input_boost_freq; +}; + +static DEFINE_PER_CPU(struct cpu_sync, sync_info); +static struct workqueue_struct *input_boost_wq; + +static struct work_struct input_boost_work; + +static bool sched_boost_active; + +static struct delayed_work input_boost_rem; +static u64 last_input_time; +#define MIN_INPUT_INTERVAL (150 * USEC_PER_MSEC) + +static DEFINE_PER_CPU(struct freq_qos_request, qos_req); + +static void boost_adjust_notify(struct cpufreq_policy *policy) +{ + unsigned int cpu = policy->cpu; + struct cpu_sync *s = &per_cpu(sync_info, cpu); + unsigned int ib_min = s->input_boost_min; + struct freq_qos_request *req = &per_cpu(qos_req, cpu); + int ret; + + pr_debug("CPU%u policy min before boost: %u kHz\n", + cpu, policy->min); + pr_debug("CPU%u boost min: %u kHz\n", cpu, ib_min); + + ret = freq_qos_update_request(req, ib_min); + + if (ret < 0) + pr_err("Failed to update freq constraint in boost_adjust: %d\n", + ib_min); + + pr_debug("CPU%u policy min after boost: %u kHz\n", cpu, policy->min); +} + +static void update_policy_online(void) +{ + unsigned int i; + struct cpufreq_policy *policy; + struct cpumask online_cpus; + + /* Re-evaluate policy to trigger adjust notifier for online CPUs */ + cpus_read_lock(); + online_cpus = *cpu_online_mask; + for_each_cpu(i, &online_cpus) { + policy = cpufreq_cpu_get(i); + if (!policy) { + pr_err("%s: cpufreq policy not found for cpu%d\n", + __func__, i); + return; + } + + cpumask_andnot(&online_cpus, &online_cpus, + policy->related_cpus); + boost_adjust_notify(policy); + } + cpus_read_unlock(); +} + +static void do_input_boost_rem(struct work_struct *work) +{ + unsigned int i, ret; + struct cpu_sync *i_sync_info; + + /* Reset the input_boost_min for all CPUs in the system */ + pr_debug("Resetting input boost min for all CPUs\n"); + for_each_possible_cpu(i) { + i_sync_info = &per_cpu(sync_info, i); + i_sync_info->input_boost_min = 0; + } + + /* Update policies for all online CPUs */ + update_policy_online(); + + if (sched_boost_active) { + ret = sched_set_boost(0); + if (!ret) + pr_err("input-boost: sched boost disable failed\n"); + sched_boost_active = false; + } +} + +static void do_input_boost(struct work_struct *work) +{ + unsigned int i, ret; + struct cpu_sync *i_sync_info; + + cancel_delayed_work_sync(&input_boost_rem); + if (sched_boost_active) { + sched_set_boost(0); + sched_boost_active = false; + } + + /* Set the input_boost_min for all CPUs in the system */ + pr_debug("Setting input boost min for all CPUs\n"); + for (i = 0; i < 8; i++) { + i_sync_info = &per_cpu(sync_info, i); + i_sync_info->input_boost_min = sysctl_input_boost_freq[i]; + } + + /* Update policies for all online CPUs */ + update_policy_online(); + + /* Enable scheduler boost to migrate tasks to big cluster */ + if (sysctl_sched_boost_on_input > 0) { + ret = sched_set_boost(sysctl_sched_boost_on_input); + if (ret) + pr_err("input-boost: sched boost enable failed\n"); + else + sched_boost_active = true; + } + + queue_delayed_work(input_boost_wq, &input_boost_rem, + msecs_to_jiffies(sysctl_input_boost_ms)); +} + +static void inputboost_input_event(struct input_handle *handle, + unsigned int type, unsigned int code, int value) +{ + u64 now; + int cpu; + int enabled = 0; + + for_each_possible_cpu(cpu) { + if (sysctl_input_boost_freq[cpu] > 0) { + enabled = 1; + break; + } + } + if (!enabled) + return; + + now = ktime_to_us(ktime_get()); + if (now - last_input_time < MIN_INPUT_INTERVAL) + return; + + if (work_pending(&input_boost_work)) + return; + + queue_work(input_boost_wq, &input_boost_work); + last_input_time = ktime_to_us(ktime_get()); +} + +static int inputboost_input_connect(struct input_handler *handler, + struct input_dev *dev, const struct input_device_id *id) +{ + struct input_handle *handle; + int error; + + handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = "cpufreq"; + + error = input_register_handle(handle); + if (error) + goto err2; + + error = input_open_device(handle); + if (error) + goto err1; + + return 0; +err1: + input_unregister_handle(handle); +err2: + kfree(handle); + return error; +} + +static void inputboost_input_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} + +static const struct input_device_id inputboost_ids[] = { + /* multi-touch touchscreen */ + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT | + INPUT_DEVICE_ID_MATCH_ABSBIT, + .evbit = { BIT_MASK(EV_ABS) }, + .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] = + BIT_MASK(ABS_MT_POSITION_X) | + BIT_MASK(ABS_MT_POSITION_Y) + }, + }, + /* touchpad */ + { + .flags = INPUT_DEVICE_ID_MATCH_KEYBIT | + INPUT_DEVICE_ID_MATCH_ABSBIT, + .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) }, + .absbit = { [BIT_WORD(ABS_X)] = + BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) + }, + }, + /* Keypad */ + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT, + .evbit = { BIT_MASK(EV_KEY) }, + }, + { }, +}; + +static struct input_handler inputboost_input_handler = { + .event = inputboost_input_event, + .connect = inputboost_input_connect, + .disconnect = inputboost_input_disconnect, + .name = "input-boost", + .id_table = inputboost_ids, +}; + +struct kobject *input_boost_kobj; +int input_boost_init(void) +{ + int cpu, ret; + struct cpu_sync *s; + struct cpufreq_policy *policy; + struct freq_qos_request *req; + + input_boost_wq = alloc_workqueue("inputboost_wq", WQ_HIGHPRI, 0); + if (!input_boost_wq) + return -EFAULT; + + INIT_WORK(&input_boost_work, do_input_boost); + INIT_DELAYED_WORK(&input_boost_rem, do_input_boost_rem); + + for_each_possible_cpu(cpu) { + s = &per_cpu(sync_info, cpu); + s->cpu = cpu; + req = &per_cpu(qos_req, cpu); + policy = cpufreq_cpu_get(cpu); + if (!policy) { + pr_err("%s: cpufreq policy not found for cpu%d\n", + __func__, cpu); + return -ESRCH; + } + + ret = freq_qos_add_request(&policy->constraints, req, + FREQ_QOS_MIN, policy->min); + if (ret < 0) { + pr_err("%s: Failed to add freq constraint (%d)\n", + __func__, ret); + return ret; + } + } + + ret = input_register_handler(&inputboost_input_handler); + return 0; +} diff --git a/kernel/sched/walt/perf_trace_counters.h b/kernel/sched/walt/perf_trace_counters.h new file mode 100644 index 000000000000..bcf318c2ebb1 --- /dev/null +++ b/kernel/sched/walt/perf_trace_counters.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2013-2014, 2017, 2021, The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM perf_trace_counters + +#if !defined(_PERF_TRACE_COUNTERS_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _PERF_TRACE_COUNTERS_H_ + +/* Ctr index for PMCNTENSET/CLR */ +#define CC 0x80000000 +#define C0 0x1 +#define C1 0x2 +#define C2 0x4 +#define C3 0x8 +#define C4 0x10 +#define C5 0x20 +#define C_ALL (CC | C0 | C1 | C2 | C3 | C4 | C5) +#define TYPE_MASK 0xFFFF +#define NUM_L1_CTRS 6 +#define NUM_AMU_CTRS 2 + +#include +#include +#include + +DECLARE_PER_CPU(u32, cntenset_val); +DECLARE_PER_CPU(unsigned long, previous_ccnt); +DECLARE_PER_CPU(unsigned long[NUM_L1_CTRS], previous_l1_cnts); +DECLARE_PER_CPU(unsigned long[NUM_AMU_CTRS], previous_amu_cnts); +TRACE_EVENT(sched_switch_with_ctrs, + + TP_PROTO(pid_t prev, pid_t next), + + TP_ARGS(prev, next), + + TP_STRUCT__entry( + __field(pid_t, old_pid) + __field(pid_t, new_pid) + __field(unsigned long, cctr) + __field(unsigned long, ctr0) + __field(unsigned long, ctr1) + __field(unsigned long, ctr2) + __field(unsigned long, ctr3) + __field(unsigned long, ctr4) + __field(unsigned long, ctr5) + __field(unsigned long, amu0) + __field(unsigned long, amu1) + ), + + TP_fast_assign( + u32 cpu = smp_processor_id(); + u32 i; + u32 cnten_val; + unsigned long total_ccnt = 0; + unsigned long total_cnt = 0; + unsigned long amu_cnt = 0; + unsigned long delta_l1_cnts[NUM_L1_CTRS] = {0}; + unsigned long delta_amu_cnts[NUM_AMU_CTRS] = {0}; + + __entry->old_pid = prev; + __entry->new_pid = next; + + cnten_val = per_cpu(cntenset_val, cpu); + + if (cnten_val & CC) { + /* Read value */ + total_ccnt = read_sysreg(pmccntr_el0); + __entry->cctr = total_ccnt - + per_cpu(previous_ccnt, cpu); + per_cpu(previous_ccnt, cpu) = total_ccnt; + } + for (i = 0; i < NUM_L1_CTRS; i++) { + if (cnten_val & (1 << i)) { + /* Select */ + write_sysreg(i, pmselr_el0); + isb(); + /* Read value */ + total_cnt = read_sysreg(pmxevcntr_el0); + delta_l1_cnts[i] = total_cnt - + per_cpu(previous_l1_cnts[i], cpu); + per_cpu(previous_l1_cnts[i], cpu) = + total_cnt; + } else + delta_l1_cnts[i] = 0; + } + + if (IS_ENABLED(CONFIG_ARM64_AMU_EXTN)) { + amu_cnt = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0); + delta_amu_cnts[0] = amu_cnt - + per_cpu(previous_amu_cnts[0], cpu); + per_cpu(previous_amu_cnts[0], cpu) = amu_cnt; + + amu_cnt = read_sysreg_s(SYS_AMEVCNTR0_INST_RET_EL0); + delta_amu_cnts[1] = amu_cnt - + per_cpu(previous_amu_cnts[1], cpu); + per_cpu(previous_amu_cnts[1], cpu) = amu_cnt; + } + + __entry->ctr0 = delta_l1_cnts[0]; + __entry->ctr1 = delta_l1_cnts[1]; + __entry->ctr2 = delta_l1_cnts[2]; + __entry->ctr3 = delta_l1_cnts[3]; + __entry->ctr4 = delta_l1_cnts[4]; + __entry->ctr5 = delta_l1_cnts[5]; + __entry->amu0 = delta_amu_cnts[0]; + __entry->amu1 = delta_amu_cnts[1]; + ), + + TP_printk("prev_pid=%d, next_pid=%d, CCNTR: %lu, CTR0: %lu, CTR1: %lu, CTR2: %lu, CTR3: %lu, CTR4: %lu, CTR5: %lu, CYC: %lu, INST: %lu", + __entry->old_pid, __entry->new_pid, + __entry->cctr, + __entry->ctr0, __entry->ctr1, + __entry->ctr2, __entry->ctr3, + __entry->ctr4, __entry->ctr5, + __entry->amu0, __entry->amu1) +); + +TRACE_EVENT(sched_switch_ctrs_cfg, + + TP_PROTO(int cpu), + + TP_ARGS(cpu), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, ctr0) + __field(unsigned long, ctr1) + __field(unsigned long, ctr2) + __field(unsigned long, ctr3) + __field(unsigned long, ctr4) + __field(unsigned long, ctr5) + ), + + TP_fast_assign( + u32 i; + u32 cnten_val; + u32 ctr_type[NUM_L1_CTRS] = {0}; + + cnten_val = per_cpu(cntenset_val, cpu); + + for (i = 0; i < NUM_L1_CTRS; i++) { + if (cnten_val & (1 << i)) { + /* Select */ + write_sysreg(i, pmselr_el0); + isb(); + /* Read type */ + ctr_type[i] = read_sysreg(pmxevtyper_el0) + & TYPE_MASK; + } else + ctr_type[i] = 0; + } + + __entry->cpu = cpu; + __entry->ctr0 = ctr_type[0]; + __entry->ctr1 = ctr_type[1]; + __entry->ctr2 = ctr_type[2]; + __entry->ctr3 = ctr_type[3]; + __entry->ctr4 = ctr_type[4]; + __entry->ctr5 = ctr_type[5]; + ), + + TP_printk("cpu=%d CTR0=%lu CTR1=%lu CTR2=%lu CTR3=%lu CTR4=%lu CTR5=%lu", + __entry->cpu, + __entry->ctr0, __entry->ctr1, + __entry->ctr2, __entry->ctr3, + __entry->ctr4, __entry->ctr5) +); + +#endif +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../kernel/sched/walt + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE perf_trace_counters +#include diff --git a/kernel/sched/walt/preemptirq_long.c b/kernel/sched/walt/preemptirq_long.c new file mode 100644 index 000000000000..17b5ea24f049 --- /dev/null +++ b/kernel/sched/walt/preemptirq_long.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#define CREATE_TRACE_POINTS +#include "preemptirq_long.h" + +#define IRQSOFF_SENTINEL 0x0fffDEAD + +static unsigned int sysctl_preemptoff_tracing_threshold_ns = 1000000; +static unsigned int sysctl_irqsoff_tracing_threshold_ns = 5000000; +static unsigned int sysctl_irqsoff_dmesg_output_enabled; +static unsigned int sysctl_irqsoff_crash_sentinel_value; +static unsigned int sysctl_irqsoff_crash_threshold_ns = 10000000; + +static unsigned int half_million = 500000; +static unsigned int one_hundred_million = 100000000; +static unsigned int one_million = 1000000; + +static DEFINE_PER_CPU(u64, irq_disabled_ts); + +/* + * preemption disable tracking require additional context + * to rule out false positives. see the comment in + * test_preempt_disable_long() for more details. + */ +struct preempt_store { + u64 ts; + int pid; + unsigned long ncsw; +}; +static DEFINE_PER_CPU(struct preempt_store, the_ps); + +static void note_irq_disable(void *u1, unsigned long u2, unsigned long u3) +{ + if (is_idle_task(current)) + return; + + /* + * We just have to note down the time stamp here. We + * use stacktrace trigger feature to print the stacktrace. + */ + this_cpu_write(irq_disabled_ts, sched_clock()); +} + +static void test_irq_disable_long(void *u1, unsigned long ip, unsigned long parent_ip) +{ + u64 ts = this_cpu_read(irq_disabled_ts); + + if (!ts) + return; + + this_cpu_write(irq_disabled_ts, 0); + ts = sched_clock() - ts; + + if (ts > sysctl_irqsoff_tracing_threshold_ns) { + trace_irq_disable_long(ts, ip, parent_ip, CALLER_ADDR4, CALLER_ADDR5); + + if (sysctl_irqsoff_dmesg_output_enabled == IRQSOFF_SENTINEL) + printk_deferred("irqs off exceeds thresh delta=%llu C:(%ps<-%ps<-%ps<-%ps)\n", + ts, (void *)CALLER_ADDR2, + (void *)CALLER_ADDR3, + (void *)CALLER_ADDR4, + (void *)CALLER_ADDR5); + } + + if (sysctl_irqsoff_crash_sentinel_value == IRQSOFF_SENTINEL && + ts > sysctl_irqsoff_crash_threshold_ns) { + printk_deferred("delta=%llu(ns) > crash_threshold=%u(ns) Task=%s\n", + ts, sysctl_irqsoff_crash_threshold_ns, + current->comm); + BUG_ON(1); + } +} + +static void note_preempt_disable(void *u1, unsigned long u2, unsigned long u3) +{ + struct preempt_store *ps = &per_cpu(the_ps, raw_smp_processor_id()); + + ps->ts = sched_clock(); + ps->pid = current->pid; + ps->ncsw = current->nvcsw + current->nivcsw; +} + +static void test_preempt_disable_long(void *u1, unsigned long ip, + unsigned long parent_ip) +{ + struct preempt_store *ps = &per_cpu(the_ps, raw_smp_processor_id()); + u64 delta = 0; + + if (!ps->ts) + return; + + /* + * schedule() calls __schedule() with preemption disabled. + * if we had entered idle and exiting idle now, we think + * preemption is disabled the whole time. Detect this by + * checking if the preemption is disabled across the same + * task. There is a possiblity that the same task is scheduled + * after idle. To rule out this possibility, compare the + * context switch count also. + */ + if (ps->pid == current->pid && (ps->ncsw == current->nvcsw + + current->nivcsw)) + delta = sched_clock() - ps->ts; + + ps->ts = 0; + if (delta > sysctl_preemptoff_tracing_threshold_ns) + trace_preempt_disable_long(delta, ip, parent_ip, CALLER_ADDR4, CALLER_ADDR5); +} + +static struct ctl_table preemptirq_long_table[] = { + { + .procname = "preemptoff_tracing_threshold_ns", + .data = &sysctl_preemptoff_tracing_threshold_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "irqsoff_tracing_threshold_ns", + .data = &sysctl_irqsoff_tracing_threshold_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = &half_million, + .extra2 = &one_hundred_million, + }, + { + .procname = "irqsoff_dmesg_output_enabled", + .data = &sysctl_irqsoff_dmesg_output_enabled, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "irqsoff_crash_sentinel_value", + .data = &sysctl_irqsoff_crash_sentinel_value, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "irqsoff_crash_threshold_ns", + .data = &sysctl_irqsoff_crash_threshold_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = &one_million, + .extra2 = &one_hundred_million, + }, + { } +}; + +int preemptirq_long_init(void) +{ + if (!register_sysctl("preemptirq", preemptirq_long_table)) { + pr_err("Fail to register sysctl table\n"); + return -EPERM; + } + + register_trace_android_rvh_irqs_disable(note_irq_disable, NULL); + register_trace_android_rvh_irqs_enable(test_irq_disable_long, NULL); + register_trace_android_rvh_preempt_disable(note_preempt_disable, NULL); + register_trace_android_rvh_preempt_enable(test_preempt_disable_long, + NULL); + + return 0; +} diff --git a/kernel/sched/walt/preemptirq_long.h b/kernel/sched/walt/preemptirq_long.h new file mode 100644 index 000000000000..89dce56d79df --- /dev/null +++ b/kernel/sched/walt/preemptirq_long.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM preemptirq_long + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#if !defined(_TRACE_PREEMPTIRQ_LONG_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_PREEMPTIRQ_LONG_H + +#include + +/* reference preemptirq_template */ +DECLARE_EVENT_CLASS(preemptirq_long_template, + + TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip, + unsigned long pparent_ip, unsigned long ppparent_ip), + + TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip), + + TP_STRUCT__entry( + __field(u64, delta) + __field(unsigned long, caller_offs) + __field(unsigned long, parent_offs) + __field(unsigned long, pparent_offs) + __field(unsigned long, ppparent_offs) + ), + + TP_fast_assign( + __entry->delta = delta; + __entry->caller_offs = ip; + __entry->parent_offs = parent_ip; + __entry->pparent_offs = pparent_ip; + __entry->ppparent_offs = ppparent_ip; + ), + + TP_printk("delta=%llu(ns) caller=%ps <- %ps <- %ps <- %ps", + __entry->delta, __entry->caller_offs, + __entry->parent_offs, __entry->pparent_offs, __entry->ppparent_offs) +); + +DEFINE_EVENT(preemptirq_long_template, irq_disable_long, + TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip, + unsigned long pparent_ip, unsigned long ppparent_ip), + TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip)); + +DEFINE_EVENT(preemptirq_long_template, preempt_disable_long, + TP_PROTO(u64 delta, unsigned long ip, unsigned long parent_ip, + unsigned long pparent_ip, unsigned long ppparent_ip), + TP_ARGS(delta, ip, parent_ip, pparent_ip, ppparent_ip)); + +#endif /* _TRACE_PREEMPTIRQ_LONG_H */ + +/* This part must be outside protection */ +#include diff --git a/kernel/sched/walt/sched_avg.c b/kernel/sched/walt/sched_avg.c new file mode 100644 index 000000000000..c18e5c2234a9 --- /dev/null +++ b/kernel/sched/walt/sched_avg.c @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012, 2015-2021, The Linux Foundation. All rights reserved. + */ + +/* + * Scheduler hook for average runqueue determination + */ +#include +#include +#include +#include +#include + +#include "walt.h" +#include "trace.h" + +static DEFINE_PER_CPU(u64, nr_prod_sum); +static DEFINE_PER_CPU(u64, last_time); +static DEFINE_PER_CPU(u64, nr_big_prod_sum); +static DEFINE_PER_CPU(u64, nr); +static DEFINE_PER_CPU(u64, nr_max); + +static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock); +static s64 last_get_time; + +static DEFINE_PER_CPU(atomic64_t, busy_hyst_end_time) = ATOMIC64_INIT(0); + +static DEFINE_PER_CPU(u64, hyst_time); +static DEFINE_PER_CPU(u64, coloc_hyst_busy); +static DEFINE_PER_CPU(u64, coloc_hyst_time); +static DEFINE_PER_CPU(u64, util_hyst_time); + +#define NR_THRESHOLD_PCT 40 +#define MAX_RTGB_TIME (sysctl_sched_coloc_busy_hyst_max_ms * NSEC_PER_MSEC) + +struct sched_avg_stats stats[WALT_NR_CPUS]; +unsigned int cstats_util_pct[MAX_CLUSTERS]; + +/** + * sched_get_cluster_util_pct + * @return: provide the percentage of this cluter that was used in the + * previous window. + * + * This routine may be called any number of times as needed during + * a window, but will always return the same result until window + * rollover. + */ +unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster) +{ + unsigned int cluster_util_pct = 0; + + if (cluster->id < MAX_CLUSTERS) + cluster_util_pct = cstats_util_pct[cluster->id]; + + return cluster_util_pct; +} + +/** + * sched_get_nr_running_avg + * @return: Average nr_running, iowait and nr_big_tasks value since last poll. + * Returns the avg * 100 to return up to two decimal points + * of accuracy. + * + * Obtains the average nr_running value since the last poll. + * This function may not be called concurrently with itself. + * + * It is assumed that this function is called at most once per window + * rollover. + */ +struct sched_avg_stats *sched_get_nr_running_avg(void) +{ + int cpu; + u64 curr_time = sched_clock(); + u64 period = curr_time - last_get_time; + u64 tmp_nr, tmp_misfit; + bool any_hyst_time = false; + struct walt_sched_cluster *cluster; + + if (!period) + goto done; + + /* read and reset nr_running counts */ + for_each_possible_cpu(cpu) { + unsigned long flags; + u64 diff; + + spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags); + curr_time = sched_clock(); + diff = curr_time - per_cpu(last_time, cpu); + BUG_ON((s64)diff < 0); + + tmp_nr = per_cpu(nr_prod_sum, cpu); + tmp_nr += per_cpu(nr, cpu) * diff; + tmp_nr = div64_u64((tmp_nr * 100), period); + + tmp_misfit = per_cpu(nr_big_prod_sum, cpu); + tmp_misfit += walt_big_tasks(cpu) * diff; + tmp_misfit = div64_u64((tmp_misfit * 100), period); + + /* + * NR_THRESHOLD_PCT is to make sure that the task ran + * at least 85% in the last window to compensate any + * over estimating being done. + */ + stats[cpu].nr = (int)div64_u64((tmp_nr + NR_THRESHOLD_PCT), + 100); + stats[cpu].nr_misfit = (int)div64_u64((tmp_misfit + + NR_THRESHOLD_PCT), 100); + stats[cpu].nr_max = per_cpu(nr_max, cpu); + stats[cpu].nr_scaled = tmp_nr; + + trace_sched_get_nr_running_avg(cpu, stats[cpu].nr, + stats[cpu].nr_misfit, stats[cpu].nr_max, + stats[cpu].nr_scaled); + + per_cpu(last_time, cpu) = curr_time; + per_cpu(nr_prod_sum, cpu) = 0; + per_cpu(nr_big_prod_sum, cpu) = 0; + per_cpu(nr_max, cpu) = per_cpu(nr, cpu); + + spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); + } + + /* collect cluster load stats */ + for_each_sched_cluster(cluster) { + unsigned int num_cpus = cpumask_weight(&cluster->cpus); + unsigned int sum_util_pct = 0; + + /* load is already scaled, see freq_policy_load/prev_runnable_sum */ + for_each_cpu(cpu, &cluster->cpus) { + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + /* compute the % this cpu's utilization of the cpu capacity, + * and sum it across all cpus + */ + sum_util_pct += + (wrq->util * 100) / arch_scale_cpu_capacity(cpu); + } + + /* calculate the averge per-cpu utilization */ + cstats_util_pct[cluster->id] = sum_util_pct / num_cpus; + } + + for_each_possible_cpu(cpu) { + if (per_cpu(coloc_hyst_time, cpu)) { + any_hyst_time = true; + break; + } + } + if (any_hyst_time && get_rtgb_active_time() >= MAX_RTGB_TIME) + sched_update_hyst_times(); + + last_get_time = curr_time; + +done: + return &stats[0]; +} +EXPORT_SYMBOL(sched_get_nr_running_avg); + +void sched_update_hyst_times(void) +{ + bool rtgb_active; + int cpu; + unsigned long cpu_cap, coloc_busy_pct; + + rtgb_active = is_rtgb_active() && (sched_boost_type != CONSERVATIVE_BOOST) + && (get_rtgb_active_time() < MAX_RTGB_TIME); + + for_each_possible_cpu(cpu) { + cpu_cap = arch_scale_cpu_capacity(cpu); + coloc_busy_pct = sysctl_sched_coloc_busy_hyst_cpu_busy_pct[cpu]; + per_cpu(hyst_time, cpu) = (BIT(cpu) + & sysctl_sched_busy_hyst_enable_cpus) ? + sysctl_sched_busy_hyst : 0; + per_cpu(coloc_hyst_time, cpu) = ((BIT(cpu) + & sysctl_sched_coloc_busy_hyst_enable_cpus) + && rtgb_active) ? + sysctl_sched_coloc_busy_hyst_cpu[cpu] : 0; + per_cpu(coloc_hyst_busy, cpu) = mult_frac(cpu_cap, + coloc_busy_pct, 100); + per_cpu(util_hyst_time, cpu) = (BIT(cpu) + & sysctl_sched_util_busy_hyst_enable_cpus) ? + sysctl_sched_util_busy_hyst_cpu[cpu] : 0; + } +} + +#define BUSY_NR_RUN 3 +#define BUSY_LOAD_FACTOR 10 +static inline void update_busy_hyst_end_time(int cpu, int enq, + unsigned long prev_nr_run, u64 curr_time) +{ + bool nr_run_trigger = false; + bool load_trigger = false, coloc_load_trigger = false; + u64 agg_hyst_time, total_util = 0; + bool util_load_trigger = false; + int i; + bool hyst_trigger, coloc_trigger; + bool dequeue = (enq < 0); + + if (!per_cpu(hyst_time, cpu) && !per_cpu(coloc_hyst_time, cpu) && + !per_cpu(util_hyst_time, cpu)) + return; + + if (prev_nr_run >= BUSY_NR_RUN && per_cpu(nr, cpu) < BUSY_NR_RUN) + nr_run_trigger = true; + + if (dequeue && (cpu_util(cpu) * BUSY_LOAD_FACTOR) > + capacity_orig_of(cpu)) + load_trigger = true; + + if (dequeue && cpu_util(cpu) > per_cpu(coloc_hyst_busy, cpu)) + coloc_load_trigger = true; + + if (dequeue) { + for_each_possible_cpu(i) { + total_util += cpu_util(i); + if (total_util >= sysctl_sched_util_busy_hyst_cpu_util[cpu]) { + util_load_trigger = true; + break; + } + } + } + + coloc_trigger = nr_run_trigger || coloc_load_trigger; +#if IS_ENABLED(CONFIG_SCHED_CONSERVATIVE_BOOST_LPM_BIAS) + hyst_trigger = nr_run_trigger || load_trigger || (sched_boost_type == CONSERVATIVE_BOOST); +#else + hyst_trigger = nr_run_trigger || load_trigger; +#endif + + agg_hyst_time = max(max(hyst_trigger ? per_cpu(hyst_time, cpu) : 0, + coloc_trigger ? per_cpu(coloc_hyst_time, cpu) : 0), + util_load_trigger ? per_cpu(util_hyst_time, cpu) : 0); + + if (agg_hyst_time) { + atomic64_set(&per_cpu(busy_hyst_end_time, cpu), + curr_time + agg_hyst_time); + trace_sched_busy_hyst_time(cpu, agg_hyst_time, prev_nr_run, + cpu_util(cpu), per_cpu(hyst_time, cpu), + per_cpu(coloc_hyst_time, cpu), + per_cpu(util_hyst_time, cpu)); + } +} + +int sched_busy_hyst_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret; + + if (table->maxlen > (sizeof(unsigned int) * num_possible_cpus())) + table->maxlen = sizeof(unsigned int) * num_possible_cpus(); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (!ret && write) + sched_update_hyst_times(); + + return ret; +} + +/** + * sched_update_nr_prod + * @cpu: The core id of the nr running driver. + * @enq: enqueue/dequeue/misfit happening on this CPU. + * @return: N/A + * + * Update average with latest nr_running value for CPU + */ +void sched_update_nr_prod(int cpu, int enq) +{ + u64 diff; + u64 curr_time; + unsigned long flags, nr_running; + + spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags); + nr_running = per_cpu(nr, cpu); + curr_time = sched_clock(); + diff = curr_time - per_cpu(last_time, cpu); + BUG_ON((s64)diff < 0); + per_cpu(last_time, cpu) = curr_time; + per_cpu(nr, cpu) = cpu_rq(cpu)->nr_running; + + if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu)) + per_cpu(nr_max, cpu) = per_cpu(nr, cpu); + + /* Don't update hyst time for misfit tasks */ + if (enq) + update_busy_hyst_end_time(cpu, enq, nr_running, curr_time); + + per_cpu(nr_prod_sum, cpu) += nr_running * diff; + per_cpu(nr_big_prod_sum, cpu) += walt_big_tasks(cpu) * diff; + spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); +} + +/* + * Returns the CPU utilization % in the last window. + */ +unsigned int sched_get_cpu_util_pct(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + u64 util; + unsigned long capacity, flags; + unsigned int busy; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + raw_spin_lock_irqsave(&rq->__lock, flags); + + capacity = capacity_orig_of(cpu); + + util = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; + util = scale_time_to_util(util); + raw_spin_unlock_irqrestore(&rq->__lock, flags); + + util = (util >= capacity) ? capacity : util; + busy = div64_ul((util * 100), capacity); + return busy; +} + +int sched_lpm_disallowed_time(int cpu, u64 *timeout) +{ + u64 now = sched_clock(); + u64 bias_end_time = atomic64_read(&per_cpu(busy_hyst_end_time, cpu)); + + if (unlikely(is_reserved(cpu))) { + *timeout = 10 * NSEC_PER_MSEC; + return 0; /* shallowest c-state */ + } + + if (now < bias_end_time) { + *timeout = bias_end_time - now; + return 0; /* shallowest c-state */ + } + + return INT_MAX; /* don't care */ +} +EXPORT_SYMBOL(sched_lpm_disallowed_time); diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c new file mode 100644 index 000000000000..310956da94e0 --- /dev/null +++ b/kernel/sched/walt/sysctl.c @@ -0,0 +1,979 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include + +#include "walt.h" +#include "trace.h" + +static int neg_three = -3; +static int three = 3; +static int two_hundred_fifty_five = 255; +static unsigned int ns_per_sec = NSEC_PER_SEC; +static unsigned int one_hundred_thousand = 100000; +static unsigned int two_hundred_million = 200000000; +static int __maybe_unused two = 2; +static int __maybe_unused four = 4; +static int one_hundred = 100; +static int one_thousand = 1000; +static int two_thousand = 2000; + +/* + * CFS task prio range is [100 ... 139] + * 120 is the default prio. + * RTG boost range is [100 ... 119] because giving + * boost for [120 .. 139] does not make sense. + * 99 means disabled and it is the default value. + */ +static unsigned int min_cfs_boost_prio = 99; +static unsigned int max_cfs_boost_prio = 119; + +unsigned int sysctl_sched_capacity_margin_up_pct[MAX_MARGIN_LEVELS]; +unsigned int sysctl_sched_capacity_margin_dn_pct[MAX_MARGIN_LEVELS]; +unsigned int sysctl_sched_busy_hyst_enable_cpus; +unsigned int sysctl_sched_busy_hyst; +unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus; +unsigned int sysctl_sched_coloc_busy_hyst_cpu[WALT_NR_CPUS]; +unsigned int sysctl_sched_coloc_busy_hyst_max_ms; +unsigned int sysctl_sched_coloc_busy_hyst_cpu_busy_pct[WALT_NR_CPUS]; +unsigned int sysctl_sched_util_busy_hyst_enable_cpus; +unsigned int sysctl_sched_util_busy_hyst_cpu[WALT_NR_CPUS]; +unsigned int sysctl_sched_util_busy_hyst_cpu_util[WALT_NR_CPUS]; +unsigned int sysctl_sched_boost; +unsigned int sysctl_sched_wake_up_idle[2]; +unsigned int sysctl_input_boost_ms; +unsigned int sysctl_input_boost_freq[8]; +unsigned int sysctl_sched_boost_on_input; + +/* sysctl nodes accesed by other files */ +unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; +unsigned int __read_mostly sysctl_sched_group_downmigrate_pct; +unsigned int __read_mostly sysctl_sched_group_upmigrate_pct; +unsigned int __read_mostly sysctl_sched_window_stats_policy; +unsigned int sysctl_sched_ravg_window_nr_ticks; +unsigned int sysctl_sched_walt_rotate_big_tasks; +unsigned int sysctl_sched_task_unfilter_period; +unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; +unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ +unsigned int sysctl_sched_conservative_pl; +unsigned int sysctl_sched_min_task_util_for_boost = 51; +unsigned int sysctl_sched_min_task_util_for_uclamp = 51; +unsigned int sysctl_sched_min_task_util_for_colocation = 35; +unsigned int sysctl_sched_many_wakeup_threshold = WALT_MANY_WAKEUP_DEFAULT; +const int sched_user_hint_max = 1000; +unsigned int sysctl_walt_rtg_cfs_boost_prio = 99; /* disabled by default */ +unsigned int sysctl_sched_sync_hint_enable = 1; +unsigned int sysctl_panic_on_walt_bug = walt_debug_initial_values(); +unsigned int sysctl_sched_suppress_region2; +unsigned int sysctl_sched_skip_sp_newly_idle_lb = 1; +unsigned int sysctl_sched_hyst_min_coloc_ns = 80000000; +unsigned int sysctl_sched_asymcap_boost; +unsigned int sysctl_sched_long_running_rt_task_ms; +unsigned int sysctl_sched_idle_enough; +unsigned int sysctl_sched_cluster_util_thres_pct; + +/* range is [1 .. INT_MAX] */ +static int sysctl_task_read_pid = 1; + +static int walt_proc_group_thresholds_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + static DEFINE_MUTEX(mutex); + struct rq *rq = cpu_rq(cpumask_first(cpu_possible_mask)); + unsigned long flags; + + if (unlikely(num_sched_clusters <= 0)) + return -EPERM; + + mutex_lock(&mutex); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write) { + mutex_unlock(&mutex); + return ret; + } + + /* + * The load scale factor update happens with all + * rqs locked. so acquiring 1 CPU rq lock and + * updating the thresholds is sufficient for + * an atomic update. + */ + raw_spin_lock_irqsave(&rq->__lock, flags); + walt_update_group_thresholds(); + raw_spin_unlock_irqrestore(&rq->__lock, flags); + + mutex_unlock(&mutex); + + return ret; +} + +static int walt_proc_user_hint_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + unsigned int old_value; + static DEFINE_MUTEX(mutex); + + mutex_lock(&mutex); + + old_value = sysctl_sched_user_hint; + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write || (old_value == sysctl_sched_user_hint)) + goto unlock; + + sched_user_hint_reset_time = jiffies + HZ; + walt_irq_work_queue(&walt_migration_irq_work); + +unlock: + mutex_unlock(&mutex); + return ret; +} + +static int sched_ravg_window_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret = -EPERM; + static DEFINE_MUTEX(mutex); + int val; + + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(val), + .mode = table->mode, + }; + + mutex_lock(&mutex); + + if (write && HZ != 250) + goto unlock; + + val = sysctl_sched_ravg_window_nr_ticks; + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + if (ret || !write || (val == sysctl_sched_ravg_window_nr_ticks)) + goto unlock; + + if (val != 2 && val != 3 && val != 4 && val != 5 && val != 8) { + ret = -EINVAL; + goto unlock; + } + + sysctl_sched_ravg_window_nr_ticks = val; + sched_window_nr_ticks_change(); + +unlock: + mutex_unlock(&mutex); + return ret; +} + +static DEFINE_MUTEX(sysctl_pid_mutex); +static int sched_task_read_pid_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + + mutex_lock(&sysctl_pid_mutex); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + mutex_unlock(&sysctl_pid_mutex); + + return ret; +} + +enum { + TASK_BEGIN = 0, + WAKE_UP_IDLE, + INIT_TASK_LOAD, + GROUP_ID, + PER_TASK_BOOST, + PER_TASK_BOOST_PERIOD_MS, + LOW_LATENCY, + PIPELINE, + LOAD_BOOST, +}; + +static int sched_task_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret, param; + struct task_struct *task; + int pid_and_val[2] = {-1, -1}; + int val; + struct walt_task_struct *wts; + + struct ctl_table tmp = { + .data = &pid_and_val, + .maxlen = sizeof(pid_and_val), + .mode = table->mode, + }; + + mutex_lock(&sysctl_pid_mutex); + + if (!write) { + task = get_pid_task(find_vpid(sysctl_task_read_pid), + PIDTYPE_PID); + if (!task) { + ret = -ENOENT; + goto unlock_mutex; + } + wts = (struct walt_task_struct *) task->android_vendor_data1; + pid_and_val[0] = sysctl_task_read_pid; + param = (unsigned long)table->data; + switch (param) { + case WAKE_UP_IDLE: + pid_and_val[1] = wts->wake_up_idle; + break; + case INIT_TASK_LOAD: + pid_and_val[1] = wts->init_load_pct; + break; + case GROUP_ID: + pid_and_val[1] = sched_get_group_id(task); + break; + case PER_TASK_BOOST: + pid_and_val[1] = wts->boost; + break; + case PER_TASK_BOOST_PERIOD_MS: + pid_and_val[1] = + div64_ul(wts->boost_period, + 1000000UL); + break; + case LOW_LATENCY: + pid_and_val[1] = wts->low_latency & + WALT_LOW_LATENCY_PROCFS; + break; + case PIPELINE: + pid_and_val[1] = wts->low_latency & + WALT_LOW_LATENCY_PIPELINE; + break; + case LOAD_BOOST: + pid_and_val[1] = wts->load_boost; + break; + default: + ret = -EINVAL; + goto put_task; + } + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + goto put_task; + } + + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + if (ret) + goto unlock_mutex; + + if (pid_and_val[0] <= 0) { + ret = -ENOENT; + goto unlock_mutex; + } + + /* parsed the values successfully in pid_and_val[] array */ + task = get_pid_task(find_vpid(pid_and_val[0]), PIDTYPE_PID); + if (!task) { + ret = -ENOENT; + goto unlock_mutex; + } + wts = (struct walt_task_struct *) task->android_vendor_data1; + param = (unsigned long)table->data; + val = pid_and_val[1]; + if (param != LOAD_BOOST && val < 0) { + ret = -EINVAL; + goto put_task; + } + switch (param) { + case WAKE_UP_IDLE: + wts->wake_up_idle = val; + break; + case INIT_TASK_LOAD: + if (pid_and_val[1] < 0 || pid_and_val[1] > 100) { + ret = -EINVAL; + goto put_task; + } + wts->init_load_pct = val; + break; + case GROUP_ID: + ret = sched_set_group_id(task, val); + break; + case PER_TASK_BOOST: + if (val < TASK_BOOST_NONE || val >= TASK_BOOST_END) { + ret = -EINVAL; + goto put_task; + } + wts->boost = val; + if (val == 0) + wts->boost_period = 0; + break; + case PER_TASK_BOOST_PERIOD_MS: + if (wts->boost == 0 && val) { + /* setting boost period w/o boost is invalid */ + ret = -EINVAL; + goto put_task; + } + wts->boost_period = (u64)val * 1000 * 1000; + wts->boost_expires = sched_clock() + wts->boost_period; + break; + case LOW_LATENCY: + if (val) + wts->low_latency |= WALT_LOW_LATENCY_PROCFS; + else + wts->low_latency &= ~WALT_LOW_LATENCY_PROCFS; + break; + case PIPELINE: + if (val) + wts->low_latency |= WALT_LOW_LATENCY_PIPELINE; + else + wts->low_latency &= ~WALT_LOW_LATENCY_PIPELINE; + break; + case LOAD_BOOST: + if (pid_and_val[1] < -90 || pid_and_val[1] > 90) { + ret = -EINVAL; + goto put_task; + } + wts->load_boost = val; + if (val) + wts->boosted_task_load = mult_frac((int64_t)1024, (int64_t)val, 100); + else + wts->boosted_task_load = 0; + break; + default: + ret = -EINVAL; + } + + trace_sched_task_handler(task, param, val, CALLER_ADDR0, CALLER_ADDR1, + CALLER_ADDR2, CALLER_ADDR3, CALLER_ADDR4, CALLER_ADDR5); +put_task: + put_task_struct(task); +unlock_mutex: + mutex_unlock(&sysctl_pid_mutex); + + return ret; +} + +#ifdef CONFIG_PROC_SYSCTL +static void sched_update_updown_migrate_values(bool up) +{ + int i = 0, cpu; + struct walt_sched_cluster *cluster; + int cap_margin_levels = num_sched_clusters - 1; + + if (cap_margin_levels > 1) { + /* + * No need to worry about CPUs in last cluster + * if there are more than 2 clusters in the system + */ + for_each_sched_cluster(cluster) { + for_each_cpu(cpu, &cluster->cpus) { + if (up) + sched_capacity_margin_up[cpu] = + SCHED_FIXEDPOINT_SCALE * 100 / + sysctl_sched_capacity_margin_up_pct[i]; + else + sched_capacity_margin_down[cpu] = + SCHED_FIXEDPOINT_SCALE * 100 / + sysctl_sched_capacity_margin_dn_pct[i]; + } + + if (++i >= cap_margin_levels) + break; + } + } else { + for_each_possible_cpu(cpu) { + if (up) + sched_capacity_margin_up[cpu] = + + SCHED_FIXEDPOINT_SCALE * 100 / + sysctl_sched_capacity_margin_up_pct[0]; + else + sched_capacity_margin_down[cpu] = + sysctl_sched_capacity_margin_dn_pct[0]; + } + } +} + +int sched_updown_migrate_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret, i; + unsigned int *data = (unsigned int *)table->data; + static DEFINE_MUTEX(mutex); + int cap_margin_levels = num_sched_clusters ? num_sched_clusters - 1 : 0; + int val[MAX_MARGIN_LEVELS]; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(int) * cap_margin_levels, + .mode = table->mode, + }; + + if (cap_margin_levels <= 0) + return -EINVAL; + + mutex_lock(&mutex); + + if (!write) { + ret = proc_dointvec(table, write, buffer, lenp, ppos); + goto unlock_mutex; + } + + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + if (ret) + goto unlock_mutex; + + /* check if valid pct values are passed in */ + for (i = 0; i < cap_margin_levels; i++) { + if (val[i] <= 0 || val[i] > 100) { + ret = -EINVAL; + goto unlock_mutex; + } + } + + /* check up pct is greater than dn pct */ + if (data == &sysctl_sched_capacity_margin_up_pct[0]) { + for (i = 0; i < cap_margin_levels; i++) { + if (val[i] < sysctl_sched_capacity_margin_dn_pct[i]) { + ret = -EINVAL; + goto unlock_mutex; + } + } + } else { + for (i = 0; i < cap_margin_levels; i++) { + if (sysctl_sched_capacity_margin_up_pct[i] < val[i]) { + ret = -EINVAL; + goto unlock_mutex; + } + } + } + + /* all things checkout update the value */ + for (i = 0; i < cap_margin_levels; i++) + data[i] = val[i]; + + /* update individual cpu thresholds */ + sched_update_updown_migrate_values(data == &sysctl_sched_capacity_margin_up_pct[0]); + +unlock_mutex: + mutex_unlock(&mutex); + + return ret; +} +#endif /* CONFIG_PROC_SYSCTL */ + +struct ctl_table input_boost_sysctls[] = { + { + .procname = "input_boost_ms", + .data = &sysctl_input_boost_ms, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred_thousand, + }, + { + .procname = "input_boost_freq", + .data = &sysctl_input_boost_freq, + .maxlen = sizeof(unsigned int) * 8, + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sched_boost_on_input", + .data = &sysctl_sched_boost_on_input, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { } +}; + +struct ctl_table walt_table[] = { + { + .procname = "sched_user_hint", + .data = &sysctl_sched_user_hint, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_user_hint_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = (void *)&sched_user_hint_max, + }, + { + .procname = "sched_window_stats_policy", + .data = &sysctl_sched_window_stats_policy, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &four, + }, + { + .procname = "sched_group_upmigrate", + .data = &sysctl_sched_group_upmigrate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_group_thresholds_handler, + .extra1 = &sysctl_sched_group_downmigrate_pct, + }, + { + .procname = "sched_group_downmigrate", + .data = &sysctl_sched_group_downmigrate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_group_thresholds_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &sysctl_sched_group_upmigrate_pct, + }, + { + .procname = "sched_boost", + .data = &sysctl_sched_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_boost_handler, + .extra1 = &neg_three, + .extra2 = &three, + }, + { + .procname = "sched_conservative_pl", + .data = &sysctl_sched_conservative_pl, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_many_wakeup_threshold", + .data = &sysctl_sched_many_wakeup_threshold, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &two, + .extra2 = &one_thousand, + }, + { + .procname = "sched_walt_rotate_big_tasks", + .data = &sysctl_sched_walt_rotate_big_tasks, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_min_task_util_for_boost", + .data = &sysctl_sched_min_task_util_for_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_min_task_util_for_uclamp", + .data = &sysctl_sched_min_task_util_for_uclamp, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_min_task_util_for_colocation", + .data = &sysctl_sched_min_task_util_for_colocation, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_asym_cap_sibling_freq_match_pct", + .data = &sysctl_sched_asym_cap_sibling_freq_match_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &one_hundred, + }, + { + .procname = "sched_coloc_downmigrate_ns", + .data = &sysctl_sched_coloc_downmigrate_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + }, + { + .procname = "sched_task_unfilter_period", + .data = &sysctl_sched_task_unfilter_period, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &two_hundred_million, + }, + { + .procname = "sched_busy_hysteresis_enable_cpus", + .data = &sysctl_sched_busy_hyst_enable_cpus, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_busy_hyst_ns", + .data = &sysctl_sched_busy_hyst, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &ns_per_sec, + }, + { + .procname = "sched_coloc_busy_hysteresis_enable_cpus", + .data = &sysctl_sched_coloc_busy_hyst_enable_cpus, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_coloc_busy_hyst_cpu_ns", + .data = &sysctl_sched_coloc_busy_hyst_cpu, + .maxlen = sizeof(unsigned int) * WALT_NR_CPUS, + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &ns_per_sec, + }, + { + .procname = "sched_coloc_busy_hyst_max_ms", + .data = &sysctl_sched_coloc_busy_hyst_max_ms, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred_thousand, + }, + { + .procname = "sched_coloc_busy_hyst_cpu_busy_pct", + .data = &sysctl_sched_coloc_busy_hyst_cpu_busy_pct, + .maxlen = sizeof(unsigned int) * WALT_NR_CPUS, + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred, + }, + { + .procname = "sched_util_busy_hysteresis_enable_cpus", + .data = &sysctl_sched_util_busy_hyst_enable_cpus, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_util_busy_hyst_cpu_ns", + .data = &sysctl_sched_util_busy_hyst_cpu, + .maxlen = sizeof(unsigned int) * WALT_NR_CPUS, + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &ns_per_sec, + }, + { + .procname = "sched_util_busy_hyst_cpu_util", + .data = &sysctl_sched_util_busy_hyst_cpu_util, + .maxlen = sizeof(unsigned int) * WALT_NR_CPUS, + .mode = 0644, + .proc_handler = sched_busy_hyst_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_ravg_window_nr_ticks", + .data = &sysctl_sched_ravg_window_nr_ticks, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_ravg_window_handler, + }, + { + .procname = "sched_upmigrate", + .data = &sysctl_sched_capacity_margin_up_pct, + .maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS, + .mode = 0644, + .proc_handler = sched_updown_migrate_handler, + }, + { + .procname = "sched_downmigrate", + .data = &sysctl_sched_capacity_margin_dn_pct, + .maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS, + .mode = 0644, + .proc_handler = sched_updown_migrate_handler, + }, + { + .procname = "walt_rtg_cfs_boost_prio", + .data = &sysctl_walt_rtg_cfs_boost_prio, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &min_cfs_boost_prio, + .extra2 = &max_cfs_boost_prio, + }, + { + .procname = "walt_low_latency_task_threshold", + .data = &sysctl_walt_low_latency_task_threshold, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_force_lb_enable", + .data = &sysctl_sched_force_lb_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_sync_hint_enable", + .data = &sysctl_sched_sync_hint_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_suppress_region2", + .data = &sysctl_sched_suppress_region2, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_skip_sp_newly_idle_lb", + .data = &sysctl_sched_skip_sp_newly_idle_lb, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_hyst_min_coloc_ns", + .data = &sysctl_sched_hyst_min_coloc_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + }, + { + .procname = "panic_on_walt_bug", + .data = &sysctl_panic_on_walt_bug, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sched_lib_name", + .data = sched_lib_name, + .maxlen = LIB_PATH_LENGTH, + .mode = 0644, + .proc_handler = proc_dostring, + }, + { + .procname = "sched_lib_mask_force", + .data = &sched_lib_mask_force, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "input_boost", + .mode = 0555, + .child = input_boost_sysctls, + }, + { + .procname = "sched_wake_up_idle", + .data = (int *) WAKE_UP_IDLE, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_init_task_load", + .data = (int *) INIT_TASK_LOAD, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_group_id", + .data = (int *) GROUP_ID, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_per_task_boost", + .data = (int *) PER_TASK_BOOST, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_per_task_boost_period_ms", + .data = (int *) PER_TASK_BOOST_PERIOD_MS, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_low_latency", + .data = (int *) LOW_LATENCY, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_pipeline", + .data = (int *) PIPELINE, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "task_load_boost", + .data = (int *) LOAD_BOOST, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, + { + .procname = "sched_task_read_pid", + .data = &sysctl_task_read_pid, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = sched_task_read_pid_handler, + .extra1 = SYSCTL_ONE, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sched_enable_tp", + .data = &sysctl_sched_dynamic_tp_enable, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_dynamic_tp_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_asymcap_boost", + .data = &sysctl_sched_asymcap_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_cluster_util_thres_pct", + .data = &sysctl_sched_cluster_util_thres_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sched_idle_enough", + .data = &sysctl_sched_idle_enough, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "sched_long_running_rt_task_ms", + .data = &sysctl_sched_long_running_rt_task_ms, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_long_running_rt_task_ms_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_thousand, + }, + { } +}; + +struct ctl_table walt_base_table[] = { + { + .procname = "walt", + .mode = 0555, + .child = walt_table, + }, + { }, +}; + +void walt_tunables(void) +{ + int i; + + for (i = 0; i < MAX_MARGIN_LEVELS; i++) { + sysctl_sched_capacity_margin_up_pct[i] = 95; /* ~5% margin */ + sysctl_sched_capacity_margin_dn_pct[i] = 85; /* ~15% margin */ + } + + sysctl_sched_group_upmigrate_pct = 100; + + sysctl_sched_group_downmigrate_pct = 95; + + sysctl_sched_asym_cap_sibling_freq_match_pct = 100; + + sysctl_sched_task_unfilter_period = 100000000; + + sysctl_sched_window_stats_policy = WINDOW_STATS_MAX_RECENT_AVG; + + sysctl_sched_ravg_window_nr_ticks = (HZ / NR_WINDOWS_PER_SEC); + + sched_load_granule = DEFAULT_SCHED_RAVG_WINDOW / NUM_LOAD_INDICES; + + for (i = 0; i < WALT_NR_CPUS; i++) { + sysctl_sched_coloc_busy_hyst_cpu[i] = 39000000; + sysctl_sched_coloc_busy_hyst_cpu_busy_pct[i] = 10; + sysctl_sched_util_busy_hyst_cpu[i] = 5000000; + sysctl_sched_util_busy_hyst_cpu_util[i] = 15; + } + + sysctl_sched_coloc_busy_hyst_enable_cpus = 112; + + sysctl_sched_util_busy_hyst_enable_cpus = 255; + + sysctl_sched_coloc_busy_hyst_max_ms = 5000; + + sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; + + sysctl_input_boost_ms = 40; + + for (i = 0; i < 8; i++) + sysctl_input_boost_freq[i] = 0; +} diff --git a/kernel/sched/walt/trace.c b/kernel/sched/walt/trace.c new file mode 100644 index 000000000000..cf9c34077f2c --- /dev/null +++ b/kernel/sched/walt/trace.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + */ + +#include "walt.h" + +static inline void __window_data(u32 *dst, u32 *src) +{ + if (src) + memcpy(dst, src, nr_cpu_ids * sizeof(u32)); + else + memset(dst, 0, nr_cpu_ids * sizeof(u32)); +} + +struct trace_seq; +const char *__window_print(struct trace_seq *p, const u32 *buf, int buf_len) +{ + int i; + const char *ret = p->buffer + seq_buf_used(&p->seq); + + for (i = 0; i < buf_len; i++) + trace_seq_printf(p, "%u ", buf[i]); + + trace_seq_putc(p, 0); + + return ret; +} + +static inline s64 __rq_update_sum(struct rq *rq, bool curr, bool new) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (curr) + if (new) + return wrq->nt_curr_runnable_sum; + else + return wrq->curr_runnable_sum; + else + if (new) + return wrq->nt_prev_runnable_sum; + else + return wrq->prev_runnable_sum; +} + +static inline s64 __grp_update_sum(struct rq *rq, bool curr, bool new) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (curr) + if (new) + return wrq->grp_time.nt_curr_runnable_sum; + else + return wrq->grp_time.curr_runnable_sum; + else + if (new) + return wrq->grp_time.nt_prev_runnable_sum; + else + return wrq->grp_time.prev_runnable_sum; +} + +static inline s64 +__get_update_sum(struct rq *rq, enum migrate_types migrate_type, + bool src, bool new, bool curr) +{ + switch (migrate_type) { + case RQ_TO_GROUP: + if (src) + return __rq_update_sum(rq, curr, new); + else + return __grp_update_sum(rq, curr, new); + case GROUP_TO_RQ: + if (src) + return __grp_update_sum(rq, curr, new); + else + return __rq_update_sum(rq, curr, new); + default: + WARN_ON_ONCE(1); + return -EINVAL; + } +} + +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h new file mode 100644 index 000000000000..d260c63a9609 --- /dev/null +++ b/kernel/sched/walt/trace.h @@ -0,0 +1,1484 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM schedwalt + +#if !defined(_TRACE_WALT_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_WALT_H + +#include + +#include "walt.h" + +struct rq; +struct group_cpu_time; +struct walt_task_struct; +struct walt_rq; +struct walt_related_thread_group; + +extern const char *task_event_names[]; + +TRACE_EVENT(sched_update_pred_demand, + + TP_PROTO(struct task_struct *p, u32 runtime, + unsigned int pred_demand_scaled, int start, + int first, int final, struct walt_task_struct *wts), + + TP_ARGS(p, runtime, pred_demand_scaled, start, first, final, wts), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned int, runtime) + __field(unsigned int, pred_demand_scaled) + __array(u8, bucket, NUM_BUSY_BUCKETS) + __field(int, cpu) + __field(int, start) + __field(int, first) + __field(int, final) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->pred_demand_scaled = pred_demand_scaled; + memcpy(__entry->bucket, wts->busy_buckets, + NUM_BUSY_BUCKETS * sizeof(u8)); + __entry->cpu = task_cpu(p); + __entry->start = start; + __entry->first = first; + __entry->final = final; + ), + + TP_printk("%d (%s): runtime %u cpu %d pred_demand_scaled %u start %d first %d final %d (buckets: %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u)", + __entry->pid, __entry->comm, + __entry->runtime, __entry->cpu, + __entry->pred_demand_scaled, __entry->start, __entry->first, __entry->final, + __entry->bucket[0], __entry->bucket[1], + __entry->bucket[2], __entry->bucket[3], __entry->bucket[4], + __entry->bucket[5], __entry->bucket[6], __entry->bucket[7], + __entry->bucket[8], __entry->bucket[9], __entry->bucket[10], + __entry->bucket[11], __entry->bucket[12], __entry->bucket[13], + __entry->bucket[14], __entry->bucket[15]) +); + +TRACE_EVENT(sched_update_history, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int samples, + enum task_event evt, struct walt_rq *wrq, struct walt_task_struct *wts), + + TP_ARGS(rq, p, runtime, samples, evt, wrq, wts), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned int, runtime) + __field(int, samples) + __field(enum task_event, evt) + __field(unsigned int, demand) + __field(unsigned int, coloc_demand) + __field(unsigned int, pred_demand_scaled) + __array(u32, hist, RAVG_HIST_SIZE) + __array(u16, hist_util, RAVG_HIST_SIZE) + __field(unsigned int, nr_big_tasks) + __field(int, cpu) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->samples = samples; + __entry->evt = evt; + __entry->demand = wts->demand; + __entry->coloc_demand = wts->coloc_demand; + __entry->pred_demand_scaled = wts->pred_demand_scaled; + memcpy(__entry->hist, wts->sum_history, + RAVG_HIST_SIZE * sizeof(u32)); + memcpy(__entry->hist_util, wts->sum_history_util, + RAVG_HIST_SIZE * sizeof(u16)); + __entry->nr_big_tasks = wrq->walt_stats.nr_big_tasks; + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u samples %d event %s demand %u (hist: %u %u %u %u %u) (hist_util: %u %u %u %u %u) coloc_demand %u pred_demand_scaled %u cpu %d nr_big %u", + __entry->pid, __entry->comm, + __entry->runtime, __entry->samples, + task_event_names[__entry->evt], + __entry->demand, + __entry->hist[0], __entry->hist[1], + __entry->hist[2], __entry->hist[3], + __entry->hist[4], + __entry->hist_util[0], __entry->hist_util[1], + __entry->hist_util[2], __entry->hist_util[3], + __entry->hist_util[4], + __entry->coloc_demand, __entry->pred_demand_scaled, + __entry->cpu, __entry->nr_big_tasks) +); + +TRACE_EVENT(sched_get_task_cpu_cycles, + + TP_PROTO(int cpu, int event, u64 cycles, + u64 exec_time, struct task_struct *p), + + TP_ARGS(cpu, event, cycles, exec_time, p), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, event) + __field(u64, cycles) + __field(u64, exec_time) + __field(u32, freq) + __field(u32, legacy_freq) + __field(u32, max_freq) + __field(pid_t, pid) + __array(char, comm, TASK_COMM_LEN) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->event = event; + __entry->cycles = cycles; + __entry->exec_time = exec_time; + __entry->freq = cpu_cycles_to_freq(cycles, exec_time); + __entry->legacy_freq = sched_cpu_legacy_freq(cpu); + __entry->max_freq = cpu_max_freq(cpu); + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + ), + + TP_printk("cpu=%d event=%d cycles=%llu exec_time=%llu freq=%u legacy_freq=%u max_freq=%u task=%d (%s)", + __entry->cpu, __entry->event, __entry->cycles, + __entry->exec_time, __entry->freq, __entry->legacy_freq, + __entry->max_freq, __entry->pid, __entry->comm) +); + +TRACE_EVENT(sched_update_task_ravg, + + TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, + u64 wallclock, u64 irqtime, + struct group_cpu_time *cpu_time, struct walt_rq *wrq, + struct walt_task_struct *wts, u64 walt_irq_work_lastq_ws), + + TP_ARGS(p, rq, evt, wallclock, irqtime, cpu_time, wrq, wts, walt_irq_work_lastq_ws), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(pid_t, cur_pid) + __field(unsigned int, cur_freq) + __field(u64, wallclock) + __field(u64, mark_start) + __field(u64, delta_m) + __field(u64, win_start) + __field(u64, delta) + __field(u64, irqtime) + __field(enum task_event, evt) + __field(unsigned int, demand) + __field(unsigned int, coloc_demand) + __field(unsigned int, sum) + __field(int, cpu) + __field(unsigned int, pred_demand_scaled) + __field(u64, rq_cs) + __field(u64, rq_ps) + __field(u64, grp_cs) + __field(u64, grp_ps) + __field(u64, grp_nt_cs) + __field(u64, grp_nt_ps) + __field(u32, curr_window) + __field(u32, prev_window) + __dynamic_array(u32, curr_sum, nr_cpu_ids) + __dynamic_array(u32, prev_sum, nr_cpu_ids) + __field(u64, nt_cs) + __field(u64, nt_ps) + __field(u64, active_time) + __field(u32, curr_top) + __field(u32, prev_top) + __field(u64, walt_irq_work_lastq_ws) + ), + + TP_fast_assign( + __entry->wallclock = wallclock; + __entry->win_start = wrq->window_start; + __entry->delta = (wallclock - wrq->window_start); + __entry->evt = evt; + __entry->cpu = rq->cpu; + __entry->cur_pid = rq->curr->pid; + __entry->cur_freq = wrq->task_exec_scale; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->mark_start = wts->mark_start; + __entry->delta_m = (wallclock - wts->mark_start); + __entry->demand = wts->demand; + __entry->coloc_demand = wts->coloc_demand; + __entry->sum = wts->sum; + __entry->irqtime = irqtime; + __entry->pred_demand_scaled = wts->pred_demand_scaled; + __entry->rq_cs = wrq->curr_runnable_sum; + __entry->rq_ps = wrq->prev_runnable_sum; + __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; + __entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0; + __entry->grp_nt_cs = cpu_time ? + cpu_time->nt_curr_runnable_sum : 0; + __entry->grp_nt_ps = cpu_time ? + cpu_time->nt_prev_runnable_sum : 0; + __entry->curr_window = wts->curr_window; + __entry->prev_window = wts->prev_window; + __window_data(__get_dynamic_array(curr_sum), + wts->curr_window_cpu); + __window_data(__get_dynamic_array(prev_sum), + wts->prev_window_cpu); + __entry->nt_cs = wrq->nt_curr_runnable_sum; + __entry->nt_ps = wrq->nt_prev_runnable_sum; + __entry->active_time = wts->active_time; + __entry->curr_top = wrq->curr_top; + __entry->prev_top = wrq->prev_top; + __entry->walt_irq_work_lastq_ws = walt_irq_work_lastq_ws; + ), + + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u coloc_demand: %u sum %u irqtime %llu pred_demand_scaled %u rq_cs %llu rq_ps %llu cur_window %u (%s) prev_window %u (%s) nt_cs %llu nt_ps %llu active_time %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu curr_top %u prev_top %u global_ws %llu", + __entry->wallclock, __entry->win_start, __entry->delta, + task_event_names[__entry->evt], __entry->cpu, + __entry->cur_freq, __entry->cur_pid, + __entry->pid, __entry->comm, __entry->mark_start, + __entry->delta_m, __entry->demand, __entry->coloc_demand, + __entry->sum, __entry->irqtime, __entry->pred_demand_scaled, + __entry->rq_cs, __entry->rq_ps, __entry->curr_window, + __window_print(p, __get_dynamic_array(curr_sum), nr_cpu_ids), + __entry->prev_window, + __window_print(p, __get_dynamic_array(prev_sum), nr_cpu_ids), + __entry->nt_cs, __entry->nt_ps, + __entry->active_time, __entry->grp_cs, + __entry->grp_ps, __entry->grp_nt_cs, __entry->grp_nt_ps, + __entry->curr_top, __entry->prev_top, __entry->walt_irq_work_lastq_ws) +); + +TRACE_EVENT(sched_update_task_ravg_mini, + + TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, + u64 wallclock, u64 irqtime, + struct group_cpu_time *cpu_time, struct walt_rq *wrq, + struct walt_task_struct *wts, u64 walt_irq_work_lastq_ws), + + TP_ARGS(p, rq, evt, wallclock, irqtime, cpu_time, wrq, wts, walt_irq_work_lastq_ws), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(u64, wallclock) + __field(u64, mark_start) + __field(u64, delta_m) + __field(u64, win_start) + __field(u64, delta) + __field(enum task_event, evt) + __field(unsigned int, demand) + __field(int, cpu) + __field(u64, rq_cs) + __field(u64, rq_ps) + __field(u64, grp_cs) + __field(u64, grp_ps) + __field(u32, curr_window) + __field(u32, prev_window) + __field(u64, walt_irq_work_lastq_ws) + ), + + TP_fast_assign( + __entry->wallclock = wallclock; + __entry->win_start = wrq->window_start; + __entry->delta = (wallclock - wrq->window_start); + __entry->evt = evt; + __entry->cpu = rq->cpu; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->mark_start = wts->mark_start; + __entry->delta_m = (wallclock - wts->mark_start); + __entry->demand = wts->demand; + __entry->rq_cs = wrq->curr_runnable_sum; + __entry->rq_ps = wrq->prev_runnable_sum; + __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; + __entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0; + __entry->curr_window = wts->curr_window; + __entry->prev_window = wts->prev_window; + __entry->walt_irq_work_lastq_ws = walt_irq_work_lastq_ws; + ), + + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d task %d (%s) ms %llu delta %llu demand %u rq_cs %llu rq_ps %llu cur_window %u prev_window %u grp_cs %lld grp_ps %lld global_ws %llu", + __entry->wallclock, __entry->win_start, __entry->delta, + task_event_names[__entry->evt], __entry->cpu, + __entry->pid, __entry->comm, __entry->mark_start, + __entry->delta_m, __entry->demand, + __entry->rq_cs, __entry->rq_ps, __entry->curr_window, + __entry->prev_window, __entry->grp_cs, __entry->grp_ps, + __entry->walt_irq_work_lastq_ws) +); + +struct migration_sum_data; +extern const char *migrate_type_names[]; + +TRACE_EVENT(sched_set_preferred_cluster, + + TP_PROTO(struct walt_related_thread_group *grp, u64 total_demand, + bool prev_skip_min), + + TP_ARGS(grp, total_demand, prev_skip_min), + + TP_STRUCT__entry( + __field(int, id) + __field(u64, total_demand) + __field(bool, skip_min) + __field(bool, prev_skip_min) + __field(u64, start_ktime_ts) + __field(u64, last_update) + __field(unsigned int, sysctl_sched_hyst_min_coloc_ns) + __field(u64, downmigrate_ts) + ), + + TP_fast_assign( + __entry->id = grp->id; + __entry->total_demand = total_demand; + __entry->skip_min = grp->skip_min; + __entry->prev_skip_min = prev_skip_min; + __entry->start_ktime_ts = grp->start_ktime_ts; + __entry->last_update = grp->last_update; + __entry->sysctl_sched_hyst_min_coloc_ns = sysctl_sched_hyst_min_coloc_ns; + __entry->downmigrate_ts = grp->downmigrate_ts; + ), + + TP_printk("group_id %d total_demand %llu skip_min %d prev_skip_min %d start_ktime_ts %llu last_update %llu min_coloc_ns %u downmigrate_ts %llu", + __entry->id, __entry->total_demand, + __entry->skip_min, __entry->prev_skip_min, + __entry->start_ktime_ts, __entry->last_update, + __entry->sysctl_sched_hyst_min_coloc_ns, + __entry->downmigrate_ts) +); + +TRACE_EVENT(sched_migration_update_sum, + + TP_PROTO(struct task_struct *p, enum migrate_types migrate_type, + struct rq *rq), + + TP_ARGS(p, migrate_type, rq), + + TP_STRUCT__entry( + __field(int, tcpu) + __field(int, pid) + __field(enum migrate_types, migrate_type) + __field(s64, src_cs) + __field(s64, src_ps) + __field(s64, dst_cs) + __field(s64, dst_ps) + __field(s64, src_nt_cs) + __field(s64, src_nt_ps) + __field(s64, dst_nt_cs) + __field(s64, dst_nt_ps) + ), + + TP_fast_assign( + __entry->tcpu = task_cpu(p); + __entry->pid = p->pid; + __entry->migrate_type = migrate_type; + __entry->src_cs = __get_update_sum(rq, migrate_type, + true, false, true); + __entry->src_ps = __get_update_sum(rq, migrate_type, + true, false, false); + __entry->dst_cs = __get_update_sum(rq, migrate_type, + false, false, true); + __entry->dst_ps = __get_update_sum(rq, migrate_type, + false, false, false); + __entry->src_nt_cs = __get_update_sum(rq, migrate_type, + true, true, true); + __entry->src_nt_ps = __get_update_sum(rq, migrate_type, + true, true, false); + __entry->dst_nt_cs = __get_update_sum(rq, migrate_type, + false, true, true); + __entry->dst_nt_ps = __get_update_sum(rq, migrate_type, + false, true, false); + ), + + TP_printk("pid %d task_cpu %d migrate_type %s src_cs %llu src_ps %llu dst_cs %lld dst_ps %lld src_nt_cs %llu src_nt_ps %llu dst_nt_cs %lld dst_nt_ps %lld", + __entry->pid, __entry->tcpu, + migrate_type_names[__entry->migrate_type], + __entry->src_cs, __entry->src_ps, __entry->dst_cs, + __entry->dst_ps, __entry->src_nt_cs, __entry->src_nt_ps, + __entry->dst_nt_cs, __entry->dst_nt_ps) +); + +TRACE_EVENT(sched_set_boost, + + TP_PROTO(int type), + + TP_ARGS(type), + + TP_STRUCT__entry( + __field(int, type) + ), + + TP_fast_assign( + __entry->type = type; + ), + + TP_printk("type %d", __entry->type) +); + +TRACE_EVENT(sched_load_to_gov, + + TP_PROTO(struct rq *rq, u64 aggr_grp_load, u32 tt_load, + int freq_aggr, u64 load, int policy, + int big_task_rotation, + unsigned int user_hint, + struct walt_rq *wrq, + unsigned int reasons), + TP_ARGS(rq, aggr_grp_load, tt_load, freq_aggr, load, policy, + big_task_rotation, user_hint, wrq, reasons), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, policy) + __field(int, ed_task_pid) + __field(u64, aggr_grp_load) + __field(int, freq_aggr) + __field(u64, tt_load) + __field(u64, rq_ps) + __field(u64, grp_rq_ps) + __field(u64, nt_ps) + __field(u64, grp_nt_ps) + __field(u64, pl) + __field(u64, load) + __field(int, big_task_rotation) + __field(unsigned int, user_hint) + __field(unsigned int, reasons) + ), + + TP_fast_assign( + __entry->cpu = cpu_of(rq); + __entry->policy = policy; + __entry->ed_task_pid = + wrq->ed_task ? wrq->ed_task->pid : -1; + __entry->aggr_grp_load = aggr_grp_load; + __entry->freq_aggr = freq_aggr; + __entry->tt_load = tt_load; + __entry->rq_ps = wrq->prev_runnable_sum; + __entry->grp_rq_ps = wrq->grp_time.prev_runnable_sum; + __entry->nt_ps = wrq->nt_prev_runnable_sum; + __entry->grp_nt_ps = wrq->grp_time.nt_prev_runnable_sum; + __entry->pl = wrq->walt_stats.pred_demands_sum_scaled; + __entry->load = load; + __entry->big_task_rotation = big_task_rotation; + __entry->user_hint = user_hint; + __entry->reasons = reasons; + ), + + TP_printk("cpu=%d policy=%d ed_task_pid=%d aggr_grp_load=%llu freq_aggr=%d tt_load=%llu rq_ps=%llu grp_rq_ps=%llu nt_ps=%llu grp_nt_ps=%llu pl=%llu load=%llu big_task_rotation=%d user_hint=%u reasons=0x%x", + __entry->cpu, __entry->policy, __entry->ed_task_pid, + __entry->aggr_grp_load, __entry->freq_aggr, + __entry->tt_load, __entry->rq_ps, __entry->grp_rq_ps, + __entry->nt_ps, __entry->grp_nt_ps, __entry->pl, __entry->load, + __entry->big_task_rotation, __entry->user_hint, __entry->reasons) +); + +TRACE_EVENT(core_ctl_eval_need, + + TP_PROTO(unsigned int cpu, unsigned int last_need, + unsigned int new_need, unsigned int active_cpus, + unsigned int adj_now, unsigned int adj_possible, + unsigned int updated, s64 need_ts), + TP_ARGS(cpu, last_need, new_need, active_cpus, adj_now, adj_possible, updated, need_ts), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, last_need) + __field(u32, new_need) + __field(u32, active_cpus) + __field(u32, adj_now) + __field(u32, adj_possible) + __field(u32, updated) + __field(s64, need_ts) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->last_need = last_need; + __entry->new_need = new_need; + __entry->active_cpus = active_cpus; + __entry->adj_now = adj_now; + __entry->adj_possible = adj_possible; + __entry->updated = updated; + __entry->need_ts = need_ts; + ), + TP_printk("cpu=%u last_need=%u new_need=%u active_cpus=%u adj_now=%u adj_possible=%u updated=%u need_ts=%llu", + __entry->cpu, __entry->last_need, __entry->new_need, + __entry->active_cpus, __entry->adj_now, __entry->adj_possible, + __entry->updated, __entry->need_ts) +); + +TRACE_EVENT(core_ctl_set_busy, + + TP_PROTO(unsigned int cpu, unsigned int busy, + unsigned int old_is_busy, unsigned int is_busy), + TP_ARGS(cpu, busy, old_is_busy, is_busy), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, busy) + __field(u32, old_is_busy) + __field(u32, is_busy) + __field(bool, high_irqload) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->busy = busy; + __entry->old_is_busy = old_is_busy; + __entry->is_busy = is_busy; + __entry->high_irqload = sched_cpu_high_irqload(cpu); + ), + TP_printk("cpu=%u, busy=%u, old_is_busy=%u, new_is_busy=%u high_irqload=%d", + __entry->cpu, __entry->busy, __entry->old_is_busy, + __entry->is_busy, __entry->high_irqload) +); + +TRACE_EVENT(core_ctl_set_boost, + + TP_PROTO(u32 refcount, s32 ret), + TP_ARGS(refcount, ret), + TP_STRUCT__entry( + __field(u32, refcount) + __field(s32, ret) + ), + TP_fast_assign( + __entry->refcount = refcount; + __entry->ret = ret; + ), + TP_printk("refcount=%u, ret=%d", __entry->refcount, __entry->ret) +); + +TRACE_EVENT(core_ctl_update_nr_need, + + TP_PROTO(int cpu, int nr_need, int prev_misfit_need, + int nrrun, int max_nr, int nr_prev_assist), + + TP_ARGS(cpu, nr_need, prev_misfit_need, nrrun, max_nr, nr_prev_assist), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, nr_need) + __field(int, prev_misfit_need) + __field(int, nrrun) + __field(int, max_nr) + __field(int, nr_prev_assist) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr_need = nr_need; + __entry->prev_misfit_need = prev_misfit_need; + __entry->nrrun = nrrun; + __entry->max_nr = max_nr; + __entry->nr_prev_assist = nr_prev_assist; + ), + + TP_printk("cpu=%d nr_need=%d prev_misfit_need=%d nrrun=%d max_nr=%d nr_prev_assist=%d", + __entry->cpu, __entry->nr_need, __entry->prev_misfit_need, + __entry->nrrun, __entry->max_nr, __entry->nr_prev_assist) +); + +TRACE_EVENT(core_ctl_notif_data, + + TP_PROTO(u32 nr_big, u32 ta_load, u32 *ta_util, u32 *cur_cap), + + TP_ARGS(nr_big, ta_load, ta_util, cur_cap), + + TP_STRUCT__entry( + __field(u32, nr_big) + __field(u32, ta_load) + __array(u32, ta_util, MAX_CLUSTERS) + __array(u32, cur_cap, MAX_CLUSTERS) + ), + + TP_fast_assign( + __entry->nr_big = nr_big; + __entry->ta_load = ta_load; + memcpy(__entry->ta_util, ta_util, MAX_CLUSTERS * sizeof(u32)); + memcpy(__entry->cur_cap, cur_cap, MAX_CLUSTERS * sizeof(u32)); + ), + + TP_printk("nr_big=%u ta_load=%u ta_util=(%u %u %u) cur_cap=(%u %u %u)", + __entry->nr_big, __entry->ta_load, + __entry->ta_util[0], __entry->ta_util[1], + __entry->ta_util[2], __entry->cur_cap[0], + __entry->cur_cap[1], __entry->cur_cap[2]) +); + +/* + * Tracepoint for sched_get_nr_running_avg + */ +TRACE_EVENT(sched_get_nr_running_avg, + + TP_PROTO(int cpu, int nr, int nr_misfit, int nr_max, int nr_scaled), + + TP_ARGS(cpu, nr, nr_misfit, nr_max, nr_scaled), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, nr) + __field(int, nr_misfit) + __field(int, nr_max) + __field(int, nr_scaled) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr = nr; + __entry->nr_misfit = nr_misfit; + __entry->nr_max = nr_max; + __entry->nr_scaled = nr_scaled; + ), + + TP_printk("cpu=%d nr=%d nr_misfit=%d nr_max=%d nr_scaled=%d", + __entry->cpu, __entry->nr, __entry->nr_misfit, __entry->nr_max, + __entry->nr_scaled) +); + +TRACE_EVENT(sched_busy_hyst_time, + + TP_PROTO(int cpu, u64 hyst_time, unsigned long nr_run, + unsigned long cpu_util, u64 busy_hyst_time, + u64 coloc_hyst_time, u64 util_hyst_time), + + TP_ARGS(cpu, hyst_time, nr_run, cpu_util, busy_hyst_time, + coloc_hyst_time, util_hyst_time), + + TP_STRUCT__entry( + __field(int, cpu) + __field(u64, hyst_time) + __field(unsigned long, nr_run) + __field(unsigned long, cpu_util) + __field(u64, busy_hyst_time) + __field(u64, coloc_hyst_time) + __field(u64, util_hyst_time) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->hyst_time = hyst_time; + __entry->nr_run = nr_run; + __entry->cpu_util = cpu_util; + __entry->busy_hyst_time = busy_hyst_time; + __entry->coloc_hyst_time = coloc_hyst_time; + __entry->util_hyst_time = util_hyst_time; + ), + + TP_printk("cpu=%d hyst_time=%llu nr_run=%lu cpu_util=%lu busy_hyst_time=%llu coloc_hyst_time=%llu util_hyst_time=%llu", + __entry->cpu, __entry->hyst_time, __entry->nr_run, + __entry->cpu_util, __entry->busy_hyst_time, + __entry->coloc_hyst_time, __entry->util_hyst_time) +); + +TRACE_EVENT(sched_ravg_window_change, + + TP_PROTO(unsigned int sched_ravg_window, unsigned int new_sched_ravg_window + , u64 change_time), + + TP_ARGS(sched_ravg_window, new_sched_ravg_window, change_time), + + TP_STRUCT__entry( + __field(unsigned int, sched_ravg_window) + __field(unsigned int, new_sched_ravg_window) + __field(u64, change_time) + ), + + TP_fast_assign( + __entry->sched_ravg_window = sched_ravg_window; + __entry->new_sched_ravg_window = new_sched_ravg_window; + __entry->change_time = change_time; + ), + + TP_printk("from=%u to=%u at=%lu", + __entry->sched_ravg_window, __entry->new_sched_ravg_window, + __entry->change_time) +); + +TRACE_EVENT(waltgov_util_update, + TP_PROTO(int cpu, + unsigned long util, unsigned long avg_cap, + unsigned long max_cap, unsigned long nl, unsigned long pl, + unsigned int rtgb, unsigned int flags), + TP_ARGS(cpu, util, avg_cap, max_cap, nl, pl, rtgb, flags), + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, util) + __field(unsigned long, avg_cap) + __field(unsigned long, max_cap) + __field(unsigned long, nl) + __field(unsigned long, pl) + __field(unsigned int, rtgb) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->util = util; + __entry->avg_cap = avg_cap; + __entry->max_cap = max_cap; + __entry->nl = nl; + __entry->pl = pl; + __entry->rtgb = rtgb; + __entry->flags = flags; + ), + TP_printk("cpu=%d util=%lu avg_cap=%lu max_cap=%lu nl=%lu pl=%lu rtgb=%u flags=0x%x", + __entry->cpu, __entry->util, __entry->avg_cap, + __entry->max_cap, __entry->nl, + __entry->pl, __entry->rtgb, __entry->flags) +); + +TRACE_EVENT(waltgov_next_freq, + TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, unsigned int raw_freq, + unsigned int freq, unsigned int policy_min_freq, unsigned int policy_max_freq, + unsigned int cached_raw_freq, bool need_freq_update, unsigned int driving_cpu, + unsigned int reason), + TP_ARGS(cpu, util, max, raw_freq, freq, policy_min_freq, policy_max_freq, + cached_raw_freq, need_freq_update, driving_cpu, reason), + TP_STRUCT__entry( + __field(unsigned int, cpu) + __field(unsigned long, util) + __field(unsigned long, max) + __field(unsigned int, raw_freq) + __field(unsigned int, freq) + __field(unsigned int, policy_min_freq) + __field(unsigned int, policy_max_freq) + __field(unsigned int, cached_raw_freq) + __field(bool, need_freq_update) + __field(unsigned int, rt_util) + __field(unsigned int, driving_cpu) + __field(unsigned int, reason) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->util = util; + __entry->max = max; + __entry->raw_freq = raw_freq; + __entry->freq = freq; + __entry->policy_min_freq = policy_min_freq; + __entry->policy_max_freq = policy_max_freq; + __entry->cached_raw_freq = cached_raw_freq; + __entry->need_freq_update = need_freq_update; + __entry->rt_util = cpu_util_rt(cpu_rq(cpu)); + __entry->driving_cpu = driving_cpu; + __entry->reason = reason; + ), + TP_printk("cpu=%u util=%lu max=%lu raw_freq=%lu freq=%u policy_min_freq=%u policy_max_freq=%u cached_raw_freq=%u need_update=%d rt_util=%u driv_cpu=%u reason=0x%x", + __entry->cpu, + __entry->util, + __entry->max, + __entry->raw_freq, + __entry->freq, + __entry->policy_min_freq, + __entry->policy_max_freq, + __entry->cached_raw_freq, + __entry->need_freq_update, + __entry->rt_util, + __entry->driving_cpu, + __entry->reason) +); + +TRACE_EVENT(walt_active_load_balance, + + TP_PROTO(struct task_struct *p, int prev_cpu, int new_cpu, struct walt_task_struct *wts), + + TP_ARGS(p, prev_cpu, new_cpu, wts), + + TP_STRUCT__entry( + __field(pid_t, pid) + __field(bool, misfit) + __field(int, prev_cpu) + __field(int, new_cpu) + ), + + TP_fast_assign( + __entry->pid = p->pid; + __entry->misfit = wts->misfit; + __entry->prev_cpu = prev_cpu; + __entry->new_cpu = new_cpu; + ), + + TP_printk("pid=%d misfit=%d prev_cpu=%d new_cpu=%d\n", + __entry->pid, __entry->misfit, __entry->prev_cpu, + __entry->new_cpu) +); + +TRACE_EVENT(walt_find_busiest_queue, + + TP_PROTO(int dst_cpu, int busiest_cpu, unsigned long src_mask), + + TP_ARGS(dst_cpu, busiest_cpu, src_mask), + + TP_STRUCT__entry( + __field(int, dst_cpu) + __field(int, busiest_cpu) + __field(unsigned long, src_mask) + ), + + TP_fast_assign( + __entry->dst_cpu = dst_cpu; + __entry->busiest_cpu = busiest_cpu; + __entry->src_mask = src_mask; + ), + + TP_printk("dst_cpu=%d busiest_cpu=%d src_mask=%lx\n", + __entry->dst_cpu, __entry->busiest_cpu, + __entry->src_mask) +); + +TRACE_EVENT(walt_nohz_balance_kick, + + TP_PROTO(struct rq *rq), + + TP_ARGS(rq), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned int, nr_running) + __field(unsigned int, nr_cfs_running) + ), + + TP_fast_assign( + __entry->cpu = rq->cpu; + __entry->nr_running = rq->nr_running; + __entry->nr_cfs_running = rq->cfs.h_nr_running; + ), + + TP_printk("cpu=%d nr_running=%u nr_cfs_running=%u", + __entry->cpu, __entry->nr_running, + __entry->nr_cfs_running) +); + +TRACE_EVENT(walt_newidle_balance, + + TP_PROTO(int this_cpu, int busy_cpu, int pulled, bool help_min_cap, bool enough_idle), + + TP_ARGS(this_cpu, busy_cpu, pulled, help_min_cap, enough_idle), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, busy_cpu) + __field(int, pulled) + __field(unsigned int, nr_running) + __field(unsigned int, rt_nr_running) + __field(int, nr_iowait) + __field(bool, help_min_cap) + __field(u64, avg_idle) + __field(bool, enough_idle) + __field(int, overload) + ), + + TP_fast_assign( + __entry->cpu = this_cpu; + __entry->busy_cpu = busy_cpu; + __entry->pulled = pulled; + __entry->nr_running = cpu_rq(this_cpu)->nr_running; + __entry->rt_nr_running = cpu_rq(this_cpu)->rt.rt_nr_running; + __entry->nr_iowait = atomic_read(&(cpu_rq(this_cpu)->nr_iowait)); + __entry->help_min_cap = help_min_cap; + __entry->avg_idle = cpu_rq(this_cpu)->avg_idle; + __entry->enough_idle = enough_idle; + __entry->overload = cpu_rq(this_cpu)->rd->overload; + ), + + TP_printk("cpu=%d busy_cpu=%d pulled=%d nr_running=%u rt_nr_running=%u nr_iowait=%d help_min_cap=%d avg_idle=%llu enough_idle=%d overload=%d", + __entry->cpu, __entry->busy_cpu, __entry->pulled, + __entry->nr_running, __entry->rt_nr_running, + __entry->nr_iowait, __entry->help_min_cap, + __entry->avg_idle, __entry->enough_idle, + __entry->overload) +); + +TRACE_EVENT(walt_lb_cpu_util, + + TP_PROTO(int cpu, struct walt_rq *wrq), + + TP_ARGS(cpu, wrq), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned int, nr_running) + __field(unsigned int, cfs_nr_running) + __field(unsigned int, nr_big) + __field(unsigned int, nr_rtg_high_prio_tasks) + __field(unsigned int, cpu_util) + __field(unsigned int, capacity_orig) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr_running = cpu_rq(cpu)->nr_running; + __entry->cfs_nr_running = cpu_rq(cpu)->cfs.h_nr_running; + __entry->nr_big = wrq->walt_stats.nr_big_tasks; + __entry->nr_rtg_high_prio_tasks = walt_nr_rtg_high_prio(cpu); + __entry->cpu_util = cpu_util(cpu); + __entry->capacity_orig = capacity_orig_of(cpu); + ), + + TP_printk("cpu=%d nr_running=%u cfs_nr_running=%u nr_big=%u nr_rtg_hp=%u cpu_util=%u capacity_orig=%u", + __entry->cpu, __entry->nr_running, __entry->cfs_nr_running, + __entry->nr_big, __entry->nr_rtg_high_prio_tasks, + __entry->cpu_util, __entry->capacity_orig) +); + +TRACE_EVENT(sched_cpu_util, + + TP_PROTO(int cpu, struct cpumask *lowest_mask), + + TP_ARGS(cpu, lowest_mask), + + TP_STRUCT__entry( + __field(unsigned int, cpu) + __field(unsigned int, nr_running) + __field(long, cpu_util) + __field(long, cpu_util_cum) + __field(unsigned long, capacity_curr) + __field(unsigned long, capacity) + __field(unsigned long, capacity_orig) + __field(unsigned int, idle_exit_latency) + __field(u64, irqload) + __field(int, online) + __field(int, inactive) + __field(int, halted) + __field(int, reserved) + __field(int, high_irq_load) + __field(unsigned int, nr_rtg_high_prio_tasks) + __field(u64, prs_gprs) + __field(unsigned int, lowest_mask) + __field(unsigned long, thermal_pressure) + ), + + TP_fast_assign( + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + __entry->cpu = cpu; + __entry->nr_running = cpu_rq(cpu)->nr_running; + __entry->cpu_util = cpu_util(cpu); + __entry->cpu_util_cum = cpu_util_cum(cpu); + __entry->capacity_curr = capacity_curr_of(cpu); + __entry->capacity = capacity_of(cpu); + __entry->capacity_orig = capacity_orig_of(cpu); + __entry->idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(cpu)); + __entry->irqload = sched_irqload(cpu); + __entry->online = cpu_online(cpu); + __entry->inactive = !cpu_active(cpu); + __entry->halted = cpu_halted(cpu); + __entry->reserved = is_reserved(cpu); + __entry->high_irq_load = sched_cpu_high_irqload(cpu); + __entry->nr_rtg_high_prio_tasks = walt_nr_rtg_high_prio(cpu); + __entry->prs_gprs = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; + if (!lowest_mask) + __entry->lowest_mask = 0; + else + __entry->lowest_mask = cpumask_bits(lowest_mask)[0]; + __entry->thermal_pressure = arch_scale_thermal_pressure(cpu); + ), + + TP_printk("cpu=%d nr_running=%d cpu_util=%ld cpu_util_cum=%ld capacity_curr=%lu capacity=%lu capacity_orig=%lu idle_exit_latency=%u irqload=%llu online=%u, inactive=%u, halted=%u, reserved=%u, high_irq_load=%u nr_rtg_hp=%u prs_gprs=%llu lowest_mask=0x%x thermal_pressure=%llu", + __entry->cpu, __entry->nr_running, __entry->cpu_util, + __entry->cpu_util_cum, __entry->capacity_curr, + __entry->capacity, __entry->capacity_orig, + __entry->idle_exit_latency, __entry->irqload, __entry->online, + __entry->inactive, __entry->halted, __entry->reserved, __entry->high_irq_load, + __entry->nr_rtg_high_prio_tasks, __entry->prs_gprs, + __entry->lowest_mask, __entry->thermal_pressure) +); + +TRACE_EVENT(sched_compute_energy, + + TP_PROTO(struct task_struct *p, int eval_cpu, + unsigned long eval_energy, + unsigned long prev_energy, + unsigned long best_energy, + unsigned long best_energy_cpu, + struct compute_energy_output *o), + + TP_ARGS(p, eval_cpu, eval_energy, prev_energy, best_energy, + best_energy_cpu, o), + + TP_STRUCT__entry( + __field(int, pid) + __array(char, comm, TASK_COMM_LEN) + __field(unsigned long, util) + __field(int, prev_cpu) + __field(unsigned long, prev_energy) + __field(int, eval_cpu) + __field(unsigned long, eval_energy) + __field(int, best_energy_cpu) + __field(unsigned long, best_energy) + __field(unsigned int, cluster_first_cpu0) + __field(unsigned int, cluster_first_cpu1) + __field(unsigned int, cluster_first_cpu2) + __field(unsigned long, s0) + __field(unsigned long, s1) + __field(unsigned long, s2) + __field(unsigned long, m0) + __field(unsigned long, m1) + __field(unsigned long, m2) + __field(unsigned long, c0) + __field(unsigned long, c1) + __field(unsigned long, c2) + ), + + TP_fast_assign( + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->util = task_util(p); + __entry->prev_cpu = task_cpu(p); + __entry->prev_energy = prev_energy; + __entry->eval_cpu = eval_cpu; + __entry->eval_energy = eval_energy; + __entry->best_energy_cpu = best_energy_cpu; + __entry->best_energy = best_energy; + __entry->cluster_first_cpu0 = o->cluster_first_cpu[0]; + __entry->cluster_first_cpu1 = o->cluster_first_cpu[1]; + __entry->cluster_first_cpu2 = o->cluster_first_cpu[2]; + __entry->s0 = o->sum_util[0]; + __entry->s1 = o->sum_util[1]; + __entry->s2 = o->sum_util[2]; + __entry->m0 = o->max_util[0]; + __entry->m1 = o->max_util[1]; + __entry->m2 = o->max_util[2]; + __entry->c0 = o->cost[0]; + __entry->c1 = o->cost[1]; + __entry->c2 = o->cost[2]; + ), + + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d prev_energy=%lu eval_cpu=%d eval_energy=%lu best_energy_cpu=%d best_energy=%lu, fcpu s m c = %u %u %u %u, %u %u %u %u, %u %u %u %u", + __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, + __entry->prev_energy, __entry->eval_cpu, __entry->eval_energy, + __entry->best_energy_cpu, __entry->best_energy, + __entry->cluster_first_cpu0, __entry->s0, __entry->m0, __entry->c0, + __entry->cluster_first_cpu1, __entry->s1, __entry->m1, __entry->c1, + __entry->cluster_first_cpu2, __entry->s2, __entry->m2, __entry->c2) +) + +TRACE_EVENT(sched_task_util, + + TP_PROTO(struct task_struct *p, unsigned long candidates, + int best_energy_cpu, bool sync, int need_idle, int fastpath, + u64 start_t, bool uclamp_boosted, int start_cpu), + + TP_ARGS(p, candidates, best_energy_cpu, sync, need_idle, fastpath, + start_t, uclamp_boosted, start_cpu), + + TP_STRUCT__entry( + __field(int, pid) + __array(char, comm, TASK_COMM_LEN) + __field(unsigned long, util) + __field(unsigned long, candidates) + __field(int, prev_cpu) + __field(int, best_energy_cpu) + __field(bool, sync) + __field(int, need_idle) + __field(int, fastpath) + __field(int, placement_boost) + __field(int, rtg_cpu) + __field(u64, latency) + __field(bool, uclamp_boosted) + __field(bool, is_rtg) + __field(bool, rtg_skip_min) + __field(int, start_cpu) + __field(u32, unfilter) + __field(unsigned long, cpus_allowed) + __field(int, task_boost) + __field(bool, low_latency) + __field(bool, iowaited) + __field(int, load_boost) + ), + + TP_fast_assign( + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->util = task_util(p); + __entry->prev_cpu = task_cpu(p); + __entry->candidates = candidates; + __entry->best_energy_cpu = best_energy_cpu; + __entry->sync = sync; + __entry->need_idle = need_idle; + __entry->fastpath = fastpath; + __entry->placement_boost = task_boost_policy(p); + __entry->latency = (sched_clock() - start_t); + __entry->uclamp_boosted = uclamp_boosted; + __entry->is_rtg = task_in_related_thread_group(p); + __entry->rtg_skip_min = walt_get_rtg_status(p); + __entry->start_cpu = start_cpu; + __entry->unfilter = + ((struct walt_task_struct *) p->android_vendor_data1)->unfilter; + __entry->cpus_allowed = cpumask_bits(p->cpus_ptr)[0]; + __entry->task_boost = per_task_boost(p); + __entry->low_latency = walt_low_latency_task(p); + __entry->iowaited = + ((struct walt_task_struct *) p->android_vendor_data1)->iowaited; + __entry->load_boost = + ((struct walt_task_struct *) p->android_vendor_data1)->load_boost; + ), + + TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d candidates=%#lx best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d unfilter=%u affinity=%lx task_boost=%d low_latency=%d iowaited=%d load_boost=%d", + __entry->pid, __entry->comm, __entry->util, __entry->prev_cpu, + __entry->candidates, __entry->best_energy_cpu, __entry->sync, + __entry->need_idle, __entry->fastpath, __entry->placement_boost, + __entry->latency, __entry->uclamp_boosted, + __entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu, + __entry->unfilter, __entry->cpus_allowed, __entry->task_boost, + __entry->low_latency, __entry->iowaited, __entry->load_boost) +); + +/* + * Tracepoint for find_best_target + */ +TRACE_EVENT(sched_find_best_target, + + TP_PROTO(struct task_struct *tsk, + unsigned long min_util, int start_cpu, + unsigned long candidates, + int most_spare_cap, + int order_index, int end_index, + int skip, bool running, + int most_spare_rq_cpu, unsigned int cpu_rq_runnable_cnt), + + TP_ARGS(tsk, min_util, start_cpu, candidates, + most_spare_cap, + order_index, end_index, skip, running, + most_spare_rq_cpu, cpu_rq_runnable_cnt), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned long, min_util) + __field(int, start_cpu) + __field(unsigned long, candidates) + __field(int, most_spare_cap) + __field(int, order_index) + __field(int, end_index) + __field(int, skip) + __field(bool, running) + __field(int, most_spare_rq_cpu) + __field(unsigned int, cpu_rq_runnable_cnt) + ), + + TP_fast_assign( + memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); + __entry->pid = tsk->pid; + __entry->min_util = min_util; + __entry->start_cpu = start_cpu; + __entry->candidates = candidates; + __entry->most_spare_cap = most_spare_cap; + __entry->order_index = order_index; + __entry->end_index = end_index; + __entry->skip = skip; + __entry->running = running; + __entry->most_spare_rq_cpu = most_spare_rq_cpu; + __entry->cpu_rq_runnable_cnt = cpu_rq_runnable_cnt; + ), + + TP_printk("pid=%d comm=%s start_cpu=%d candidates=%#lx most_spare_cap=%d order_index=%d end_index=%d skip=%d running=%d min_util=%lu spare_rq_cpu=%d min_runnable=%u", + __entry->pid, __entry->comm, + __entry->start_cpu, + __entry->candidates, + __entry->most_spare_cap, + __entry->order_index, + __entry->end_index, + __entry->skip, + __entry->running, + __entry->min_util, + __entry->most_spare_rq_cpu, + __entry->cpu_rq_runnable_cnt) +); + +TRACE_EVENT(sched_enq_deq_task, + + TP_PROTO(struct task_struct *p, bool enqueue, unsigned int cpus_allowed, bool mvp), + + TP_ARGS(p, enqueue, cpus_allowed, mvp), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, prio) + __field(int, cpu) + __field(bool, enqueue) + __field(unsigned int, nr_running) + __field(unsigned int, rt_nr_running) + __field(unsigned int, cpus_allowed) + __field(unsigned int, demand) + __field(unsigned int, pred_demand_scaled) + __field(bool, compat_thread) + __field(bool, mvp) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->cpu = task_cpu(p); + __entry->enqueue = enqueue; + __entry->nr_running = task_rq(p)->nr_running; + __entry->rt_nr_running = task_rq(p)->rt.rt_nr_running; + __entry->cpus_allowed = cpus_allowed; + __entry->demand = task_load(p); + __entry->pred_demand_scaled = + ((struct walt_task_struct *) p->android_vendor_data1)->pred_demand_scaled; + __entry->compat_thread = is_compat_thread(task_thread_info(p)); + __entry->mvp = mvp; + ), + + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand_scaled=%u is_compat_t=%d mvp=%d", + __entry->cpu, + __entry->enqueue ? "enqueue" : "dequeue", + __entry->comm, __entry->pid, + __entry->prio, __entry->nr_running, + __entry->rt_nr_running, + __entry->cpus_allowed, __entry->demand, + __entry->pred_demand_scaled, + __entry->compat_thread, __entry->mvp) +); + +TRACE_EVENT(walt_window_rollover, + + TP_PROTO(u64 window_start), + + TP_ARGS(window_start), + + TP_STRUCT__entry( + __field(u64, window_start) + ), + + TP_fast_assign( + __entry->window_start = window_start; + ), + + TP_printk("window_start=%llu", __entry->window_start) +); + +DECLARE_EVENT_CLASS(walt_cfs_mvp_task_template, + + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + + TP_ARGS(p, wts, limit), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, prio) + __field(int, mvp_prio) + __field(int, cpu) + __field(u64, exec) + __field(unsigned int, limit) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->mvp_prio = wts->mvp_prio; + __entry->cpu = task_cpu(p); + __entry->exec = wts->total_exec; + __entry->limit = limit; + ), + + TP_printk("comm=%s pid=%d prio=%d mvp_prio=%d cpu=%d exec=%llu limit=%u", + __entry->comm, __entry->pid, __entry->prio, + __entry->mvp_prio, __entry->cpu, __entry->exec, + __entry->limit) +); + +/* called upon MVP task de-activation. exec will be more than limit */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_deactivate_mvp_task, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP is returned to run next */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_pick_next, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP (current) is not preempted by waking task */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_nopreempt, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +/* called upon when MVP (waking task) preempts the current */ +DEFINE_EVENT(walt_cfs_mvp_task_template, walt_cfs_mvp_wakeup_preempt, + TP_PROTO(struct task_struct *p, struct walt_task_struct *wts, unsigned int limit), + TP_ARGS(p, wts, limit)); + +#define SPAN_SIZE (NR_CPUS/4) + +TRACE_EVENT(sched_overutilized, + + TP_PROTO(int overutilized, char *span), + + TP_ARGS(overutilized, span), + + TP_STRUCT__entry( + __field(int, overutilized) + __array(char, span, SPAN_SIZE) + ), + + TP_fast_assign( + __entry->overutilized = overutilized; + strscpy(__entry->span, span, SPAN_SIZE); + ), + + TP_printk("overutilized=%d span=0x%s", + __entry->overutilized, __entry->span) +); + +TRACE_EVENT(sched_cgroup_attach, + + TP_PROTO(struct task_struct *p, unsigned int grp_id, int ret), + + TP_ARGS(p, grp_id, ret), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(unsigned int, grp_id) + __field(int, ret) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->grp_id = grp_id; + __entry->ret = ret; + ), + + TP_printk("comm=%s pid=%d grp_id=%u ret=%d", + __entry->comm, __entry->pid, + __entry->grp_id, __entry->ret) + +); + +TRACE_EVENT(halt_cpus_start, + TP_PROTO(struct cpumask *cpus, unsigned char halt), + + TP_ARGS(cpus, halt), + + TP_STRUCT__entry( + __field(unsigned int, cpus) + __field(unsigned int, halted_cpus) + __field(unsigned char, halt) + ), + + TP_fast_assign( + __entry->cpus = cpumask_bits(cpus)[0]; + __entry->halted_cpus = cpumask_bits(cpu_halt_mask)[0]; + __entry->halt = halt; + ), + + TP_printk("req_cpus=0x%x halt_cpus=0x%x halt=%d", + __entry->cpus, __entry->halted_cpus, __entry->halt) + +); + +TRACE_EVENT(halt_cpus, + TP_PROTO(struct cpumask *cpus, u64 start_time, unsigned char halt, int err), + + TP_ARGS(cpus, start_time, halt, err), + + TP_STRUCT__entry( + __field(unsigned int, cpus) + __field(unsigned int, halted_cpus) + __field(unsigned int, time) + __field(unsigned char, halt) + __field(unsigned char, success) + ), + + TP_fast_assign( + __entry->cpus = cpumask_bits(cpus)[0]; + __entry->halted_cpus = cpumask_bits(cpu_halt_mask)[0]; + __entry->time = div64_u64(sched_clock() - start_time, 1000); + __entry->halt = halt; + __entry->success = ((err >= 0)?1:0); + ), + + TP_printk("req_cpus=0x%x halt_cpus=0x%x time=%u us halt=%d success=%d", + __entry->cpus, __entry->halted_cpus, + __entry->time, __entry->halt, __entry->success) +); + +TRACE_EVENT(sched_task_handler, + TP_PROTO(struct task_struct *p, int param, int val, unsigned long c0, + unsigned long c1, unsigned long c2, unsigned long c3, + unsigned long c4, unsigned long c5), + + TP_ARGS(p, param, val, c0, c1, c2, c3, c4, c5), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, param) + __field(int, val) + __field(unsigned long, c0) + __field(unsigned long, c1) + __field(unsigned long, c2) + __field(unsigned long, c3) + __field(unsigned long, c4) + __field(unsigned long, c5) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->param = param; + __entry->val = val; + __entry->c0 = c0; + __entry->c1 = c1; + __entry->c2 = c2; + __entry->c3 = c3; + __entry->c4 = c4; + __entry->c5 = c5; + ), + + TP_printk("comm=%s pid=%d param=%d val=%d callers=%ps <- %ps <- %ps <- %ps <- %ps <- %ps", + __entry->comm, __entry->pid, __entry->param, __entry->val, __entry->c0, + __entry->c1, __entry->c2, __entry->c3, __entry->c4, __entry->c5) +); + +TRACE_EVENT(update_cpu_capacity, + + TP_PROTO(int cpu, unsigned long rt_pressure, unsigned long capacity), + + TP_ARGS(cpu, rt_pressure, capacity), + + TP_STRUCT__entry( + __field(int, cpu) + __field(unsigned long, rt_pressure) + __field(unsigned long, capacity) + __field(unsigned long, arch_capacity) + __field(unsigned long, thermal_cap) + __field(unsigned long, max_possible_freq) + __field(unsigned long, max_freq) + ), + + TP_fast_assign( + struct walt_sched_cluster *cluster = cpu_cluster(cpu); + + __entry->cpu = cpu; + __entry->rt_pressure = rt_pressure; + __entry->capacity = capacity; + __entry->arch_capacity = arch_scale_cpu_capacity(cpu); + __entry->thermal_cap = arch_scale_cpu_capacity(cpu) - + arch_scale_thermal_pressure(cpu); + __entry->max_freq = cluster->max_freq; + __entry->max_possible_freq = cluster->max_possible_freq; + ), + + TP_printk("cpu=%d arch_capacity=%lu thermal_cap=%lu rt_pressure=%lu max_freq=%lu max_possible_freq=%lu capacity=%lu", + __entry->cpu, __entry->arch_capacity, + __entry->thermal_cap, __entry->rt_pressure, + __entry->max_freq, __entry->max_possible_freq, + __entry->capacity) +); + +#endif /* _TRACE_WALT_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../kernel/sched/walt +#define TRACE_INCLUDE_FILE trace + +#include diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c new file mode 100644 index 000000000000..702e6a492269 --- /dev/null +++ b/kernel/sched/walt/walt.c @@ -0,0 +1,4707 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "walt.h" +#include "trace.h" + +const char *task_event_names[] = { + "PUT_PREV_TASK", + "PICK_NEXT_TASK", + "TASK_WAKE", + "TASK_MIGRATE", + "TASK_UPDATE", + "IRQ_UPDATE" +}; + +const char *migrate_type_names[] = { + "GROUP_TO_RQ", + "RQ_TO_GROUP", + "RQ_TO_RQ", + "GROUP_TO_GROUP" +}; + +#define SCHED_FREQ_ACCOUNT_WAIT_TIME 0 +#define SCHED_ACCOUNT_WAIT_TIME 1 + +#define EARLY_DETECTION_DURATION 9500000 +#define MAX_NUM_CGROUP_COLOC_ID 20 + +#define MAX_NR_CLUSTERS 3 + +#define NEW_TASK_ACTIVE_TIME 100000000 + +unsigned int sysctl_sched_user_hint; +static u64 sched_clock_last; +static bool walt_clock_suspended; + +static bool use_cycle_counter; +static DEFINE_MUTEX(cluster_lock); +static u64 walt_load_reported_window; + +static struct irq_work walt_cpufreq_irq_work; +struct irq_work walt_migration_irq_work; +unsigned int walt_rotation_enabled; +cpumask_t asym_cap_sibling_cpus = CPU_MASK_NONE; + +unsigned int __read_mostly sched_ravg_window = 20000000; +int min_possible_cluster_id; +int max_possible_cluster_id; +/* Initial task load. Newly created tasks are assigned this load. */ +unsigned int __read_mostly sched_init_task_load_windows; +/* + * Task load is categorized into buckets for the purpose of top task tracking. + * The entire range of load from 0 to sched_ravg_window needs to be covered + * in NUM_LOAD_INDICES number of buckets. Therefore the size of each bucket + * is given by sched_ravg_window / NUM_LOAD_INDICES. Since the default value + * of sched_ravg_window is DEFAULT_SCHED_RAVG_WINDOW, use that to compute + * sched_load_granule. + */ +unsigned int __read_mostly sched_load_granule; + +u64 walt_sched_clock(void) +{ + if (unlikely(walt_clock_suspended)) + return sched_clock_last; + return sched_clock(); +} + +static void walt_resume(void) +{ + walt_clock_suspended = false; +} + +static int walt_suspend(void) +{ + sched_clock_last = sched_clock(); + walt_clock_suspended = true; + return 0; +} + +static struct syscore_ops walt_syscore_ops = { + .resume = walt_resume, + .suspend = walt_suspend +}; + +/* + *@boost:should be 0,1,2. + *@period:boost time based on ms units. + */ +int set_task_boost(int boost, u64 period) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + + if (boost < TASK_BOOST_NONE || boost >= TASK_BOOST_END) + return -EINVAL; + if (boost) { + wts->boost = boost; + wts->boost_period = (u64)period * 1000 * 1000; + wts->boost_expires = walt_sched_clock() + wts->boost_period; + } else { + wts->boost = 0; + wts->boost_expires = 0; + wts->boost_period = 0; + } + return 0; +} +EXPORT_SYMBOL(set_task_boost); + +static inline void acquire_rq_locks_irqsave(const cpumask_t *cpus, + unsigned long *flags) +{ + int cpu; + int level = 0; + + local_irq_save(*flags); + + for_each_cpu(cpu, cpus) { + if (level == 0) + raw_spin_lock(&cpu_rq(cpu)->__lock); + else + raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); + level++; + } +} + +static inline void release_rq_locks_irqrestore(const cpumask_t *cpus, + unsigned long *flags) +{ + int cpu; + + for_each_cpu(cpu, cpus) + raw_spin_unlock(&cpu_rq(cpu)->__lock); + local_irq_restore(*flags); +} + +static inline u64 walt_rq_clock(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (unlikely(walt_clock_suspended)) + return sched_clock_last; + + lockdep_assert_held(&rq->__lock); + + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + + return max(rq_clock(rq), wrq->latest_clock); +} + +static unsigned int walt_cpu_high_irqload; + +static __read_mostly unsigned int sched_io_is_busy = 1; + +/* Window size (in ns) */ +static __read_mostly unsigned int new_sched_ravg_window = DEFAULT_SCHED_RAVG_WINDOW; + +static DEFINE_SPINLOCK(sched_ravg_window_lock); +static u64 sched_ravg_window_change_time; + +static unsigned int __read_mostly sched_init_task_load_windows_scaled; +static unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; + +/* Size of bitmaps maintained to track top tasks */ +static const unsigned int top_tasks_bitmap_size = + BITS_TO_LONGS(NUM_LOAD_INDICES + 1) * sizeof(unsigned long); + +__read_mostly unsigned int walt_scale_demand_divisor; + +#define SCHED_PRINT(arg) printk_deferred("%s=%llu", #arg, arg) +#define STRG(arg) #arg + +void walt_task_dump(struct task_struct *p) +{ + char buff[WALT_NR_CPUS * 16]; + int i, j = 0; + int buffsz = WALT_NR_CPUS * 16; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + bool is_32bit_thread = is_compat_thread(task_thread_info(p)); + + printk_deferred("Task: %.16s-%d\n", p->comm, p->pid); + SCHED_PRINT(READ_ONCE(p->__state)); + SCHED_PRINT(p->cpu); + SCHED_PRINT(p->policy); + SCHED_PRINT(p->prio); + SCHED_PRINT(wts->mark_start); + SCHED_PRINT(wts->demand); + SCHED_PRINT(wts->coloc_demand); + SCHED_PRINT(wts->enqueue_after_migration); + SCHED_PRINT(wts->last_sleep_ts); + SCHED_PRINT(wts->prev_cpu); + SCHED_PRINT(wts->new_cpu); + SCHED_PRINT(wts->misfit); + SCHED_PRINT(wts->prev_on_rq); + SCHED_PRINT(wts->prev_on_rq_cpu); + SCHED_PRINT(wts->mvp_prio); + SCHED_PRINT(wts->iowaited); + SCHED_PRINT(sched_ravg_window); + SCHED_PRINT(new_sched_ravg_window); + + for (i = 0 ; i < nr_cpu_ids; i++) + j += scnprintf(buff + j, buffsz - j, "%u ", + wts->curr_window_cpu[i]); + printk_deferred("%s=%u (%s)\n", STRG(wts->curr_window), + wts->curr_window, buff); + + for (i = 0, j = 0 ; i < nr_cpu_ids; i++) + j += scnprintf(buff + j, buffsz - j, "%u ", + wts->prev_window_cpu[i]); + printk_deferred("%s=%u (%s)\n", STRG(wts->prev_window), + wts->prev_window, buff); + + SCHED_PRINT(wts->last_wake_ts); + SCHED_PRINT(wts->last_enqueued_ts); + SCHED_PRINT(wts->misfit); + SCHED_PRINT(wts->unfilter); + SCHED_PRINT(is_32bit_thread); + SCHED_PRINT(wts->grp); + SCHED_PRINT(p->on_cpu); + SCHED_PRINT(p->on_rq); +} + +void walt_rq_dump(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct task_struct *tsk = cpu_curr(cpu); + int i; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + /* + * Increment the task reference so that it can't be + * freed on a remote CPU. Since we are going to + * enter panic, there is no need to decrement the + * task reference. Decrementing the task reference + * can't be done in atomic context, especially with + * rq locks held. + */ + get_task_struct(tsk); + printk_deferred("CPU:%d nr_running:%u current: %d (%s)\n", + cpu, rq->nr_running, tsk->pid, tsk->comm); + + printk_deferred("=========================================="); + SCHED_PRINT(wrq->latest_clock); + SCHED_PRINT(wrq->window_start); + SCHED_PRINT(wrq->prev_window_size); + SCHED_PRINT(wrq->curr_runnable_sum); + SCHED_PRINT(wrq->prev_runnable_sum); + SCHED_PRINT(wrq->nt_curr_runnable_sum); + SCHED_PRINT(wrq->nt_prev_runnable_sum); + SCHED_PRINT(wrq->task_exec_scale); + SCHED_PRINT(wrq->grp_time.curr_runnable_sum); + SCHED_PRINT(wrq->grp_time.prev_runnable_sum); + SCHED_PRINT(wrq->grp_time.nt_curr_runnable_sum); + SCHED_PRINT(wrq->grp_time.nt_prev_runnable_sum); + for (i = 0 ; i < NUM_TRACKED_WINDOWS; i++) { + printk_deferred("wrq->load_subs[%d].window_start=%llu)\n", i, + wrq->load_subs[i].window_start); + printk_deferred("wrq->load_subs[%d].subs=%llu)\n", i, + wrq->load_subs[i].subs); + printk_deferred("wrq->load_subs[%d].new_subs=%llu)\n", i, + wrq->load_subs[i].new_subs); + } + walt_task_dump(tsk); + SCHED_PRINT(sched_capacity_margin_up[cpu]); + SCHED_PRINT(sched_capacity_margin_down[cpu]); +} + +void walt_dump(void) +{ + int cpu; + + printk_deferred("============ WALT RQ DUMP START ==============\n"); + printk_deferred("Sched clock: %llu\n", walt_sched_clock()); + printk_deferred("Time last window changed=%lu\n", + sched_ravg_window_change_time); + printk_deferred("global_ws=%llu\n", + atomic64_read(&walt_irq_work_lastq_ws)); + for_each_online_cpu(cpu) + walt_rq_dump(cpu); + SCHED_PRINT(max_possible_cluster_id); + printk_deferred("============ WALT RQ DUMP END ==============\n"); +} + +int in_sched_bug; + +static inline void +fixup_cumulative_runnable_avg(struct rq *rq, + struct task_struct *p, + struct walt_sched_stats *stats, + s64 demand_scaled_delta, + s64 pred_demand_scaled_delta) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + s64 cumulative_runnable_avg_scaled = + stats->cumulative_runnable_avg_scaled + demand_scaled_delta; + s64 pred_demands_sum_scaled = + stats->pred_demands_sum_scaled + pred_demand_scaled_delta; + + lockdep_assert_held(&rq->__lock); + + if (task_rq(p) != rq) + WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on rq %d", + raw_smp_processor_id(), p->comm, p->pid, rq->cpu); + + if (cumulative_runnable_avg_scaled < 0) { + WALT_BUG(WALT_BUG_WALT, p, "on CPU %d task ds=%llu is higher than cra=%llu\n", + raw_smp_processor_id(), wts->demand_scaled, + stats->cumulative_runnable_avg_scaled); + cumulative_runnable_avg_scaled = 0; + } + stats->cumulative_runnable_avg_scaled = (u64)cumulative_runnable_avg_scaled; + + if (pred_demands_sum_scaled < 0) { + WALT_BUG(WALT_BUG_WALT, p, "on CPU %d task pds=%llu is higher than pds_sum=%llu\n", + raw_smp_processor_id(), wts->pred_demand_scaled, + stats->pred_demands_sum_scaled); + pred_demands_sum_scaled = 0; + } + stats->pred_demands_sum_scaled = (u64)pred_demands_sum_scaled; +} + +static void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, + u16 updated_demand_scaled, + u16 updated_pred_demand_scaled) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + s64 task_load_delta = (s64)updated_demand_scaled - + wts->demand_scaled; + s64 pred_demand_delta = (s64)updated_pred_demand_scaled - + wts->pred_demand_scaled; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, task_load_delta, + pred_demand_delta); +} + +static void rollover_cpu_window(struct rq *rq, bool full_window); +static void rollover_top_tasks(struct rq *rq, bool full_window); + +/* walt_find_cluster_packing_cpu - Return a packing_cpu choice common for this cluster. + * @start_cpu: The cpu from the cluster to choose from + * + * If the cluster has a 32bit capable cpu return it regardless + * of whether it is halted or not. + * + * If the cluster does not have a 32 bit capable cpu, find the + * first unhalted, active cpu in this cluster. + */ +int walt_find_cluster_packing_cpu(int start_cpu) +{ + struct rq *rq = cpu_rq(start_cpu); + struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + cpumask_t unhalted_cpus; + cpumask_t cluster_32bit_cpus; + + /* find all 32 bit capable cpus in this cluster */ + cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask()); + + /* pack 32 bit and 64 bit tasks on the same cpu, if possible */ + if (cpumask_weight(&cluster_32bit_cpus) > 0) + return cpumask_first(&cluster_32bit_cpus); + + /* find all unhalted active cpus */ + cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); + + /* find all unhalted active cpus in this cluster */ + cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); + + /* return the first found unhalted, active cpu, in this cluster */ + return cpumask_first(&unhalted_cpus); +} + +/* for cfs and rt, determine if packing_cpu should be used */ +bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p) +{ + struct rq *rq; + struct walt_rq *wrq; + struct walt_sched_cluster *cluster; + + /* packing cpu must be a valid cpu for runqueue lookup */ + if (packing_cpu >= nr_cpu_ids) + return false; + + rq = cpu_rq(packing_cpu); + wrq = (struct walt_rq *)rq->android_vendor_data1; + cluster = wrq->cluster; + + /* if idle_enough feature is not enabled */ + if (!sysctl_sched_idle_enough) + return false; + + /* if cpu is not allowed for this task */ + if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr)) + return false; + + /* if cluster util is high */ + if (sched_get_cluster_util_pct(cluster) >= sysctl_sched_cluster_util_thres_pct) + return false; + + /* if cpu utilization is high */ + if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough) + return false; + + /* don't pack big tasks */ + if (task_util(p) >= sysctl_sched_idle_enough) + return false; + + /* the packing cpu can be used, so pack! */ + return true; +} + +/* + * Demand aggregation for frequency purpose: + * + * CPU demand of tasks from various related groups is aggregated per-cluster and + * added to the "max_busy_cpu" in that cluster, where max_busy_cpu is determined + * by just wrq->prev_runnable_sum. + * + * Some examples follow, which assume: + * Cluster0 = CPU0-3, Cluster1 = CPU4-7 + * One related thread group A that has tasks A0, A1, A2 + * + * A->cpu_time[X].curr/prev_sum = counters in which cpu execution stats of + * tasks belonging to group A are accumulated when they run on cpu X. + * + * CX->curr/prev_sum = counters in which cpu execution stats of all tasks + * not belonging to group A are accumulated when they run on cpu X + * + * Lets say the stats for window M was as below: + * + * C0->prev_sum = 1ms, A->cpu_time[0].prev_sum = 5ms + * Task A0 ran 5ms on CPU0 + * Task B0 ran 1ms on CPU0 + * + * C1->prev_sum = 5ms, A->cpu_time[1].prev_sum = 6ms + * Task A1 ran 4ms on CPU1 + * Task A2 ran 2ms on CPU1 + * Task B1 ran 5ms on CPU1 + * + * C2->prev_sum = 0ms, A->cpu_time[2].prev_sum = 0 + * CPU2 idle + * + * C3->prev_sum = 0ms, A->cpu_time[3].prev_sum = 0 + * CPU3 idle + * + * In this case, CPU1 was most busy going by just its prev_sum counter. Demand + * from all group A tasks are added to CPU1. IOW, at end of window M, cpu busy + * time reported to governor will be: + * + * + * C0 busy time = 1ms + * C1 busy time = 5 + 5 + 6 = 16ms + * + */ +__read_mostly bool sched_freq_aggr_en; + +static u64 +update_window_start(struct rq *rq, u64 wallclock, int event) +{ + s64 delta; + int nr_windows; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 old_window_start = wrq->window_start; + bool full_window; + + if (wallclock < wrq->latest_clock) { + printk_deferred("WALT-BUG CPU%d; wallclock=%llu(0x%llx) is lesser than latest_clock=%llu(0x%llx)", + rq->cpu, wallclock, wallclock, wrq->latest_clock, + wrq->latest_clock); + WALT_PANIC(1); + } + delta = wallclock - wrq->window_start; + if (delta < 0) { + printk_deferred("WALT-BUG CPU%d; wallclock=%llu(0x%llx) is lesser than window_start=%llu(0x%llx)", + rq->cpu, wallclock, wallclock, + wrq->window_start, wrq->window_start); + WALT_PANIC(1); + } + wrq->latest_clock = wallclock; + if (delta < sched_ravg_window) + return old_window_start; + + nr_windows = div64_u64(delta, sched_ravg_window); + wrq->window_start += (u64)nr_windows * (u64)sched_ravg_window; + + wrq->prev_window_size = sched_ravg_window; + + full_window = nr_windows > 1; + rollover_cpu_window(rq, full_window); + rollover_top_tasks(rq, full_window); + + return old_window_start; +} + +/* + * Assumes rq_lock is held and wallclock was recorded in the same critical + * section as this function's invocation. + */ +static inline u64 read_cycle_counter(int cpu, u64 wallclock) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + if (wrq->last_cc_update != wallclock) { + wrq->cycles = qcom_cpufreq_get_cpu_cycle_counter(cpu); + wrq->last_cc_update = wallclock; + } + + return wrq->cycles; +} + +static void update_task_cpu_cycles(struct task_struct *p, int cpu, + u64 wallclock) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (use_cycle_counter) + wts->cpu_cycles = read_cycle_counter(cpu, wallclock); +} + +static inline bool is_ed_enabled(void) +{ + return (walt_rotation_enabled || (boost_policy != SCHED_BOOST_NONE)); +} + +static inline bool is_ed_task(struct task_struct *p, u64 wallclock) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wallclock - wts->last_wake_ts >= EARLY_DETECTION_DURATION); +} + +static bool is_ed_task_present(struct rq *rq, u64 wallclock, struct task_struct *deq_task) +{ + struct task_struct *p; + int loop_max = 10; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + wrq->ed_task = NULL; + + if (!is_ed_enabled() || !rq->cfs.h_nr_running) + return false; + + list_for_each_entry(p, &rq->cfs_tasks, se.group_node) { + if (!loop_max) + break; + + if (p == deq_task) + continue; + + if (is_ed_task(p, wallclock)) { + wrq->ed_task = p; + return true; + } + + loop_max--; + } + + return false; +} + +static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime); +/* + * Return total number of tasks "eligible" to run on higher capacity cpus + */ +unsigned int walt_big_tasks(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->walt_stats.nr_big_tasks; +} + +static void clear_walt_request(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + clear_reserved(cpu); + if (wrq->push_task) { + struct task_struct *push_task = NULL; + + raw_spin_lock_irqsave(&rq->__lock, flags); + if (wrq->push_task) { + clear_reserved(rq->push_cpu); + push_task = wrq->push_task; + wrq->push_task = NULL; + } + rq->active_balance = 0; + raw_spin_unlock_irqrestore(&rq->__lock, flags); + if (push_task) + put_task_struct(push_task); + } +} + +/* + * Special case the last index and provide a fast path for index = 0. + * Note that sched_load_granule can change underneath us if we are not + * holding any runqueue locks while calling the two functions below. + */ +static u32 top_task_load(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + int index = wrq->prev_top; + u8 prev = 1 - wrq->curr_table; + + if (!index) { + int msb = NUM_LOAD_INDICES - 1; + + if (!test_bit(msb, wrq->top_tasks_bitmap[prev])) + return 0; + else + return sched_load_granule; + } else if (index == NUM_LOAD_INDICES - 1) { + return sched_ravg_window; + } else { + return (index + 1) * sched_load_granule; + } +} + +unsigned long sched_user_hint_reset_time; +static bool is_cluster_hosting_top_app(struct walt_sched_cluster *cluster); + +static inline bool +should_apply_suh_freq_boost(struct walt_sched_cluster *cluster) +{ + if (sched_freq_aggr_en || !sysctl_sched_user_hint || + !cluster->aggr_grp_load) + return false; + + return is_cluster_hosting_top_app(cluster); +} + +static inline u64 freq_policy_load(struct rq *rq, unsigned int *reason) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + u64 aggr_grp_load = cluster->aggr_grp_load; + u64 load, tt_load = 0, kload = 0; + struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu_of(rq)); + + if (wrq->ed_task != NULL) { + load = sched_ravg_window; + *reason = CPUFREQ_REASON_EARLY_DET; + goto done; + } + + if (sched_freq_aggr_en) { + load = wrq->prev_runnable_sum + aggr_grp_load; + *reason = CPUFREQ_REASON_FREQ_AGR; + } + else + load = wrq->prev_runnable_sum + + wrq->grp_time.prev_runnable_sum; + + if (cpu_ksoftirqd && READ_ONCE(cpu_ksoftirqd->__state) == TASK_RUNNING) { + kload = task_load(cpu_ksoftirqd); + if (kload > load) { + load = kload; + *reason = CPUFREQ_REASON_KSOFTIRQD; + } + } + + tt_load = top_task_load(rq); + if (tt_load > load) { + load = tt_load; + *reason = CPUFREQ_REASON_TT_LOAD; + } + + if (should_apply_suh_freq_boost(cluster)) { + if (is_suh_max()) + load = sched_ravg_window; + else + load = div64_u64(load * sysctl_sched_user_hint, + (u64)100); + *reason = CPUFREQ_REASON_SUH; + } + +done: + trace_sched_load_to_gov(rq, aggr_grp_load, tt_load, sched_freq_aggr_en, + load, 0, walt_rotation_enabled, + sysctl_sched_user_hint, wrq, *reason); + return load; +} + +static bool rtgb_active; + +static inline unsigned long +__cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reason) +{ + u64 util; + struct rq *rq = cpu_rq(cpu); + unsigned long capacity = capacity_orig_of(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + util = scale_time_to_util(freq_policy_load(rq, reason)); + + /* + * util is on a scale of 0 to 1024. this is the utilization + * of the cpu in the last window + */ + wrq->util = util; + + if (walt_load) { + u64 nl = wrq->nt_prev_runnable_sum + + wrq->grp_time.nt_prev_runnable_sum; + u64 pl = wrq->walt_stats.pred_demands_sum_scaled; + + wrq->old_busy_time = util; + wrq->old_estimated_time = pl; + + nl = scale_time_to_util(nl); + walt_load->nl = nl; + walt_load->pl = pl; + walt_load->ws = walt_load_reported_window; + walt_load->rtgb_active = rtgb_active; + } + + return (util >= capacity) ? capacity : util; +} + +#define ADJUSTED_ASYM_CAP_CPU_UTIL(orig, other, x) \ + (max(orig, mult_frac(other, x, 100))) + +unsigned long +cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *reason) +{ + struct walt_cpu_load wl_other = {0}; + unsigned long util = 0, util_other = 0; + unsigned long capacity = capacity_orig_of(cpu); + int i, mpct = sysctl_sched_asym_cap_sibling_freq_match_pct; + + if (!cpumask_test_cpu(cpu, &asym_cap_sibling_cpus)) + return __cpu_util_freq_walt(cpu, walt_load, reason); + + for_each_cpu(i, &asym_cap_sibling_cpus) { + if (i == cpu) + util = __cpu_util_freq_walt(cpu, walt_load, reason); + else + util_other = __cpu_util_freq_walt(i, &wl_other, reason); + } + + if (cpu == cpumask_last(&asym_cap_sibling_cpus)) + mpct = 100; + + util = ADJUSTED_ASYM_CAP_CPU_UTIL(util, util_other, mpct); + + walt_load->nl = ADJUSTED_ASYM_CAP_CPU_UTIL(walt_load->nl, wl_other.nl, + mpct); + walt_load->pl = ADJUSTED_ASYM_CAP_CPU_UTIL(walt_load->pl, wl_other.pl, + mpct); + + return (util >= capacity) ? capacity : util; +} + +/* + * In this function we match the accumulated subtractions with the current + * and previous windows we are operating with. Ignore any entries where + * the window start in the load_subtraction struct does not match either + * the curent or the previous window. This could happen whenever CPUs + * become idle or busy with interrupts disabled for an extended period. + */ +static inline void account_load_subtractions(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 ws = wrq->window_start; + u64 prev_ws = ws - wrq->prev_window_size; + struct load_subtractions *ls = wrq->load_subs; + int i; + + for (i = 0; i < NUM_TRACKED_WINDOWS; i++) { + if (ls[i].window_start == ws) { + wrq->curr_runnable_sum -= ls[i].subs; + wrq->nt_curr_runnable_sum -= ls[i].new_subs; + } else if (ls[i].window_start == prev_ws) { + wrq->prev_runnable_sum -= ls[i].subs; + wrq->nt_prev_runnable_sum -= ls[i].new_subs; + } + + ls[i].subs = 0; + ls[i].new_subs = 0; + } + + if ((s64)wrq->prev_runnable_sum < 0) { + WALT_BUG(WALT_BUG_WALT, NULL, "wrq->prev_runnable_sum=%llu < 0", + (s64)wrq->prev_runnable_sum); + wrq->prev_runnable_sum = 0; + } + if ((s64)wrq->curr_runnable_sum < 0) { + WALT_BUG(WALT_BUG_WALT, NULL, "wrq->curr_runnable_sum=%llu < 0", + (s64)wrq->curr_runnable_sum); + wrq->curr_runnable_sum = 0; + } + if ((s64)wrq->nt_prev_runnable_sum < 0) { + WALT_BUG(WALT_BUG_WALT, NULL, "wrq->nt_prev_runnable_sum=%llu < 0", + (s64)wrq->nt_prev_runnable_sum); + wrq->nt_prev_runnable_sum = 0; + } + if ((s64)wrq->nt_curr_runnable_sum < 0) { + WALT_BUG(WALT_BUG_WALT, NULL, "wrq->nt_curr_runnable_sum=%llu < 0", + (s64)wrq->nt_curr_runnable_sum); + wrq->nt_curr_runnable_sum = 0; + } +} + +static inline void create_subtraction_entry(struct rq *rq, u64 ws, int index) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + wrq->load_subs[index].window_start = ws; + wrq->load_subs[index].subs = 0; + wrq->load_subs[index].new_subs = 0; +} + +static int get_top_index(unsigned long *bitmap, unsigned long old_top) +{ + int index = find_next_bit(bitmap, NUM_LOAD_INDICES, old_top); + + if (index == NUM_LOAD_INDICES) + return 0; + + return NUM_LOAD_INDICES - 1 - index; +} + +static bool get_subtraction_index(struct rq *rq, u64 ws) +{ + int i; + u64 oldest = ULLONG_MAX; + int oldest_index = 0; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + for (i = 0; i < NUM_TRACKED_WINDOWS; i++) { + u64 entry_ws = wrq->load_subs[i].window_start; + + if (ws == entry_ws) + return i; + + if (entry_ws < oldest) { + oldest = entry_ws; + oldest_index = i; + } + } + + create_subtraction_entry(rq, ws, oldest_index); + return oldest_index; +} + +static void update_rq_load_subtractions(int index, struct rq *rq, + u32 sub_load, bool new_task) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + wrq->load_subs[index].subs += sub_load; + if (new_task) + wrq->load_subs[index].new_subs += sub_load; +} + +static void update_cluster_load_subtractions(struct task_struct *p, + int cpu, u64 ws, bool new_task) +{ + struct walt_sched_cluster *cluster = cpu_cluster(cpu); + struct cpumask cluster_cpus = cluster->cpus; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + u64 prev_ws = ws - wrq->prev_window_size; + int i; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + cpumask_clear_cpu(cpu, &cluster_cpus); + raw_spin_lock(&cluster->load_lock); + + for_each_cpu(i, &cluster_cpus) { + struct rq *rq = cpu_rq(i); + int index; + + if (wts->curr_window_cpu[i]) { + index = get_subtraction_index(rq, ws); + update_rq_load_subtractions(index, rq, + wts->curr_window_cpu[i], new_task); + wts->curr_window_cpu[i] = 0; + } + + if (wts->prev_window_cpu[i]) { + index = get_subtraction_index(rq, prev_ws); + update_rq_load_subtractions(index, rq, + wts->prev_window_cpu[i], new_task); + wts->prev_window_cpu[i] = 0; + } + } + + raw_spin_unlock(&cluster->load_lock); +} + +static inline void migrate_inter_cluster_subtraction(struct task_struct *p, int task_cpu, + bool new_task) +{ + struct rq *src_rq = cpu_rq(task_cpu); + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (src_wrq->curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, + src_wrq->curr_runnable_sum, + wts->curr_window_cpu[task_cpu]); + src_wrq->curr_runnable_sum = wts->curr_window_cpu[task_cpu]; + } + src_wrq->curr_runnable_sum -= wts->curr_window_cpu[task_cpu]; + + if (src_wrq->prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, + src_wrq->prev_runnable_sum, + wts->prev_window_cpu[task_cpu]); + src_wrq->prev_runnable_sum = wts->prev_window_cpu[task_cpu]; + } + src_wrq->prev_runnable_sum -= wts->prev_window_cpu[task_cpu]; + + if (new_task) { + if (src_wrq->nt_curr_runnable_sum < wts->curr_window_cpu[task_cpu]) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, + src_wrq->nt_curr_runnable_sum, + wts->curr_window_cpu[task_cpu]); + src_wrq->nt_curr_runnable_sum = wts->curr_window_cpu[task_cpu]; + } + src_wrq->nt_curr_runnable_sum -= + wts->curr_window_cpu[task_cpu]; + + if (src_wrq->nt_prev_runnable_sum < wts->prev_window_cpu[task_cpu]) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, src_rq->cpu, + src_wrq->nt_prev_runnable_sum, + wts->prev_window_cpu[task_cpu]); + src_wrq->nt_prev_runnable_sum = wts->prev_window_cpu[task_cpu]; + } + src_wrq->nt_prev_runnable_sum -= + wts->prev_window_cpu[task_cpu]; + } + + wts->curr_window_cpu[task_cpu] = 0; + wts->prev_window_cpu[task_cpu] = 0; + + update_cluster_load_subtractions(p, task_cpu, + src_wrq->window_start, new_task); +} + +static inline void migrate_inter_cluster_addition(struct task_struct *p, int new_cpu, + bool new_task) +{ + struct rq *dest_rq = cpu_rq(new_cpu); + struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + + wts->curr_window_cpu[new_cpu] = wts->curr_window; + wts->prev_window_cpu[new_cpu] = wts->prev_window; + + dest_wrq->curr_runnable_sum += wts->curr_window; + dest_wrq->prev_runnable_sum += wts->prev_window; + + if (new_task) { + dest_wrq->nt_curr_runnable_sum += wts->curr_window; + dest_wrq->nt_prev_runnable_sum += wts->prev_window; + } +} + +static u32 load_to_index(u32 load) +{ + u32 index = load / sched_load_granule; + + return min(index, (u32)(NUM_LOAD_INDICES - 1)); +} + +static void migrate_top_tasks_subtraction(struct task_struct *p, struct rq *src_rq) +{ + int index; + int top_index; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u32 curr_window = wts->curr_window; + u32 prev_window = wts->prev_window; + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + u8 src = src_wrq->curr_table; + u8 *src_table; + + if (curr_window) { + src_table = src_wrq->top_tasks[src]; + index = load_to_index(curr_window); + src_table[index] -= 1; + + if (!src_table[index]) + __clear_bit(NUM_LOAD_INDICES - index - 1, + src_wrq->top_tasks_bitmap[src]); + + top_index = src_wrq->curr_top; + if (index == top_index && !src_table[index]) + src_wrq->curr_top = get_top_index( + src_wrq->top_tasks_bitmap[src], top_index); + } + + if (prev_window) { + src = 1 - src; + src_table = src_wrq->top_tasks[src]; + index = load_to_index(prev_window); + src_table[index] -= 1; + + if (!src_table[index]) + __clear_bit(NUM_LOAD_INDICES - index - 1, + src_wrq->top_tasks_bitmap[src]); + + top_index = src_wrq->prev_top; + if (index == top_index && !src_table[index]) + src_wrq->prev_top = get_top_index( + src_wrq->top_tasks_bitmap[src], top_index); + } +} + +static void migrate_top_tasks_addition(struct task_struct *p, struct rq *rq) +{ + int index; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u32 curr_window = wts->curr_window; + u32 prev_window = wts->prev_window; + struct walt_rq *dst_wrq = (struct walt_rq *) rq->android_vendor_data1; + u8 dst = dst_wrq->curr_table; + u8 *dst_table; + + if (curr_window) { + dst_table = dst_wrq->top_tasks[dst]; + index = load_to_index(curr_window); + dst_table[index] += 1; + + if (dst_table[index] == 1) + __set_bit(NUM_LOAD_INDICES - index - 1, + dst_wrq->top_tasks_bitmap[dst]); + + if (index > dst_wrq->curr_top) + dst_wrq->curr_top = index; + } + + if (prev_window) { + dst = 1 - dst; + dst_table = dst_wrq->top_tasks[dst]; + index = load_to_index(prev_window); + dst_table[index] += 1; + + if (dst_table[index] == 1) + __set_bit(NUM_LOAD_INDICES - index - 1, + dst_wrq->top_tasks_bitmap[dst]); + + if (index > dst_wrq->prev_top) + dst_wrq->prev_top = index; + } +} + +static inline bool is_new_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->active_time < NEW_TASK_ACTIVE_TIME; +} +static inline void run_walt_irq_work_rollover(u64 old_window_start, struct rq *rq); + +static void migrate_busy_time_subtraction(struct task_struct *p, int new_cpu) +{ + struct rq *src_rq = task_rq(p); + u64 wallclock; + u64 *src_curr_runnable_sum, *src_prev_runnable_sum; + u64 *src_nt_curr_runnable_sum, *src_nt_prev_runnable_sum; + bool new_task; + struct walt_related_thread_group *grp; + long pstate; + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!p->on_rq && READ_ONCE(p->__state) != TASK_WAKING) + return; + + pstate = READ_ONCE(p->__state); + + if (pstate == TASK_WAKING) + raw_spin_rq_lock(src_rq); + + lockdep_assert_held(&src_rq->__lock); + + if (task_rq(p) != src_rq) + WALT_BUG(WALT_BUG_UPSTREAM, p, "on CPU %d task %s(%d) not on src_rq %d", + raw_smp_processor_id(), p->comm, p->pid, src_rq->cpu); + + wts->new_cpu = new_cpu; + + if (!same_freq_domain(task_cpu(p), new_cpu)) + wts->enqueue_after_migration = 2; /* 2 is intercluster */ + else + wts->enqueue_after_migration = 1; /* 1 is within cluster */ + + wallclock = walt_sched_clock(); + walt_update_task_ravg(p, task_rq(p), TASK_MIGRATE, wallclock, 0); + + if (wts->window_start != src_wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s task %s(%d)'s ws=%llu not equal to src_rq %d's ws=%llu", + __func__, raw_smp_processor_id(), p->comm, p->pid, + wts->window_start, src_rq->cpu, src_wrq->window_start); + + + /* safe to update the task cyc cntr for new_cpu without the new_cpu rq_lock */ + update_task_cpu_cycles(p, new_cpu, wallclock); + + new_task = is_new_task(p); + /* Protected by rq_lock */ + grp = wts->grp; + + /* + * For frequency aggregation, we continue to do migration fixups + * even for intra cluster migrations. This is because, the aggregated + * load has to reported on a single CPU regardless. + */ + if (grp) { + struct group_cpu_time *cpu_time = &src_wrq->grp_time; + + src_curr_runnable_sum = &cpu_time->curr_runnable_sum; + src_prev_runnable_sum = &cpu_time->prev_runnable_sum; + src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + if (wts->curr_window) { + *src_curr_runnable_sum -= wts->curr_window; + if (new_task) + *src_nt_curr_runnable_sum -= wts->curr_window; + } + + if (wts->prev_window) { + *src_prev_runnable_sum -= wts->prev_window; + if (new_task) + *src_nt_prev_runnable_sum -= wts->prev_window; + } + } else { + if (wts->enqueue_after_migration == 2) + migrate_inter_cluster_subtraction(p, task_cpu(p), new_task); + } + + migrate_top_tasks_subtraction(p, src_rq); + + if (is_ed_enabled() && (p == src_wrq->ed_task)) + src_wrq->ed_task = NULL; + + wts->prev_cpu = task_cpu(p); + + if (pstate == TASK_WAKING) + raw_spin_rq_unlock(src_rq); +} + +static void migrate_busy_time_addition(struct task_struct *p, int new_cpu, u64 wallclock) +{ + struct rq *dest_rq = cpu_rq(new_cpu); + u64 *dst_curr_runnable_sum, *dst_prev_runnable_sum; + u64 *dst_nt_curr_runnable_sum, *dst_nt_prev_runnable_sum; + bool new_task; + struct walt_related_thread_group *grp; + struct walt_rq *dest_wrq = (struct walt_rq *) dest_rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + int src_cpu = wts->prev_cpu; + struct rq *src_rq = cpu_rq(src_cpu); + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + + walt_update_task_ravg(p, dest_rq, TASK_UPDATE, wallclock, 0); + + if (wts->window_start != dest_wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s task %s(%d)'s ws=%llu not equal to dest_rq %d's ws=%llu", + __func__, raw_smp_processor_id(), p->comm, p->pid, + wts->window_start, dest_rq->cpu, dest_wrq->window_start); + + new_task = is_new_task(p); + /* Protected by rq_lock */ + grp = wts->grp; + + /* + * For frequency aggregation, we continue to do migration fixups + * even for intra cluster migrations. This is because, the aggregated + * load has to reported on a single CPU regardless. + */ + if (grp) { + struct group_cpu_time *cpu_time = &dest_wrq->grp_time; + + dst_curr_runnable_sum = &cpu_time->curr_runnable_sum; + dst_prev_runnable_sum = &cpu_time->prev_runnable_sum; + dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + if (wts->curr_window) { + *dst_curr_runnable_sum += wts->curr_window; + if (new_task) + *dst_nt_curr_runnable_sum += wts->curr_window; + } + + if (wts->prev_window) { + *dst_prev_runnable_sum += wts->prev_window; + if (new_task) + *dst_nt_prev_runnable_sum += wts->prev_window; + } + } else { + if (wts->enqueue_after_migration == 2) + migrate_inter_cluster_addition(p, new_cpu, new_task); + } + + migrate_top_tasks_addition(p, dest_rq); + + if (wts->enqueue_after_migration == 2) { + src_wrq->notif_pending = true; + dest_wrq->notif_pending = true; + walt_irq_work_queue(&walt_migration_irq_work); + } + + if (is_ed_enabled() && is_ed_task(p, wallclock)) + dest_wrq->ed_task = p; + + wts->enqueue_after_migration = 0; + wts->new_cpu = -1; +} + +#define INC_STEP 8 +#define DEC_STEP 2 +#define CONSISTENT_THRES 16 +#define INC_STEP_BIG 16 +/* + * bucket_increase - update the count of all buckets + * + * @buckets: array of buckets tracking busy time of a task + * @idx: the index of bucket to be incremented + * + * Each time a complete window finishes, count of bucket that runtime + * falls in (@idx) is incremented. Counts of all other buckets are + * decayed. The rate of increase and decay could be different based + * on current count in the bucket. + */ +static inline void bucket_increase(u8 *buckets, u16 *bucket_bitmask, int idx) +{ + int i, step; + + for (i = 0; i < NUM_BUSY_BUCKETS; i++) { + if (idx != i) { + if (buckets[i] > DEC_STEP) + buckets[i] -= DEC_STEP; + else { + buckets[i] = 0; + *bucket_bitmask &= ~BIT_MASK(i); + } + } else { + step = buckets[i] >= CONSISTENT_THRES ? + INC_STEP_BIG : INC_STEP; + if (buckets[i] > U8_MAX - step) + buckets[i] = U8_MAX; + else + buckets[i] += step; + *bucket_bitmask |= BIT_MASK(i); + } + } +} + +static inline int busy_to_bucket(u16 normalized_rt) +{ + int bidx; + + bidx = normalized_rt >> (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); + bidx = min(bidx, NUM_BUSY_BUCKETS - 1); + + /* + * Combine lowest two buckets. The lowest frequency falls into + * 2nd bucket and thus keep predicting lowest bucket is not + * useful. + */ + if (!bidx) + bidx++; + + return bidx; +} + +/* + * get_pred_busy - calculate predicted demand for a task on runqueue + * + * @p: task whose prediction is being updated + * @start: starting bucket. returned prediction should not be lower than + * this bucket. + * @runtime: runtime of the task. returned prediction should not be lower + * than this runtime. + * Note: @start can be derived from @runtime. It's passed in only to + * avoid duplicated calculation in some cases. + * + * A new predicted busy time is returned for task @p based on @runtime + * passed in. The function searches through buckets that represent busy + * time equal to or bigger than @runtime and attempts to find the bucket + * to use for prediction. Once found, it searches through historical busy + * time and returns the latest that falls into the bucket. If no such busy + * time exists, it returns the medium of that bucket. + */ +static u32 get_pred_busy(struct task_struct *p, + int start, u16 runtime_scaled, u16 bucket_bitmask) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u16 dmin, dmax; + int first = NUM_BUSY_BUCKETS, final = NUM_BUSY_BUCKETS; + u16 ret = runtime_scaled; + u16 next_mask = bucket_bitmask >> start; + u16 *hist_util = wts->sum_history_util; + int i; + + /* skip prediction for new tasks due to lack of history */ + if (unlikely(is_new_task(p))) + goto out; + + /* find minimal bucket index to pick */ + if (next_mask) + first = ffs(next_mask) - 1 + start; + + /* if no higher buckets are filled, predict runtime */ + if (first >= NUM_BUSY_BUCKETS) + goto out; + + /* compute the bucket for prediction */ + final = first; + + /* determine demand range for the predicted bucket */ + if (final < 2) { + /* lowest two buckets are combined */ + dmin = 0; + final = 1; + } else { + dmin = final << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); + } + dmax = (final + 1) << (SCHED_CAPACITY_SHIFT - NUM_BUSY_BUCKETS_SHIFT); + + /* + * search through runtime history and return first runtime that falls + * into the range of predicted bucket. + */ + for (i = 0; i < RAVG_HIST_SIZE; i++) { + if (hist_util[i] >= dmin && hist_util[i] < dmax) { + ret = hist_util[i]; + break; + } + } + /* no historical runtime within bucket found, use average of the bin */ + if (ret < dmin) + ret = (u16) (((u32)dmin + dmax) / 2); + /* + * when updating in middle of a window, runtime could be higher + * than all recorded history. Always predict at least runtime. + */ + ret = max(runtime_scaled, ret); +out: + trace_sched_update_pred_demand(p, runtime_scaled, + ret, start, first, final, wts); + return ret; +} + +/* + * predictive demand of a task was calculated at the last window roll-over. + * if the task current window busy time exceeds the predicted + * demand, update it here to reflect the task needs. + */ +static void update_task_pred_demand(struct rq *rq, struct task_struct *p, int event) +{ + u16 new_pred_demand_scaled; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u16 curr_window_scaled; + + if (is_idle_task(p)) + return; + + if (event != PUT_PREV_TASK && event != TASK_UPDATE && + (!SCHED_FREQ_ACCOUNT_WAIT_TIME || + (event != TASK_MIGRATE && + event != PICK_NEXT_TASK))) + return; + + /* + * TASK_UPDATE can be called on sleeping task, when its moved between + * related groups + */ + if (event == TASK_UPDATE) { + if (!p->on_rq && !SCHED_FREQ_ACCOUNT_WAIT_TIME) + return; + } + + curr_window_scaled = scale_time_to_util(wts->curr_window); + if (wts->pred_demand_scaled >= curr_window_scaled) + return; + + new_pred_demand_scaled = get_pred_busy(p, busy_to_bucket(curr_window_scaled), + curr_window_scaled, wts->bucket_bitmask); + + if (task_on_rq_queued(p) && (!task_has_dl_policy(p) || + !p->dl.dl_throttled)) + fixup_walt_sched_stats_common(rq, p, + wts->demand_scaled, + new_pred_demand_scaled); + + wts->pred_demand_scaled = new_pred_demand_scaled; +} + +static void clear_top_tasks_bitmap(unsigned long *bitmap) +{ + memset(bitmap, 0, top_tasks_bitmap_size); + __set_bit(NUM_LOAD_INDICES, bitmap); +} + +static inline void clear_top_tasks_table(u8 *table) +{ + memset(table, 0, NUM_LOAD_INDICES * sizeof(u8)); +} + +static void update_top_tasks(struct task_struct *p, struct rq *rq, + u32 old_curr_window, int new_window, bool full_window) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u8 curr = wrq->curr_table; + u8 prev = 1 - curr; + u8 *curr_table = wrq->top_tasks[curr]; + u8 *prev_table = wrq->top_tasks[prev]; + int old_index, new_index, update_index; + u32 curr_window = wts->curr_window; + u32 prev_window = wts->prev_window; + bool zero_index_update; + + if (old_curr_window == curr_window && !new_window) + return; + + old_index = load_to_index(old_curr_window); + new_index = load_to_index(curr_window); + + if (!new_window) { + zero_index_update = !old_curr_window && curr_window; + if (old_index != new_index || zero_index_update) { + if (old_curr_window) + curr_table[old_index] -= 1; + if (curr_window) + curr_table[new_index] += 1; + if (new_index > wrq->curr_top) + wrq->curr_top = new_index; + } + + if (!curr_table[old_index]) + __clear_bit(NUM_LOAD_INDICES - old_index - 1, + wrq->top_tasks_bitmap[curr]); + + if (curr_table[new_index] == 1) + __set_bit(NUM_LOAD_INDICES - new_index - 1, + wrq->top_tasks_bitmap[curr]); + + return; + } + + /* + * The window has rolled over for this task. By the time we get + * here, curr/prev swaps would has already occurred. So we need + * to use prev_window for the new index. + */ + update_index = load_to_index(prev_window); + + if (full_window) { + /* + * Two cases here. Either 'p' ran for the entire window or + * it didn't run at all. In either case there is no entry + * in the prev table. If 'p' ran the entire window, we just + * need to create a new entry in the prev table. In this case + * update_index will be correspond to sched_ravg_window + * so we can unconditionally update the top index. + */ + if (prev_window) { + prev_table[update_index] += 1; + wrq->prev_top = update_index; + } + + if (prev_table[update_index] == 1) + __set_bit(NUM_LOAD_INDICES - update_index - 1, + wrq->top_tasks_bitmap[prev]); + } else { + zero_index_update = !old_curr_window && prev_window; + if (old_index != update_index || zero_index_update) { + if (old_curr_window) + prev_table[old_index] -= 1; + + prev_table[update_index] += 1; + + if (update_index > wrq->prev_top) + wrq->prev_top = update_index; + + if (!prev_table[old_index]) + __clear_bit(NUM_LOAD_INDICES - old_index - 1, + wrq->top_tasks_bitmap[prev]); + + if (prev_table[update_index] == 1) + __set_bit(NUM_LOAD_INDICES - update_index - 1, + wrq->top_tasks_bitmap[prev]); + } + } + + if (curr_window) { + curr_table[new_index] += 1; + + if (new_index > wrq->curr_top) + wrq->curr_top = new_index; + + if (curr_table[new_index] == 1) + __set_bit(NUM_LOAD_INDICES - new_index - 1, + wrq->top_tasks_bitmap[curr]); + } +} + +static void rollover_top_tasks(struct rq *rq, bool full_window) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u8 curr_table = wrq->curr_table; + u8 prev_table = 1 - curr_table; + int curr_top = wrq->curr_top; + + clear_top_tasks_table(wrq->top_tasks[prev_table]); + clear_top_tasks_bitmap(wrq->top_tasks_bitmap[prev_table]); + + if (full_window) { + curr_top = 0; + clear_top_tasks_table(wrq->top_tasks[curr_table]); + clear_top_tasks_bitmap(wrq->top_tasks_bitmap[curr_table]); + } + + wrq->curr_table = prev_table; + wrq->prev_top = curr_top; + wrq->curr_top = 0; +} + +static u32 empty_windows[WALT_NR_CPUS]; + +static void rollover_task_window(struct task_struct *p, bool full_window) +{ + u32 *curr_cpu_windows = empty_windows; + u32 curr_window; + int i; + struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + /* Rollover the sum */ + curr_window = 0; + + if (!full_window) { + curr_window = wts->curr_window; + curr_cpu_windows = wts->curr_window_cpu; + } + + wts->prev_window = curr_window; + wts->curr_window = 0; + + /* Roll over individual CPU contributions */ + for (i = 0; i < nr_cpu_ids; i++) { + wts->prev_window_cpu[i] = curr_cpu_windows[i]; + wts->curr_window_cpu[i] = 0; + } + + if (is_new_task(p)) + wts->active_time += wrq->prev_window_size; +} + +static inline int cpu_is_waiting_on_io(struct rq *rq) +{ + if (!sched_io_is_busy) + return 0; + + return atomic_read(&rq->nr_iowait); +} + +static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, + u64 irqtime, int event) +{ + if (is_idle_task(p)) { + /* TASK_WAKE && TASK_MIGRATE is not possible on idle task! */ + if (event == PICK_NEXT_TASK) + return 0; + + /* PUT_PREV_TASK, TASK_UPDATE && IRQ_UPDATE are left */ + return irqtime || cpu_is_waiting_on_io(rq); + } + + if (event == TASK_WAKE) + return 0; + + if (event == PUT_PREV_TASK || event == IRQ_UPDATE) + return 1; + + /* + * TASK_UPDATE can be called on sleeping task, when its moved between + * related groups + */ + if (event == TASK_UPDATE) { + if (rq->curr == p) + return 1; + + return p->on_rq ? SCHED_FREQ_ACCOUNT_WAIT_TIME : 0; + } + + /* TASK_MIGRATE, PICK_NEXT_TASK left */ + return SCHED_FREQ_ACCOUNT_WAIT_TIME; +} + +#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) + +static inline u64 scale_exec_time(u64 delta, struct rq *rq, struct walt_task_struct *wts) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + delta = (delta * wrq->task_exec_scale) >> SCHED_CAPACITY_SHIFT; + + if (wts->load_boost && wts->grp && wts->grp->skip_min) + delta = (delta * (1024 + wts->boosted_task_load) >> 10); + + return delta; +} + +/* Convert busy time to frequency equivalent + * Assumes load is scaled to 1024 + */ +static inline unsigned int load_to_freq(struct rq *rq, unsigned int load) +{ + return mult_frac(cpu_max_possible_freq(cpu_of(rq)), load, + (unsigned int)arch_scale_cpu_capacity(cpu_of(rq))); +} + +static bool do_pl_notif(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 prev = wrq->old_busy_time; + u64 pl = wrq->walt_stats.pred_demands_sum_scaled; + int cpu = cpu_of(rq); + + /* If already at max freq, bail out */ + if (capacity_orig_of(cpu) == capacity_curr_of(cpu)) + return false; + + prev = max(prev, wrq->old_estimated_time); + + /* 400 MHz filter. */ + return (pl > prev) && (load_to_freq(rq, pl - prev) > 400000); +} + +static void rollover_cpu_window(struct rq *rq, bool full_window) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 curr_sum = wrq->curr_runnable_sum; + u64 nt_curr_sum = wrq->nt_curr_runnable_sum; + u64 grp_curr_sum = wrq->grp_time.curr_runnable_sum; + u64 grp_nt_curr_sum = wrq->grp_time.nt_curr_runnable_sum; + + if (unlikely(full_window)) { + curr_sum = 0; + nt_curr_sum = 0; + grp_curr_sum = 0; + grp_nt_curr_sum = 0; + } + + wrq->prev_runnable_sum = curr_sum; + wrq->nt_prev_runnable_sum = nt_curr_sum; + wrq->grp_time.prev_runnable_sum = grp_curr_sum; + wrq->grp_time.nt_prev_runnable_sum = grp_nt_curr_sum; + + wrq->curr_runnable_sum = 0; + wrq->nt_curr_runnable_sum = 0; + wrq->grp_time.curr_runnable_sum = 0; + wrq->grp_time.nt_curr_runnable_sum = 0; +} + +/* + * Account cpu activity in its + * busy time counters(wrq->curr/prev_runnable_sum) + * + * While the comments at the top of update_task_demand() apply, irqtime handling + * needs some explanation. + * + * Note that update_task_ravg() with irqtime is only called when idle, i.e. p is + * always idle + * + * ms_i = mark_start of idle task + * ws = wrq->window_start + * irq_s = start time of irq + * irq_e = end time of irq = wallclock + * + * note irqtime = irq_e - irq_s + * + * Similar to the explanation at update_task_demand() we have few sitautions for irqtime + * + * ws ms_i is ie + * | | | | + * V V V V + * --------|--------------------| + * prev curr + * + * In the above case, new_window is false and irqtime is accounted in curr_runnable_sum, this is + * done in the if (!new_window) block. + * + * ms_i ws is ie + * | | | | + * V V V V + * -------------|--------------------- + * prev curr + * + * In this case, new_window is true, however the irqtime falls within the current window, the + * entire irqtime is accounted in curr_runnable_sum. This is handled in the if (irqtime) block and + * within that if (mark_start > window_start) block + * + * ms_i is ws ie + * | | | | + * V V V V + * --------------------|--------------- + * prev curr + * + * In this case, new_window is true, portion of the irqtime needs to be accounted in + * prev_runnable_sum while the rest is in curr_runnable_sum. This is handled in the + * if (irqtime) block + */ +static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) +{ + int new_window, full_window = 0; + int p_is_curr_task = (p == rq->curr); + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u64 mark_start = wts->mark_start; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 window_start = wrq->window_start; + u32 window_size = wrq->prev_window_size; + u64 delta; + u64 *curr_runnable_sum = &wrq->curr_runnable_sum; + u64 *prev_runnable_sum = &wrq->prev_runnable_sum; + u64 *nt_curr_runnable_sum = &wrq->nt_curr_runnable_sum; + u64 *nt_prev_runnable_sum = &wrq->nt_prev_runnable_sum; + bool new_task; + struct walt_related_thread_group *grp; + int cpu = rq->cpu; + u32 old_curr_window = wts->curr_window; + + new_window = mark_start < window_start; + if (new_window) + full_window = (window_start - mark_start) >= window_size; + + /* + * Handle per-task window rollover. We don't care about the + * idle task. + */ + if (new_window) { + if (!is_idle_task(p)) + rollover_task_window(p, full_window); + wts->window_start = window_start; + } + + new_task = is_new_task(p); + + if (!account_busy_for_cpu_time(rq, p, irqtime, event)) + goto done; + + grp = wts->grp; + if (grp) { + struct group_cpu_time *cpu_time = &wrq->grp_time; + + curr_runnable_sum = &cpu_time->curr_runnable_sum; + prev_runnable_sum = &cpu_time->prev_runnable_sum; + + nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + } + + if (wts->window_start != wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s task %s(%d)'s ws=%llu not equal to rq %d's ws=%llu", + __func__, raw_smp_processor_id(), p->comm, p->pid, + wts->window_start, rq->cpu, wrq->window_start); + + if (!new_window) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. No rollover + * since we didn't start a new window. An example of this is + * when a task starts execution and then sleeps within the + * same window. + */ + + if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) + delta = wallclock - mark_start; + else + delta = irqtime; + delta = scale_exec_time(delta, rq, wts); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + if (!is_idle_task(p)) { + wts->curr_window += delta; + wts->curr_window_cpu[cpu] += delta; + } + + goto done; + } + + /* + * situations below this need window rollover, + * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done + * in update_window_start() + * + * For task counters curr/prev_window[_cpu] are rolled over in the early part of + * this function. If full_window(s) have expired and time since last update needs + * to be accounted as busy time, set the prev to a complete window size time, else + * add the prev window portion. + * + * For task curr counters a new window has begun, always assign + */ + + if (!p_is_curr_task) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * must have been started in udpate_window_start() + * - just split up and account as necessary into curr and prev. + * + * Irqtime can't be accounted by a task that isn't the + * currently running task. + */ + + if (!full_window) { + /* + * A full window hasn't elapsed, account partial + * contribution to previous completed window. + */ + delta = scale_exec_time(window_start - mark_start, rq, wts); + wts->prev_window += delta; + wts->prev_window_cpu[cpu] += delta; + } else { + /* + * Since at least one full window has elapsed, + * the contribution to the previous window is the + * full window (window_size). + */ + delta = scale_exec_time(window_size, rq, wts); + wts->prev_window = delta; + wts->prev_window_cpu[cpu] = delta; + } + + *prev_runnable_sum += delta; + if (new_task) + *nt_prev_runnable_sum += delta; + + /* Account piece of busy time in the current window. */ + delta = scale_exec_time(wallclock - window_start, rq, wts); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + wts->curr_window = delta; + wts->curr_window_cpu[cpu] = delta; + + goto done; + } + + if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * must have been started in udpate_window_start() + * If any of these three above conditions are true + * then this busy time can't be accounted as irqtime. + * + * Busy time for the idle task need not be accounted. + * + * An example of this would be a task that starts execution + * and then sleeps once a new window has begun. + */ + + if (!full_window) { + /* + * A full window hasn't elapsed, account partial + * contribution to previous completed window. + */ + delta = scale_exec_time(window_start - mark_start, rq, wts); + if (!is_idle_task(p)) { + wts->prev_window += delta; + wts->prev_window_cpu[cpu] += delta; + } + } else { + /* + * Since at least one full window has elapsed, + * the contribution to the previous window is the + * full window (window_size). + */ + delta = scale_exec_time(window_size, rq, wts); + if (!is_idle_task(p)) { + wts->prev_window = delta; + wts->prev_window_cpu[cpu] = delta; + } + } + + *prev_runnable_sum += delta; + if (new_task) + *nt_prev_runnable_sum += delta; + + /* Account piece of busy time in the current window. */ + delta = scale_exec_time(wallclock - window_start, rq, wts); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + if (!is_idle_task(p)) { + wts->curr_window = delta; + wts->curr_window_cpu[cpu] = delta; + } + + goto done; + } + + if (irqtime) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * must have been started in udpate_window_start() + * The current task must be the idle task because + * irqtime is not accounted for any other task. + * + * Irqtime will be accounted each time we process IRQ activity + * after a period of idleness, so we know the IRQ busy time + * started at wallclock - irqtime. + */ + + WALT_PANIC(!is_idle_task(p)); + /* mark_start here becomes the starting time of interrupt */ + mark_start = wallclock - irqtime; + + /* + * If IRQ busy time was just in the current + * window then that is all that need be accounted. + */ + if (mark_start > window_start) { + *curr_runnable_sum += scale_exec_time(irqtime, rq, wts); + return; + } + + /* + * The IRQ busy time spanned multiple windows. Process the + * busy time preceding the current window start first. + */ + delta = window_start - mark_start; + if (delta > window_size) + delta = window_size; + delta = scale_exec_time(delta, rq, wts); + *prev_runnable_sum += delta; + + /* Process the remaining IRQ busy time in the current window. */ + delta = wallclock - window_start; + wrq->curr_runnable_sum += scale_exec_time(delta, rq, wts); + + return; + } + +done: + if (!is_idle_task(p)) + update_top_tasks(p, rq, old_curr_window, + new_window, full_window); +} + +static inline u16 predict_and_update_buckets( + struct task_struct *p, u16 runtime_scaled) { + int bidx; + u32 pred_demand_scaled; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + bidx = busy_to_bucket(runtime_scaled); + pred_demand_scaled = get_pred_busy(p, bidx, runtime_scaled, wts->bucket_bitmask); + bucket_increase(wts->busy_buckets, &wts->bucket_bitmask, bidx); + + return pred_demand_scaled; +} + +static int +account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event) +{ + /* + * No need to bother updating task demand for the idle task. + */ + if (is_idle_task(p)) + return 0; + + /* + * When a task is waking up it is completing a segment of non-busy + * time. Likewise, if wait time is not treated as busy time, then + * when a task begins to run or is migrated, it is not running and + * is completing a segment of non-busy time. + */ + if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && + (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) + return 0; + + /* + * The idle exit time is not accounted for the first task _picked_ up to + * run on the idle CPU. + */ + if (event == PICK_NEXT_TASK && rq->curr == rq->idle) + return 0; + + /* + * TASK_UPDATE can be called on sleeping task, when its moved between + * related groups + */ + if (event == TASK_UPDATE) { + if (rq->curr == p) + return 1; + + return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0; + } + + return 1; +} + +/* + * Called when new window is starting for a task, to record cpu usage over + * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 + * when, say, a real-time task runs without preemption for several windows at a + * stretch. + */ +static void update_history(struct rq *rq, struct task_struct *p, + u32 runtime, int samples, int event) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u32 *hist = &wts->sum_history[0]; + u16 *hist_util = &wts->sum_history_util[0]; + int i; + u32 max = 0, avg, demand; + u64 sum = 0; + u16 demand_scaled, pred_demand_scaled, runtime_scaled; + + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + /* Ignore windows where task had no activity */ + if (!runtime || is_idle_task(p) || !samples) + goto done; + + runtime_scaled = scale_time_to_util(runtime); + /* Push new 'runtime' value onto stack */ + for (; samples > 0; samples--) { + hist[wts->cidx] = runtime; + hist_util[wts->cidx] = runtime_scaled; + wts->cidx = ++(wts->cidx) % RAVG_HIST_SIZE; + } + + for (i = 0; i < RAVG_HIST_SIZE; i++) { + sum += hist[i]; + if (hist[i] > max) + max = hist[i]; + } + + wts->sum = 0; + + if (sysctl_sched_window_stats_policy == WINDOW_STATS_RECENT) { + demand = runtime; + } else if (sysctl_sched_window_stats_policy == WINDOW_STATS_MAX) { + demand = max; + } else { + avg = div64_u64(sum, RAVG_HIST_SIZE); + if (sysctl_sched_window_stats_policy == WINDOW_STATS_AVG) + demand = avg; + else + demand = max(avg, runtime); + } + pred_demand_scaled = predict_and_update_buckets(p, runtime_scaled); + demand_scaled = scale_time_to_util(demand); + + /* + * A throttled deadline sched class task gets dequeued without + * changing p->on_rq. Since the dequeue decrements walt stats + * avoid decrementing it here again. + * + * When window is rolled over, the cumulative window demand + * is reset to the cumulative runnable average (contribution from + * the tasks on the runqueue). If the current task is dequeued + * already, it's demand is not included in the cumulative runnable + * average. So add the task demand separately to cumulative window + * demand. + */ + if (!task_has_dl_policy(p) || !p->dl.dl_throttled) { + if (task_on_rq_queued(p)) + fixup_walt_sched_stats_common(rq, p, + demand_scaled, pred_demand_scaled); + } + + wts->demand = demand; + wts->demand_scaled = demand_scaled; + wts->coloc_demand = div64_u64(sum, RAVG_HIST_SIZE); + wts->pred_demand_scaled = pred_demand_scaled; + + if (demand_scaled > sysctl_sched_min_task_util_for_colocation) + wts->unfilter = sysctl_sched_task_unfilter_period; + else + if (wts->unfilter) + wts->unfilter = max_t(int, 0, + wts->unfilter - wrq->prev_window_size); + +done: + trace_sched_update_history(rq, p, runtime, samples, event, wrq, wts); +} + +static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + delta = scale_exec_time(delta, rq, wts); + wts->sum += delta; + if (unlikely(wts->sum > sched_ravg_window)) + wts->sum = sched_ravg_window; + + return delta; +} + +/* + * Account cpu demand of task and/or update task's cpu demand history + * + * ms = wts->mark_start; + * wc = wallclock + * ws = wrq->window_start + * + * Three possibilities: + * + * a) Task event is contained within one window. + * window_start < mark_start < wallclock + * + * ws ms wc + * | | | + * V V V + * |---------------| + * + * In this case, wts->sum is updated *iff* event is appropriate + * (ex: event == PUT_PREV_TASK) + * + * b) Task event spans two windows. + * mark_start < window_start < wallclock + * + * ms ws wc + * | | | + * V V V + * -----|------------------- + * + * In this case, wts->sum is updated with (ws - ms) *iff* event + * is appropriate, then a new window sample is recorded followed + * by wts->sum being set to (wc - ws) *iff* event is appropriate. + * + * c) Task event spans more than two windows. + * + * ms ws_tmp ws wc + * | | | | + * V V V V + * ---|-------|-------|-------|-------|------ + * | | + * |<------ nr_full_windows ------>| + * + * In this case, wts->sum is updated with (ws_tmp - ms) first *iff* + * event is appropriate, window sample of wts->sum is recorded, + * 'nr_full_window' samples of window_size is also recorded *iff* + * event is appropriate and finally wts->sum is set to (wc - ws) + * *iff* event is appropriate. + * + * IMPORTANT : Leave wts->mark_start unchanged, as update_cpu_busy_time() + * depends on it! + */ +static u64 update_task_demand(struct task_struct *p, struct rq *rq, + int event, u64 wallclock) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + u64 mark_start = wts->mark_start; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 delta, window_start = wrq->window_start; + int new_window, nr_full_windows; + u32 window_size = sched_ravg_window; + u64 runtime; + + new_window = mark_start < window_start; + if (!account_busy_for_task_demand(rq, p, event)) { + if (new_window) + /* + * If the time accounted isn't being accounted as + * busy time, and a new window started, only the + * previous window need be closed out with the + * pre-existing demand. Multiple windows may have + * elapsed, but since empty windows are dropped, + * it is not necessary to account those. + */ + update_history(rq, p, wts->sum, 1, event); + return 0; + } + + if (!new_window) { + /* + * The simple case - busy time contained within the existing + * window. + */ + return add_to_task_demand(rq, p, wallclock - mark_start); + } + + /* + * Busy time spans at least two windows. Temporarily rewind + * window_start to first window boundary after mark_start. + */ + delta = window_start - mark_start; + nr_full_windows = div64_u64(delta, window_size); + window_start -= (u64)nr_full_windows * (u64)window_size; + + /* Process (window_start - mark_start) first */ + runtime = add_to_task_demand(rq, p, window_start - mark_start); + + /* Push new sample(s) into task's demand history */ + update_history(rq, p, wts->sum, 1, event); + if (nr_full_windows) { + u64 scaled_window = scale_exec_time(window_size, rq, wts); + + update_history(rq, p, scaled_window, nr_full_windows, event); + runtime += nr_full_windows * scaled_window; + } + + /* + * Roll window_start back to current to process any remainder + * in current window. + */ + window_start += (u64)nr_full_windows * (u64)window_size; + + /* Process (wallclock - window_start) next */ + mark_start = window_start; + runtime += add_to_task_demand(rq, p, wallclock - mark_start); + + return runtime; +} + +static inline unsigned int cpu_cur_freq(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster->cur_freq; +} + +static void +update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime) +{ + u64 cur_cycles; + u64 cycles_delta; + u64 time_delta; + int cpu = cpu_of(rq); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + lockdep_assert_held(&rq->__lock); + + if (!use_cycle_counter) { + wrq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) * + arch_scale_cpu_capacity(cpu), + wrq->cluster->max_possible_freq); + return; + } + + cur_cycles = read_cycle_counter(cpu, wallclock); + + /* + * If current task is idle task and irqtime == 0 CPU was + * indeed idle and probably its cycle counter was not + * increasing. We still need estimatied CPU frequency + * for IO wait time accounting. Use the previously + * calculated frequency in such a case. + */ + if (!is_idle_task(rq->curr) || irqtime) { + if (unlikely(cur_cycles < wts->cpu_cycles)) + cycles_delta = cur_cycles + (U64_MAX - + wts->cpu_cycles); + else + cycles_delta = cur_cycles - wts->cpu_cycles; + cycles_delta = cycles_delta * NSEC_PER_MSEC; + + if (event == IRQ_UPDATE && is_idle_task(p)) + /* + * Time between mark_start of idle task and IRQ handler + * entry time is CPU cycle counter stall period. + * Upon IRQ handler entry walt_sched_account_irqstart() + * replenishes idle task's cpu cycle counter so + * cycles_delta now represents increased cycles during + * IRQ handler rather than time between idle entry and + * IRQ exit. Thus use irqtime as time delta. + */ + time_delta = irqtime; + else + time_delta = wallclock - wts->mark_start; + + if ((s64)time_delta < 0) { + printk_deferred("WALT-BUG pid=%u CPU%d wallclock=%llu(0x%llx) < mark_start=%llu(0x%llx) event=%d irqtime=%llu", + p->pid, rq->cpu, wallclock, wallclock, + wts->mark_start, wts->mark_start, event, irqtime); + WALT_PANIC((s64)time_delta < 0); + } + + wrq->task_exec_scale = DIV64_U64_ROUNDUP(cycles_delta * + arch_scale_cpu_capacity(cpu), + time_delta * + wrq->cluster->max_possible_freq); + + trace_sched_get_task_cpu_cycles(cpu, event, + cycles_delta, time_delta, p); + } + + wts->cpu_cycles = cur_cycles; +} + +static inline void run_walt_irq_work_rollover(u64 old_window_start, struct rq *rq) +{ + u64 result; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (old_window_start == wrq->window_start) + return; + + result = atomic64_cmpxchg(&walt_irq_work_lastq_ws, old_window_start, + wrq->window_start); + if (result == old_window_start) { + walt_irq_work_queue(&walt_cpufreq_irq_work); + trace_walt_window_rollover(wrq->window_start); + } +} + +/* Reflect task activity on its demand and cpu's busy time statistics */ +static void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime) +{ + u64 old_window_start; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!wrq->window_start || wts->mark_start == wallclock) + return; + + lockdep_assert_held(&rq->__lock); + + old_window_start = update_window_start(rq, wallclock, event); + + if (!wts->window_start) + wts->window_start = wrq->window_start; + + if (!wts->mark_start) { + update_task_cpu_cycles(p, cpu_of(rq), wallclock); + goto done; + } + + update_task_rq_cpu_cycles(p, rq, event, wallclock, irqtime); + update_task_demand(p, rq, event, wallclock); + update_cpu_busy_time(p, rq, event, wallclock, irqtime); + update_task_pred_demand(rq, p, event); + if (event == PUT_PREV_TASK && READ_ONCE(p->__state)) + wts->iowaited = p->in_iowait; + + trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime, + &wrq->grp_time, wrq, wts, atomic64_read(&walt_irq_work_lastq_ws)); + trace_sched_update_task_ravg_mini(p, rq, event, wallclock, irqtime, + &wrq->grp_time, wrq, wts, atomic64_read(&walt_irq_work_lastq_ws)); + +done: + wts->mark_start = wallclock; + + run_walt_irq_work_rollover(old_window_start, rq); +} + +static inline void __sched_fork_init(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->last_sleep_ts = 0; + wts->wake_up_idle = false; + wts->boost = 0; + wts->boost_expires = 0; + wts->boost_period = false; + wts->low_latency = false; + wts->iowaited = false; + wts->load_boost = 0; + wts->boosted_task_load = 0; +} + +static void init_new_task_load(struct task_struct *p) +{ + int i; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_task_struct *cur_wts = + (struct walt_task_struct *) current->android_vendor_data1; + u32 init_load_windows = sched_init_task_load_windows; + u32 init_load_windows_scaled = sched_init_task_load_windows_scaled; + u32 init_load_pct = cur_wts->init_load_pct; + + wts->init_load_pct = 0; + rcu_assign_pointer(wts->grp, NULL); + INIT_LIST_HEAD(&wts->grp_list); + + wts->prev_cpu = raw_smp_processor_id(); + wts->new_cpu = -1; + wts->enqueue_after_migration = 0; + wts->mark_start = 0; + wts->window_start = 0; + wts->sum = 0; + wts->curr_window = 0; + wts->prev_window = 0; + wts->active_time = 0; + wts->prev_on_rq = 0; + wts->prev_on_rq_cpu = -1; + + for (i = 0; i < NUM_BUSY_BUCKETS; ++i) + wts->busy_buckets[i] = 0; + wts->bucket_bitmask = 0; + wts->cpu_cycles = 0; + + memset(wts->curr_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); + memset(wts->prev_window_cpu, 0, sizeof(u32) * WALT_NR_CPUS); + + if (init_load_pct) { + init_load_windows = div64_u64((u64)init_load_pct * + (u64)sched_ravg_window, 100); + init_load_windows_scaled = scale_time_to_util(init_load_windows); + } + + wts->demand = init_load_windows; + wts->demand_scaled = init_load_windows_scaled; + wts->coloc_demand = init_load_windows; + wts->pred_demand_scaled = 0; + for (i = 0; i < RAVG_HIST_SIZE; ++i) + wts->sum_history[i] = init_load_windows; + wts->misfit = false; + wts->rtg_high_prio = false; + wts->unfilter = sysctl_sched_task_unfilter_period; + + INIT_LIST_HEAD(&wts->mvp_list); + wts->sum_exec_snapshot_for_slice = 0; + wts->sum_exec_snapshot_for_total = 0; + wts->total_exec = 0; + wts->mvp_prio = WALT_NOT_MVP; + wts->cidx = 0; + __sched_fork_init(p); +} + +static void init_existing_task_load(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + init_new_task_load(p); + cpumask_copy(&wts->cpus_requested, &p->cpus_mask); +} + +static void walt_task_dead(struct task_struct *p) +{ + sched_set_group_id(p, 0); +} + +static void mark_task_starting(struct task_struct *p) +{ + u64 wallclock; + struct rq *rq = task_rq(p); + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wallclock = walt_rq_clock(rq); + wts->mark_start = wts->last_wake_ts = wallclock; + wts->last_enqueued_ts = wallclock; + update_task_cpu_cycles(p, cpu_of(rq), wallclock); +} + +/* + * Task groups whose aggregate demand on a cpu is more than + * sched_group_upmigrate need to be up-migrated if possible. + */ +static unsigned int __read_mostly sched_group_upmigrate = 20000000; + +/* + * Task groups, once up-migrated, will need to drop their aggregate + * demand to less than sched_group_downmigrate before they are "down" + * migrated. + */ +static unsigned int __read_mostly sched_group_downmigrate = 19000000; + +void walt_update_group_thresholds(void) +{ + unsigned int min_scale = arch_scale_cpu_capacity( + cluster_first_cpu(sched_cluster[0])); + u64 min_ms = min_scale * (sched_ravg_window >> SCHED_CAPACITY_SHIFT); + + sched_group_upmigrate = div64_ul(min_ms * + sysctl_sched_group_upmigrate_pct, 100); + sched_group_downmigrate = div64_ul(min_ms * + sysctl_sched_group_downmigrate_pct, 100); +} + +struct walt_sched_cluster *sched_cluster[WALT_NR_CPUS]; +__read_mostly int num_sched_clusters; + +struct list_head cluster_head; + +static struct walt_sched_cluster init_cluster = { + .list = LIST_HEAD_INIT(init_cluster.list), + .id = 0, + .cur_freq = 1, + .max_possible_freq = 1, + .aggr_grp_load = 0, +}; + +static void init_clusters(void) +{ + init_cluster.cpus = *cpu_possible_mask; + raw_spin_lock_init(&init_cluster.load_lock); + INIT_LIST_HEAD(&cluster_head); + list_add(&init_cluster.list, &cluster_head); +} + +static void +insert_cluster(struct walt_sched_cluster *cluster, struct list_head *head) +{ + struct walt_sched_cluster *tmp; + struct list_head *iter = head; + + list_for_each_entry(tmp, head, list) { + if (arch_scale_cpu_capacity(cluster_first_cpu(cluster)) + < arch_scale_cpu_capacity(cluster_first_cpu(tmp))) + break; + iter = &tmp->list; + } + + list_add(&cluster->list, iter); +} + +static struct walt_sched_cluster *alloc_new_cluster(const struct cpumask *cpus) +{ + struct walt_sched_cluster *cluster = NULL; + + cluster = kzalloc(sizeof(struct walt_sched_cluster), GFP_ATOMIC); + BUG_ON(!cluster); + + INIT_LIST_HEAD(&cluster->list); + cluster->cur_freq = 1; + cluster->max_freq = 1; + cluster->max_possible_freq = 1; + + raw_spin_lock_init(&cluster->load_lock); + cluster->cpus = *cpus; + + return cluster; +} + +static void add_cluster(const struct cpumask *cpus, struct list_head *head) +{ + struct walt_sched_cluster *cluster = alloc_new_cluster(cpus); + int i; + struct walt_rq *wrq; + + BUG_ON(num_sched_clusters >= MAX_CLUSTERS); + + for_each_cpu(i, cpus) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq->cluster = cluster; + } + + insert_cluster(cluster, head); + num_sched_clusters++; +} + +static void cleanup_clusters(struct list_head *head) +{ + struct walt_sched_cluster *cluster, *tmp; + int i; + struct walt_rq *wrq; + + list_for_each_entry_safe(cluster, tmp, head, list) { + for_each_cpu(i, &cluster->cpus) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + wrq->cluster = &init_cluster; + } + list_del(&cluster->list); + num_sched_clusters--; + kfree(cluster); + } +} + +static inline void assign_cluster_ids(struct list_head *head) +{ + struct walt_sched_cluster *cluster; + int pos = 0; + + list_for_each_entry(cluster, head, list) { + cluster->id = pos; + sched_cluster[pos++] = cluster; + } + + WARN_ON(pos > MAX_NR_CLUSTERS); +} + +static inline void +move_list(struct list_head *dst, struct list_head *src, bool sync_rcu) +{ + struct list_head *first, *last; + + first = src->next; + last = src->prev; + + if (sync_rcu) { + INIT_LIST_HEAD_RCU(src); + synchronize_rcu(); + } + + first->prev = dst; + dst->prev = last; + last->next = dst; + + /* Ensure list sanity before making the head visible to all CPUs. */ + smp_mb(); + dst->next = first; +} + +static void update_all_clusters_stats(void) +{ + struct walt_sched_cluster *cluster; + u64 highest_mpc = 0, lowest_mpc = U64_MAX; + + for_each_sched_cluster(cluster) { + u64 mpc = arch_scale_cpu_capacity( + cluster_first_cpu(cluster)); + int cluster_id = cluster->id; + + if (mpc > highest_mpc) { + highest_mpc = mpc; + max_possible_cluster_id = cluster_id; + } + + if (mpc < lowest_mpc) { + lowest_mpc = mpc; + min_possible_cluster_id = cluster_id; + } + } + walt_update_group_thresholds(); +} + +static bool walt_clusters_parsed; +cpumask_t __read_mostly **cpu_array; + +static void init_cpu_array(void) +{ + int i; + + cpu_array = kcalloc(num_sched_clusters, sizeof(cpumask_t *), + GFP_ATOMIC | __GFP_NOFAIL); + if (!cpu_array) + WALT_PANIC(1); + + for (i = 0; i < num_sched_clusters; i++) { + cpu_array[i] = kcalloc(num_sched_clusters, sizeof(cpumask_t), + GFP_ATOMIC | __GFP_NOFAIL); + if (!cpu_array[i]) + WALT_PANIC(1); + } +} + +static void build_cpu_array(void) +{ + int i; + + if (!cpu_array) + WALT_PANIC(1); + /* Construct cpu_array row by row */ + for (i = 0; i < num_sched_clusters; i++) { + int j, k = 1; + + /* Fill out first column with appropriate cpu arrays */ + cpumask_copy(&cpu_array[i][0], &sched_cluster[i]->cpus); + /* + * k starts from column 1 because 0 is filled + * Fill clusters for the rest of the row, + * above i in ascending order + */ + for (j = i + 1; j < num_sched_clusters; j++) { + cpumask_copy(&cpu_array[i][k], + &sched_cluster[j]->cpus); + k++; + } + + /* + * k starts from where we left off above. + * Fill clusters below i in descending order. + */ + for (j = i - 1; j >= 0; j--) { + cpumask_copy(&cpu_array[i][k], + &sched_cluster[j]->cpus); + k++; + } + } +} + +static void walt_get_possible_siblings(int cpuid, struct cpumask *cluster_cpus) +{ + int cpu; + struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; + + if (cpuid_topo->package_id == -1) + return; + + for_each_possible_cpu(cpu) { + cpu_topo = &cpu_topology[cpu]; + + if (cpuid_topo->package_id != cpu_topo->package_id) + continue; + cpumask_set_cpu(cpu, cluster_cpus); + } +} + +int cpu_l2_sibling[WALT_NR_CPUS] = {[0 ... WALT_NR_CPUS-1] = -1}; +static void find_cache_siblings(void) +{ + int cpu, cpu2; + struct device_node *cpu_dev, *cpu_dev2, *cpu_l2_cache_node, *cpu_l2_cache_node2; + + for_each_possible_cpu(cpu) { + cpu_dev = of_get_cpu_node(cpu, NULL); + if (!cpu_dev) + continue; + + cpu_l2_cache_node = of_parse_phandle(cpu_dev, "next-level-cache", 0); + if (!cpu_l2_cache_node) + continue; + + for_each_possible_cpu(cpu2) { + if (cpu == cpu2) + continue; + + cpu_dev2 = of_get_cpu_node(cpu2, NULL); + if (!cpu_dev2) + continue; + + cpu_l2_cache_node2 = of_parse_phandle(cpu_dev2, "next-level-cache", 0); + if (!cpu_l2_cache_node2) + continue; + + if (cpu_l2_cache_node == cpu_l2_cache_node2) { + cpu_l2_sibling[cpu] = cpu2; + break; + } + } + } +} + +static void walt_update_cluster_topology(void) +{ + struct cpumask cpus = *cpu_possible_mask; + struct cpumask cluster_cpus; + struct walt_sched_cluster *cluster; + struct list_head new_head; + int i; + struct walt_rq *wrq; + + INIT_LIST_HEAD(&new_head); + + for_each_cpu(i, &cpus) { + cpumask_clear(&cluster_cpus); + walt_get_possible_siblings(i, &cluster_cpus); + if (cpumask_empty(&cluster_cpus)) { + WARN(1, "WALT: Invalid cpu topology!!"); + cleanup_clusters(&new_head); + return; + } + cpumask_andnot(&cpus, &cpus, &cluster_cpus); + add_cluster(&cluster_cpus, &new_head); + } + + assign_cluster_ids(&new_head); + + list_for_each_entry(cluster, &new_head, list) { + struct cpufreq_policy *policy; + + policy = cpufreq_cpu_get_raw(cluster_first_cpu(cluster)); + /* + * walt_update_cluster_topology() must be called AFTER policies + * for all cpus are initialized. If not, simply BUG(). + */ + WALT_PANIC(!policy); + + if (policy) { + cluster->max_possible_freq = policy->cpuinfo.max_freq; + cluster->max_freq = policy->max; + for_each_cpu(i, &cluster->cpus) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + cpumask_copy(&wrq->freq_domain_cpumask, + policy->related_cpus); + } + cpuinfo_max_freq_cached = (cpuinfo_max_freq_cached > + policy->cpuinfo.max_freq) ?: policy->cpuinfo.max_freq; + } + } + + /* + * Ensure cluster ids are visible to all CPUs before making + * cluster_head visible. + */ + move_list(&cluster_head, &new_head, false); + update_all_clusters_stats(); + cluster = NULL; + + for_each_sched_cluster(cluster) { + if (cpumask_weight(&cluster->cpus) == 1) + cpumask_or(&asym_cap_sibling_cpus, + &asym_cap_sibling_cpus, &cluster->cpus); + } + + if (cpumask_weight(&asym_cap_sibling_cpus) == 1) + cpumask_clear(&asym_cap_sibling_cpus); + + init_cpu_array(); + build_cpu_array(); + find_cache_siblings(); + + create_util_to_cost(); + walt_clusters_parsed = true; +} + +static int cpufreq_notifier_trans(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data; + unsigned int cpu = freq->policy->cpu, new_freq = freq->new; + unsigned long flags; + struct walt_sched_cluster *cluster; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct cpumask policy_cpus = wrq->freq_domain_cpumask; + int i, j; + + if (use_cycle_counter) + return NOTIFY_DONE; + wrq = (struct walt_rq *) cpu_rq(cpumask_first(&policy_cpus))->android_vendor_data1; + if (wrq->cluster == &init_cluster) + return NOTIFY_DONE; + + if (val != CPUFREQ_POSTCHANGE) + return NOTIFY_DONE; + + if (cpu_cur_freq(cpu) == new_freq) + return NOTIFY_OK; + + for_each_cpu(i, &policy_cpus) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + cluster = wrq->cluster; + + for_each_cpu(j, &cluster->cpus) { + struct rq *rq = cpu_rq(j); + + raw_spin_lock_irqsave(&rq->__lock, flags); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, walt_sched_clock(), 0); + raw_spin_unlock_irqrestore(&rq->__lock, flags); + } + + cluster->cur_freq = new_freq; + cpumask_andnot(&policy_cpus, &policy_cpus, &cluster->cpus); + } + + return NOTIFY_OK; +} + +static struct notifier_block notifier_trans_block = { + .notifier_call = cpufreq_notifier_trans +}; + +static void walt_init_cycle_counter(void) +{ + if (qcom_cpufreq_get_cpu_cycle_counter(smp_processor_id()) != U64_MAX) { + use_cycle_counter = true; + return; + } + + cpufreq_register_notifier(¬ifier_trans_block, + CPUFREQ_TRANSITION_NOTIFIER); +} + +static void transfer_busy_time(struct rq *rq, + struct walt_related_thread_group *grp, + struct task_struct *p, int event); + +/* + * Enable colocation and frequency aggregation for all threads in a process. + * The children inherits the group id from the parent. + */ + +static struct walt_related_thread_group + *related_thread_groups[MAX_NUM_CGROUP_COLOC_ID]; +static LIST_HEAD(active_related_thread_groups); +static DEFINE_RWLOCK(related_thread_group_lock); + +static inline +void update_best_cluster(struct walt_related_thread_group *grp, + u64 combined_demand, bool boost) +{ + if (boost) { + /* + * since we are in boost, we can keep grp on min, the boosts + * will ensure tasks get to bigs + */ + grp->skip_min = false; + return; + } + + if (is_suh_max()) + combined_demand = sched_group_upmigrate; + + if (!grp->skip_min) { + if (combined_demand >= sched_group_upmigrate) + grp->skip_min = true; + return; + } + if (combined_demand < sched_group_downmigrate) { + if (!sysctl_sched_coloc_downmigrate_ns || + (grp->last_update - grp->start_ktime_ts) < + sysctl_sched_hyst_min_coloc_ns) { + grp->downmigrate_ts = 0; + grp->skip_min = false; + return; + } + if (!grp->downmigrate_ts) { + grp->downmigrate_ts = grp->last_update; + return; + } + if (grp->last_update - grp->downmigrate_ts > + sysctl_sched_coloc_downmigrate_ns) { + grp->downmigrate_ts = 0; + grp->skip_min = false; + } + } else if (grp->downmigrate_ts) + grp->downmigrate_ts = 0; +} + +static void _set_preferred_cluster(struct walt_related_thread_group *grp) +{ + struct task_struct *p; + u64 combined_demand = 0; + bool group_boost = false; + u64 wallclock; + bool prev_skip_min = grp->skip_min; + struct walt_task_struct *wts; + + if (list_empty(&grp->tasks)) { + grp->skip_min = false; + goto out; + } + + if (!hmp_capable()) { + grp->skip_min = false; + goto out; + } + + wallclock = walt_sched_clock(); + + /* + * wakeup of two or more related tasks could race with each other and + * could result in multiple calls to _set_preferred_cluster being issued + * at same time. Avoid overhead in such cases of rechecking preferred + * cluster + */ + if (wallclock - grp->last_update < sched_ravg_window / 10) + return; + + list_for_each_entry(wts, &grp->tasks, grp_list) { + p = wts_to_ts(wts); + if (task_boost_policy(p) == SCHED_BOOST_ON_BIG) { + group_boost = true; + break; + } + + if (wts->mark_start < wallclock - + (sched_ravg_window * RAVG_HIST_SIZE)) + continue; + + combined_demand += wts->coloc_demand; + if (!trace_sched_set_preferred_cluster_enabled()) { + if (combined_demand > sched_group_upmigrate) + break; + } + } + + grp->last_update = wallclock; + update_best_cluster(grp, combined_demand, group_boost); + +out: + trace_sched_set_preferred_cluster(grp, combined_demand, prev_skip_min); + if (grp->id == DEFAULT_CGROUP_COLOC_ID + && grp->skip_min != prev_skip_min) { + if (grp->skip_min) + grp->start_ktime_ts = wallclock; + else + grp->start_ktime_ts = 0; + sched_update_hyst_times(); + } +} + +static void set_preferred_cluster(struct walt_related_thread_group *grp) +{ + raw_spin_lock(&grp->lock); + _set_preferred_cluster(grp); + raw_spin_unlock(&grp->lock); +} + +static int update_preferred_cluster(struct walt_related_thread_group *grp, + struct task_struct *p, u32 old_load, bool from_tick) +{ + u32 new_load = task_load(p); + + if (!grp) + return 0; + + if (unlikely(from_tick && is_suh_max())) + return 1; + + /* + * Update if task's load has changed significantly or a complete window + * has passed since we last updated preference + */ + + if (abs(new_load - old_load) > sched_ravg_window / 4) + return 1; + + if (walt_sched_clock() - grp->last_update > sched_ravg_window) + return 1; + + return 0; +} + +#define ADD_TASK 0 +#define REM_TASK 1 + +static inline struct walt_related_thread_group* +lookup_related_thread_group(unsigned int group_id) +{ + return related_thread_groups[group_id]; +} + +static int alloc_related_thread_groups(void) +{ + int i; + struct walt_related_thread_group *grp; + + /* groupd_id = 0 is invalid as it's special id to remove group. */ + for (i = 1; i < MAX_NUM_CGROUP_COLOC_ID; i++) { + grp = kzalloc(sizeof(*grp), GFP_ATOMIC | GFP_NOWAIT); + BUG_ON(!grp); + + grp->id = i; + INIT_LIST_HEAD(&grp->tasks); + INIT_LIST_HEAD(&grp->list); + raw_spin_lock_init(&grp->lock); + + related_thread_groups[i] = grp; + } + + return 0; +} + +static void remove_task_from_group(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_related_thread_group *grp = wts->grp; + struct rq *rq; + int empty_group = 1; + struct rq_flags rf; + + raw_spin_lock(&grp->lock); + + rq = __task_rq_lock(p, &rf); + transfer_busy_time(rq, wts->grp, p, REM_TASK); + list_del_init(&wts->grp_list); + rcu_assign_pointer(wts->grp, NULL); + __task_rq_unlock(rq, &rf); + + if (!list_empty(&grp->tasks)) { + empty_group = 0; + _set_preferred_cluster(grp); + } + + raw_spin_unlock(&grp->lock); + + /* Reserved groups cannot be destroyed */ + if (empty_group && grp->id != DEFAULT_CGROUP_COLOC_ID) + /* + * We test whether grp->list is attached with list_empty() + * hence re-init the list after deletion. + */ + list_del_init(&grp->list); +} + +static int +add_task_to_group(struct task_struct *p, struct walt_related_thread_group *grp) +{ + struct rq *rq; + struct rq_flags rf; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + raw_spin_lock(&grp->lock); + + /* + * Change wts->grp under rq->lock. Will prevent races with read-side + * reference of wts->grp in various hot-paths + */ + rq = __task_rq_lock(p, &rf); + transfer_busy_time(rq, grp, p, ADD_TASK); + list_add(&wts->grp_list, &grp->tasks); + rcu_assign_pointer(wts->grp, grp); + __task_rq_unlock(rq, &rf); + + _set_preferred_cluster(grp); + + raw_spin_unlock(&grp->lock); + + return 0; +} + +#ifdef CONFIG_UCLAMP_TASK_GROUP +static inline bool uclamp_task_colocated(struct task_struct *p) +{ + struct cgroup_subsys_state *css; + struct task_group *tg; + bool colocate; + struct walt_task_group *wtg; + + rcu_read_lock(); + css = task_css(p, cpu_cgrp_id); + if (!css) { + rcu_read_unlock(); + return false; + } + tg = container_of(css, struct task_group, css); + wtg = (struct walt_task_group *) tg->android_vendor_data1; + colocate = wtg->colocate; + rcu_read_unlock(); + + return colocate; +} +#else +static inline bool uclamp_task_colocated(struct task_struct *p) +{ + return false; +} +#endif /* CONFIG_UCLAMP_TASK_GROUP */ + +static void add_new_task_to_grp(struct task_struct *new) +{ + unsigned long flags; + struct walt_related_thread_group *grp; + struct walt_task_struct *wts = (struct walt_task_struct *) new->android_vendor_data1; + + /* + * If the task does not belong to colocated schedtune + * cgroup, nothing to do. We are checking this without + * lock. Even if there is a race, it will be added + * to the co-located cgroup via cgroup attach. + */ + if (!uclamp_task_colocated(new)) + return; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + write_lock_irqsave(&related_thread_group_lock, flags); + + /* + * It's possible that someone already added the new task to the + * group. or it might have taken out from the colocated schedtune + * cgroup. check these conditions under lock. + */ + if (!uclamp_task_colocated(new) || wts->grp) { + write_unlock_irqrestore(&related_thread_group_lock, flags); + return; + } + + raw_spin_lock(&grp->lock); + + rcu_assign_pointer(wts->grp, grp); + list_add(&wts->grp_list, &grp->tasks); + + raw_spin_unlock(&grp->lock); + write_unlock_irqrestore(&related_thread_group_lock, flags); +} + +static int __sched_set_group_id(struct task_struct *p, unsigned int group_id) +{ + int rc = 0; + unsigned long flags; + struct walt_related_thread_group *grp = NULL; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (group_id >= MAX_NUM_CGROUP_COLOC_ID) + return -EINVAL; + + raw_spin_lock_irqsave(&p->pi_lock, flags); + write_lock(&related_thread_group_lock); + + /* Switching from one group to another directly is not permitted */ + if ((!wts->grp && !group_id) || (wts->grp && group_id)) + goto done; + + if (!group_id) { + remove_task_from_group(p); + goto done; + } + + grp = lookup_related_thread_group(group_id); + if (list_empty(&grp->list)) + list_add(&grp->list, &active_related_thread_groups); + + rc = add_task_to_group(p, grp); +done: + write_unlock(&related_thread_group_lock); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + + return rc; +} + +int sched_set_group_id(struct task_struct *p, unsigned int group_id) +{ + /* DEFAULT_CGROUP_COLOC_ID is a reserved id */ + if (group_id == DEFAULT_CGROUP_COLOC_ID) + return -EINVAL; + + return __sched_set_group_id(p, group_id); +} + +unsigned int sched_get_group_id(struct task_struct *p) +{ + unsigned int group_id; + struct walt_related_thread_group *grp; + + rcu_read_lock(); + grp = task_related_thread_group(p); + group_id = grp ? grp->id : 0; + rcu_read_unlock(); + + return group_id; +} + +/* + * We create a default colocation group at boot. There is no need to + * synchronize tasks between cgroups at creation time because the + * correct cgroup hierarchy is not available at boot. Therefore cgroup + * colocation is turned off by default even though the colocation group + * itself has been allocated. Furthermore this colocation group cannot + * be destroyted once it has been created. All of this has been as part + * of runtime optimizations. + * + * The job of synchronizing tasks to the colocation group is done when + * the colocation flag in the cgroup is turned on. + */ +static int create_default_coloc_group(void) +{ + struct walt_related_thread_group *grp = NULL; + unsigned long flags; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + write_lock_irqsave(&related_thread_group_lock, flags); + list_add(&grp->list, &active_related_thread_groups); + write_unlock_irqrestore(&related_thread_group_lock, flags); + return 0; +} + +static void walt_update_tg_pointer(struct cgroup_subsys_state *css) +{ + if (!strcmp(css->cgroup->kn->name, "top-app")) + walt_init_topapp_tg(css_tg(css)); + else if (!strcmp(css->cgroup->kn->name, "foreground")) + walt_init_foreground_tg(css_tg(css)); + else + walt_init_tg(css_tg(css)); +} + +void walt_kick_cpu(int cpu) +{ + unsigned int flags = NOHZ_KICK_MASK; + + if (cpu == -1) + return; + + /* + * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets + * the first flag owns it; cleared by nohz_csd_func(). + */ + flags = atomic_fetch_or(flags, nohz_flags(cpu)); + if (flags & NOHZ_KICK_MASK) + return; + + /* + * This way we generate an IPI on the target CPU which + * is idle. And the softirq performing nohz idle load balance + * will be run before returning from the IPI. + */ + smp_call_function_single_async(cpu, &cpu_rq(cpu)->nohz_csd); +} + + +static void android_rvh_cpu_cgroup_online(void *unused, struct cgroup_subsys_state *css) +{ + if (unlikely(walt_disabled)) + return; + + walt_update_tg_pointer(css); +} + +static void android_rvh_cpu_cgroup_attach(void *unused, + struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + struct task_group *tg; + struct walt_task_group *wtg; + unsigned int grp_id; + int ret; + + if (unlikely(walt_disabled)) + return; + + cgroup_taskset_first(tset, &css); + if (!css) + return; + + tg = container_of(css, struct task_group, css); + wtg = (struct walt_task_group *) tg->android_vendor_data1; + + cgroup_taskset_for_each(task, css, tset) { + grp_id = wtg->colocate ? DEFAULT_CGROUP_COLOC_ID : 0; + ret = __sched_set_group_id(task, grp_id); + trace_sched_cgroup_attach(task, grp_id, ret); + } +} + +static bool is_cluster_hosting_top_app(struct walt_sched_cluster *cluster) +{ + struct walt_related_thread_group *grp; + bool grp_on_min; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + + if (!grp) + return false; + + grp_on_min = !grp->skip_min && (boost_policy != SCHED_BOOST_ON_BIG); + + return (is_min_capacity_cluster(cluster) == grp_on_min); +} + +static void note_task_waking(struct task_struct *p, u64 wallclock) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wts->last_wake_ts = wallclock; +} + +/* + * Task's cpu usage is accounted in: + * wrq->curr/prev_runnable_sum, when its ->grp is NULL + * grp->cpu_time[cpu]->curr/prev_runnable_sum, when its ->grp is !NULL + * + * Transfer task's cpu usage between those counters when transitioning between + * groups + */ +static void transfer_busy_time(struct rq *rq, + struct walt_related_thread_group *grp, + struct task_struct *p, int event) +{ + u64 wallclock; + struct group_cpu_time *cpu_time; + u64 *src_curr_runnable_sum, *dst_curr_runnable_sum; + u64 *src_prev_runnable_sum, *dst_prev_runnable_sum; + u64 *src_nt_curr_runnable_sum, *dst_nt_curr_runnable_sum; + u64 *src_nt_prev_runnable_sum, *dst_nt_prev_runnable_sum; + int migrate_type; + int cpu = cpu_of(rq); + bool new_task; + int i; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + wallclock = walt_sched_clock(); + + walt_update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); + + if (wts->window_start != wrq->window_start) + WALT_BUG(WALT_BUG_WALT, p, + "CPU%d: %s event=%d task %s(%d)'s ws=%llu not equal to rq %d's ws=%llu", + __func__, raw_smp_processor_id(), event, p->comm, p->pid, + wts->window_start, rq->cpu, wrq->window_start); + + new_task = is_new_task(p); + + cpu_time = &wrq->grp_time; + if (event == ADD_TASK) { + migrate_type = RQ_TO_GROUP; + + src_curr_runnable_sum = &wrq->curr_runnable_sum; + dst_curr_runnable_sum = &cpu_time->curr_runnable_sum; + src_prev_runnable_sum = &wrq->prev_runnable_sum; + dst_prev_runnable_sum = &cpu_time->prev_runnable_sum; + + src_nt_curr_runnable_sum = &wrq->nt_curr_runnable_sum; + dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &wrq->nt_prev_runnable_sum; + dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + if (*src_curr_runnable_sum < wts->curr_window_cpu[cpu]) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_curr_runnable_sum, + wts->curr_window_cpu[cpu]); + *src_curr_runnable_sum = wts->curr_window_cpu[cpu]; + } + *src_curr_runnable_sum -= wts->curr_window_cpu[cpu]; + + if (*src_prev_runnable_sum < wts->prev_window_cpu[cpu]) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_prev_runnable_sum, + wts->prev_window_cpu[cpu]); + *src_prev_runnable_sum = wts->prev_window_cpu[cpu]; + } + *src_prev_runnable_sum -= wts->prev_window_cpu[cpu]; + + if (new_task) { + if (*src_nt_curr_runnable_sum < wts->curr_window_cpu[cpu]) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_curr_runnable_sum, + wts->curr_window_cpu[cpu]); + *src_nt_curr_runnable_sum = wts->curr_window_cpu[cpu]; + } + *src_nt_curr_runnable_sum -= + wts->curr_window_cpu[cpu]; + + if (*src_nt_prev_runnable_sum < wts->prev_window_cpu[cpu]) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_prev_runnable_sum, + wts->prev_window_cpu[cpu]); + *src_nt_prev_runnable_sum = wts->prev_window_cpu[cpu]; + } + *src_nt_prev_runnable_sum -= + wts->prev_window_cpu[cpu]; + } + + update_cluster_load_subtractions(p, cpu, + wrq->window_start, new_task); + + } else { + migrate_type = GROUP_TO_RQ; + + src_curr_runnable_sum = &cpu_time->curr_runnable_sum; + dst_curr_runnable_sum = &wrq->curr_runnable_sum; + src_prev_runnable_sum = &cpu_time->prev_runnable_sum; + dst_prev_runnable_sum = &wrq->prev_runnable_sum; + + src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + dst_nt_curr_runnable_sum = &wrq->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + dst_nt_prev_runnable_sum = &wrq->nt_prev_runnable_sum; + + if (*src_curr_runnable_sum < wts->curr_window) { + WALT_BUG(WALT_BUG_WALT, p, + "WALT-UG pid=%u CPU=%d event=%d src_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_curr_runnable_sum, + wts->curr_window); + *src_curr_runnable_sum = wts->curr_window; + } + *src_curr_runnable_sum -= wts->curr_window; + + if (*src_prev_runnable_sum < wts->prev_window) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, *src_prev_runnable_sum, + wts->prev_window); + *src_prev_runnable_sum = wts->prev_window; + } + *src_prev_runnable_sum -= wts->prev_window; + + if (new_task) { + if (*src_nt_curr_runnable_sum < wts->curr_window) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_nt_crs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_curr_runnable_sum, + wts->curr_window); + *src_nt_curr_runnable_sum = wts->curr_window; + } + *src_nt_curr_runnable_sum -= wts->curr_window; + + if (*src_nt_prev_runnable_sum < wts->prev_window) { + WALT_BUG(WALT_BUG_WALT, p, + "pid=%u CPU=%d event=%d src_nt_prs=%llu is lesser than task_contrib=%llu", + p->pid, cpu, event, + *src_nt_prev_runnable_sum, + wts->prev_window); + *src_nt_prev_runnable_sum = wts->prev_window; + } + *src_nt_prev_runnable_sum -= wts->prev_window; + } + + /* + * Need to reset curr/prev windows for all CPUs, not just the + * ones in the same cluster. Since inter cluster migrations + * did not result in the appropriate book keeping, the values + * per CPU would be inaccurate. + */ + for_each_possible_cpu(i) { + wts->curr_window_cpu[i] = 0; + wts->prev_window_cpu[i] = 0; + } + } + + *dst_curr_runnable_sum += wts->curr_window; + *dst_prev_runnable_sum += wts->prev_window; + if (new_task) { + *dst_nt_curr_runnable_sum += wts->curr_window; + *dst_nt_prev_runnable_sum += wts->prev_window; + } + + /* + * When a task enter or exits a group, it's curr and prev windows are + * moved to a single CPU. This behavior might be sub-optimal in the + * exit case, however, it saves us the overhead of handling inter + * cluster migration fixups while the task is part of a related group. + */ + wts->curr_window_cpu[cpu] = wts->curr_window; + wts->prev_window_cpu[cpu] = wts->prev_window; + + trace_sched_migration_update_sum(p, migrate_type, rq); +} + +bool is_rtgb_active(void) +{ + struct walt_related_thread_group *grp; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + return grp && grp->skip_min; +} + +u64 get_rtgb_active_time(void) +{ + struct walt_related_thread_group *grp; + u64 now = walt_sched_clock(); + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + + if (grp && grp->skip_min && grp->start_ktime_ts) + return now - grp->start_ktime_ts; + + return 0; +} + +static void walt_init_window_dep(void); +static void walt_tunables_fixup(void) +{ + if (likely(num_sched_clusters > 0)) + walt_update_group_thresholds(); + walt_init_window_dep(); +} + +static void walt_update_irqload(struct rq *rq) +{ + u64 irq_delta = 0; + unsigned int nr_windows = 0; + u64 cur_irq_time; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + u64 last_irq_window = READ_ONCE(wrq->last_irq_window); + + if (wrq->window_start > last_irq_window) + nr_windows = div64_u64(wrq->window_start - last_irq_window, + sched_ravg_window); + + /* Decay CPU's irqload by 3/4 for each window. */ + if (nr_windows < 10) + wrq->avg_irqload = mult_frac(wrq->avg_irqload, 3, 4); + else + wrq->avg_irqload = 0; + + cur_irq_time = irq_time_read(cpu_of(rq)); + if (cur_irq_time > wrq->prev_irq_time) + irq_delta = cur_irq_time - wrq->prev_irq_time; + + wrq->avg_irqload += irq_delta; + wrq->prev_irq_time = cur_irq_time; + + if (nr_windows < SCHED_HIGH_IRQ_TIMEOUT) + wrq->high_irqload = (wrq->avg_irqload >= + walt_cpu_high_irqload); + else + wrq->high_irqload = false; +} + +/** + * __walt_irq_work_locked() - common function to process work + * @is_migration: if true, performing migration work, else rollover + * @lock_cpus: mask of the cpus involved in the operation. + * + * In rq locked context, update the cluster group load and find + * the load of the min cluster, while tracking the total aggregate + * work load. Update the cpufreq through the walt governor, + * based upon the new load calculated. + * + * For the window rollover case lock_cpus will be all possible cpus, + * and for migrations it will include the cpus from the two clusters + * involved in the migration. + */ +static inline void __walt_irq_work_locked(bool is_migration, struct cpumask *lock_cpus) +{ + struct walt_sched_cluster *cluster; + struct rq *rq; + int cpu; + u64 wc; + bool is_asym_migration = false; + u64 total_grp_load = 0, min_cluster_grp_load = 0; + unsigned long flags; + struct walt_rq *wrq; + + wc = walt_sched_clock(); + if (!is_migration) + walt_load_reported_window = atomic64_read(&walt_irq_work_lastq_ws); + for_each_sched_cluster(cluster) { + u64 aggr_grp_load = 0; + + raw_spin_lock(&cluster->load_lock); + for_each_cpu(cpu, &cluster->cpus) { + rq = cpu_rq(cpu); + wrq = (struct walt_rq *) rq->android_vendor_data1; + if (rq->curr) { + /* only update ravg for locked cpus */ + if (cpumask_intersects(lock_cpus, &cluster->cpus)) { + walt_update_task_ravg(rq->curr, rq, + TASK_UPDATE, wc, 0); + account_load_subtractions(rq); + } + + /* update aggr_grp_load for all clusters, all cpus */ + aggr_grp_load += + wrq->grp_time.prev_runnable_sum; + } + if (is_migration && wrq->notif_pending && + cpumask_test_cpu(cpu, &asym_cap_sibling_cpus)) { + is_asym_migration = true; + wrq->notif_pending = false; + } + } + raw_spin_unlock(&cluster->load_lock); + + cluster->aggr_grp_load = aggr_grp_load; + total_grp_load += aggr_grp_load; + + if (is_min_capacity_cluster(cluster)) + min_cluster_grp_load = aggr_grp_load; + } + + if (total_grp_load) { + if (cpumask_weight(&asym_cap_sibling_cpus)) { + u64 big_grp_load = + total_grp_load - min_cluster_grp_load; + + for_each_cpu(cpu, &asym_cap_sibling_cpus) + cpu_cluster(cpu)->aggr_grp_load = big_grp_load; + } + rtgb_active = is_rtgb_active(); + } else { + rtgb_active = false; + } + + if (!is_migration && sysctl_sched_user_hint && time_after(jiffies, + sched_user_hint_reset_time)) + sysctl_sched_user_hint = 0; + + for_each_sched_cluster(cluster) { + cpumask_t cluster_online_cpus; + unsigned int num_cpus, i = 1; + + /* for migration, skip unnotified clusters */ + if (is_migration && !cpumask_intersects(lock_cpus, &cluster->cpus)) + continue; + + cpumask_and(&cluster_online_cpus, &cluster->cpus, + cpu_online_mask); + num_cpus = cpumask_weight(&cluster_online_cpus); + for_each_cpu(cpu, &cluster_online_cpus) { + int wflag = 0; + + rq = cpu_rq(cpu); + wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (is_migration) { + if (wrq->notif_pending) { + wrq->notif_pending = false; + + wflag |= WALT_CPUFREQ_IC_MIGRATION; + } + } else { + wflag |= WALT_CPUFREQ_ROLLOVER; + } + + if (is_asym_migration && cpumask_test_cpu(cpu, + &asym_cap_sibling_cpus)) { + wflag |= WALT_CPUFREQ_IC_MIGRATION; + } + + if (i == num_cpus) + waltgov_run_callback(cpu_rq(cpu), wflag); + else + waltgov_run_callback(cpu_rq(cpu), wflag | + WALT_CPUFREQ_CONTINUE); + i++; + + if (!is_migration) + walt_update_irqload(rq); + } + } + + /* + * If the window change request is in pending, good place to + * change sched_ravg_window since all rq locks are acquired. + * + * If the current window roll over is delayed such that the + * mark_start (current wallclock with which roll over is done) + * of the current task went past the window start with the + * updated new window size, delay the update to the next + * window roll over. Otherwise the CPU counters (prs and crs) are + * not rolled over properly as mark_start > window_start. + */ + if (!is_migration) { + spin_lock_irqsave(&sched_ravg_window_lock, flags); + wrq = (struct walt_rq *) this_rq()->android_vendor_data1; + if ((sched_ravg_window != new_sched_ravg_window) && + (wc < wrq->window_start + new_sched_ravg_window)) { + sched_ravg_window_change_time = walt_sched_clock(); + trace_sched_ravg_window_change(sched_ravg_window, + new_sched_ravg_window, + sched_ravg_window_change_time); + sched_ravg_window = new_sched_ravg_window; + walt_tunables_fixup(); + } + spin_unlock_irqrestore(&sched_ravg_window_lock, flags); + } +} + +/** + * irq_work_restrict_to_mig_clusters() - only allow notified clusters + * @lock_cpus: mask of the cpus for which the runque should be locked. + * + * Remove cpus in clusters that are not part of the migration, using + * the notif_pending flag to track. + * + * This is only valid for the migration irq work. + */ +static inline void irq_work_restrict_to_mig_clusters(cpumask_t *lock_cpus) +{ + struct walt_sched_cluster *cluster; + struct rq *rq; + struct walt_rq *wrq; + int cpu; + + for_each_sched_cluster(cluster) { + bool keep_locked = false; + for_each_cpu(cpu, &cluster->cpus) { + rq = cpu_rq(cpu); + wrq = (struct walt_rq *)rq->android_vendor_data1; + + /* remove this cluster if it's not being notified */ + if (wrq->notif_pending) { + keep_locked = true; + break; + } + } + if (!keep_locked) + cpumask_andnot(lock_cpus, lock_cpus, &cluster->cpus); + } +} + +/** + * walt_irq_work() - perform walt irq work for rollover and migration + * + * Process a workqueue call scheduled, while running in a hard irq + * protected context. Handle migration and window rollover work + * with common funtionality, and on window rollover ask core control + * to decide if it needs to adjust the active cpus. + */ +static void walt_irq_work(struct irq_work *irq_work) +{ + cpumask_t lock_cpus; + struct walt_rq *wrq; + int level = 0; + int cpu; + bool is_migration = false; + + if (irq_work == &walt_migration_irq_work) + is_migration = true; + + cpumask_copy(&lock_cpus, cpu_possible_mask); + + if (is_migration) { + irq_work_restrict_to_mig_clusters(&lock_cpus); + + /* + * if the notif_pending was handled by a previous + * walt_irq_work invocation, there is no migration + * work. + */ + if (cpumask_empty(&lock_cpus)) + return; + } + + for_each_cpu(cpu, &lock_cpus) { + if (level == 0) + raw_spin_lock(&cpu_rq(cpu)->__lock); + else + raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); + level++; + } + + __walt_irq_work_locked(is_migration, &lock_cpus); + + for_each_cpu(cpu, &lock_cpus) + raw_spin_unlock(&cpu_rq(cpu)->__lock); + + if (!is_migration) { + wrq = (struct walt_rq *) this_rq()->android_vendor_data1; + core_ctl_check(wrq->window_start); + } +} + +void walt_rotation_checkpoint(int nr_big) +{ + if (!hmp_capable()) + return; + + if (!sysctl_sched_walt_rotate_big_tasks || sched_boost_type != NO_BOOST) { + walt_rotation_enabled = 0; + return; + } + + walt_rotation_enabled = nr_big >= num_possible_cpus(); +} + +void walt_fill_ta_data(struct core_ctl_notif_data *data) +{ + struct walt_related_thread_group *grp; + unsigned long flags; + u64 total_demand = 0, wallclock; + int min_cap_cpu, scale = 1024; + struct walt_sched_cluster *cluster; + int i = 0; + struct walt_task_struct *wts; + + grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID); + + raw_spin_lock_irqsave(&grp->lock, flags); + if (list_empty(&grp->tasks)) { + raw_spin_unlock_irqrestore(&grp->lock, flags); + goto fill_util; + } + + wallclock = walt_sched_clock(); + + list_for_each_entry(wts, &grp->tasks, grp_list) { + if (wts->mark_start < wallclock - + (sched_ravg_window * RAVG_HIST_SIZE)) + continue; + + total_demand += wts->coloc_demand; + } + + raw_spin_unlock_irqrestore(&grp->lock, flags); + + /* + * Scale the total demand to the lowest capacity CPU and + * convert into percentage. + * + * P = total_demand/sched_ravg_window * 1024/scale * 100 + */ + + min_cap_cpu = cpumask_first(&cpu_array[0][0]); + if (min_cap_cpu != -1) + scale = arch_scale_cpu_capacity(min_cap_cpu); + + data->coloc_load_pct = div64_u64(total_demand * 1024 * 100, + (u64)sched_ravg_window * scale); + +fill_util: + for_each_sched_cluster(cluster) { + int fcpu = cluster_first_cpu(cluster); + + if (i == MAX_CLUSTERS) + break; + + scale = arch_scale_cpu_capacity(fcpu); + data->ta_util_pct[i] = div64_u64(cluster->aggr_grp_load * 1024 * + 100, (u64)sched_ravg_window * scale); + + scale = arch_scale_freq_capacity(fcpu); + data->cur_cap_pct[i] = (scale * 100)/1024; + i++; + } +} + +static void walt_init_window_dep(void) +{ + walt_scale_demand_divisor = sched_ravg_window >> SCHED_CAPACITY_SHIFT; + + sched_init_task_load_windows = + div64_u64((u64)sysctl_sched_init_task_load_pct * + (u64)sched_ravg_window, 100); + sched_init_task_load_windows_scaled = + scale_time_to_util(sched_init_task_load_windows); + + walt_cpu_high_irqload = div64_u64((u64)sched_ravg_window * 95, (u64) 100); +} + +static void walt_init_once(void) +{ + init_irq_work(&walt_migration_irq_work, walt_irq_work); + init_irq_work(&walt_cpufreq_irq_work, walt_irq_work); + walt_init_window_dep(); +} + +static void walt_sched_init_rq(struct rq *rq) +{ + int j; + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (cpu_of(rq) == 0) + walt_init_once(); + + cpumask_set_cpu(cpu_of(rq), &wrq->freq_domain_cpumask); + + wrq->walt_stats.cumulative_runnable_avg_scaled = 0; + wrq->prev_window_size = sched_ravg_window; + wrq->window_start = 0; + wrq->walt_stats.nr_big_tasks = 0; + wrq->walt_flags = 0; + wrq->avg_irqload = 0; + wrq->prev_irq_time = 0; + wrq->last_irq_window = 0; + wrq->high_irqload = false; + wrq->task_exec_scale = 1024; + wrq->push_task = NULL; + + /* + * All cpus part of same cluster by default. This avoids the + * need to check for wrq->cluster being non-NULL in hot-paths + * like select_best_cpu() + */ + wrq->cluster = &init_cluster; + wrq->curr_runnable_sum = wrq->prev_runnable_sum = 0; + wrq->nt_curr_runnable_sum = wrq->nt_prev_runnable_sum = 0; + memset(&wrq->grp_time, 0, sizeof(struct group_cpu_time)); + wrq->old_busy_time = 0; + wrq->old_estimated_time = 0; + wrq->walt_stats.pred_demands_sum_scaled = 0; + wrq->walt_stats.nr_rtg_high_prio_tasks = 0; + wrq->ed_task = NULL; + wrq->curr_table = 0; + wrq->prev_top = 0; + wrq->curr_top = 0; + wrq->last_cc_update = 0; + wrq->cycles = 0; + for (j = 0; j < NUM_TRACKED_WINDOWS; j++) { + memset(&wrq->load_subs[j], 0, + sizeof(struct load_subtractions)); + wrq->top_tasks[j] = kcalloc(NUM_LOAD_INDICES, + sizeof(u8), GFP_ATOMIC | GFP_NOWAIT); + /* No other choice */ + BUG_ON(!wrq->top_tasks[j]); + clear_top_tasks_bitmap(wrq->top_tasks_bitmap[j]); + } + wrq->notif_pending = false; + + wrq->num_mvp_tasks = 0; + INIT_LIST_HEAD(&wrq->mvp_tasks); +} + +void sched_window_nr_ticks_change(void) +{ + unsigned long flags; + + spin_lock_irqsave(&sched_ravg_window_lock, flags); + new_sched_ravg_window = mult_frac(sysctl_sched_ravg_window_nr_ticks, + NSEC_PER_SEC, HZ); + spin_unlock_irqrestore(&sched_ravg_window_lock, flags); +} + +static void +walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, wts->demand_scaled, + wts->pred_demand_scaled); +} + +static void +walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + fixup_cumulative_runnable_avg(rq, p, &wrq->walt_stats, + -(s64)wts->demand_scaled, + -(s64)wts->pred_demand_scaled); +} + +static void inc_rq_walt_stats(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (wts->misfit) + wrq->walt_stats.nr_big_tasks++; + + wts->rtg_high_prio = task_rtg_high_prio(p); + if (wts->rtg_high_prio) + wrq->walt_stats.nr_rtg_high_prio_tasks++; +} + +static void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (wts->misfit) + wrq->walt_stats.nr_big_tasks--; + + if (wts->rtg_high_prio) + wrq->walt_stats.nr_rtg_high_prio_tasks--; + + BUG_ON(wrq->walt_stats.nr_big_tasks < 0); +} + +static void android_rvh_wake_up_new_task(void *unused, struct task_struct *new) +{ + if (unlikely(walt_disabled)) + return; + init_new_task_load(new); + add_new_task_to_grp(new); +} + +static void walt_cpu_frequency_limits(void *unused, struct cpufreq_policy *policy) +{ + if (unlikely(walt_disabled)) + return; + + cpu_cluster(policy->cpu)->max_freq = policy->max; +} + +/* + * The intention of this hook is to update cpu_capacity_orig as well as + * (*capacity), otherwise we will end up capacity_of() > capacity_orig_of(). + */ +static void android_rvh_update_cpu_capacity(void *unused, int cpu, unsigned long *capacity) +{ + unsigned long fmax_capacity = arch_scale_cpu_capacity(cpu); + unsigned long thermal_pressure = arch_scale_thermal_pressure(cpu); + unsigned long thermal_cap, old; + unsigned long rt_pressure = fmax_capacity - *capacity; + struct walt_sched_cluster *cluster; + struct rq *rq = cpu_rq(cpu); + + if (unlikely(walt_disabled)) + return; + + /* + * thermal_pressure = cpu_scale - curr_cap_as_per_thermal. + * so, + * curr_cap_as_per_thermal = cpu_scale - thermal_pressure. + */ + + thermal_cap = fmax_capacity - thermal_pressure; + + cluster = cpu_cluster(cpu); + /* reduce the fmax_capacity under cpufreq constraints */ + if (cluster->max_freq != cluster->max_possible_freq) + fmax_capacity = mult_frac(fmax_capacity, cluster->max_freq, + cluster->max_possible_freq); + + old = rq->cpu_capacity_orig; + rq->cpu_capacity_orig = min(fmax_capacity, thermal_cap); + + if (old != rq->cpu_capacity_orig) + trace_update_cpu_capacity(cpu, rt_pressure, *capacity); + + *capacity = max(rq->cpu_capacity_orig - rt_pressure, 1UL); +} + +static void android_rvh_sched_cpu_starting(void *unused, int cpu) +{ + if (unlikely(walt_disabled)) + return; + clear_walt_request(cpu); +} + +static void android_rvh_sched_cpu_dying(void *unused, int cpu) +{ + if (unlikely(walt_disabled)) + return; + clear_walt_request(cpu); +} + +static void android_rvh_set_task_cpu(void *unused, struct task_struct *p, unsigned int new_cpu) +{ + if (unlikely(walt_disabled)) + return; + + migrate_busy_time_subtraction(p, (int) new_cpu); + + if (!cpumask_test_cpu(new_cpu, p->cpus_ptr)) + WALT_BUG(WALT_BUG_WALT, p, "selecting unaffined cpu=%d comm=%s(%d) affinity=0x%x", + new_cpu, p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); + + if (!p->in_execve && + is_compat_thread(task_thread_info(p)) && + !cpumask_test_cpu(new_cpu, system_32bit_el0_cpumask())) + WALT_BUG(WALT_BUG_WALT, p, + "selecting non 32 bit cpu=%d comm=%s(%d) 32bit_cpus=0x%x", + new_cpu, p->comm, p->pid, (*(cpumask_bits(system_32bit_el0_cpumask())))); +} + +static void android_rvh_new_task_stats(void *unused, struct task_struct *p) +{ + if (unlikely(walt_disabled)) + return; + mark_task_starting(p); +} + +static void android_rvh_account_irq_start(void *unused, struct task_struct *curr, int cpu, + s64 delta) +{ + struct rq *rq; + struct walt_rq *wrq; + + if (unlikely(walt_disabled)) + return; + + if (!is_idle_task(curr)) + return; + + rq = cpu_rq(cpu); + wrq = (struct walt_rq *) rq->android_vendor_data1; + + if (!wrq->window_start) + return; + + /* We're here without rq->lock held, IRQ disabled */ + raw_spin_lock(&rq->__lock); + update_task_cpu_cycles(curr, cpu, walt_sched_clock()); + raw_spin_unlock(&rq->__lock); +} + +static void android_rvh_account_irq_end(void *unused, struct task_struct *curr, int cpu, s64 delta) +{ + struct rq *rq; + unsigned long flags; + struct walt_rq *wrq; + + if (unlikely(walt_disabled)) + return; + + if (!is_idle_task(curr)) + return; + + rq = cpu_rq(cpu); + wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + raw_spin_lock_irqsave(&rq->__lock, flags); + walt_update_task_ravg(curr, rq, IRQ_UPDATE, walt_sched_clock(), delta); + raw_spin_unlock_irqrestore(&rq->__lock, flags); + + wrq->last_irq_window = wrq->window_start; +} + +static void android_rvh_flush_task(void *unused, struct task_struct *p) +{ + if (unlikely(walt_disabled)) + return; + walt_task_dead(p); +} + +static void android_rvh_enqueue_task(void *unused, struct rq *rq, struct task_struct *p) +{ + u64 wallclock; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + bool double_enqueue = false; + + if (unlikely(walt_disabled)) + return; + + lockdep_assert_held(&rq->__lock); + + if (p->cpu != cpu_of(rq)) + WALT_BUG(WALT_BUG_UPSTREAM, p, "enqueuing on rq %d when task->cpu is %d\n", + cpu_of(rq), p->cpu); + + /* catch double enqueue */ + if (wts->prev_on_rq == 1) { + WALT_BUG(WALT_BUG_UPSTREAM, p, "double enqueue detected: task_cpu=%d new_cpu=%d\n", + task_cpu(p), cpu_of(rq)); + double_enqueue = true; + } + + if (cpu_halted(cpu_of(rq)) && !(p->flags & PF_KTHREAD) && !walt_halt_check_last(cpu_of(rq))) + WALT_BUG(WALT_BUG_NONCRITICAL, p, + "Non Kthread Started on halted cpu_of(rq)=%d comm=%s(%d) affinity=0x%x\n", + cpu_of(rq), p->comm, p->pid, (*(cpumask_bits(p->cpus_ptr)))); + + wallclock = walt_rq_clock(rq); + if (wts->enqueue_after_migration != 0) { + wallclock = walt_sched_clock(); + migrate_busy_time_addition(p, cpu_of(rq), wallclock); + } + + wts->prev_on_rq = 1; + wts->prev_on_rq_cpu = cpu_of(rq); + + wts->last_enqueued_ts = wallclock; + sched_update_nr_prod(rq->cpu, 1); + + if (walt_fair_task(p)) { + wts->misfit = !task_fits_max(p, rq->cpu); + if (!double_enqueue) + inc_rq_walt_stats(rq, p); + walt_cfs_enqueue_task(rq, p); + } + + if (!double_enqueue) + walt_inc_cumulative_runnable_avg(rq, p); + trace_sched_enq_deq_task(p, 1, cpumask_bits(p->cpus_ptr)[0], is_mvp(wts)); +} + +static void android_rvh_dequeue_task(void *unused, struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + bool double_dequeue = false; + + if (unlikely(walt_disabled)) + return; + + lockdep_assert_held(&rq->__lock); + + /* + * a task can be enqueued before walt is started, and dequeued after. + * therefore the check to ensure that prev_on_rq_cpu is needed to prevent + * an invalid failure. + */ + if (wts->prev_on_rq_cpu >= 0 && wts->prev_on_rq_cpu != cpu_of(rq)) + WALT_BUG(WALT_BUG_UPSTREAM, p, "dequeue cpu %d not same as enqueue %d\n", + cpu_of(rq), wts->prev_on_rq_cpu); + + /* no longer on a cpu */ + wts->prev_on_rq_cpu = -1; + + /* catch double deq */ + if (wts->prev_on_rq == 2) { + WALT_BUG(WALT_BUG_UPSTREAM, p, "double dequeue detected: task_cpu=%d new_cpu=%d\n", + task_cpu(p), cpu_of(rq)); + double_dequeue = true; + } + + wts->prev_on_rq = 2; + if (p == wrq->ed_task) + is_ed_task_present(rq, walt_rq_clock(rq), p); + + sched_update_nr_prod(rq->cpu, -1); + + if (walt_fair_task(p)) { + if (!double_dequeue) + dec_rq_walt_stats(rq, p); + walt_cfs_dequeue_task(rq, p); + } + + if (!double_dequeue) + walt_dec_cumulative_runnable_avg(rq, p); + + trace_sched_enq_deq_task(p, 0, cpumask_bits(p->cpus_ptr)[0], is_mvp(wts)); +} + +static void android_rvh_update_misfit_status(void *unused, struct task_struct *p, + struct rq *rq, bool *need_update) +{ + struct walt_task_struct *wts; + struct walt_rq *wrq; + bool old_misfit, misfit; + int change; + + if (unlikely(walt_disabled)) + return; + *need_update = false; + + if (!p) { + rq->misfit_task_load = 0; + return; + } + + wrq = (struct walt_rq *) rq->android_vendor_data1; + wts = (struct walt_task_struct *) p->android_vendor_data1; + old_misfit = wts->misfit; + + if (task_fits_max(p, rq->cpu)) + rq->misfit_task_load = 0; + else + rq->misfit_task_load = task_util(p); + + misfit = rq->misfit_task_load; + + change = misfit - old_misfit; + if (change) { + sched_update_nr_prod(rq->cpu, 0); + wts->misfit = misfit; + wrq->walt_stats.nr_big_tasks += change; + BUG_ON(wrq->walt_stats.nr_big_tasks < 0); + } +} + +/* utility function to update walt signals at wakeup */ +static void android_rvh_try_to_wake_up(void *unused, struct task_struct *p) +{ + struct rq *rq = cpu_rq(task_cpu(p)); + struct rq_flags rf; + u64 wallclock; + unsigned int old_load; + struct walt_related_thread_group *grp = NULL; + + if (unlikely(walt_disabled)) + return; + rq_lock_irqsave(rq, &rf); + old_load = task_load(p); + wallclock = walt_sched_clock(); + + if (is_idle_task(rq->curr) && p->in_iowait) + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); + note_task_waking(p, wallclock); + rq_unlock_irqrestore(rq, &rf); + + rcu_read_lock(); + grp = task_related_thread_group(p); + if (update_preferred_cluster(grp, p, old_load, false)) + set_preferred_cluster(grp); + rcu_read_unlock(); +} + +static void android_rvh_try_to_wake_up_success(void *unused, struct task_struct *p) +{ + unsigned long flags; + int cpu = p->cpu; + + if (unlikely(walt_disabled)) + return; + + raw_spin_lock_irqsave(&cpu_rq(cpu)->__lock, flags); + if (do_pl_notif(cpu_rq(cpu))) + waltgov_run_callback(cpu_rq(cpu), WALT_CPUFREQ_PL); + raw_spin_unlock_irqrestore(&cpu_rq(cpu)->__lock, flags); +} + +static u64 tick_sched_clock; +static DECLARE_COMPLETION(tick_sched_clock_completion); + +static void android_rvh_tick_entry(void *unused, struct rq *rq) +{ + u64 wallclock; + + if (unlikely(walt_disabled)) + return; + + lockdep_assert_held(&rq->__lock); + wallclock = walt_rq_clock(rq); + + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + + if (is_ed_task_present(rq, wallclock, NULL)) + waltgov_run_callback(rq, WALT_CPUFREQ_EARLY_DET); +} + +static void android_vh_scheduler_tick(void *unused, struct rq *rq) +{ + struct walt_related_thread_group *grp; + u32 old_load; + + if (!tick_sched_clock) { + /* + * Let the window begin 20us prior to the tick, + * that way we are guaranteed a rollover when the tick occurs. + */ + tick_sched_clock = rq_clock(rq) - 20000; + complete(&tick_sched_clock_completion); + } + + if (unlikely(walt_disabled)) + return; + + old_load = task_load(rq->curr); + rcu_read_lock(); + grp = task_related_thread_group(rq->curr); + if (update_preferred_cluster(grp, rq->curr, old_load, true)) + set_preferred_cluster(grp); + rcu_read_unlock(); + + walt_lb_tick(rq); +} + +static void android_rvh_schedule(void *unused, struct task_struct *prev, + struct task_struct *next, struct rq *rq) +{ + u64 wallclock; + struct walt_task_struct *wts = (struct walt_task_struct *) prev->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + wallclock = walt_rq_clock(rq); + + if (likely(prev != next)) { + if (!prev->on_rq) + wts->last_sleep_ts = wallclock; + walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); + walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); + } else { + walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0); + } +} + +static void android_rvh_update_cpus_allowed(void *unused, struct task_struct *p, + cpumask_var_t cpus_requested, + const struct cpumask *new_mask, int *ret) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + if (cpumask_subset(&wts->cpus_requested, cpus_requested)) + *ret = set_cpus_allowed_ptr(p, &wts->cpus_requested); +} + +static void android_rvh_sched_getaffinity(void *unused, struct task_struct *p, + struct cpumask *in_mask) +{ + if (unlikely(walt_disabled)) + return; + + if (!(p->flags & PF_KTHREAD)) + cpumask_andnot(in_mask, in_mask, cpu_halt_mask); +} + +static void android_rvh_sched_setaffinity(void *unused, struct task_struct *p, + const struct cpumask *in_mask, + int *retval) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + /* nothing to do if the affinity call failed */ + if (*retval) + return; + + /* + * cache the affinity for user space tasks so that they + * can be restored during cpuset cgroup change. + */ + if (!(p->flags & PF_KTHREAD)) + cpumask_and(&wts->cpus_requested, in_mask, cpu_possible_mask); +} + +static void android_rvh_sched_fork_init(void *unused, struct task_struct *p) +{ + if (unlikely(walt_disabled)) + return; + + __sched_fork_init(p); +} + +static void android_rvh_ttwu_cond(void *unused, int cpu, bool *cond) +{ + if (unlikely(walt_disabled)) + return; + *cond = (sysctl_sched_many_wakeup_threshold < WALT_MANY_WAKEUP_DEFAULT) && + (cpu != smp_processor_id()); +} + +static void android_rvh_sched_exec(void *unused, bool *cond) +{ + if (unlikely(walt_disabled)) + return; + *cond = true; +} + +static void android_rvh_build_perf_domains(void *unused, bool *eas_check) +{ + if (unlikely(walt_disabled)) + return; + *eas_check = true; +} + +static void walt_do_sched_yield(void *unused, struct rq *rq) +{ + struct task_struct *curr = rq->curr; + struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + lockdep_assert_held(&rq->__lock); + if (!list_empty(&wts->mvp_list) && wts->mvp_list.next) + walt_cfs_deactivate_mvp_task(rq, curr); + + if (per_cpu(rt_task_arrival_time, cpu_of(rq))) + per_cpu(rt_task_arrival_time, cpu_of(rq)) = 0; +} + +static void register_walt_hooks(void) +{ + register_trace_android_rvh_wake_up_new_task(android_rvh_wake_up_new_task, NULL); + register_trace_android_rvh_update_cpu_capacity(android_rvh_update_cpu_capacity, NULL); + register_trace_android_rvh_sched_cpu_starting(android_rvh_sched_cpu_starting, NULL); + register_trace_android_rvh_sched_cpu_dying(android_rvh_sched_cpu_dying, NULL); + register_trace_android_rvh_set_task_cpu(android_rvh_set_task_cpu, NULL); + register_trace_android_rvh_new_task_stats(android_rvh_new_task_stats, NULL); + register_trace_android_rvh_account_irq_start(android_rvh_account_irq_start, NULL); + register_trace_android_rvh_account_irq_end(android_rvh_account_irq_end, NULL); + register_trace_android_rvh_flush_task(android_rvh_flush_task, NULL); + register_trace_android_rvh_update_misfit_status(android_rvh_update_misfit_status, NULL); + register_trace_android_rvh_after_enqueue_task(android_rvh_enqueue_task, NULL); + register_trace_android_rvh_after_dequeue_task(android_rvh_dequeue_task, NULL); + register_trace_android_rvh_try_to_wake_up(android_rvh_try_to_wake_up, NULL); + register_trace_android_rvh_try_to_wake_up_success(android_rvh_try_to_wake_up_success, NULL); + register_trace_android_rvh_tick_entry(android_rvh_tick_entry, NULL); + register_trace_android_vh_scheduler_tick(android_vh_scheduler_tick, NULL); + register_trace_android_rvh_schedule(android_rvh_schedule, NULL); + register_trace_android_rvh_cpu_cgroup_attach(android_rvh_cpu_cgroup_attach, NULL); + register_trace_android_rvh_cpu_cgroup_online(android_rvh_cpu_cgroup_online, NULL); + register_trace_android_rvh_update_cpus_allowed(android_rvh_update_cpus_allowed, NULL); + register_trace_android_rvh_sched_setaffinity(android_rvh_sched_setaffinity, NULL); + register_trace_android_rvh_sched_getaffinity(android_rvh_sched_getaffinity, NULL); + register_trace_android_rvh_sched_fork_init(android_rvh_sched_fork_init, NULL); + register_trace_android_rvh_ttwu_cond(android_rvh_ttwu_cond, NULL); + register_trace_android_rvh_sched_exec(android_rvh_sched_exec, NULL); + register_trace_android_rvh_build_perf_domains(android_rvh_build_perf_domains, NULL); + register_trace_cpu_frequency_limits(walt_cpu_frequency_limits, NULL); + register_trace_android_rvh_do_sched_yield(walt_do_sched_yield, NULL); +} + +atomic64_t walt_irq_work_lastq_ws; +bool walt_disabled = true; + +static int walt_init_stop_handler(void *data) +{ + int cpu; + struct task_struct *g, *p; + struct walt_rq *wrq; + int level = 0; + + read_lock(&tasklist_lock); + for_each_possible_cpu(cpu) { + if (level == 0) + raw_spin_lock(&cpu_rq(cpu)->__lock); + else + raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level); + level++; + } + + do_each_thread(g, p) { + init_existing_task_load(p); + } while_each_thread(g, p); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + /* Create task members for idle thread */ + init_new_task_load(rq->idle); + + walt_sched_init_rq(rq); + + wrq = (struct walt_rq *) rq->android_vendor_data1; + wrq->window_start = tick_sched_clock; + } + + atomic64_set(&walt_irq_work_lastq_ws, tick_sched_clock); + + create_default_coloc_group(); + + walt_update_cluster_topology(); + + walt_disabled = false; + + for_each_possible_cpu(cpu) { + raw_spin_unlock(&cpu_rq(cpu)->__lock); + } + read_unlock(&tasklist_lock); + return 0; +} + +static void walt_init_tg_pointers(void) +{ + struct cgroup_subsys_state *css = &root_task_group.css; + struct cgroup_subsys_state *top_css = css; + + rcu_read_lock(); + css_for_each_child(css, top_css) + walt_update_tg_pointer(css); + rcu_read_unlock(); +} + +static void walt_init(struct work_struct *work) +{ + struct ctl_table_header *hdr; + static atomic_t already_inited = ATOMIC_INIT(0); + int i; + + might_sleep(); + + if (atomic_cmpxchg(&already_inited, 0, 1)) + return; + + walt_tunables(); + + register_syscore_ops(&walt_syscore_ops); + BUG_ON(alloc_related_thread_groups()); + walt_init_cycle_counter(); + init_clusters(); + walt_init_tg_pointers(); + + register_walt_hooks(); + walt_fixup_init(); + walt_lb_init(); + walt_rt_init(); + walt_cfs_init(); + walt_halt_init(); + wait_for_completion_interruptible(&tick_sched_clock_completion); + stop_machine(walt_init_stop_handler, NULL, NULL); + + hdr = register_sysctl_table(walt_base_table); + kmemleak_not_leak(hdr); + + input_boost_init(); + core_ctl_init(); + walt_boost_init(); + waltgov_register(); + + i = match_string(sched_feat_names, __SCHED_FEAT_NR, "TTWU_QUEUE"); + if (i >= 0) { + static_key_disable(&sched_feat_keys[i]); + sysctl_sched_features &= ~(1UL << i); + } + + topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_ARCH, cpu_online_mask); +} + +static DECLARE_WORK(walt_init_work, walt_init); +static void android_vh_update_topology_flags_workfn(void *unused, void *unused2) +{ + schedule_work(&walt_init_work); +} + +#define WALT_VENDOR_DATA_SIZE_TEST(wstruct, kstruct) \ + BUILD_BUG_ON(sizeof(wstruct) > (sizeof(u64) * \ + ARRAY_SIZE(((kstruct *)0)->android_vendor_data1))) + +static int walt_module_init(void) +{ + /* compile time checks for vendor data size */ + WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_struct, struct task_struct); + WALT_VENDOR_DATA_SIZE_TEST(struct walt_rq, struct rq); + WALT_VENDOR_DATA_SIZE_TEST(struct walt_task_group, struct task_group); + + register_trace_android_vh_update_topology_flags_workfn( + android_vh_update_topology_flags_workfn, NULL); + + if (topology_update_done) + schedule_work(&walt_init_work); + + return 0; +} + +module_init(walt_module_init); +MODULE_LICENSE("GPL v2"); + +#if IS_ENABLED(CONFIG_SCHED_WALT_DEBUG) +MODULE_SOFTDEP("pre: sched-walt-debug"); +#endif diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h new file mode 100644 index 000000000000..b8a0e5bf8096 --- /dev/null +++ b/kernel/sched/walt/walt.h @@ -0,0 +1,1021 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _WALT_H +#define _WALT_H + +#include "../../../kernel/sched/sched.h" +#include "../../../fs/proc/internal.h" +#include +#include + +#include + +#ifdef CONFIG_HZ_300 +/* + * Tick interval becomes to 3333333 due to + * rounding error when HZ=300. + */ +#define DEFAULT_SCHED_RAVG_WINDOW (3333333 * 5) +#else +/* Min window size (in ns) = 16ms */ +#define DEFAULT_SCHED_RAVG_WINDOW 16000000 +#endif + +/* Max window size (in ns) = 1s */ +#define MAX_SCHED_RAVG_WINDOW 1000000000 + +#define NR_WINDOWS_PER_SEC (NSEC_PER_SEC / DEFAULT_SCHED_RAVG_WINDOW) + +/* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */ +#define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1) + +extern bool walt_disabled; + +enum task_event { + PUT_PREV_TASK = 0, + PICK_NEXT_TASK = 1, + TASK_WAKE = 2, + TASK_MIGRATE = 3, + TASK_UPDATE = 4, + IRQ_UPDATE = 5, +}; + +/* Note: this need to be in sync with migrate_type_names array */ +enum migrate_types { + GROUP_TO_RQ, + RQ_TO_GROUP, +}; + +#define WALT_LOW_LATENCY_PROCFS BIT(0) +#define WALT_LOW_LATENCY_BINDER BIT(1) +#define WALT_LOW_LATENCY_PIPELINE BIT(2) + +struct walt_cpu_load { + unsigned long nl; + unsigned long pl; + bool rtgb_active; + u64 ws; +}; + +#define DECLARE_BITMAP_ARRAY(name, nr, bits) \ + unsigned long name[nr][BITS_TO_LONGS(bits)] + +struct walt_sched_stats { + int nr_big_tasks; + u64 cumulative_runnable_avg_scaled; + u64 pred_demands_sum_scaled; + unsigned int nr_rtg_high_prio_tasks; +}; + +#define NUM_TRACKED_WINDOWS 2 +#define NUM_LOAD_INDICES 1000 + +struct group_cpu_time { + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; +}; + +struct load_subtractions { + u64 window_start; + u64 subs; + u64 new_subs; +}; + +struct walt_rq { + struct task_struct *push_task; + struct walt_sched_cluster *cluster; + struct cpumask freq_domain_cpumask; + struct walt_sched_stats walt_stats; + + u64 window_start; + u32 prev_window_size; + unsigned long walt_flags; + + u64 avg_irqload; + u64 last_irq_window; + u64 prev_irq_time; + struct task_struct *ed_task; + u64 task_exec_scale; + u64 old_busy_time; + u64 old_estimated_time; + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; + struct group_cpu_time grp_time; + struct load_subtractions load_subs[NUM_TRACKED_WINDOWS]; + DECLARE_BITMAP_ARRAY(top_tasks_bitmap, + NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES); + u8 *top_tasks[NUM_TRACKED_WINDOWS]; + u8 curr_table; + int prev_top; + int curr_top; + bool notif_pending; + bool high_irqload; + u64 last_cc_update; + u64 cycles; + u64 util; + struct list_head mvp_tasks; + int num_mvp_tasks; + u64 latest_clock; +}; + +struct walt_sched_cluster { + raw_spinlock_t load_lock; + struct list_head list; + struct cpumask cpus; + int id; + /* + * max_possible_freq = maximum supported by hardware + * max_freq = max freq as per cpufreq limits + */ + unsigned int cur_freq; + unsigned int max_possible_freq; + unsigned int max_freq; + u64 aggr_grp_load; + unsigned long util_to_cost[1024]; +}; + +extern struct walt_sched_cluster *sched_cluster[WALT_NR_CPUS]; + +/*END SCHED.H PORT*/ + +extern int num_sched_clusters; +extern unsigned int sched_capacity_margin_up[WALT_NR_CPUS]; +extern unsigned int sched_capacity_margin_down[WALT_NR_CPUS]; +extern cpumask_t asym_cap_sibling_cpus; +extern cpumask_t __read_mostly **cpu_array; +extern int cpu_l2_sibling[WALT_NR_CPUS]; +extern void sched_update_nr_prod(int cpu, int enq); +extern unsigned int walt_big_tasks(int cpu); +extern void walt_rotation_checkpoint(int nr_big); +extern void walt_fill_ta_data(struct core_ctl_notif_data *data); +extern int sched_set_group_id(struct task_struct *p, unsigned int group_id); +extern unsigned int sched_get_group_id(struct task_struct *p); +extern void core_ctl_check(u64 wallclock); +extern int sched_set_boost(int enable); +extern void walt_boost_init(void); +extern int sched_pause_cpus(struct cpumask *pause_cpus); +extern int sched_unpause_cpus(struct cpumask *unpause_cpus); +extern void walt_kick_cpu(int cpu); +extern void walt_smp_call_newidle_balance(int cpu); + +extern unsigned int sched_get_cpu_util_pct(int cpu); +extern void sched_update_hyst_times(void); +extern int sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); +extern int sched_busy_hyst_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); +extern u64 walt_sched_clock(void); +extern void walt_init_tg(struct task_group *tg); +extern void walt_init_topapp_tg(struct task_group *tg); +extern void walt_init_foreground_tg(struct task_group *tg); +extern int register_walt_callback(void); +extern int input_boost_init(void); +extern int core_ctl_init(void); + +extern atomic64_t walt_irq_work_lastq_ws; +extern unsigned int __read_mostly sched_ravg_window; +extern int min_possible_cluster_id; +extern int max_possible_cluster_id; +extern unsigned int __read_mostly sched_init_task_load_windows; +extern unsigned int __read_mostly sched_load_granule; + +extern unsigned int sysctl_sched_idle_enough; +extern unsigned int sysctl_sched_cluster_util_thres_pct; + +/* 1ms default for 20ms window size scaled to 1024 */ +extern unsigned int sysctl_sched_min_task_util_for_boost; +extern unsigned int sysctl_sched_min_task_util_for_uclamp; +/* 0.68ms default for 20ms window size scaled to 1024 */ +extern unsigned int sysctl_sched_min_task_util_for_colocation; +extern unsigned int sysctl_sched_busy_hyst_enable_cpus; +extern unsigned int sysctl_sched_busy_hyst; +extern unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus; +extern unsigned int sysctl_sched_coloc_busy_hyst_cpu[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_coloc_busy_hyst_max_ms; +extern unsigned int sysctl_sched_coloc_busy_hyst_cpu_busy_pct[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_util_busy_hyst_enable_cpus; +extern unsigned int sysctl_sched_util_busy_hyst_cpu[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_util_busy_hyst_cpu_util[WALT_NR_CPUS]; +extern unsigned int sysctl_sched_boost; /* To/from userspace */ +extern unsigned int sysctl_sched_capacity_margin_up[MAX_MARGIN_LEVELS]; +extern unsigned int sysctl_sched_capacity_margin_down[MAX_MARGIN_LEVELS]; +extern unsigned int sched_boost_type; /* currently activated sched boost */ +extern enum sched_boost_policy boost_policy; +extern unsigned int sysctl_input_boost_ms; +extern unsigned int sysctl_input_boost_freq[8]; +extern unsigned int sysctl_sched_boost_on_input; +extern unsigned int sysctl_sched_user_hint; +extern unsigned int sysctl_sched_conservative_pl; +extern unsigned int sysctl_sched_hyst_min_coloc_ns; +extern unsigned int sysctl_sched_long_running_rt_task_ms; + +#define WALT_MANY_WAKEUP_DEFAULT 1000 +extern unsigned int sysctl_sched_many_wakeup_threshold; +extern unsigned int sysctl_walt_rtg_cfs_boost_prio; +extern __read_mostly unsigned int sysctl_sched_force_lb_enable; +extern const int sched_user_hint_max; +extern unsigned int sysctl_sched_dynamic_tp_enable; +extern unsigned int sysctl_panic_on_walt_bug; +extern int sched_dynamic_tp_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +extern struct list_head cluster_head; +#define for_each_sched_cluster(cluster) \ + list_for_each_entry_rcu(cluster, &cluster_head, list) + +static inline struct walt_sched_cluster *cpu_cluster(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster; +} + +static inline u32 cpu_cycles_to_freq(u64 cycles, u64 period) +{ + return div64_u64(cycles, period); +} + +static inline unsigned int sched_cpu_legacy_freq(int cpu) +{ + unsigned long curr_cap = arch_scale_freq_capacity(cpu); + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return (curr_cap * (u64) wrq->cluster->max_possible_freq) >> + SCHED_CAPACITY_SHIFT; +} + +extern __read_mostly bool sched_freq_aggr_en; +static inline void walt_enable_frequency_aggregation(bool enable) +{ + sched_freq_aggr_en = enable; +} + +#ifndef CONFIG_IRQ_TIME_ACCOUNTING +static inline u64 irq_time_read(int cpu) { return 0; } +#endif + +/*Sysctl related interface*/ +#define WINDOW_STATS_RECENT 0 +#define WINDOW_STATS_MAX 1 +#define WINDOW_STATS_MAX_RECENT_AVG 2 +#define WINDOW_STATS_AVG 3 +#define WINDOW_STATS_INVALID_POLICY 4 + +extern unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; +extern unsigned int __read_mostly sysctl_sched_group_downmigrate_pct; +extern unsigned int __read_mostly sysctl_sched_group_upmigrate_pct; +extern unsigned int __read_mostly sysctl_sched_window_stats_policy; +extern unsigned int sysctl_sched_ravg_window_nr_ticks; +extern unsigned int sysctl_sched_walt_rotate_big_tasks; +extern unsigned int sysctl_sched_task_unfilter_period; +extern unsigned int __read_mostly sysctl_sched_asym_cap_sibling_freq_match_pct; +extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */ +extern unsigned int sysctl_sched_sync_hint_enable; +extern unsigned int sysctl_sched_suppress_region2; +extern unsigned int sysctl_sched_skip_sp_newly_idle_lb; +extern unsigned int sysctl_sched_asymcap_boost; +extern struct ctl_table walt_table[]; +extern struct ctl_table walt_base_table[]; +extern void walt_tunables(void); +extern void walt_update_group_thresholds(void); +extern void sched_window_nr_ticks_change(void); +extern unsigned long sched_user_hint_reset_time; +extern struct irq_work walt_migration_irq_work; + +#define LIB_PATH_LENGTH 512 +extern unsigned int cpuinfo_max_freq_cached; +extern char sched_lib_name[LIB_PATH_LENGTH]; +extern unsigned int sched_lib_mask_force; + +/* WALT cpufreq interface */ +#define WALT_CPUFREQ_ROLLOVER 0x1 +#define WALT_CPUFREQ_CONTINUE 0x2 +#define WALT_CPUFREQ_IC_MIGRATION 0x4 +#define WALT_CPUFREQ_PL 0x8 +#define WALT_CPUFREQ_EARLY_DET 0x10 +#define WALT_CPUFREQ_BOOST_UPDATE 0x20 + +#define CPUFREQ_REASON_LOAD 0 +#define CPUFREQ_REASON_PL 0x2 +#define CPUFREQ_REASON_EARLY_DET 0x4 +#define CPUFREQ_REASON_RTG_BOOST 0x8 +#define CPUFREQ_REASON_HISPEED 0x10 +#define CPUFREQ_REASON_NWD 0x20 +#define CPUFREQ_REASON_FREQ_AGR 0x40 +#define CPUFREQ_REASON_KSOFTIRQD 0x80 +#define CPUFREQ_REASON_TT_LOAD 0x100 +#define CPUFREQ_REASON_SUH 0x200 +#define CPUFREQ_REASON_ADAPTIVE_LOW 0x400 +#define CPUFREQ_REASON_ADAPTIVE_HIGH 0x800 + +#define NO_BOOST 0 +#define FULL_THROTTLE_BOOST 1 +#define CONSERVATIVE_BOOST 2 +#define RESTRAINED_BOOST 3 +#define FULL_THROTTLE_BOOST_DISABLE -1 +#define CONSERVATIVE_BOOST_DISABLE -2 +#define RESTRAINED_BOOST_DISABLE -3 +#define MAX_NUM_BOOST_TYPE (RESTRAINED_BOOST+1) + +enum sched_boost_policy { + SCHED_BOOST_NONE, + SCHED_BOOST_ON_BIG, + SCHED_BOOST_ON_ALL, +}; + +struct walt_task_group { + /* + * Controls whether tasks of this cgroup should be colocated with each + * other and tasks of other cgroups that have the same flag turned on. + */ + bool colocate; + /* + * array indicating whether this task group participates in the + * particular boost type + */ + bool sched_boost_enable[MAX_NUM_BOOST_TYPE]; +}; + +struct sched_avg_stats { + int nr; + int nr_misfit; + int nr_max; + int nr_scaled; +}; + +struct waltgov_callback { + void (*func)(struct waltgov_callback *cb, u64 time, unsigned int flags); +}; + +DECLARE_PER_CPU(struct waltgov_callback *, waltgov_cb_data); + +static inline void waltgov_add_callback(int cpu, struct waltgov_callback *cb, + void (*func)(struct waltgov_callback *cb, u64 time, + unsigned int flags)) +{ + if (WARN_ON(!cb || !func)) + return; + + if (WARN_ON(per_cpu(waltgov_cb_data, cpu))) + return; + + cb->func = func; + rcu_assign_pointer(per_cpu(waltgov_cb_data, cpu), cb); +} + +static inline void waltgov_remove_callback(int cpu) +{ + rcu_assign_pointer(per_cpu(waltgov_cb_data, cpu), NULL); +} + +static inline void waltgov_run_callback(struct rq *rq, unsigned int flags) +{ + struct waltgov_callback *cb; + + cb = rcu_dereference_sched(*per_cpu_ptr(&waltgov_cb_data, cpu_of(rq))); + if (cb) + cb->func(cb, walt_sched_clock(), flags); +} + +extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, + unsigned int *reason); +int waltgov_register(void); + +extern void walt_lb_init(void); +extern unsigned int walt_rotation_enabled; + +/* + * Returns the current capacity of cpu after applying both + * cpu and freq scaling. + */ +static inline unsigned long capacity_curr_of(int cpu) +{ + unsigned long max_cap = cpu_rq(cpu)->cpu_capacity_orig; + unsigned long scale_freq = arch_scale_freq_capacity(cpu); + + return cap_scale(max_cap, scale_freq); +} + +static inline unsigned long task_util(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->demand_scaled; +} + +static inline unsigned long cpu_util(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + u64 walt_cpu_util = wrq->walt_stats.cumulative_runnable_avg_scaled; + + return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu)); +} + +static inline unsigned long cpu_util_cum(int cpu) +{ + return READ_ONCE(cpu_rq(cpu)->cfs.avg.util_avg); +} + +/* applying the task threshold for all types of low latency tasks. */ +static inline bool walt_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->low_latency && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_binder_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_BINDER) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_procfs_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_PROCFS) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_pipeline_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->low_latency & WALT_LOW_LATENCY_PIPELINE; +} + +static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) +{ + struct cpuidle_state *idle = idle_get_state(rq); + + if (idle) + return idle->exit_latency; + + return 0; /* CPU is not idle */ +} + +static inline bool rt_boost_on_big(void) +{ + return sched_boost_type == FULL_THROTTLE_BOOST ? + (boost_policy == SCHED_BOOST_ON_BIG) : false; +} + +static inline bool is_full_throttle_boost(void) +{ + return sched_boost_type == FULL_THROTTLE_BOOST; +} + +static inline bool task_sched_boost(struct task_struct *p) +{ + struct cgroup_subsys_state *css; + struct task_group *tg; + bool sched_boost_enabled; + struct walt_task_group *wtg; + + /* optimization for FT boost, skip looking at tg */ + if (sched_boost_type == FULL_THROTTLE_BOOST) + return true; + + rcu_read_lock(); + css = task_css(p, cpu_cgrp_id); + if (!css) { + rcu_read_unlock(); + return false; + } + tg = container_of(css, struct task_group, css); + wtg = (struct walt_task_group *) tg->android_vendor_data1; + sched_boost_enabled = wtg->sched_boost_enable[sched_boost_type]; + rcu_read_unlock(); + + return sched_boost_enabled; +} + +static inline bool task_placement_boost_enabled(struct task_struct *p) +{ + if (likely(boost_policy == SCHED_BOOST_NONE)) + return false; + + return task_sched_boost(p); +} + +static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) +{ + enum sched_boost_policy policy; + + if (likely(boost_policy == SCHED_BOOST_NONE)) + return SCHED_BOOST_NONE; + + policy = task_sched_boost(p) ? boost_policy : SCHED_BOOST_NONE; + if (policy == SCHED_BOOST_ON_BIG) { + /* + * Filter out tasks less than min task util threshold + * under conservative boost. + */ + if (sched_boost_type == CONSERVATIVE_BOOST && + task_util(p) <= sysctl_sched_min_task_util_for_boost && + !walt_pipeline_low_latency_task(p)) + policy = SCHED_BOOST_NONE; + } + + return policy; +} + +static inline bool walt_uclamp_boosted(struct task_struct *p) +{ + return ((uclamp_eff_value(p, UCLAMP_MIN) > 0) && + (task_util(p) > sysctl_sched_min_task_util_for_uclamp)); +} + +static inline unsigned long capacity_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity; +} + +static inline bool __cpu_overutilized(int cpu, int delta) +{ + return (capacity_orig_of(cpu) * 1024) < + ((cpu_util(cpu) + delta) * sched_capacity_margin_up[cpu]); +} + +static inline bool cpu_overutilized(int cpu) +{ + return __cpu_overutilized(cpu, 0); +} + +static inline int asym_cap_siblings(int cpu1, int cpu2) +{ + return (cpumask_test_cpu(cpu1, &asym_cap_sibling_cpus) && + cpumask_test_cpu(cpu2, &asym_cap_sibling_cpus)); +} + +static inline bool asym_cap_sibling_group_has_capacity(int dst_cpu, int margin) +{ + int sib1, sib2; + int nr_running; + unsigned long total_util, total_capacity; + + if (cpumask_empty(&asym_cap_sibling_cpus) || + cpumask_test_cpu(dst_cpu, &asym_cap_sibling_cpus)) + return false; + + sib1 = cpumask_first(&asym_cap_sibling_cpus); + sib2 = cpumask_last(&asym_cap_sibling_cpus); + + if (!cpu_active(sib1) || !cpu_active(sib2)) + return false; + + nr_running = cpu_rq(sib1)->cfs.h_nr_running + + cpu_rq(sib2)->cfs.h_nr_running; + + if (nr_running <= 2) + return true; + + total_capacity = capacity_of(sib1) + capacity_of(sib2); + total_util = cpu_util(sib1) + cpu_util(sib2); + + return ((total_capacity * 100) > (total_util * margin)); +} + +/* Is frequency of two cpus synchronized with each other? */ +static inline int same_freq_domain(int src_cpu, int dst_cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; + + if (src_cpu == dst_cpu) + return 1; + + if (asym_cap_siblings(src_cpu, dst_cpu)) + return 1; + + return cpumask_test_cpu(dst_cpu, &wrq->freq_domain_cpumask); +} + +static inline unsigned long task_util_est(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->demand_scaled; +} + +#ifdef CONFIG_UCLAMP_TASK +static inline unsigned long uclamp_task_util(struct task_struct *p) +{ + return clamp(task_util_est(p), + uclamp_eff_value(p, UCLAMP_MIN), + uclamp_eff_value(p, UCLAMP_MAX)); +} +#else +static inline unsigned long uclamp_task_util(struct task_struct *p) +{ + return task_util_est(p); +} +#endif + +static inline int per_task_boost(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (wts->boost_period) { + if (walt_sched_clock() > wts->boost_expires) { + wts->boost_period = 0; + wts->boost_expires = 0; + wts->boost = 0; + } + } + return wts->boost; +} + +static inline int cluster_first_cpu(struct walt_sched_cluster *cluster) +{ + return cpumask_first(&cluster->cpus); +} + +static inline bool hmp_capable(void) +{ + return max_possible_cluster_id != min_possible_cluster_id; +} + +static inline bool is_max_cluster_cpu(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster->id == max_possible_cluster_id; +} + +static inline bool is_min_cluster_cpu(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster->id == min_possible_cluster_id; +} + +static inline bool is_min_capacity_cluster(struct walt_sched_cluster *cluster) +{ + return cluster->id == min_possible_cluster_id; +} + +/* + * This is only for tracepoints to print the avg irq load. For + * task placment considerations, use sched_cpu_high_irqload(). + */ +#define SCHED_HIGH_IRQ_TIMEOUT 3 +static inline u64 sched_irqload(int cpu) +{ + s64 delta; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + delta = wrq->window_start - wrq->last_irq_window; + if (delta < SCHED_HIGH_IRQ_TIMEOUT) + return wrq->avg_irqload; + else + return 0; +} + +static inline int sched_cpu_high_irqload(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->high_irqload; +} + +static inline u64 +scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq) +{ + return div64_u64(load * (u64)src_freq, (u64)dst_freq); +} + +static inline unsigned int max_task_load(void) +{ + return sched_ravg_window; +} + +static inline int same_cluster(int src_cpu, int dst_cpu) +{ + struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; + struct walt_rq *dest_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + + return src_wrq->cluster == dest_wrq->cluster; +} + +static inline bool is_suh_max(void) +{ + return sysctl_sched_user_hint == sched_user_hint_max; +} + +#define DEFAULT_CGROUP_COLOC_ID 1 +static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_related_thread_group *rtg = wts->grp; + + if (is_suh_max() && rtg && rtg->id == DEFAULT_CGROUP_COLOC_ID && + rtg->skip_min && wts->unfilter) + return is_min_cluster_cpu(cpu); + + return false; +} + +extern bool is_rtgb_active(void); +extern u64 get_rtgb_active_time(void); + +static inline unsigned int walt_nr_rtg_high_prio(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->walt_stats.nr_rtg_high_prio_tasks; +} + +static inline bool task_fits_capacity(struct task_struct *p, + long capacity, + int cpu) +{ + unsigned int margin; + struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + /* + * Derive upmigration/downmigrate margin wrt the src/dest CPU. + */ + if (src_wrq->cluster->id > dst_wrq->cluster->id) + margin = sched_capacity_margin_down[cpu]; + else + margin = sched_capacity_margin_up[task_cpu(p)]; + + return capacity * 1024 > uclamp_task_util(p) * margin; +} + +static inline bool task_in_related_thread_group(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (rcu_access_pointer(wts->grp) != NULL); +} + +static inline bool task_fits_max(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_orig_of(cpu); + unsigned long task_boost = per_task_boost(p); + + if (is_max_cluster_cpu(cpu)) + return true; + + if (is_min_cluster_cpu(cpu)) { + if (task_boost_policy(p) == SCHED_BOOST_ON_BIG || + task_boost > 0 || + walt_uclamp_boosted(p) || + walt_should_kick_upmigrate(p, cpu)) + return false; + } else { /* mid cap cpu */ + if (task_boost > TASK_BOOST_ON_MID) + return false; + if (!task_in_related_thread_group(p) && p->prio >= 124) + /* a non topapp low prio task fits on gold */ + return true; + } + + return task_fits_capacity(p, capacity, cpu); +} + +extern struct sched_avg_stats *sched_get_nr_running_avg(void); +extern unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster); +extern void sched_update_hyst_times(void); + +extern void walt_rt_init(void); +extern void walt_cfs_init(void); +extern void walt_halt_init(void); +extern void walt_fixup_init(void); +extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, + int sync, int sibling_count_hint); +extern int walt_find_cluster_packing_cpu(int start_cpu); +extern bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p); + +static inline unsigned int cpu_max_possible_freq(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->cluster->max_possible_freq; +} + +static inline unsigned int cpu_max_freq(int cpu) +{ + return mult_frac(cpu_max_possible_freq(cpu), capacity_orig_of(cpu), + arch_scale_cpu_capacity(cpu)); +} + +static inline unsigned int task_load(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->demand; +} + +static inline bool task_rtg_high_prio(struct task_struct *p) +{ + return task_in_related_thread_group(p) && + (p->prio <= sysctl_walt_rtg_cfs_boost_prio); +} + +static inline struct walt_related_thread_group +*task_related_thread_group(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return rcu_dereference(wts->grp); +} + +static inline bool walt_get_rtg_status(struct task_struct *p) +{ + struct walt_related_thread_group *grp; + bool ret = false; + + rcu_read_lock(); + + grp = task_related_thread_group(p); + if (grp) + ret = grp->skip_min; + + rcu_read_unlock(); + + return ret; +} + +#define CPU_RESERVED 1 +static inline int is_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + return test_bit(CPU_RESERVED, &wrq->walt_flags); +} + +static inline int mark_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + return test_and_set_bit(CPU_RESERVED, &wrq->walt_flags); +} + +static inline void clear_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + + clear_bit(CPU_RESERVED, &wrq->walt_flags); +} + +static inline void walt_irq_work_queue(struct irq_work *work) +{ + if (likely(cpu_online(raw_smp_processor_id()))) + irq_work_queue(work); + else + irq_work_queue_on(work, cpumask_any(cpu_online_mask)); +} + +static inline struct task_group *css_tg(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct task_group, css) : NULL; +} + +/* + * The policy of a RT boosted task (via PI mutex) still indicates it is + * a fair task, so use prio check as well. The prio check alone is not + * sufficient since idle task also has 120 priority. + */ +static inline bool walt_fair_task(struct task_struct *p) +{ + return p->prio >= MAX_RT_PRIO && !is_idle_task(p); +} + +extern int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); + +#define WALT_MVP_SLICE 3000000U +#define WALT_MVP_LIMIT (4 * WALT_MVP_SLICE) + +#define WALT_RTG_MVP 0 +#define WALT_BINDER_MVP 1 +#define WALT_TASK_BOOST_MVP 2 + +#define WALT_NOT_MVP -1 + +#define is_mvp(wts) (wts->mvp_prio != WALT_NOT_MVP) +void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p); +void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p); +void walt_cfs_tick(struct rq *rq); +void walt_lb_tick(struct rq *rq); + +extern __read_mostly unsigned int walt_scale_demand_divisor; + +static inline u64 scale_time_to_util(u64 d) +{ + /* + * The denominator at most could be (8 * tick_size) >> SCHED_CAPACITY_SHIFT, + * a value that easily fits a 32bit integer. + */ + do_div(d, walt_scale_demand_divisor); + return d; +} + +#define ASYMCAP_BOOST(cpu) (sysctl_sched_asymcap_boost && !is_min_cluster_cpu(cpu)) + +void create_util_to_cost(void); +struct compute_energy_output { + unsigned long sum_util[MAX_CLUSTERS]; + unsigned long max_util[MAX_CLUSTERS]; + unsigned long cost[MAX_CLUSTERS]; + unsigned int cluster_first_cpu[MAX_CLUSTERS]; +}; + +bool walt_halt_check_last(int cpu); +extern struct cpumask __cpu_halt_mask; +#define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask) +#define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask) + +extern void walt_task_dump(struct task_struct *p); +extern void walt_rq_dump(int cpu); +extern void walt_dump(void); +extern int in_sched_bug; + +extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, + struct task_struct *p, int dest_cpu); + +DECLARE_PER_CPU(u64, rt_task_arrival_time); +extern int walt_get_mvp_task_prio(struct task_struct *p); +extern void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p); + +enum WALT_DEBUG_FEAT { + WALT_BUG_UPSTREAM, + WALT_BUG_WALT, + WALT_BUG_NONCRITICAL, + WALT_BUG_UNUSED, + + /* maximum 4 entries allowed */ + WALT_DEBUG_FEAT_NR, +}; + +#define WALT_PANIC(condition) \ +({ \ + if (unlikely(!!(condition)) && !in_sched_bug) { \ + in_sched_bug = 1; \ + walt_dump(); \ + BUG_ON(condition); \ + } \ +}) + +/* the least signifcant byte is the bitmask for features and printk */ +#define WALT_PANIC_SENTINEL 0x4544DE00 + +#define walt_debug_bitmask_panic(x) (1UL << x) +#define walt_debug_bitmask_print(x) (1UL << (x + WALT_DEBUG_FEAT_NR)) + +/* setup initial values, bug and print on upstream and walt, ignore noncritical */ +#define walt_debug_initial_values() \ + (WALT_PANIC_SENTINEL | \ + walt_debug_bitmask_panic(WALT_BUG_UPSTREAM) | \ + walt_debug_bitmask_print(WALT_BUG_UPSTREAM) | \ + walt_debug_bitmask_panic(WALT_BUG_WALT) | \ + walt_debug_bitmask_print(WALT_BUG_WALT)) + +/* least significant nibble is the bug feature itself */ +#define walt_debug_feat_panic(x) (!!(sysctl_panic_on_walt_bug & (1UL << x))) + +/* 2nd least significant nibble is the print capability */ +#define walt_debug_feat_print(x) (!!(sysctl_panic_on_walt_bug & (1UL << (x + WALT_DEBUG_FEAT_NR)))) + +/* return true if the sentinel is set, regardless of feature set */ +static inline bool is_walt_sentinel(void) +{ + if (unlikely((sysctl_panic_on_walt_bug & 0xFFFFFF00) == WALT_PANIC_SENTINEL)) + return true; + return false; +} + +/* if the sentinel is properly set, print and/or panic as configured */ +#define WALT_BUG(feat, p, format, args...) \ +({ \ + if (is_walt_sentinel()) { \ + if (walt_debug_feat_print(feat)) \ + printk_deferred("WALT-BUG " format, args); \ + if (walt_debug_feat_panic(feat)) { \ + if (p) \ + walt_task_dump(p); \ + WALT_PANIC(1); \ + } \ + } \ +}) + +#endif /* _WALT_H */ diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c new file mode 100644 index 000000000000..da6489938e69 --- /dev/null +++ b/kernel/sched/walt/walt_cfs.c @@ -0,0 +1,1370 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include + +#include "walt.h" +#include "trace.h" +#include <../../../drivers/android/binder_internal.h> +#include "../../../drivers/android/binder_trace.h" + +static void create_util_to_cost_pd(struct em_perf_domain *pd) +{ + int util, cpu = cpumask_first(to_cpumask(pd->cpus)); + unsigned long fmax; + unsigned long scale_cpu; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_sched_cluster *cluster = wrq->cluster; + + fmax = (u64)pd->table[pd->nr_perf_states - 1].frequency; + scale_cpu = arch_scale_cpu_capacity(cpu); + + for (util = 0; util < 1024; util++) { + int j; + + int f = (fmax * util) / scale_cpu; + struct em_perf_state *ps = &pd->table[0]; + + for (j = 0; j < pd->nr_perf_states; j++) { + ps = &pd->table[j]; + if (ps->frequency >= f) + break; + } + cluster->util_to_cost[util] = ps->cost; + } +} + +void create_util_to_cost(void) +{ + struct perf_domain *pd; + struct root_domain *rd = cpu_rq(smp_processor_id())->rd; + + rcu_read_lock(); + pd = rcu_dereference(rd->pd); + for (; pd; pd = pd->next) + create_util_to_cost_pd(pd->em_pd); + rcu_read_unlock(); +} + +DECLARE_PER_CPU(unsigned long, gov_last_util); + +/* Migration margins */ +unsigned int sched_capacity_margin_up[WALT_NR_CPUS] = { + [0 ... WALT_NR_CPUS-1] = 1078 /* ~5% margin */ +}; +unsigned int sched_capacity_margin_down[WALT_NR_CPUS] = { + [0 ... WALT_NR_CPUS-1] = 1205 /* ~15% margin */ +}; + +static inline bool +bias_to_this_cpu(struct task_struct *p, int cpu, int start_cpu) +{ + bool base_test = cpumask_test_cpu(cpu, p->cpus_ptr) && + cpu_active(cpu); + + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; + bool start_cap_test = (wrq->cluster->id >= start_wrq->cluster->id); + + return base_test && start_cap_test; +} + +static inline bool task_demand_fits(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_orig_of(cpu); + + if (is_max_cluster_cpu(cpu)) + return true; + + if (!task_in_related_thread_group(p) && p->prio >= 124 && + !is_min_cluster_cpu(cpu) && !is_max_cluster_cpu(cpu)) { + /* a non topapp low prio task fits on gold */ + return true; + } + return task_fits_capacity(p, capacity, cpu); +} + +struct find_best_target_env { + bool is_rtg; + int need_idle; + int fastpath; + int start_cpu; + int order_index; + int end_index; + bool strict_max; + int skip_cpu; + u64 prs[8]; +}; + +/* + * cpu_util_without: compute cpu utilization without any contributions from *p + * @cpu: the CPU which utilization is requested + * @p: the task which utilization should be discounted + * + * The utilization of a CPU is defined by the utilization of tasks currently + * enqueued on that CPU as well as tasks which are currently sleeping after an + * execution on that CPU. + * + * This method returns the utilization of the specified CPU by discounting the + * utilization of the specified task, whenever the task is currently + * contributing to the CPU utilization. + */ +static unsigned long cpu_util_without(int cpu, struct task_struct *p) +{ + unsigned int util; + + /* + * WALT does not decay idle tasks in the same manner + * as PELT, so it makes little sense to subtract task + * utilization from cpu utilization. Instead just use + * cpu_util for this case. + */ + if (likely(READ_ONCE(p->__state) == TASK_WAKING)) + return cpu_util(cpu); + + /* Task has no contribution or is new */ + if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) + return cpu_util(cpu); + + util = max_t(long, cpu_util(cpu) - task_util(p), 0); + + /* + * Utilization (estimated) can exceed the CPU capacity, thus let's + * clamp to the maximum CPU capacity to ensure consistency with + * the cpu_util call. + */ + return min_t(unsigned long, util, capacity_orig_of(cpu)); +} + +static inline bool walt_task_skip_min_cpu(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (sched_boost_type != CONSERVATIVE_BOOST) && + walt_get_rtg_status(p) && (wts->unfilter || + walt_pipeline_low_latency_task(p)); +} + +static inline bool walt_is_many_wakeup(int sibling_count_hint) +{ + return sibling_count_hint >= sysctl_sched_many_wakeup_threshold; +} + +static inline bool walt_target_ok(int target_cpu, int order_index) +{ + return !((order_index != num_sched_clusters - 1) && + (cpumask_weight(&cpu_array[order_index][0]) == 1) && + (target_cpu == cpumask_first(&cpu_array[order_index][0]))); +} + +#define MIN_UTIL_FOR_ENERGY_EVAL 52 +static void walt_get_indicies(struct task_struct *p, int *order_index, + int *end_index, int per_task_boost, bool is_uclamp_boosted, + bool *energy_eval_needed) +{ + int i = 0; + + *order_index = 0; + *end_index = 0; + + if (num_sched_clusters <= 1) + return; + + if (per_task_boost > TASK_BOOST_ON_MID) { + *order_index = num_sched_clusters - 1; + *energy_eval_needed = false; + return; + } + + if (is_full_throttle_boost()) { + *energy_eval_needed = false; + *order_index = num_sched_clusters - 1; + if ((*order_index > 1) && task_demand_fits(p, + cpumask_first(&cpu_array[*order_index][1]))) + *end_index = 1; + return; + } + + if (is_uclamp_boosted || per_task_boost || + task_boost_policy(p) == SCHED_BOOST_ON_BIG || + walt_task_skip_min_cpu(p)) { + *energy_eval_needed = false; + *order_index = 1; + if (sysctl_sched_asymcap_boost) { + *end_index = 1; + return; + } + } + + for (i = *order_index ; i < num_sched_clusters - 1; i++) { + if (task_demand_fits(p, cpumask_first(&cpu_array[i][0]))) + break; + } + + *order_index = i; + + if (*order_index == 0 && + (task_util(p) >= MIN_UTIL_FOR_ENERGY_EVAL) && + !(p->in_iowait && task_in_related_thread_group(p)) && + !walt_get_rtg_status(p) && + !(sched_boost_type == CONSERVATIVE_BOOST && task_sched_boost(p)) && + !sysctl_sched_suppress_region2 + ) + *end_index = 1; + + if (p->in_iowait && task_in_related_thread_group(p)) + *energy_eval_needed = false; +} + +enum fastpaths { + NONE = 0, + SYNC_WAKEUP, + PREV_CPU_FASTPATH, + CLUSTER_PACKING_FASTPATH, +}; + +static inline bool is_complex_sibling_idle(int cpu) +{ + if (cpu_l2_sibling[cpu] != -1) + return available_idle_cpu(cpu_l2_sibling[cpu]); + return false; +} + +#define DIRE_STRAITS_PREV_NR_LIMIT 10 +static void walt_find_best_target(struct sched_domain *sd, + cpumask_t *candidates, + struct task_struct *p, + struct find_best_target_env *fbt_env) +{ + unsigned long min_task_util = uclamp_task_util(p); + long target_max_spare_cap = 0; + unsigned long best_idle_cuml_util = ULONG_MAX; + unsigned int min_exit_latency = UINT_MAX; + int i, start_cpu; + long spare_wake_cap, most_spare_wake_cap = 0; + int most_spare_cap_cpu = -1; + int least_nr_cpu = -1; + unsigned int cpu_rq_runnable_cnt = UINT_MAX; + int prev_cpu = task_cpu(p); + int order_index = fbt_env->order_index, end_index = fbt_env->end_index; + int stop_index = INT_MAX; + int cluster; + unsigned int target_nr_rtg_high_prio = UINT_MAX; + bool rtg_high_prio_task = task_rtg_high_prio(p); + cpumask_t visit_cpus; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + int packing_cpu; + struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1; + struct walt_rq *start_wrq; + + /* Find start CPU based on boost value */ + start_cpu = fbt_env->start_cpu; + start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; + + /* + * For higher capacity worth I/O tasks, stop the search + * at the end of higher capacity cluster(s). + */ + if (order_index > 0 && wts->iowaited) { + stop_index = num_sched_clusters - 2; + most_spare_wake_cap = LONG_MIN; + } + + if (fbt_env->strict_max) { + stop_index = 0; + most_spare_wake_cap = LONG_MIN; + } + + /* fast path for packing_cpu */ + packing_cpu = walt_find_cluster_packing_cpu(start_cpu); + if (walt_choose_packing_cpu(packing_cpu, p)) { + fbt_env->fastpath = CLUSTER_PACKING_FASTPATH; + cpumask_set_cpu(packing_cpu, candidates); + goto out; + } + + /* fast path for prev_cpu */ + if (((prev_wrq->cluster->id == start_wrq->cluster->id) || + asym_cap_siblings(prev_cpu, start_cpu)) && + cpu_active(prev_cpu) && + available_idle_cpu(prev_cpu) && + cpumask_test_cpu(prev_cpu, p->cpus_ptr) && + !cpu_halted(prev_cpu)) { + + fbt_env->fastpath = PREV_CPU_FASTPATH; + cpumask_set_cpu(prev_cpu, candidates); + goto out; + } + + for (cluster = 0; cluster < num_sched_clusters; cluster++) { + int best_idle_cpu_cluster = -1; + int target_cpu_cluster = -1; + int this_complex_idle = 0; + int best_complex_idle = 0; + + target_max_spare_cap = 0; + min_exit_latency = INT_MAX; + best_idle_cuml_util = ULONG_MAX; + + cpumask_and(&visit_cpus, p->cpus_ptr, + &cpu_array[order_index][cluster]); + for_each_cpu(i, &visit_cpus) { + unsigned long capacity_orig = capacity_orig_of(i); + unsigned long wake_cpu_util, new_cpu_util, new_util_cuml; + long spare_cap; + unsigned int idle_exit_latency = UINT_MAX; + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + + trace_sched_cpu_util(i, NULL); + /* record the prss as we visit cpus in a cluster */ + fbt_env->prs[i] = wrq->prev_runnable_sum + wrq->grp_time.prev_runnable_sum; + + if (!cpu_active(i)) + continue; + + if (cpu_halted(i)) + continue; + + /* + * This CPU is the target of an active migration that's + * yet to complete. Avoid placing another task on it. + */ + if (is_reserved(i)) + continue; + + if (sched_cpu_high_irqload(i)) + continue; + + if (fbt_env->skip_cpu == i) + continue; + + if (wrq->num_mvp_tasks > 0) + continue; + + /* + * p's blocked utilization is still accounted for on prev_cpu + * so prev_cpu will receive a negative bias due to the double + * accounting. However, the blocked utilization may be zero. + */ + wake_cpu_util = cpu_util_without(i, p); + spare_wake_cap = capacity_orig - wake_cpu_util; + + if (spare_wake_cap > most_spare_wake_cap) { + most_spare_wake_cap = spare_wake_cap; + most_spare_cap_cpu = i; + } + + /* + * Keep track of runnables for each CPU, if none of the + * CPUs have spare capacity then use CPU with less + * number of runnables. + */ + if (cpu_rq(i)->nr_running < cpu_rq_runnable_cnt) { + cpu_rq_runnable_cnt = cpu_rq(i)->nr_running; + least_nr_cpu = i; + } + + /* + * Ensure minimum capacity to grant the required boost. + * The target CPU can be already at a capacity level higher + * than the one required to boost the task. + */ + new_cpu_util = wake_cpu_util + min_task_util; + if (new_cpu_util > capacity_orig) + continue; + + /* + * Find an optimal backup IDLE CPU for non latency + * sensitive tasks. + * + * Looking for: + * - favoring shallowest idle states + * i.e. avoid to wakeup deep-idle CPUs + * + * The following code path is used by non latency + * sensitive tasks if IDLE CPUs are available. If at + * least one of such CPUs are available it sets the + * best_idle_cpu to the most suitable idle CPU to be + * selected. + * + * If idle CPUs are available, favour these CPUs to + * improve performances by spreading tasks. + * Indeed, the energy_diff() computed by the caller + * will take care to ensure the minimization of energy + * consumptions without affecting performance. + */ + if (available_idle_cpu(i)) { + idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(i)); + + this_complex_idle = is_complex_sibling_idle(i) ? 1 : 0; + + if (this_complex_idle < best_complex_idle) + continue; + /* + * Prefer shallowest over deeper idle state cpu, + * of same capacity cpus. + */ + if (idle_exit_latency > min_exit_latency) + continue; + + new_util_cuml = cpu_util_cum(i); + if (min_exit_latency == idle_exit_latency && + (best_idle_cpu_cluster == prev_cpu || + (i != prev_cpu && + new_util_cuml > best_idle_cuml_util))) + continue; + + min_exit_latency = idle_exit_latency; + best_idle_cuml_util = new_util_cuml; + best_idle_cpu_cluster = i; + best_complex_idle = this_complex_idle; + continue; + } + + /* skip visiting any more busy if idle was found */ + if (best_idle_cpu_cluster != -1) + continue; + + /* + * Compute the maximum possible capacity we expect + * to have available on this CPU once the task is + * enqueued here. + */ + spare_cap = capacity_orig - new_cpu_util; + + /* + * Try to spread the rtg high prio tasks so that they + * don't preempt each other. This is a optimisitc + * check assuming rtg high prio can actually preempt + * the current running task with the given vruntime + * boost. + */ + if (rtg_high_prio_task) { + if (walt_nr_rtg_high_prio(i) > target_nr_rtg_high_prio) + continue; + + /* Favor CPUs with maximum spare capacity */ + if (walt_nr_rtg_high_prio(i) == target_nr_rtg_high_prio && + spare_cap < target_max_spare_cap) + continue; + } else { + /* Favor CPUs with maximum spare capacity */ + if (spare_cap < target_max_spare_cap) + continue; + } + + target_max_spare_cap = spare_cap; + target_nr_rtg_high_prio = walt_nr_rtg_high_prio(i); + target_cpu_cluster = i; + } + + if (best_idle_cpu_cluster != -1) + cpumask_set_cpu(best_idle_cpu_cluster, candidates); + else if (target_cpu_cluster != -1) + cpumask_set_cpu(target_cpu_cluster, candidates); + + if ((cluster >= end_index) && (!cpumask_empty(candidates)) && + walt_target_ok(target_cpu_cluster, order_index)) + break; + + if (most_spare_cap_cpu != -1 && cluster >= stop_index) + break; + } + + /* + * We have set idle or target as long as they are valid CPUs. + * If we don't find either, then we fallback to most_spare_cap, + * If we don't find most spare cap, we fallback to prev_cpu, + * provided that the prev_cpu is active and has less than + * DIRE_STRAITS_PREV_NR_LIMIT runnables otherwise, we fallback to cpu + * with least number of runnables. + */ + + if (unlikely(cpumask_empty(candidates))) { + if (most_spare_cap_cpu != -1) + cpumask_set_cpu(most_spare_cap_cpu, candidates); + else if (cpu_active(prev_cpu) + && (cpu_rq(prev_cpu)->nr_running < DIRE_STRAITS_PREV_NR_LIMIT)) + cpumask_set_cpu(prev_cpu, candidates); + else if (least_nr_cpu != -1) + cpumask_set_cpu(least_nr_cpu, candidates); + } + +out: + trace_sched_find_best_target(p, min_task_util, start_cpu, cpumask_bits(candidates)[0], + most_spare_cap_cpu, order_index, end_index, + fbt_env->skip_cpu, task_on_rq_queued(p), least_nr_cpu, + cpu_rq_runnable_cnt); +} + +static inline unsigned long +cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + unsigned long util = wrq->walt_stats.cumulative_runnable_avg_scaled; + bool queued = task_on_rq_queued(p); + + /* + * When task is queued, + * (a) The evaluating CPU (cpu) is task's current CPU. If the + * task is migrating, discount the task contribution from the + * evaluation cpu. + * (b) The evaluating CPU (cpu) is task's current CPU. If the + * task is NOT migrating, nothing to do. The contribution is + * already present on the evaluation CPU. + * (c) The evaluating CPU (cpu) is not task's current CPU. But + * the task is migrating to the evaluating CPU. So add the + * task contribution to it. + * (d) The evaluating CPU (cpu) is neither the current CPU nor + * the destination CPU. don't care. + * + * When task is NOT queued i.e waking. Task contribution is not + * present on any CPU. + * + * (a) If the evaluating CPU is the destination CPU, add the task + * contribution. + * (b) The evaluation CPU is not the destination CPU, don't care. + */ + if (unlikely(queued)) { + if (task_cpu(p) == cpu) { + if (dst_cpu != cpu) + util = max_t(long, util - task_util(p), 0); + } else if (dst_cpu == cpu) { + util += task_util(p); + } + } else if (dst_cpu == cpu) { + util += task_util(p); + } + + return min_t(unsigned long, util, capacity_orig_of(cpu)); +} + +static inline u64 +cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_dst_same_cluster, + u64 *prs) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + long util = prs[cpu]; + + if (wts->prev_window) { + if (!prev_dst_same_cluster) { + /* intercluster migration of non rtg task - mimic fixups */ + util -= wts->prev_window_cpu[cpu]; + if (util < 0) + util = 0; + if (cpu == dst_cpu) + util += wts->prev_window; + } + } else { + if (cpu == dst_cpu) + util += wts->demand; + } + + return util; +} + +/** + * walt_em_cpu_energy() - Estimates the energy consumed by the CPUs of a + performance domain + * @pd : performance domain for which energy has to be estimated + * @max_util : highest utilization among CPUs of the domain + * @sum_util : sum of the utilization of all CPUs in the domain + * + * This function must be used only for CPU devices. There is no validation, + * i.e. if the EM is a CPU type and has cpumask allocated. It is called from + * the scheduler code quite frequently and that is why there is not checks. + * + * Return: the sum of the energy consumed by the CPUs of the domain assuming + * a capacity state satisfying the max utilization of the domain. + */ +static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, + unsigned long max_util, unsigned long sum_util, + struct compute_energy_output *output, unsigned int x) +{ + unsigned long scale_cpu; + int cpu; + struct walt_rq *wrq; + + if (!sum_util) + return 0; + + /* + * In order to predict the capacity state, map the utilization of the + * most utilized CPU of the performance domain to a requested frequency, + * like schedutil. + */ + cpu = cpumask_first(to_cpumask(pd->cpus)); + scale_cpu = arch_scale_cpu_capacity(cpu); + + max_util = max_util + (max_util >> 2); /* account for TARGET_LOAD usually 80 */ + max_util = max(max_util, + (arch_scale_freq_capacity(cpu) * scale_cpu) >> + SCHED_CAPACITY_SHIFT); + + /* + * The capacity of a CPU in the domain at the performance state (ps) + * can be computed as: + * + * ps->freq * scale_cpu + * ps->cap = -------------------- (1) + * cpu_max_freq + * + * So, ignoring the costs of idle states (which are not available in + * the EM), the energy consumed by this CPU at that performance state + * is estimated as: + * + * ps->power * cpu_util + * cpu_nrg = -------------------- (2) + * ps->cap + * + * since 'cpu_util / ps->cap' represents its percentage of busy time. + * + * NOTE: Although the result of this computation actually is in + * units of power, it can be manipulated as an energy value + * over a scheduling period, since it is assumed to be + * constant during that interval. + * + * By injecting (1) in (2), 'cpu_nrg' can be re-expressed as a product + * of two terms: + * + * ps->power * cpu_max_freq cpu_util + * cpu_nrg = ------------------------ * --------- (3) + * ps->freq scale_cpu + * + * The first term is static, and is stored in the em_perf_state struct + * as 'ps->cost'. + * + * Since all CPUs of the domain have the same micro-architecture, they + * share the same 'ps->cost', and the same CPU capacity. Hence, the + * total energy of the domain (which is the simple sum of the energy of + * all of its CPUs) can be factorized as: + * + * ps->cost * \Sum cpu_util + * pd_nrg = ------------------------ (4) + * scale_cpu + */ + if (max_util >= 1024) + max_util = 1023; + + wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + if (output) { + output->cost[x] = wrq->cluster->util_to_cost[max_util]; + output->max_util[x] = max_util; + output->sum_util[x] = sum_util; + } + return wrq->cluster->util_to_cost[max_util] * sum_util / scale_cpu; +} + +/* + * walt_pd_compute_energy(): Estimates the energy that @pd would consume if @p was + * migrated to @dst_cpu. compute_energy() predicts what will be the utilization + * landscape of @pd's CPUs after the task migration, and uses the Energy Model + * to compute what would be the energy if we decided to actually migrate that + * task. + */ +static long +walt_pd_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, u64 *prs, + struct compute_energy_output *output, unsigned int x) +{ + struct cpumask *pd_mask = perf_domain_span(pd); + unsigned long max_util = 0, sum_util = 0; + int cpu; + unsigned long cpu_util; + bool prev_dst_same_cluster = false; + + if (same_cluster(task_cpu(p), dst_cpu)) + prev_dst_same_cluster = true; + + /* + * The capacity state of CPUs of the current rd can be driven by CPUs + * of another rd if they belong to the same pd. So, account for the + * utilization of these CPUs too by masking pd with cpu_online_mask + * instead of the rd span. + * + * If an entire pd is outside of the current rd, it will not appear in + * its pd list and will not be accounted by compute_energy(). + */ + for_each_cpu_and(cpu, pd_mask, cpu_online_mask) { + sum_util += cpu_util_next_walt(cpu, p, dst_cpu); + cpu_util = cpu_util_next_walt_prs(cpu, p, dst_cpu, prev_dst_same_cluster, prs); + max_util = max(max_util, cpu_util); + } + + max_util = scale_time_to_util(max_util); + + if (output) + output->cluster_first_cpu[x] = cpumask_first(pd_mask); + + return walt_em_cpu_energy(pd->em_pd, max_util, sum_util, output, x); +} + +static inline long +walt_compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd, + cpumask_t *candidates, u64 *prs, struct compute_energy_output *output) +{ + long energy = 0; + unsigned int x = 0; + + for (; pd; pd = pd->next) { + struct cpumask *pd_mask = perf_domain_span(pd); + + if (cpumask_intersects(candidates, pd_mask) + || cpumask_test_cpu(task_cpu(p), pd_mask)) { + energy += walt_pd_compute_energy(p, dst_cpu, pd, prs, output, x); + x++; + } + } + + return energy; +} + +static inline int wake_to_idle(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_task_struct *cur_wts = + (struct walt_task_struct *) current->android_vendor_data1; + + return (cur_wts->wake_up_idle || wts->wake_up_idle); +} + +/* return true if cpu should be chosen over best_energy_cpu */ +static inline bool select_cpu_same_energy(int cpu, int best_cpu, int prev_cpu) +{ + bool new_cpu_is_idle = available_idle_cpu(cpu); + bool best_cpu_is_idle = available_idle_cpu(best_cpu); + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + struct walt_rq *best_wrq = (struct walt_rq *) cpu_rq(best_cpu)->android_vendor_data1; + + if (best_wrq->cluster->id < wrq->cluster->id) + return false; + if (wrq->cluster->id < best_wrq->cluster->id) + return true; + + if (best_cpu_is_idle && walt_get_idle_exit_latency(cpu_rq(best_cpu)) <= 1) + return false; + if (new_cpu_is_idle && walt_get_idle_exit_latency(cpu_rq(cpu)) <= 1) + return true; + + if (best_cpu_is_idle && !new_cpu_is_idle) + return false; + if (new_cpu_is_idle && !best_cpu_is_idle) + return true; + + if (best_cpu == prev_cpu) + return false; + if (cpu == prev_cpu) + return true; + + if (best_cpu_is_idle && new_cpu_is_idle) + return false; + + if (cpu_util(best_cpu) <= cpu_util(cpu)) + return false; + + return true; +} + +static inline unsigned int capacity_spare_of(int cpu) +{ + return capacity_orig_of(cpu) - cpu_util(cpu); +} + +static DEFINE_PER_CPU(cpumask_t, energy_cpus); +int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, + int sync, int sibling_count_hint) +{ + unsigned long prev_energy = ULONG_MAX, best_energy = ULONG_MAX; + struct root_domain *rd = cpu_rq(cpumask_first(cpu_active_mask))->rd; + int weight, cpu = smp_processor_id(), best_energy_cpu = prev_cpu; + struct perf_domain *pd; + unsigned long cur_energy; + cpumask_t *candidates; + bool is_rtg, curr_is_rtg; + struct find_best_target_env fbt_env; + bool need_idle = wake_to_idle(p); + u64 start_t = 0; + int delta = 0; + int task_boost = per_task_boost(p); + bool uclamp_boost = walt_uclamp_boosted(p); + int start_cpu, order_index, end_index; + int first_cpu; + bool energy_eval_needed = true; + struct compute_energy_output output; + struct walt_rq *prev_wrq = (struct walt_rq *) cpu_rq(prev_cpu)->android_vendor_data1; + struct walt_rq *start_wrq; + + if (walt_is_many_wakeup(sibling_count_hint) && prev_cpu != cpu && + cpumask_test_cpu(prev_cpu, p->cpus_ptr)) + return prev_cpu; + + if (unlikely(!cpu_array)) + return prev_cpu; + + walt_get_indicies(p, &order_index, &end_index, task_boost, uclamp_boost, + &energy_eval_needed); + start_cpu = cpumask_first(&cpu_array[order_index][0]); + start_wrq = (struct walt_rq *) cpu_rq(start_cpu)->android_vendor_data1; + + is_rtg = task_in_related_thread_group(p); + curr_is_rtg = task_in_related_thread_group(cpu_rq(cpu)->curr); + + if (trace_sched_task_util_enabled()) + start_t = sched_clock(); + + /* Pre-select a set of candidate CPUs. */ + candidates = this_cpu_ptr(&energy_cpus); + cpumask_clear(candidates); + + rcu_read_lock(); + need_idle |= uclamp_latency_sensitive(p); + + fbt_env.fastpath = 0; + fbt_env.need_idle = need_idle; + + if (sync && (need_idle || (is_rtg && curr_is_rtg))) + sync = 0; + + if (sysctl_sched_sync_hint_enable && sync + && bias_to_this_cpu(p, cpu, start_cpu) && !cpu_halted(cpu)) { + best_energy_cpu = cpu; + fbt_env.fastpath = SYNC_WAKEUP; + goto unlock; + } + + pd = rcu_dereference(rd->pd); + if (!pd) + goto fail; + + fbt_env.is_rtg = is_rtg; + fbt_env.start_cpu = start_cpu; + fbt_env.order_index = order_index; + fbt_env.end_index = end_index; + fbt_env.strict_max = is_rtg && + (task_boost == TASK_BOOST_STRICT_MAX); + fbt_env.skip_cpu = walt_is_many_wakeup(sibling_count_hint) ? + cpu : -1; + + walt_find_best_target(NULL, candidates, p, &fbt_env); + + /* Bail out if no candidate was found. */ + weight = cpumask_weight(candidates); + if (!weight) + goto unlock; + + first_cpu = cpumask_first(candidates); + + if (fbt_env.fastpath == CLUSTER_PACKING_FASTPATH) { + best_energy_cpu = first_cpu; + goto unlock; + } + + if (weight == 1) { + if (available_idle_cpu(first_cpu) || first_cpu == prev_cpu) { + best_energy_cpu = first_cpu; + goto unlock; + } + } + + if (need_idle && available_idle_cpu(first_cpu)) { + best_energy_cpu = first_cpu; + goto unlock; + } + + if (!energy_eval_needed) { + int max_spare_cpu = first_cpu; + + for_each_cpu(cpu, candidates) { + if (capacity_spare_of(max_spare_cpu) < capacity_spare_of(cpu)) + max_spare_cpu = cpu; + } + best_energy_cpu = max_spare_cpu; + goto unlock; + } + + if (READ_ONCE(p->__state) == TASK_WAKING) + delta = task_util(p); + + if (cpumask_test_cpu(prev_cpu, p->cpus_ptr) && !__cpu_overutilized(prev_cpu, delta)) { + if (trace_sched_compute_energy_enabled()) { + memset(&output, 0, sizeof(output)); + prev_energy = walt_compute_energy(p, prev_cpu, pd, candidates, fbt_env.prs, + &output); + } else { + prev_energy = walt_compute_energy(p, prev_cpu, pd, candidates, fbt_env.prs, + NULL); + } + + best_energy = prev_energy; + trace_sched_compute_energy(p, prev_cpu, prev_energy, 0, 0, 0, &output); + } else { + prev_energy = best_energy = ULONG_MAX; + } + + /* Select the best candidate energy-wise. */ + for_each_cpu(cpu, candidates) { + if (cpu == prev_cpu) + continue; + + if (trace_sched_compute_energy_enabled()) { + memset(&output, 0, sizeof(output)); + cur_energy = walt_compute_energy(p, cpu, pd, candidates, fbt_env.prs, + &output); + } else { + cur_energy = walt_compute_energy(p, cpu, pd, candidates, fbt_env.prs, + NULL); + } + + if (cur_energy < best_energy) { + best_energy = cur_energy; + best_energy_cpu = cpu; + } else if (cur_energy == best_energy) { + if (select_cpu_same_energy(cpu, best_energy_cpu, + prev_cpu)) { + best_energy = cur_energy; + best_energy_cpu = cpu; + } + } + + trace_sched_compute_energy(p, cpu, cur_energy, + prev_energy, best_energy, best_energy_cpu, &output); + } + + /* + * Pick the prev CPU, if best energy CPU can't saves at least 6% of + * the energy used by prev_cpu. + */ + if (!(available_idle_cpu(best_energy_cpu) && + walt_get_idle_exit_latency(cpu_rq(best_energy_cpu)) <= 1) && + (prev_energy != ULONG_MAX) && (best_energy_cpu != prev_cpu) && + ((prev_energy - best_energy) <= prev_energy >> 5) && + (prev_wrq->cluster->id <= start_wrq->cluster->id)) + best_energy_cpu = prev_cpu; + +unlock: + rcu_read_unlock(); + + if (best_energy_cpu < 0 || best_energy_cpu >= WALT_NR_CPUS) + best_energy_cpu = prev_cpu; + + trace_sched_task_util(p, cpumask_bits(candidates)[0], best_energy_cpu, + sync, fbt_env.need_idle, fbt_env.fastpath, + start_t, uclamp_boost, start_cpu); + + return best_energy_cpu; + +fail: + rcu_read_unlock(); + return prev_cpu; +} + +static void +walt_select_task_rq_fair(void *unused, struct task_struct *p, int prev_cpu, + int sd_flag, int wake_flags, int *target_cpu) +{ + int sync; + int sibling_count_hint; + + if (unlikely(walt_disabled)) + return; + + sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); + sibling_count_hint = p->wake_q_count; + p->wake_q_count = 0; + + *target_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, sync, sibling_count_hint); +} + +static void walt_binder_low_latency_set(void *unused, struct task_struct *task, + bool sync, struct binder_proc *proc) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + if (task && current->signal && + (current->signal->oom_score_adj == 0) && + ((current->prio < DEFAULT_PRIO) || + (task->group_leader->prio < MAX_RT_PRIO))) + wts->low_latency |= WALT_LOW_LATENCY_BINDER; +} + +static void walt_binder_low_latency_clear(void *unused, struct binder_transaction *t) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + if (wts->low_latency & WALT_LOW_LATENCY_BINDER) + wts->low_latency &= ~WALT_LOW_LATENCY_BINDER; +} + +static void binder_set_priority_hook(void *data, + struct binder_transaction *bndrtrans, struct task_struct *task) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; + struct walt_task_struct *current_wts = + (struct walt_task_struct *) current->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + if (bndrtrans && bndrtrans->need_reply && current_wts->boost == TASK_BOOST_STRICT_MAX) { + bndrtrans->android_vendor_data1 = wts->boost; + wts->boost = TASK_BOOST_STRICT_MAX; + } +} + +static void binder_restore_priority_hook(void *data, + struct binder_transaction *bndrtrans, struct task_struct *task) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) task->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + if (bndrtrans && wts->boost == TASK_BOOST_STRICT_MAX) + wts->boost = bndrtrans->android_vendor_data1; +} + +/* + * Higher prio mvp can preempt lower prio mvp. + * + * However, the lower prio MVP slice will be more since we expect them to + * be the work horses. For example, binders will have higher prio MVP and + * they can preempt long running rtg prio tasks but binders loose their + * powers with in 3 msec where as rtg prio tasks can run more than that. + */ +int walt_get_mvp_task_prio(struct task_struct *p) +{ + if (per_task_boost(p) == TASK_BOOST_STRICT_MAX) + return WALT_TASK_BOOST_MVP; + + if (walt_binder_low_latency_task(p)) + return WALT_BINDER_MVP; + + if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p) || + walt_pipeline_low_latency_task(p)) + return WALT_RTG_MVP; + + return WALT_NOT_MVP; +} + +static inline unsigned int walt_cfs_mvp_task_limit(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + /* Binder MVP tasks are high prio but have only single slice */ + if (wts->mvp_prio == WALT_BINDER_MVP) + return WALT_MVP_SLICE; + + return WALT_MVP_LIMIT; +} + +static void walt_cfs_insert_mvp_task(struct walt_rq *wrq, struct walt_task_struct *wts, + bool at_front) +{ + struct list_head *pos; + + list_for_each(pos, &wrq->mvp_tasks) { + struct walt_task_struct *tmp_wts = container_of(pos, struct walt_task_struct, + mvp_list); + + if (at_front) { + if (wts->mvp_prio >= tmp_wts->mvp_prio) + break; + } else { + if (wts->mvp_prio > tmp_wts->mvp_prio) + break; + } + } + + list_add(&wts->mvp_list, pos->prev); + wrq->num_mvp_tasks++; +} + +void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + list_del_init(&wts->mvp_list); + wts->mvp_prio = WALT_NOT_MVP; + wrq->num_mvp_tasks--; +} + +/* + * MVP task runtime update happens here. Three possibilities: + * + * de-activated: The MVP consumed its runtime. Non MVP can preempt. + * slice expired: MVP slice is expired and other MVP can preempt. + * slice not expired: This MVP task can continue to run. + */ +static void walt_cfs_account_mvp_runtime(struct rq *rq, struct task_struct *curr) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) curr->android_vendor_data1; + u64 slice; + unsigned int limit; + + lockdep_assert_held(&rq->__lock); + + /* + * RQ clock update happens in tick path in the scheduler. + * Since we drop the lock in the scheduler before calling + * into vendor hook, it is possible that update flags are + * reset by another rq lock and unlock. Do the update here + * if required. + */ + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + + if (curr->se.sum_exec_runtime > wts->sum_exec_snapshot_for_total) + wts->total_exec = curr->se.sum_exec_runtime - wts->sum_exec_snapshot_for_total; + else + wts->total_exec = 0; + + if (curr->se.sum_exec_runtime > wts->sum_exec_snapshot_for_slice) + slice = curr->se.sum_exec_runtime - wts->sum_exec_snapshot_for_slice; + else + slice = 0; + + /* slice is not expired */ + if (slice < WALT_MVP_SLICE) + return; + + wts->sum_exec_snapshot_for_slice = curr->se.sum_exec_runtime; + /* + * slice is expired, check if we have to deactivate the + * MVP task, otherwise requeue the task in the list so + * that other MVP tasks gets a chance. + */ + + limit = walt_cfs_mvp_task_limit(curr); + if (wts->total_exec > limit) { + walt_cfs_deactivate_mvp_task(rq, curr); + trace_walt_cfs_deactivate_mvp_task(curr, wts, limit); + return; + } + + if (wrq->num_mvp_tasks == 1) + return; + + /* slice expired. re-queue the task */ + list_del(&wts->mvp_list); + wrq->num_mvp_tasks--; + walt_cfs_insert_mvp_task(wrq, wts, false); +} + +void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + int mvp_prio = walt_get_mvp_task_prio(p); + + if (mvp_prio == WALT_NOT_MVP) + return; + + /* + * This can happen during migration or enq/deq for prio/class change. + * it was once MVP but got demoted, it will not be MVP until + * it goes to sleep again. + */ + if (wts->total_exec > walt_cfs_mvp_task_limit(p)) + return; + + wts->mvp_prio = mvp_prio; + walt_cfs_insert_mvp_task(wrq, wts, task_running(rq, p)); + + /* + * We inserted the task at the appropriate position. Take the + * task runtime snapshot. From now onwards we use this point as a + * baseline to enforce the slice and demotion. + */ + if (!wts->total_exec) /* queue after sleep */ { + wts->sum_exec_snapshot_for_total = p->se.sum_exec_runtime; + wts->sum_exec_snapshot_for_slice = p->se.sum_exec_runtime; + } +} + +void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + if (!list_empty(&wts->mvp_list) && wts->mvp_list.next) + walt_cfs_deactivate_mvp_task(rq, p); + + /* + * Reset the exec time during sleep so that it starts + * from scratch upon next wakeup. total_exec should + * be preserved when task is enq/deq while it is on + * runqueue. + */ + if (READ_ONCE(p->__state) != TASK_RUNNING) + wts->total_exec = 0; +} + +void walt_cfs_tick(struct rq *rq) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + + raw_spin_lock(&rq->__lock); + + if (list_empty(&wts->mvp_list) || (wts->mvp_list.next == NULL)) + goto out; + + walt_cfs_account_mvp_runtime(rq, rq->curr); + /* + * If the current is not MVP means, we have to re-schedule to + * see if we can run any other task including MVP tasks. + */ + if ((wrq->mvp_tasks.next != &wts->mvp_list) && rq->cfs.h_nr_running > 1) + resched_curr(rq); + +out: + raw_spin_unlock(&rq->__lock); +} + +/* + * When preempt = false and nopreempt = false, we leave the preemption + * decision to CFS. + */ +static void walt_cfs_check_preempt_wakeup(void *unused, struct rq *rq, struct task_struct *p, + bool *preempt, bool *nopreempt, int wake_flags, + struct sched_entity *se, struct sched_entity *pse, + int next_buddy_marked, unsigned int granularity) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts_p = (struct walt_task_struct *) p->android_vendor_data1; + struct task_struct *c = rq->curr; + struct walt_task_struct *wts_c = (struct walt_task_struct *) rq->curr->android_vendor_data1; + bool resched = false; + bool p_is_mvp, curr_is_mvp; + + if (unlikely(walt_disabled)) + return; + + p_is_mvp = !list_empty(&wts_p->mvp_list) && wts_p->mvp_list.next; + curr_is_mvp = !list_empty(&wts_c->mvp_list) && wts_c->mvp_list.next; + + /* + * current is not MVP, so preemption decision + * is simple. + */ + if (!curr_is_mvp) { + if (p_is_mvp) + goto preempt; + return; /* CFS decides preemption */ + } + + /* + * current is MVP. update its runtime before deciding the + * preemption. + */ + walt_cfs_account_mvp_runtime(rq, c); + resched = (wrq->mvp_tasks.next != &wts_c->mvp_list); + + /* + * current is no longer eligible to run. It must have been + * picked (because of MVP) ahead of other tasks in the CFS + * tree, so drive preemption to pick up the next task from + * the tree, which also includes picking up the first in + * the MVP queue. + */ + if (resched) + goto preempt; + + /* current is the first in the queue, so no preemption */ + *nopreempt = true; + trace_walt_cfs_mvp_wakeup_nopreempt(c, wts_c, walt_cfs_mvp_task_limit(c)); + return; +preempt: + *preempt = true; + trace_walt_cfs_mvp_wakeup_preempt(p, wts_p, walt_cfs_mvp_task_limit(p)); +} + +#ifdef CONFIG_FAIR_GROUP_SCHED +/* Walk up scheduling entities hierarchy */ +#define for_each_sched_entity(se) \ + for (; se; se = se->parent) +#else /* !CONFIG_FAIR_GROUP_SCHED */ +#define for_each_sched_entity(se) \ + for (; se; se = NULL) +#endif + +extern void set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se); +static void walt_cfs_replace_next_task_fair(void *unused, struct rq *rq, struct task_struct **p, + struct sched_entity **se, bool *repick, bool simple, + struct task_struct *prev) +{ + struct walt_rq *wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_task_struct *wts; + struct task_struct *mvp; + struct cfs_rq *cfs_rq; + + if (unlikely(walt_disabled)) + return; + + if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || + (*p)->on_rq == TASK_ON_RQ_MIGRATING || + (*p)->cpu != cpu_of(rq))) + WALT_BUG(WALT_BUG_UPSTREAM, *p, + "picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", + (*p)->comm, (*p)->pid, (*p)->on_cpu, + (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); + + /* We don't have MVP tasks queued */ + if (list_empty(&wrq->mvp_tasks)) + return; + + /* Return the first task from MVP queue */ + wts = list_first_entry(&wrq->mvp_tasks, struct walt_task_struct, mvp_list); + mvp = wts_to_ts(wts); + + *p = mvp; + *se = &mvp->se; + *repick = true; + + if (simple) { + for_each_sched_entity((*se)) { + /* + * TODO If CFS_BANDWIDTH is enabled, we might pick + * from a throttled cfs_rq + */ + cfs_rq = cfs_rq_of(*se); + set_next_entity(cfs_rq, *se); + } + } + + if ((*p) && (*p) != prev && ((*p)->on_cpu == 1 || (*p)->on_rq == 0 || + (*p)->on_rq == TASK_ON_RQ_MIGRATING || + (*p)->cpu != cpu_of(rq))) + WALT_BUG(WALT_BUG_UPSTREAM, *p, + "picked %s(%d) on_cpu=%d on_rq=%d p->cpu=%d cpu_of(rq)=%d kthread=%d\n", + (*p)->comm, (*p)->pid, (*p)->on_cpu, + (*p)->on_rq, (*p)->cpu, cpu_of(rq), ((*p)->flags & PF_KTHREAD)); + + trace_walt_cfs_mvp_pick_next(mvp, wts, walt_cfs_mvp_task_limit(mvp)); +} + +void walt_cfs_init(void) +{ + register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL); + + register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL); + register_trace_binder_transaction_received(walt_binder_low_latency_clear, NULL); + + register_trace_android_vh_binder_set_priority(binder_set_priority_hook, NULL); + register_trace_android_vh_binder_restore_priority(binder_restore_priority_hook, NULL); + + register_trace_android_rvh_check_preempt_wakeup(walt_cfs_check_preempt_wakeup, NULL); + register_trace_android_rvh_replace_next_task_fair(walt_cfs_replace_next_task_fair, NULL); +} diff --git a/kernel/sched/walt/walt_debug.c b/kernel/sched/walt/walt_debug.c new file mode 100644 index 000000000000..26458c7ee307 --- /dev/null +++ b/kernel/sched/walt/walt_debug.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include + +#include + +#include "walt.h" +#include "walt_debug.h" + +static void android_rvh_schedule_bug(void *unused, void *unused2) +{ + BUG(); +} + +static int __init walt_debug_init(void) +{ + int ret; + + ret = preemptirq_long_init(); + if (ret) + return ret; + + register_trace_android_rvh_schedule_bug(android_rvh_schedule_bug, NULL); + + return 0; +} +module_init(walt_debug_init); + +MODULE_DESCRIPTION("QTI WALT Debug Module"); +MODULE_LICENSE("GPL v2"); diff --git a/kernel/sched/walt/walt_debug.h b/kernel/sched/walt/walt_debug.h new file mode 100644 index 000000000000..282836c14743 --- /dev/null +++ b/kernel/sched/walt/walt_debug.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ +int preemptirq_long_init(void); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c new file mode 100644 index 000000000000..f61528ef7cb2 --- /dev/null +++ b/kernel/sched/walt/walt_halt.c @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021-2021 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ +#include +#include +#include +#include +#include +#include "trace.h" + +#ifdef CONFIG_HOTPLUG_CPU + +/* if a cpu is halting */ +struct cpumask __cpu_halt_mask; + +/* spin lock to allow calling from non-preemptible context */ +static DEFINE_RAW_SPINLOCK(halt_lock); + +struct halt_cpu_state { + u64 last_halt; + u8 reason; +}; + +static DEFINE_PER_CPU(struct halt_cpu_state, halt_state); +static DEFINE_RAW_SPINLOCK(walt_drain_pending_lock); + +/* the amount of time allowed for enqueue operations that happen + * just after a halt operation. + */ +#define WALT_HALT_CHECK_THRESHOLD_NS 400000 + +/* + * Remove a task from the runqueue and pretend that it's migrating. This + * should prevent migrations for the detached task and disallow further + * changes to tsk_cpus_allowed. + */ +void +detach_one_task_core(struct task_struct *p, struct rq *rq, + struct list_head *tasks) +{ + lockdep_assert_held(&rq->__lock); + + p->on_rq = TASK_ON_RQ_MIGRATING; + deactivate_task(rq, p, 0); + list_add(&p->se.group_node, tasks); +} + +void attach_tasks_core(struct list_head *tasks, struct rq *rq) +{ + struct task_struct *p; + + lockdep_assert_held(&rq->__lock); + + while (!list_empty(tasks)) { + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); + + BUG_ON(task_rq(p) != rq); + activate_task(rq, p, 0); + p->on_rq = TASK_ON_RQ_QUEUED; + } +} + +/* + * Migrate all tasks from the rq, sleeping tasks will be migrated by + * try_to_wake_up()->select_task_rq(). + * + * Called with rq->__lock held even though we'er in stop_machine() and + * there's no concurrency possible, we hold the required locks anyway + * because of lock validation efforts. + * + * The function will skip CPU pinned kthreads. + */ +static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) +{ + struct rq *rq = dead_rq; + struct task_struct *next, *stop = rq->stop; + LIST_HEAD(percpu_kthreads); + unsigned int num_pinned_kthreads = 1; + struct rq_flags orf = *rf; + int dest_cpu; + + /* + * Fudge the rq selection such that the below task selection loop + * doesn't get stuck on the currently eligible stop task. + * + * We're currently inside stop_machine() and the rq is either stuck + * in the stop_machine_cpu_stop() loop, or we're executing this code, + * either way we should never end up calling schedule() until we're + * done here. + */ + rq->stop = NULL; + + /* + * put_prev_task() and pick_next_task() sched + * class method both need to have an up-to-date + * value of rq->clock[_task] + */ + update_rq_clock(rq); + +#ifdef CONFIG_SCHED_DEBUG + /* note the clock update in orf */ + orf.clock_update_flags |= RQCF_UPDATED; +#endif + + for (;;) { + /* + * There's this thread running, bail when that's the only + * remaining thread: + */ + if (rq->nr_running == 1) + break; + + next = pick_migrate_task(rq); + + /* + * Argh ... no iterator for tasks, we need to remove the + * kthread from the run-queue to continue. + */ + + if (is_per_cpu_kthread(next)) { + detach_one_task_core(next, rq, &percpu_kthreads); + num_pinned_kthreads += 1; + continue; + } + + /* + * Rules for changing task_struct::cpus_mask are holding + * both pi_lock and rq->__lock, such that holding either + * stabilizes the mask. + * + * Drop rq->__lock is not quite as disastrous as it usually is + * because !cpu_active at this point, which means load-balance + * will not interfere. Also, stop-machine. + */ + rq_unlock(rq, rf); + raw_spin_lock(&next->pi_lock); + rq_relock(rq, rf); + + /* + * Since we're inside stop-machine, _nothing_ should have + * changed the task, WARN if weird stuff happened, because in + * that case the above rq->__lock drop is a fail too. + */ + if (task_rq(next) != rq || !task_on_rq_queued(next)) { + raw_spin_unlock(&next->pi_lock); + continue; + } + + /* Find suitable destination for @next */ + dest_cpu = select_fallback_rq(dead_rq->cpu, next); + + if (cpu_of(rq) != dest_cpu && !is_migration_disabled(next)) { + /* only perform a required migration */ + rq = __migrate_task(rq, rf, next, dest_cpu); + + if (rq != dead_rq) { + rq_unlock(rq, rf); + rq = dead_rq; + *rf = orf; + rq_relock(rq, rf); + } + } else { + detach_one_task_core(next, rq, &percpu_kthreads); + num_pinned_kthreads += 1; + } + + raw_spin_unlock(&next->pi_lock); + } + + if (num_pinned_kthreads > 1) + attach_tasks_core(&percpu_kthreads, rq); + + rq->stop = stop; +} + +static int drain_rq_cpu_stop(void *data) +{ + struct rq *rq = this_rq(); + struct rq_flags rf; + + rq_lock_irqsave(rq, &rf); + migrate_tasks(rq, &rf); + rq_unlock_irqrestore(rq, &rf); + + return 0; +} + +static int cpu_drain_rq(unsigned int cpu) +{ + if (!cpu_online(cpu)) + return 0; + + if (available_idle_cpu(cpu)) + return 0; + + /* this will schedule, must not be in atomic context */ + return stop_one_cpu(cpu, drain_rq_cpu_stop, NULL); +} + +/* + * returns true if last halt is within threshold + * note: do not take halt_lock, called from atomic context + */ +bool walt_halt_check_last(int cpu) +{ + u64 last_halt = per_cpu_ptr(&halt_state, cpu)->last_halt; + + /* last_halt is valid, check it against sched_clock */ + if (last_halt != 0 && sched_clock() - last_halt > WALT_HALT_CHECK_THRESHOLD_NS) + return false; + + return true; +} + +struct drain_thread_data { + cpumask_t cpus_to_drain; +}; + +static struct drain_thread_data drain_data = { + .cpus_to_drain = { CPU_BITS_NONE } +}; + +static int __ref try_drain_rqs(void *data) +{ + cpumask_t *cpus_ptr = &((struct drain_thread_data *)data)->cpus_to_drain; + int cpu; + unsigned long flags; + + while (!kthread_should_stop()) { + raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); + if (cpumask_weight(cpus_ptr)) { + cpumask_t local_cpus; + + cpumask_copy(&local_cpus, cpus_ptr); + raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); + + for_each_cpu(cpu, &local_cpus) + cpu_drain_rq(cpu); + + raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); + cpumask_andnot(cpus_ptr, cpus_ptr, &local_cpus); + + } + raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + set_current_state(TASK_RUNNING); + } + + return 0; +} + +struct task_struct *walt_drain_thread; + +/* + * 1) add the cpus to the halt mask + * 2) migrate tasks off the cpu + */ +static int halt_cpus(struct cpumask *cpus) +{ + int cpu; + int ret = 0; + u64 start_time = sched_clock(); + struct halt_cpu_state *halt_cpu_state; + unsigned long flags; + + trace_halt_cpus_start(cpus, 1); + + for_each_cpu(cpu, cpus) { + + if (cpu == cpumask_first(system_32bit_el0_cpumask())) { + ret = -EINVAL; + goto out; + } + + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); + + /* set the cpu as halted */ + cpumask_set_cpu(cpu, cpu_halt_mask); + + /* guarantee mask written before updating last_halt */ + wmb(); + + halt_cpu_state->last_halt = start_time; + } + + /* signal and wakeup the drain kthread */ + raw_spin_lock_irqsave(&walt_drain_pending_lock, flags); + cpumask_or(&drain_data.cpus_to_drain, &drain_data.cpus_to_drain, cpus); + raw_spin_unlock_irqrestore(&walt_drain_pending_lock, flags); + + if (!IS_ERR(walt_drain_thread)) + wake_up_process(walt_drain_thread); + +out: + trace_halt_cpus(cpus, start_time, 1, ret); + + return ret; +} + +/* + * 1) remove the cpus from the halt mask + * + */ +static int start_cpus(struct cpumask *cpus) +{ + u64 start_time = sched_clock(); + struct halt_cpu_state *halt_cpu_state; + int cpu; + + trace_halt_cpus_start(cpus, 0); + + for_each_cpu(cpu, cpus) { + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); + halt_cpu_state->last_halt = 0; + wmb(); + + /* wmb to guarantee zero'd last_halt before clearing from the mask */ + + cpumask_clear_cpu(cpu, cpu_halt_mask); + + /* kick the cpu so it can pull tasks + * after the mask has been cleared. + */ + walt_smp_call_newidle_balance(cpu); + } + + trace_halt_cpus(cpus, start_time, 0, 0); + + return 0; +} + +/* update reason for cpus in yield/halt mask */ +static void update_reasons(struct cpumask *cpus, bool halt, enum pause_reason reason) +{ + int cpu; + struct halt_cpu_state *halt_cpu_state; + + for_each_cpu(cpu, cpus) { + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); + if (halt) + halt_cpu_state->reason |= reason; + else + halt_cpu_state->reason &= ~reason; + } +} + +/* remove cpus that are already halted */ +static void update_halt_cpus(struct cpumask *cpus) +{ + int cpu; + struct halt_cpu_state *halt_cpu_state; + + for_each_cpu(cpu, cpus) { + halt_cpu_state = per_cpu_ptr(&halt_state, cpu); + if (halt_cpu_state->reason) + cpumask_clear_cpu(cpu, cpus); + } +} + +/* cpus will be modified */ +int walt_halt_cpus(struct cpumask *cpus, enum pause_reason reason) +{ + int ret = 0; + cpumask_t requested_cpus; + unsigned long flags; + + raw_spin_lock_irqsave(&halt_lock, flags); + + cpumask_copy(&requested_cpus, cpus); + + /* remove cpus that are already halted */ + update_halt_cpus(cpus); + + if (cpumask_empty(cpus)) { + update_reasons(&requested_cpus, true, reason); + goto unlock; + } + + ret = halt_cpus(cpus); + + if (ret < 0) + pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret, + cpumask_pr_args(&requested_cpus)); + else + update_reasons(&requested_cpus, true, reason); +unlock: + raw_spin_unlock_irqrestore(&halt_lock, flags); + + return ret; +} + +int walt_pause_cpus(struct cpumask *cpus, enum pause_reason reason) +{ + return walt_halt_cpus(cpus, reason); +} +EXPORT_SYMBOL(walt_pause_cpus); + +/* cpus will be modified */ +int walt_start_cpus(struct cpumask *cpus, enum pause_reason reason) +{ + int ret = 0; + cpumask_t requested_cpus; + unsigned long flags; + + raw_spin_lock_irqsave(&halt_lock, flags); + cpumask_copy(&requested_cpus, cpus); + update_reasons(&requested_cpus, false, reason); + + /* remove cpus that should still be halted */ + update_halt_cpus(cpus); + + ret = start_cpus(cpus); + + if (ret < 0) { + pr_debug("halt_cpus failure ret=%d cpus=%*pbl\n", ret, + cpumask_pr_args(&requested_cpus)); + /* restore/increment ref counts in case of error */ + update_reasons(&requested_cpus, true, reason); + } + + raw_spin_unlock_irqrestore(&halt_lock, flags); + + return ret; +} + +int walt_resume_cpus(struct cpumask *cpus, enum pause_reason reason) +{ + return walt_start_cpus(cpus, reason); +} +EXPORT_SYMBOL(walt_resume_cpus); + +static void android_rvh_get_nohz_timer_target(void *unused, int *cpu, bool *done) +{ + int i, default_cpu = -1; + struct sched_domain *sd; + cpumask_t unhalted; + + *done = true; + + if (housekeeping_cpu(*cpu, HK_FLAG_TIMER) && !cpu_halted(*cpu)) { + if (!available_idle_cpu(*cpu)) + return; + default_cpu = *cpu; + } + + rcu_read_lock(); + for_each_domain(*cpu, sd) { + for_each_cpu_and(i, sched_domain_span(sd), + housekeeping_cpumask(HK_FLAG_TIMER)) { + if (*cpu == i) + continue; + + if (!available_idle_cpu(i) && !cpu_halted(i)) { + *cpu = i; + goto unlock; + } + } + } + + if (default_cpu == -1) { + cpumask_complement(&unhalted, cpu_halt_mask); + for_each_cpu_and(i, &unhalted, + housekeeping_cpumask(HK_FLAG_TIMER)) { + if (*cpu == i) + continue; + + if (!available_idle_cpu(i)) { + *cpu = i; + goto unlock; + } + } + + /* no active, non-halted, not-idle, choose any */ + default_cpu = cpumask_any(&unhalted); + + if (unlikely(default_cpu >= nr_cpu_ids)) + goto unlock; + } + + *cpu = default_cpu; +unlock: + rcu_read_unlock(); +} + +static void android_rvh_set_cpus_allowed_ptr_locked(void *unused, + const struct cpumask *cpu_valid_mask, + const struct cpumask *new_mask, + unsigned int *dest_cpu) +{ + cpumask_t allowed_cpus; + + if (unlikely(walt_disabled)) + return; + + if (cpu_halted(*dest_cpu)) { + /* remove halted cpus from the valid mask, and store locally */ + cpumask_andnot(&allowed_cpus, cpu_valid_mask, cpu_halt_mask); + *dest_cpu = cpumask_any_and_distribute(&allowed_cpus, new_mask); + } +} + +static void android_rvh_rto_next_cpu(void *unused, int rto_cpu, struct cpumask *rto_mask, int *cpu) +{ + cpumask_t allowed_cpus; + + if (unlikely(walt_disabled)) + return; + + if (cpu_halted(*cpu)) { + /* remove halted cpus from the valid mask, and store locally */ + cpumask_andnot(&allowed_cpus, rto_mask, cpu_halt_mask); + *cpu = cpumask_next(rto_cpu, &allowed_cpus); + } +} + +/** + * android_rvh_is_cpu_allowed: disallow cpus that are halted + * + * Caveat: For 32 bit tasks that are being directed to a halted cpu, allow the halted cpu + * in a particular case (32 bit task, in execve, moving to a 32 bit cpu) + * This is to handle the call to is_cpu_allowed() from __migrate_task, in the + * event that a 32bit task is being execve'd. + */ +static void android_rvh_is_cpu_allowed(void *unused, struct task_struct *p, int cpu, bool *allowed) +{ + if (unlikely(walt_disabled)) + return; + + if (cpumask_test_cpu(cpu, cpu_halt_mask)) { + + /* default reject for any halted cpu */ + *allowed = false; + + if (is_compat_thread(task_thread_info(p)) && + p->in_execve) { + /* + * the task is 32 bit capable and + * the context is execve. allow this cpu. + */ + *allowed = true; + } + } +} + +void walt_halt_init(void) +{ + struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; + + walt_drain_thread = kthread_run(try_drain_rqs, &drain_data, "halt_drain_rqs"); + if (IS_ERR(walt_drain_thread)) { + pr_err("Error creating walt drain thread\n"); + return; + } + + sched_setscheduler_nocheck(walt_drain_thread, SCHED_FIFO, ¶m); + + register_trace_android_rvh_get_nohz_timer_target(android_rvh_get_nohz_timer_target, NULL); + register_trace_android_rvh_set_cpus_allowed_ptr_locked( + android_rvh_set_cpus_allowed_ptr_locked, NULL); + register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL); + register_trace_android_rvh_is_cpu_allowed(android_rvh_is_cpu_allowed, NULL); + +} + +#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c new file mode 100644 index 000000000000..a3020210bbc3 --- /dev/null +++ b/kernel/sched/walt/walt_lb.c @@ -0,0 +1,1080 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include + +#include "walt.h" +#include "trace.h" + +static inline unsigned long walt_lb_cpu_util(int cpu) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + return wrq->walt_stats.cumulative_runnable_avg_scaled; +} + +static void walt_detach_task(struct task_struct *p, struct rq *src_rq, + struct rq *dst_rq) +{ + //TODO can we just replace with detach_task in fair.c?? + deactivate_task(src_rq, p, 0); + set_task_cpu(p, dst_rq->cpu); +} + +static void walt_attach_task(struct task_struct *p, struct rq *rq) +{ + activate_task(rq, p, 0); + check_preempt_curr(rq, p, 0); +} + +static int stop_walt_lb_active_migration(void *data) +{ + struct rq *busiest_rq = data; + int busiest_cpu = cpu_of(busiest_rq); + int target_cpu = busiest_rq->push_cpu; + struct rq *target_rq = cpu_rq(target_cpu); + struct walt_rq *wrq = (struct walt_rq *) busiest_rq->android_vendor_data1; + struct task_struct *push_task; + int push_task_detached = 0; + + raw_spin_lock_irq(&busiest_rq->__lock); + push_task = wrq->push_task; + + /* sanity checks before initiating the pull */ + if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu) || !push_task) + goto out_unlock; + + if (unlikely(busiest_cpu != raw_smp_processor_id() || + !busiest_rq->active_balance)) + goto out_unlock; + + if (busiest_rq->nr_running <= 1) + goto out_unlock; + + BUG_ON(busiest_rq == target_rq); + + if (task_on_rq_queued(push_task) && + READ_ONCE(push_task->__state) == TASK_RUNNING && + task_cpu(push_task) == busiest_cpu && + cpu_active(target_cpu) && + cpumask_test_cpu(target_cpu, push_task->cpus_ptr)) { + walt_detach_task(push_task, busiest_rq, target_rq); + push_task_detached = 1; + } + +out_unlock: /* called with busiest_rq lock */ + busiest_rq->active_balance = 0; + target_cpu = busiest_rq->push_cpu; + clear_reserved(target_cpu); + wrq->push_task = NULL; + raw_spin_unlock(&busiest_rq->__lock); + + if (push_task_detached) { + raw_spin_lock(&target_rq->__lock); + walt_attach_task(push_task, target_rq); + raw_spin_unlock(&target_rq->__lock); + } + + if (push_task) + put_task_struct(push_task); + + local_irq_enable(); + + return 0; +} + +struct walt_lb_rotate_work { + struct work_struct w; + struct task_struct *src_task; + struct task_struct *dst_task; + int src_cpu; + int dst_cpu; +}; + +DEFINE_PER_CPU(struct walt_lb_rotate_work, walt_lb_rotate_works); + +static void walt_lb_rotate_work_func(struct work_struct *work) +{ + struct walt_lb_rotate_work *wr = container_of(work, + struct walt_lb_rotate_work, w); + struct rq *src_rq = cpu_rq(wr->src_cpu), *dst_rq = cpu_rq(wr->dst_cpu); + unsigned long flags; + + migrate_swap(wr->src_task, wr->dst_task, wr->dst_cpu, wr->src_cpu); + + put_task_struct(wr->src_task); + put_task_struct(wr->dst_task); + + local_irq_save(flags); + double_rq_lock(src_rq, dst_rq); + dst_rq->active_balance = 0; + src_rq->active_balance = 0; + double_rq_unlock(src_rq, dst_rq); + local_irq_restore(flags); + + clear_reserved(wr->src_cpu); + clear_reserved(wr->dst_cpu); +} + +static void walt_lb_rotate_work_init(void) +{ + int i; + + for_each_possible_cpu(i) { + struct walt_lb_rotate_work *wr = &per_cpu(walt_lb_rotate_works, i); + + INIT_WORK(&wr->w, walt_lb_rotate_work_func); + } +} + +#define WALT_ROTATION_THRESHOLD_NS 16000000 +static void walt_lb_check_for_rotation(struct rq *src_rq) +{ + u64 wc, wait, max_wait = 0, run, max_run = 0; + int deserved_cpu = nr_cpu_ids, dst_cpu = nr_cpu_ids; + int i, src_cpu = cpu_of(src_rq); + struct rq *dst_rq; + struct walt_lb_rotate_work *wr = NULL; + struct walt_task_struct *wts; + + if (!is_min_cluster_cpu(src_cpu)) + return; + + /* + * Use src_rq->clock directly instead of rq_clock() since + * we do not have the rq lock and + * src_rq->clock was updated in the tick callpath. + */ + wc = src_rq->clock; + + for_each_possible_cpu(i) { + struct rq *rq = cpu_rq(i); + + if (!is_min_cluster_cpu(i)) + break; + + if (is_reserved(i)) + continue; + + if (!rq->misfit_task_load || !walt_fair_task(rq->curr)) + continue; + + wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; + wait = wc - wts->last_enqueued_ts; + if (wait > max_wait) { + max_wait = wait; + deserved_cpu = i; + } + } + + if (deserved_cpu != src_cpu) + return; + + for_each_possible_cpu(i) { + struct rq *rq = cpu_rq(i); + + if (is_min_cluster_cpu(i)) + continue; + + if (is_reserved(i)) + continue; + + if (!walt_fair_task(rq->curr)) + continue; + + if (rq->nr_running > 1) + continue; + + wts = (struct walt_task_struct *) rq->curr->android_vendor_data1; + run = wc - wts->last_enqueued_ts; + + if (run < WALT_ROTATION_THRESHOLD_NS) + continue; + + if (run > max_run) { + max_run = run; + dst_cpu = i; + } + } + + if (dst_cpu == nr_cpu_ids) + return; + + dst_rq = cpu_rq(dst_cpu); + + double_rq_lock(src_rq, dst_rq); + if (walt_fair_task(dst_rq->curr) && + !src_rq->active_balance && !dst_rq->active_balance && + cpumask_test_cpu(dst_cpu, src_rq->curr->cpus_ptr) && + cpumask_test_cpu(src_cpu, dst_rq->curr->cpus_ptr)) { + get_task_struct(src_rq->curr); + get_task_struct(dst_rq->curr); + + mark_reserved(src_cpu); + mark_reserved(dst_cpu); + wr = &per_cpu(walt_lb_rotate_works, src_cpu); + + wr->src_task = src_rq->curr; + wr->dst_task = dst_rq->curr; + + wr->src_cpu = src_cpu; + wr->dst_cpu = dst_cpu; + + dst_rq->active_balance = 1; + src_rq->active_balance = 1; + } + double_rq_unlock(src_rq, dst_rq); + + if (wr) + queue_work_on(src_cpu, system_highpri_wq, &wr->w); +} + +static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, + bool to_lower, bool to_higher, bool force) +{ + struct walt_rq *wrq = (struct walt_rq *) task_rq(p)->android_vendor_data1; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + /* Don't detach task if it is under active migration */ + if (wrq->push_task == p) + return false; + + if (to_lower) { + if (wts->iowaited) + return false; + if (per_task_boost(p) == TASK_BOOST_STRICT_MAX && + task_in_related_thread_group(p)) + return false; + if (walt_pipeline_low_latency_task(p)) + return false; + if (!force && walt_get_rtg_status(p)) + return false; + if (!force && !task_fits_max(p, dst_cpu)) + return false; + } else if (!to_higher) { + if (!task_fits_max(p, dst_cpu) && + wrq->walt_stats.nr_big_tasks < 2) + return false; + } + + /* Don't detach task if dest cpu is halted */ + if (cpu_halted(dst_cpu)) + return false; + + return true; +} + +static inline bool need_active_lb(struct task_struct *p, int dst_cpu, + int src_cpu) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + struct walt_rq *src_wrq = (struct walt_rq *) cpu_rq(src_cpu)->android_vendor_data1; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + + if (cpu_rq(src_cpu)->active_balance) + return false; + + if (dst_wrq->cluster->id <= src_wrq->cluster->id) + return false; + + if (!wts->misfit) + return false; + + return true; +} + +static int walt_lb_pull_tasks(int dst_cpu, int src_cpu) +{ + struct rq *dst_rq = cpu_rq(dst_cpu); + struct rq *src_rq = cpu_rq(src_cpu); + unsigned long flags; + struct task_struct *pulled_task = NULL, *p; + bool active_balance = false, to_lower, to_higher; + struct walt_rq *src_wrq = (struct walt_rq *) src_rq->android_vendor_data1; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + struct walt_task_struct *wts; + + BUG_ON(src_cpu == dst_cpu); + + to_lower = dst_wrq->cluster->id < src_wrq->cluster->id; + + to_higher = dst_wrq->cluster->id > src_wrq->cluster->id; + + raw_spin_lock_irqsave(&src_rq->__lock, flags); + + list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { + + if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) + continue; + + if (task_running(src_rq, p)) + continue; + + if (!_walt_can_migrate_task(p, dst_cpu, to_lower, to_higher, + false)) + continue; + + walt_detach_task(p, src_rq, dst_rq); + pulled_task = p; + goto unlock; + } + + list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { + + if (!cpumask_test_cpu(dst_cpu, p->cpus_ptr)) + continue; + + if (task_running(src_rq, p)) + continue; + + if (!_walt_can_migrate_task(p, dst_cpu, to_lower, to_higher, + true)) + continue; + + walt_detach_task(p, src_rq, dst_rq); + pulled_task = p; + goto unlock; + } + + list_for_each_entry_reverse(p, &src_rq->cfs_tasks, se.group_node) { + + if (task_running(src_rq, p)) { + if (cpumask_test_cpu(dst_cpu, p->cpus_ptr) + && need_active_lb(p, dst_cpu, src_cpu)) { + bool success; + active_balance = true; + src_rq->active_balance = 1; + src_rq->push_cpu = dst_cpu; + get_task_struct(p); + src_wrq->push_task = p; + mark_reserved(dst_cpu); + + /* lock must be dropped before waking the stopper */ + raw_spin_unlock_irqrestore(&src_rq->__lock, flags); + + /* + * Using our custom active load balance callback so that + * the push_task is really pulled onto this CPU. + */ + wts = (struct walt_task_struct *) p->android_vendor_data1; + trace_walt_active_load_balance(p, src_cpu, dst_cpu, wts); + success = stop_one_cpu_nowait(src_cpu, + stop_walt_lb_active_migration, + src_rq, &src_rq->active_balance_work); + if (!success) + clear_reserved(dst_cpu); + + return 0; /* we did not pull any task here */ + } + goto unlock; + } + } +unlock: + /* lock must be dropped before waking the stopper */ + raw_spin_unlock_irqrestore(&src_rq->__lock, flags); + + if (!pulled_task) + return 0; + + raw_spin_lock_irqsave(&dst_rq->__lock, flags); + walt_attach_task(p, dst_rq); + raw_spin_unlock_irqrestore(&dst_rq->__lock, flags); + + return 1; /* we pulled 1 task */ +} + +#define SMALL_TASK_THRESHOLD 102 +/* + * find_first_idle_if_others_are_busy + * + * Get an idle cpu in the src_mask, iif the other cpus are busy + * with larger tasks or more than one task. + * + * Returns -1 if there are no idle cpus in the mask, or if there + * is a CPU that is about to be come newly idle. Returns + * if there is an idle cpu in a cluster that's not about to do + * newly idle load balancing. + */ +static int find_first_idle_if_others_are_busy(const cpumask_t *src_mask) +{ + int i, first_idle = -1; + + for_each_cpu(i, src_mask) { + if (available_idle_cpu(i)) + first_idle = i; + + if (cpu_util(i) < SMALL_TASK_THRESHOLD && cpu_rq(i)->nr_running == 1) { + /* + * there was one CPU in the mask that was almost idle, but not + * quite. when it becomes idle, it will do newidle load-balance, + * and start off with it's own cluster. So no reason to kick + * anything in this cluster. i.e. don't perform a wasted kick. + */ + first_idle = -1; + + /* return -1, a cpu in this cluster is about to do + * newly idle load balancing + */ + break; + } + } + + return first_idle; +} + +static int walt_lb_find_busiest_similar_cap_cpu(int dst_cpu, const cpumask_t *src_mask, + int *has_misfit) +{ + int i; + int busiest_cpu = -1; + unsigned long util, busiest_util = 0; + struct walt_rq *wrq; + + for_each_cpu(i, src_mask) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + trace_walt_lb_cpu_util(i, wrq); + + if (cpu_rq(i)->nr_running < 2 || !cpu_rq(i)->cfs.h_nr_running) + continue; + + util = walt_lb_cpu_util(i); + if (util < busiest_util) + continue; + + busiest_util = util; + busiest_cpu = i; + } + + return busiest_cpu; +} + +static int walt_lb_find_busiest_from_higher_cap_cpu(int dst_cpu, const cpumask_t *src_mask, + int *has_misfit) +{ + int i; + int busiest_cpu = -1; + unsigned long util, busiest_util = 0; + unsigned long total_capacity = 0, total_util = 0, total_nr = 0; + int total_cpus = 0; + struct walt_rq *wrq; + bool asymcap_boost = ASYMCAP_BOOST(dst_cpu); + for_each_cpu(i, src_mask) { + + if (!cpu_active(i)) + continue; + + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + trace_walt_lb_cpu_util(i, wrq); + + util = walt_lb_cpu_util(i); + total_cpus += 1; + total_util += util; + total_capacity += capacity_orig_of(i); + total_nr += cpu_rq(i)->cfs.h_nr_running; + + if (cpu_rq(i)->cfs.h_nr_running < 2) + continue; + + if (cpu_rq(i)->cfs.h_nr_running == 2 && + task_util(cpu_rq(i)->curr) < SMALL_TASK_THRESHOLD) + continue; + + /* + * During rotation, two silver fmax tasks gets + * placed on gold/prime and the CPU may not be + * overutilized but for rotation, we have to + * spread out. + */ + if (!walt_rotation_enabled && !cpu_overutilized(i) && + !asymcap_boost) + continue; + + if (util < busiest_util) + continue; + + busiest_util = util; + busiest_cpu = i; + } + + /* + * Don't allow migrating to lower cluster unless this high + * capacity cluster is sufficiently loaded. + */ + if (!walt_rotation_enabled && !asymcap_boost) { + if (total_nr <= total_cpus || total_util * 1280 < total_capacity * 1024) + busiest_cpu = -1; + } + + return busiest_cpu; +} + +static int walt_lb_find_busiest_from_lower_cap_cpu(int dst_cpu, const cpumask_t *src_mask, + int *has_misfit) +{ + int i; + int busiest_cpu = -1; + unsigned long util, busiest_util = 0; + unsigned long total_capacity = 0, total_util = 0, total_nr = 0; + int total_cpus = 0; + int busy_nr_big_tasks = 0; + struct walt_rq *wrq; + + /* + * A higher capacity CPU is looking at a lower capacity + * cluster. active balance and big tasks are in play. + * other than that, it is very much same as above. we + * really don't need this as a separate block. will + * refactor this after final testing is done. + */ + for_each_cpu(i, src_mask) { + wrq = (struct walt_rq *) cpu_rq(i)->android_vendor_data1; + + if (!cpu_active(i)) + continue; + + trace_walt_lb_cpu_util(i, wrq); + + util = walt_lb_cpu_util(i); + total_cpus += 1; + total_util += util; + total_capacity += capacity_orig_of(i); + total_nr += cpu_rq(i)->cfs.h_nr_running; + + /* + * no point in selecting this CPU as busy, as + * active balance is in progress. + */ + if (cpu_rq(i)->active_balance) + continue; + + /* active migration is allowed only to idle cpu */ + if (cpu_rq(i)->cfs.h_nr_running < 2 && + (!wrq->walt_stats.nr_big_tasks || !available_idle_cpu(dst_cpu))) + continue; + + if (!walt_rotation_enabled && !cpu_overutilized(i) && + !ASYMCAP_BOOST(i)) + continue; + + if (util < busiest_util) + continue; + + busiest_util = util; + busiest_cpu = i; + busy_nr_big_tasks = wrq->walt_stats.nr_big_tasks; + } + + if (!walt_rotation_enabled && !busy_nr_big_tasks && + !(busiest_cpu != -1 && ASYMCAP_BOOST(busiest_cpu))) { + if (total_nr <= total_cpus || total_util * 1280 < total_capacity * 1024) + busiest_cpu = -1; + } + + if (busy_nr_big_tasks && busiest_cpu != -1) + *has_misfit = true; + + return busiest_cpu; +} + +static int walt_lb_find_busiest_cpu(int dst_cpu, const cpumask_t *src_mask, int *has_misfit) +{ + int fsrc_cpu = cpumask_first(src_mask); + int busiest_cpu; + struct walt_rq *fsrc_wrq = (struct walt_rq *) cpu_rq(fsrc_cpu)->android_vendor_data1; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + + if (dst_wrq->cluster->id == fsrc_wrq->cluster->id) + busiest_cpu = walt_lb_find_busiest_similar_cap_cpu(dst_cpu, + src_mask, has_misfit); + else if (dst_wrq->cluster->id > fsrc_wrq->cluster->id) + busiest_cpu = walt_lb_find_busiest_from_lower_cap_cpu(dst_cpu, + src_mask, has_misfit); + else + busiest_cpu = walt_lb_find_busiest_from_higher_cap_cpu(dst_cpu, + src_mask, has_misfit); + + return busiest_cpu; +} + +static DEFINE_RAW_SPINLOCK(walt_lb_migration_lock); +void walt_lb_tick(struct rq *rq) +{ + int prev_cpu = rq->cpu, new_cpu, ret; + struct task_struct *p = rq->curr; + unsigned long flags; + struct walt_rq *prev_wrq = (struct walt_rq *) rq->android_vendor_data1; + struct walt_rq *new_wrq; + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + raw_spin_lock(&rq->__lock); + if (available_idle_cpu(prev_cpu) && is_reserved(prev_cpu) && !rq->active_balance) + clear_reserved(prev_cpu); + raw_spin_unlock(&rq->__lock); + + if (!walt_fair_task(p)) + return; + + walt_cfs_tick(rq); + + if (!rq->misfit_task_load) + return; + + if (READ_ONCE(p->__state) != TASK_RUNNING || p->nr_cpus_allowed == 1) + return; + + raw_spin_lock_irqsave(&walt_lb_migration_lock, flags); + + if (walt_rotation_enabled) { + walt_lb_check_for_rotation(rq); + goto out_unlock; + } + + rcu_read_lock(); + new_cpu = walt_find_energy_efficient_cpu(p, prev_cpu, 0, 1); + rcu_read_unlock(); + + new_wrq = (struct walt_rq *) cpu_rq(new_cpu)->android_vendor_data1; + + /* prevent active task migration to busy or same/lower capacity CPU */ + if (!available_idle_cpu(new_cpu) || new_wrq->cluster->id <= prev_wrq->cluster->id) + goto out_unlock; + + raw_spin_lock(&rq->__lock); + if (rq->active_balance) { + raw_spin_unlock(&rq->__lock); + goto out_unlock; + } + rq->active_balance = 1; + rq->push_cpu = new_cpu; + get_task_struct(p); + prev_wrq->push_task = p; + raw_spin_unlock(&rq->__lock); + + mark_reserved(new_cpu); + raw_spin_unlock_irqrestore(&walt_lb_migration_lock, flags); + + trace_walt_active_load_balance(p, prev_cpu, new_cpu, wts); + ret = stop_one_cpu_nowait(prev_cpu, + stop_walt_lb_active_migration, rq, + &rq->active_balance_work); + if (!ret) + clear_reserved(new_cpu); + else + wake_up_if_idle(new_cpu); + + return; + +out_unlock: + raw_spin_unlock_irqrestore(&walt_lb_migration_lock, flags); +} + +static inline int has_pushable_tasks(struct rq *rq) +{ + return !plist_head_empty(&rq->rt.pushable_tasks); +} + +#define WALT_RT_PULL_THRESHOLD_NS 250000 +static bool walt_balance_rt(struct rq *this_rq) +{ + int i, this_cpu = this_rq->cpu, src_cpu = this_cpu; + struct rq *src_rq; + struct task_struct *p; + struct walt_task_struct *wts; + bool pulled = false; + u64 wallclock; + + /* can't help if this has a runnable RT */ + if (sched_rt_runnable(this_rq)) + return false; + + /* check if any CPU has a pushable RT task */ + for_each_possible_cpu(i) { + struct rq *rq = cpu_rq(i); + + if (!has_pushable_tasks(rq)) + continue; + + src_cpu = i; + break; + } + + if (src_cpu == this_cpu) + return false; + + src_rq = cpu_rq(src_cpu); + double_lock_balance(this_rq, src_rq); + + /* lock is dropped, so check again */ + if (sched_rt_runnable(this_rq)) + goto unlock; + + p = pick_highest_pushable_task(src_rq, this_cpu); + + if (!p) + goto unlock; + + if (!cpumask_test_cpu(this_cpu, p->cpus_ptr)) + goto unlock; + + wts = (struct walt_task_struct *) p->android_vendor_data1; + + /* + * Use rq->clock directly instead of rq_clock() since + * rq->clock was updated recently in the __schedule() -> pick_next_task() callpath. + * Time lost in grabbing rq locks will likely be corrected via max. + */ + wallclock = max(this_rq->clock, src_rq->clock); + + if (wallclock > wts->last_wake_ts && + wallclock - wts->last_wake_ts < WALT_RT_PULL_THRESHOLD_NS) + goto unlock; + + pulled = true; + deactivate_task(src_rq, p, 0); + set_task_cpu(p, this_cpu); + activate_task(this_rq, p, 0); +unlock: + double_unlock_balance(this_rq, src_rq); + return pulled; +} + +__read_mostly unsigned int sysctl_sched_force_lb_enable = 1; +static bool should_help_min_cap(int this_cpu) +{ + int cpu; + + if (!sysctl_sched_force_lb_enable || is_min_cluster_cpu(this_cpu)) + return false; + + for_each_cpu(cpu, &cpu_array[0][0]) { + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + if (wrq->walt_stats.nr_big_tasks) + return true; + } + + return false; +} + +/* similar to sysctl_sched_migration_cost */ +#define NEWIDLE_BALANCE_THRESHOLD 500000 +static void walt_newidle_balance(void *unused, struct rq *this_rq, + struct rq_flags *rf, int *pulled_task, + int *done) +{ + int this_cpu = this_rq->cpu; + struct walt_rq *wrq = (struct walt_rq *) this_rq->android_vendor_data1; + int order_index; + int busy_cpu = -1; + bool enough_idle = (this_rq->avg_idle > NEWIDLE_BALANCE_THRESHOLD); + bool help_min_cap = false, find_next_cluster = false; + int first_idle; + int has_misfit = 0; + + if (unlikely(walt_disabled)) + return; + + /* + * newly idle load balance is completely handled here, so + * set done to skip the load balance by the caller. + */ + *done = 1; + *pulled_task = 0; + + /* + * This CPU is about to enter idle, so clear the + * misfit_task_load and mark the idle stamp. + */ + this_rq->misfit_task_load = 0; + this_rq->idle_stamp = rq_clock(this_rq); + + if (!cpu_active(this_cpu)) + return; + + if (cpu_halted(this_cpu)) + return; + + if (is_reserved(this_cpu)) + return; + + /*Cluster isn't initialized until after WALT is enabled*/ + order_index = wrq->cluster->id; + + rq_unpin_lock(this_rq, rf); + + /* + * Since we drop rq lock while doing RT balance, + * check if any tasks are queued on this and bail out + * early. + */ + if (walt_balance_rt(this_rq) || this_rq->nr_running) + goto rt_pulled; + + if (!READ_ONCE(this_rq->rd->overload)) + goto repin; + + if (atomic_read(&this_rq->nr_iowait) && !enough_idle) + goto repin; + + help_min_cap = should_help_min_cap(this_cpu); + raw_spin_unlock(&this_rq->__lock); + + /* + * careful, we dropped the lock, and has to be acquired + * before returning. Since rq lock is dropped, tasks + * can be queued remotely, so keep a check on nr_running + * and bail out. + */ + if (num_sched_clusters <= 2) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + + if (num_sched_clusters == 2) { + has_misfit = false; + find_next_cluster = (order_index == 0) ? enough_idle : 1; + if (find_next_cluster) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, + &cpu_array[order_index][1], &has_misfit); + if (busy_cpu != -1 && (enough_idle || has_misfit)) + goto found_busy_cpu; + } + } + + goto unlock; + } + + if (order_index == 0) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + + if (enough_idle) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + } + + /* + * help the farthest cluster by kicking an idle cpu in the next + * cluster. In case no idle is found, pull it in. + */ + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], + &has_misfit); + if (busy_cpu != -1) { + first_idle = + find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); + if (first_idle != -1) + walt_kick_cpu(first_idle); + else + goto found_busy_cpu; + } + } else if (order_index == 2) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + + /* help gold only if prime has had enough idle or gold has a misfit */ + has_misfit = false; + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], + &has_misfit); + if (busy_cpu != -1 && (enough_idle || has_misfit)) + goto found_busy_cpu; + + /* help the farthest cluster indirectly if it needs help */ + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], + &has_misfit); + if (busy_cpu != -1) { + first_idle = + find_first_idle_if_others_are_busy(&cpu_array[order_index][1]); + walt_kick_cpu(first_idle); + } + } else { + busy_cpu = + walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][0], &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + + if (enough_idle) { + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][1], + &has_misfit); + if (busy_cpu != -1) + goto found_busy_cpu; + } + + has_misfit = false; + busy_cpu = walt_lb_find_busiest_cpu(this_cpu, &cpu_array[order_index][2], + &has_misfit); + if (busy_cpu != -1 && (enough_idle || has_misfit)) + goto found_busy_cpu; + } + goto unlock; + +found_busy_cpu: + /* sanity checks before attempting the pull */ + if (this_rq->nr_running > 0 || (busy_cpu == this_cpu)) + goto unlock; + + *pulled_task = walt_lb_pull_tasks(this_cpu, busy_cpu); + +unlock: + raw_spin_lock(&this_rq->__lock); +rt_pulled: + if (this_rq->cfs.h_nr_running && !*pulled_task) + *pulled_task = 1; + + /* Is there a task of a high priority class? */ + if (this_rq->nr_running != this_rq->cfs.h_nr_running) + *pulled_task = -1; + + /* reset the idle time stamp if we pulled any task */ + if (*pulled_task) + this_rq->idle_stamp = 0; + +repin: + rq_repin_lock(this_rq, rf); + + trace_walt_newidle_balance(this_cpu, busy_cpu, *pulled_task, + help_min_cap, enough_idle); +} + +/* run newidle balance as a result of an unhalt operation */ +void walt_smp_newidle_balance(void *ignored) +{ + int cpu = raw_smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + int pulled_task; + int done = 0; + + rq_lock(rq, &rf); + update_rq_clock(rq); + walt_newidle_balance(NULL, rq, &rf, &pulled_task, &done); + resched_curr(rq); + rq_unlock(rq, &rf); +} + +static DEFINE_PER_CPU(call_single_data_t, nib_csd); + +void walt_smp_call_newidle_balance(int cpu) +{ + call_single_data_t *csd = &per_cpu(nib_csd, cpu); + + if (unlikely(walt_disabled)) + return; + + smp_call_function_single_async(cpu, csd); +} + +static void walt_find_busiest_queue(void *unused, int dst_cpu, + struct sched_group *group, + struct cpumask *env_cpus, + struct rq **busiest, int *done) +{ + int fsrc_cpu = group_first_cpu(group); + int busiest_cpu = -1; + struct cpumask src_mask; + int has_misfit; + + if (unlikely(walt_disabled)) + return; + *done = 1; + *busiest = NULL; + + /* + * same cluster means, there will only be 1 + * CPU in the busy group, so just select it. + */ + if (same_cluster(dst_cpu, fsrc_cpu)) { + busiest_cpu = fsrc_cpu; + goto done; + } + + /* + * We will allow inter cluster migrations + * only if the source group is sufficiently + * loaded. The upstream load balancer is a + * bit more generous. + * + * re-using the same code that we use it + * for newly idle load balance. The policies + * remain same. + */ + cpumask_and(&src_mask, sched_group_span(group), env_cpus); + busiest_cpu = walt_lb_find_busiest_cpu(dst_cpu, &src_mask, &has_misfit); +done: + if (busiest_cpu != -1) + *busiest = cpu_rq(busiest_cpu); + + trace_walt_find_busiest_queue(dst_cpu, busiest_cpu, src_mask.bits[0]); +} + +/* + * we only decide if nohz balance kick is needed or not. the + * first CPU in the nohz.idle will come out of idle and do + * load balance on behalf of every CPU. adding another hook + * to decide which cpu to kick is useless. most of the time, + * it is impossible to decide which CPU has to come out because + * we get to kick only once. + */ +static void walt_nohz_balancer_kick(void *unused, struct rq *rq, + unsigned int *flags, int *done) +{ + if (unlikely(walt_disabled)) + return; + *done = 1; + + /* + * tick path migration takes care of misfit task. + * so we have to check for nr_running >= 2 here. + */ + if (rq->nr_running >= 2 && cpu_overutilized(rq->cpu)) { + *flags = NOHZ_KICK_MASK; + trace_walt_nohz_balance_kick(rq); + } +} + +static void walt_can_migrate_task(void *unused, struct task_struct *p, + int dst_cpu, int *can_migrate) +{ + bool to_lower, to_higher; + struct walt_rq *dst_wrq = (struct walt_rq *) cpu_rq(dst_cpu)->android_vendor_data1; + struct walt_rq *task_wrq = (struct walt_rq *) cpu_rq(task_cpu(p))->android_vendor_data1; + + if (unlikely(walt_disabled)) + return; + to_lower = dst_wrq->cluster->id < task_wrq->cluster->id; + to_higher = dst_wrq->cluster->id > task_wrq->cluster->id; + + if (_walt_can_migrate_task(p, dst_cpu, to_lower, + to_higher, true)) + return; + + *can_migrate = 0; +} + +void walt_lb_init(void) +{ + int cpu; + + walt_lb_rotate_work_init(); + + register_trace_android_rvh_sched_nohz_balancer_kick(walt_nohz_balancer_kick, NULL); + register_trace_android_rvh_can_migrate_task(walt_can_migrate_task, NULL); + register_trace_android_rvh_find_busiest_queue(walt_find_busiest_queue, NULL); + register_trace_android_rvh_sched_newidle_balance(walt_newidle_balance, NULL); + + for_each_cpu(cpu, cpu_possible_mask) { + call_single_data_t *csd; + + csd = &per_cpu(nib_csd, cpu); + INIT_CSD(csd, walt_smp_newidle_balance, (void *)(unsigned long)cpu); + } +} diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c new file mode 100644 index 000000000000..e69fe2a9e9de --- /dev/null +++ b/kernel/sched/walt/walt_rt.c @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include + +#include "walt.h" +#include "trace.h" + +#define MSEC_TO_NSEC (1000 * 1000) + +static DEFINE_PER_CPU(cpumask_var_t, walt_local_cpu_mask); +DEFINE_PER_CPU(u64, rt_task_arrival_time) = 0; +static bool long_running_rt_task_trace_rgstrd; + +static void rt_task_arrival_marker(void *unused, bool preempt, + struct task_struct *prev, struct task_struct *next) +{ + unsigned int cpu = raw_smp_processor_id(); + + if (next->policy == SCHED_FIFO && next != cpu_rq(cpu)->stop) + per_cpu(rt_task_arrival_time, cpu) = rq_clock_task(this_rq()); + else + per_cpu(rt_task_arrival_time, cpu) = 0; +} + +static void long_running_rt_task_notifier(void *unused, struct rq *rq) +{ + struct task_struct *curr = rq->curr; + unsigned int cpu = raw_smp_processor_id(); + + if (!sysctl_sched_long_running_rt_task_ms) + return; + + if (!per_cpu(rt_task_arrival_time, cpu)) + return; + + if (per_cpu(rt_task_arrival_time, cpu) && curr->policy != SCHED_FIFO) { + /* + * It is possible that the scheduling policy for the current + * task might get changed after task arrival time stamp is + * noted during sched_switch of RT task. To avoid such false + * positives, reset arrival time stamp. + */ + per_cpu(rt_task_arrival_time, cpu) = 0; + return; + } + + /* + * Since we are called from the main tick, rq clock task must have + * been updated very recently. Use it directly, instead of + * update_rq_clock_task() to avoid warnings. + */ + if (rq->clock_task - + per_cpu(rt_task_arrival_time, cpu) + > sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC) { + printk_deferred("RT task %s (%d) runtime > %u now=%llu task arrival time=%llu runtime=%llu\n", + curr->comm, curr->pid, + sysctl_sched_long_running_rt_task_ms * MSEC_TO_NSEC, + rq->clock_task, + per_cpu(rt_task_arrival_time, cpu), + rq->clock_task - + per_cpu(rt_task_arrival_time, cpu)); + BUG(); + } +} + +int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + static DEFINE_MUTEX(mutex); + + mutex_lock(&mutex); + + ret = proc_douintvec_minmax(table, write, buffer, lenp, ppos); + + if (sysctl_sched_long_running_rt_task_ms > 0 && + sysctl_sched_long_running_rt_task_ms < 800) + sysctl_sched_long_running_rt_task_ms = 800; + + if (write && !long_running_rt_task_trace_rgstrd) { + register_trace_sched_switch(rt_task_arrival_marker, NULL); + register_trace_android_vh_scheduler_tick(long_running_rt_task_notifier, NULL); + long_running_rt_task_trace_rgstrd = true; + } + + mutex_unlock(&mutex); + + return ret; +} + +static void walt_rt_energy_aware_wake_cpu(struct task_struct *task, struct cpumask *lowest_mask, + int ret, int *best_cpu) +{ + int cpu; + unsigned long util, best_cpu_util = ULONG_MAX; + unsigned long best_cpu_util_cum = ULONG_MAX; + unsigned long util_cum; + unsigned long tutil = task_util(task); + unsigned int best_idle_exit_latency = UINT_MAX; + unsigned int cpu_idle_exit_latency = UINT_MAX; + bool boost_on_big = rt_boost_on_big(); + int cluster; + int order_index = (boost_on_big && num_sched_clusters > 1) ? 1 : 0; + bool best_cpu_lt = true; + + if (unlikely(walt_disabled)) + return; + + if (!ret) + return; /* No targets found */ + + rcu_read_lock(); + for (cluster = 0; cluster < num_sched_clusters; cluster++) { + for_each_cpu_and(cpu, lowest_mask, &cpu_array[order_index][cluster]) { + bool lt; + + trace_sched_cpu_util(cpu, lowest_mask); + + if (!cpu_active(cpu)) + continue; + + if (cpu_halted(cpu)) + continue; + + if (sched_cpu_high_irqload(cpu)) + continue; + + if (__cpu_overutilized(cpu, tutil)) + continue; + + util = cpu_util(cpu); + + lt = (walt_low_latency_task(cpu_rq(cpu)->curr) || + walt_nr_rtg_high_prio(cpu)); + + /* + * When the best is suitable and the current is not, + * skip it + */ + if (lt && !best_cpu_lt) + continue; + + /* + * Either both are sutilable or unsuitable, load takes + * precedence. + */ + if (!(best_cpu_lt ^ lt) && (util > best_cpu_util)) + continue; + + /* + * If the previous CPU has same load, keep it as + * best_cpu. + */ + if (best_cpu_util == util && *best_cpu == task_cpu(task)) + continue; + + /* + * If candidate CPU is the previous CPU, select it. + * Otherwise, if its load is same with best_cpu and in + * a shallower C-state, select it. If all above + * conditions are same, select the least cumulative + * window demand CPU. + */ + cpu_idle_exit_latency = walt_get_idle_exit_latency(cpu_rq(cpu)); + + util_cum = cpu_util_cum(cpu); + if (cpu != task_cpu(task) && best_cpu_util == util) { + if (best_idle_exit_latency < cpu_idle_exit_latency) + continue; + + if (best_idle_exit_latency == cpu_idle_exit_latency && + best_cpu_util_cum < util_cum) + continue; + } + + best_idle_exit_latency = cpu_idle_exit_latency; + best_cpu_util_cum = util_cum; + best_cpu_util = util; + *best_cpu = cpu; + best_cpu_lt = lt; + } + + if (*best_cpu != -1) + break; + } + + rcu_read_unlock(); +} + +#ifdef CONFIG_UCLAMP_TASK +static inline bool walt_rt_task_fits_capacity(struct task_struct *p, int cpu) +{ + unsigned int min_cap; + unsigned int max_cap; + unsigned int cpu_cap; + + min_cap = uclamp_eff_value(p, UCLAMP_MIN); + max_cap = uclamp_eff_value(p, UCLAMP_MAX); + + cpu_cap = capacity_orig_of(cpu); + + return cpu_cap >= min(min_cap, max_cap); +} +#else +static inline bool walt_rt_task_fits_capacity(struct task_struct *p, int cpu) +{ + return true; +} +#endif + +/* + * walt specific should_honor_rt_sync (see rt.c). this will honor + * the sync flag regardless of whether the current waker is cfs or rt + */ +static inline bool walt_should_honor_rt_sync(struct rq *rq, struct task_struct *p, + bool sync) +{ + return sync && + p->prio <= rq->rt.highest_prio.next && + rq->rt.rt_nr_running <= 2; +} + +static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int cpu, + int sd_flag, int wake_flags, int *new_cpu) +{ + struct task_struct *curr; + struct rq *rq, *this_cpu_rq; + bool may_not_preempt; + bool sync = !!(wake_flags & WF_SYNC); + int ret, target = -1, this_cpu; + struct cpumask *lowest_mask; + int packing_cpu; + + if (unlikely(walt_disabled)) + return; + + /* For anything but wake ups, just return the task_cpu */ + if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) + return; + + this_cpu = raw_smp_processor_id(); + this_cpu_rq = cpu_rq(this_cpu); + + /* + * Respect the sync flag as long as the task can run on this CPU. + */ + if (sysctl_sched_sync_hint_enable && cpu_active(this_cpu) && !cpu_halted(this_cpu) && + cpumask_test_cpu(this_cpu, task->cpus_ptr) && + walt_should_honor_rt_sync(this_cpu_rq, task, sync)) { + *new_cpu = this_cpu; + return; + } + + *new_cpu = cpu; /* previous CPU as back up */ + rq = cpu_rq(cpu); + + rcu_read_lock(); + curr = READ_ONCE(rq->curr); /* unlocked access */ + + /* + * If the current task on @p's runqueue is a softirq task, + * it may run without preemption for a time that is + * ill-suited for a waiting RT task. Therefore, try to + * wake this RT task on another runqueue. + * + * Otherwise, just let it ride on the affined RQ and the + * post-schedule router will push the preempted task away + * + * This test is optimistic, if we get it wrong the load-balancer + * will have to sort it out. + * + * We take into account the capacity of the CPU to ensure it fits the + * requirement of the task - which is only important on heterogeneous + * systems like big.LITTLE. + */ + may_not_preempt = task_may_not_preempt(curr, cpu); + + lowest_mask = this_cpu_cpumask_var_ptr(walt_local_cpu_mask); + + /* + * If we're on asym system ensure we consider the different capacities + * of the CPUs when searching for the lowest_mask. + */ + ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, task, + lowest_mask, walt_rt_task_fits_capacity); + + /* create a fastpath for finding a packing cpu */ + packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); + if (walt_choose_packing_cpu(packing_cpu, task)) { + *new_cpu = packing_cpu; + goto unlock; + } + + walt_rt_energy_aware_wake_cpu(task, lowest_mask, ret, &target); + + /* + * If cpu is non-preemptible, prefer remote cpu + * even if it's running a higher-prio task. + * Otherwise: Don't bother moving it if the destination CPU is + * not running a lower priority task. + */ + if (target != -1 && + (may_not_preempt || task->prio < cpu_rq(target)->rt.highest_prio.curr)) + *new_cpu = target; + + /* if backup or chosen cpu is halted, pick something else */ + if (cpu_halted(*new_cpu)) { + cpumask_t non_halted; + + /* choose the lowest-order, unhalted, allowed CPU */ + cpumask_andnot(&non_halted, task->cpus_ptr, cpu_halt_mask); + target = cpumask_first(&non_halted); + if (target < nr_cpu_ids) + *new_cpu = target; + } +unlock: + rcu_read_unlock(); +} + + +static void walt_rt_find_lowest_rq(void *unused, struct task_struct *task, + struct cpumask *lowest_mask, int ret, int *best_cpu) + +{ + int packing_cpu; + + if (unlikely(walt_disabled)) + return; + + /* create a fastpath for finding a packing cpu */ + packing_cpu = walt_find_cluster_packing_cpu(task_cpu(task)); + if (walt_choose_packing_cpu(packing_cpu, task)) { + *best_cpu = packing_cpu; + return; + } + + walt_rt_energy_aware_wake_cpu(task, lowest_mask, ret, best_cpu); + + /* + * Walt was not able to find a non-halted best cpu. Ensure that + * find_lowest_rq doesn't use a halted cpu going forward, but + * does a best effort itself to find a good CPU. + */ + if (*best_cpu == -1) + cpumask_andnot(lowest_mask, lowest_mask, cpu_halt_mask); +} + +void walt_rt_init(void) +{ + unsigned int i; + + for_each_possible_cpu(i) { + if(!(zalloc_cpumask_var_node(&per_cpu(walt_local_cpu_mask, i), + GFP_KERNEL, cpu_to_node(i)))) { + pr_err("walt_local_cpu_mask alloc failed for cpu%d\n", i); + return; + } + } + + register_trace_android_rvh_select_task_rq_rt(walt_select_task_rq_rt, NULL); + register_trace_android_rvh_find_lowest_rq(walt_rt_find_lowest_rq, NULL); +} diff --git a/kernel/sched/walt/walt_tp.c b/kernel/sched/walt/walt_tp.c new file mode 100644 index 000000000000..62d3acb2b482 --- /dev/null +++ b/kernel/sched/walt/walt_tp.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include "trace.h" +#define CREATE_TRACE_POINTS +#include "perf_trace_counters.h" + +unsigned int sysctl_sched_dynamic_tp_enable; + +#define USE_CPUHP_STATE CPUHP_AP_ONLINE_DYN + +DEFINE_PER_CPU(u32, cntenset_val); +DEFINE_PER_CPU(unsigned long, previous_ccnt); +DEFINE_PER_CPU(unsigned long[NUM_L1_CTRS], previous_l1_cnts); +DEFINE_PER_CPU(unsigned long[NUM_AMU_CTRS], previous_amu_cnts); +DEFINE_PER_CPU(u32, old_pid); +DEFINE_PER_CPU(u32, hotplug_flag); +DEFINE_PER_CPU(u64, prev_time); + +static int tracectr_cpu_hotplug_coming_up(unsigned int cpu) +{ + per_cpu(hotplug_flag, cpu) = 1; + + return 0; +} + +static void setup_prev_cnts(u32 cpu, u32 cnten_val) +{ + int i; + + if (cnten_val & CC) + per_cpu(previous_ccnt, cpu) = + read_sysreg(pmccntr_el0); + + for (i = 0; i < NUM_L1_CTRS; i++) { + if (cnten_val & (1 << i)) { + /* Select */ + write_sysreg(i, pmselr_el0); + isb(); + /* Read value */ + per_cpu(previous_l1_cnts[i], cpu) = + read_sysreg(pmxevcntr_el0); + } + } +} + +void tracectr_notifier(void *ignore, bool preempt, + struct task_struct *prev, struct task_struct *next) +{ + u32 cnten_val; + int current_pid; + u32 cpu = task_cpu(next); + u64 now; + + if (!trace_sched_switch_with_ctrs_enabled()) + return; + + current_pid = next->pid; + if (per_cpu(old_pid, cpu) != -1) { + cnten_val = read_sysreg(pmcntenset_el0); + per_cpu(cntenset_val, cpu) = cnten_val; + /* Disable all the counters that were enabled */ + write_sysreg(cnten_val, pmcntenclr_el0); + + if (per_cpu(hotplug_flag, cpu) == 1) { + per_cpu(hotplug_flag, cpu) = 0; + setup_prev_cnts(cpu, cnten_val); + } else { + trace_sched_switch_with_ctrs(per_cpu(old_pid, cpu), + current_pid); + now = sched_clock(); + if ((now - per_cpu(prev_time, cpu)) > NSEC_PER_SEC) { + trace_sched_switch_ctrs_cfg(cpu); + per_cpu(prev_time, cpu) = now; + } + } + + /* Enable all the counters that were disabled */ + write_sysreg(cnten_val, pmcntenset_el0); + } + per_cpu(old_pid, cpu) = current_pid; +} + +static void register_sched_switch_ctrs(void) +{ + int cpu, rc; + + for_each_possible_cpu(cpu) + per_cpu(old_pid, cpu) = -1; + + rc = cpuhp_setup_state_nocalls(USE_CPUHP_STATE, "tracectr_cpu_hotplug", + tracectr_cpu_hotplug_coming_up, NULL); + if (rc >= 0) + register_trace_sched_switch(tracectr_notifier, NULL); +} + +static void unregister_sched_switch_ctrs(void) +{ + unregister_trace_sched_switch(tracectr_notifier, NULL); + cpuhp_remove_state_nocalls(USE_CPUHP_STATE); +} + +static void sched_overutilized(void *data, struct root_domain *rd, + bool overutilized) +{ + if (trace_sched_overutilized_enabled()) { + char span[SPAN_SIZE]; + + cpumap_print_to_pagebuf(false, span, sched_trace_rd_span(rd)); + trace_sched_overutilized(overutilized, span); + } +} + +static void walt_register_dynamic_tp_events(void) +{ + register_trace_sched_overutilized_tp(sched_overutilized, NULL); + register_sched_switch_ctrs(); +} + +static void walt_unregister_dynamic_tp_events(void) +{ + unregister_trace_sched_overutilized_tp(sched_overutilized, NULL); + unregister_sched_switch_ctrs(); +} + +int sched_dynamic_tp_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + static DEFINE_MUTEX(mutex); + int ret = 0, *val = (unsigned int *)table->data; + unsigned int old_val; + + mutex_lock(&mutex); + old_val = sysctl_sched_dynamic_tp_enable; + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write || (old_val == sysctl_sched_dynamic_tp_enable)) + goto done; + + if (*val) + walt_register_dynamic_tp_events(); + else + walt_unregister_dynamic_tp_events(); +done: + mutex_unlock(&mutex); + return ret; +}