diff --git a/kernel/sched/walt/core_ctl.c b/kernel/sched/walt/core_ctl.c index cdb123d90b8f..ebd0b01da5f7 100644 --- a/kernel/sched/walt/core_ctl.c +++ b/kernel/sched/walt/core_ctl.c @@ -392,7 +392,7 @@ static ssize_t show_not_preferred(const struct cluster_data *state, char *buf) for (i = 0; i < state->num_cpus; i++) { c = &per_cpu(cpu_state, i + state->first_cpu); count += scnprintf(buf + count, PAGE_SIZE - count, - "CPU#%d: %u\n", c->cpu, c->not_preferred); + "%u ", c->not_preferred); } spin_unlock_irqrestore(&state_lock, flags); diff --git a/kernel/sched/walt/cpufreq_walt.c b/kernel/sched/walt/cpufreq_walt.c index f36038918132..83f802199f64 100644 --- a/kernel/sched/walt/cpufreq_walt.c +++ b/kernel/sched/walt/cpufreq_walt.c @@ -25,6 +25,8 @@ struct waltgov_tunables { unsigned int rtg_boost_freq; unsigned int adaptive_low_freq; unsigned int adaptive_high_freq; + unsigned int adaptive_low_freq_kernel; + unsigned int adaptive_high_freq_kernel; unsigned int target_load_thresh; unsigned int target_load_shift; bool pl; @@ -220,6 +222,18 @@ static inline unsigned long walt_map_util_freq(unsigned long util, return (fmax + (fmax >> 2)) * util / cap; } +static inline unsigned int get_adaptive_low_freq(struct waltgov_policy *wg_policy) +{ + return(max(wg_policy->tunables->adaptive_low_freq, + wg_policy->tunables->adaptive_low_freq_kernel)); +} + +static inline unsigned int get_adaptive_high_freq(struct waltgov_policy *wg_policy) +{ + return(max(wg_policy->tunables->adaptive_high_freq, + wg_policy->tunables->adaptive_high_freq_kernel)); +} + static unsigned int get_next_freq(struct waltgov_policy *wg_policy, unsigned long util, unsigned long max, struct waltgov_cpu *wg_cpu, u64 time) @@ -232,11 +246,11 @@ static unsigned int get_next_freq(struct waltgov_policy *wg_policy, freq = raw_freq; if (wg_policy->tunables->adaptive_high_freq) { - if (raw_freq < wg_policy->tunables->adaptive_low_freq) { - freq = wg_policy->tunables->adaptive_low_freq; + if (raw_freq < get_adaptive_low_freq(wg_policy)) { + freq = get_adaptive_low_freq(wg_policy); wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_LOW; - } else if (raw_freq <= wg_policy->tunables->adaptive_high_freq) { - freq = wg_policy->tunables->adaptive_high_freq; + } else if (raw_freq <= get_adaptive_high_freq(wg_policy)) { + freq = get_adaptive_high_freq(wg_policy); wg_driv_cpu->reasons = CPUFREQ_REASON_ADAPTIVE_HIGH; } } @@ -296,10 +310,11 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut bool is_migration = wg_cpu->flags & WALT_CPUFREQ_IC_MIGRATION; bool is_rtg_boost = wg_cpu->walt_load.rtgb_active; bool is_hiload; - bool is_ed_boost = wg_cpu->walt_load.ed_active; + bool big_task_rotation = wg_cpu->walt_load.big_task_rotation; + bool employ_ed_boost = wg_cpu->walt_load.ed_active && sysctl_ed_boost_pct; unsigned long pl = wg_cpu->walt_load.pl; - if (is_ed_boost) { + if (employ_ed_boost) { cpu_util = mult_frac(cpu_util, 100 + sysctl_ed_boost_pct, 100); max_and_reason(util, cpu_util, wg_cpu, CPUFREQ_REASON_EARLY_DET); } @@ -323,8 +338,11 @@ static void waltgov_walt_adjust(struct waltgov_cpu *wg_cpu, unsigned long cpu_ut max_and_reason(util, pl, wg_cpu, CPUFREQ_REASON_PL); } - if (is_ed_boost) + if (employ_ed_boost) wg_cpu->reasons |= CPUFREQ_REASON_EARLY_DET; + + if (big_task_rotation) + max_and_reason(util, *max, wg_cpu, CPUFREQ_REASON_BTR); } static inline unsigned long target_util(struct waltgov_policy *wg_policy, @@ -665,6 +683,91 @@ static ssize_t boost_store(struct gov_attr_set *attr_set, const char *buf, return count; } +/** + * cpufreq_walt_set_adaptive_freq() - set the waltgov adaptive freq for cpu + * @cpu: the cpu for which the values should be set + * @adaptive_low_freq: low freq + * @adaptive_high_freq:high_freq + * + * Configure the adaptive_low/high_freq for the cpu specified. This will impact all + * cpus governed by the policy (e.g. all cpus in a cluster). The actual value used + * for adaptive frequencies will be governed by the user space setting for the + * policy, and this value. + * + * Return: 0 if successful, error otherwise + */ +int cpufreq_walt_set_adaptive_freq(unsigned int cpu, unsigned int adaptive_low_freq, + unsigned int adaptive_high_freq) +{ + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + struct cpufreq_policy *policy = wg_policy->policy; + + if (!cpu_possible(cpu)) + return -EFAULT; + + if (policy->min <= adaptive_low_freq && policy->max >= adaptive_high_freq) { + wg_policy->tunables->adaptive_low_freq_kernel = adaptive_low_freq; + wg_policy->tunables->adaptive_high_freq_kernel = adaptive_high_freq; + return 0; + } + + return -EINVAL; +} +EXPORT_SYMBOL(cpufreq_walt_set_adaptive_freq); + +/** + * cpufreq_walt_get_adaptive_freq() - get the waltgov adaptive freq for cpu + * @cpu: the cpu for which the values should be returned + * @adaptive_low_freq: pointer to write the current kernel adaptive_low_freq value + * @adaptive_high_freq:pointer to write the current kernel adaptive_high_freq value + * + * Get the currently active adaptive_low/high_freq for the cpu specified. + * + * Return: 0 if successful, error otherwise + */ +int cpufreq_walt_get_adaptive_freq(unsigned int cpu, unsigned int *adaptive_low_freq, + unsigned int *adaptive_high_freq) +{ + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + + if (!cpu_possible(cpu)) + return -EFAULT; + + if (adaptive_low_freq && adaptive_high_freq) { + *adaptive_low_freq = get_adaptive_low_freq(wg_policy); + *adaptive_high_freq = get_adaptive_high_freq(wg_policy); + return 0; + } + + return -EINVAL; +} +EXPORT_SYMBOL(cpufreq_walt_get_adaptive_freq); + +/** + * cpufreq_walt_reset_adaptive_freq() - reset the waltgov adaptive freq for cpu + * @cpu: the cpu for which the values should be set + * + * Reset the kernel adaptive_low/high_freq to zero. + * + * Return: 0 if successful, error otherwise + */ +int cpufreq_walt_reset_adaptive_freq(unsigned int cpu) +{ + struct waltgov_cpu *wg_cpu = &per_cpu(waltgov_cpu, cpu); + struct waltgov_policy *wg_policy = wg_cpu->wg_policy; + + if (!cpu_possible(cpu)) + return -EFAULT; + + wg_policy->tunables->adaptive_low_freq_kernel = 0; + wg_policy->tunables->adaptive_high_freq_kernel = 0; + + return 0; +} +EXPORT_SYMBOL(cpufreq_walt_reset_adaptive_freq); + #define WALTGOV_ATTR_RW(_name) \ static struct governor_attr _name = \ __ATTR(_name, 0644, show_##_name, store_##_name) \ @@ -822,6 +925,8 @@ static void waltgov_tunables_save(struct cpufreq_policy *policy, cached->boost = tunables->boost; cached->adaptive_low_freq = tunables->adaptive_low_freq; cached->adaptive_high_freq = tunables->adaptive_high_freq; + cached->adaptive_low_freq_kernel = tunables->adaptive_low_freq_kernel; + cached->adaptive_high_freq_kernel = tunables->adaptive_high_freq_kernel; cached->target_load_thresh = tunables->target_load_thresh; cached->target_load_shift = tunables->target_load_shift; } @@ -844,6 +949,8 @@ static void waltgov_tunables_restore(struct cpufreq_policy *policy) tunables->boost = cached->boost; tunables->adaptive_low_freq = cached->adaptive_low_freq; tunables->adaptive_high_freq = cached->adaptive_high_freq; + tunables->adaptive_low_freq_kernel = cached->adaptive_low_freq_kernel; + tunables->adaptive_high_freq_kernel = cached->adaptive_high_freq_kernel; tunables->target_load_thresh = cached->target_load_thresh; tunables->target_load_shift = cached->target_load_shift; } diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 0e8439d33799..38b0c731eae4 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -19,6 +19,7 @@ static int __maybe_unused two = 2; static int __maybe_unused four = 4; static int one_hundred = 100; static int one_thousand = 1000; +static int one_thousand_twenty_four = 1024; static int two_thousand = 2000; /* @@ -47,6 +48,8 @@ unsigned int sysctl_sched_wake_up_idle[2]; unsigned int sysctl_input_boost_ms; unsigned int sysctl_input_boost_freq[8]; unsigned int sysctl_sched_boost_on_input; +unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS]; +unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS]; /* sysctl nodes accesed by other files */ unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns; @@ -75,6 +78,8 @@ unsigned int sysctl_sched_long_running_rt_task_ms; unsigned int sysctl_sched_idle_enough; unsigned int sysctl_sched_cluster_util_thres_pct; unsigned int sysctl_ed_boost_pct; +unsigned int sysctl_em_inflate_pct = 100; +unsigned int sysctl_em_inflate_thres = 1024; /* range is [1 .. INT_MAX] */ static int sysctl_task_read_pid = 1; @@ -457,6 +462,69 @@ int sched_updown_migrate_handler(struct ctl_table *table, int write, /* update individual cpu thresholds */ sched_update_updown_migrate_values(data == &sysctl_sched_capacity_margin_up_pct[0]); +unlock_mutex: + mutex_unlock(&mutex); + + return ret; +} + +int sched_updown_early_migrate_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret, i; + unsigned int *data = (unsigned int *)table->data; + static DEFINE_MUTEX(mutex); + int cap_margin_levels = num_sched_clusters ? num_sched_clusters - 1 : 0; + int val[MAX_MARGIN_LEVELS]; + struct ctl_table tmp = { + .data = &val, + .maxlen = sizeof(int) * cap_margin_levels, + .mode = table->mode, + }; + + if (cap_margin_levels <= 0) + return -EINVAL; + + mutex_lock(&mutex); + + if (!write) { + ret = proc_dointvec(table, write, buffer, lenp, ppos); + goto unlock_mutex; + } + + ret = proc_dointvec(&tmp, write, buffer, lenp, ppos); + if (ret) + goto unlock_mutex; + + for (i = 0; i < cap_margin_levels; i++) { + if (val[i] < 1024) { + ret = -EINVAL; + goto unlock_mutex; + } + } + + /* check up pct is greater than dn pct */ + if (data == &sysctl_sched_early_up[0]) { + for (i = 0; i < cap_margin_levels; i++) { + if (val[i] >= sysctl_sched_early_down[i]) { + ret = -EINVAL; + goto unlock_mutex; + } + } + } else { + for (i = 0; i < cap_margin_levels; i++) { + if (sysctl_sched_early_up[i] >= val[i]) { + ret = -EINVAL; + goto unlock_mutex; + } + } + } + + /* all things checkout update the value */ + for (i = 0; i < cap_margin_levels; i++) + data[i] = val[i]; + unlock_mutex: mutex_unlock(&mutex); @@ -721,6 +789,20 @@ struct ctl_table walt_table[] = { .mode = 0644, .proc_handler = sched_updown_migrate_handler, }, + { + .procname = "sched_early_upmigrate", + .data = &sysctl_sched_early_up, + .maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS, + .mode = 0644, + .proc_handler = sched_updown_early_migrate_handler, + }, + { + .procname = "sched_early_downmigrate", + .data = &sysctl_sched_early_down, + .maxlen = sizeof(unsigned int) * MAX_MARGIN_LEVELS, + .mode = 0644, + .proc_handler = sched_updown_early_migrate_handler, + }, { .procname = "walt_rtg_cfs_boost_prio", .data = &sysctl_walt_rtg_cfs_boost_prio, @@ -932,6 +1014,24 @@ struct ctl_table walt_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &one_hundred, }, + { + .procname = "sched_em_inflate_pct", + .data = &sysctl_em_inflate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = &one_hundred, + .extra2 = &one_thousand, + }, + { + .procname = "sched_em_inflate_thres", + .data = &sysctl_em_inflate_thres, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand_twenty_four, + }, { } }; @@ -951,6 +1051,8 @@ void walt_tunables(void) for (i = 0; i < MAX_MARGIN_LEVELS; i++) { sysctl_sched_capacity_margin_up_pct[i] = 95; /* ~5% margin */ sysctl_sched_capacity_margin_dn_pct[i] = 85; /* ~15% margin */ + sysctl_sched_early_up[i] = 1077; + sysctl_sched_early_down[i] = 1204; } sysctl_sched_group_upmigrate_pct = 100; diff --git a/kernel/sched/walt/trace.h b/kernel/sched/walt/trace.h index d260c63a9609..bed2f38942e0 100644 --- a/kernel/sched/walt/trace.h +++ b/kernel/sched/walt/trace.h @@ -1055,6 +1055,28 @@ TRACE_EVENT(sched_compute_energy, __entry->cluster_first_cpu2, __entry->s2, __entry->m2, __entry->c2) ) +TRACE_EVENT(sched_select_task_rt, + + TP_PROTO(struct task_struct *p, int fastpath), + + TP_ARGS(p, fastpath), + + TP_STRUCT__entry( + __field(int, pid) + __array(char, comm, TASK_COMM_LEN) + __field(int, fastpath) + ), + + TP_fast_assign( + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->fastpath = fastpath; + ), + + TP_printk("pid=%d comm=%s fastpath=%u", + __entry->pid, __entry->comm, __entry->fastpath) +); + TRACE_EVENT(sched_task_util, TP_PROTO(struct task_struct *p, unsigned long candidates, diff --git a/kernel/sched/walt/walt.c b/kernel/sched/walt/walt.c index 3defd6833e3c..2bba5fa165f3 100644 --- a/kernel/sched/walt/walt.c +++ b/kernel/sched/walt/walt.c @@ -464,7 +464,7 @@ static void update_task_cpu_cycles(struct task_struct *p, int cpu, static inline bool is_ed_enabled(void) { - return (walt_rotation_enabled || (boost_policy != SCHED_BOOST_NONE)); + return (boost_policy != SCHED_BOOST_NONE); } static inline bool is_ed_task(struct task_struct *p, u64 wallclock) @@ -652,6 +652,7 @@ __cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load, unsigned int *rea walt_load->pl = pl; walt_load->ws = walt_load_reported_window; walt_load->rtgb_active = rtgb_active; + walt_load->big_task_rotation = walt_rotation_enabled; if (wrq->ed_task) walt_load->ed_active = true; else diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index 42dd9a79ab99..9e074bcc5a14 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -60,6 +60,7 @@ struct walt_cpu_load { bool rtgb_active; u64 ws; bool ed_active; + bool big_task_rotation; }; #define DECLARE_BITMAP_ARRAY(name, nr, bits) \ @@ -219,6 +220,14 @@ extern unsigned int sysctl_sched_conservative_pl; extern unsigned int sysctl_sched_hyst_min_coloc_ns; extern unsigned int sysctl_sched_long_running_rt_task_ms; extern unsigned int sysctl_ed_boost_pct; +extern unsigned int sysctl_em_inflate_pct; +extern unsigned int sysctl_em_inflate_thres; + +extern int cpufreq_walt_set_adaptive_freq(unsigned int cpu, unsigned int adaptive_low_freq, + unsigned int adaptive_high_freq); +extern int cpufreq_walt_get_adaptive_freq(unsigned int cpu, unsigned int *adaptive_low_freq, + unsigned int *adaptive_high_freq); +extern int cpufreq_walt_reset_adaptive_freq(unsigned int cpu); #define WALT_MANY_WAKEUP_DEFAULT 1000 extern unsigned int sysctl_sched_many_wakeup_threshold; @@ -307,6 +316,7 @@ extern unsigned int sched_lib_mask_force; #define WALT_CPUFREQ_BOOST_UPDATE 0x20 #define CPUFREQ_REASON_LOAD 0 +#define CPUFREQ_REASON_BTR 0x1 #define CPUFREQ_REASON_PL 0x2 #define CPUFREQ_REASON_EARLY_DET 0x4 #define CPUFREQ_REASON_RTG_BOOST 0x8 @@ -740,6 +750,15 @@ static inline unsigned int walt_nr_rtg_high_prio(int cpu) return wrq->walt_stats.nr_rtg_high_prio_tasks; } +static inline bool task_in_related_thread_group(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (rcu_access_pointer(wts->grp) != NULL); +} + +extern unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS]; +extern unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS]; static inline bool task_fits_capacity(struct task_struct *p, long capacity, int cpu) @@ -751,21 +770,27 @@ static inline bool task_fits_capacity(struct task_struct *p, /* * Derive upmigration/downmigrate margin wrt the src/dest CPU. */ - if (src_wrq->cluster->id > dst_wrq->cluster->id) + if (src_wrq->cluster->id > dst_wrq->cluster->id) { margin = sched_capacity_margin_down[cpu]; - else + if (task_in_related_thread_group(p)) { + if (is_min_cluster_cpu(cpu)) + margin = sysctl_sched_early_down[0]; + else if (!is_max_cluster_cpu(cpu)) + margin = sysctl_sched_early_down[1]; + } + } else { margin = sched_capacity_margin_up[task_cpu(p)]; + if (task_in_related_thread_group(p)) { + if (is_min_cluster_cpu(task_cpu(p))) + margin = sysctl_sched_early_up[0]; + else if (!is_max_cluster_cpu(task_cpu(p))) + margin = sysctl_sched_early_up[1]; + } + } return capacity * 1024 > uclamp_task_util(p) * margin; } -static inline bool task_in_related_thread_group(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return (rcu_access_pointer(wts->grp) != NULL); -} - static inline bool task_fits_max(struct task_struct *p, int cpu) { unsigned long capacity = capacity_orig_of(cpu); @@ -985,7 +1010,6 @@ static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct struct walt_rq *wrq = (struct walt_rq *)rq->android_vendor_data1; struct walt_sched_cluster *cluster = wrq->cluster; cpumask_t unhalted_cpus; - cpumask_t cluster_32bit_cpus; int packing_cpu; /* if idle_enough feature is not enabled */ @@ -994,22 +1018,19 @@ static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct if (!sysctl_sched_cluster_util_thres_pct) return -1; - /* find all 32 bit capable cpus in this cluster */ - cpumask_and(&cluster_32bit_cpus, &cluster->cpus, system_32bit_el0_cpumask()); - /* pack 32 bit and 64 bit tasks on the same cpu, if possible */ - if (cpumask_weight(&cluster_32bit_cpus) > 0) { - packing_cpu = cpumask_first(&cluster_32bit_cpus); - } else { - /* find all unhalted active cpus */ - cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); + /* find all unhalted active cpus */ + cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask); - /* find all unhalted active cpus in this cluster */ - cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); + /* find all unhalted active cpus in this cluster */ + cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus); - /* return the first found unhalted, active cpu, in this cluster */ - packing_cpu = cpumask_first(&unhalted_cpus); - } + if (is_compat_thread(task_thread_info(p))) + /* try to find a packing cpu within 32 bit subset */ + cpumask_and(&unhalted_cpus, &unhalted_cpus, system_32bit_el0_cpumask()); + + /* return the first found unhalted, active cpu, in this cluster */ + packing_cpu = cpumask_first(&unhalted_cpus); /* packing cpu must be a valid cpu for runqueue lookup */ if (packing_cpu >= nr_cpu_ids) @@ -1031,6 +1052,10 @@ static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct if (task_util(p) >= sysctl_sched_idle_enough) return -1; + /* don't pack if running at a freq higher than 43.9pct of its fmax */ + if (arch_scale_freq_capacity(packing_cpu) > 450) + return -1; + /* the packing cpu can be used, so pack! */ return packing_cpu; } diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 6a44c6c9400c..f53e96478325 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -343,7 +343,8 @@ static void walt_find_best_target(struct sched_domain *sd, if (fbt_env->skip_cpu == i) continue; - if (wrq->num_mvp_tasks > 0) + if (wrq->num_mvp_tasks > 0 && + per_task_boost(p) != TASK_BOOST_STRICT_MAX) continue; /* @@ -568,6 +569,16 @@ cpu_util_next_walt_prs(int cpu, struct task_struct *p, int dst_cpu, bool prev_ds return util; } +static inline unsigned long get_util_to_cost(int cpu, unsigned long util) +{ + struct walt_rq *wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + + if (cpu == 0 && util > sysctl_em_inflate_thres) + return mult_frac(wrq->cluster->util_to_cost[util], sysctl_em_inflate_pct, 100); + else + return wrq->cluster->util_to_cost[util]; +} + /** * walt_em_cpu_energy() - Estimates the energy consumed by the CPUs of a performance domain @@ -586,9 +597,8 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, unsigned long max_util, unsigned long sum_util, struct compute_energy_output *output, unsigned int x) { - unsigned long scale_cpu; + unsigned long scale_cpu, cost; int cpu; - struct walt_rq *wrq; if (!sum_util) return 0; @@ -651,14 +661,14 @@ static inline unsigned long walt_em_cpu_energy(struct em_perf_domain *pd, if (max_util >= 1024) max_util = 1023; - wrq = (struct walt_rq *) cpu_rq(cpu)->android_vendor_data1; + cost = get_util_to_cost(cpu, max_util); if (output) { - output->cost[x] = wrq->cluster->util_to_cost[max_util]; + output->cost[x] = cost; output->max_util[x] = max_util; output->sum_util[x] = sum_util; } - return wrq->cluster->util_to_cost[max_util] * sum_util / scale_cpu; + return cost * sum_util / scale_cpu; } /* @@ -987,21 +997,23 @@ static void walt_binder_low_latency_set(void *unused, struct task_struct *task, if (unlikely(walt_disabled)) return; - if (task && current->signal && - (current->signal->oom_score_adj == 0) && - ((current->prio < DEFAULT_PRIO) || - (task->group_leader->prio < MAX_RT_PRIO))) + if (task && ((task_in_related_thread_group(current) && + task->group_leader->prio < MAX_RT_PRIO) || + (current->group_leader->prio < MAX_RT_PRIO && + task_in_related_thread_group(task)))) wts->low_latency |= WALT_LOW_LATENCY_BINDER; -} - -static void walt_binder_low_latency_clear(void *unused, struct binder_transaction *t) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) current->android_vendor_data1; - - if (unlikely(walt_disabled)) - return; - if (wts->low_latency & WALT_LOW_LATENCY_BINDER) + else + /* + * Clear low_latency flag if criterion above is not met, this + * will handle usecase where for a binder thread WALT_LOW_LATENCY_BINDER + * is set by one task and before WALT clears this flag after timer expiry + * some other task tries to use same binder thread. + * + * The only gets cleared when binder transaction is initiated + * and the above condition to set flasg is nto satisfied. + */ wts->low_latency &= ~WALT_LOW_LATENCY_BINDER; + } static void binder_set_priority_hook(void *data, @@ -1364,7 +1376,6 @@ void walt_cfs_init(void) register_trace_android_rvh_select_task_rq_fair(walt_select_task_rq_fair, NULL); register_trace_android_vh_binder_wakeup_ilocked(walt_binder_low_latency_set, NULL); - register_trace_binder_transaction_received(walt_binder_low_latency_clear, NULL); register_trace_android_vh_binder_set_priority(binder_set_priority_hook, NULL); register_trace_android_vh_binder_restore_priority(binder_restore_priority_hook, NULL); diff --git a/kernel/sched/walt/walt_halt.c b/kernel/sched/walt/walt_halt.c index 663efed493c2..48a82da53cf5 100644 --- a/kernel/sched/walt/walt_halt.c +++ b/kernel/sched/walt/walt_halt.c @@ -500,17 +500,18 @@ static void android_rvh_get_nohz_timer_target(void *unused, int *cpu, bool *done rcu_read_unlock(); } -static void android_rvh_set_cpus_allowed_ptr_locked(void *unused, - const struct cpumask *cpu_valid_mask, - const struct cpumask *new_mask, - unsigned int *dest_cpu) +static void android_rvh_set_cpus_allowed_by_task(void *unused, + const struct cpumask *cpu_valid_mask, + const struct cpumask *new_mask, + struct task_struct *p, + unsigned int *dest_cpu) { cpumask_t allowed_cpus; if (unlikely(walt_disabled)) return; - if (cpu_halted(*dest_cpu)) { + if (cpu_halted(*dest_cpu) && !p->migration_disabled) { /* remove halted cpus from the valid mask, and store locally */ cpumask_andnot(&allowed_cpus, cpu_valid_mask, cpu_halt_mask); *dest_cpu = cpumask_any_and_distribute(&allowed_cpus, new_mask); @@ -573,8 +574,8 @@ void walt_halt_init(void) sched_setscheduler_nocheck(walt_drain_thread, SCHED_FIFO, ¶m); register_trace_android_rvh_get_nohz_timer_target(android_rvh_get_nohz_timer_target, NULL); - register_trace_android_rvh_set_cpus_allowed_ptr_locked( - android_rvh_set_cpus_allowed_ptr_locked, NULL); + register_trace_android_rvh_set_cpus_allowed_by_task( + android_rvh_set_cpus_allowed_by_task, NULL); register_trace_android_rvh_rto_next_cpu(android_rvh_rto_next_cpu, NULL); register_trace_android_rvh_is_cpu_allowed(android_rvh_is_cpu_allowed, NULL); diff --git a/kernel/sched/walt/walt_rt.c b/kernel/sched/walt/walt_rt.c index 3265be5ba664..18d531791ff2 100644 --- a/kernel/sched/walt/walt_rt.c +++ b/kernel/sched/walt/walt_rt.c @@ -225,6 +225,14 @@ static inline bool walt_should_honor_rt_sync(struct rq *rq, struct task_struct * rq->rt.rt_nr_running <= 2; } +enum rt_fastpaths { + NONE = 0, + NON_WAKEUP, + SYNC_WAKEUP, + CLUSTER_PACKING_FASTPATH, +}; + + static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int cpu, int sd_flag, int wake_flags, int *new_cpu) { @@ -235,13 +243,16 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c int ret, target = -1, this_cpu; struct cpumask *lowest_mask; int packing_cpu; + int fastpath = NONE; if (unlikely(walt_disabled)) return; /* For anything but wake ups, just return the task_cpu */ - if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) - return; + if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) { + fastpath = NON_WAKEUP; + goto out; + } this_cpu = raw_smp_processor_id(); this_cpu_rq = cpu_rq(this_cpu); @@ -252,8 +263,9 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c if (sysctl_sched_sync_hint_enable && cpu_active(this_cpu) && !cpu_halted(this_cpu) && cpumask_test_cpu(this_cpu, task->cpus_ptr) && walt_should_honor_rt_sync(this_cpu_rq, task, sync)) { + fastpath = SYNC_WAKEUP; *new_cpu = this_cpu; - return; + goto out; } *new_cpu = cpu; /* previous CPU as back up */ @@ -292,6 +304,7 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c /* create a fastpath for finding a packing cpu */ packing_cpu = walt_find_and_choose_cluster_packing_cpu(task_cpu(task), task); if (packing_cpu >= 0) { + fastpath = CLUSTER_PACKING_FASTPATH; *new_cpu = packing_cpu; goto unlock; } @@ -320,6 +333,8 @@ static void walt_select_task_rq_rt(void *unused, struct task_struct *task, int c } unlock: rcu_read_unlock(); +out: + trace_sched_select_task_rt(task, fastpath); }