diff --git a/kernel/sched/walt/sysctl.c b/kernel/sched/walt/sysctl.c index 5c0840029da3..03cef3c1c05c 100644 --- a/kernel/sched/walt/sysctl.c +++ b/kernel/sched/walt/sysctl.c @@ -186,6 +186,7 @@ enum { PER_TASK_BOOST, PER_TASK_BOOST_PERIOD_MS, LOW_LATENCY, + PIPELINE, }; static int sched_task_handler(struct ctl_table *table, int write, @@ -238,6 +239,10 @@ static int sched_task_handler(struct ctl_table *table, int write, pid_and_val[1] = wts->low_latency & WALT_LOW_LATENCY_PROCFS; break; + case PIPELINE: + pid_and_val[1] = wts->low_latency & + WALT_LOW_LATENCY_PIPELINE; + break; default: ret = -EINVAL; goto put_task; @@ -302,6 +307,12 @@ static int sched_task_handler(struct ctl_table *table, int write, else wts->low_latency &= ~WALT_LOW_LATENCY_PROCFS; break; + case PIPELINE: + if (val) + wts->low_latency |= WALT_LOW_LATENCY_PIPELINE; + else + wts->low_latency &= ~WALT_LOW_LATENCY_PIPELINE; + break; default: ret = -EINVAL; } @@ -823,6 +834,13 @@ struct ctl_table walt_table[] = { .mode = 0644, .proc_handler = sched_task_handler, }, + { + .procname = "sched_pipeline", + .data = (int *) PIPELINE, + .maxlen = sizeof(unsigned int) * 2, + .mode = 0644, + .proc_handler = sched_task_handler, + }, { .procname = "sched_task_read_pid", .data = &sysctl_task_read_pid, diff --git a/kernel/sched/walt/walt.h b/kernel/sched/walt/walt.h index ffd3e92c49e1..0fb5a831cba7 100644 --- a/kernel/sched/walt/walt.h +++ b/kernel/sched/walt/walt.h @@ -49,8 +49,9 @@ enum migrate_types { RQ_TO_GROUP, }; -#define WALT_LOW_LATENCY_PROCFS BIT(0) -#define WALT_LOW_LATENCY_BINDER BIT(1) +#define WALT_LOW_LATENCY_PROCFS BIT(0) +#define WALT_LOW_LATENCY_BINDER BIT(1) +#define WALT_LOW_LATENCY_PIPELINE BIT(2) struct walt_cpu_load { unsigned long nl; @@ -419,6 +420,48 @@ static inline unsigned long cpu_util_cum(int cpu) return READ_ONCE(cpu_rq(cpu)->cfs.avg.util_avg); } +/* applying the task threshold for all types of low latency tasks. */ +static inline bool walt_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->low_latency && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_binder_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_BINDER) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_procfs_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return (wts->low_latency & WALT_LOW_LATENCY_PROCFS) && + (task_util(p) < sysctl_walt_low_latency_task_threshold); +} + +static inline bool walt_pipeline_low_latency_task(struct task_struct *p) +{ + struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; + + return wts->low_latency & WALT_LOW_LATENCY_PIPELINE; +} + +static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) +{ + struct cpuidle_state *idle = idle_get_state(rq); + + if (idle) + return idle->exit_latency; + + return 0; /* CPU is not idle */ +} + static inline bool rt_boost_on_big(void) { return sched_boost_type == FULL_THROTTLE_BOOST ? @@ -477,7 +520,8 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) * under conservative boost. */ if (sched_boost_type == CONSERVATIVE_BOOST && - task_util(p) <= sysctl_sched_min_task_util_for_boost) + task_util(p) <= sysctl_sched_min_task_util_for_boost && + !walt_pipeline_low_latency_task(p)) policy = SCHED_BOOST_NONE; } @@ -725,41 +769,6 @@ static inline bool task_fits_max(struct task_struct *p, int cpu) return task_fits_capacity(p, capacity, cpu); } -/* applying the task threshold for all types of low latency tasks. */ -static inline bool walt_low_latency_task(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return wts->low_latency && - (task_util(p) < sysctl_walt_low_latency_task_threshold); -} - -static inline bool walt_binder_low_latency_task(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return (wts->low_latency & WALT_LOW_LATENCY_BINDER) && - (task_util(p) < sysctl_walt_low_latency_task_threshold); -} - -static inline bool walt_procfs_low_latency_task(struct task_struct *p) -{ - struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; - - return (wts->low_latency & WALT_LOW_LATENCY_PROCFS) && - (task_util(p) < sysctl_walt_low_latency_task_threshold); -} - -static inline unsigned int walt_get_idle_exit_latency(struct rq *rq) -{ - struct cpuidle_state *idle = idle_get_state(rq); - - if (idle) - return idle->exit_latency; - - return 0; /* CPU is not idle */ -} - extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); extern void sched_update_hyst_times(void); diff --git a/kernel/sched/walt/walt_cfs.c b/kernel/sched/walt/walt_cfs.c index 6288bb794375..13086c193a71 100644 --- a/kernel/sched/walt/walt_cfs.c +++ b/kernel/sched/walt/walt_cfs.c @@ -138,7 +138,8 @@ static inline bool walt_task_skip_min_cpu(struct task_struct *p) struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1; return (sched_boost_type != CONSERVATIVE_BOOST) && - walt_get_rtg_status(p) && wts->unfilter; + walt_get_rtg_status(p) && (wts->unfilter || + walt_pipeline_low_latency_task(p)); } static inline bool walt_is_many_wakeup(int sibling_count_hint) @@ -1007,7 +1008,8 @@ static inline int walt_get_mvp_task_prio(struct task_struct *p) if (walt_binder_low_latency_task(p)) return WALT_BINDER_MVP; - if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p)) + if (task_rtg_high_prio(p) || walt_procfs_low_latency_task(p) || + walt_pipeline_low_latency_task(p)) return WALT_RTG_MVP; return WALT_NOT_MVP; diff --git a/kernel/sched/walt/walt_lb.c b/kernel/sched/walt/walt_lb.c index 29e117b011ce..e760f83cf3c3 100644 --- a/kernel/sched/walt/walt_lb.c +++ b/kernel/sched/walt/walt_lb.c @@ -225,6 +225,8 @@ static inline bool _walt_can_migrate_task(struct task_struct *p, int dst_cpu, if (per_task_boost(p) == TASK_BOOST_STRICT_MAX && task_in_related_thread_group(p)) return false; + if (walt_pipeline_low_latency_task(p)) + return false; } /* Don't detach task if it is under active migration */