From 758c67bc727dc78b6de94644e07a5283482487c3 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 3 Jan 2017 18:27:01 +0000 Subject: [PATCH] UPSTREAM: arm64: restore get_current() optimisation commit 9d84fb27fa135c99c9fe3de33628774a336a70a8 upstream. Commit c02433dd6de32f04 ("arm64: split thread_info from task stack") inverted the relationship between get_current() and current_thread_info(), with sp_el0 now holding the current task_struct rather than the current thead_info. The new implementation of get_current() prevents the compiler from being able to optimize repeated calls to either, resulting in a noticeable penalty in some microbenchmarks. This patch restores the previous optimisation by implementing get_current() in the same way as our old current_thread_info(), using a non-volatile asm statement. Acked-by: Will Deacon Signed-off-by: Mark Rutland Reported-by: Davidlohr Bueso Signed-off-by: Catalin Marinas Signed-off-by: Amit Pundir --- arch/arm64/include/asm/current.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/current.h b/arch/arm64/include/asm/current.h index 2e61d21294ba..483a6c9d3e10 100644 --- a/arch/arm64/include/asm/current.h +++ b/arch/arm64/include/asm/current.h @@ -10,9 +10,17 @@ #ifdef CONFIG_THREAD_INFO_IN_TASK struct task_struct; +/* + * We don't use read_sysreg() as we want the compiler to cache the value where + * possible. + */ static __always_inline struct task_struct *get_current(void) { - return (struct task_struct *)read_sysreg(sp_el0); + unsigned long sp_el0; + + asm ("mrs %0, sp_el0" : "=r" (sp_el0)); + + return (struct task_struct *)sp_el0; } #define current get_current() #else