LoongArch: Add THREAD_INFO_IN_TASK implementation

Like other architectures such as x86, arm64, riscv, powerpc and s390,
select THREAD_INFO_IN_TASK for LoongArch to move thread_info off the
stack into task_struct. This follows modern kernel standards and also
makes the system more secure.

With this patch, thread_info is included in task_struct at an offset
of 0 instead of being placed at the bottom of the kernel stack. Thus,
the $tp register points to both thread_info and task_struct.

To support this, introduce a per-CPU variable cpu_tasks to store the
pointer to the current task_struct. This decouples the recovery	of the
$tp register from the stack pointer during exception entry.

Then initialize cpu_tasks for the primary and secondary CPUs during
arch-specific setup and SMP boot paths. To eliminate the dangerous
windows during the early initialization where the cpu_tasks remains
uninitialized, set_current() is invoked as early as possible in both
setup_arch() and start_secondary(). This ensures the $tp recovery
barrier is armed in case any early boot exceptions or kernel panics
occur.

Modify SAVE_SOME and handle_syscall to restore the $tp register from
cpu_tasks, and also use the la_abs absolute addressing for cpu_tasks
access in assembly to bypass the relocation limits within exception
handling sections. By advancing the preservation of u0 in SAVE_SOME,
we reuse the PERCPU_BASE_KS value in u0 for the cpu_tasks calculation,
effectively eliminating a duplicate csrrd instruction execution on SMP
platforms.

Update <asm/switch_to.h> and <kernel/switch.S> to fully support the
CONFIG_THREAD_INFO_IN_TASK feature.

Remove the obsolete next_ti argument from __switch_to(), which shifts
the remaining arguments ahead in the calling convention (sched_ra from
a3 to a2, and sched_cfa from a4 to a3). Under the new configuration,
__switch_to() now directly derives the thread pointer ($tp) from the
next task_struct pointer in a1.

To preserve the optimal and clean "move tp, a1" path for 64-bit kernels,
the thread pointer ($tp) is assigned directly from a1 in the core path.
For 32-bit kernels, where a1 carries a 2000-byte structural pointer bias
at entry, an explicit adjustment "PTR_ADDI tp, tp, -TASK_STRUCT_OFFSET"
is introduced at the function exit.

In the context of __switch_to(), local interrupts are disabled, and the
kernel is in a critical switching phase where handling any synchronous
exception is practically impossible and prohibited.

If any synchronous exception or watchpoint does trigger in this narrow
window, it constitutes a fatal double fault and the kernel is expected
to die/panic immediately anyway. Therefore, the temporary biased value
in $tp is safe and acceptable here.

Additionally, evaluate the stack lookup as a single load instruction
"LONG_LPTR t0, a1, (TASK_STACK - TASK_STRUCT_OFFSET)", this perfectly
satisfies both 32-bit and 64-bit kernels. Using the "next" pointer in
a1 as the base register, rather than $tp, effectively unchains the data
dependency (RAW hazard) from the preceding move instruction, maximizing
the instruction-level parallelism and superscalar execution efficiency
while naturally adapting the structural shift.

With CONFIG_THREAD_INFO_IN_TASK enabled, the kernel stack life cycle is
decoupled from task_struct and can be freed concurrently.

Currently, show_stacktrace() reads raw stack data via __get_addr() and
subsequently calls show_backtrace() to unwind the frame, without holding
any reference to the target task's stack. If show_stacktrace() is called
on a concurrently exiting task, it could attempt to read from a freed or
reallocated kernel stack. This introduces a severe use-after-free (UAF)
read risk or kernel panics.

Wrap the entire stack inspection process inside show_stacktrace() with
a try_get_task_stack() and put_task_stack() pair. This ensures the task
stack remains pinned safely during both the raw stack data dump loop and
the subsequent stack unwinding phase.

Also, ensure that the task pointer is initialized to "current" early if
it is NULL, so that try_get_task_stack() always operates on a valid task
reference.

Signed-off-by: Tiezhu Yang <yangtiezhu@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

Documentation/features/core/thread-info-in-task/arch-support.txt

@@ -12,7 +12,7 @@
     |       arm64: |  ok  |
     |        csky: | TODO |
     |     hexagon: | TODO |
-    |   loongarch: | TODO |
+    |   loongarch: |  ok  |
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |

arch/loongarch/Kconfig

@@ -210,6 +210,7 @@ config LOONGARCH
 	select SYSCTL_ARCH_UNALIGN_NO_WARN
 	select SYSCTL_EXCEPTION_TRACE
 	select SWIOTLB if 64BIT
+	select THREAD_INFO_IN_TASK
 	select TRACE_IRQFLAGS_SUPPORT
 	select USE_PERCPU_NUMA_NODE_ID
 	select USER_STACKTRACE_SUPPORT

arch/loongarch/include/asm/current.h

@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_LOONGARCH_CURRENT_H
+#define __ASM_LOONGARCH_CURRENT_H
+
+#include <linux/compiler.h>
+
+#ifndef __ASSEMBLER__
+
+#include <asm/percpu.h>
+
+struct task_struct;
+
+DECLARE_PER_CPU(struct task_struct *, cpu_tasks);
+
+register struct task_struct *current_thread_pointer __asm__("$tp");
+
+static __always_inline struct task_struct *get_current(void)
+{
+	return current_thread_pointer;
+}
+
+#define current get_current()
+
+static __always_inline void set_current(struct task_struct *task)
+{
+	__this_cpu_write(cpu_tasks, task);
+}
+
+#endif /* __ASSEMBLER__ */
+
+#endif /* __ASM_LOONGARCH_CURRENT_H */

arch/loongarch/include/asm/percpu.h

@@ -5,7 +5,6 @@
 #ifndef __ASM_PERCPU_H
 #define __ASM_PERCPU_H
 
-#include <asm/cmpxchg.h>
 #include <asm/loongarch.h>
 
 /*

arch/loongarch/include/asm/smp.h

@@ -81,8 +81,9 @@ extern int __cpu_logical_map[NR_CPUS];
 struct seq_file;
 
 struct secondary_data {
+	unsigned long task;
 	unsigned long stack;
-	unsigned long thread_info;
+	unsigned long offset;
 };
 extern struct secondary_data cpuboot_data;
 

arch/loongarch/include/asm/stackframe.h

@@ -184,6 +184,7 @@
 	.cfi_rel_offset ra, PT_ERA
 	.endif
 	cfi_st	tp, PT_R2, \docfi
+	cfi_st  u0, PT_R21, \docfi
 	cfi_st	fp, PT_R22, \docfi
 
 	/* Set thread_info if we're coming from user mode */
@@ -191,10 +192,13 @@
 	andi	t0, t0, 0x3	/* extract pplv bit */
 	beqz	t0, 9f
 
-	LONG_LI	tp, ~_THREAD_MASK
-	and	tp, tp, sp
-	cfi_st  u0, PT_R21, \docfi
 	csrrd	u0, PERCPU_BASE_KS
+
+	la_abs	t1, cpu_tasks
+#ifdef CONFIG_SMP
+	LONG_ADD t1, t1, u0
+#endif
+	LONG_L	tp, t1, 0
 9:
 #ifdef CONFIG_KGDB
 	li.w	t0, CSR_CRMD_WE

arch/loongarch/include/asm/switch_to.h

@@ -15,7 +15,6 @@ struct task_struct;
  * __switch_to - switch execution of a task
  * @prev:	The task previously executed.
  * @next:	The task to begin executing.
- * @next_ti:	task_thread_info(next).
  * @sched_ra:	__schedule return address.
  * @sched_cfa:	__schedule call frame address.
  *
@@ -23,8 +22,7 @@ struct task_struct;
  * the context of next. Returns prev.
  */
 extern asmlinkage struct task_struct *__switch_to(struct task_struct *prev,
-			struct task_struct *next, struct thread_info *next_ti,
-			void *sched_ra, void *sched_cfa);
+			struct task_struct *next, void *sched_ra, void *sched_cfa);
 
 /*
  * For newly created kernel threads switch_to() will return to
@@ -37,7 +35,8 @@ do {										\
 	lose_fpu_inatomic(1, prev);						\
 	lose_lbt_inatomic(1, prev);						\
 	hw_breakpoint_thread_switch(next);					\
-	(last) = __switch_to(prev, next, task_thread_info(next),		\
+	set_current(next);							\
+	(last) = __switch_to(prev, next,					\
 		 __builtin_return_address(0), __builtin_frame_address(0));	\
 } while (0)
 

arch/loongarch/include/asm/thread_info.h

@@ -22,7 +22,6 @@
  *   must also be changed
  */
 struct thread_info {
-	struct task_struct	*task;		/* main task structure */
 	unsigned long		flags;		/* low level flags */
 	unsigned long		tp_value;	/* thread pointer */
 	__u32			cpu;		/* current CPU */
@@ -37,20 +36,11 @@ struct thread_info {
  */
 #define INIT_THREAD_INFO(tsk)			\
 {						\
-	.task		= &tsk,			\
 	.flags		= _TIF_FIXADE,		\
 	.cpu		= 0,			\
 	.preempt_count	= INIT_PREEMPT_COUNT,	\
 }
 
-/* How to get the thread information struct from C. */
-register struct thread_info *__current_thread_info __asm__("$tp");
-
-static inline struct thread_info *current_thread_info(void)
-{
-	return __current_thread_info;
-}
-
 register unsigned long current_stack_pointer __asm__("$sp");
 
 #endif /* !__ASSEMBLER__ */

arch/loongarch/kernel/asm-offsets.c

@@ -70,7 +70,7 @@ static void __used output_task_defines(void)
 {
 	COMMENT("LoongArch task_struct offsets.");
 	OFFSET(TASK_STATE, task_struct, __state);
-	OFFSET(TASK_THREAD_INFO, task_struct, stack);
+	OFFSET(TASK_STACK, task_struct, stack);
 	OFFSET(TASK_FLAGS, task_struct, flags);
 	OFFSET(TASK_MM, task_struct, mm);
 	OFFSET(TASK_PID, task_struct, pid);
@@ -84,7 +84,6 @@ static void __used output_task_defines(void)
 static void __used output_thread_info_defines(void)
 {
 	COMMENT("LoongArch thread_info offsets.");
-	OFFSET(TI_TASK, thread_info, task);
 	OFFSET(TI_FLAGS, thread_info, flags);
 	OFFSET(TI_TP_VALUE, thread_info, tp_value);
 	OFFSET(TI_CPU, thread_info, cpu);
@@ -266,8 +265,9 @@ static void __used output_signal_defines(void)
 static void __used output_smpboot_defines(void)
 {
 	COMMENT("Linux smp cpu boot offsets.");
+	OFFSET(CPU_BOOT_TASK, secondary_data, task);
 	OFFSET(CPU_BOOT_STACK, secondary_data, stack);
-	OFFSET(CPU_BOOT_TINFO, secondary_data, thread_info);
+	OFFSET(CPU_BOOT_OFFSET, secondary_data, offset);
 	BLANK();
 }
 #endif

arch/loongarch/kernel/entry.S

@@ -67,8 +67,11 @@ SYM_CODE_START(handle_syscall)
 #endif
 
 	move		u0, t0
-	LONG_LI		tp, ~_THREAD_MASK
-	and		tp, tp, sp
+	la_abs		t1, cpu_tasks
+#ifdef CONFIG_SMP
+	LONG_ADD	t1, t1, u0
+#endif
+	LONG_L		tp, t1, 0
 
 	move		a0, sp
 	bl		do_syscall

arch/loongarch/kernel/head.S

@@ -74,10 +74,11 @@ SYM_CODE_START(kernel_entry)			# kernel entry point
 	/* GPR21 used for percpu base (runtime), initialized as 0 */
 	move		u0, zero
 
-	la.pcrel	tp, init_thread_union
-	/* Set the SP after an empty pt_regs.  */
-	PTR_LI		sp, (_THREAD_SIZE - PT_SIZE)
-	PTR_ADD		sp, sp, tp
+	la.pcrel	tp, init_task
+	la.pcrel	t0, init_stack
+	PTR_LI		t1, _THREAD_SIZE
+	PTR_ADD		t0, t0, t1
+	PTR_ADDI	sp, t0, -PT_SIZE
 	set_saved_sp	sp, t0, t1
 
 #ifdef CONFIG_RELOCATABLE
@@ -86,8 +87,10 @@ SYM_CODE_START(kernel_entry)			# kernel entry point
 
 #ifdef CONFIG_RANDOMIZE_BASE
 	/* Repoint the sp into the new kernel */
-	PTR_LI		sp, (_THREAD_SIZE - PT_SIZE)
-	PTR_ADD		sp, sp, tp
+	LONG_LPTR	t0, tp, TASK_STACK
+	PTR_LI		t1, _THREAD_SIZE
+	PTR_ADD		t0, t0, t1
+	PTR_ADDI	sp, t0, -PT_SIZE
 	set_saved_sp	sp, t0, t1
 
 	/* Jump to the new kernel: new_pc = current_pc + random_offset */
@@ -127,8 +130,9 @@ SYM_CODE_START(smpboot_entry)
 	csrxchg		t0, t1, LOONGARCH_CSR_IMPCTL1
 #endif
 	la.pcrel	t0, cpuboot_data
+	ld.d		tp, t0, CPU_BOOT_TASK
 	ld.d		sp, t0, CPU_BOOT_STACK
-	ld.d		tp, t0, CPU_BOOT_TINFO
+	ld.d		u0, t0, CPU_BOOT_OFFSET
 
 	bl		start_secondary
 	ASM_BUG()

arch/loongarch/kernel/process.c

@@ -60,6 +60,8 @@ unsigned long __stack_chk_guard __read_mostly;
 EXPORT_SYMBOL(__stack_chk_guard);
 #endif
 
+DEFINE_PER_CPU(struct task_struct *, cpu_tasks);
+
 /*
  * Idle related variables and functions
  */

arch/loongarch/kernel/relocate.c

@@ -313,7 +313,7 @@ unsigned long __init relocate_kernel(void)
 		reloc_offset += random_offset;
 
 		/* The current thread is now within the relocated kernel */
-		__current_thread_info = RELOCATED_KASLR(__current_thread_info);
+		current_thread_pointer = RELOCATED_KASLR(current_thread_pointer);
 
 		update_reloc_offset(&reloc_offset, random_offset);
 	}

arch/loongarch/kernel/setup.c

@@ -593,6 +593,7 @@ void __init setup_arch(char **cmdline_p)
 {
 	cpu_probe();
 	unwind_init();
+	set_current(current);
 
 	init_environ();
 	efi_init();

arch/loongarch/kernel/smp.c

@@ -400,8 +400,9 @@ void loongson_boot_secondary(int cpu, struct task_struct *idle)
 	pr_info("Booting CPU#%d...\n", cpu);
 
 	entry = __pa_symbol((unsigned long)&smpboot_entry);
-	cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle);
-	cpuboot_data.thread_info = (unsigned long)task_thread_info(idle);
+	cpuboot_data.task = (unsigned long)idle;
+	cpuboot_data.stack = (unsigned long)task_pt_regs(idle);
+	cpuboot_data.offset = per_cpu_offset(cpu);
 
 	csr_mail_send(entry, cpu_logical_map(cpu), 0);
 
@@ -663,6 +664,7 @@ asmlinkage void start_secondary(void)
 	set_my_cpu_offset(per_cpu_offset(cpu));
 
 	cpu_probe();
+	set_current(current);
 	constant_clockevent_init();
 	loongson_init_secondary();
 

arch/loongarch/kernel/switch.S

@@ -12,7 +12,7 @@
 
 /*
  * task_struct *__switch_to(task_struct *prev, task_struct *next,
- *			    struct thread_info *next_ti, void *sched_ra, void *sched_cfa)
+ *			    void *sched_ra, void *sched_cfa)
  */
 	.align	5
 SYM_FUNC_START(__switch_to)
@@ -24,8 +24,8 @@ SYM_FUNC_START(__switch_to)
 	LONG_SPTR	t1, a0, (THREAD_CSRPRMD - TASK_STRUCT_OFFSET)
 
 	cpu_save_nonscratch a0
-	LONG_SPTR	a3, a0, (THREAD_SCHED_RA - TASK_STRUCT_OFFSET)
-	LONG_SPTR	a4, a0, (THREAD_SCHED_CFA - TASK_STRUCT_OFFSET)
+	LONG_SPTR	a2, a0, (THREAD_SCHED_RA - TASK_STRUCT_OFFSET)
+	LONG_SPTR	a3, a0, (THREAD_SCHED_CFA - TASK_STRUCT_OFFSET)
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP)
 	la		t7, __stack_chk_guard
@@ -33,11 +33,12 @@ SYM_FUNC_START(__switch_to)
 	LONG_SPTR	t8, t7, 0
 #endif
 
-	move	tp, a2
+	move		tp, a1
 	cpu_restore_nonscratch a1
 
-	li.w		t0, _THREAD_SIZE
-	PTR_ADD		t0, t0, tp
+	LONG_LPTR	t0, a1, (TASK_STACK - TASK_STRUCT_OFFSET)
+	PTR_LI		t1, _THREAD_SIZE
+	PTR_ADD		t0, t0, t1
 	set_saved_sp	t0, t1, t2
 
 	LONG_LPTR	t1, a1, (THREAD_CSRPRMD - TASK_STRUCT_OFFSET)
@@ -45,6 +46,7 @@ SYM_FUNC_START(__switch_to)
 
 #ifdef CONFIG_32BIT
 	PTR_ADDI	a0, a0, -TASK_STRUCT_OFFSET
+	PTR_ADDI	tp, tp, -TASK_STRUCT_OFFSET
 #endif
 	jr	ra
 SYM_FUNC_END(__switch_to)

arch/loongarch/kernel/traps.c

@@ -107,6 +107,12 @@ static void show_stacktrace(struct task_struct *task,
 	unsigned long stackdata;
 	unsigned long *sp = (unsigned long *)regs->regs[3];
 
+	if (!task)
+		task = current;
+
+	if (!try_get_task_stack(task))
+		return;
+
 	printk("%sStack :", loglvl);
 	i = 0;
 	while ((unsigned long) sp & (PAGE_SIZE - 1)) {
@@ -129,6 +135,8 @@ static void show_stacktrace(struct task_struct *task,
 	}
 	pr_cont("\n");
 	show_backtrace(task, regs, loglvl, user);
+
+	put_task_stack(task);
 }
 
 void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl)