github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-31 02:24:24 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

x86/percpu: Move pcpu_hot to percpu hot section

Browse Source 
Also change the alignment of the percpu hot section:

 -       PERCPU_SECTION(INTERNODE_CACHE_BYTES)
 +       PERCPU_SECTION(L1_CACHE_BYTES)

As vSMP will muck with INTERNODE_CACHE_BYTES that invalidates the
too-large-section assert we do:

  ASSERT(__per_cpu_hot_end - __per_cpu_hot_start <= 64, "percpu cache hot section too large")

[ mingo: Added INTERNODE_CACHE_BYTES fix & explanation. ]

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250303165246.2175811-3-brgerst@gmail.com
This commit is contained in:
Brian Gerst 2025-03-03 11:52:37 -05:00 committed by Ingo Molnar
parent ab2bb9c084
commit 972f9cdff9
3 changed files with 16 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
arch/x86/include/asm/current.h
View File

@ -13,32 +13,26 @@
struct task_struct;
struct pcpu_hot {
union {
struct {
struct task_struct *current_task;
int preempt_count;
int cpu_number;
struct task_struct *current_task;
int preempt_count;
int cpu_number;
#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
u64 call_depth;
u64 call_depth;
#endif
unsigned long top_of_stack;
void *hardirq_stack_ptr;
u16 softirq_pending;
unsigned long top_of_stack;
void *hardirq_stack_ptr;
u16 softirq_pending;
#ifdef CONFIG_X86_64
bool hardirq_stack_inuse;
bool hardirq_stack_inuse;
#else
void *softirq_stack_ptr;
void *softirq_stack_ptr;
#endif
};
u8 pad[64];
};
};
static_assert(sizeof(struct pcpu_hot) == 64);
DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
DECLARE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot);
/* const-qualified alias to pcpu_hot, aliased by linker. */
DECLARE_PER_CPU_ALIGNED(const struct pcpu_hot __percpu_seg_override,
DECLARE_PER_CPU_CACHE_HOT(const struct pcpu_hot __percpu_seg_override,
const_pcpu_hot);
static __always_inline struct task_struct *get_current(void)

2
arch/x86/kernel/cpu/common.c
View File

@ -2064,7 +2064,7 @@ static __init int setup_setcpuid(char *arg)
}
__setup("setcpuid=", setup_setcpuid);
DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
DEFINE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot) = {
.current_task = &init_task,
.preempt_count = INIT_PREEMPT_COUNT,
.top_of_stack = TOP_OF_INIT_STACK,

5
arch/x86/kernel/vmlinux.lds.S
View File

@ -187,6 +187,8 @@ SECTIONS
PAGE_ALIGNED_DATA(PAGE_SIZE)
CACHE_HOT_DATA(L1_CACHE_BYTES)
CACHELINE_ALIGNED_DATA(L1_CACHE_BYTES)
DATA_DATA
@ -327,7 +329,8 @@ SECTIONS
EXIT_DATA
}
PERCPU_SECTION(INTERNODE_CACHE_BYTES)
PERCPU_SECTION(L1_CACHE_BYTES)
ASSERT(__per_cpu_hot_end - __per_cpu_hot_start <= 64, "percpu cache hot section too large")
RUNTIME_CONST_VARIABLES
RUNTIME_CONST(ptr, USER_PTR_MAX)