mirror of
https://github.com/torvalds/linux.git
synced 2026-10-08 11:36:02 +02:00
efebf64966
__pcpu_freelist_push() can loop forever when only one CPU is possible
and an NMI re-enters pcpu_freelist_push() while the interrupted context
holds that CPU's freelist lock.
After the current-CPU fast path fails, the fallback loop walks
cpu_possible_mask while skipping the current CPU. With CONFIG_SMP=n, or
when an SMP kernel is limited to one possible CPU with nr_cpus=1 or
possible_cpus=1, there are no other possible CPUs to examine. The loop
therefore makes no lock acquisition attempt and can never make progress.
The following stack was observed on a UP system:
NMI context:
pcpu_freelist_push
free_htab_elem
htab_map_delete_elem
[perf-event BPF program]
__perf_event_overflow
perf_event_nmi_handler
exc_nmi
Interrupted context:
__pcpu_freelist_push
pcpu_freelist_push
free_htab_elem
htab_map_delete_elem
[raw_tp/sys_enter BPF program]
__bpf_trace_sys_enter
do_syscall_64
raw_res_spin_lock() detects the same-CPU recursive acquisition and
returns -EDEADLK, but the subsequent fallback loop has no candidate head
on a system with one possible CPU.
Restore the extra fallback head that existed before the rqspinlock
conversion. Keep the current-CPU fast path, then try the other possible
CPUs and finally the extra head. The additional head lets a push, which
cannot fail without losing a preallocated element, make progress when the
only per-CPU head is held by the interrupted context.
Also check the extra head from the pop path so that nodes placed there
can be reused.
Fixes: f2ac0e5d1c ("bpf: Convert percpu_freelist.c to rqspinlock")
Signed-off-by: Hui Su <sh_def@163.com>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/bpf/20260806175600.1993595-1-sh_def@163.com
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
35 lines
1.1 KiB
C
35 lines
1.1 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/* Copyright (c) 2016 Facebook
|
|
*/
|
|
#ifndef __PERCPU_FREELIST_H__
|
|
#define __PERCPU_FREELIST_H__
|
|
#include <linux/spinlock.h>
|
|
#include <linux/percpu.h>
|
|
#include <asm/rqspinlock.h>
|
|
|
|
struct pcpu_freelist_head {
|
|
struct pcpu_freelist_node *first;
|
|
rqspinlock_t lock;
|
|
};
|
|
|
|
struct pcpu_freelist {
|
|
struct pcpu_freelist_head __percpu *freelist;
|
|
struct pcpu_freelist_head extralist;
|
|
};
|
|
|
|
struct pcpu_freelist_node {
|
|
struct pcpu_freelist_node *next;
|
|
};
|
|
|
|
/* pcpu_freelist_* do spin_lock_irqsave. */
|
|
void pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *);
|
|
struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *);
|
|
/* __pcpu_freelist_* do spin_lock only. caller must disable irqs. */
|
|
void __pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *);
|
|
struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *);
|
|
void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
|
|
u32 nr_elems);
|
|
int pcpu_freelist_init(struct pcpu_freelist *);
|
|
void pcpu_freelist_destroy(struct pcpu_freelist *s);
|
|
#endif
|