github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

dm-snapshot: fix 'scheduling while atomic' on real-time kernels

Browse Source 
There is reported 'scheduling while atomic' bug when using dm-snapshot on
real-time kernels. The reason for the bug is that the hlist_bl code does
preempt_disable() when taking the lock and the kernel attempts to take
other spinlocks while holding the hlist_bl lock.

Fix this by converting a hlist_bl spinlock into a regular spinlock.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Reported-by: Jiping Ma <jiping.ma2@windriver.com>
This commit is contained in:
Mikulas Patocka 2025-12-01 22:13:10 +01:00
parent f4412c7d5a
commit 8581b19eb2
2 changed files with 35 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/md/dm-exception-store.h
View File

@ -29,7 +29,7 @@ typedef sector_t chunk_t;
* chunk within the device. * chunk within the device.
*/ */
struct dm_exception { struct dm_exception {
struct hlist_bl_node hash_list; struct hlist_node hash_list;
chunk_t old_chunk; chunk_t old_chunk;
chunk_t new_chunk; chunk_t new_chunk;

73
drivers/md/dm-snap.c
View File

@ -40,10 +40,15 @@ static const char dm_snapshot_merge_target_name[] = "snapshot-merge";
#define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \ #define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \
(DM_TRACKED_CHUNK_HASH_SIZE - 1)) (DM_TRACKED_CHUNK_HASH_SIZE - 1))
struct dm_hlist_head {
struct hlist_head head;
spinlock_t lock;
};
struct dm_exception_table { struct dm_exception_table {
uint32_t hash_mask; uint32_t hash_mask;
unsigned int hash_shift; unsigned int hash_shift;
struct hlist_bl_head *table; struct dm_hlist_head *table;
}; };
struct dm_snapshot { struct dm_snapshot {
@ -628,8 +633,8 @@ static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk);
/* Lock to protect access to the completed and pending exception hash tables. */ /* Lock to protect access to the completed and pending exception hash tables. */
struct dm_exception_table_lock { struct dm_exception_table_lock {
struct hlist_bl_head *complete_slot; spinlock_t *complete_slot;
struct hlist_bl_head *pending_slot; spinlock_t *pending_slot;
}; };
static void dm_exception_table_lock_init(struct dm_snapshot *s, chunk_t chunk, static void dm_exception_table_lock_init(struct dm_snapshot *s, chunk_t chunk,
@ -638,20 +643,20 @@ static void dm_exception_table_lock_init(struct dm_snapshot *s, chunk_t chunk,
struct dm_exception_table *complete = &s->complete; struct dm_exception_table *complete = &s->complete;
struct dm_exception_table *pending = &s->pending; struct dm_exception_table *pending = &s->pending;
lock->complete_slot = &complete->table[exception_hash(complete, chunk)]; lock->complete_slot = &complete->table[exception_hash(complete, chunk)].lock;
lock->pending_slot = &pending->table[exception_hash(pending, chunk)]; lock->pending_slot = &pending->table[exception_hash(pending, chunk)].lock;
} }
static void dm_exception_table_lock(struct dm_exception_table_lock *lock) static void dm_exception_table_lock(struct dm_exception_table_lock *lock)
{ {
hlist_bl_lock(lock->complete_slot); spin_lock_nested(lock->complete_slot, 1);
hlist_bl_lock(lock->pending_slot); spin_lock_nested(lock->pending_slot, 2);
} }
static void dm_exception_table_unlock(struct dm_exception_table_lock *lock) static void dm_exception_table_unlock(struct dm_exception_table_lock *lock)
{ {
hlist_bl_unlock(lock->pending_slot); spin_unlock(lock->pending_slot);
hlist_bl_unlock(lock->complete_slot); spin_unlock(lock->complete_slot);
} }
static int dm_exception_table_init(struct dm_exception_table *et, static int dm_exception_table_init(struct dm_exception_table *et,
@ -661,13 +666,15 @@ static int dm_exception_table_init(struct dm_exception_table *et,
et->hash_shift = hash_shift; et->hash_shift = hash_shift;
et->hash_mask = size - 1; et->hash_mask = size - 1;
et->table = kvmalloc_array(size, sizeof(struct hlist_bl_head), et->table = kvmalloc_array(size, sizeof(struct dm_hlist_head),
GFP_KERNEL); GFP_KERNEL);
if (!et->table) if (!et->table)
return -ENOMEM; return -ENOMEM;
for (i = 0; i < size; i++) for (i = 0; i < size; i++) {
INIT_HLIST_BL_HEAD(et->table + i); INIT_HLIST_HEAD(&et->table[i].head);
spin_lock_init(&et->table[i].lock);
}
return 0; return 0;
} }
@ -675,16 +682,17 @@ static int dm_exception_table_init(struct dm_exception_table *et,
static void dm_exception_table_exit(struct dm_exception_table *et, static void dm_exception_table_exit(struct dm_exception_table *et,
struct kmem_cache *mem) struct kmem_cache *mem)
{ {
struct hlist_bl_head *slot; struct dm_hlist_head *slot;
struct dm_exception *ex; struct dm_exception *ex;
struct hlist_bl_node *pos, *n; struct hlist_node *pos;
int i, size; int i, size;
size = et->hash_mask + 1; size = et->hash_mask + 1;
for (i = 0; i < size; i++) { for (i = 0; i < size; i++) {
slot = et->table + i; slot = et->table + i;
hlist_bl_for_each_entry_safe(ex, pos, n, slot, hash_list) { hlist_for_each_entry_safe(ex, pos, &slot->head, hash_list) {
hlist_del(&ex->hash_list);
kmem_cache_free(mem, ex); kmem_cache_free(mem, ex);
cond_resched(); cond_resched();
} }
@ -700,7 +708,7 @@ static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk)
static void dm_remove_exception(struct dm_exception *e) static void dm_remove_exception(struct dm_exception *e)
{ {
hlist_bl_del(&e->hash_list); hlist_del(&e->hash_list);
} }
/* /*
@ -710,12 +718,11 @@ static void dm_remove_exception(struct dm_exception *e)
static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et, static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et,
chunk_t chunk) chunk_t chunk)
{ {
struct hlist_bl_head *slot; struct hlist_head *slot;
struct hlist_bl_node *pos;
struct dm_exception *e; struct dm_exception *e;
slot = &et->table[exception_hash(et, chunk)]; slot = &et->table[exception_hash(et, chunk)].head;
hlist_bl_for_each_entry(e, pos, slot, hash_list) hlist_for_each_entry(e, slot, hash_list)
if (chunk >= e->old_chunk && if (chunk >= e->old_chunk &&
chunk <= e->old_chunk + dm_consecutive_chunk_count(e)) chunk <= e->old_chunk + dm_consecutive_chunk_count(e))
return e; return e;
@ -762,18 +769,17 @@ static void free_pending_exception(struct dm_snap_pending_exception *pe)
static void dm_insert_exception(struct dm_exception_table *eh, static void dm_insert_exception(struct dm_exception_table *eh,
struct dm_exception *new_e) struct dm_exception *new_e)
{ {
struct hlist_bl_head *l; struct hlist_head *l;
struct hlist_bl_node *pos;
struct dm_exception *e = NULL; struct dm_exception *e = NULL;
l = &eh->table[exception_hash(eh, new_e->old_chunk)]; l = &eh->table[exception_hash(eh, new_e->old_chunk)].head;
/* Add immediately if this table doesn't support consecutive chunks */ /* Add immediately if this table doesn't support consecutive chunks */
if (!eh->hash_shift) if (!eh->hash_shift)
goto out; goto out;
/* List is ordered by old_chunk */ /* List is ordered by old_chunk */
hlist_bl_for_each_entry(e, pos, l, hash_list) { hlist_for_each_entry(e, l, hash_list) {
/* Insert after an existing chunk? */ /* Insert after an existing chunk? */
if (new_e->old_chunk == (e->old_chunk + if (new_e->old_chunk == (e->old_chunk +
dm_consecutive_chunk_count(e) + 1) && dm_consecutive_chunk_count(e) + 1) &&
@ -804,13 +810,13 @@ static void dm_insert_exception(struct dm_exception_table *eh,
* Either the table doesn't support consecutive chunks or slot * Either the table doesn't support consecutive chunks or slot
* l is empty. * l is empty.
*/ */
hlist_bl_add_head(&new_e->hash_list, l); hlist_add_head(&new_e->hash_list, l);
} else if (new_e->old_chunk < e->old_chunk) { } else if (new_e->old_chunk < e->old_chunk) {
/* Add before an existing exception */ /* Add before an existing exception */
hlist_bl_add_before(&new_e->hash_list, &e->hash_list); hlist_add_before(&new_e->hash_list, &e->hash_list);
} else { } else {
/* Add to l's tail: e is the last exception in this slot */ /* Add to l's tail: e is the last exception in this slot */
hlist_bl_add_behind(&new_e->hash_list, &e->hash_list); hlist_add_behind(&new_e->hash_list, &e->hash_list);
} }
} }
@ -820,7 +826,6 @@ static void dm_insert_exception(struct dm_exception_table *eh,
*/ */
static int dm_add_exception(void *context, chunk_t old, chunk_t new) static int dm_add_exception(void *context, chunk_t old, chunk_t new)
{ {
struct dm_exception_table_lock lock;
struct dm_snapshot *s = context; struct dm_snapshot *s = context;
struct dm_exception *e; struct dm_exception *e;
@ -833,17 +838,7 @@ static int dm_add_exception(void *context, chunk_t old, chunk_t new)
/* Consecutive_count is implicitly initialised to zero */ /* Consecutive_count is implicitly initialised to zero */
e->new_chunk = new; e->new_chunk = new;
/*
* Although there is no need to lock access to the exception tables
* here, if we don't then hlist_bl_add_head(), called by
* dm_insert_exception(), will complain about accessing the
* corresponding list without locking it first.
*/
dm_exception_table_lock_init(s, old, &lock);
dm_exception_table_lock(&lock);
dm_insert_exception(&s->complete, e); dm_insert_exception(&s->complete, e);
dm_exception_table_unlock(&lock);
return 0; return 0;
} }
@ -873,7 +868,7 @@ static int calc_max_buckets(void)
/* use a fixed size of 2MB */ /* use a fixed size of 2MB */
unsigned long mem = 2 * 1024 * 1024; unsigned long mem = 2 * 1024 * 1024;
mem /= sizeof(struct hlist_bl_head); mem /= sizeof(struct dm_hlist_head);
return mem; return mem;
} }