github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

blk-cgroup: don't nest queue_lock under rcu in blkg_lookup_create()

Browse Source 
Change this in two steps:

1) hold rcu lock and do blkg_lookup() from fast path;
2) hold queue_lock directly from slow path, and don't nest it under rcu
   lock;

Prepare to convert protecting blkcg with blkcg_mutex instead of
queue_lock.

Signed-off-by: Yu Kuai <yukuai@fygo.io>
Link: https://patch.msgid.link/93f33cc9e5a39dddb78dcd934d0c1d04b564fb00.1780621988.git.yukuai@fygo.io
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Yu Kuai 2026-06-08 11:42:45 +08:00 committed by Jens Axboe
parent 56cc24f59c
commit 9327a865e3
1 changed files with 36 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
block/blk-cgroup.c
View File

@ -469,22 +469,17 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
{
struct request_queue *q = disk->queue;
struct blkcg_gq *blkg;
unsigned long flags;
WARN_ON_ONCE(!rcu_read_lock_held());
blkg = blkg_lookup(blkcg, q);
if (blkg)
return blkg;
spin_lock_irqsave(&q->queue_lock, flags);
rcu_read_lock();
blkg = blkg_lookup(blkcg, q);
if (blkg) {
if (blkcg != &blkcg_root &&
blkg != rcu_dereference(blkcg->blkg_hint))
rcu_assign_pointer(blkcg->blkg_hint, blkg);
goto found;
rcu_read_unlock();
return blkg;
}
rcu_read_unlock();
/*
* Create blkgs walking down from blkcg_root to @blkcg, so that all
@ -516,8 +511,6 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
break;
}
found:
spin_unlock_irqrestore(&q->queue_lock, flags);
return blkg;
}
@ -2046,6 +2039,18 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
atomic64_add(delta, &blkg->delay_nsec);
}
static inline struct blkcg_gq *blkg_lookup_tryget(struct blkcg_gq *blkg)
{
retry:
if (blkg_tryget(blkg))
return blkg;
blkg = blkg->parent;
if (blkg)
goto retry;
return NULL;
}
/**
* blkg_tryget_closest - try and get a blkg ref on the closet blkg
* @bio: target bio
@ -2058,20 +2063,30 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
struct cgroup_subsys_state *css)
{
struct blkcg_gq *blkg, *ret_blkg = NULL;
struct request_queue *q = bio->bi_bdev->bd_queue;
struct blkcg *blkcg = css_to_blkcg(css);
struct blkcg_gq *blkg;
rcu_read_lock();
blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk);
while (blkg) {
if (blkg_tryget(blkg)) {
ret_blkg = blkg;
break;
}
blkg = blkg->parent;
}
blkg = blkg_lookup(blkcg, q);
if (likely(blkg))
blkg = blkg_lookup_tryget(blkg);
rcu_read_unlock();
return ret_blkg;
if (blkg)
return blkg;
/*
* Fast path failed, we're probably issuing IO in this cgroup the first
* time, hold lock to create new blkg.
*/
spin_lock_irq(&q->queue_lock);
blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk);
if (blkg)
blkg = blkg_lookup_tryget(blkg);
spin_unlock_irq(&q->queue_lock);
return blkg;
}
/**