diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 9891cd2cf712..766789a7bbfa 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -506,7 +506,7 @@ static void vduse_dev_reset(struct vduse_dev *dev) } scoped_guard(rwsem_write, &dev->rwsem) { - dev->suspended = false; + WRITE_ONCE(dev->suspended, false); dev->status = 0; dev->driver_features = 0; dev->generation++; @@ -567,11 +567,17 @@ static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx, static void vduse_vq_kick(struct vduse_virtqueue *vq) { - guard(rwsem_read)(&vq->dev->rwsem); - if (vq->dev->suspended) + /* + * This runs in the context of the vdpa kick_vq op, which may be + * atomic (e.g. virtio-blk kicks from blk-mq dispatch under + * rcu_read_lock()), so dev->rwsem must not be taken here. + * dev->suspended is checked under kick_lock instead and + * vduse_vdpa_suspend() cycles every kick_lock after setting it. + */ + guard(spinlock)(&vq->kick_lock); + if (READ_ONCE(vq->dev->suspended)) return; - guard(spinlock)(&vq->kick_lock); scoped_guard(spinlock_bh, &vq->ready_lock) if (!vq->ready) return; @@ -946,7 +952,17 @@ static int vduse_vdpa_suspend(struct vdpa_device *vdpa) ret = vduse_dev_msg_sync(dev, &msg); if (ret == 0) { scoped_guard(rwsem_write, &dev->rwsem) - dev->suspended = true; + WRITE_ONCE(dev->suspended, true); + + /* + * Kicks check dev->suspended under kick_lock without taking + * the rwsem: cycle each kick_lock so that no kick that has + * already passed the check is still in flight after this. + */ + for (u32 i = 0; i < dev->vq_num; i++) { + spin_lock(&dev->vqs[i]->kick_lock); + spin_unlock(&dev->vqs[i]->kick_lock); + } cancel_work_sync(&dev->inject); for (u32 i = 0; i < dev->vq_num; i++)