diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 2cc730729e08..4570a4ffbab2 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -1452,7 +1452,7 @@ static int vhost_net_release(struct inode *inode, struct file *f) vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); vhost_dev_stop(&n->dev); - vhost_dev_cleanup(&n->dev); + vhost_dev_cleanup(&n->dev, false); vhost_net_vq_reset(n); if (tx_sock) sockfd_put(tx_sock); @@ -1661,7 +1661,7 @@ static long vhost_net_reset_owner(struct vhost_net *n) vhost_net_stop(n, &tx_sock, &rx_sock); vhost_net_flush(n); vhost_dev_stop(&n->dev); - vhost_dev_reset_owner(&n->dev, umem); + vhost_dev_reset_owner(&n->dev, umem, false); vhost_net_vq_reset(n); done: mutex_unlock(&n->dev.mutex); diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 4f8c0260bc9e..885fac5ed042 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -2390,7 +2390,7 @@ static int vhost_scsi_release(struct inode *inode, struct file *f) mutex_unlock(&vs->dev.mutex); vhost_scsi_clear_endpoint(vs, &t); vhost_dev_stop(&vs->dev); - vhost_dev_cleanup(&vs->dev); + vhost_dev_cleanup(&vs->dev, false); kfree(vs->dev.vqs); kvfree(vs->vqs); kfree(vs->old_inflight); diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c index 24514c8fdee4..84ed9ea81219 100644 --- a/drivers/vhost/test.c +++ b/drivers/vhost/test.c @@ -163,7 +163,7 @@ static int vhost_test_release(struct inode *inode, struct file *f) vhost_test_stop(n, &private); vhost_test_flush(n); vhost_dev_stop(&n->dev); - vhost_dev_cleanup(&n->dev); + vhost_dev_cleanup(&n->dev, false); kfree(n->dev.vqs); kfree(n); return 0; @@ -238,7 +238,7 @@ static long vhost_test_reset_owner(struct vhost_test *n) vhost_test_stop(n, &priv); vhost_test_flush(n); vhost_dev_stop(&n->dev); - vhost_dev_reset_owner(&n->dev, umem); + vhost_dev_reset_owner(&n->dev, umem, false); done: mutex_unlock(&n->dev.mutex); return err; diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index a31786796d4c..ff7d90719171 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -948,7 +948,7 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, case VHOST_SET_OWNER: r = vhost_vdpa_bind_mm(v); if (r) - vhost_dev_reset_owner(d, NULL); + vhost_dev_reset_owner(d, NULL, false); break; } out: @@ -1453,7 +1453,7 @@ static void vhost_vdpa_cleanup(struct vhost_vdpa *v) } vhost_vdpa_free_domain(v); - vhost_dev_cleanup(&v->vdev); + vhost_dev_cleanup(&v->vdev, false); kfree(v->vdev.vqs); v->vdev.vqs = NULL; } diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 44cac11b68d2..4d5d32e68d35 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -731,7 +731,7 @@ static void vhost_worker_destroy(struct vhost_dev *dev, kfree(worker); } -static void vhost_workers_free(struct vhost_dev *dev) +static void vhost_workers_free(struct vhost_dev *dev, bool sync) { struct vhost_worker *worker; unsigned long i; @@ -741,6 +741,21 @@ static void vhost_workers_free(struct vhost_dev *dev) for (i = 0; i < dev->nvqs; i++) rcu_assign_pointer(dev->vqs[i]->worker, NULL); + + /* + * This path can be reached by lockless work queuers. In this case + * vhost_vq_work_queue() reads vq->worker under rcu_read_lock(), so a + * RCU reader that fetched a worker before we cleared the pointers above + * may still be queueing work on it. Wait for those readers to finish, + * then flush so any work they queued runs (clearing VHOST_WORK_QUEUED) + * before the workers are freed. Notably, that is the case for the + * vsock RESET_OWNER path. + */ + if (sync) { + synchronize_rcu(); + vhost_dev_flush(dev); + } + /* * Free the default worker we created and cleanup workers userspace * created but couldn't clean up (it forgot or crashed). @@ -1163,11 +1178,12 @@ struct vhost_iotlb *vhost_dev_reset_owner_prepare(void) EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); /* Caller should have device mutex */ -void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem) +void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem, + bool sync) { int i; - vhost_dev_cleanup(dev); + vhost_dev_cleanup(dev, sync); dev->fork_owner = fork_from_owner_default; dev->umem = umem; @@ -1227,7 +1243,7 @@ void vhost_clear_msg(struct vhost_dev *dev) } EXPORT_SYMBOL_GPL(vhost_clear_msg); -void vhost_dev_cleanup(struct vhost_dev *dev) +void vhost_dev_cleanup(struct vhost_dev *dev, bool sync) { int i; @@ -1251,7 +1267,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev) dev->iotlb = NULL; vhost_clear_msg(dev); wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); - vhost_workers_free(dev); + vhost_workers_free(dev, sync); vhost_detach_mm(dev); } EXPORT_SYMBOL_GPL(vhost_dev_cleanup); diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 39e6121f7525..5a684ba5e389 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -219,8 +219,9 @@ long vhost_dev_set_owner(struct vhost_dev *dev); bool vhost_dev_has_owner(struct vhost_dev *dev); long vhost_dev_check_owner(struct vhost_dev *); struct vhost_iotlb *vhost_dev_reset_owner_prepare(void); -void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb); -void vhost_dev_cleanup(struct vhost_dev *); +void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb, + bool sync); +void vhost_dev_cleanup(struct vhost_dev *dev, bool sync); void vhost_dev_stop(struct vhost_dev *); long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp); diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index 30d4b16a5e50..e6ac9b41c2a5 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -824,7 +824,7 @@ static int vhost_vsock_dev_release(struct inode *inode, struct file *file) virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); - vhost_dev_cleanup(&vsock->dev); + vhost_dev_cleanup(&vsock->dev, false); put_net_track(vsock->net, &vsock->ns_tracker); kfree(vsock->dev.vqs); vhost_vsock_free(vsock);