diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c
index fd5651c003ae..a7cbe643e33c 100644
--- a/drivers/infiniband/core/rdma_core.c
+++ b/drivers/infiniband/core/rdma_core.c
@@ -69,7 +69,6 @@ void ib_uverbs_release_file(struct kref *ref)
 
 	if (file->disassociate_page)
 		__free_pages(file->disassociate_page, 0);
-	mutex_destroy(&file->disassociation_lock);
 	mutex_destroy(&file->umap_lock);
 	mutex_destroy(&file->ucontext_lock);
 	kfree(file);
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 0d88b2ee68ff..2a046c888dfa 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -644,12 +644,15 @@ static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)
 		goto out;
 	}
 
-	mutex_lock(&file->disassociation_lock);
+	if (!down_read_trylock(&file->hw_destroy_rwsem)) {
+		ret = -EIO;
+		goto out;
+	}
 
 	vma->vm_ops = &rdma_umap_ops;
 	ret = ucontext->device->ops.mmap(ucontext, vma);
 
-	mutex_unlock(&file->disassociation_lock);
+	up_read(&file->hw_destroy_rwsem);
 out:
 	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 	return ret;
@@ -671,7 +674,6 @@ static void rdma_umap_open(struct vm_area_struct *vma)
 	/* We are racing with disassociation */
 	if (!down_read_trylock(&ufile->hw_destroy_rwsem))
 		goto out_zap;
-	mutex_lock(&ufile->disassociation_lock);
 
 	/*
 	 * Disassociation already completed, the VMA should already be zapped.
@@ -684,12 +686,10 @@ static void rdma_umap_open(struct vm_area_struct *vma)
 		goto out_unlock;
 	rdma_umap_priv_init(priv, vma, opriv->entry);
 
-	mutex_unlock(&ufile->disassociation_lock);
 	up_read(&ufile->hw_destroy_rwsem);
 	return;
 
 out_unlock:
-	mutex_unlock(&ufile->disassociation_lock);
 	up_read(&ufile->hw_destroy_rwsem);
 out_zap:
 	/*
@@ -773,7 +773,7 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 {
 	struct rdma_umap_priv *priv, *next_priv;
 
-	mutex_lock(&ufile->disassociation_lock);
+	lockdep_assert_held_write(&ufile->hw_destroy_rwsem);
 
 	while (1) {
 		struct mm_struct *mm = NULL;
@@ -799,10 +799,8 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 			break;
 		}
 		mutex_unlock(&ufile->umap_lock);
-		if (!mm) {
-			mutex_unlock(&ufile->disassociation_lock);
+		if (!mm)
 			return;
-		}
 
 		/*
 		 * The umap_lock is nested under mmap_lock since it used within
@@ -832,8 +830,6 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 		mmap_read_unlock(mm);
 		mmput(mm);
 	}
-
-	mutex_unlock(&ufile->disassociation_lock);
 }
 
 /**
@@ -851,8 +847,11 @@ void rdma_user_mmap_disassociate(struct ib_device *device)
 
 	mutex_lock(&uverbs_dev->lists_mutex);
 	list_for_each_entry(ufile, &uverbs_dev->uverbs_file_list, list) {
-		if (ufile->ucontext)
+		if (ufile->ucontext) {
+			down_write(&ufile->hw_destroy_rwsem);
 			uverbs_user_mmap_disassociate(ufile);
+			up_write(&ufile->hw_destroy_rwsem);
+		}
 	}
 	mutex_unlock(&uverbs_dev->lists_mutex);
 }
@@ -927,8 +926,6 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)
 	mutex_init(&file->umap_lock);
 	INIT_LIST_HEAD(&file->umaps);
 
-	mutex_init(&file->disassociation_lock);
-
 	filp->private_data = file;
 	list_add_tail(&file->list, &dev->uverbs_file_list);
 	mutex_unlock(&dev->lists_mutex);
diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h
index 5a07f9a6dcd1..6f3622892c0c 100644
--- a/include/rdma/uverbs_types.h
+++ b/include/rdma/uverbs_types.h
@@ -180,8 +180,6 @@ struct ib_uverbs_file {
 	struct page *disassociate_page;
 
 	struct xarray		idr;
-
-	struct mutex disassociation_lock;
 };
 
 extern const struct uverbs_obj_type_class uverbs_idr_class;