diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c
index 3f05fa8811af..08e22829f72d 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.c
+++ b/drivers/accel/amdxdna/amdxdna_gem.c
@@ -697,6 +697,8 @@ static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *fi
 	int ret;
 
 	guard(mutex)(&abo->lock);
+	if (abo->open_ref > 0 && filp->driver_priv != abo->client)
+		return -EPERM;
 	abo->open_ref++;
 	if (abo->open_ref > 1)
 		return 0;