diff --git a/drivers/block/swim.c b/drivers/block/swim.c
index 49c894af7c13..baeb05f3e003 100644
--- a/drivers/block/swim.c
+++ b/drivers/block/swim.c
@@ -700,10 +700,11 @@ static int floppy_ioctl(struct block_device *bdev, blk_mode_t mode,
 
 	switch (cmd) {
 	case FDEJECT:
-		if (fs->ref_count != 1)
-			return -EBUSY;
 		mutex_lock(&swim_mutex);
-		err = floppy_eject(fs);
+		if (fs->ref_count == -1 || fs->ref_count == 1)
+			err = floppy_eject(fs);
+		else
+			err = -EBUSY;
 		mutex_unlock(&swim_mutex);
 		return err;