diff --git a/drivers/spi/spi-ma35d1-qspi.c b/drivers/spi/spi-ma35d1-qspi.c
index 929546dbe38f..b4fd0ab4bd31 100644
--- a/drivers/spi/spi-ma35d1-qspi.c
+++ b/drivers/spi/spi-ma35d1-qspi.c
@@ -131,11 +131,15 @@ static int nuvoton_qspi_reset_fifo(struct nuvoton_qspi *qspi)
 					 1, NUVOTON_QSPI_TIMEOUT_US);
 }
 
-static int nuvoton_qspi_set_speed(struct nuvoton_qspi *qspi, u32 speed_hz)
+static int nuvoton_qspi_set_speed(struct spi_device *spi, u32 speed_hz)
 {
+	struct nuvoton_qspi *qspi = spi_controller_get_devdata(spi->controller);
 	unsigned long clk_rate;
 	u32 div;
 
+	if (!speed_hz)
+		speed_hz = spi->max_speed_hz;
+
 	if (!speed_hz)
 		return -EINVAL;
 
@@ -174,24 +178,16 @@ static int nuvoton_qspi_set_bits_per_word(struct nuvoton_qspi *qspi, u8 bpw)
 	return 0;
 }
 
-static int nuvoton_qspi_setup_transfer(struct spi_device *spi,
-				       u32 speed_hz, u8 bpw)
+static int nuvoton_qspi_setup_transfer(struct spi_device *spi, u8 bpw)
 {
 	struct nuvoton_qspi *qspi = spi_controller_get_devdata(spi->controller);
 	u32 mode = spi->mode & SPI_MODE_X_MASK;
 	u32 ctl = 0;
 	int ret;
 
-	if (!speed_hz)
-		speed_hz = spi->max_speed_hz;
-
 	if (!bpw)
 		bpw = NUVOTON_QSPI_DEFAULT_BPW;
 
-	ret = nuvoton_qspi_set_speed(qspi, speed_hz);
-	if (ret)
-		return ret;
-
 	ret = nuvoton_qspi_set_bits_per_word(qspi, bpw);
 	if (ret)
 		return ret;
@@ -216,11 +212,18 @@ static int nuvoton_qspi_setup_transfer(struct spi_device *spi,
 	return 0;
 }
 
-static void nuvoton_qspi_set_bus_width(struct nuvoton_qspi *qspi,
-				       unsigned int buswidth,
-				       enum spi_mem_data_dir dir)
+static int nuvoton_qspi_configure_bus(struct spi_device *spi,
+				      unsigned int buswidth,
+				      enum spi_mem_data_dir dir,
+				      u32 speed_hz)
 {
+	struct nuvoton_qspi *qspi = spi_controller_get_devdata(spi->controller);
 	u32 ctl = 0;
+	int ret;
+
+	ret = nuvoton_qspi_set_speed(spi, speed_hz);
+	if (ret)
+		return ret;
 
 	if (buswidth == 4)
 		ctl |= NUVOTON_QSPI_CTL_QUADIOEN_MASK;
@@ -234,6 +237,8 @@ static void nuvoton_qspi_set_bus_width(struct nuvoton_qspi *qspi,
 				 NUVOTON_QSPI_CTL_QUADIOEN_MASK |
 				 NUVOTON_QSPI_CTL_DUALIOEN_MASK |
 				 NUVOTON_QSPI_CTL_DATDIR_MASK, ctl);
+
+	return 0;
 }
 
 static u32 nuvoton_qspi_tx_byte(const void *txbuf, unsigned int idx)
@@ -459,14 +464,17 @@ static int nuvoton_qspi_mem_exec_op(struct spi_mem *mem,
 	int ret;
 	int i;
 
-	ret = nuvoton_qspi_setup_transfer(spi, op->max_freq,
-					  NUVOTON_QSPI_DEFAULT_BPW);
+	ret = nuvoton_qspi_setup_transfer(spi, NUVOTON_QSPI_DEFAULT_BPW);
 	if (ret)
 		return ret;
 
 	nuvoton_qspi_mem_set_cs(spi, true);
 
-	nuvoton_qspi_set_bus_width(qspi, op->cmd.buswidth, SPI_MEM_DATA_OUT);
+	ret = nuvoton_qspi_configure_bus(spi, op->cmd.buswidth, SPI_MEM_DATA_OUT,
+					 op->max_freq);
+	if (ret)
+		goto out_deassert_cs;
+
 	ret = nuvoton_qspi_txrx(qspi, &opcode, NULL, 1);
 	if (ret)
 		goto out_deassert_cs;
@@ -475,24 +483,33 @@ static int nuvoton_qspi_mem_exec_op(struct spi_mem *mem,
 		for (i = 0; i < op->addr.nbytes; i++)
 			addr[i] = op->addr.val >> (8 * (op->addr.nbytes - i - 1));
 
-		nuvoton_qspi_set_bus_width(qspi, op->addr.buswidth,
-					   SPI_MEM_DATA_OUT);
+		ret = nuvoton_qspi_configure_bus(spi, op->addr.buswidth, SPI_MEM_DATA_OUT,
+						 op->max_freq);
+		if (ret)
+			goto out_deassert_cs;
+
 		ret = nuvoton_qspi_txrx(qspi, addr, NULL, op->addr.nbytes);
 		if (ret)
 			goto out_deassert_cs;
 	}
 
 	if (op->dummy.nbytes) {
-		nuvoton_qspi_set_bus_width(qspi, op->dummy.buswidth,
-					   SPI_MEM_DATA_OUT);
+		ret = nuvoton_qspi_configure_bus(spi, op->dummy.buswidth, SPI_MEM_DATA_OUT,
+						 op->max_freq);
+		if (ret)
+			goto out_deassert_cs;
+
 		ret = nuvoton_qspi_txrx(qspi, NULL, NULL, op->dummy.nbytes);
 		if (ret)
 			goto out_deassert_cs;
 	}
 
 	if (op->data.nbytes) {
-		nuvoton_qspi_set_bus_width(qspi, op->data.buswidth,
-					   op->data.dir);
+		ret = nuvoton_qspi_configure_bus(spi, op->data.buswidth, op->data.dir,
+						 op->max_freq);
+		if (ret)
+			goto out_deassert_cs;
+
 		ret = nuvoton_qspi_txrx(qspi,
 					op->data.dir == SPI_MEM_DATA_OUT ?
 					op->data.buf.out : NULL,
@@ -528,8 +545,7 @@ static int nuvoton_qspi_transfer_one(struct spi_controller *ctlr,
 	unsigned int buswidth = 1;
 	int ret;
 
-	ret = nuvoton_qspi_setup_transfer(spi, xfer->speed_hz,
-					  xfer->bits_per_word);
+	ret = nuvoton_qspi_setup_transfer(spi, xfer->bits_per_word);
 	if (ret)
 		return ret;
 
@@ -552,7 +568,10 @@ static int nuvoton_qspi_transfer_one(struct spi_controller *ctlr,
 			buswidth = 2;
 	}
 
-	nuvoton_qspi_set_bus_width(qspi, buswidth, dir);
+	ret = nuvoton_qspi_configure_bus(spi, buswidth, dir, xfer->speed_hz);
+	if (ret)
+		return ret;
+
 	ret = nuvoton_qspi_txrx(qspi, xfer->tx_buf, xfer->rx_buf,
 				xfer->len);