diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c index ed6e688cfafb..1f0e93800908 100644 --- a/drivers/spi/spi-dw-core.c +++ b/drivers/spi/spi-dw-core.c @@ -937,6 +937,15 @@ static void dw_spi_enh_write_cmd_addr(struct dw_spi *dws, const struct spi_mem_o dw_spi_set_cs(mem->spi, false); } + + /* + * FIXME: The exact reason for this delay is not fully understood, + * but empirical testing shows it significantly improves the stability + * of read/write operations. Without this delay, occasional transfer + * errors or timeouts may occur under certain conditions. + * Keeping it as a safeguard based on practical validation. + */ + udelay(5); } static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) @@ -948,6 +957,8 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op * unsigned long long ms; int ret; + dws->dma_mapped = false; + switch (op->data.buswidth) { case 0: case 1: @@ -971,10 +982,13 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op * cfg.dfs = 8; cfg.freq = clamp(op->max_freq, 0U, dws->max_mem_freq); cfg.ndf = op->data.nbytes; - if (op->data.dir == SPI_MEM_DATA_IN) + if (op->data.dir == SPI_MEM_DATA_IN) { cfg.tmode = DW_SPI_CTRLR0_TMOD_RO; - else + dws->tx_dir = false; + } else { cfg.tmode = DW_SPI_CTRLR0_TMOD_TO; + dws->tx_dir = true; + } if (op->data.buswidth == op->addr.buswidth && op->data.buswidth == op->cmd.buswidth) @@ -1010,47 +1024,61 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op * } } - dw_spi_enh_write_cmd_addr(dws, op, mem); - - /* - * FIXME: The exact reason for this delay is not fully understood, - * but empirical testing shows it significantly improves the stability - * of read/write operations. Without this delay, occasional transfer - * errors or timeouts may occur under certain conditions. - * Keeping it as a safeguard based on practical validation. - */ - udelay(5); - - dw_spi_enh_irq_setup(dws); - - /* Use timeout calculation from spi_transfer_wait() */ - ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len); - do_div(ms, dws->current_freq); - - /* - * Increase it twice and add 200 ms tolerance, use - * predefined maximum in case of overflow. - */ - ms += ms + 200; - if (ms > UINT_MAX) - ms = UINT_MAX; - - ms = wait_for_completion_timeout(&ctlr->xfer_completion, - msecs_to_jiffies(ms)); - if (ms == 0) { - dw_spi_mask_intr(dws, 0xff); - synchronize_irq(dws->irq); - dws->rx = NULL; - dws->tx = NULL; - dws->rx_len = 0; - dws->tx_len = 0; - dw_spi_stop_mem_op(dws, mem->spi); - return -EIO; + if (dws->dma_nr_chans && dws->dma_ops->dma_enh_mem_get_chan && + op->data.nbytes > dws->fifo_len) { + ret = dws->dma_ops->dma_enh_mem_get_chan(ctlr->dev.parent, dws); + if (ret) { + /* fall back to the PIO/IRQ path */ + dev_dbg(&ctlr->dev, "DMA enh mem channel unavailable (%d)\n", ret); + } else if (dws->dma_ops->can_dma_enh_mem(ctlr)) { + ret = dws->dma_ops->dma_enh_mem_setup(dws); + if (ret) + dev_err(&ctlr->dev, "DMA enh mem setup failed (%d)\n", ret); + else + dws->dma_mapped = true; + } } - ret = dw_spi_wait_mem_op_done(dws); + if (dws->dma_mapped) { + dw_spi_enh_write_cmd_addr(dws, op, mem); - dw_spi_stop_mem_op(dws, mem->spi); + ret = dws->dma_ops->dma_enh_mem_transfer(mem, op); + + dw_spi_stop_mem_op(dws, mem->spi); + } else { + dw_spi_enh_write_cmd_addr(dws, op, mem); + + dw_spi_enh_irq_setup(dws); + + /* Use timeout calculation from spi_transfer_wait() */ + ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len); + do_div(ms, dws->current_freq); + + /* + * Increase it twice and add 200 ms tolerance, use + * predefined maximum in case of overflow. + */ + ms += ms + 200; + if (ms > UINT_MAX) + ms = UINT_MAX; + + ms = wait_for_completion_timeout(&ctlr->xfer_completion, + msecs_to_jiffies(ms)); + if (ms == 0) { + dw_spi_mask_intr(dws, 0xff); + synchronize_irq(dws->irq); + dws->rx = NULL; + dws->tx = NULL; + dws->rx_len = 0; + dws->tx_len = 0; + dw_spi_stop_mem_op(dws, mem->spi); + return -EIO; + } + + ret = dw_spi_wait_mem_op_done(dws); + + dw_spi_stop_mem_op(dws, mem->spi); + } return ret; } @@ -1361,6 +1389,9 @@ int dw_spi_add_controller(struct device *dev, struct dw_spi *dws) goto err_free_irq; } else if (ret) { dev_warn(dev, "DMA init failed\n"); + } else if (dws->dma_nr_chans == 1) { + /* Full-duplex DMA transfers need both channels */ + dev_info(dev, "Single DMA channel, full-duplex DMA disabled\n"); } else { ctlr->can_dma = dws->dma_ops->can_dma; ctlr->flags |= SPI_CONTROLLER_MUST_TX; diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c index f7d848fec9ab..a86039057988 100644 --- a/drivers/spi/spi-dw-dma.c +++ b/drivers/spi/spi-dw-dma.c @@ -42,14 +42,19 @@ static void dw_spi_dma_maxburst_init(struct dw_spi *dws) def_burst = dws->fifo_len / 2; - ret = dma_get_slave_caps(dws->rxchan, &caps); - if (!ret && caps.max_burst) - max_burst = caps.max_burst; - else - max_burst = DW_SPI_RX_BURST_LEVEL; + if (dws->rxchan) { + ret = dma_get_slave_caps(dws->rxchan, &caps); + if (!ret && caps.max_burst) + max_burst = caps.max_burst; + else + max_burst = DW_SPI_RX_BURST_LEVEL; - dws->rxburst = min(max_burst, def_burst); - dw_writel(dws, DW_SPI_DMARDLR, dws->rxburst - 1); + dws->rxburst = min(max_burst, def_burst); + dw_writel(dws, DW_SPI_DMARDLR, dws->rxburst - 1); + } + + if (!dws->txchan) + return; ret = dma_get_slave_caps(dws->txchan, &caps); if (!ret && caps.max_burst) @@ -74,20 +79,39 @@ static void dw_spi_dma_maxburst_init(struct dw_spi *dws) static int dw_spi_dma_caps_init(struct dw_spi *dws) { - struct dma_slave_caps tx, rx; + struct dma_slave_caps tx = {}, rx = {}; int ret; - ret = dma_get_slave_caps(dws->txchan, &tx); - if (ret) - return ret; + if (dws->txchan) { + ret = dma_get_slave_caps(dws->txchan, &tx); + if (ret) + return ret; - ret = dma_get_slave_caps(dws->rxchan, &rx); - if (ret) - return ret; + if (!(tx.directions & BIT(DMA_MEM_TO_DEV))) + return -ENXIO; + } - if (!(tx.directions & BIT(DMA_MEM_TO_DEV) && - rx.directions & BIT(DMA_DEV_TO_MEM))) - return -ENXIO; + if (dws->rxchan) { + ret = dma_get_slave_caps(dws->rxchan, &rx); + if (ret) + return ret; + + if (!(rx.directions & BIT(DMA_DEV_TO_MEM))) + return -ENXIO; + } + + /* With a single channel only one direction is available at a time */ + if (!dws->rxchan) { + dws->dma_sg_burst = tx.max_sg_burst; + dws->dma_addr_widths = tx.dst_addr_widths; + return 0; + } + + if (!dws->txchan) { + dws->dma_sg_burst = rx.max_sg_burst; + dws->dma_addr_widths = rx.src_addr_widths; + return 0; + } if (tx.max_sg_burst > 0 && rx.max_sg_burst > 0) dws->dma_sg_burst = min(tx.max_sg_burst, rx.max_sg_burst); @@ -169,40 +193,54 @@ static int dw_spi_dma_init_generic(struct device *dev, struct dw_spi *dws) { int ret; + dws->dma_nr_chans = 0; + dws->rxchan = dma_request_chan(dev, "rx"); if (IS_ERR(dws->rxchan)) { ret = PTR_ERR(dws->rxchan); dws->rxchan = NULL; goto err_exit; } + dws->dma_nr_chans++; + /* + * Some platforms have only one DMA channel for the controller. Keep + * the Rx channel in that case, it can still serve half-duplex + * transfers. + */ dws->txchan = dma_request_chan(dev, "tx"); if (IS_ERR(dws->txchan)) { ret = PTR_ERR(dws->txchan); dws->txchan = NULL; - goto free_rxchan; + if (ret == -EPROBE_DEFER) + goto free_rxchan; + } else { + dws->dma_nr_chans++; } - dws->ctlr->dma_rx = dws->rxchan; - dws->ctlr->dma_tx = dws->txchan; - - init_completion(&dws->dma_completion); - ret = dw_spi_dma_caps_init(dws); if (ret) goto free_txchan; dw_spi_dma_maxburst_init(dws); + init_completion(&dws->dma_completion); + + dws->ctlr->dma_rx = dws->rxchan; + dws->ctlr->dma_tx = dws->txchan; + return 0; free_txchan: - dma_release_channel(dws->txchan); - dws->txchan = NULL; + if (dws->txchan) { + dma_release_channel(dws->txchan); + dws->txchan = NULL; + } free_rxchan: dma_release_channel(dws->rxchan); dws->rxchan = NULL; err_exit: + dws->dma_nr_chans = 0; return ret; } @@ -286,19 +324,27 @@ static inline bool dw_spi_dma_tx_busy(struct dw_spi *dws) (DW_SPI_SR_BUSY | DW_SPI_SR_TF_EMPT)) != DW_SPI_SR_TF_EMPT; } -static int dw_spi_dma_wait_tx_done(struct dw_spi *dws, - struct spi_transfer *xfer) +static int dw_spi_dma_wait_tx_done(struct dw_spi *dws, u32 speed_hz) { int retry = DW_SPI_WAIT_RETRIES; struct spi_delay delay; + unsigned long ns, us; u32 nents; nents = dw_readl(dws, DW_SPI_TXFLR); - delay.unit = SPI_DELAY_UNIT_SCK; - delay.value = nents * dws->n_bytes * BITS_PER_BYTE; + ns = DIV_ROUND_UP(NSEC_PER_SEC, speed_hz) * nents * + dws->n_bytes * BITS_PER_BYTE; + if (ns <= NSEC_PER_USEC) { + delay.unit = SPI_DELAY_UNIT_NSECS; + delay.value = ns; + } else { + us = DIV_ROUND_UP(ns, NSEC_PER_USEC); + delay.unit = SPI_DELAY_UNIT_USECS; + delay.value = clamp_val(us, 0, USHRT_MAX); + } while (dw_spi_dma_tx_busy(dws) && retry--) - spi_delay_exec(&delay, xfer); + spi_delay_exec(&delay, NULL); if (retry < 0) { dev_err(&dws->ctlr->dev, "Tx hanged up\n"); @@ -659,7 +705,7 @@ static int dw_spi_dma_transfer(struct dw_spi *dws, struct spi_transfer *xfer) return ret; if (dws->ctlr->cur_msg->status == -EINPROGRESS) { - ret = dw_spi_dma_wait_tx_done(dws, xfer); + ret = dw_spi_dma_wait_tx_done(dws, xfer->effective_speed_hz); if (ret) return ret; } @@ -682,6 +728,183 @@ static void dw_spi_dma_stop(struct dw_spi *dws) } } +/* + * The enhanced SPI memory operations are half-duplex, so a single DMA channel + * is enough for them. On a platform with only one channel, the channel is + * bound to a direction by its "tx"/"rx" DMA specifier, and whichever channel + * is currently held tells which direction it serves. Keep it across + * operations and swap it only when the transfer direction changes. + */ +static int dw_spi_enh_mem_dma_get_chan(struct device *dev, struct dw_spi *dws) +{ + struct dma_chan **chan, **other; + const char *name; + int ret; + + if (dws->dma_nr_chans == 2) + return 0; + + if (dws->tx_dir) { + chan = &dws->txchan; + other = &dws->rxchan; + name = "tx"; + } else { + chan = &dws->rxchan; + other = &dws->txchan; + name = "rx"; + } + + if (*chan) + return 0; + + dws->ctlr->dma_tx = NULL; + dws->ctlr->dma_rx = NULL; + + /* + * With a single physical channel, the held channel must be released + * before it can be requested again with the handshake of the other + * direction. On failure both stay NULL, and the next operation + * retries the request and falls back to PIO meanwhile. + */ + + if (*other) { + dma_release_channel(*other); + *other = NULL; + } + + *chan = dma_request_chan(dev, name); + if (IS_ERR(*chan)) { + ret = PTR_ERR(*chan); + *chan = NULL; + return ret; + } + + ret = dw_spi_dma_caps_init(dws); + if (ret) { + dma_release_channel(*chan); + *chan = NULL; + return ret; + } + + dw_spi_dma_maxburst_init(dws); + + dws->ctlr->dma_tx = dws->txchan; + dws->ctlr->dma_rx = dws->rxchan; + + return 0; +} + +static int dw_spi_enh_mem_dma_setup(struct dw_spi *dws) +{ + u16 dma_ctrl, level; + int ret; + + /* Setup DMA channels */ + if (dws->tx_dir) { + ret = dw_spi_dma_config_tx(dws); + if (ret) + return ret; + + dma_ctrl = DW_SPI_DMACR_TDMAE; + } else { + ret = dw_spi_dma_config_rx(dws); + if (ret) + return ret; + + dma_ctrl = DW_SPI_DMACR_RDMAE; + } + + dw_writel(dws, DW_SPI_DMACR, dma_ctrl); + + /* Clear stale error status, it is checked after the transfer */ + dw_readl(dws, DW_SPI_ICR); + + reinit_completion(&dws->dma_completion); + + level = min_t(unsigned int, dws->fifo_len / 2, dws->tx_len); + dw_writel(dws, DW_SPI_TXFTLR, level); + + level = min_t(unsigned int, dws->fifo_len / 2, dws->rx_len); + dw_writel(dws, DW_SPI_RXFTLR, level ? level - 1 : 0); + + return 0; +} + +static bool dw_spi_enh_mem_can_dma(struct spi_controller *ctlr) +{ + struct dw_spi *dws = spi_controller_get_devdata(ctlr); + unsigned int len = dws->tx_dir ? dws->tx_len : dws->rx_len; + enum dma_slave_buswidth dma_bus_width; + + if (len <= dws->fifo_len) + return false; + + dma_bus_width = dw_spi_dma_convert_width(dws->n_bytes); + + return dws->dma_addr_widths & BIT(dma_bus_width); +} + +static void dw_spi_enh_mem_dma_stop(struct dw_spi *dws) +{ + if (dws->tx_dir) { + if (test_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy)) { + dmaengine_terminate_sync(dws->txchan); + clear_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy); + } + } else { + if (test_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy)) { + dmaengine_terminate_sync(dws->rxchan); + clear_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy); + } + } +} + +static int dw_spi_enh_mem_dma_transfer(struct spi_mem *mem, const struct spi_mem_op *op) +{ + struct spi_controller *ctlr = mem->spi->controller; + struct dw_spi *dws = spi_controller_get_devdata(ctlr); + struct sg_table sgt; + int ret; + + ret = spi_controller_dma_map_mem_op_data(ctlr, op, &sgt); + if (ret) + goto out_clear_dmac; + + if (dws->tx_dir) { + ret = dw_spi_dma_submit_tx(dws, sgt.sgl, sgt.nents); + if (ret) + goto out_unmap; + + dma_async_issue_pending(dws->txchan); + + ret = dw_spi_dma_wait(dws, dws->tx_len, dws->current_freq); + if (!ret) + ret = dw_spi_dma_wait_tx_done(dws, dws->current_freq); + } else { + ret = dw_spi_dma_submit_rx(dws, sgt.sgl, sgt.nents); + if (ret) + goto out_unmap; + + dma_async_issue_pending(dws->rxchan); + + ret = dw_spi_dma_wait(dws, dws->rx_len, dws->current_freq); + if (!ret) + ret = dw_spi_dma_wait_rx_done(dws); + } + + if (ret) + dw_spi_enh_mem_dma_stop(dws); + else + ret = dw_spi_check_status(dws, true); + +out_unmap: + spi_controller_dma_unmap_mem_op_data(ctlr, op, &sgt); +out_clear_dmac: + dw_writel(dws, DW_SPI_DMACR, 0); + + return ret; +} + static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops = { .dma_init = dw_spi_dma_init_mfld, .dma_exit = dw_spi_dma_exit, @@ -704,6 +927,11 @@ static const struct dw_spi_dma_ops dw_spi_dma_generic_ops = { .can_dma = dw_spi_can_dma, .dma_transfer = dw_spi_dma_transfer, .dma_stop = dw_spi_dma_stop, + + .dma_enh_mem_get_chan = dw_spi_enh_mem_dma_get_chan, + .dma_enh_mem_setup = dw_spi_enh_mem_dma_setup, + .can_dma_enh_mem = dw_spi_enh_mem_can_dma, + .dma_enh_mem_transfer = dw_spi_enh_mem_dma_transfer, }; void dw_spi_dma_setup_generic(struct dw_spi *dws) diff --git a/drivers/spi/spi-dw-mmio.c b/drivers/spi/spi-dw-mmio.c index 8cdb0351605b..9c70947c969c 100644 --- a/drivers/spi/spi-dw-mmio.c +++ b/drivers/spi/spi-dw-mmio.c @@ -363,6 +363,8 @@ static int dw_spi_jhb100_init(struct platform_device *pdev, dw_spi_jhb100_mask_intr(&dwsmmio->dws, 0xff); + dw_spi_dma_setup_generic(&dwsmmio->dws); + return 0; } diff --git a/drivers/spi/spi-dw.h b/drivers/spi/spi-dw.h index e62f919f1188..bfdf371bcacb 100644 --- a/drivers/spi/spi-dw.h +++ b/drivers/spi/spi-dw.h @@ -183,6 +183,12 @@ struct dw_spi_dma_ops { struct spi_transfer *xfer); int (*dma_transfer)(struct dw_spi *dws, struct spi_transfer *xfer); void (*dma_stop)(struct dw_spi *dws); + + /* enh mem dma ops */ + int (*dma_enh_mem_get_chan)(struct device *dev, struct dw_spi *dws); + int (*dma_enh_mem_setup)(struct dw_spi *dws); + bool (*can_dma_enh_mem)(struct spi_controller *ctlr); + int (*dma_enh_mem_transfer)(struct spi_mem *mem, const struct spi_mem_op *op); }; struct dw_spi { @@ -227,6 +233,8 @@ struct dw_spi { u32 txburst; struct dma_chan *rxchan; u32 rxburst; + unsigned int dma_nr_chans; + bool tx_dir; u32 dma_sg_burst; u32 dma_addr_widths; unsigned long dma_chan_busy;