diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c index 6889f4446ecb..892b3a8787eb 100644 --- a/drivers/spi/spi-dw-core.c +++ b/drivers/spi/spi-dw-core.c @@ -937,6 +937,15 @@ static void dw_spi_enh_write_cmd_addr(struct dw_spi *dws, const struct spi_mem_o dw_spi_set_cs(mem->spi, false); } + + /* + * FIXME: The exact reason for this delay is not fully understood, + * but empirical testing shows it significantly improves the stability + * of read/write operations. Without this delay, occasional transfer + * errors or timeouts may occur under certain conditions. + * Keeping it as a safeguard based on practical validation. + */ + udelay(5); } static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) @@ -948,6 +957,8 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op * unsigned long long ms; int ret; + dws->dma_mapped = false; + switch (op->data.buswidth) { case 0: case 1: @@ -971,10 +982,13 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op * cfg.dfs = 8; cfg.freq = clamp(op->max_freq, 0U, dws->max_mem_freq); cfg.ndf = op->data.nbytes; - if (op->data.dir == SPI_MEM_DATA_IN) + if (op->data.dir == SPI_MEM_DATA_IN) { cfg.tmode = DW_SPI_CTRLR0_TMOD_RO; - else + dws->tx_dir = false; + } else { cfg.tmode = DW_SPI_CTRLR0_TMOD_TO; + dws->tx_dir = true; + } if (op->data.buswidth == op->addr.buswidth && op->data.buswidth == op->cmd.buswidth) @@ -1010,47 +1024,61 @@ static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op * } } - dw_spi_enh_write_cmd_addr(dws, op, mem); - - /* - * FIXME: The exact reason for this delay is not fully understood, - * but empirical testing shows it significantly improves the stability - * of read/write operations. Without this delay, occasional transfer - * errors or timeouts may occur under certain conditions. - * Keeping it as a safeguard based on practical validation. - */ - udelay(5); - - dw_spi_enh_irq_setup(dws); - - /* Use timeout calculation from spi_transfer_wait() */ - ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len); - do_div(ms, dws->current_freq); - - /* - * Increase it twice and add 200 ms tolerance, use - * predefined maximum in case of overflow. - */ - ms += ms + 200; - if (ms > UINT_MAX) - ms = UINT_MAX; - - ms = wait_for_completion_timeout(&ctlr->xfer_completion, - msecs_to_jiffies(ms)); - if (ms == 0) { - dw_spi_mask_intr(dws, 0xff); - synchronize_irq(dws->irq); - dws->rx = NULL; - dws->tx = NULL; - dws->rx_len = 0; - dws->tx_len = 0; - dw_spi_stop_mem_op(dws, mem->spi); - return -EIO; + if (dws->dma_nr_chans && dws->dma_ops->dma_enh_mem_get_chan && + op->data.nbytes > dws->fifo_len) { + ret = dws->dma_ops->dma_enh_mem_get_chan(ctlr->dev.parent, dws); + if (ret) { + /* fall back to the PIO/IRQ path */ + dev_dbg(&ctlr->dev, "DMA enh mem channel unavailable (%d)\n", ret); + } else if (dws->dma_ops->can_dma_enh_mem(ctlr)) { + ret = dws->dma_ops->dma_enh_mem_setup(dws); + if (ret) + dev_err(&ctlr->dev, "DMA enh mem setup failed (%d)\n", ret); + else + dws->dma_mapped = true; + } } - ret = dw_spi_wait_mem_op_done(dws); + if (dws->dma_mapped) { + dw_spi_enh_write_cmd_addr(dws, op, mem); - dw_spi_stop_mem_op(dws, mem->spi); + ret = dws->dma_ops->dma_enh_mem_transfer(mem, op); + + dw_spi_stop_mem_op(dws, mem->spi); + } else { + dw_spi_enh_write_cmd_addr(dws, op, mem); + + dw_spi_enh_irq_setup(dws); + + /* Use timeout calculation from spi_transfer_wait() */ + ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len); + do_div(ms, dws->current_freq); + + /* + * Increase it twice and add 200 ms tolerance, use + * predefined maximum in case of overflow. + */ + ms += ms + 200; + if (ms > UINT_MAX) + ms = UINT_MAX; + + ms = wait_for_completion_timeout(&ctlr->xfer_completion, + msecs_to_jiffies(ms)); + if (ms == 0) { + dw_spi_mask_intr(dws, 0xff); + synchronize_irq(dws->irq); + dws->rx = NULL; + dws->tx = NULL; + dws->rx_len = 0; + dws->tx_len = 0; + dw_spi_stop_mem_op(dws, mem->spi); + return -EIO; + } + + ret = dw_spi_wait_mem_op_done(dws); + + dw_spi_stop_mem_op(dws, mem->spi); + } return ret; } diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c index 76d7814f0dcf..a86039057988 100644 --- a/drivers/spi/spi-dw-dma.c +++ b/drivers/spi/spi-dw-dma.c @@ -728,6 +728,183 @@ static void dw_spi_dma_stop(struct dw_spi *dws) } } +/* + * The enhanced SPI memory operations are half-duplex, so a single DMA channel + * is enough for them. On a platform with only one channel, the channel is + * bound to a direction by its "tx"/"rx" DMA specifier, and whichever channel + * is currently held tells which direction it serves. Keep it across + * operations and swap it only when the transfer direction changes. + */ +static int dw_spi_enh_mem_dma_get_chan(struct device *dev, struct dw_spi *dws) +{ + struct dma_chan **chan, **other; + const char *name; + int ret; + + if (dws->dma_nr_chans == 2) + return 0; + + if (dws->tx_dir) { + chan = &dws->txchan; + other = &dws->rxchan; + name = "tx"; + } else { + chan = &dws->rxchan; + other = &dws->txchan; + name = "rx"; + } + + if (*chan) + return 0; + + dws->ctlr->dma_tx = NULL; + dws->ctlr->dma_rx = NULL; + + /* + * With a single physical channel, the held channel must be released + * before it can be requested again with the handshake of the other + * direction. On failure both stay NULL, and the next operation + * retries the request and falls back to PIO meanwhile. + */ + + if (*other) { + dma_release_channel(*other); + *other = NULL; + } + + *chan = dma_request_chan(dev, name); + if (IS_ERR(*chan)) { + ret = PTR_ERR(*chan); + *chan = NULL; + return ret; + } + + ret = dw_spi_dma_caps_init(dws); + if (ret) { + dma_release_channel(*chan); + *chan = NULL; + return ret; + } + + dw_spi_dma_maxburst_init(dws); + + dws->ctlr->dma_tx = dws->txchan; + dws->ctlr->dma_rx = dws->rxchan; + + return 0; +} + +static int dw_spi_enh_mem_dma_setup(struct dw_spi *dws) +{ + u16 dma_ctrl, level; + int ret; + + /* Setup DMA channels */ + if (dws->tx_dir) { + ret = dw_spi_dma_config_tx(dws); + if (ret) + return ret; + + dma_ctrl = DW_SPI_DMACR_TDMAE; + } else { + ret = dw_spi_dma_config_rx(dws); + if (ret) + return ret; + + dma_ctrl = DW_SPI_DMACR_RDMAE; + } + + dw_writel(dws, DW_SPI_DMACR, dma_ctrl); + + /* Clear stale error status, it is checked after the transfer */ + dw_readl(dws, DW_SPI_ICR); + + reinit_completion(&dws->dma_completion); + + level = min_t(unsigned int, dws->fifo_len / 2, dws->tx_len); + dw_writel(dws, DW_SPI_TXFTLR, level); + + level = min_t(unsigned int, dws->fifo_len / 2, dws->rx_len); + dw_writel(dws, DW_SPI_RXFTLR, level ? level - 1 : 0); + + return 0; +} + +static bool dw_spi_enh_mem_can_dma(struct spi_controller *ctlr) +{ + struct dw_spi *dws = spi_controller_get_devdata(ctlr); + unsigned int len = dws->tx_dir ? dws->tx_len : dws->rx_len; + enum dma_slave_buswidth dma_bus_width; + + if (len <= dws->fifo_len) + return false; + + dma_bus_width = dw_spi_dma_convert_width(dws->n_bytes); + + return dws->dma_addr_widths & BIT(dma_bus_width); +} + +static void dw_spi_enh_mem_dma_stop(struct dw_spi *dws) +{ + if (dws->tx_dir) { + if (test_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy)) { + dmaengine_terminate_sync(dws->txchan); + clear_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy); + } + } else { + if (test_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy)) { + dmaengine_terminate_sync(dws->rxchan); + clear_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy); + } + } +} + +static int dw_spi_enh_mem_dma_transfer(struct spi_mem *mem, const struct spi_mem_op *op) +{ + struct spi_controller *ctlr = mem->spi->controller; + struct dw_spi *dws = spi_controller_get_devdata(ctlr); + struct sg_table sgt; + int ret; + + ret = spi_controller_dma_map_mem_op_data(ctlr, op, &sgt); + if (ret) + goto out_clear_dmac; + + if (dws->tx_dir) { + ret = dw_spi_dma_submit_tx(dws, sgt.sgl, sgt.nents); + if (ret) + goto out_unmap; + + dma_async_issue_pending(dws->txchan); + + ret = dw_spi_dma_wait(dws, dws->tx_len, dws->current_freq); + if (!ret) + ret = dw_spi_dma_wait_tx_done(dws, dws->current_freq); + } else { + ret = dw_spi_dma_submit_rx(dws, sgt.sgl, sgt.nents); + if (ret) + goto out_unmap; + + dma_async_issue_pending(dws->rxchan); + + ret = dw_spi_dma_wait(dws, dws->rx_len, dws->current_freq); + if (!ret) + ret = dw_spi_dma_wait_rx_done(dws); + } + + if (ret) + dw_spi_enh_mem_dma_stop(dws); + else + ret = dw_spi_check_status(dws, true); + +out_unmap: + spi_controller_dma_unmap_mem_op_data(ctlr, op, &sgt); +out_clear_dmac: + dw_writel(dws, DW_SPI_DMACR, 0); + + return ret; +} + static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops = { .dma_init = dw_spi_dma_init_mfld, .dma_exit = dw_spi_dma_exit, @@ -750,6 +927,11 @@ static const struct dw_spi_dma_ops dw_spi_dma_generic_ops = { .can_dma = dw_spi_can_dma, .dma_transfer = dw_spi_dma_transfer, .dma_stop = dw_spi_dma_stop, + + .dma_enh_mem_get_chan = dw_spi_enh_mem_dma_get_chan, + .dma_enh_mem_setup = dw_spi_enh_mem_dma_setup, + .can_dma_enh_mem = dw_spi_enh_mem_can_dma, + .dma_enh_mem_transfer = dw_spi_enh_mem_dma_transfer, }; void dw_spi_dma_setup_generic(struct dw_spi *dws) diff --git a/drivers/spi/spi-dw-mmio.c b/drivers/spi/spi-dw-mmio.c index 8cdb0351605b..9c70947c969c 100644 --- a/drivers/spi/spi-dw-mmio.c +++ b/drivers/spi/spi-dw-mmio.c @@ -363,6 +363,8 @@ static int dw_spi_jhb100_init(struct platform_device *pdev, dw_spi_jhb100_mask_intr(&dwsmmio->dws, 0xff); + dw_spi_dma_setup_generic(&dwsmmio->dws); + return 0; } diff --git a/drivers/spi/spi-dw.h b/drivers/spi/spi-dw.h index 01eb24ae7fe0..bfdf371bcacb 100644 --- a/drivers/spi/spi-dw.h +++ b/drivers/spi/spi-dw.h @@ -183,6 +183,12 @@ struct dw_spi_dma_ops { struct spi_transfer *xfer); int (*dma_transfer)(struct dw_spi *dws, struct spi_transfer *xfer); void (*dma_stop)(struct dw_spi *dws); + + /* enh mem dma ops */ + int (*dma_enh_mem_get_chan)(struct device *dev, struct dw_spi *dws); + int (*dma_enh_mem_setup)(struct dw_spi *dws); + bool (*can_dma_enh_mem)(struct spi_controller *ctlr); + int (*dma_enh_mem_transfer)(struct spi_mem *mem, const struct spi_mem_op *op); }; struct dw_spi { @@ -228,6 +234,7 @@ struct dw_spi { struct dma_chan *rxchan; u32 rxburst; unsigned int dma_nr_chans; + bool tx_dir; u32 dma_sg_burst; u32 dma_addr_widths; unsigned long dma_chan_busy;