mirror of
https://github.com/torvalds/linux.git
synced 2026-05-28 17:13:52 +02:00
Support ECSPI dynamic burst feature for DMA mode
Merge series from Carlos Song <carlos.song@nxp.com>:
ECSPI has a low throughput because of no dynamic burst support, it
transfers only one word per frame in DMA mode, causing SCLK stalls
between words due to BURST_LENGTH updates.
This patch set is to support ECSPI dynamic burst feature to help improve
the ECSPI DMA mode performance.
Performance test (spidev_test @10MHz, 4KB):
Before: tx/rx ~6651.9 kbps
After: tx/rx ~9922.2 kbps (~50% improvement)
For compatibility with slow SPI devices, add configurable word delay in
DMA mode. When word delay is set, dynamic burst is disabled and
BURST_LENGTH equals word length.
Also support target DMA mode with enabled dynamic burst.
---
Changes since v2:
* Patch1~4: no change
* Patch5: remove __counted_by from *dma_data, because spi_imx struct
is allocated with a fixed memory in probe, flexible array is not
suitable, so just remove it to avoid the build error. Also add
review-by tag
* Patch6: add review-by tag
Changes since v1:
* Patch1~3:
* Add review-by tag
* Patch4:
* Add review-by tag
* Improve patch commit log
* Patch5:
* Clean up tab instead of space for dma_tx_addr and dma_rx_addr
* Add __counted_by(dma_package_num) for dma_data_package *dma_data
* Move temp and bytes_per_word to if()
* Add comment for spi_imx_dma_rx_data_handle() and
spi_imx_dma_tx_data_handle()
* Use kzalloc instead of kmalloc(x, x | __GFP_ZERO)
* Use sizeof(*temp) instead of BYTES_PER_32BITS_WORD in suitable place
* Add comments and fix comments according to discussion
* Propagate the error value of dma_mapping_error()
* Use round_down() to replace DIV_ROUND_DOWN_ULL()
* Improve transfer->error |= SPI_TRANS_FAIL_NO_START judgment for
better readability
* Patch6:
* Improve patch commit log
* Add READ_ONCE for spi_imx->target_aborted
* Add comments for data length limit
* Move data length limit to spi_imx_can_dma()
Carlos Song (6):
spi: imx: group spi_imx_dma_configure() with spi_imx_dma_transfer()
spi: imx: introduce helper to clear DMA mode logic
spi: imx: avoid dmaengine_terminate_all() on TX prep failure
spi: imx: handle DMA submission errors with dma_submit_error()
spi: imx: support dynamic burst length for ECSPI DMA mode
spi: imx: enable DMA mode for target operation
drivers/spi/spi-imx.c | 625 +++++++++++++++++++++++++++++++++++-------
1 file changed, 519 insertions(+), 106 deletions(-)
--
2.34.1
This commit is contained in:
Mark Brown
commit
c73618541c
|
|
@ -60,6 +60,7 @@ MODULE_PARM_DESC(polling_limit_us,
|
|||
#define MX51_ECSPI_CTRL_MAX_BURST 512
|
||||
/* The maximum bytes that IMX53_ECSPI can transfer in target mode.*/
|
||||
#define MX53_MAX_TRANSFER_BYTES 512
|
||||
#define BYTES_PER_32BITS_WORD 4
|
||||
|
||||
enum spi_imx_devtype {
|
||||
IMX1_CSPI,
|
||||
|
|
@ -95,6 +96,16 @@ struct spi_imx_devtype_data {
|
|||
enum spi_imx_devtype devtype;
|
||||
};
|
||||
|
||||
struct dma_data_package {
|
||||
u32 cmd_word;
|
||||
void *dma_rx_buf;
|
||||
void *dma_tx_buf;
|
||||
dma_addr_t dma_tx_addr;
|
||||
dma_addr_t dma_rx_addr;
|
||||
int dma_len;
|
||||
int data_len;
|
||||
};
|
||||
|
||||
struct spi_imx_data {
|
||||
struct spi_controller *controller;
|
||||
struct device *dev;
|
||||
|
|
@ -130,6 +141,9 @@ struct spi_imx_data {
|
|||
u32 wml;
|
||||
struct completion dma_rx_completion;
|
||||
struct completion dma_tx_completion;
|
||||
size_t dma_package_num;
|
||||
struct dma_data_package *dma_data;
|
||||
int rx_offset;
|
||||
|
||||
const struct spi_imx_devtype_data *devtype_data;
|
||||
};
|
||||
|
|
@ -189,6 +203,9 @@ MXC_SPI_BUF_TX(u16)
|
|||
MXC_SPI_BUF_RX(u32)
|
||||
MXC_SPI_BUF_TX(u32)
|
||||
|
||||
/* Align to cache line to avoid swiotlo bounce */
|
||||
#define DMA_CACHE_ALIGNED_LEN(x) ALIGN((x), dma_get_cache_alignment())
|
||||
|
||||
/* First entry is reserved, second entry is valid only if SDHC_SPIEN is set
|
||||
* (which is currently not the case in this driver)
|
||||
*/
|
||||
|
|
@ -247,12 +264,26 @@ static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device
|
|||
if (!controller->dma_rx)
|
||||
return false;
|
||||
|
||||
if (spi_imx->target_mode)
|
||||
/*
|
||||
* Due to Freescale errata ERR003775 "eCSPI: Burst completion by Chip
|
||||
* Select (SS) signal in Slave mode is not functional" burst size must
|
||||
* be set exactly to the size of the transfer. This limit SPI transaction
|
||||
* with maximum 2^12 bits.
|
||||
*/
|
||||
if (transfer->len > MX53_MAX_TRANSFER_BYTES && spi_imx->target_mode)
|
||||
return false;
|
||||
|
||||
if (transfer->len < spi_imx->devtype_data->fifo_size)
|
||||
return false;
|
||||
|
||||
/* DMA only can transmit data in bytes */
|
||||
if (spi_imx->bits_per_word != 8 && spi_imx->bits_per_word != 16 &&
|
||||
spi_imx->bits_per_word != 32)
|
||||
return false;
|
||||
|
||||
if (transfer->len >= MAX_SDMA_BD_BYTES)
|
||||
return false;
|
||||
|
||||
spi_imx->dynamic_burst = 0;
|
||||
|
||||
return true;
|
||||
|
|
@ -1282,50 +1313,6 @@ static irqreturn_t spi_imx_isr(int irq, void *dev_id)
|
|||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
static int spi_imx_dma_configure(struct spi_controller *controller)
|
||||
{
|
||||
int ret;
|
||||
enum dma_slave_buswidth buswidth;
|
||||
struct dma_slave_config rx = {}, tx = {};
|
||||
struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
|
||||
|
||||
switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {
|
||||
case 4:
|
||||
buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
break;
|
||||
case 2:
|
||||
buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
|
||||
break;
|
||||
case 1:
|
||||
buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
tx.direction = DMA_MEM_TO_DEV;
|
||||
tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
|
||||
tx.dst_addr_width = buswidth;
|
||||
tx.dst_maxburst = spi_imx->wml;
|
||||
ret = dmaengine_slave_config(controller->dma_tx, &tx);
|
||||
if (ret) {
|
||||
dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
rx.direction = DMA_DEV_TO_MEM;
|
||||
rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
|
||||
rx.src_addr_width = buswidth;
|
||||
rx.src_maxburst = spi_imx->wml;
|
||||
ret = dmaengine_slave_config(controller->dma_rx, &rx);
|
||||
if (ret) {
|
||||
dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int spi_imx_setupxfer(struct spi_device *spi,
|
||||
struct spi_transfer *t)
|
||||
{
|
||||
|
|
@ -1442,8 +1429,6 @@ static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,
|
|||
|
||||
init_completion(&spi_imx->dma_rx_completion);
|
||||
init_completion(&spi_imx->dma_tx_completion);
|
||||
controller->can_dma = spi_imx_can_dma;
|
||||
controller->max_dma_len = MAX_SDMA_BD_BYTES;
|
||||
spi_imx->controller->flags = SPI_CONTROLLER_MUST_RX |
|
||||
SPI_CONTROLLER_MUST_TX;
|
||||
|
||||
|
|
@ -1481,31 +1466,445 @@ static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size)
|
|||
return secs_to_jiffies(2 * timeout);
|
||||
}
|
||||
|
||||
static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
|
||||
struct spi_transfer *transfer)
|
||||
static void spi_imx_dma_unmap(struct spi_imx_data *spi_imx,
|
||||
struct dma_data_package *dma_data)
|
||||
{
|
||||
struct device *tx_dev = spi_imx->controller->dma_tx->device->dev;
|
||||
struct device *rx_dev = spi_imx->controller->dma_rx->device->dev;
|
||||
|
||||
dma_unmap_single(tx_dev, dma_data->dma_tx_addr,
|
||||
DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
|
||||
DMA_TO_DEVICE);
|
||||
dma_unmap_single(rx_dev, dma_data->dma_rx_addr,
|
||||
DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
|
||||
DMA_FROM_DEVICE);
|
||||
}
|
||||
|
||||
static void spi_imx_dma_rx_data_handle(struct spi_imx_data *spi_imx,
|
||||
struct dma_data_package *dma_data, void *rx_buf,
|
||||
bool word_delay)
|
||||
{
|
||||
void *copy_ptr;
|
||||
int unaligned;
|
||||
|
||||
/*
|
||||
* On little-endian CPUs, adjust byte order:
|
||||
* - Swap bytes when bpw = 8
|
||||
* - Swap half-words when bpw = 16
|
||||
* This ensures correct data ordering for DMA transfers.
|
||||
*/
|
||||
#ifdef __LITTLE_ENDIAN
|
||||
if (!word_delay) {
|
||||
unsigned int bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
|
||||
u32 *temp = dma_data->dma_rx_buf;
|
||||
|
||||
for (int i = 0; i < DIV_ROUND_UP(dma_data->dma_len, sizeof(*temp)); i++) {
|
||||
if (bytes_per_word == 1)
|
||||
swab32s(temp + i);
|
||||
else if (bytes_per_word == 2)
|
||||
swahw32s(temp + i);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* When dynamic burst enabled, DMA RX always receives 32-bit words from RXFIFO with
|
||||
* buswidth = 4, but when data_len is not 4-bytes alignment, the RM shows when
|
||||
* burst length = 32*n + m bits, a SPI burst contains the m LSB in first word and all
|
||||
* 32 bits in other n words. So if garbage bytes in the first word, trim first word then
|
||||
* copy the actual data to rx_buf.
|
||||
*/
|
||||
if (dma_data->data_len % BYTES_PER_32BITS_WORD && !word_delay) {
|
||||
unaligned = dma_data->data_len % BYTES_PER_32BITS_WORD;
|
||||
copy_ptr = (u8 *)dma_data->dma_rx_buf + BYTES_PER_32BITS_WORD - unaligned;
|
||||
} else {
|
||||
copy_ptr = dma_data->dma_rx_buf;
|
||||
}
|
||||
|
||||
memcpy(rx_buf, copy_ptr, dma_data->data_len);
|
||||
}
|
||||
|
||||
static int spi_imx_dma_map(struct spi_imx_data *spi_imx,
|
||||
struct dma_data_package *dma_data)
|
||||
{
|
||||
struct spi_controller *controller = spi_imx->controller;
|
||||
struct device *tx_dev = controller->dma_tx->device->dev;
|
||||
struct device *rx_dev = controller->dma_rx->device->dev;
|
||||
int ret;
|
||||
|
||||
dma_data->dma_tx_addr = dma_map_single(tx_dev, dma_data->dma_tx_buf,
|
||||
DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
|
||||
DMA_TO_DEVICE);
|
||||
ret = dma_mapping_error(tx_dev, dma_data->dma_tx_addr);
|
||||
if (ret < 0) {
|
||||
dev_err(spi_imx->dev, "DMA TX map failed %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
dma_data->dma_rx_addr = dma_map_single(rx_dev, dma_data->dma_rx_buf,
|
||||
DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
|
||||
DMA_FROM_DEVICE);
|
||||
ret = dma_mapping_error(rx_dev, dma_data->dma_rx_addr);
|
||||
if (ret < 0) {
|
||||
dev_err(spi_imx->dev, "DMA RX map failed %d\n", ret);
|
||||
dma_unmap_single(tx_dev, dma_data->dma_tx_addr,
|
||||
DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
|
||||
DMA_TO_DEVICE);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int spi_imx_dma_tx_data_handle(struct spi_imx_data *spi_imx,
|
||||
struct dma_data_package *dma_data,
|
||||
const void *tx_buf,
|
||||
bool word_delay)
|
||||
{
|
||||
void *copy_ptr;
|
||||
int unaligned;
|
||||
|
||||
if (word_delay) {
|
||||
dma_data->dma_len = dma_data->data_len;
|
||||
} else {
|
||||
/*
|
||||
* As per the reference manual, when burst length = 32*n + m bits, ECSPI
|
||||
* sends m LSB bits in the first word, followed by n full 32-bit words.
|
||||
* Since actual data may not be 4-byte aligned, allocate DMA TX/RX buffers
|
||||
* to ensure alignment. For TX, DMA pushes 4-byte aligned words to TXFIFO,
|
||||
* while ECSPI uses BURST_LENGTH settings to maintain correct bit count.
|
||||
* For RX, DMA always receives 32-bit words from RXFIFO, when data len is
|
||||
* not 4-byte aligned, trim the first word to drop garbage bytes, then group
|
||||
* all transfer DMA bounse buffer and copy all valid data to rx_buf.
|
||||
*/
|
||||
dma_data->dma_len = ALIGN(dma_data->data_len, BYTES_PER_32BITS_WORD);
|
||||
}
|
||||
|
||||
dma_data->dma_tx_buf = kzalloc(dma_data->dma_len, GFP_KERNEL);
|
||||
if (!dma_data->dma_tx_buf)
|
||||
return -ENOMEM;
|
||||
|
||||
dma_data->dma_rx_buf = kzalloc(dma_data->dma_len, GFP_KERNEL);
|
||||
if (!dma_data->dma_rx_buf) {
|
||||
kfree(dma_data->dma_tx_buf);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
if (dma_data->data_len % BYTES_PER_32BITS_WORD && !word_delay) {
|
||||
unaligned = dma_data->data_len % BYTES_PER_32BITS_WORD;
|
||||
copy_ptr = (u8 *)dma_data->dma_tx_buf + BYTES_PER_32BITS_WORD - unaligned;
|
||||
} else {
|
||||
copy_ptr = dma_data->dma_tx_buf;
|
||||
}
|
||||
|
||||
memcpy(copy_ptr, tx_buf, dma_data->data_len);
|
||||
|
||||
/*
|
||||
* When word_delay is enabled, DMA transfers an entire word in one minor loop.
|
||||
* In this case, no data requires additional handling.
|
||||
*/
|
||||
if (word_delay)
|
||||
return 0;
|
||||
|
||||
#ifdef __LITTLE_ENDIAN
|
||||
/*
|
||||
* On little-endian CPUs, adjust byte order:
|
||||
* - Swap bytes when bpw = 8
|
||||
* - Swap half-words when bpw = 16
|
||||
* This ensures correct data ordering for DMA transfers.
|
||||
*/
|
||||
unsigned int bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
|
||||
u32 *temp = dma_data->dma_tx_buf;
|
||||
|
||||
for (int i = 0; i < DIV_ROUND_UP(dma_data->dma_len, sizeof(*temp)); i++) {
|
||||
if (bytes_per_word == 1)
|
||||
swab32s(temp + i);
|
||||
else if (bytes_per_word == 2)
|
||||
swahw32s(temp + i);
|
||||
}
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int spi_imx_dma_data_prepare(struct spi_imx_data *spi_imx,
|
||||
struct spi_transfer *transfer,
|
||||
bool word_delay)
|
||||
{
|
||||
u32 pre_bl, tail_bl;
|
||||
u32 ctrl;
|
||||
int ret;
|
||||
|
||||
/*
|
||||
* ECSPI supports a maximum burst of 512 bytes. When xfer->len exceeds 512
|
||||
* and is not a multiple of 512, a tail transfer is required. BURST_LEGTH
|
||||
* is used for SPI HW to maintain correct bit count. BURST_LENGTH should
|
||||
* update with data length. After DMA request submit, SPI can not update the
|
||||
* BURST_LENGTH, in this case, we must split two package, update the register
|
||||
* then setup second DMA transfer.
|
||||
*/
|
||||
ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
|
||||
if (word_delay) {
|
||||
/*
|
||||
* When SPI IMX need to support word delay, according to "Sample Period Control
|
||||
* Register" shows, The Sample Period Control Register (ECSPI_PERIODREG)
|
||||
* provides software a way to insert delays (wait states) between consecutive
|
||||
* SPI transfers. As a result, ECSPI can only transfer one word per frame, and
|
||||
* the delay occurs between frames.
|
||||
*/
|
||||
spi_imx->dma_package_num = 1;
|
||||
pre_bl = spi_imx->bits_per_word - 1;
|
||||
} else if (transfer->len <= MX51_ECSPI_CTRL_MAX_BURST) {
|
||||
spi_imx->dma_package_num = 1;
|
||||
pre_bl = transfer->len * BITS_PER_BYTE - 1;
|
||||
} else if (!(transfer->len % MX51_ECSPI_CTRL_MAX_BURST)) {
|
||||
spi_imx->dma_package_num = 1;
|
||||
pre_bl = MX51_ECSPI_CTRL_MAX_BURST * BITS_PER_BYTE - 1;
|
||||
} else {
|
||||
spi_imx->dma_package_num = 2;
|
||||
pre_bl = MX51_ECSPI_CTRL_MAX_BURST * BITS_PER_BYTE - 1;
|
||||
tail_bl = (transfer->len % MX51_ECSPI_CTRL_MAX_BURST) * BITS_PER_BYTE - 1;
|
||||
}
|
||||
|
||||
spi_imx->dma_data = kmalloc_array(spi_imx->dma_package_num,
|
||||
sizeof(struct dma_data_package),
|
||||
GFP_KERNEL | __GFP_ZERO);
|
||||
if (!spi_imx->dma_data) {
|
||||
dev_err(spi_imx->dev, "Failed to allocate DMA package buffer!\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
if (spi_imx->dma_package_num == 1) {
|
||||
ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
|
||||
ctrl |= pre_bl << MX51_ECSPI_CTRL_BL_OFFSET;
|
||||
spi_imx->dma_data[0].cmd_word = ctrl;
|
||||
spi_imx->dma_data[0].data_len = transfer->len;
|
||||
ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[0], transfer->tx_buf,
|
||||
word_delay);
|
||||
if (ret) {
|
||||
kfree(spi_imx->dma_data);
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
|
||||
ctrl |= pre_bl << MX51_ECSPI_CTRL_BL_OFFSET;
|
||||
spi_imx->dma_data[0].cmd_word = ctrl;
|
||||
spi_imx->dma_data[0].data_len = round_down(transfer->len,
|
||||
MX51_ECSPI_CTRL_MAX_BURST);
|
||||
ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[0], transfer->tx_buf,
|
||||
false);
|
||||
if (ret) {
|
||||
kfree(spi_imx->dma_data);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
|
||||
ctrl |= tail_bl << MX51_ECSPI_CTRL_BL_OFFSET;
|
||||
spi_imx->dma_data[1].cmd_word = ctrl;
|
||||
spi_imx->dma_data[1].data_len = transfer->len % MX51_ECSPI_CTRL_MAX_BURST;
|
||||
ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[1],
|
||||
transfer->tx_buf + spi_imx->dma_data[0].data_len,
|
||||
false);
|
||||
if (ret) {
|
||||
kfree(spi_imx->dma_data[0].dma_tx_buf);
|
||||
kfree(spi_imx->dma_data[0].dma_rx_buf);
|
||||
kfree(spi_imx->dma_data);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int spi_imx_dma_submit(struct spi_imx_data *spi_imx,
|
||||
struct dma_data_package *dma_data,
|
||||
struct spi_transfer *transfer)
|
||||
{
|
||||
struct spi_controller *controller = spi_imx->controller;
|
||||
struct dma_async_tx_descriptor *desc_tx, *desc_rx;
|
||||
unsigned long transfer_timeout;
|
||||
unsigned long time_left;
|
||||
struct spi_controller *controller = spi_imx->controller;
|
||||
struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
|
||||
struct scatterlist *last_sg = sg_last(rx->sgl, rx->nents);
|
||||
unsigned int bytes_per_word, i;
|
||||
int ret;
|
||||
dma_cookie_t cookie;
|
||||
|
||||
/*
|
||||
* The TX DMA setup starts the transfer, so make sure RX is configured
|
||||
* before TX.
|
||||
*/
|
||||
desc_rx = dmaengine_prep_slave_single(controller->dma_rx, dma_data->dma_rx_addr,
|
||||
dma_data->dma_len, DMA_DEV_TO_MEM,
|
||||
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
|
||||
if (!desc_rx) {
|
||||
transfer->error |= SPI_TRANS_FAIL_NO_START;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
desc_rx->callback = spi_imx_dma_rx_callback;
|
||||
desc_rx->callback_param = (void *)spi_imx;
|
||||
cookie = dmaengine_submit(desc_rx);
|
||||
if (dma_submit_error(cookie)) {
|
||||
dev_err(spi_imx->dev, "submitting DMA RX failed\n");
|
||||
transfer->error |= SPI_TRANS_FAIL_NO_START;
|
||||
goto dmaengine_terminate_rx;
|
||||
}
|
||||
|
||||
reinit_completion(&spi_imx->dma_rx_completion);
|
||||
dma_async_issue_pending(controller->dma_rx);
|
||||
|
||||
desc_tx = dmaengine_prep_slave_single(controller->dma_tx, dma_data->dma_tx_addr,
|
||||
dma_data->dma_len, DMA_MEM_TO_DEV,
|
||||
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
|
||||
if (!desc_tx)
|
||||
goto dmaengine_terminate_rx;
|
||||
|
||||
desc_tx->callback = spi_imx_dma_tx_callback;
|
||||
desc_tx->callback_param = (void *)spi_imx;
|
||||
cookie = dmaengine_submit(desc_tx);
|
||||
if (dma_submit_error(cookie)) {
|
||||
dev_err(spi_imx->dev, "submitting DMA TX failed\n");
|
||||
goto dmaengine_terminate_tx;
|
||||
}
|
||||
reinit_completion(&spi_imx->dma_tx_completion);
|
||||
dma_async_issue_pending(controller->dma_tx);
|
||||
|
||||
spi_imx->devtype_data->trigger(spi_imx);
|
||||
|
||||
transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
|
||||
|
||||
if (!spi_imx->target_mode) {
|
||||
/* Wait SDMA to finish the data transfer.*/
|
||||
time_left = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
|
||||
transfer_timeout);
|
||||
if (!time_left) {
|
||||
dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
|
||||
dmaengine_terminate_all(controller->dma_tx);
|
||||
dmaengine_terminate_all(controller->dma_rx);
|
||||
return -ETIMEDOUT;
|
||||
}
|
||||
|
||||
time_left = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
|
||||
transfer_timeout);
|
||||
if (!time_left) {
|
||||
dev_err(&controller->dev, "I/O Error in DMA RX\n");
|
||||
spi_imx->devtype_data->reset(spi_imx);
|
||||
dmaengine_terminate_all(controller->dma_rx);
|
||||
return -ETIMEDOUT;
|
||||
}
|
||||
} else {
|
||||
spi_imx->target_aborted = false;
|
||||
|
||||
if (wait_for_completion_interruptible(&spi_imx->dma_tx_completion) ||
|
||||
READ_ONCE(spi_imx->target_aborted)) {
|
||||
dev_dbg(spi_imx->dev, "I/O Error in DMA TX interrupted\n");
|
||||
dmaengine_terminate_all(controller->dma_tx);
|
||||
dmaengine_terminate_all(controller->dma_rx);
|
||||
return -EINTR;
|
||||
}
|
||||
|
||||
if (wait_for_completion_interruptible(&spi_imx->dma_rx_completion) ||
|
||||
READ_ONCE(spi_imx->target_aborted)) {
|
||||
dev_dbg(spi_imx->dev, "I/O Error in DMA RX interrupted\n");
|
||||
dmaengine_terminate_all(controller->dma_rx);
|
||||
return -EINTR;
|
||||
}
|
||||
|
||||
/*
|
||||
* ECSPI has a HW issue when works in Target mode, after 64 words
|
||||
* writtern to TXFIFO, even TXFIFO becomes empty, ECSPI_TXDATA keeps
|
||||
* shift out the last word data, so we have to disable ECSPI when in
|
||||
* target mode after the transfer completes.
|
||||
*/
|
||||
if (spi_imx->devtype_data->disable)
|
||||
spi_imx->devtype_data->disable(spi_imx);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
dmaengine_terminate_tx:
|
||||
dmaengine_terminate_all(controller->dma_tx);
|
||||
dmaengine_terminate_rx:
|
||||
dmaengine_terminate_all(controller->dma_rx);
|
||||
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static void spi_imx_dma_max_wml_find(struct spi_imx_data *spi_imx,
|
||||
struct dma_data_package *dma_data,
|
||||
bool word_delay)
|
||||
{
|
||||
unsigned int bytes_per_word = word_delay ?
|
||||
spi_imx_bytes_per_word(spi_imx->bits_per_word) :
|
||||
BYTES_PER_32BITS_WORD;
|
||||
unsigned int i;
|
||||
|
||||
/* Get the right burst length from the last sg to ensure no tail data */
|
||||
bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word);
|
||||
for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
|
||||
if (!(sg_dma_len(last_sg) % (i * bytes_per_word)))
|
||||
if (!dma_data->dma_len % (i * bytes_per_word))
|
||||
break;
|
||||
}
|
||||
/* Use 1 as wml in case no available burst length got */
|
||||
if (i == 0)
|
||||
i = 1;
|
||||
|
||||
spi_imx->wml = i;
|
||||
spi_imx->wml = i;
|
||||
}
|
||||
|
||||
ret = spi_imx_dma_configure(controller);
|
||||
static int spi_imx_dma_configure(struct spi_controller *controller, bool word_delay)
|
||||
{
|
||||
int ret;
|
||||
enum dma_slave_buswidth buswidth;
|
||||
struct dma_slave_config rx = {}, tx = {};
|
||||
struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
|
||||
|
||||
if (word_delay) {
|
||||
switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {
|
||||
case 4:
|
||||
buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
break;
|
||||
case 2:
|
||||
buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
|
||||
break;
|
||||
case 1:
|
||||
buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
} else {
|
||||
buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
}
|
||||
|
||||
tx.direction = DMA_MEM_TO_DEV;
|
||||
tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
|
||||
tx.dst_addr_width = buswidth;
|
||||
tx.dst_maxburst = spi_imx->wml;
|
||||
ret = dmaengine_slave_config(controller->dma_tx, &tx);
|
||||
if (ret) {
|
||||
dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
rx.direction = DMA_DEV_TO_MEM;
|
||||
rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
|
||||
rx.src_addr_width = buswidth;
|
||||
rx.src_maxburst = spi_imx->wml;
|
||||
ret = dmaengine_slave_config(controller->dma_rx, &rx);
|
||||
if (ret) {
|
||||
dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int spi_imx_dma_package_transfer(struct spi_imx_data *spi_imx,
|
||||
struct dma_data_package *dma_data,
|
||||
struct spi_transfer *transfer,
|
||||
bool word_delay)
|
||||
{
|
||||
struct spi_controller *controller = spi_imx->controller;
|
||||
int ret;
|
||||
|
||||
spi_imx_dma_max_wml_find(spi_imx, dma_data, word_delay);
|
||||
|
||||
ret = spi_imx_dma_configure(controller, word_delay);
|
||||
if (ret)
|
||||
goto dma_failure_no_start;
|
||||
|
||||
|
|
@ -1516,61 +1915,16 @@ static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
|
|||
}
|
||||
spi_imx->devtype_data->setup_wml(spi_imx);
|
||||
|
||||
/*
|
||||
* The TX DMA setup starts the transfer, so make sure RX is configured
|
||||
* before TX.
|
||||
*/
|
||||
desc_rx = dmaengine_prep_slave_sg(controller->dma_rx,
|
||||
rx->sgl, rx->nents, DMA_DEV_TO_MEM,
|
||||
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
|
||||
if (!desc_rx) {
|
||||
ret = -EINVAL;
|
||||
goto dma_failure_no_start;
|
||||
}
|
||||
ret = spi_imx_dma_submit(spi_imx, dma_data, transfer);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
desc_rx->callback = spi_imx_dma_rx_callback;
|
||||
desc_rx->callback_param = (void *)spi_imx;
|
||||
dmaengine_submit(desc_rx);
|
||||
reinit_completion(&spi_imx->dma_rx_completion);
|
||||
dma_async_issue_pending(controller->dma_rx);
|
||||
|
||||
desc_tx = dmaengine_prep_slave_sg(controller->dma_tx,
|
||||
tx->sgl, tx->nents, DMA_MEM_TO_DEV,
|
||||
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
|
||||
if (!desc_tx) {
|
||||
dmaengine_terminate_all(controller->dma_tx);
|
||||
dmaengine_terminate_all(controller->dma_rx);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
desc_tx->callback = spi_imx_dma_tx_callback;
|
||||
desc_tx->callback_param = (void *)spi_imx;
|
||||
dmaengine_submit(desc_tx);
|
||||
reinit_completion(&spi_imx->dma_tx_completion);
|
||||
dma_async_issue_pending(controller->dma_tx);
|
||||
|
||||
spi_imx->devtype_data->trigger(spi_imx);
|
||||
|
||||
transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
|
||||
|
||||
/* Wait SDMA to finish the data transfer.*/
|
||||
time_left = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
|
||||
transfer_timeout);
|
||||
if (!time_left) {
|
||||
dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
|
||||
dmaengine_terminate_all(controller->dma_tx);
|
||||
dmaengine_terminate_all(controller->dma_rx);
|
||||
return -ETIMEDOUT;
|
||||
}
|
||||
|
||||
time_left = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
|
||||
transfer_timeout);
|
||||
if (!time_left) {
|
||||
dev_err(&controller->dev, "I/O Error in DMA RX\n");
|
||||
spi_imx->devtype_data->reset(spi_imx);
|
||||
dmaengine_terminate_all(controller->dma_rx);
|
||||
return -ETIMEDOUT;
|
||||
}
|
||||
/* Trim the DMA RX buffer and copy the actual data to rx_buf */
|
||||
dma_sync_single_for_cpu(controller->dma_rx->device->dev, dma_data->dma_rx_addr,
|
||||
dma_data->dma_len, DMA_FROM_DEVICE);
|
||||
spi_imx_dma_rx_data_handle(spi_imx, dma_data, transfer->rx_buf + spi_imx->rx_offset,
|
||||
word_delay);
|
||||
spi_imx->rx_offset += dma_data->data_len;
|
||||
|
||||
return 0;
|
||||
/* fallback to pio */
|
||||
|
|
@ -1579,6 +1933,57 @@ static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
|
||||
struct spi_transfer *transfer)
|
||||
{
|
||||
bool word_delay = transfer->word_delay.value != 0 && !spi_imx->target_mode;
|
||||
int ret;
|
||||
int i;
|
||||
|
||||
ret = spi_imx_dma_data_prepare(spi_imx, transfer, word_delay);
|
||||
if (ret < 0) {
|
||||
transfer->error |= SPI_TRANS_FAIL_NO_START;
|
||||
dev_err(spi_imx->dev, "DMA data prepare fail\n");
|
||||
goto fallback_pio;
|
||||
}
|
||||
|
||||
spi_imx->rx_offset = 0;
|
||||
|
||||
/* Each dma_package performs a separate DMA transfer once */
|
||||
for (i = 0; i < spi_imx->dma_package_num; i++) {
|
||||
ret = spi_imx_dma_map(spi_imx, &spi_imx->dma_data[i]);
|
||||
if (ret < 0) {
|
||||
if (i == 0)
|
||||
transfer->error |= SPI_TRANS_FAIL_NO_START;
|
||||
dev_err(spi_imx->dev, "DMA map fail\n");
|
||||
break;
|
||||
}
|
||||
|
||||
/* Update the CTRL register BL field */
|
||||
writel(spi_imx->dma_data[i].cmd_word, spi_imx->base + MX51_ECSPI_CTRL);
|
||||
|
||||
ret = spi_imx_dma_package_transfer(spi_imx, &spi_imx->dma_data[i],
|
||||
transfer, word_delay);
|
||||
|
||||
/* Whether the dma transmission is successful or not, dma unmap is necessary */
|
||||
spi_imx_dma_unmap(spi_imx, &spi_imx->dma_data[i]);
|
||||
|
||||
if (ret < 0) {
|
||||
dev_dbg(spi_imx->dev, "DMA %d transfer not really finish\n", i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (int j = 0; j < spi_imx->dma_package_num; j++) {
|
||||
kfree(spi_imx->dma_data[j].dma_tx_buf);
|
||||
kfree(spi_imx->dma_data[j].dma_rx_buf);
|
||||
}
|
||||
kfree(spi_imx->dma_data);
|
||||
|
||||
fallback_pio:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int spi_imx_pio_transfer(struct spi_device *spi,
|
||||
struct spi_transfer *transfer)
|
||||
{
|
||||
|
|
@ -1737,7 +2142,7 @@ static int spi_imx_transfer_one(struct spi_controller *controller,
|
|||
while (spi_imx->devtype_data->rx_available(spi_imx))
|
||||
readl(spi_imx->base + MXC_CSPIRXDATA);
|
||||
|
||||
if (spi_imx->target_mode)
|
||||
if (spi_imx->target_mode && !spi_imx->usedma)
|
||||
return spi_imx_pio_transfer_target(spi, transfer);
|
||||
|
||||
/*
|
||||
|
|
@ -1745,9 +2150,17 @@ static int spi_imx_transfer_one(struct spi_controller *controller,
|
|||
* transfer, the SPI transfer has already been mapped, so we
|
||||
* have to do the DMA transfer here.
|
||||
*/
|
||||
if (spi_imx->usedma)
|
||||
return spi_imx_dma_transfer(spi_imx, transfer);
|
||||
|
||||
if (spi_imx->usedma) {
|
||||
ret = spi_imx_dma_transfer(spi_imx, transfer);
|
||||
if (transfer->error & SPI_TRANS_FAIL_NO_START) {
|
||||
spi_imx->usedma = false;
|
||||
if (spi_imx->target_mode)
|
||||
return spi_imx_pio_transfer_target(spi, transfer);
|
||||
else
|
||||
return spi_imx_pio_transfer(spi, transfer);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
/* run in polling mode for short transfers */
|
||||
if (transfer->len == 1 || (polling_limit_us &&
|
||||
spi_imx_transfer_estimate_time_us(transfer) < polling_limit_us))
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user