github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-25 15:41:52 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

dmaengine: axi-dmac: Add support for scatter-gather transfers

Browse Source 
Implement support for scatter-gather transfers. Build a chain of
hardware descriptors, each one corresponding to a segment of the
transfer, and linked to the next one. The hardware will transfer the
chain and only fire interrupts when the whole chain has been
transferred.

Support for scatter-gather is automatically enabled when the driver
detects that the hardware supports it, by writing then reading the
AXI_DMAC_REG_SG_ADDRESS register. If not available, the driver will fall
back to standard DMA transfers.

Signed-off-by: Paul Cercueil <paul@crapouillou.net>
Link: https://lore.kernel.org/r/20231215131313.23840-4-paul@crapouillou.net
Signed-off-by: Vinod Koul <vkoul@kernel.org>
This commit is contained in:
Paul Cercueil 2023-12-15 14:13:11 +01:00 committed by Vinod Koul
parent 3f8fd25936
commit e97dc74359
1 changed files with 96 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
drivers/dma/dma-axi-dmac.c
View File

@ -81,9 +81,13 @@
#define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438
#define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c
#define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450
#define AXI_DMAC_REG_CURRENT_SG_ID 0x454
#define AXI_DMAC_REG_SG_ADDRESS 0x47c
#define AXI_DMAC_REG_SG_ADDRESS_HIGH 0x4bc
#define AXI_DMAC_CTRL_ENABLE BIT(0) #define AXI_DMAC_CTRL_ENABLE BIT(0)
#define AXI_DMAC_CTRL_PAUSE BIT(1) #define AXI_DMAC_CTRL_PAUSE BIT(1)
#define AXI_DMAC_CTRL_ENABLE_SG BIT(2)
#define AXI_DMAC_IRQ_SOT BIT(0) #define AXI_DMAC_IRQ_SOT BIT(0)
#define AXI_DMAC_IRQ_EOT BIT(1) #define AXI_DMAC_IRQ_EOT BIT(1)
@ -97,12 +101,16 @@
/* The maximum ID allocated by the hardware is 31 */ /* The maximum ID allocated by the hardware is 31 */
#define AXI_DMAC_SG_UNUSED 32U #define AXI_DMAC_SG_UNUSED 32U
/* Flags for axi_dmac_hw_desc.flags */
#define AXI_DMAC_HW_FLAG_LAST BIT(0)
#define AXI_DMAC_HW_FLAG_IRQ BIT(1)
struct axi_dmac_hw_desc { struct axi_dmac_hw_desc {
u32 flags; u32 flags;
u32 id; u32 id;
u64 dest_addr; u64 dest_addr;
u64 src_addr; u64 src_addr;
u64 __unused; u64 next_sg_addr;
u32 y_len; u32 y_len;
u32 x_len; u32 x_len;
u32 src_stride; u32 src_stride;
@ -150,6 +158,7 @@ struct axi_dmac_chan {
bool hw_partial_xfer; bool hw_partial_xfer;
bool hw_cyclic; bool hw_cyclic;
bool hw_2d; bool hw_2d;
bool hw_sg;
}; };
struct axi_dmac { struct axi_dmac {
@ -224,9 +233,11 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
unsigned int flags = 0; unsigned int flags = 0;
unsigned int val; unsigned int val;
if (!chan->hw_sg) {
val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER);
if (val) /* Queue is full, wait for the next SOT IRQ */ if (val) /* Queue is full, wait for the next SOT IRQ */
return; return;
}
desc = chan->next_desc; desc = chan->next_desc;
@ -245,8 +256,9 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
return; return;
} }
desc->num_submitted++; if (chan->hw_sg) {
if (desc->num_submitted == desc->num_sgs || chan->next_desc = NULL;
} else if (++desc->num_submitted == desc->num_sgs ||
desc->have_partial_xfer) { desc->have_partial_xfer) {
if (desc->cyclic) if (desc->cyclic)
desc->num_submitted = 0; /* Start again */ desc->num_submitted = 0; /* Start again */
@ -259,6 +271,7 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID);
if (!chan->hw_sg) {
if (axi_dmac_dest_is_mem(chan)) { if (axi_dmac_dest_is_mem(chan)) {
axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr);
axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride);
@ -268,6 +281,7 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr);
axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride);
} }
}
/* /*
* If the hardware supports cyclic transfers and there is no callback to * If the hardware supports cyclic transfers and there is no callback to
@ -281,8 +295,14 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
if (chan->hw_partial_xfer) if (chan->hw_partial_xfer)
flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; flags |= AXI_DMAC_FLAG_PARTIAL_REPORT;
if (chan->hw_sg) {
axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys);
axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH,
(u64)sg->hw_phys >> 32);
} else {
axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len);
axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len);
}
axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags);
axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1);
} }
@ -359,6 +379,9 @@ static void axi_dmac_compute_residue(struct axi_dmac_chan *chan,
rslt->result = DMA_TRANS_NOERROR; rslt->result = DMA_TRANS_NOERROR;
rslt->residue = 0; rslt->residue = 0;
if (chan->hw_sg)
return;
/* /*
* We get here if the last completed segment is partial, which * We get here if the last completed segment is partial, which
* means we can compute the residue from that segment onwards * means we can compute the residue from that segment onwards
@ -385,6 +408,15 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
(completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE))
axi_dmac_dequeue_partial_xfers(chan); axi_dmac_dequeue_partial_xfers(chan);
if (chan->hw_sg) {
if (active->cyclic) {
vchan_cyclic_callback(&active->vdesc);
} else {
list_del(&active->vdesc.node);
vchan_cookie_complete(&active->vdesc);
active = axi_dmac_active_desc(chan);
}
} else {
do { do {
sg = &active->sg[active->num_completed]; sg = &active->sg[active->num_completed];
if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */
@ -415,6 +447,7 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
} }
} }
} while (active); } while (active);
}
return start_next; return start_next;
} }
@ -478,8 +511,12 @@ static void axi_dmac_issue_pending(struct dma_chan *c)
struct axi_dmac_chan *chan = to_axi_dmac_chan(c); struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
struct axi_dmac *dmac = chan_to_axi_dmac(chan); struct axi_dmac *dmac = chan_to_axi_dmac(chan);
unsigned long flags; unsigned long flags;
u32 ctrl = AXI_DMAC_CTRL_ENABLE;
axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); if (chan->hw_sg)
ctrl |= AXI_DMAC_CTRL_ENABLE_SG;
axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl);
spin_lock_irqsave(&chan->vchan.lock, flags); spin_lock_irqsave(&chan->vchan.lock, flags);
if (vchan_issue_pending(&chan->vchan)) if (vchan_issue_pending(&chan->vchan))
@ -516,8 +553,14 @@ axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs)
hws[i].id = AXI_DMAC_SG_UNUSED; hws[i].id = AXI_DMAC_SG_UNUSED;
hws[i].flags = 0; hws[i].flags = 0;
/* Link hardware descriptors */
hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws);
} }
/* The last hardware descriptor will trigger an interrupt */
desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ;
return desc; return desc;
} }
@ -753,6 +796,9 @@ static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg)
case AXI_DMAC_REG_CURRENT_DEST_ADDR: case AXI_DMAC_REG_CURRENT_DEST_ADDR:
case AXI_DMAC_REG_PARTIAL_XFER_LEN: case AXI_DMAC_REG_PARTIAL_XFER_LEN:
case AXI_DMAC_REG_PARTIAL_XFER_ID: case AXI_DMAC_REG_PARTIAL_XFER_ID:
case AXI_DMAC_REG_CURRENT_SG_ID:
case AXI_DMAC_REG_SG_ADDRESS:
case AXI_DMAC_REG_SG_ADDRESS_HIGH:
return true; return true;
default: default:
return false; return false;
@ -905,6 +951,10 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version)
if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC)
chan->hw_cyclic = true; chan->hw_cyclic = true;
axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff);
if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS))
chan->hw_sg = true;
axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1); axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1);
if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1) if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1)
chan->hw_2d = true; chan->hw_2d = true;
@ -1005,6 +1055,7 @@ static int axi_dmac_probe(struct platform_device *pdev)
dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width);
dma_dev->directions = BIT(dmac->chan.direction); dma_dev->directions = BIT(dmac->chan.direction);
dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */
INIT_LIST_HEAD(&dma_dev->channels); INIT_LIST_HEAD(&dma_dev->channels);
dmac->chan.vchan.desc_free = axi_dmac_desc_free; dmac->chan.vchan.desc_free = axi_dmac_desc_free;