github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-14 08:01:12 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

iommu/riscv: Serialize command queue publishing

Browse Source 
Serialize command queue publishing so software producer state advances only
after a command is written and the hardware tail is updated. Wait for
hardware consumption outside the queue lock when the command queue is full
so other CPUs are not blocked behind a long poll.

Fixes: 856c0cfe5c ("iommu/riscv: Command and fault queue support")
Signed-off-by: Fangyu Yu <fangyu.yu@linux.alibaba.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
This commit is contained in:
Fangyu Yu 2026-09-01 21:39:19 +08:00 committed by Joerg Roedel
parent d5bcf9ccaa
commit ca58afa409
1 changed files with 59 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
drivers/iommu/riscv/iommu.c
View File

@ -382,77 +382,93 @@ static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue,
(int)(cons - index) > 0, 0, timeout_us);
}
/* Enqueue an entry and wait to be processed if timeout_us > 0
*
* Error handling for IOMMU hardware not responding in reasonable time
* will be added as separate patch series along with other RAS features.
* For now, only report hardware failure and continue.
*/
static int riscv_iommu_queue_wait_for_space(struct riscv_iommu_queue *queue,
unsigned int last)
{
unsigned int head;
unsigned int tail;
unsigned int hw_head;
unsigned long flags;
int ret;
ret = riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), hw_head,
!(hw_head & ~queue->mask) && hw_head != last,
0, RISCV_IOMMU_QUEUE_TIMEOUT);
if (ret)
return ret;
raw_spin_lock_irqsave(&queue->lock, flags);
head = atomic_read(&queue->head);
tail = atomic_read(&queue->tail);
if ((tail - head) >= queue->mask) {
last = Q_ITEM(queue, head);
/*
* Re-read hw_head under the lock so that it is consistent with
* the freshly computed 'last'. Using the pre-lock snapshot
* could produce a stale value that wraps around relative to the
* new 'last', advancing the shadow head past entries that have
* not yet been consumed by the hardware.
*/
hw_head = riscv_iommu_readl(queue->iommu, Q_HEAD(queue));
if (!(hw_head & ~queue->mask) && hw_head != last)
atomic_add((hw_head - last) & queue->mask, &queue->head);
}
raw_spin_unlock_irqrestore(&queue->lock, flags);
return 0;
}
/* Enqueue an entry and publish it to the hardware queue. */
static unsigned int riscv_iommu_queue_send(struct riscv_iommu_queue *queue,
void *entry, size_t entry_size)
{
unsigned int prod;
unsigned int head;
unsigned int tail;
unsigned long flags;
int ret;
/* Do not preempt submission flow. */
local_irq_save(flags);
/* 1. Wait for space availability and reserve the next slot. */
for (;;) {
raw_spin_lock_irqsave(&queue->lock, flags);
/* 1. Allocate some space in the queue */
prod = atomic_inc_return(&queue->prod) - 1;
head = atomic_read(&queue->head);
prod = atomic_read(&queue->tail);
head = atomic_read(&queue->head);
/* 2. Wait for space availability. */
if ((prod - head) > queue->mask) {
if (readx_poll_timeout(atomic_read, &queue->head,
head, (prod - head) < queue->mask,
0, RISCV_IOMMU_QUEUE_TIMEOUT))
if ((prod - head) < queue->mask)
break;
head = Q_ITEM(queue, head);
raw_spin_unlock_irqrestore(&queue->lock, flags);
ret = riscv_iommu_queue_wait_for_space(queue, head);
if (ret)
goto err_busy;
} else if ((prod - head) == queue->mask) {
const unsigned int last = Q_ITEM(queue, head);
if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head,
!(head & ~queue->mask) && head != last,
0, RISCV_IOMMU_QUEUE_TIMEOUT))
goto err_busy;
atomic_add((head - last) & queue->mask, &queue->head);
}
/* 3. Store entry in the ring buffer */
/* 2. Store entry in the ring buffer. */
memcpy(queue->base + Q_ITEM(queue, prod) * entry_size, entry, entry_size);
/* 4. Wait for all previous entries to be ready */
if (readx_poll_timeout(atomic_read, &queue->tail, tail, prod == tail,
0, RISCV_IOMMU_QUEUE_TIMEOUT))
goto err_busy;
/*
* 5. Make sure the ring buffer update (whether in normal or I/O memory) is
* completed and visible before signaling the tail doorbell to fetch
* the next command. 'fence ow, ow'
*/
/* 3. Make sure the entry is visible before updating the queue tail. */
dma_wmb();
riscv_iommu_writel(queue->iommu, Q_TAIL(queue), Q_ITEM(queue, prod + 1));
/*
* 6. Make sure the doorbell write to the device has finished before updating
* the shadow tail index in normal memory. 'fence o, w'
* 4. Make sure the doorbell write to the device has finished before
* updating the shadow tail index in normal memory. 'fence o, w'
*/
#ifdef CONFIG_MMIOWB
mmiowb();
#endif
atomic_inc(&queue->tail);
atomic_set(&queue->tail, prod + 1);
atomic_set(&queue->prod, prod + 1);
/* 7. Complete submission and restore local interrupts */
local_irq_restore(flags);
raw_spin_unlock_irqrestore(&queue->lock, flags);
return prod;
err_busy:
local_irq_restore(flags);
/* Report the failure and continue; full RAS recovery is not implemented. */
dev_err_once(queue->iommu->dev, "Hardware error: command enqueue failed\n");
return prod;
}