github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-28 09:04:39 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

drm/xe: Prep TLB invalidation fence before sending

Browse Source 
It is a bit backwards to add a TLB invalidation fence to the pending
list after issuing the invalidation. Perform this step before issuing
the TLB invalidation in a helper function.

v2: Make sure the seqno_lock mutex covers the send as well (Matt)

Signed-off-by: Stuart Summers <stuart.summers@intel.com>
Reviewed-by: Stuart Summers <stuart.summers@intel.com>
Signed-off-by: Matthew Brost <matthew.brost@intel.com>
Link: https://lore.kernel.org/r/20250826182911.392550-8-stuart.summers@intel.com
This commit is contained in:
Matthew Brost 2025-08-26 18:29:09 +00:00
parent 15366239e2
commit 9aff63cf37
1 changed files with 55 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
drivers/gpu/drm/xe/xe_tlb_inval.c
View File

@ -66,19 +66,19 @@ __inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence)
static void
inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence)
{
lockdep_assert_held(&fence->tlb_inval->pending_lock);
list_del(&fence->link);
__inval_fence_signal(xe, fence);
}
void xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence)
static void
inval_fence_signal_unlocked(struct xe_device *xe,
struct xe_tlb_inval_fence *fence)
{
struct xe_gt *gt;
if (WARN_ON_ONCE(!fence->tlb_inval))
return;
gt = fence->tlb_inval->private;
__inval_fence_signal(gt_to_xe(gt), fence);
spin_lock_irq(&fence->tlb_inval->pending_lock);
inval_fence_signal(xe, fence);
spin_unlock_irq(&fence->tlb_inval->pending_lock);
}
static void xe_gt_tlb_fence_timeout(struct work_struct *work)
@ -221,14 +221,10 @@ static bool tlb_inval_seqno_past(struct xe_gt *gt, int seqno)
return seqno_recv >= seqno;
}
static int send_tlb_inval(struct xe_guc *guc,
struct xe_tlb_inval_fence *fence,
static int send_tlb_inval(struct xe_guc *guc, struct xe_tlb_inval_fence *fence,
u32 *action, int len)
{
struct xe_gt *gt = guc_to_gt(guc);
struct xe_device *xe = gt_to_xe(gt);
int seqno;
int ret;
xe_gt_assert(gt, fence);
@ -238,47 +234,36 @@ static int send_tlb_inval(struct xe_guc *guc,
* need to be updated.
*/
mutex_lock(&gt->tlb_inval.seqno_lock);
seqno = gt->tlb_inval.seqno;
fence->seqno = seqno;
trace_xe_tlb_inval_fence_send(xe, fence);
action[1] = seqno;
ret = xe_guc_ct_send(&guc->ct, action, len,
G2H_LEN_DW_TLB_INVALIDATE, 1);
if (!ret) {
spin_lock_irq(&gt->tlb_inval.pending_lock);
/*
* We haven't actually published the TLB fence as per
* pending_fences, but in theory our seqno could have already
* been written as we acquired the pending_lock. In such a case
* we can just go ahead and signal the fence here.
*/
if (tlb_inval_seqno_past(gt, seqno)) {
__inval_fence_signal(xe, fence);
} else {
fence->inval_time = ktime_get();
list_add_tail(&fence->link,
&gt->tlb_inval.pending_fences);
if (list_is_singular(&gt->tlb_inval.pending_fences))
queue_delayed_work(system_wq,
&gt->tlb_inval.fence_tdr,
tlb_timeout_jiffies(gt));
}
spin_unlock_irq(&gt->tlb_inval.pending_lock);
} else {
__inval_fence_signal(xe, fence);
}
if (!ret) {
gt->tlb_inval.seqno = (gt->tlb_inval.seqno + 1) %
TLB_INVALIDATION_SEQNO_MAX;
if (!gt->tlb_inval.seqno)
gt->tlb_inval.seqno = 1;
}
mutex_unlock(&gt->tlb_inval.seqno_lock);
xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1);
action[1] = fence->seqno;
return ret;
return xe_guc_ct_send(&guc->ct, action, len,
G2H_LEN_DW_TLB_INVALIDATE, 1);
}
static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence)
{
struct xe_tlb_inval *tlb_inval = fence->tlb_inval;
struct xe_gt *gt = tlb_inval->private;
struct xe_device *xe = gt_to_xe(gt);
fence->seqno = tlb_inval->seqno;
trace_xe_tlb_inval_fence_send(xe, fence);
spin_lock_irq(&tlb_inval->pending_lock);
fence->inval_time = ktime_get();
list_add_tail(&fence->link, &tlb_inval->pending_fences);
if (list_is_singular(&tlb_inval->pending_fences))
queue_delayed_work(system_wq,
&tlb_inval->fence_tdr,
tlb_timeout_jiffies(gt));
spin_unlock_irq(&tlb_inval->pending_lock);
tlb_inval->seqno = (tlb_inval->seqno + 1) %
TLB_INVALIDATION_SEQNO_MAX;
if (!tlb_inval->seqno)
tlb_inval->seqno = 1;
}
#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
@ -306,7 +291,14 @@ static int xe_tlb_inval_guc(struct xe_gt *gt,
};
int ret;
mutex_lock(&gt->tlb_inval.seqno_lock);
xe_tlb_inval_fence_prep(fence);
ret = send_tlb_inval(&gt->uc.guc, fence, action, ARRAY_SIZE(action));
if (ret < 0)
inval_fence_signal_unlocked(gt_to_xe(gt), fence);
mutex_unlock(&gt->tlb_inval.seqno_lock);
/*
* -ECANCELED indicates the CT is stopped for a GT reset. TLB caches
* should be nuked on a GT reset so this error can be ignored.
@ -433,7 +425,7 @@ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
#define MAX_TLB_INVALIDATION_LEN 7
u32 action[MAX_TLB_INVALIDATION_LEN];
u64 length = end - start;
int len = 0;
int len = 0, ret;
xe_gt_assert(gt, fence);
@ -494,7 +486,16 @@ int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
return send_tlb_inval(&gt->uc.guc, fence, action, len);
mutex_lock(&gt->tlb_inval.seqno_lock);
xe_tlb_inval_fence_prep(fence);
ret = send_tlb_inval(&gt->uc.guc, fence, action,
ARRAY_SIZE(action));
if (ret < 0)
inval_fence_signal_unlocked(xe, fence);
mutex_unlock(&gt->tlb_inval.seqno_lock);
return ret;
}
/**