diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 3dd8cf4e2131..bc4dea606b38 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -504,6 +504,68 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); } +static int emit_ulls_preamble(struct xe_lrc *lrc, u32 *dw, int i, u32 seqno) +{ + u32 addr = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno); + + return emit_store_imm_ggtt(addr, LRC_MIGRATION_ULLS_SEMAPHORE_CLEAR, + dw, i); +} + +/* + * Advance the ring tail from within the ring, so submitting the next ULLS job + * needs nothing from the CPU beyond signalling the semaphore. All ULLS jobs + * occupy exactly ULLS_JOB_SIZE_BYTES, so the tail the next job ends at is two + * job slots on from where this job started, even though that job has not been + * emitted yet. Both LRC and MMIO ring tail advanced in step. + */ +static int emit_ulls_ring_tail(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw, + int i, u32 head) +{ + u32 next_tail = (head + 2 * ULLS_JOB_SIZE_BYTES) & (lrc->ring.size - 1); + + xe_gt_assert(gt, IS_ALIGNED(next_tail, 8)); + + i = emit_store_imm_ggtt(xe_lrc_ring_tail_ggtt_addr(lrc), next_tail, + dw, i); + + dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | + MI_LRI_LRM_CS_MMIO; + dw[i++] = RING_TAIL(0).addr; + dw[i++] = next_tail; + + return i; +} + +/* Publish the next job's tail, then park the engine on its semaphore */ +static int emit_ulls_postamble(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw, + int i, u32 seqno, u32 head) +{ + i = emit_ulls_ring_tail(gt, lrc, dw, i, head); + + dw[i++] = MI_SEMAPHORE_WAIT | + MI_SEMW_GGTT | + MI_SEMW_POLL | + MI_SEMW_COMPARE(SAD_EQ_SDD); + dw[i++] = LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL; + dw[i++] = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno + 1); + dw[i++] = 0; + dw[i++] = 0; + + return i; +} + +/* Pad out to the fixed ULLS job size */ +static int emit_ulls_pad(struct xe_gt *gt, u32 *dw, int i) +{ + xe_gt_assert(gt, i <= ULLS_JOB_SIZE_DW); + + while (i < ULLS_JOB_SIZE_DW) + dw[i++] = MI_NOOP; + + return i; +} + static void emit_migration_job_gen12(struct xe_sched_job *job, struct xe_lrc *lrc, u32 *head, u32 seqno) @@ -517,10 +579,16 @@ static void emit_migration_job_gen12(struct xe_sched_job *job, xe_gt_assert(gt, !job->ring_ops_force_reset); + if (xe_sched_job_is_ulls(job)) + i = emit_ulls_preamble(lrc, dw, i, seqno); + i = emit_copy_timestamp(xe, lrc, dw, i); i = emit_store_imm_ggtt(saddr, seqno, dw, i); + if (!xe_sched_job_ulls_has_batch(job)) + goto seqno_write; + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i); @@ -531,12 +599,19 @@ static void emit_migration_job_gen12(struct xe_sched_job *job, i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); +seqno_write: i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, job->migrate_flush_flags, dw, i); i = emit_user_interrupt(dw, i); + if (xe_sched_job_ulls_parks(job)) + i = emit_ulls_postamble(gt, lrc, dw, i, seqno, *head); + + if (xe_sched_job_is_ulls(job)) + i = emit_ulls_pad(gt, dw, i); + xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); diff --git a/drivers/gpu/drm/xe/xe_ring_ops_types.h b/drivers/gpu/drm/xe/xe_ring_ops_types.h index 52ff96bc4100..ea4af321dd7c 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops_types.h +++ b/drivers/gpu/drm/xe/xe_ring_ops_types.h @@ -6,6 +6,7 @@ #ifndef _XE_RING_OPS_TYPES_H_ #define _XE_RING_OPS_TYPES_H_ +#include #include struct xe_gt; @@ -14,6 +15,29 @@ struct xe_sched_job; #define MAX_JOB_SIZE_DW 74 #define MAX_JOB_SIZE_BYTES (MAX_JOB_SIZE_DW * 4) +/* + * ULLS migration jobs advance the ring tail from within the ring itself, so a + * job has to know where its successor will end before that successor has been + * emitted. Every ULLS job is therefore padded to a fixed size, letting the + * next tail be derived arithmetically. + * + * Sized for the largest such job, emitted by emit_migration_job_gen12(): + * preamble (4), copy timestamp (8, its size on an SRIOV VF), start seqno + * store (4), arbitration off (1), batch buffer starts (2 * 3), pre-parser + * bracketed flush invalidate (6), seqno flush (4), user interrupt (3) and + * postamble (7 + 5). + */ +#define ULLS_JOB_SIZE_DW 48 +#define ULLS_JOB_SIZE_BYTES (ULLS_JOB_SIZE_DW * 4) + +/* + * RING_TAIL only encodes a qword aligned offset, and xe_lrc_write_ring() + * appends a NOP to anything shorter, either of which would desynchronise the + * ring from the tail a job predicts for its successor. + */ +static_assert(ULLS_JOB_SIZE_BYTES % 8 == 0); +static_assert(ULLS_JOB_SIZE_DW <= MAX_JOB_SIZE_DW); + /** * struct xe_ring_ops - Ring operations */