From b4b15e4f12ad1dd83b69d453ef826553917eb8cb Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Fri, 29 May 2026 12:36:02 -0700 Subject: [PATCH 01/14] Revert "drm/xe/nvls: Define GuC firmware for NVL-S" This reverts commit 4e88de313ff4d1c67b644b1f39f9fb4089711b71. The early GuC FW definition meant for our CI branch was accidentally merged to the drm-xe-next branch instead. This GuC FW will never be released to linux-firmware, so we do not want the definition to be available in the mainline Linux codebase. Fixes: 4e88de313ff4 ("drm/xe/nvls: Define GuC firmware for NVL-S") Signed-off-by: Daniele Ceraolo Spurio Cc: Julia Filipchuk Cc: Rodrigo Vivi Cc: Matt Roper Cc: stable@vger.kernel.org # v7.0+ Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20260529193558.185436-11-daniele.ceraolospurio@intel.com Signed-off-by: Rodrigo Vivi (cherry picked from commit 65b8e0ac86e48cfc9128c04dfc53ea3395d030dd) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_uc_fw.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c index df2aa196f6f9..3f08a3b54062 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.c +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -115,7 +115,6 @@ struct fw_blobs_by_type { #define XE_GT_TYPE_ANY XE_GT_TYPE_UNINITIALIZED #define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver) \ - fw_def(NOVALAKE_S, GT_TYPE_ANY, mmp_ver(xe, guc, nvl, 70, 55, 4)) \ fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 54, 0)) \ fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 54, 0)) \ fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 53, 0)) \ From 65b1f95802f90fea0283f226317dc02dba494c81 Mon Sep 17 00:00:00 2001 From: Tangudu Tilak Tirumalesh Date: Wed, 3 Jun 2026 12:22:15 +0530 Subject: [PATCH 02/14] Revert "drm/xe: Skip exec queue schedule toggle if queue is idle during suspend" This reverts commit 8533051ce92015e9cc6f75e0d52119b9d91610b6. The idle-skip optimization bypasses GuC suspend, so the GPU may not perform the context switch that flushes TLB entries for invalidated userptr VMAs. In LR/preempt-fence VM mode, this can lead to missed TLB invalidation and page faults during userptr invalidation tests. Restore unconditional schedule toggling on suspend so the context-switch TLB flush is always performed. This optimization will be reintroduced with a fix that does not skip suspend in LR/preempt-fence VM mode. Fixes: 8533051ce920 ("drm/xe: Skip exec queue schedule toggle if queue is idle during suspend") Cc: stable@vger.kernel.org # v7.0+ Suggested-by: Thomas Hellstrom Signed-off-by: Tangudu Tilak Tirumalesh Reviewed-by: Thomas Hellstrom Signed-off-by: Daniele Ceraolo Spurio Link: https://patch.msgid.link/20260603065217.3131066-2-tilak.tirumalesh.tangudu@intel.com (cherry picked from commit 6a1e7934d9a6cf46aecae00a99c2603d1295e170) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_exec_queue.h | 17 -------- drivers/gpu/drm/xe/xe_guc_submit.c | 55 ++----------------------- drivers/gpu/drm/xe/xe_hw_engine_group.c | 10 +---- 3 files changed, 5 insertions(+), 77 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index a82d99bd77bc..0225426c57b0 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -162,21 +162,4 @@ int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch); struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q); struct xe_lrc *xe_exec_queue_get_lrc(struct xe_exec_queue *q, u16 idx); -/** - * xe_exec_queue_idle_skip_suspend() - Can exec queue skip suspend - * @q: The exec_queue - * - * If an exec queue is not parallel and is idle, the suspend steps can be - * skipped in the submission backend immediatley signaling the suspend fence. - * Parallel queues cannot skip this step due to limitations in the submission - * backend. - * - * Return: True if exec queue is idle and can skip suspend steps, False - * otherwise - */ -static inline bool xe_exec_queue_idle_skip_suspend(struct xe_exec_queue *q) -{ - return !xe_exec_queue_is_parallel(q) && xe_exec_queue_is_idle(q); -} - #endif diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index ab501513d806..d1ab66ca1856 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -71,7 +71,6 @@ exec_queue_to_guc(struct xe_exec_queue *q) #define EXEC_QUEUE_STATE_WEDGED (1 << 8) #define EXEC_QUEUE_STATE_BANNED (1 << 9) #define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 10) -#define EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND (1 << 11) static bool exec_queue_registered(struct xe_exec_queue *q) { @@ -218,21 +217,6 @@ static void clear_exec_queue_pending_resume(struct xe_exec_queue *q) atomic_and(~EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); } -static bool exec_queue_idle_skip_suspend(struct xe_exec_queue *q) -{ - return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND; -} - -static void set_exec_queue_idle_skip_suspend(struct xe_exec_queue *q) -{ - atomic_or(EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state); -} - -static void clear_exec_queue_idle_skip_suspend(struct xe_exec_queue *q) -{ - atomic_and(~EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state); -} - static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) { return (atomic_read(&q->guc->state) & @@ -1157,7 +1141,7 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) if (!job->restore_replay || job->last_replay) { if (xe_exec_queue_is_parallel(q)) wq_item_append(q); - else if (!exec_queue_idle_skip_suspend(q)) + else xe_lrc_set_ring_tail(lrc, lrc->ring.tail); job->last_replay = false; } @@ -1812,10 +1796,9 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; struct xe_guc *guc = exec_queue_to_guc(q); - bool idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q); - if (!idle_skip_suspend && guc_exec_queue_allowed_to_change_state(q) && - !exec_queue_suspended(q) && exec_queue_enabled(q)) { + if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && + exec_queue_enabled(q)) { wait_event(guc->ct.wq, vf_recovery(guc) || ((q->guc->resume_time != RESUME_PENDING || xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q))); @@ -1834,33 +1817,11 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) disable_scheduling(q, false); } } else if (q->guc->suspend_pending) { - if (idle_skip_suspend) - set_exec_queue_idle_skip_suspend(q); set_exec_queue_suspended(q); suspend_fence_signal(q); } } -static void sched_context(struct xe_exec_queue *q) -{ - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_lrc *lrc = q->lrc[0]; - u32 action[] = { - XE_GUC_ACTION_SCHED_CONTEXT, - q->guc->id, - }; - - xe_gt_assert(guc_to_gt(guc), !xe_exec_queue_is_parallel(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); - xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); - - trace_xe_exec_queue_submit(q); - - xe_lrc_set_ring_tail(lrc, lrc->ring.tail); - xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); -} - static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; @@ -1868,22 +1829,12 @@ static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) if (guc_exec_queue_allowed_to_change_state(q)) { clear_exec_queue_suspended(q); if (!exec_queue_enabled(q)) { - if (exec_queue_idle_skip_suspend(q)) { - struct xe_lrc *lrc = q->lrc[0]; - - clear_exec_queue_idle_skip_suspend(q); - xe_lrc_set_ring_tail(lrc, lrc->ring.tail); - } q->guc->resume_time = RESUME_PENDING; set_exec_queue_pending_resume(q); enable_scheduling(q); - } else if (exec_queue_idle_skip_suspend(q)) { - clear_exec_queue_idle_skip_suspend(q); - sched_context(q); } } else { clear_exec_queue_suspended(q); - clear_exec_queue_idle_skip_suspend(q); } } diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c index 4c2b113364d3..02cf32ae5aa9 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_group.c +++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c @@ -208,21 +208,15 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group lockdep_assert_held_write(&group->mode_sem); list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { - bool idle_skip_suspend; if (!xe_vm_in_fault_mode(q->vm)) continue; - idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q); - if (!idle_skip_suspend && has_deps) + if (has_deps) return -EAGAIN; xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); - if (idle_skip_suspend) - xe_gt_stats_incr(q->gt, - XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, 1); - - need_resume |= !idle_skip_suspend; + need_resume = true; q->ops->suspend(q); gt = q->gt; } From 1141574cb85db5226042b3d903157f95bfc631b1 Mon Sep 17 00:00:00 2001 From: Tangudu Tilak Tirumalesh Date: Wed, 3 Jun 2026 12:22:16 +0530 Subject: [PATCH 03/14] drm/xe: Clear pending_disable before signaling suspend fence In the schedule-disable done path for suspend, we signal the suspend fence before clearing pending_disable. That wakeup can let suspend_wait complete and resume be queued immediately. The resume path may then reach enable_scheduling() while pending_disable is still set and hit the !exec_queue_pending_disable(q) assertion. Fix this by clearing pending_disable before signaling the suspend fence, so any resumed transition observes a consistent state. Fixes: 87651f31ae4e ("drm/xe/guc_submit: fix race around suspend_pending") Cc: stable@vger.kernel.org # v7.0+ Signed-off-by: Tangudu Tilak Tirumalesh Reviewed-by: Thomas Hellstrom Signed-off-by: Daniele Ceraolo Spurio Link: https://patch.msgid.link/20260603065217.3131066-3-tilak.tirumalesh.tangudu@intel.com (cherry picked from commit 4b1ae138b0e103d753773956a84eebc2edbf62c4) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_guc_submit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index d1ab66ca1856..122a0983df18 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -2791,8 +2791,8 @@ static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); if (q->guc->suspend_pending) { - suspend_fence_signal(q); clear_exec_queue_pending_disable(q); + suspend_fence_signal(q); } else { if (exec_queue_banned(q)) { smp_wmb(); From 65280af331aa15eab7012bd7f07823be13754ba2 Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Wed, 3 Jun 2026 16:39:47 -0700 Subject: [PATCH 04/14] drm/xe/multi_queue: skip submit when primary queue is suspended Return early in submit path when the multi-queue primary exec queue is suspended to avoid submitting while suspended. v2: Remove idle_skip_suspend fix as that feature is being reverted here https://patchwork.freedesktop.org/series/167262/ Fixes: bc5775c59258 ("drm/xe/multi_queue: Add GuC interface for multi queue support") Cc: stable@vger.kernel.org # v7.0+ Assisted-by: GitHub-Copilot:claude-sonnet-4.6 Reviewed-by: Daniele Ceraolo Spurio Signed-off-by: Niranjana Vishwanathapura Link: https://patch.msgid.link/20260603233946.863663-2-niranjana.vishwanathapura@intel.com (cherry picked from commit b7fb55cc3364ca128cfff9d50649ffd4327cd01e) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_guc_submit.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 122a0983df18..4b247a3019d2 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1151,9 +1151,12 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) /* * All queues in a multi-queue group will use the primary queue - * of the group to interface with GuC. + * of the group to interface with GuC. If primay is suspended, + * just return. Jobs will get scheduled once primary is resumed. */ q = xe_exec_queue_multi_queue_primary(q); + if (exec_queue_suspended(q)) + return; if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; From 45fa032b68570a9ccd80328a857ac5872217be92 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Tue, 2 Jun 2026 10:18:42 +0530 Subject: [PATCH 05/14] drm/xe/drm_ras: Make counter allocation drm managed cleanup_node_param() is not registered for previous node in case of counter allocation failure, which results in stale memory of previous node that isn't cleaned up on unwind. Fix this using drm managed allocation, which is guaranteed to be cleaned up on unwind. Fixes: b40db12b542f ("drm/xe/xe_drm_ras: Add support for XE DRM RAS") Signed-off-by: Raag Jadav Reviewed-by: Riana Tauro Link: https://patch.msgid.link/20260602044919.702209-3-raag.jadav@intel.com Signed-off-by: Matt Roper (cherry picked from commit 58d77c77ea0c5cb2b755ebe23e973c8272acd896) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_drm_ras.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c index c21c8b428de6..c1d5ac198a7c 100644 --- a/drivers/gpu/drm/xe/xe_drm_ras.c +++ b/drivers/gpu/drm/xe/xe_drm_ras.c @@ -80,7 +80,7 @@ static struct xe_drm_ras_counter *allocate_and_copy_counters(struct xe_device *x struct xe_drm_ras_counter *counter; int i; - counter = kcalloc(DRM_XE_RAS_ERR_COMP_MAX, sizeof(*counter), GFP_KERNEL); + counter = drmm_kcalloc(&xe->drm, DRM_XE_RAS_ERR_COMP_MAX, sizeof(*counter), GFP_KERNEL); if (!counter) return ERR_PTR(-ENOMEM); @@ -135,7 +135,6 @@ static void cleanup_node_param(struct xe_drm_ras *ras, const enum drm_xe_ras_err { struct drm_ras_node *node = &ras->node[severity]; - kfree(ras->info[severity]); ras->info[severity] = NULL; kfree(node->device_name); From 042f1d5f20d63fb9b0e2fbba97648d16fe1a845d Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Tue, 2 Jun 2026 10:18:43 +0530 Subject: [PATCH 06/14] drm/xe/drm_ras: Add per node cleanup action cleanup_node_param() is not registered for previous node in case of counter allocation failure, which results in stale memory of previous node that isn't cleaned up on unwind. Add per node cleanup action which guarantees cleanup on unwind and also simplifies the cleanup logic. Fixes: b40db12b542f ("drm/xe/xe_drm_ras: Add support for XE DRM RAS") Signed-off-by: Raag Jadav Reviewed-by: Riana Tauro Link: https://patch.msgid.link/20260602044919.702209-4-raag.jadav@intel.com Signed-off-by: Matt Roper (cherry picked from commit 67fc5543d8274b2fcbef87734fad0469358f4478) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_drm_ras.c | 58 +++++++++++++-------------------- 1 file changed, 23 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c index c1d5ac198a7c..cd236f53699e 100644 --- a/drivers/gpu/drm/xe/xe_drm_ras.c +++ b/drivers/gpu/drm/xe/xe_drm_ras.c @@ -131,53 +131,47 @@ static int assign_node_params(struct xe_device *xe, struct drm_ras_node *node, return 0; } -static void cleanup_node_param(struct xe_drm_ras *ras, const enum drm_xe_ras_error_severity severity) +static void cleanup_node_param(struct drm_ras_node *node) { - struct drm_ras_node *node = &ras->node[severity]; - - ras->info[severity] = NULL; - kfree(node->device_name); node->device_name = NULL; } +static void cleanup_node(struct drm_device *drm, void *node) +{ + drm_ras_node_unregister(node); + cleanup_node_param(node); +} + static int register_nodes(struct xe_device *xe) { struct xe_drm_ras *ras = &xe->ras; - int i; + struct drm_ras_node *node; + int i, ret; for_each_error_severity(i) { - struct drm_ras_node *node = &ras->node[i]; - int ret; + node = &ras->node[i]; ret = assign_node_params(xe, node, i); - if (ret) { - cleanup_node_param(ras, i); - return ret; - } + if (ret) + goto free_param; ret = drm_ras_node_register(node); - if (ret) { - cleanup_node_param(ras, i); - return ret; - } + if (ret) + goto free_param; + + ret = drmm_add_action_or_reset(&xe->drm, cleanup_node, node); + if (ret) + goto null_info; } return 0; -} -static void xe_drm_ras_unregister_nodes(struct drm_device *device, void *arg) -{ - struct xe_device *xe = arg; - struct xe_drm_ras *ras = &xe->ras; - int i; - - for_each_error_severity(i) { - struct drm_ras_node *node = &ras->node[i]; - - drm_ras_node_unregister(node); - cleanup_node_param(ras, i); - } +free_param: + cleanup_node_param(node); +null_info: + ras->info[i] = NULL; + return ret; } /** @@ -206,11 +200,5 @@ int xe_drm_ras_init(struct xe_device *xe) return err; } - err = drmm_add_action_or_reset(&xe->drm, xe_drm_ras_unregister_nodes, xe); - if (err) { - drm_err(&xe->drm, "Failed to add action for Xe DRM RAS (%pe)\n", ERR_PTR(err)); - return err; - } - return 0; } From d46319bc8273c1018fa532a383eb515ee1b1de4b Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Tue, 2 Jun 2026 10:18:44 +0530 Subject: [PATCH 07/14] drm/xe/hw_error: Use HW_ERR prefix in log Hardware errors should be logged with HW_ERR prefix. Make them consistent with existing logs. Fixes: 01aab7e1c9d4 ("drm/xe/xe_hw_error: Add support for PVC SoC errors") Signed-off-by: Raag Jadav Reviewed-by: Riana Tauro Link: https://patch.msgid.link/20260602044919.702209-5-raag.jadav@intel.com Signed-off-by: Matt Roper (cherry picked from commit ad60a618c49fef07d1860bfb1091140d29f5eddb) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_hw_error.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c index 5135e8e4093f..4b72959b2276 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.c +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -223,9 +223,9 @@ static void log_hw_error(struct xe_tile *tile, const char *name, struct xe_device *xe = tile_to_xe(tile); if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) - drm_warn(&xe->drm, "%s %s detected\n", name, severity_str); + drm_warn(&xe->drm, HW_ERR "%s %s detected\n", name, severity_str); else - drm_err_ratelimited(&xe->drm, "%s %s detected\n", name, severity_str); + drm_err_ratelimited(&xe->drm, HW_ERR "%s %s detected\n", name, severity_str); } static void log_gt_err(struct xe_tile *tile, const char *name, int i, u32 err, @@ -235,10 +235,10 @@ static void log_gt_err(struct xe_tile *tile, const char *name, int i, u32 err, struct xe_device *xe = tile_to_xe(tile); if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) - drm_warn(&xe->drm, "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n", + drm_warn(&xe->drm, HW_ERR "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n", name, severity_str, i, err); else - drm_err_ratelimited(&xe->drm, "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n", + drm_err_ratelimited(&xe->drm, HW_ERR "%s %s detected, ERROR_STAT_GT_VECTOR%d:0x%08x\n", name, severity_str, i, err); } @@ -255,9 +255,9 @@ static void log_soc_error(struct xe_tile *tile, const char * const *reg_info, if (strcmp(name, "Undefined")) { if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) - drm_warn(&xe->drm, "%s SOC %s detected", name, severity_str); + drm_warn(&xe->drm, HW_ERR "%s SOC %s detected", name, severity_str); else - drm_err_ratelimited(&xe->drm, "%s SOC %s detected", name, severity_str); + drm_err_ratelimited(&xe->drm, HW_ERR "%s SOC %s detected", name, severity_str); atomic_inc(&info[index].counter); } } From f2d238408db2b963951f0f2ae70ff7f9e337b540 Mon Sep 17 00:00:00 2001 From: Tangudu Tilak Tirumalesh Date: Mon, 8 Jun 2026 21:57:44 +0530 Subject: [PATCH 08/14] drm/xe: include all registered queues in TLB invalidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context-based TLB invalidation currently selects only scheduling-active exec queues via q->ops->active(). During rebind flows, queues may be suspended (or transitioning through resume) while still owning valid translations, causing them to be skipped from invalidation and leading to missed TLB invalidations on LR rebinds. The underlying issue is a TOCTOU: q->guc->state bits are flipped lock-free from enable_scheduling(), disable_scheduling{,_deregister}(), the suspend/resume sched-msg handlers, handle_sched_done(), and guc_exec_queue_stop(); nothing in send_tlb_inval_ctx_ppgtt() serializes against them, so any state-based predicate can race. Include all the registered queues so that TLB invalidations are not missed. This is race-free because list membership on vm->exec_queues.list is stable under vm->exec_queues.lock held by the caller. The performance impact is expected to be minimal and harmless. If it does turn out to be a concern, we can come back with a race-safe solution to ignore certain queues. Fixes: 6cdaa5346d6f ("drm/xe: Add context-based invalidation to GuC TLB invalidation backend") Assisted-by: Claude:claude-opus-4.6 Suggested-by: Thomas Hellstrom Signed-off-by: Tangudu Tilak Tirumalesh Reviewed-by: Thomas Hellström Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260608162745.338725-2-tilak.tirumalesh.tangudu@intel.com Signed-off-by: Shuicheng Lin (cherry picked from commit aa625e1e9f0710e424fe4f0e3f032807df81b5b0) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_guc_tlb_inval.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c index ced58f46f846..cf6d106e6036 100644 --- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c @@ -255,9 +255,8 @@ static int send_tlb_inval_ctx_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, #undef EXEC_QUEUE_COUNT_FULL_THRESHOLD /* - * Move exec queues to a temporary list to issue invalidations. The exec - * queue must active and a reference must be taken to prevent concurrent - * deregistrations. + * Move exec queues to a temporary list to issue invalidations. A + * reference must be taken to prevent concurrent deregistrations. * * List modification is safe because we hold 'vm->exec_queues.lock' for * reading, which prevents external modifications. Using a per-GT list @@ -266,7 +265,7 @@ static int send_tlb_inval_ctx_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, */ list_for_each_entry_safe(q, next, &vm->exec_queues.list[id], vm_exec_queue_link) { - if (q->ops->active(q) && xe_exec_queue_get_unless_zero(q)) { + if (xe_exec_queue_get_unless_zero(q)) { last_q = q; list_move_tail(&q->vm_exec_queue_link, &tlb_inval_list); } From 0b837315ca0adc317e5c8a9c7e484a0e29199b9b Mon Sep 17 00:00:00 2001 From: Wentao Liang Date: Wed, 10 Jun 2026 10:27:05 -0700 Subject: [PATCH 09/14] drm/xe: fix refcount leak in xe_range_fence_insert() xe_range_fence_insert() acquires a reference on fence via dma_fence_get() and stores it in rfence->fence. It then calls dma_fence_add_callback() and handles two cases: when the callback is successfully registered (err == 0) the fence is transferred to the tree for later cleanup; when the fence is already signaled (err == -ENOENT) it manually drops the extra reference with dma_fence_put(fence). However, dma_fence_add_callback() can fail with other errors (e.g. -EINVAL) and in that case the code falls through to the free: label without releasing the acquired reference, leaking it. Fix the leak by adding an else branch that calls dma_fence_put() before jumping to free: for any error other than -ENOENT. Fixes: 845f64bdbfc9 ("drm/xe: Introduce a range-fence utility") Signed-off-by: Wentao Liang Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260610172705.3450560-1-matthew.brost@intel.com (cherry picked from commit 98c4a4201290823c2c5c7ba21692bd9a64b61021) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_range_fence.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_range_fence.c b/drivers/gpu/drm/xe/xe_range_fence.c index 372378e89e98..3d8fa194a7b0 100644 --- a/drivers/gpu/drm/xe/xe_range_fence.c +++ b/drivers/gpu/drm/xe/xe_range_fence.c @@ -77,6 +77,8 @@ int xe_range_fence_insert(struct xe_range_fence_tree *tree, } else if (err == 0) { xe_range_fence_tree_insert(rfence, &tree->root); return 0; + } else { + dma_fence_put(fence); } free: From 770031ec2312bfab307d05db5469f24fd297e758 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 10 Jun 2026 11:25:49 -0400 Subject: [PATCH 10/14] drm/xe: fix job timeout recovery for unstarted jobs and kernel queues A job that GuC never scheduled (never started) indicates a GuC scheduling failure; previously such jobs were silently errored out instead of triggering a GT reset to recover. Trigger a GT reset and resubmit them, but only when the queue was not already killed or banned: an unstarted job on an already banned queue is the ban working as intended and must neither clear the ban nor kick off a reset, otherwise a banned userspace queue could be resurrected and spam GT resets. Kernel queues are always recovered this way and wedge the device once recovery attempts are exhausted, since kernel work must not silently fail. A started job that times out on a userspace VM bind queue stays banned rather than being reset and retried. The queue is banned early in the timeout handler to signal the G2H scheduling-done handler so it wakes the disable-scheduling waiter; without it the waiter sleeps the full 5s timeout. When a reset is warranted the ban is cleared before rearming so that guc_exec_queue_start() can resubmit jobs after the GT reset - a still-banned queue would block resubmission and cause an infinite TDR loop. The already-banned case is gated out before this point via skip_timeout_check, so it is unaffected. v2: (Himal) Do it for any queue type, not just kernel/migration v3: - (Sashiko and Sanjay): don't clear the ban / GT reset for already killed/banned queues on unstarted-job timeout - Update commit message - (Matt) Add Fixes tag Fixes: fe05cee4d953 ("drm/xe: Don't short circuit TDR on jobs not started") Cc: Matthew Auld Cc: Matthew Brost Cc: Sanjay Yadav Cc: Himal Prasad Ghimiray Assisted-by: GitHub-Copilot:claude-sonnet-4.6 Assisted-by: GitHub-Copilot:claude-opus-4.8 Tested-by: Sanjay Yadav Reviewed-by: Sanjay Yadav Reviewed-by: Matthew Brost Reviewed-by: Himal Prasad Ghimiray Link: https://patch.msgid.link/20260610152548.404575-3-rodrigo.vivi@intel.com Signed-off-by: Rodrigo Vivi (cherry picked from commit b1107d085e7e8ed15ba6f80c102528a9c8a6cb0e) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_guc_submit.c | 49 +++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 4b247a3019d2..12a410458df6 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -157,6 +157,11 @@ static void set_exec_queue_banned(struct xe_exec_queue *q) atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); } +static void clear_exec_queue_banned(struct xe_exec_queue *q) +{ + atomic_andnot(EXEC_QUEUE_STATE_BANNED, &q->guc->state); +} + static bool exec_queue_suspended(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; @@ -1363,7 +1368,8 @@ static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), q->guc->id); - return xe_sched_invalidate_job(job, 2); + /* GuC never scheduled this job - let the caller trigger a GT reset. */ + return true; } ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0])); @@ -1460,6 +1466,21 @@ static void disable_scheduling(struct xe_exec_queue *q, bool immediate) G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); } +/* + * Recover via GT reset for a kernel queue, or for a GuC scheduling failure (job + * never started) on a queue that was not already killed or banned. An already + * banned queue must stay banned, so its unstarted jobs do not clear the ban or + * trigger a reset. + */ +static bool timeout_needs_gt_reset(struct xe_exec_queue *q, struct xe_sched_job *job, + bool skip_timeout_check) +{ + if (q->flags & EXEC_QUEUE_FLAG_KERNEL) + return true; + + return !skip_timeout_check && !xe_sched_job_started(job); +} + static enum drm_gpu_sched_stat guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) { @@ -1608,19 +1629,19 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), q->guc->id, q->flags); - /* - * Kernel jobs should never fail, nor should VM jobs if they do - * somethings has gone wrong and the GT needs a reset - */ - xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL, - "Kernel-submitted job timed out\n"); - xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q), - "VM job timed out on non-killed execqueue\n"); - if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL || - (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) { - if (!xe_sched_invalidate_job(job, 2)) { - xe_gt_reset_async(q->gt); - goto rearm; + if (!wedged) { + if (timeout_needs_gt_reset(q, job, skip_timeout_check)) { + if (!xe_sched_invalidate_job(job, 2)) { + clear_exec_queue_banned(q); + xe_gt_reset_async(q->gt); + goto rearm; + } + if (q->flags & EXEC_QUEUE_FLAG_KERNEL) { + xe_gt_WARN(q->gt, true, "Kernel-submitted job timed out\n"); + xe_device_declare_wedged(gt_to_xe(q->gt)); + } + } else if (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)) { + xe_gt_WARN(q->gt, true, "VM job timed out on non-killed execqueue\n"); } } From 92dc59ab2a09097cdf249e0288ff9b69261761c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Thu, 4 Jun 2026 09:45:00 +0200 Subject: [PATCH 11/14] drm/xe: Fix wa_oob codegen recipe for external module builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building with 'make M=drivers/gpu/drm/xe modules', kbuild invokes scripts/Makefile.build with obj=., causing $(obj) to expand to '.'. Make normalizes './xe_gen_wa_oob' to 'xe_gen_wa_oob' when constructing the $^ automatic variable (target name normalization), so the recipe command becomes just 'xe_gen_wa_oob ...' without any path prefix, and the shell cannot find the tool. Fix by replacing $^ with explicit $(obj)/xe_gen_wa_oob and $(src)/ references in both wa_oob recipe commands. In recipe strings, make does not apply target name normalization, so $(obj)/xe_gen_wa_oob correctly expands to './xe_gen_wa_oob' and the shell can execute it. This matches the pattern already used by other DRM drivers (e.g. radeon's mkregtable). Fixes: f037e0b78e6d ("drm/xe: add xe_device_wa infrastructure") Cc: Matt Atwood Cc: Matthew Brost Cc: Rodrigo Vivi Cc: intel-xe@lists.freedesktop.org Assisted-by: GitHub_Copilot:claude-sonnet-4.6 Signed-off-by: Thomas Hellström Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20260604074501.172129-1-thomas.hellstrom@linux.intel.com (cherry picked from commit 3a11a63cc16660d514ff584e7551589655337e87) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 09661f079d03..8e7b146880f4 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -16,14 +16,14 @@ subdir-ccflags-y += -I$(obj) -I$(src) hostprogs := xe_gen_wa_oob generated_oob := $(obj)/generated/xe_wa_oob.c $(obj)/generated/xe_wa_oob.h quiet_cmd_wa_oob = GEN $(notdir $(generated_oob)) - cmd_wa_oob = mkdir -p $(@D); $^ $(generated_oob) + cmd_wa_oob = mkdir -p $(@D); $(obj)/xe_gen_wa_oob $(src)/xe_wa_oob.rules $(generated_oob) $(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \ $(src)/xe_wa_oob.rules $(call cmd,wa_oob) generated_device_oob := $(obj)/generated/xe_device_wa_oob.c $(obj)/generated/xe_device_wa_oob.h quiet_cmd_device_wa_oob = GEN $(notdir $(generated_device_oob)) - cmd_device_wa_oob = mkdir -p $(@D); $^ $(generated_device_oob) + cmd_device_wa_oob = mkdir -p $(@D); $(obj)/xe_gen_wa_oob $(src)/xe_device_wa_oob.rules $(generated_device_oob) $(obj)/generated/%_device_wa_oob.c $(obj)/generated/%_device_wa_oob.h: $(obj)/xe_gen_wa_oob \ $(src)/xe_device_wa_oob.rules $(call cmd,device_wa_oob) From ba7fd163422877ad5a5cf31306a38c07d3932b0c Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 11 Jun 2026 16:58:44 -0700 Subject: [PATCH 12/14] drm/xe: Set TTM device beneficial_order to 9 (2M) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set the TTM device beneficial_order to 9 (2M), which is the sweet spot for Xe when attempting reclaim on system memory BOs, as it matches the large GPU page size. This ensures reclaim is attempted at the most effective order for the driver. This fixes an issue where an order-10 (4M) allocation cannot be found despite an abundance of memory. The 4M allocation triggers reclaim, unnecessarily evicting the working set and hurting performance. Since the TTM infrastructure was introduced recently, we are tagging the TTM patch as the Fixes target, even though this resolves an Xe-side problem. Fixes: 7e9c548d3709 ("drm/ttm: Allow drivers to specify maximum beneficial TTM pool size") Cc: stable@vger.kernel.org Signed-off-by: Matthew Brost Reviewed-by: Andi Shyti Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20260611235844.3725147-1-matthew.brost@intel.com (cherry picked from commit 0d81db90d364cb3d733410829118759f28957c5a) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_device.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index d224861b6f6f..abe25aedeead 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -526,7 +526,8 @@ int xe_device_init_early(struct xe_device *xe) err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, xe->drm.anon_inode->i_mapping, - xe->drm.vma_offset_manager, 0); + xe->drm.vma_offset_manager, + TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M))); if (err) return err; From 632ecc90e1ca5d3b6822bb4d08f84a175b6c42c0 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Fri, 12 Jun 2026 12:34:02 +0530 Subject: [PATCH 13/14] drm/xe/guc: Fix buffer overflow in steered register list allocation The size calculation for the steered register extarray uses only the geometry DSS mask (g_dss_mask) to determine the number of entries to allocate: total = bitmap_weight(gt->fuse_topo.g_dss_mask, ...) * steer_reg_num; However, the filling loop uses for_each_dss_steering(), which iterates over for_each_dss(), defined as the union of g_dss_mask and c_dss_mask (geometry + compute DSS). On platforms with compute-only DSS bits, the loop writes past the allocated buffer, corrupting adjacent slab objects. This manifests as list_del corruption and SLUB redzone overwrites during drm_managed_release on device unbind, since the overflow corrupts the drmres list_head of neighboring allocations. Fix by computing the allocation size using the union of both DSS masks, matching the iteration pattern of for_each_dss_steering(). -- v2: - use bitmap_weighted_or() (Zhanjun) Fixes: b170d696c1e2 ("drm/xe/guc: Add XE_LP steered register lists") Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/8049 Cc: Zhanjun Dong Cc: stable@vger.kernel.org Assisted-by: GitHub-Copilot:claude-opus-4.6 Reviewed-by: Zhanjun Dong Link: https://patch.msgid.link/20260612070401.543305-2-tejas.upadhyay@intel.com Signed-off-by: Tejas Upadhyay (cherry picked from commit 0a78a44f4901aa6c9263e66be7fce02282f1109f) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/xe_guc_capture.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index 21f7caf9ea08..1a019137ddf4 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -461,8 +461,14 @@ static void guc_capture_alloc_steered_lists(struct xe_guc *guc) if (!list || guc->capture->extlists) return; - total = bitmap_weight(gt->fuse_topo.g_dss_mask, sizeof(gt->fuse_topo.g_dss_mask) * 8) * - guc_capture_get_steer_reg_num(guc_to_xe(guc)); + { + xe_dss_mask_t all_dss; + + total = bitmap_weighted_or(all_dss, gt->fuse_topo.g_dss_mask, + gt->fuse_topo.c_dss_mask, + XE_MAX_DSS_FUSE_BITS) * + guc_capture_get_steer_reg_num(guc_to_xe(guc)); + } if (!total) return; From 0b5ed2756d45b04669502a1f13b1657ec7664571 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Fri, 5 Jun 2026 22:42:58 +0000 Subject: [PATCH 14/14] drm/xe: Add compact-PT and addr mask handling for page reclaim Current implementation of generate_reclaim_entry() overlooks some differences between the different page implementations: address masking and compact 64K page handling. Address masking of each leaf varies depending on the leaf entry size. generate_reclaim_entry() is using XE_PTE_ADDR_MASK [51:12] for all leaf entries. For 2MB PTEs, bit 12 (PAT) is part of the flags so the old mask corrupts the physical address extraction. 64K pages can be represented as PS64 and a compact PT, which the latter was not handled. Compact pages aren't walked by the unbind walker, so we separately walk through the compact PT to ensure none of the leaf 64K PTEs are dropped. Previously, compact PT were causing an abort since it was considered covered and not descended into. v2: - Update 64K entry/unbind walker for 64K compact PT handling. (Matthew) - Rework calculations of reclamation and address mask size. - Add new func abstracting the error handling before generating the reclaim entry. v3: - Report finer addr granularity in abort debug print for compact. (Zongyao) - Add comments for ADDR_MASK usage. (Zongyao) - Drop existing phys_addr asserts, the new XE_PAGE_ADDR_MASK clears bits checked, so redundant asserts. (Sashiko) - WARN_ON to verify compact pt and edge pt won't be possible. Fixes: b912138df299 ("drm/xe: Create page reclaim list on unbind") Assisted-by: Sashiko-Review:gemini-3.1-pro-preview Cc: stable@vger.kernel.org Cc: Matthew Auld Suggested-by: Zongyao Bai Signed-off-by: Brian Nguyen Reviewed-by: Matthew Auld Reviewed-by: Zongyao Bai Link: https://patch.msgid.link/20260605224257.2194194-2-brian3.nguyen@intel.com Signed-off-by: Matt Roper (cherry picked from commit 669252801a4aa4098fbc5dd9dd0bd93f0625abd7) Signed-off-by: Matthew Brost --- drivers/gpu/drm/xe/regs/xe_gtt_defs.h | 6 +- drivers/gpu/drm/xe/xe_pt.c | 131 +++++++++++++++----------- 2 files changed, 82 insertions(+), 55 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h index 4d83461e538b..d6bc19ef277b 100644 --- a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h +++ b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h @@ -9,7 +9,11 @@ #define XELPG_GGTT_PTE_PAT0 BIT_ULL(52) #define XELPG_GGTT_PTE_PAT1 BIT_ULL(53) -#define XE_PTE_ADDR_MASK GENMASK_ULL(51, 12) +/* + * Mask for PTE address bits [51:shift]. + * shift is the lower address boundary of page. + */ +#define XE_PAGE_ADDR_MASK(shift) GENMASK_ULL(51, (shift)) #define GGTT_PTE_VFID GENMASK_ULL(11, 2) #define GUC_GGTT_TOP 0xFEE00000 diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 2669ff5ee747..18a98667c0e6 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -1602,23 +1602,21 @@ static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, return false; } -/* page_size = 2^(reclamation_size + XE_PTE_SHIFT) */ -#define COMPUTE_RECLAIM_ADDRESS_MASK(page_size) \ -({ \ - BUILD_BUG_ON(!__builtin_constant_p(page_size)); \ - ilog2(page_size) - XE_PTE_SHIFT; \ -}) - static int generate_reclaim_entry(struct xe_tile *tile, struct xe_page_reclaim_list *prl, u64 pte, struct xe_pt *xe_child) { struct xe_gt *gt = tile->primary_gt; struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries; - u64 phys_addr = pte & XE_PTE_ADDR_MASK; + bool is_2m = xe_child->level == 1 && (pte & XE_PDE_PS_2M); + bool is_64k = xe_child->level == 0 && ((pte & XE_PTE_PS64) || xe_child->is_compact); + u32 page_shift = is_2m ? ilog2(SZ_2M) : is_64k ? ilog2(SZ_64K) : ilog2(SZ_4K); + /* Physical address bits start at page shift: 2M->[51:21], 64K->[51:16], 4K->[51:12] */ + u64 phys_addr = pte & XE_PAGE_ADDR_MASK(page_shift); + /* Page address is relative to 4K page regardless of entry level */ u64 phys_page = phys_addr >> XE_PTE_SHIFT; int num_entries = prl->num_entries; - u32 reclamation_size; + u32 reclamation_size = page_shift - XE_PTE_SHIFT; xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL); xe_tile_assert(tile, reclaim_entries); @@ -1633,18 +1631,12 @@ static int generate_reclaim_entry(struct xe_tile *tile, * Page size is computed as 2^(reclamation_size + XE_PTE_SHIFT) bytes. * Only 4K, 64K (level 0), and 2M pages are supported by hardware for page reclaim */ - if (xe_child->level == 0 && !(pte & XE_PTE_PS64)) { - xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT, 1); - reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_4K); /* reclamation_size = 0 */ - xe_tile_assert(tile, phys_addr % SZ_4K == 0); - } else if (xe_child->level == 0) { - xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT, 1); - reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_64K); /* reclamation_size = 4 */ - xe_tile_assert(tile, phys_addr % SZ_64K == 0); - } else if (xe_child->level == 1 && pte & XE_PDE_PS_2M) { + if (is_2m) { xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT, 1); - reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_2M); /* reclamation_size = 9 */ - xe_tile_assert(tile, phys_addr % SZ_2M == 0); + } else if (is_64k) { + xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT, 1); + } else if (xe_child->level == 0) { + xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT, 1); } else { xe_page_reclaim_list_abort(tile->primary_gt, prl, "unsupported PTE level=%u pte=%#llx", @@ -1665,6 +1657,48 @@ static int generate_reclaim_entry(struct xe_tile *tile, return 0; } +static int add_pte_to_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl, + struct xe_pt *xe_child, u64 pte, u64 addr) +{ + /* + * In rare scenarios, pte may not be written yet due to racy conditions. + * In such cases, invalidate the PRL and fallback to full PPC invalidation. + */ + if (!pte) { + xe_page_reclaim_list_abort(tile->primary_gt, prl, + "found zero pte at addr=%#llx", addr); + return -EINVAL; + } + + /* Ensure it is a defined page */ + xe_tile_assert(tile, xe_child->level == 0 || + (pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G))); + + /* Account for NULL terminated entry on end (-1) */ + if (prl->num_entries >= XE_PAGE_RECLAIM_MAX_ENTRIES - 1) { + xe_page_reclaim_list_abort(tile->primary_gt, prl, + "overflow while adding pte=%#llx", pte); + return -ENOSPC; + } + + return generate_reclaim_entry(tile, prl, pte, xe_child); +} + +static bool add_compact_pt_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl, + struct xe_device *xe, struct xe_pt *compact_pt, u64 addr) +{ + struct iosys_map *map = &compact_pt->bo->vmap; + + for (pgoff_t i = 0; i < SZ_2M / SZ_64K && xe_page_reclaim_list_valid(prl); i++) { + u64 pte = xe_map_rd(xe, map, i * sizeof(u64), u64); + + if (add_pte_to_prl(tile, prl, compact_pt, pte, addr + i * SZ_64K)) + break; + } + + return xe_page_reclaim_list_valid(prl); +} + static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, unsigned int level, u64 addr, u64 next, struct xe_ptw **child, @@ -1674,21 +1708,22 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); struct xe_pt_stage_unbind_walk *xe_walk = container_of(walk, typeof(*xe_walk), base); - struct xe_device *xe = tile_to_xe(xe_walk->tile); + struct xe_page_reclaim_list *prl = xe_walk->prl; + struct xe_tile *tile = xe_walk->tile; + struct xe_device *xe = tile_to_xe(tile); pgoff_t first = xe_pt_offset(addr, xe_child->level, walk); bool killed; XE_WARN_ON(!*child); XE_WARN_ON(!level); /* Check for leaf node */ - if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) && + if (prl && xe_page_reclaim_list_valid(prl) && xe_child->level <= MAX_HUGEPTE_LEVEL) { struct iosys_map *leaf_map = &xe_child->bo->vmap; pgoff_t count = xe_pt_num_entries(addr, next, xe_child->level, walk); for (pgoff_t i = 0; i < count; i++) { u64 pte; - int ret; /* * If not a leaf pt, skip unless non-leaf pt is interleaved between @@ -1698,10 +1733,23 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, u64 pt_size = 1ULL << walk->shifts[xe_child->level]; bool edge_pt = (i == 0 && !IS_ALIGNED(addr, pt_size)) || (i == count - 1 && !IS_ALIGNED(next, pt_size)); + struct xe_pt *child_pt = + container_of(xe_child->base.children[first + i], + struct xe_pt, base); - if (!edge_pt) { - xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, - xe_walk->prl, + /* Compact PTs always fill a full 2M-aligned slot, never an edge. */ + XE_WARN_ON(child_pt->is_compact && edge_pt); + if (edge_pt) + continue; + + /* Walker never descends into compact PTs, descend now */ + if (child_pt->is_compact) { + if (!add_compact_pt_prl(tile, prl, xe, child_pt, + addr + (u64)i * pt_size)) + break; + } else { + xe_page_reclaim_list_abort(tile->primary_gt, + prl, "PT is skipped by walk at level=%u offset=%lu", xe_child->level, first + i); break; @@ -1711,37 +1759,12 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64); - /* - * In rare scenarios, pte may not be written yet due to racy conditions. - * In such cases, invalidate the PRL and fallback to full PPC invalidation. - */ - if (!pte) { - xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, xe_walk->prl, - "found zero pte at addr=%#llx", addr); + if (add_pte_to_prl(tile, prl, xe_child, pte, addr)) break; - } - - /* Ensure it is a defined page */ - xe_tile_assert(xe_walk->tile, xe_child->level == 0 || - (pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G))); /* An entry should be added for 64KB but contigious 4K have XE_PTE_PS64 */ if (pte & XE_PTE_PS64) i += 15; /* Skip other 15 consecutive 4K pages in the 64K page */ - - /* Account for NULL terminated entry on end (-1) */ - if (xe_walk->prl->num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1) { - ret = generate_reclaim_entry(xe_walk->tile, xe_walk->prl, - pte, xe_child); - if (ret) - break; - } else { - /* overflow, mark as invalid */ - xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, xe_walk->prl, - "overflow while adding pte=%#llx", - pte); - break; - } } } @@ -1751,7 +1774,7 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, * Verify if any PTE are potentially dropped at non-leaf levels, either from being * killed or the page walk covers the region. */ - if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) && + if (prl && xe_page_reclaim_list_valid(prl) && xe_child->level > MAX_HUGEPTE_LEVEL && xe_child->num_live) { bool covered = xe_pt_covers(addr, next, xe_child->level, &xe_walk->base); @@ -1760,7 +1783,7 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, * we need to invalidate the PRL. */ if (killed || covered) - xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, xe_walk->prl, + xe_page_reclaim_list_abort(tile->primary_gt, prl, "kill at level=%u addr=%#llx next=%#llx num_live=%u", level, addr, next, xe_child->num_live); }