From 188d8139945c8dc37374efbf8552d0c2b13659a8 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Wed, 2 Sep 2026 13:41:19 +0100 Subject: [PATCH 01/76] drm/xe/migrate: support 4K PTEs for identity map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When VRAM limit (usable_size) is no longer 2M aligned (e.g. because flat CCS storage size rounds it down to 4K), we must map the tail of the identity mapping using 4K pages. This removes the 2M alignment assert (which pops) and introduces a 4K fallback in the identity map programming loop to handle the last 2M chunk of usable VRAM. This now also ensures we are only mapping the usable portion of VRAM. Might be possible to skip backporting this. The assert that pops should be harmless and is only on debug builds plus only a stray write would hit CCS, which technically get mapped. Assisted-by: Gemini:gemini-3.1-pro-preview Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Matthew Brost Cc: Rodrigo Vivi Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260902124117.918018-8-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_migrate.c | 57 +++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 75b83687f1b5..0bf000d7c901 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -87,7 +87,7 @@ struct xe_migrate { #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ #define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ #define NUM_KERNEL_PDE 15 -#define NUM_PT_SLOTS 32 +#define NUM_PT_SLOTS 48 #define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M #define MAX_NUM_PTE 512 #define IDENTITY_OFFSET 256ULL @@ -163,22 +163,20 @@ static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte) } static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo, - u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs) + u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs, + u64 pt_4k_ofs) { struct xe_vram_region *vram = xe->mem.vram; resource_size_t dpa_base = xe_vram_region_dpa_base(vram); u64 pos, ofs, flags; u64 entry; - /* XXX: Unclear if this should be usable_size? */ - u64 vram_limit = xe_vram_region_actual_physical_size(vram) + dpa_base; + u64 vram_limit = xe_vram_region_usable_size(vram) + dpa_base; u32 level = 2; ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8; flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, true, 0); - xe_assert(xe, IS_ALIGNED(xe_vram_region_usable_size(vram), SZ_2M)); - /* * Use 1GB pages when possible, last chunk always use 2M * pages as mixing reserved memory (stolen, WOCPM) with a single @@ -196,8 +194,24 @@ static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, true, 0); for (ofs = pt_2m_ofs; pos < vram_limit; - pos += SZ_2M, ofs += 8) + pos += SZ_2M, ofs += 8) { + if (pos + SZ_2M > vram_limit) { + entry = vm->pt_ops->pde_encode_bo(bo, pt_4k_ofs); + xe_map_wr(xe, &bo->vmap, ofs, u64, entry); + + flags = vm->pt_ops->pte_encode_addr(xe, 0, + pat_index, + level - 2, + true, 0); + + for (ofs = pt_4k_ofs; pos < vram_limit; + pos += SZ_4K, ofs += 8) + xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); + break; + } + xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); + } break; /* Ensure pos == vram_limit assert correct */ } @@ -242,16 +256,17 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); u8 id = tile->id; u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; -#define VRAM_IDENTITY_MAP_COUNT 2 - u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT; -#undef VRAM_IDENTITY_MAP_COUNT +#define VRAM_IDENTITY_MAP_PT_COUNT 4 + u32 num_setup = num_level + VRAM_IDENTITY_MAP_PT_COUNT; +#undef VRAM_IDENTITY_MAP_PT_COUNT u32 map_ofs, level, i; struct xe_bo *bo = m->pt_bo, *batch = tile->mem.kernel_bb_pool->bo; - u64 entry, pt29_ofs; + u64 entry; - /* PT30 & PT31 reserved for 2M identity map */ - pt29_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE; - entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs); + /* PT44..PT47 reserved for 4K and 2M identity map */ + u64 l1_pt_ofs = xe_bo_size(bo) - 5 * XE_PAGE_SIZE; + + entry = vm->pt_ops->pde_encode_bo(bo, l1_pt_ofs); xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; @@ -347,11 +362,12 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, /* Identity map the entire vram at 256GiB offset */ if (IS_DGFX(xe)) { - u64 pt30_ofs = xe_bo_size(bo) - 2 * XE_PAGE_SIZE; + u64 pt46_ofs = xe_bo_size(bo) - 2 * XE_PAGE_SIZE; resource_size_t actual_phy_size = xe_vram_region_actual_physical_size(xe->mem.vram); + u64 pt44_ofs = xe_bo_size(bo) - 4 * XE_PAGE_SIZE; xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET, - pat_index, pt30_ofs); + pat_index, pt46_ofs, pt44_ofs); xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G); /* @@ -362,12 +378,13 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, u16 comp_pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION); u64 vram_offset = IDENTITY_OFFSET + DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G); - u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE; + u64 pt47_ofs = xe_bo_size(bo) - XE_PAGE_SIZE; xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET - IDENTITY_OFFSET / 2) * SZ_1G); + u64 pt45_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE; xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset, - comp_pat_index, pt31_ofs); + comp_pat_index, pt47_ofs, pt45_ofs); } } @@ -381,8 +398,8 @@ static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs) * Example layout created above, with root level = 3: * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's - * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's - * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map] + * [PT9...PT40]: Userspace PT's for VM_BIND, 4 KiB PTE's + * [PT41 = PDE 0] [PT44...PT47 = 4K and 2M vram identity maps] * * This makes the lowest part of the VM point to the pagetables. * Hence the lowest 2M in the vm should point to itself, with a few writes From d00b7f4f03bbeb2efad872f1686130e18c2b4141 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Wed, 2 Sep 2026 13:41:20 +0100 Subject: [PATCH 02/76] drm/xe/vram: report FLAT_CCS base misalignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So we can easily check if a machine had the CCS bug, when looking back over bug reports where we have the same machine with newer kernel. Example print for a machine with the CCS bug: FLAT_CCS base:27bbff800, aligned:no v2 (Matt B): - Unconditionally print the base + alignment Fixes: 37173392741c ("drm/xe/vram: fix ccs offset calculation") Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Matthew Brost Cc: Rodrigo Vivi Cc: stable@kernel.org Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260902124117.918018-9-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_vram.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 7b4478fb1697..56cff1e44530 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -90,6 +90,9 @@ static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset) offset |= offset_lo << 6; /* HW view bits 31:6 */ offset *= num_enabled; /* convert to SW view */ + drm_info(&xe->drm, "FLAT_CCS base:%llx, aligned:%s\n", offset, + str_yes_no(IS_ALIGNED(offset, SZ_128K))); + /* * Everything below this offset is handed to the VRAM * allocator, so it has to be the *first* address the From bab69ad31cd35b282921efe825a496cf89dd7374 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Wed, 2 Sep 2026 13:41:21 +0100 Subject: [PATCH 03/76] drm/xe/vram: revamp CPU VRAM mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, we called devm_ioremap_wc() for the entire PCI LMEM BAR (which can be significantly larger than usable memory, e.g., mapping 16G for a 10G card), and then simply assigned subsets of this global mapping to each tile. By moving the devm_ioremap_wc() call into vram_region_init() and mapping on a per-tile basis, we restrict the virtual address space to exactly the usable_size of each tile. The other big win is that the core kernel will place a guard page at the end of each per-tile mapping to help catch OOB CPU writes (e.g. into the flat CCS storage) by triggering an immediate page fault instead of silent memory corruption. As a consequence the global vram->mapping is now NULL. But that was unused anyway, with CPU access already correctly routed through the per tile mapping. v2 (Sashiko) - Make sure to update the panic flow to now use the root tile. Assisted-by: Gemini:gemini-3.1-pro-preview Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Matthew Brost Cc: Rodrigo Vivi Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260902124117.918018-10-matthew.auld@intel.com --- drivers/gpu/drm/xe/display/xe_panic.c | 3 ++- drivers/gpu/drm/xe/xe_vram.c | 14 +++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/display/xe_panic.c b/drivers/gpu/drm/xe/display/xe_panic.c index 12c6fb99015d..1a6cee25e9d7 100644 --- a/drivers/gpu/drm/xe/display/xe_panic.c +++ b/drivers/gpu/drm/xe/display/xe_panic.c @@ -52,7 +52,8 @@ static void xe_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int if (new_page != panic->page) { if (xe_bo_is_vram(bo)) { /* Display is always mapped on root tile */ - struct xe_vram_region *vram = xe_bo_device(bo)->mem.vram; + struct xe_vram_region *vram = + xe_device_get_root_tile(xe_bo_device(bo))->mem.vram; if (panic->page < 0 || new_page < panic->page) { xe_res_first(bo->ttm.resource, new_page * PAGE_SIZE, diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 56cff1e44530..04d831b101bd 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -55,9 +55,6 @@ static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region * /* XXX: Need to change when xe link code is ready */ lmem_bar->dpa_base = 0; - /* set up a map to the total memory area. */ - lmem_bar->mapping = devm_ioremap_wc(&pdev->dev, lmem_bar->io_start, lmem_bar->io_size); - return 0; } @@ -196,7 +193,7 @@ static void vram_fini(void *arg) struct xe_tile *tile; int id; - xe->mem.vram->mapping = NULL; + xe_assert(xe, !xe->mem.vram->mapping); for_each_tile(tile, xe, id) { tile->mem.vram->mapping = NULL; @@ -257,8 +254,15 @@ static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram, return -ENODEV; } + if (vram != xe->mem.vram) { + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + vram->mapping = devm_ioremap_wc(&pdev->dev, vram->io_start, vram->io_size); + if (!vram->mapping) + return -ENOMEM; + } + vram->dpa_base = lmem_bar->dpa_base + offset; - vram->mapping = lmem_bar->mapping + offset; vram->usable_size = usable_size; print_vram_region_info(xe, vram); From 6df8aa0e8a323201a1eb85a98864e90b6b59230a Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Wed, 2 Sep 2026 13:41:22 +0100 Subject: [PATCH 04/76] drm/xe: add force option for global invalidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flush is gated by the WA, since there would no valid use for it otherwise, but for some very special usecases allow bypassing the WA. Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Matthew Brost Cc: Rodrigo Vivi Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260902124117.918018-11-matthew.auld@intel.com --- drivers/gpu/drm/xe/display/xe_dsb_buffer.c | 2 +- drivers/gpu/drm/xe/display/xe_fb_pin.c | 2 +- drivers/gpu/drm/xe/xe_device.c | 6 +++--- drivers/gpu/drm/xe/xe_device.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c index a7158c73a14c..82974e933e8d 100644 --- a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c +++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c @@ -88,7 +88,7 @@ static void xe_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf) * both for weak ordering archs and discrete cards. */ xe_device_wmb(xe); - xe_device_l2_flush(xe); + xe_device_l2_flush(xe, false); } const struct intel_display_dsb_interface xe_display_dsb_interface = { diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index 73469ea5f333..b46a2c32ac07 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -203,7 +203,7 @@ static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj, vma->node = dpt->ggtt_node[tile0->id]; /* Ensure DPT writes are flushed */ - xe_device_l2_flush(xe); + xe_device_l2_flush(xe, false); return 0; } diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 396d02eb2af8..fa98b2d12204 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1270,7 +1270,7 @@ bool xe_device_is_l2_flush_optimized(struct xe_device *xe) return false; } -void xe_device_l2_flush(struct xe_device *xe) +void xe_device_l2_flush(struct xe_device *xe, bool force) { struct xe_gt *gt; @@ -1278,7 +1278,7 @@ void xe_device_l2_flush(struct xe_device *xe) if (!gt) return; - if (!XE_GT_WA(gt, 16023588340)) + if (!force && !XE_GT_WA(gt, 16023588340)) return; CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); @@ -1333,7 +1333,7 @@ void xe_device_td_flush(struct xe_device *xe) if (XE_GT_WA(root_gt, 16023588340)) { /* A transient flush is not sufficient: flush the L2 */ - xe_device_l2_flush(xe); + xe_device_l2_flush(xe, false); } else { xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc); tdf_request_sync(xe); diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index 6c4cfaebc44a..6d3d6d5eba29 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -205,7 +205,7 @@ u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address); bool xe_device_is_l2_flush_optimized(struct xe_device *xe); void xe_device_td_flush(struct xe_device *xe); -void xe_device_l2_flush(struct xe_device *xe); +void xe_device_l2_flush(struct xe_device *xe, bool force); static inline bool xe_device_wedged(struct xe_device *xe) { From c874bc70c897c554cc460cfe1fa2c715f780d368 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Wed, 2 Sep 2026 13:41:23 +0100 Subject: [PATCH 05/76] drm/xe/vram: add early VRAM health check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During early probe, use the last page as a canary for BAR sizing, CCS sizing, identity map setup etc. If something is wrong the last page is where we will likely find it. Hit it with everything we have. For now this is gated behind a debug config option, so shouldn't trigger on production. Main motivation is around CCS sizing where on some BMG cards the CCS offset is programmed misaligned, for whatever reason, and our handling of that was busted, as found by Linus, leading to some amount of CCS storage getting pulled into the allocator as normal VRAM. Nothing in our CI farm has such a misaligned offset it would seem, however I did get this to pop on my b570, which does also have the misaligned CCS offset: Tile 0: Running VRAM memtest... Tile 0: VRAM bounds overlap CCS region! VRAM sizing is incorrect. With the fix from Linus, this goes away: Tile 0: Running VRAM memtest... Tile 0: VRAM memtest completed. For the VRAM health check itself, this adds: - CPU access to the last page (BAR). - GPU access to the last page (identity map). - CCS overlap check. This one is more involved, but overall idea is fill the last page with a known pattern, and also save the CCS state for the first 4M of VRAM to some scratch memory. We then zero the CCS storage for that same range, all using the proper CCS copy instruction. At this point we readback the last page, and check if the pattern we wrote changed. Finally we restore the CCS state. This works since CCS 1:1 maps with VRAM, so the start of the raw CCS, should map to the start of VRAM. This successfully catches the issue that Linus found and fixed. Assisted-by: Gemini:gemini-3.1-pro-preview Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Linus Torvalds Cc: Stuart Summers Cc: Matthew Brost Cc: Rodrigo Vivi Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260902124117.918018-12-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_device.c | 8 + drivers/gpu/drm/xe/xe_migrate.c | 64 ++++++++ drivers/gpu/drm/xe/xe_migrate.h | 6 + drivers/gpu/drm/xe/xe_tile_types.h | 4 + drivers/gpu/drm/xe/xe_vram.c | 241 +++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_vram.h | 10 ++ 6 files changed, 333 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index fa98b2d12204..8583b2e9ecf4 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -1051,6 +1051,10 @@ int xe_device_probe(struct xe_device *xe) if (err) return err; + err = xe_vram_reserve_memtest_bo(xe); + if (err) + return err; + for_each_tile(tile, xe, id) { err = xe_tile_init(tile); if (err) @@ -1067,6 +1071,10 @@ int xe_device_probe(struct xe_device *xe) return err; } + err = xe_vram_memtest(xe); + if (err) + return err; + err = xe_pagefault_init(xe); if (err) return err; diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 0bf000d7c901..ff45c24d8889 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -2650,3 +2650,67 @@ void xe_migrate_job_lock_assert(struct xe_exec_queue *q) #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) #include "tests/xe_migrate.c" #endif + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) +int xe_migrate_debug_ccs_overlap(struct xe_migrate *m, + struct xe_bo *scratch_bo, + bool write_to_ccs) +{ + struct xe_device *xe = tile_to_xe(m->tile); + struct xe_gt *gt = m->tile->primary_gt; + struct dma_fence *fence; + struct xe_bb *bb; + struct xe_sched_job *job; + u64 first_page_dpa, clear_L0_ofs, scratch_dpa, scratch_L0_ofs; + + if (!xe_device_has_flat_ccs(xe)) + return -EINVAL; + + first_page_dpa = xe_vram_region_dpa_base(m->tile->mem.vram); + clear_L0_ofs = xe_migrate_vram_ofs(xe, first_page_dpa, true); + + scratch_dpa = xe_bo_addr(scratch_bo, 0, XE_PAGE_SIZE); + scratch_L0_ofs = xe_migrate_vram_ofs(xe, scratch_dpa, false); + + bb = xe_bb_new(gt, EMIT_COPY_CCS_DW + 1, xe->info.has_usm); + if (IS_ERR(bb)) { + drm_warn(&xe->drm, "Failed to create bb for VRAM overlap check\n"); + return PTR_ERR(bb); + } + + /* 4MB payload = 8KB CCS metadata */ + if (write_to_ccs) { + emit_copy_ccs(gt, bb, clear_L0_ofs, true, + scratch_L0_ofs, false, SZ_4M); + } else { + emit_copy_ccs(gt, bb, scratch_L0_ofs, false, + clear_L0_ofs, true, SZ_4M); + } + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + + job = xe_bb_create_migration_job(m->q, bb, + xe_migrate_batch_base(m, xe->info.has_usm), + 0); + if (!IS_ERR(job)) { + xe_sched_job_add_migrate_flush(job, MI_FLUSH_DW_CCS); + + mutex_lock(&m->job_mutex); + xe_sched_job_arm(job); + + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + mutex_unlock(&m->job_mutex); + + dma_fence_wait(fence, false); + dma_fence_put(fence); + } else { + drm_warn(&xe->drm, "Failed to create job for VRAM overlap check\n"); + xe_bb_free(bb, NULL); + return PTR_ERR(job); + } + + xe_bb_free(bb, NULL); + return 0; +} +#endif diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index c3a268b01768..a9acc62f78f0 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -182,4 +182,10 @@ static inline void xe_migrate_job_lock_assert(struct xe_exec_queue *q) void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q); void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q); +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) +int xe_migrate_debug_ccs_overlap(struct xe_migrate *m, + struct xe_bo *scratch_bo, + bool write_to_ccs); +#endif + #endif diff --git a/drivers/gpu/drm/xe/xe_tile_types.h b/drivers/gpu/drm/xe/xe_tile_types.h index 0048100ccb72..e1368c04846a 100644 --- a/drivers/gpu/drm/xe/xe_tile_types.h +++ b/drivers/gpu/drm/xe/xe_tile_types.h @@ -97,6 +97,10 @@ struct xe_tile { * Only main GT has page reclaim list allocations. */ struct xe_sa_manager *reclaim_pool; +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) + /** @mem.memtest_bo: VRAM overlap check BO */ + struct xe_bo *memtest_bo; +#endif } mem; /** @sriov: tile level virtualization data */ diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 04d831b101bd..dcedd8cfd731 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -17,8 +17,11 @@ #include "xe_device.h" #include "xe_force_wake.h" #include "xe_gt_mcr.h" +#include "xe_map.h" +#include "xe_migrate.h" #include "xe_mmio.h" #include "xe_sriov.h" +#include "xe_tile.h" #include "xe_tile_sriov_vf.h" #include "xe_ttm_vram_mgr.h" #include "xe_vram.h" @@ -407,3 +410,241 @@ resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region return vram ? vram->actual_physical_size : 0; } EXPORT_SYMBOL_IF_KUNIT(xe_vram_region_actual_physical_size); + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) +static void memtest_bo_cleanup(void *arg) +{ + struct xe_device *xe = arg; + + xe_vram_free_memtest_bos(xe); +} + +int xe_vram_reserve_memtest_bo(struct xe_device *xe) +{ + struct xe_tile *tile; + u8 id; + + if (IS_SRIOV_VF(xe)) + return 0; + + for_each_tile(tile, xe, id) { + u64 vram_size; + + if (!tile->mem.vram) + continue; + + if (tile->mem.vram->io_size < tile->mem.vram->usable_size) { + drm_info(&xe->drm, + "Tile %d: Small-BAR system detected, skipping VRAM memtest\n", + id); + continue; + } + + vram_size = tile->mem.vram->usable_size; + + tile->mem.memtest_bo = xe_bo_create_pin_map_at_novm(xe, tile, SZ_64K, + vram_size - SZ_64K, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile), + 0, false); + if (IS_ERR(tile->mem.memtest_bo)) { + drm_warn(&xe->drm, "Tile %d: Failed to reserve memtest BO\n", id); + tile->mem.memtest_bo = NULL; + continue; + } + + drm_info(&xe->drm, "Tile %d: Reserved memtest BO at offset 0x%llx\n", + id, vram_size - SZ_64K); + } + + return devm_add_action_or_reset(xe->drm.dev, memtest_bo_cleanup, xe); +} + +void xe_vram_free_memtest_bos(struct xe_device *xe) +{ + struct xe_tile *tile; + u8 id; + + for_each_tile(tile, xe, id) { + if (tile->mem.memtest_bo) { + xe_bo_unpin_map_no_vm(tile->mem.memtest_bo); + tile->mem.memtest_bo = NULL; + } + } +} + +int xe_vram_memtest(struct xe_device *xe) +{ + struct xe_tile *tile; + u8 id; + int err = 0; + + if (IS_SRIOV_VF(xe)) + return 0; + + for_each_tile(tile, xe, id) { + struct xe_bo *last_page_bo = tile->mem.memtest_bo; + struct dma_fence *fence; + bool overlap = false; + int i; + u8 val; + + if (!last_page_bo || !tile->migrate) + continue; + + drm_info(&xe->drm, "Tile %d: Running VRAM memtest...\n", id); + + /* CPU write and readback first and last byte of the last page */ + xe_map_wr(xe, &last_page_bo->vmap, 0, u8, 0xA5); + xe_map_wr(xe, &last_page_bo->vmap, SZ_64K - 1, u8, 0x5A); + + val = xe_map_rd(xe, &last_page_bo->vmap, 0, u8); + if (drm_WARN(&xe->drm, val != 0xA5, + "Tile %d: CPU memtest failed at offset 0 (expected 0xA5, got 0x%02x)\n", + id, val)) { + err = -EIO; + goto unpin; + } + + val = xe_map_rd(xe, &last_page_bo->vmap, SZ_64K - 1, u8); + if (drm_WARN(&xe->drm, val != 0x5A, + "Tile %d: CPU memtest failed at offset 65535 (expected 0x5A, got 0x%02x)\n", + id, val)) { + err = -EIO; + goto unpin; + } + + /* Non-CCS access via GPU on the last page */ + xe_bo_lock(last_page_bo, false); + fence = xe_migrate_clear(tile->migrate, last_page_bo, + last_page_bo->ttm.resource, + XE_MIGRATE_CLEAR_FLAG_BO_DATA); + xe_bo_unlock(last_page_bo); + + if (!IS_ERR(fence)) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } else { + err = PTR_ERR(fence); + goto unpin; + } + + val = xe_map_rd(xe, &last_page_bo->vmap, 0, u8); + if (drm_WARN(&xe->drm, val != 0x00, + "Tile %d: GPU memtest clear failed at offset 0 (expected 0x00, got 0x%02x)\n", + id, val)) { + err = -EIO; + goto unpin; + } + + /* + * Check for CCS overlap on the root tile. + * + * TODO: maybe extend if we ever get multi-tile + CCS. Pay + * special attention to the l2 flush below. Currently that is + * hard coded to the root tile. + */ + if (!id && xe_device_has_flat_ccs(xe) && + GRAPHICS_VERx100(xe) >= 2000) { + struct xe_bo *scratch_bo_before; + struct xe_bo *scratch_bo_after; + + scratch_bo_before = xe_bo_create_pin_map_novm(xe, tile, SZ_64K, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile), + false); + if (IS_ERR(scratch_bo_before)) { + err = PTR_ERR(scratch_bo_before); + goto unpin; + } + + scratch_bo_after = xe_bo_create_pin_map_novm(xe, tile, SZ_64K, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile), + false); + if (IS_ERR(scratch_bo_after)) { + xe_bo_unpin_map_no_vm(scratch_bo_before); + err = PTR_ERR(scratch_bo_after); + goto unpin; + } + + /* Save original CCS metadata for PA 0 + */ + err = xe_migrate_debug_ccs_overlap(tile->migrate, scratch_bo_before, false); + if (err) { + xe_bo_unpin_map_no_vm(scratch_bo_before); + xe_bo_unpin_map_no_vm(scratch_bo_after); + goto unpin; + } + + /* + * Fill last page. If there is CCS overlap in the last + * page this will snag the raw CCS storage. + */ + xe_map_memset(xe, &last_page_bo->vmap, 0, 0x5A, SZ_64K); + xe_device_wmb(xe); + + /* + * Global invalidation. Some BMG SKUs will cache the BAR + * writes in the GPU side VRAM cache. Make sure above + * writes are fully flushed out to VRAM, so this is + * hopefully more well behaved with the CCS unit, if + * there is indeed CCS overlap with normal VRAM. Since + * there is a separate CCS cache, the CCS unit might not + * respect the GPU VRAM cache for CCS accesses, so opt + * for being super careful here. + */ + xe_device_l2_flush(xe, true); + + /* Use GPU to clear CCS state for PA 0 */ + xe_map_memset(xe, &scratch_bo_after->vmap, 0, 0x00, SZ_64K); + err = xe_migrate_debug_ccs_overlap(tile->migrate, scratch_bo_after, true); + if (err) { + xe_bo_unpin_map_no_vm(scratch_bo_before); + xe_bo_unpin_map_no_vm(scratch_bo_after); + goto unpin; + } + /* + * Global invalidation. Ensure CCS caches really are + * nuked and the raw CCS data is visible in VRAM, for + * the below access. + */ + xe_device_l2_flush(xe, true); + + /* Check if last_page_bo was corrupted by the GPU CCS clear */ + for (i = 0; i < SZ_64K; i += 8) { + u64 payload = xe_map_rd(xe, &last_page_bo->vmap, i, u64); + + if (payload != 0x5A5A5A5A5A5A5A5AULL) { + overlap = true; + break; + } + } + + /* Restore original CCS metadata for PA 0 + */ + err = xe_migrate_debug_ccs_overlap(tile->migrate, scratch_bo_before, true); + if (err) + drm_warn(&xe->drm, "Failed to restore CCS metadata\n"); + + xe_bo_unpin_map_no_vm(scratch_bo_before); + xe_bo_unpin_map_no_vm(scratch_bo_after); + } + + if (drm_WARN(&xe->drm, overlap, + "Tile %d: VRAM bounds overlap CCS region! VRAM sizing is incorrect.\n", + id)) { + err = -EINVAL; + goto unpin; + } + + drm_info(&xe->drm, "Tile %d: VRAM memtest completed.\n", id); + +unpin: + if (err) + break; + } + + xe_vram_free_memtest_bos(xe); + + return err; +} +#endif diff --git a/drivers/gpu/drm/xe/xe_vram.h b/drivers/gpu/drm/xe/xe_vram.h index dd1c8bf17922..38425d81f777 100644 --- a/drivers/gpu/drm/xe/xe_vram.h +++ b/drivers/gpu/drm/xe/xe_vram.h @@ -23,4 +23,14 @@ resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram); resource_size_t xe_vram_region_usable_size(const struct xe_vram_region *vram); resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region *vram); +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) +int xe_vram_reserve_memtest_bo(struct xe_device *xe); +void xe_vram_free_memtest_bos(struct xe_device *xe); +int xe_vram_memtest(struct xe_device *xe); +#else +static inline int xe_vram_reserve_memtest_bo(struct xe_device *xe) { return 0; } +static inline void xe_vram_free_memtest_bos(struct xe_device *xe) {} +static inline int xe_vram_memtest(struct xe_device *xe) { return 0; } +#endif + #endif From e0ba981c317209e8464ead7dc81439b1210a2fe3 Mon Sep 17 00:00:00 2001 From: Jonathan Cavitt Date: Thu, 3 Sep 2026 05:43:57 +0800 Subject: [PATCH 06/76] drm/xe/pagefault: Add SRCID to pagefault struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add SRCID information to pagefault struct for the purpose of reporting the hardware unit that resulted in the pagefault. v2: - Squash SRCID with ASID to keep the struct compact (Matthew) v3: - Use BUILD_BUG_ON and move ASID definition in one function (Matthew) v4: - Various formatting fixes Signed-off-by: Jonathan Cavitt Reviewed-by: Matthew Brost Cc: Matthew Brost Cc: Thomas Hellström Cc: Rodrigo Vivi Cc: David Airlie Cc: Simona Vetter Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260902214358.348399-2-jonathan.cavitt@intel.com --- drivers/gpu/drm/xe/xe_guc_pagefault.c | 8 +++++++- drivers/gpu/drm/xe/xe_pagefault.c | 24 +++++++++++++++--------- drivers/gpu/drm/xe/xe_pagefault_types.h | 9 +++++++-- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_pagefault.c b/drivers/gpu/drm/xe/xe_guc_pagefault.c index 8f8210a732e9..df237fd40551 100644 --- a/drivers/gpu/drm/xe/xe_guc_pagefault.c +++ b/drivers/gpu/drm/xe/xe_guc_pagefault.c @@ -108,7 +108,13 @@ int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len) << PFD_VIRTUAL_ADDR_HI_SHIFT) | (FIELD_GET(PFD_VIRTUAL_ADDR_LO, msg[2]) << PFD_VIRTUAL_ADDR_LO_SHIFT); - pf.consumer.asid = FIELD_GET(PFD_ASID, msg[1]); + + BUILD_BUG_ON(XE_MAX_ASID > XE_PAGEFAULT_ASID_MASK); + + pf.consumer.id = FIELD_PREP(XE_PAGEFAULT_ASID_MASK, + FIELD_GET(PFD_ASID, msg[1])) | + FIELD_PREP(XE_PAGEFAULT_SRCID_MASK, + FIELD_GET(PFD_SRC_ID, msg[0])); pf.consumer.access_type = FIELD_GET(PFD_ACCESS_TYPE, msg[2]) | (FIELD_GET(PFD_PREFETCH, msg[2]) ? XE_PAGEFAULT_ACCESS_PREFETCH : 0); if (FIELD_GET(XE2_PFD_TRVA_FAULT, msg[0])) diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index 2e415995f067..e81ca24df37f 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -252,12 +252,13 @@ static int xe_pagefault_service(struct xe_pagefault *pf) struct xe_vma *vma = NULL; int err; bool atomic; + u32 asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id); /* Producer flagged this fault to be nacked */ if (pf->consumer.fault_type_level == XE_PAGEFAULT_TYPE_LEVEL_NACK) return -EFAULT; - vm = xe_pagefault_asid_to_vm(xe, pf->consumer.asid); + vm = xe_pagefault_asid_to_vm(xe, asid); if (IS_ERR(vm)) return PTR_ERR(vm); @@ -374,7 +375,7 @@ static bool xe_pagefault_match(struct xe_pagefault *pf, u64 start, { struct xe_device *xe = gt_to_xe(pf->gt); u64 page_addr = pf->consumer.page_addr; - u32 pf_asid = pf->consumer.asid; + u32 pf_asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id); xe_assert(xe, pf->consumer.alloc_state != XE_PAGEFAULT_ALLOC_STATE_FREE); @@ -499,7 +500,7 @@ static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue, align = SZ_4K; pf_work->cache.start = ALIGN_DOWN(lpf->consumer.page_addr, align); pf_work->cache.end = pf_work->cache.start + align; - pf_work->cache.asid = lpf->consumer.asid; + pf_work->cache.asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, lpf->consumer.id); pf_work->cache.pf = lpf; lpf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_ACTIVE; @@ -545,14 +546,16 @@ static void xe_pagefault_print(struct xe_pagefault *pf) u8 engine_class = FIELD_GET(XE_PAGEFAULT_ENGINE_CLASS_MASK, pf->consumer.engine_class_instance); - xe_gt_info(pf->gt, "\n\tASID: %d\n" + xe_gt_info(pf->gt, "\n\tASID: %lu\n" "\tFaulted Address: 0x%08x%08x\n" "\tFaultType: %lu\n" "\tAccessType: %lu\n" "\tFaultLevel: %lu\n" "\tEngineClass: %d %s\n" - "\tEngineInstance: %lu\n", - pf->consumer.asid, + "\tEngineInstance: %lu\n" + "\tSRCID: 0x%02lx\n", + FIELD_GET(XE_PAGEFAULT_ASID_MASK, + pf->consumer.id), upper_32_bits(pf->consumer.page_addr), lower_32_bits(pf->consumer.page_addr), FIELD_GET(XE_PAGEFAULT_TYPE_MASK, @@ -564,7 +567,9 @@ static void xe_pagefault_print(struct xe_pagefault *pf) engine_class, xe_hw_engine_class_to_str(engine_class), FIELD_GET(XE_PAGEFAULT_ENGINE_INSTANCE_MASK, - pf->consumer.engine_class_instance)); + pf->consumer.engine_class_instance), + FIELD_GET(XE_PAGEFAULT_SRCID_MASK, + pf->consumer.id)); } static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *pf) @@ -577,7 +582,8 @@ static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *p * mode, return VM anyways. */ down_read(&xe->usm.lock); - vm = xa_load(&xe->usm.asid_to_vm, pf->consumer.asid); + vm = xa_load(&xe->usm.asid_to_vm, + FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id)); if (vm) xe_vm_get(vm); else @@ -611,7 +617,7 @@ static void xe_pagefault_queue_work(struct work_struct *w) const struct xe_pagefault_ops *ops = pf->producer.ops; void *private = pf->producer.private; struct xe_gt *gt = pf->gt; - u32 asid = pf->consumer.asid; + u32 asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id); int err = 0; bool invalidated = false; diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h index efeba5c3a58b..f16bab29bfc3 100644 --- a/drivers/gpu/drm/xe/xe_pagefault_types.h +++ b/drivers/gpu/drm/xe/xe_pagefault_types.h @@ -112,8 +112,13 @@ struct xe_pagefault { u8 engine_class_instance; #define XE_PAGEFAULT_ENGINE_CLASS_MASK GENMASK(3, 0) #define XE_PAGEFAULT_ENGINE_INSTANCE_MASK GENMASK(7, 4) - /** @consumer.asid: address space ID */ - u32 asid; + /** + * @consumer.id: address space ID and SRCID, folded into one + * to keep size compact + */ + u32 id; +#define XE_PAGEFAULT_ASID_MASK GENMASK(23, 0) +#define XE_PAGEFAULT_SRCID_MASK GENMASK(31, 24) }; /** * @consumer.end_addr: end address of page fault, From 88064894bcce6d239804e54bd7f577645d4739ab Mon Sep 17 00:00:00 2001 From: Jonathan Cavitt Date: Thu, 3 Sep 2026 05:43:58 +0800 Subject: [PATCH 07/76] drm/xe/vm: Add srcid to xe_vm_get_property_ioctl fault report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the SRCID of the faulting hardware unit to the return of the xe_vm_get_property_ioctl fault report. v2: - Readd pad check, as the pad in the ioctl struct was not changed (jcavitt) v3: - Squash SRCID with ASID to keep the struct compact (Matthew) Signed-off-by: Jonathan Cavitt Reviewed-by: Matthew Brost Cc: Matthew Brost Cc: Thomas Hellström Cc: Rodrigo Vivi Cc: David Airlie Cc: Simona Vetter Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Thomas Zimmermann Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260902214358.348399-3-jonathan.cavitt@intel.com --- drivers/gpu/drm/xe/xe_vm.c | 8 ++++++++ drivers/gpu/drm/xe/xe_vm_types.h | 2 ++ include/uapi/drm/xe_drm.h | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 19b3d0be7928..753a5fc55baa 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -655,6 +655,7 @@ void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf) pf->consumer.fault_type_level); e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK, pf->consumer.fault_type_level); + e->srcid = FIELD_GET(XE_PAGEFAULT_SRCID_MASK, pf->consumer.id); list_add_tail(&e->list, &vm->faults.list); vm->faults.len++; @@ -4277,6 +4278,11 @@ static u8 xe_to_user_fault_level(u8 fault_level) return fault_level; } +static u8 xe_to_user_srcid(u8 srcid) +{ + return srcid; +} + static int fill_faults(struct xe_vm *vm, struct drm_xe_vm_get_property *args) { @@ -4304,6 +4310,8 @@ static int fill_faults(struct xe_vm *vm, fault_entry.fault_type = xe_to_user_fault_type(entry->fault_type); fault_entry.fault_level = xe_to_user_fault_level(entry->fault_level); + fault_entry.srcid = xe_to_user_srcid(entry->srcid); + memcpy(&fault_list[i], &fault_entry, entry_size); i++; diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 68588b624212..648031e64145 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -202,6 +202,7 @@ struct xe_device; * @access_type: type of address access that resulted in fault * @fault_type: type of fault reported * @fault_level: fault level of the fault + * @srcid: ID of the faulting hardware unit */ struct xe_vm_fault_entry { struct list_head list; @@ -210,6 +211,7 @@ struct xe_vm_fault_entry { u8 access_type; u8 fault_type; u8 fault_level; + u8 srcid; }; struct xe_vm { diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 509202a7b13e..8c852abcacf0 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1294,8 +1294,8 @@ struct xe_vm_fault { #define FAULT_LEVEL_PML4 3 #define FAULT_LEVEL_PML5 4 __u8 fault_level; - /** @pad: MBZ */ - __u8 pad; + /** @srcid: ID of the faulting hardware unit */ + __u8 srcid; /** @reserved: MBZ */ __u64 reserved[4]; }; From c1b1906f86ee34bae4a80d0ed4176b64d82e8dec Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:45:54 +0530 Subject: [PATCH 08/76] drm/xe: Link VRAM resource allocation with gpu buddy Setup to link allocation resource structure inside gpu buddy. This functionality is critical for supporting the memory page offline feature on CRI, where identified faulty pages must be traced back to their originating buffer for safe removal. v3(MattB): store memory resource(res) instead of tbo V2(MattB): Clear block->private in xe_ttm_vram_mgr_del as well Reviewed-by: Andi Shyti Reviewed-by: Himal Prasad Ghimiray Reviewed-by: Matthew Brost Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-18-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 05911904c1f9..0f9cf1c3a576 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -57,6 +57,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); struct xe_ttm_vram_mgr_resource *vres; struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; u64 size, min_page_size; unsigned long lpfn; int err; @@ -141,6 +142,8 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, } mgr->visible_avail -= vres->used_visible_size; + list_for_each_entry(block, &vres->blocks, link) + block->private = &vres->base; mutex_unlock(&mgr->lock); if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) && @@ -179,8 +182,11 @@ static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, to_xe_ttm_vram_mgr_resource(res); struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; mutex_lock(&mgr->lock); + list_for_each_entry(block, &vres->blocks, link) + block->private = NULL; gpu_buddy_free_list(mm, &vres->blocks, 0); mgr->visible_avail += vres->used_visible_size; mutex_unlock(&mgr->lock); From 4356370caf36bc9f7ec9a7a376b9876d53b92878 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:45:55 +0530 Subject: [PATCH 09/76] drm/xe: Link LRC BO and its execution queue with safe lifetime rules Introduce an execution queue back-pointer (`q`) within `struct xe_bo`, primarily for Logical Ring Context (LRC) Buffer Objects. This back-pointer allows the driver to identify and execute targeted corrective actions on a specific queue if its associated LRC BO encounters errors like memory corruption or eviction. Because this back-pointer takes no reference on its target execution queue, strict lifetime and serialization rules are implemented to prevent concurrent readers from encountering use-after-free or dangling pointer bugs: - Encapsulate tracking logic inside xe_exec_queue_set_lrc_bo_backpointer() and xe_exec_queue_clear_lrc_bo_backpointer(). - Explicitly wrap all back-pointer writes and clears under the BO's dma_resv lock via xe_bo_lock(). Readers must hold this same lock across both the pointer read and its subsequent xe_exec_queue_get_unless_zero() call to guarantee serialization against teardown. - Defer publishing the back-pointer until the very end of xe_exec_queue_create(). This ensures that early initialization failure paths (which bypass the kref mechanism and immediately free the queue structure) never leak a transient pointer to concurrent readers. - Clear the back-pointer at the absolute top of __xe_exec_queue_fini(). This strips the pointer before q->ops->fini() destroys the hardware backend, ensuring that any reader holding the BO lock either observes a fully functional queue or NULL. For multi-queue engines, secondary LRC BOs safely point to the primary queue, which is guaranteed to outlive the teardown pass due to active references held by its secondaries. V4 (MattB): - Add LRC BO's execution queue safe lifetime rules V3 (Sashiko): - Use 8-byte placeholder structure compatibility for non-LRC BO cases. - Wrap assignments and clears securely under dma_resv locks. V2 (Matt B): - Add native support handling multi-queue configuration tracking. Assisted-by: Copilot:claude-opus-4.6 Reviewed-by: Andi Shyti Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-19-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_bo_types.h | 8 ++++ drivers/gpu/drm/xe/xe_exec_queue.c | 64 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index e45f24301050..0eb93052c1d5 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -20,6 +20,7 @@ struct xe_device; struct xe_mem_pool_node; struct xe_vm; +struct xe_exec_queue; #define XE_BO_MAX_PLACEMENTS 3 @@ -42,6 +43,13 @@ struct xe_bo { u32 flags; /** @vm: VM this BO is attached to, for extobj this will be NULL */ struct xe_vm *vm; + /** + * @q: Queue this BO is attached to, mostly for LRC BO, NULL otherwise. + * Protected by the BO dma_resv: readers must hold it across both the + * read and xe_exec_queue_get_unless_zero(). The BO holds no reference + * on the queue. + */ + struct xe_exec_queue *q; /** @tile: Tile this BO is attached to (kernel BO only) */ struct xe_tile *tile; /** @placements: valid placements for this BO */ diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index c4213bb9c137..f3ea4807ac33 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -322,10 +322,66 @@ struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) return q->lrc[0]; } +/* + * Publish the queue back-pointer in the LRC BOs. + * + * The BO holds no reference on the queue; the queue owns the LRCs, and + * therefore the BOs, instead. The back-pointer is made safe by two rules: + * + * - It is published only once the queue is fully constructed and can no + * longer be destroyed by an error path that bypasses the kref (see + * xe_exec_queue_create()), so a reader that successfully takes a + * reference can never be handed a queue that is freed without going + * through xe_exec_queue_destroy(). + * + * - It is written and cleared under the BO dma_resv. Readers must hold + * the same lock across both the read and + * xe_exec_queue_get_unless_zero(), which serializes them against + * xe_exec_queue_clear_lrc_bo_backpointer() below. + * + * For a multi-queue group the LRC BOs point at the primary queue, which is + * kept alive by the reference every secondary holds on it. + */ +static void xe_exec_queue_set_lrc_bo_backpointer(struct xe_exec_queue *q) +{ + struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); + int i; + + for (i = 0; i < q->width; ++i) { + struct xe_bo *bo = q->lrc[i]->bo; + + xe_bo_lock(bo, false); + bo->q = primary; + xe_bo_unlock(bo); + } +} + +/* + * Drop the queue back-pointer before anything belonging to the queue is + * torn down. This must happen before q->ops->fini(), otherwise a reader + * could take a reference and then operate on an already destroyed backend. + */ +static void xe_exec_queue_clear_lrc_bo_backpointer(struct xe_exec_queue *q) +{ + int i; + + for (i = 0; i < q->width; ++i) { + struct xe_bo *bo = q->lrc[i] ? q->lrc[i]->bo : NULL; + + if (!bo) + continue; + + xe_bo_lock(bo, false); + bo->q = NULL; + xe_bo_unlock(bo); + } +} + static void __xe_exec_queue_fini(struct xe_exec_queue *q) { int i; + xe_exec_queue_clear_lrc_bo_backpointer(q); q->ops->fini(q); for (i = 0; i < q->width; ++i) @@ -450,6 +506,14 @@ struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *v goto err_post_init; } + /* + * Publish the LRC BO back-pointers last: past this point the queue can + * only be destroyed through xe_exec_queue_destroy(), so a concurrent + * reader that takes a reference via bo->q cannot race with the + * kref-bypassing error paths below. + */ + xe_exec_queue_set_lrc_bo_backpointer(q); + return q; err_post_init: From 1faafb979f2da6662bc62aa4030020f38b2044aa Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:45:56 +0530 Subject: [PATCH 10/76] drm/xe: Export xe_ttm_bo_purge() Remove the static qualifier from xe_ttm_bo_purge() and add its prototype to xe_bo.h. This allows the function to be called from other parts of the driver outside of xe_bo.c, specifically needed for the memory page offline feature. Cc: Andi Shyti Reviewed-by: Matthew Brost Reviewed-by: Arvind Yadav Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-20-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_bo.c | 2 +- drivers/gpu/drm/xe/xe_bo.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index dde309821237..8f93e09e51bb 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -921,7 +921,7 @@ void xe_bo_set_purgeable_state(struct xe_bo *bo, * * Return: 0 on success, negative error code on failure */ -static int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx) +int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx) { struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); struct ttm_placement place = {}; diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index e8081af5bfc1..90b15fff36c7 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -600,6 +600,7 @@ struct xe_bo_shrink_flags { long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, const struct xe_bo_shrink_flags flags, unsigned long *scanned); +int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx); /** * xe_bo_is_mem_type - Whether the bo currently resides in the given From 529f45af3e65e17448c6b402a4dce2245fb2df08 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:45:57 +0530 Subject: [PATCH 11/76] drm/xe: Handle NULL resource and allow purging of VRAM pages Add NULL checks for the buffer object's resource pointer in xe_bo_is_vram() and xe_bo_is_stolen() to prevent potential NULL pointer dereferences when checking memory location during page offlining. Additionally, remove the `!ttm_bo->ttm` check from xe_ttm_bo_purge(). This restriction is stripped away to explicitly allow the driver to purge VRAM pages during the memory page offline sequence. v2(Sashiko): - Split export and this patch also Check NULL res in xe_bo_is_vram() and xe_bo_is_stolen() Reviewed-by: Himal Prasad Ghimiray Cc: Andi Shyti Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-21-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_bo.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 8f93e09e51bb..d30a8411d2a3 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -104,13 +104,16 @@ static bool resource_is_vram(struct ttm_resource *res) bool xe_bo_is_vram(struct xe_bo *bo) { - return resource_is_vram(bo->ttm.resource) || - resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource); + struct ttm_resource *res = bo->ttm.resource; + + return res && (resource_is_vram(res) || resource_is_stolen_vram(xe_bo_device(bo), res)); } bool xe_bo_is_stolen(struct xe_bo *bo) { - return bo->ttm.resource->mem_type == XE_PL_STOLEN; + struct ttm_resource *res = bo->ttm.resource; + + return res && res->mem_type == XE_PL_STOLEN; } /** @@ -929,9 +932,6 @@ int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx * xe_bo_assert_held(bo); - if (!ttm_bo->ttm) - return 0; - if (!xe_bo_madv_is_dontneed(bo)) return 0; From e86c44c4c92ba4a30ef775d6d713e7dc50ca20fa Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:45:58 +0530 Subject: [PATCH 12/76] drm/xe/bo: Make xe_bo_is_user() public Export xe_bo_is_user() so it can be used by the VRAM page offline code to distinguish user-created BOs from kernel BOs when deciding whether a faulty page can be safely purged or requires a full reset. Reviewed-by: Andi Shyti Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-22-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_bo.c | 8 +++++++- drivers/gpu/drm/xe/xe_bo.h | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index d30a8411d2a3..c255327669da 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -161,7 +161,13 @@ bool xe_bo_is_vm_bound(struct xe_bo *bo) return !list_empty(&bo->ttm.base.gpuva.list); } -static bool xe_bo_is_user(struct xe_bo *bo) +/** + * xe_bo_is_user - Check if BO is user-created + * @bo: The BO + * + * Returns: true if @bo was created by userspace + */ +bool xe_bo_is_user(struct xe_bo *bo) { return bo->flags & XE_BO_FLAG_USER; } diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 90b15fff36c7..eede678ad303 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -601,6 +601,7 @@ long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, const struct xe_bo_shrink_flags flags, unsigned long *scanned); int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx); +bool xe_bo_is_user(struct xe_bo *bo); /** * xe_bo_is_mem_type - Whether the bo currently resides in the given From f34e06068b0fa6708dcb958a99e651f60c770f99 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:45:59 +0530 Subject: [PATCH 13/76] drm/xe: Guard teardown paths against purged BOs VRAM page offlining can purge BOs that are still referenced by page tables, exec queues, and DMA-buf exports. Add xe_bo_is_purged() guards in the teardown paths to prevent unpinning or mapping an already-purged BO: - xe_bo_unpin_map_no_vm(): skip unpin if purged - xe_dma_buf_map(): return -ENOENT early if purged - xe_exec_queue_update_run_ticks(): skip LRC timestamp read if purged - xe_pt_destroy(): skip unpin if purged v4(Sashiko): - Move bo purge check in xe_bo_unpin() to handle all instances v3(Sashiko): - Remove dma_resv is already held v2(Himal): - take dma_resv lock before calling xe_bo_is_purged() Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-23-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_bo.c | 3 +++ drivers/gpu/drm/xe/xe_dma_buf.c | 3 +++ drivers/gpu/drm/xe/xe_exec_queue.c | 8 ++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index c255327669da..b162753cebb7 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -3267,6 +3267,9 @@ void xe_bo_unpin(struct xe_bo *bo) struct ttm_place *place = &bo->placements[0]; struct xe_device *xe = xe_bo_device(bo); + if (xe_bo_is_purged(bo)) + return; + xe_assert(xe, !bo->ttm.base.import_attach); xe_assert(xe, xe_bo_is_pinned(bo)); diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index bf0728838ead..5d9f1cd24b7f 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -104,6 +104,9 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach, struct sg_table *sgt; int r = 0; + if (xe_bo_is_purged(bo)) + return ERR_PTR(-ENOENT); + if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT)) return ERR_PTR(-EOPNOTSUPP); diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index f3ea4807ac33..e63559a2f582 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -1630,8 +1630,12 @@ void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) * errors. */ lrc = q->lrc[0]; - new_ts = xe_lrc_update_timestamp(lrc, &old_ts); - q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; + xe_bo_lock(lrc->bo, false); + if (!xe_bo_is_purged(lrc->bo)) { + new_ts = xe_lrc_update_timestamp(lrc, &old_ts); + q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; + } + xe_bo_unlock(lrc->bo); drm_dev_exit(idx); } From afc588a89015f55c52a3e83efa83253e79abab04 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:00 +0530 Subject: [PATCH 14/76] drm/xe/vram: Extract buddy allocation and free helpers Factor out xe_ttm_vram_buddy_alloc() and xe_ttm_vram_buddy_free() from xe_ttm_vram_mgr_new() and xe_ttm_vram_mgr_del(). These new helpers consolidate block allocation/deallocation with visible-size tracking. This modularization makes the logic reusable by the upcoming VRAM page offlining reservation path. No functional change. Reviewed-by: Andi Shyti Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-24-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 90 ++++++++++++++++++---------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 0f9cf1c3a576..a5116dc05166 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -49,6 +49,48 @@ static inline bool xe_is_vram_mgr_blocks_contiguous(struct gpu_buddy *mm, return true; } +static int xe_ttm_vram_buddy_alloc(struct xe_ttm_vram_mgr *mgr, u64 start, + u64 end, u64 size, u64 min_page_size, + struct list_head *blocks, unsigned long flags, + struct ttm_resource *res, u64 *used_visible) +{ + struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; + int err; + + err = gpu_buddy_alloc_blocks(mm, start, end, size, min_page_size, blocks, flags); + if (err) + return err; + + /* + * Track the owning resource, never the owning BO. A BO backpointer + * cached here goes stale the moment TTM hands the resource to a ghost + * object (ttm_buffer_object_transfer()), which happens on every + * accelerated move and on pipelined gutting. The resource, in + * contrast, has exactly the same lifetime as these blocks and TTM + * keeps &ttm_resource.bo pointing at the current owner for us. + */ + list_for_each_entry(block, blocks, link) + block->private = res; + + if (end <= mgr->visible_size) { + *used_visible = size; + } else { + list_for_each_entry(block, blocks, link) { + u64 blk_start = gpu_buddy_block_offset(block); + + if (blk_start < mgr->visible_size) { + u64 blk_end = blk_start + gpu_buddy_block_size(mm, block); + + *used_visible += min(blk_end, mgr->visible_size) - blk_start; + } + } + } + + mgr->visible_avail -= *used_visible; + return 0; +} + static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, struct ttm_buffer_object *tbo, const struct ttm_place *place, @@ -57,7 +99,6 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); struct xe_ttm_vram_mgr_resource *vres; struct gpu_buddy *mm = &mgr->mm; - struct gpu_buddy_block *block; u64 size, min_page_size; unsigned long lpfn; int err; @@ -118,32 +159,12 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, goto error_unlock; } - err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, - (u64)lpfn << PAGE_SHIFT, size, - min_page_size, &vres->blocks, vres->flags); + err = xe_ttm_vram_buddy_alloc(mgr, (u64)place->fpfn << PAGE_SHIFT, + (u64)lpfn << PAGE_SHIFT, size, + min_page_size, &vres->blocks, vres->flags, + &vres->base, &vres->used_visible_size); if (err) goto error_unlock; - - if (lpfn <= mgr->visible_size >> PAGE_SHIFT) { - vres->used_visible_size = size; - } else { - struct gpu_buddy_block *block; - - list_for_each_entry(block, &vres->blocks, link) { - u64 start = gpu_buddy_block_offset(block); - - if (start < mgr->visible_size) { - u64 end = start + gpu_buddy_block_size(mm, block); - - vres->used_visible_size += - min(end, mgr->visible_size) - start; - } - } - } - - mgr->visible_avail -= vres->used_visible_size; - list_for_each_entry(block, &vres->blocks, link) - block->private = &vres->base; mutex_unlock(&mgr->lock); if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) && @@ -175,20 +196,27 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, return err; } +static void xe_ttm_vram_buddy_free(struct xe_ttm_vram_mgr *mgr, + struct list_head *blocks, + u64 used_visible) +{ + struct gpu_buddy_block *block; + + list_for_each_entry(block, blocks, link) + block->private = NULL; + gpu_buddy_free_list(&mgr->mm, blocks, 0); + mgr->visible_avail += used_visible; +} + static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, struct ttm_resource *res) { struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); - struct gpu_buddy *mm = &mgr->mm; - struct gpu_buddy_block *block; mutex_lock(&mgr->lock); - list_for_each_entry(block, &vres->blocks, link) - block->private = NULL; - gpu_buddy_free_list(mm, &vres->blocks, 0); - mgr->visible_avail += vres->used_visible_size; + xe_ttm_vram_buddy_free(mgr, &vres->blocks, vres->used_visible_size); mutex_unlock(&mgr->lock); ttm_resource_fini(man, res); From 556774b11e1d87a4235ca87d6921cf557df1340a Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:01 +0530 Subject: [PATCH 15/76] drm/xe/vram: Add page offline data structures and lifecycle Add xe_ttm_vram_offline_resource to track individual offlined VRAM pages, and extend xe_ttm_vram_mgr with offlined_pages/queued_pages lists and their counters. Initialize the lists in __xe_ttm_vram_mgr_init() and add xe_ttm_vram_free_bad_pages() to release all tracked pages during xe_ttm_vram_mgr_fini() teardown. v3(Sashiko): - Reorder xe_ttm_vram_buddy_free and list_del_rcu - Introduce reservation status v2(Himal): - Address possible leak in xe_ttm_vram_mgr_fini() - Remove unused dev and add comment for used_visible_size 0 Reviewed-by: Andi Shyti Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-25-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 25 ++++++++++++++ drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h | 38 ++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index a5116dc05166..d97739a84a2d 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -346,12 +346,35 @@ static void xe_ttm_vram_mgr_set_unused(struct drm_device *dev, void *arg) ttm_resource_manager_set_used(man, false); } +static void xe_ttm_vram_free_bad_pages(struct xe_ttm_vram_mgr *mgr) +{ + struct xe_ttm_vram_offline_resource *pos, *n; + + list_for_each_entry_safe(pos, n, &mgr->offlined_pages, offlined_link) { + list_del_rcu(&pos->offlined_link); + xe_ttm_vram_buddy_free(mgr, &pos->blocks, pos->used_visible_size); + --mgr->n_offlined_pages; + kfree_rcu(pos, rcu); + } + list_for_each_entry_safe(pos, n, &mgr->queued_pages, queued_link) { + list_del_rcu(&pos->queued_link); + /* queued entries have no buddy reservation yet */ + xe_ttm_vram_buddy_free(mgr, &pos->blocks, 0); + --mgr->n_queued_pages; + kfree_rcu(pos, rcu); + } +} + static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg) { struct xe_device *xe = to_xe_device(dev); struct xe_ttm_vram_mgr *mgr = arg; struct ttm_resource_manager *man = &mgr->manager; + mutex_lock(&mgr->lock); + xe_ttm_vram_free_bad_pages(mgr); + mutex_unlock(&mgr->lock); + if (ttm_resource_manager_evict_all(&xe->ttm, man)) return; @@ -378,6 +401,8 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, err = drmm_mutex_init(&xe->drm, &mgr->lock); if (err) return err; + INIT_LIST_HEAD(&mgr->offlined_pages); + INIT_LIST_HEAD(&mgr->queued_pages); mgr->default_page_size = default_page_size; mgr->visible_size = io_size; mgr->visible_avail = io_size; diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h index 9106da056b49..dc97b0ad0e51 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h @@ -19,6 +19,14 @@ struct xe_ttm_vram_mgr { struct ttm_resource_manager manager; /** @mm: DRM buddy allocator which manages the VRAM */ struct gpu_buddy mm; + /** @offlined_pages: List of offlined pages */ + struct list_head offlined_pages; + /** @n_offlined_pages: Number of offlined pages */ + u16 n_offlined_pages; + /** @queued_pages: List of queued pages */ + struct list_head queued_pages; + /** @n_queued_pages: Number of queued pages */ + u16 n_queued_pages; /** @visible_size: Proped size of the CPU visible portion */ u64 visible_size; /** @visible_avail: CPU visible portion still unallocated */ @@ -45,4 +53,34 @@ struct xe_ttm_vram_mgr_resource { unsigned long flags; }; +/** + * enum xe_page_reserve_status - Buddy reservation status + * @XE_PAGE_RESERVE_PENDING: reservation in progress + * @XE_PAGE_RESERVE_FAIL: reservation failed + */ +enum xe_page_reserve_status { + XE_PAGE_RESERVE_PENDING = 0, + XE_PAGE_RESERVE_FAIL, +}; + +/** + * struct xe_ttm_vram_offline_resource - Tracks a single offlined VRAM page + */ +struct xe_ttm_vram_offline_resource { + /** @offlined_link: Link into mgr->offlined_pages */ + struct list_head offlined_link; + /** @queued_link: Link into mgr->queued_pages */ + struct list_head queued_link; + /** @blocks: Buddy blocks reserved for this page */ + struct list_head blocks; + /** @used_visible_size: CPU-visible bytes consumed */ + u64 used_visible_size; + /** @addr: Faulty DPA reported by HW */ + u64 addr; + /** @status: buddy reservation status */ + enum xe_page_reserve_status status; + /** @rcu: RCU head for deferred freeing */ + struct rcu_head rcu; +}; + #endif From 7737f819e9f1a0cbd653e1a58bda4be18e0306b8 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:02 +0530 Subject: [PATCH 16/76] drm/xe/vram: Add VRAM page offline fault handler Add the core VRAM page offlining logic to handle HW-reported faulty physical addresses: - xe_ttm_vram_purge_page(): Purges the BO containing the faulty address. Bans the associated VM (if page table BO) and exec queue (if LRC BO). Moves xe_exec_queue_kill() outside xe_bo_lock() to avoid AB-BA deadlock with vm->lock. Uses READ_ONCE(bo->q) to safely access the exec queue pointer. - xe_ttm_vram_page_already_processed(): Checks if an address is already tracked in offlined_pages or queued_pages lists to avoid double-processing. - xe_ttm_vram_reserve_page_at_addr(): Two-phase reservation that first queues the page, purges the BO outside the lock, then reserves the buddy block. Handles both allocated (BO present) and free page cases. Returns -EIO for critical kernel BOs to trigger system reset. - xe_ttm_vram_addr_to_region(): Maps a DPA to its corresponding VRAM region. Checks if the address falls within usable space, or infrastructure zones (CCS, GSM, DSM) where it returns NULL to flag a reset path. If the target address is outside any known region returns ERR_PTR(-EOPNOTSUPP) - xe_ttm_vram_handle_addr_fault(): Entry point called by RAS. Returns -EEXIST if already processed, -EIO for GSM/critical BO, -EOPNOTSUPP if out of bounds. v15: - Add immediate reservation for free pages - Use lock before accessing tbo v14(Sashiko/MattB): - Retry queued pages during xe_ttm_vram_mgr_del() - Move all changes to find owner of bo to its API v13(Himal): - Remove redundant code v12(Sashiko): - Handle multi tile and add assert for 4K align - Remove unaligned action comment for ENXIO v11(Himal): - match everywhere with enum vs bool for status member - Fix comment and remove unused var - if purge fail let next alloc confirm failure - pass absolute address, useful for multi tile Assisted-by: Copilot:claude-opus-4.6 Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-26-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 328 +++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_ttm_vram_mgr.h | 1 + 2 files changed, 329 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index d97739a84a2d..aea4cb0b66fd 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -13,10 +13,16 @@ #include #include +#include "regs/xe_regs.h" #include "xe_bo.h" +#include "xe_configfs.h" #include "xe_device.h" +#include "xe_exec_queue.h" +#include "xe_lrc.h" +#include "xe_mmio.h" #include "xe_pm.h" #include "xe_res_cursor.h" +#include "xe_ttm_stolen_mgr.h" #include "xe_ttm_vram_mgr.h" #include "xe_vram_types.h" @@ -208,6 +214,38 @@ static void xe_ttm_vram_buddy_free(struct xe_ttm_vram_mgr *mgr, mgr->visible_avail += used_visible; } +/* + * Retry pending page-offline reservations. + * + * A reservation can fail because the blocks backing the bad page are still + * allocated: either the owning BO could not be purged, or the purge was + * pipelined and TTM handed the resource to a ghost object which frees it + * only once the move fences signal. Rather than giving up, entries stay on + * @queued_pages and are retried here every time VRAM blocks come back. + * + * Called with @mgr->lock held. + */ +static void xe_ttm_vram_retry_queued_pages(struct xe_ttm_vram_mgr *mgr) +{ + struct xe_ttm_vram_offline_resource *pos, *n; + + lockdep_assert_held(&mgr->lock); + + list_for_each_entry_safe(pos, n, &mgr->queued_pages, queued_link) { + if (xe_ttm_vram_buddy_alloc(mgr, pos->addr, pos->addr + PAGE_SIZE, + PAGE_SIZE, PAGE_SIZE, &pos->blocks, + GPU_BUDDY_RANGE_ALLOCATION, NULL, + &pos->used_visible_size)) { + pos->status = XE_PAGE_RESERVE_FAIL; + continue; + } + --mgr->n_queued_pages; + list_del_rcu(&pos->queued_link); + ++mgr->n_offlined_pages; + list_add_rcu(&pos->offlined_link, &mgr->offlined_pages); + } +} + static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, struct ttm_resource *res) { @@ -217,6 +255,8 @@ static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, mutex_lock(&mgr->lock); xe_ttm_vram_buddy_free(mgr, &vres->blocks, vres->used_visible_size); + if (unlikely(!list_empty(&mgr->queued_pages))) + xe_ttm_vram_retry_queued_pages(mgr); mutex_unlock(&mgr->lock); ttm_resource_fini(man, res); @@ -580,3 +620,291 @@ u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man) return avail; } + +static int xe_ttm_vram_purge_page(struct xe_device *xe, struct xe_bo *bo) +{ + struct ttm_operation_ctx ctx = {}; + struct xe_exec_queue *q_to_put = NULL; + struct xe_exec_queue *q = NULL; + struct xe_vm *vm = NULL; + u32 flags; + int ret = 0; + + xe_bo_lock(bo, false); + if (bo->vm) + vm = xe_vm_get(bo->vm); + flags = bo->flags; + xe_bo_unlock(bo); + /* Ban VM if BO is PPGTT */ + if (vm && (flags & XE_BO_FLAG_PAGETABLE)) { + down_write(&vm->lock); + xe_vm_kill(vm, true); + up_write(&vm->lock); + } + if (vm) + xe_vm_put(vm); + + xe_bo_lock(bo, false); + q = READ_ONCE(bo->q); + /* Ban exec queue if BO is lrc */ + if (q && xe_exec_queue_get_unless_zero(q)) { + /* ban queue */ + q_to_put = q; + } + + if (bo->purgeable.state == XE_MADV_PURGEABLE_PURGED) { + /* Already purged by shrinker during unlocked window — nothing to do */ + xe_bo_unlock(bo); + goto out; + } + + xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_DONTNEED); + ttm_bo_unmap_virtual(&bo->ttm); /* nuke CPU mmap + VRAM IO mappings */ + if (xe_bo_is_pinned(bo)) + xe_bo_unpin(bo); + ret = xe_ttm_bo_purge(&bo->ttm, &ctx); + xe_bo_unlock(bo); + +out: + if (q_to_put) { + xe_exec_queue_kill(q_to_put); + xe_exec_queue_put(q_to_put); + } + + return ret; +} + +static bool xe_ttm_vram_page_already_processed(struct xe_ttm_vram_mgr *mgr, + u64 addr) +{ + struct xe_ttm_vram_offline_resource *pos; + + lockdep_assert_held(&mgr->lock); + + list_for_each_entry(pos, &mgr->offlined_pages, offlined_link) { + if (pos->addr == addr) + return true; + } + + list_for_each_entry(pos, &mgr->queued_pages, queued_link) { + if (pos->addr == addr) + return true; + } + + return false; +} + +/* + * Resolve the BO currently owning @block and take a reference on it. + * + * Called with @mgr->lock held, which serializes against + * xe_ttm_vram_buddy_free() clearing block->private. + * + * Returns NULL when there is no xe_bo we can act on: either the block is + * free, or the resource is temporarily owned by a TTM ghost object because + * a move or a pipelined gutting is still in flight. In both cases the + * blocks will hit xe_ttm_vram_mgr_del() on their own and the pending + * reservation is retried from there. + */ +static struct xe_bo *xe_ttm_vram_block_owner_get(struct xe_device *xe, + struct gpu_buddy_block *block) +{ + struct ttm_resource *res = block->private; + struct ttm_buffer_object *tbo; + struct xe_bo *bo; + + if (!res) + return NULL; + + guard(spinlock)(&xe->ttm.lru_lock); + + /* + * res->bo is updated under bdev->lru_lock by ttm_resource_set_bo(). + * Racing with a ghost transfer here is benign: we either see the old + * owner (whose purge is a no-op and the retry path recovers) or the + * ghost (rejected below). + * + * A ghost is a bare ttm_transfer_obj, not an xe_bo, so ttm_to_xe_bo() + * on one would be out of bounds. xe_bo_is_xe_bo() rejects it since + * only our own BOs carry xe_ttm_bo_destroy(). + */ + tbo = READ_ONCE(res->bo); + if (!tbo || !xe_bo_is_xe_bo(tbo)) + return NULL; + + bo = ttm_to_xe_bo(tbo); + + /* The BO may already be in teardown with a zero refcount */ + return xe_bo_get_unless_zero(bo) ? bo : NULL; +} + +static int xe_ttm_vram_reserve_page_at_addr(struct xe_device *xe, u64 addr, + struct xe_ttm_vram_mgr *vram_mgr, struct gpu_buddy *mm) +{ + struct xe_ttm_vram_offline_resource *nentry; + struct xe_bo *pbo_to_put = NULL; + struct xe_bo *pbo = NULL; + struct gpu_buddy_block *block; + u64 size = PAGE_SIZE; + int ret = 0; + + scoped_guard(mutex, &vram_mgr->lock) { + if (xe_ttm_vram_page_already_processed(vram_mgr, addr)) + return -EEXIST; + block = gpu_buddy_allocated_addr_to_block(mm, addr); + if (WARN_ON(IS_ERR(block))) + return PTR_ERR(block); + + nentry = kzalloc_obj(*nentry); + if (!nentry) + return -ENOMEM; + INIT_LIST_HEAD(&nentry->blocks); + nentry->status = XE_PAGE_RESERVE_PENDING; + nentry->addr = addr; + + if (block) { + pbo = xe_ttm_vram_block_owner_get(xe, block); + + /* + * Critical kernel BO? Best-effort check without resv lock; + * worst case a concurrent pin causes reset path unnecessarily. + */ + if (pbo && ((pbo->ttm.type == ttm_bo_type_kernel && + !(pbo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) || + (xe_bo_is_user(pbo) && xe_bo_is_pinned(pbo)))) { + kfree(nentry); + pbo_to_put = pbo; + drm_err(&xe->drm, + "%s: addr: 0x%llx is critical kernel bo, requesting SBR\n", + __func__, addr); + break; + } + /* Queue free(to-be-purged) pages */ + ++vram_mgr->n_queued_pages; + list_add_rcu(&nentry->queued_link, &vram_mgr->queued_pages); + } else { + /* Immediately offline unoccupied pages */ + /* Queue free(to-be-reserved) pages */ + ret = xe_ttm_vram_buddy_alloc(vram_mgr, addr, addr + size, + size, size, &nentry->blocks, + GPU_BUDDY_RANGE_ALLOCATION, + NULL, &nentry->used_visible_size); + if (ret) { + nentry->status = XE_PAGE_RESERVE_FAIL; + drm_dbg(&xe->drm, + "Page at addr:0x%llx still busy (%d), deferring reservation\n", + addr, ret); + ++vram_mgr->n_queued_pages; + list_add_rcu(&nentry->queued_link, &vram_mgr->queued_pages); + return 0; + } + ++vram_mgr->n_offlined_pages; + list_add_rcu(&nentry->offlined_link, &vram_mgr->offlined_pages); + return ret; + } + } + + /* Deferred put outside lock to avoid recursive deadlock */ + if (pbo_to_put) { + xe_bo_put(pbo_to_put); + /* Hint System controller driver for reset with -EIO */ + return -EIO; + } + + if (pbo) { + /* + * Purge BO containing address - reference held from above. + * This does not necessarily free the blocks synchronously: if + * the BO is not idle, ttm_bo_pipeline_gutting() hands the + * resource to a ghost object and it is released only once the + * move fences signal. The reservation below then fails and is + * retried from xe_ttm_vram_mgr_del(). + */ + ret = xe_ttm_vram_purge_page(xe, pbo); + xe_bo_put(pbo); + if (ret) + drm_warn(&xe->drm, "Purge failed at addr:0x%llx, ret:%d\n", addr, ret); + } + + return 0; +} + +static struct xe_vram_region *xe_ttm_vram_addr_to_region(struct xe_device *xe, u64 addr) +{ + struct xe_tile *tile; + u8 id; + + for_each_tile(tile, xe, id) { + struct xe_vram_region *vr = tile->mem.vram; + + if (!vr) + continue; + + if (addr >= vr->dpa_base && addr < (vr->dpa_base + vr->usable_size)) + return vr; + + /* CCS, GSM, or DSM — infrastructure zone, needs reset */ + if (addr >= (vr->dpa_base + vr->usable_size) && + addr < (vr->dpa_base + vr->actual_physical_size)) + return NULL; + } + + /* + * Return an explicit error pointer so the caller knows the addr + * is invalid and should be ignored, NOT SBR. + */ + return ERR_PTR(-EOPNOTSUPP); +} + +/** + * xe_ttm_vram_handle_addr_fault - Handle vram physical address error flaged + * @xe: pointer to parent device + * @addr: physical faulty address + * + * Handle the physcial faulty address error on specific tile. + * + * Returns 0 for success, negative error code otherwise as follow: + * * %-EIO - critical BO or address outside any VRAM region; next action is reset. + * * %-EOPNOTSUPP - log-only policy or unknown address; no further action. + * * %-ENOMEM - allocation failure; next action is reset. + * * %-ENXIO - address not found in buddy; no further action. + * * %-EEXIST - address already processed; no further action. + * + * A return of 0 means the page is tracked. It may still be listed as + * pending if the blocks backing it could not be freed immediately; the + * reservation is then completed from xe_ttm_vram_mgr_del(). + */ +int xe_ttm_vram_handle_addr_fault(struct xe_device *xe, u64 addr) +{ + struct xe_ttm_vram_mgr *vram_mgr; + struct xe_vram_region *vr; + struct gpu_buddy *mm; + + /* Assert that the address is PAGE_SIZE aligned */ + if (WARN_ON_ONCE(!IS_ALIGNED(addr, PAGE_SIZE))) { + drm_err(&xe->drm, "Address %llx is not %lu aligned!\n", addr, PAGE_SIZE); + return -EINVAL; + } + + vr = xe_ttm_vram_addr_to_region(xe, addr); + if (IS_ERR(vr)) { + /* + * The addr is outside VRAM and GSM. + * Log a debug message if needed, and safely exit/ignore. + */ + drm_dbg(&xe->drm, "Address %llx is out of bounds, ignoring fault.\n", addr); + return PTR_ERR(vr); + } + if (!vr) { + drm_err(&xe->drm, "%s:%d GSM addr:%llx error requesting SBR\n", + __func__, __LINE__, addr); + /* Hint System controller driver for reset with -EIO */ + return -EIO; + } + vram_mgr = &vr->ttm; + mm = &vram_mgr->mm; + + /* Reserve page at address */ + return xe_ttm_vram_reserve_page_at_addr(xe, addr - vr->dpa_base, vram_mgr, mm); +} +EXPORT_SYMBOL(xe_ttm_vram_handle_addr_fault); diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h index 87b7fae5edba..764d493b4d50 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h @@ -30,6 +30,7 @@ u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man); u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man); void xe_ttm_vram_get_used(struct ttm_resource_manager *man, u64 *used, u64 *used_visible); +int xe_ttm_vram_handle_addr_fault(struct xe_device *xe, u64 addr); static inline struct xe_ttm_vram_mgr_resource * to_xe_ttm_vram_mgr_resource(struct ttm_resource *res) From 12f8e9ed28bc5c7713d25b873b4da6b67e77c8fa Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:03 +0530 Subject: [PATCH 17/76] drm/xe/configfs: Add disable_vram_page_offline attribute Add a new configfs attribute 'disable_vram_page_offline' to control how bad VRAM pages are handled: 0, n, N, false - Do not disable (Offlining is active - default) 1, y, Y, true - Disable vram page offline (Logging only) The attribute can only be set before binding to the device and defaults to false (offlining enabled). This gives administrators control over whether corrupted VRAM pages detected by hardware (e.g., ECC errors) are actively offlined or only logged. v3(Sashiko): - fix doc v2(Sashiko): - keep default value in case configfs not present Reviewed-by: Himal Prasad Ghimiray Cc: Michal Wajdeczko Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-27-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_configfs.c | 72 +++++++++++++++++++++++++++++++- drivers/gpu/drm/xe/xe_configfs.h | 2 + 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 052cce962161..f5c828cf7e8f 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -61,7 +61,8 @@ * ├── survivability_mode * ├── gt_types_allowed * ├── engines_allowed - * └── enable_psmi + * ├── enable_psmi + * └── disable_vram_page_offline * * After configuring the attributes as per next section, the device can be * probed with:: @@ -159,6 +160,18 @@ * * This attribute can only be set before binding to the device. * + * Disable VRAM page offline: + * ---------------------------- + * + * 0, n, N, false - Do not disable (Offlining is active - default) + * 1, y, Y, true - Disable vram page offline (Logging only) + * + * Example to disable VRAM offline:: + * + * # echo 1 > /sys/kernel/config/xe/0000:03:00.0/disable_vram_page_offline + * + * This attribute can only be set on CRI before binding to the device. + * * Context restore BB * ------------------ * @@ -275,6 +288,7 @@ struct xe_config_group_device { bool survivability_mode; bool enable_psmi; bool enable_multi_queue; + bool disable_vram_page_offline; struct { unsigned int max_vfs; bool admin_only_pf; @@ -295,6 +309,7 @@ static const struct xe_config_device device_defaults = { .survivability_mode = false, .enable_psmi = false, .enable_multi_queue = true, + .disable_vram_page_offline = false, .sriov = { .max_vfs = XE_DEFAULT_MAX_VFS, .admin_only_pf = XE_DEFAULT_ADMIN_ONLY_PF, @@ -616,6 +631,33 @@ static ssize_t enable_multi_queue_store(struct config_item *item, const char *pa return len; } +static ssize_t disable_vram_page_offline_show(struct config_item *item, char *page) +{ + struct xe_config_device *dev = to_xe_config_device(item); + + return sprintf(page, "%s\n", str_yes_no(dev->disable_vram_page_offline)); +} + +static ssize_t disable_vram_page_offline_store(struct config_item *item, + const char *page, size_t len) +{ + struct xe_config_group_device *dev = to_xe_config_group_device(item); + bool val; + int ret; + + ret = kstrtobool(page, &val); + if (ret) + return ret; + + guard(mutex)(&dev->lock); + if (is_bound(dev)) + return -EBUSY; + + dev->config.disable_vram_page_offline = val; + + return len; +} + static bool wa_bb_read_advance(bool dereference, char **p, const char *append, size_t len, size_t *max_size) @@ -855,6 +897,7 @@ CONFIGFS_ATTR(, ctx_restore_mid_bb); CONFIGFS_ATTR(, ctx_restore_post_bb); CONFIGFS_ATTR(, enable_multi_queue); CONFIGFS_ATTR(, enable_psmi); +CONFIGFS_ATTR(, disable_vram_page_offline); CONFIGFS_ATTR(, engines_allowed); CONFIGFS_ATTR(, gt_types_allowed); CONFIGFS_ATTR(, survivability_mode); @@ -864,6 +907,7 @@ static struct configfs_attribute *xe_config_device_attrs[] = { &attr_ctx_restore_post_bb, &attr_enable_multi_queue, &attr_enable_psmi, + &attr_disable_vram_page_offline, &attr_engines_allowed, &attr_gt_types_allowed, &attr_survivability_mode, @@ -895,6 +939,11 @@ static bool xe_config_device_is_visible(struct config_item *item, return false; } + if (attr == &attr_disable_vram_page_offline) { + if (!dev->desc->is_dgfx || dev->desc->platform != XE_CRESCENTISLAND) + return false; + } + return true; } @@ -1142,6 +1191,7 @@ static void dump_custom_dev_config(struct pci_dev *pdev, PRI_CUSTOM_ATTR("%llx", engines_allowed); PRI_CUSTOM_ATTR("%d", enable_multi_queue); PRI_CUSTOM_ATTR("%d", enable_psmi); + PRI_CUSTOM_ATTR("%d", disable_vram_page_offline); PRI_CUSTOM_ATTR("%d", survivability_mode); PRI_CUSTOM_ATTR("%u", sriov.admin_only_pf); @@ -1290,6 +1340,26 @@ bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) return ret; } +/** + * xe_configfs_get_disable_vram_page_offline - get configfs disable_vram_page_offline setting + * @pdev: pci device + * + * Return: disable_vram_page_offline setting in configfs + */ +bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev) +{ + struct xe_config_group_device *dev = find_xe_config_group_device(pdev); + bool ret; + + if (!dev) + return device_defaults.disable_vram_page_offline; + + ret = dev->config.disable_vram_page_offline; + config_group_put(&dev->group); + + return ret; +} + /** * xe_configfs_get_ctx_restore_mid_bb - get configfs ctx_restore_mid_bb setting * @pdev: pci device diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index 4fbbeafba473..42cd1a491d01 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -24,6 +24,7 @@ bool xe_configfs_media_gt_allowed(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev); +bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev); u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class class, const u32 **cs); @@ -44,6 +45,7 @@ static inline bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) { return t static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } static inline bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) { return true; } +static inline bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev) { return false; } static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class class, const u32 **cs) { return 0; } From 586041a29187ac05a66118a53baeb9a757f9926e Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:04 +0530 Subject: [PATCH 18/76] drm/xe/ras: Cache disable_vram_page_offline policy at init The configfs disable_vram_page_offline attribute can only be set before device bind, so its value is immutable at runtime. Cache it in struct xe_drm_ras during xe_ras_init() to avoid repeated configfs lookups on every fault. Reviewed-by: Himal Prasad Ghimiray Cc: Michal Wajdeczko Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-28-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_drm_ras_types.h | 3 +++ drivers/gpu/drm/xe/xe_ras.c | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_drm_ras_types.h b/drivers/gpu/drm/xe/xe_drm_ras_types.h index 8d729ad6a264..0be218ba2db7 100644 --- a/drivers/gpu/drm/xe/xe_drm_ras_types.h +++ b/drivers/gpu/drm/xe/xe_drm_ras_types.h @@ -43,6 +43,9 @@ struct xe_drm_ras { /** @info: info array for all types of errors */ struct xe_drm_ras_counter *info[DRM_XE_RAS_ERR_SEV_MAX]; + + /** @disable_vram_page_offline: cached configfs policy, immutable after init */ + bool disable_vram_page_offline; }; #endif diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c index de4cb9ef7355..7a85735c57d5 100644 --- a/drivers/gpu/drm/xe/xe_ras.c +++ b/drivers/gpu/drm/xe/xe_ras.c @@ -3,6 +3,7 @@ * Copyright © 2026 Intel Corporation */ +#include "xe_configfs.h" #include "xe_debugfs.h" #include "xe_device.h" #include "xe_drm_ras.h" @@ -915,6 +916,14 @@ void xe_ras_init(struct xe_device *xe) { int ret; + /* + * TODO: Replace platform check with xe->info.has_disable_vram_page_offline + * once the feature flag is plumbed through device info. + */ + if (xe->info.platform == XE_CRESCENTISLAND) + xe->ras.disable_vram_page_offline = + xe_configfs_get_disable_vram_page_offline(to_pci_dev(xe->drm.dev)); + xe_drm_ras_init(xe); if (!xe->info.has_sysctrl) From 6ff99f4335f7ff5c210a2130438535c625f61f06 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:05 +0530 Subject: [PATCH 19/76] drm/xe/vram: Check disable_vram_page_offline policy in fault handler Before reserving a page at a faulting address, check the cached disable_vram_page_offline policy from xe->ras. If the policy is enabled (logging only), log the corrupted address and return -EOPNOTSUPP so that RAS can report to firmware to drop the address from the SRAM queue without attempting to offline the page. Reviewed-by: Himal Prasad Ghimiray Cc: Michal Wajdeczko Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-29-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index aea4cb0b66fd..f4810cb961df 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -21,6 +21,7 @@ #include "xe_lrc.h" #include "xe_mmio.h" #include "xe_pm.h" +#include "xe_printk.h" #include "xe_res_cursor.h" #include "xe_ttm_stolen_mgr.h" #include "xe_ttm_vram_mgr.h" @@ -904,6 +905,12 @@ int xe_ttm_vram_handle_addr_fault(struct xe_device *xe, u64 addr) vram_mgr = &vr->ttm; mm = &vram_mgr->mm; + if (xe->ras.disable_vram_page_offline) { + xe_err(xe, "0x%llx is reported as corrupted address by HW\n", + addr); + return -EOPNOTSUPP; + } + /* Reserve page at address */ return xe_ttm_vram_reserve_page_at_addr(xe, addr - vr->dpa_base, vram_mgr, mm); } From fb19798cc366eaacd68b6f59226120045e0b8f7e Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:06 +0530 Subject: [PATCH 20/76] drm/xe: Expose bad VRAM pages via debugfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a debugfs file "vram_bad_pages" that shows offlined and queued VRAM pages across all tiles. Each entry displays the page frame number, GPU page size, and status flag (R=reserved, P=pending, F=failed). example, cat /sys/kernel/debug/dri/0/vram_bad_pages max_pages: 10000 0x0000000000000000 : 0x0000000000001000 : R 0x0000000000001234 : 0x0000000000001000 : P 0x0000000000080000 : 0x0000000000001000 : R ← tile 1 addr v3(Michal): -add kernel doc v2(Sashikoi/Michal/Himal): -Remove block iteration, use offline and queue list only -Move platform check inside api Reviewed-by: Himal Prasad Ghimiray Reviewed-by: Rodrigo Vivi Cc: Michal Wajdeczko Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-30-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_debugfs.c | 3 + drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 71 ++++++++++++++++++++++ drivers/gpu/drm/xe/xe_ttm_vram_mgr.h | 1 + drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h | 2 + 4 files changed, 77 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index 28135f84e286..cb24e001b142 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -32,6 +32,7 @@ #include "xe_sriov_vf.h" #include "xe_step.h" #include "xe_tile_debugfs.h" +#include "xe_ttm_vram_mgr.h" #include "xe_vsec.h" #include "xe_wa.h" @@ -773,6 +774,8 @@ void xe_debugfs_register(struct xe_device *xe) if (man) ttm_resource_manager_create_debugfs(man, root, "stolen_mm"); + xe_ttm_vram_debugfs_init(xe, root); + for_each_tile(tile, xe, tile_id) xe_tile_debugfs_register(tile); diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index f4810cb961df..3dcee4590126 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -5,6 +5,7 @@ */ #include +#include #include #include @@ -915,3 +916,73 @@ int xe_ttm_vram_handle_addr_fault(struct xe_device *xe, u64 addr) return xe_ttm_vram_reserve_page_at_addr(xe, addr - vr->dpa_base, vram_mgr, mm); } EXPORT_SYMBOL(xe_ttm_vram_handle_addr_fault); + +static int vram_bad_pages_show(struct seq_file *m, void *unused) +{ + struct xe_device *xe = m->private; + struct xe_ttm_vram_offline_resource *pos; + struct ttm_resource_manager *man; + struct xe_ttm_vram_mgr *mgr; + struct xe_tile *tile; + u8 id; + + man = ttm_manager_type(&xe->ttm, XE_PL_VRAM0); + if (man) + /* TODO Hook with RAS to show max_pages fetched from FW */ + seq_printf(m, "max_pages: %d\n", + to_xe_ttm_vram_mgr(man)->max_pages); + + for_each_tile(tile, xe, id) { + struct xe_vram_region *vr = tile->mem.vram; + + man = ttm_manager_type(&xe->ttm, XE_PL_VRAM0 + id); + if (!man || !vr) + continue; + mgr = to_xe_ttm_vram_mgr(man); + + rcu_read_lock(); + + list_for_each_entry_rcu(pos, &mgr->offlined_pages, offlined_link) { + u64 pfn; + + pfn = (pos->addr + vr->dpa_base) >> PAGE_SHIFT; + seq_printf(m, "0x%016llx : 0x%016lx : R\n", pfn, PAGE_SIZE); + } + + list_for_each_entry_rcu(pos, &mgr->queued_pages, queued_link) { + u64 pfn; + + pfn = (pos->addr + vr->dpa_base) >> PAGE_SHIFT; + seq_printf(m, "0x%016llx : 0x%016lx : %c\n", + pfn, PAGE_SIZE, pos->status ? 'F' : 'P'); + } + + rcu_read_unlock(); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(vram_bad_pages); + +/** + * xe_ttm_vram_debugfs_init - Initialize VRAM debugfs interfaces + * @xe: The xe device structure pointer + * @root: The root dentry of the debugfs directory + * + * This function registers platform-specific VRAM debugfs files used for + * testing and debugging. Currently, it exposes the "vram_bad_pages" interface + * to inspect marked faulty memory pages, restricted specifically to the + * %XE_CRESCENTISLAND platform. + * + * Return: Void. + */ +void xe_ttm_vram_debugfs_init(struct xe_device *xe, struct dentry *root) +{ + /* + * TODO: Replace platform check with xe->info + * once the feature flag is plumbed through device info. + */ + if (xe->info.platform != XE_CRESCENTISLAND) + return; + debugfs_create_file("vram_bad_pages", 0444, root, xe, &vram_bad_pages_fops); +} diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h index 764d493b4d50..48f39fd20e44 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h @@ -17,6 +17,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, u32 mem_type, u64 size, u64 io_size, u64 default_page_size); int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram); +void xe_ttm_vram_debugfs_init(struct xe_device *xe, struct dentry *root); int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, struct ttm_resource *res, u64 offset, u64 length, diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h index dc97b0ad0e51..efcf3e1d4e80 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h @@ -37,6 +37,8 @@ struct xe_ttm_vram_mgr { struct mutex lock; /** @mem_type: The TTM memory type */ u32 mem_type; + /** @max_pages: max pages that can be in offline queue retrieved from FW */ + u16 max_pages; }; /** From 69183b7062061ce9985a6e116890ffb7929bd1f4 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:07 +0530 Subject: [PATCH 21/76] drm/xe/uapi: Expose ban reason in EXEC_QUEUE_GET_PROPERTY_BAN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN to return a bitmask indicating the reason for the ban, rather than a simple boolean. This allows userspace to distinguish between different ban causes: - DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG (bit 0): exec queue was banned due to a GPU hang or job timeout detected by the TDR. - DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE (bit 1): exec queue was banned because a VRAM page backing its resources was taken offline. The ban_reason field is added to struct xe_exec_queue and set at the point where the ban is triggered: - In guc_exec_queue_timedout_job() for GPU hang. - In xe_ttm_vram_purge_page() for memory page offline, before calling xe_exec_queue_kill() or xe_vm_kill(). The reset_status op is updated to return u64 with the reason bitmask. When a queue is banned but no explicit reason was recorded (e.g., from a generic CAT error), it defaults to GPU_HANG for backward compatibility. A value of 0 means the exec queue is not banned. v5 (Sashiko/MattB): - Take the write lock for the traversal to tag ban_reason v4(Sashiko): - Add ban reason for non-LR exec queues - Add TODO for multiqueue v3(Rodrigo): - Add doc in xe_drm.h v2(Sashiko): - Use atomic_t for ban_reason to fix concurrent updates from TDR and page-offline - Guard GPU_HANG bit with !exec_queue_killed to avoid masking page-offline reason - Clear ban_reason on queue recovery (clear_exec_queue_banned path) - Use atomic_read in guc_exec_queue_reset_status for lockless read Assisted-by: Copilot:claude-opus-4.6 Acked-by: José Roberto de Souza Acked-by: Michal Mrozek Reviewed-by: Rodrigo Vivi Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-31-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_exec_queue_types.h | 7 +++-- drivers/gpu/drm/xe/xe_execlist.c | 4 +-- drivers/gpu/drm/xe/xe_guc_submit.c | 36 ++++++++++++++++++++---- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 27 ++++++++++++++++++ include/uapi/drm/xe_drm.h | 18 +++++++++++- 5 files changed, 82 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 95f75d61a647..836f88fc0faa 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -154,6 +154,9 @@ struct xe_exec_queue { */ unsigned long flags; + /** @ban_reason: Bitmask of ban reasons (DRM_XE_EXEC_QUEUE_BAN_REASON_*) */ + atomic_t ban_reason; + union { /** @multi_gt_list: list head for VM bind engines if multi-GT */ struct list_head multi_gt_list; @@ -348,8 +351,8 @@ struct xe_exec_queue_ops { * signalled when this function is called. */ void (*resume)(struct xe_exec_queue *q); - /** @reset_status: check exec queue reset status */ - bool (*reset_status)(struct xe_exec_queue *q); + /** @reset_status: check exec queue ban status, returns ban reason bitmask */ + u64 (*reset_status)(struct xe_exec_queue *q); }; #endif diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c index 0d0db66c6ea2..a36db39dcda8 100644 --- a/drivers/gpu/drm/xe/xe_execlist.c +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -453,10 +453,10 @@ static void execlist_exec_queue_resume(struct xe_exec_queue *q) /* NIY */ } -static bool execlist_exec_queue_reset_status(struct xe_exec_queue *q) +static u64 execlist_exec_queue_reset_status(struct xe_exec_queue *q) { /* NIY */ - return false; + return 0; } static const struct xe_exec_queue_ops execlist_exec_queue_ops = { diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 99d8c807ff05..0a6e2b81b5a5 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -6,6 +6,7 @@ #include "xe_guc_submit.h" #include +#include #include #include #include @@ -1599,6 +1600,12 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) else wedged = xe_device_wedged(xe); + /* + * Only tag as GPU hang if this is the original timeout, not a + * consequence of a prior kill (e.g., page-offline). + */ + if (!exec_queue_killed(q)) + atomic_or(DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG, &q->ban_reason); set_exec_queue_banned(q); /* Kick job / queue off hardware */ @@ -1682,6 +1689,9 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) if (timeout_needs_gt_reset(q, job, skip_timeout_check)) { if (!xe_sched_invalidate_job(job, 2)) { clear_exec_queue_banned(q); + /* protect concurrent page offline reasons */ + atomic_andnot(DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG, + &q->ban_reason); xe_gt_reset_async(q->gt); goto rearm; } @@ -2580,13 +2590,29 @@ static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q) } } -static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) +static u64 guc_exec_queue_reset_status(struct xe_exec_queue *q) { - if (xe_exec_queue_is_multi_queue_secondary(q) && - guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q))) - return true; + /* TODO: In case of multiqueue, if a secondary queue is banned due to + * page offlining, checking only the primary queue's GuC reset status + * may mask the true reason or race with it. + */ + if (xe_exec_queue_is_multi_queue_secondary(q)) { + u64 status = guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q)); - return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); + if (status) + return status; + } + + if (exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q)) { + u64 reason = atomic_read_acquire(&q->ban_reason); + + /* If no specific reason was recorded, default to GPU hang */ + if (!reason) + reason = DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG; + return reason; + } + + return 0; } /* diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 3dcee4590126..6aa24eb064eb 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -625,6 +626,7 @@ u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man) static int xe_ttm_vram_purge_page(struct xe_device *xe, struct xe_bo *bo) { + u32 q_flag = DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE; struct ttm_operation_ctx ctx = {}; struct xe_exec_queue *q_to_put = NULL; struct xe_exec_queue *q = NULL; @@ -639,7 +641,30 @@ static int xe_ttm_vram_purge_page(struct xe_device *xe, struct xe_bo *bo) xe_bo_unlock(bo); /* Ban VM if BO is PPGTT */ if (vm && (flags & XE_BO_FLAG_PAGETABLE)) { + struct xe_exec_queue *eq; + int id; + down_write(&vm->lock); + if (xe->info.has_ctx_tlb_inval) { + /* + * Must be the write lock: send_tlb_inval_ctx_ppgtt() + * mutates this list (list_move_tail() onto an on-stack + * head) while holding only the read lock, relying on + * tlb_inval->seqno_lock to keep itself the sole + * mutator. Traversing it under down_read() would let + * this walk follow entries onto that stack list. + */ + down_write(&vm->exec_queues.lock); + for (id = 0; id < ARRAY_SIZE(vm->exec_queues.list); id++) + list_for_each_entry(eq, &vm->exec_queues.list[id], + vm_exec_queue_link) + atomic_or(q_flag, &eq->ban_reason); + up_write(&vm->exec_queues.lock); + } else { + list_for_each_entry(eq, &vm->preempt.exec_queues, lr.link) + atomic_or(q_flag, &eq->ban_reason); + } + smp_wmb(); /* Force all queue bits to be visible before killing the VM */ xe_vm_kill(vm, true); up_write(&vm->lock); } @@ -651,6 +676,8 @@ static int xe_ttm_vram_purge_page(struct xe_device *xe, struct xe_bo *bo) /* Ban exec queue if BO is lrc */ if (q && xe_exec_queue_get_unless_zero(q)) { /* ban queue */ + atomic_or(q_flag, &q->ban_reason); + smp_wmb(); /* Force bit change to finish before state change triggers */ q_to_put = q; } diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 8c852abcacf0..578f0c1eb43e 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1491,6 +1491,12 @@ struct drm_xe_exec_queue_destroy { * * The @property can be: * - %DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN + * + * For %DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN, @value is a bitmask of ban reasons: + * - %DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG - banned due to GPU hang/timeout + * - %DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE - banned due to memory page offline + * + * A @value of 0 means the exec queue is not banned. */ struct drm_xe_exec_queue_get_property { /** @extensions: Pointer to the first extension struct, if any */ @@ -1503,7 +1509,17 @@ struct drm_xe_exec_queue_get_property { /** @property: property to get */ __u32 property; - /** @value: property value */ + /** + * @value: property value + * + * For %DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN, this is a bitmask of: + * - %DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG - banned due to GPU hang/timeout + * - %DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE - banned due to memory page offline + * + * Value of 0 means the exec queue is not banned. + */ +#define DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG (1 << 0) +#define DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE (1 << 1) __u64 value; /** @reserved: Reserved */ From 6ef3eb10aea4260f752c06489a5369c2b67848b0 Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Thu, 3 Sep 2026 21:46:08 +0530 Subject: [PATCH 22/76] drm/xe: Add fault-inject based VRAM page offline injection Add a fault-inject based debugfs interface for testing VRAM page offlining. This replaces the previous standalone debugfs approach with the standard kernel fault-inject infrastructure. Two debugfs entries are created under the xe debugfs root for CRI platforms: - inject_mempage_offline/: Standard fault-inject knobs (probability, times, interval, etc.) created by fault_create_debugfs_attr(). Without CONFIG_FAULT_INJECTION_DEBUG_FS, the stub returns ERR_PTR(-ENODEV) and no knobs are created, making the trigger effectively a no-op. - inject_mempage_offline_trigger: Write a PFN value to inject a specific page, or write "0" to auto-pick the last unallocated VRAM page The trigger accepts: - "0" : auto-pick last unallocated page - "0xPFN" : inject fault at a specific PFN address Usage: echo 100 > inject_mempage_offline/probability echo 1 > inject_mempage_offline/times echo 0 > inject_mempage_offline_trigger probability: likelihood of should_fail() returning true (0-100) times: number of times injection is allowed (-1 for unlimited) v7: - Use PAGE_SIZE v6(Himal): - Add warning to rebind driver post test run v5(Sashiko): - exclude SRIOV and remove dpa_base addition, already absolute dpa v4(Himal): - Use xe_fault_mempage_offline() instead of IS_ENABLED() + direct should_fail(). CONFIG_FAULT_INJECTION_DEBUG_FS is now an implicit requirement for the trigger to function. v3(Himal): - Use FAULT_ACTION v2(sashiko): - use cond_resched() - validate input first and fix addr < 0 case - validate vr, move block, found var as local to scope_guard Reviewed-by: Himal Prasad Ghimiray Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260903161553.528932-32-tejas.upadhyay@intel.com --- drivers/gpu/drm/xe/xe_debugfs.c | 48 +++++++++++++++++++++++ drivers/gpu/drm/xe/xe_debugfs.h | 2 + drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 58 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_ttm_vram_mgr.h | 1 + 4 files changed, 109 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index cb24e001b142..80f62634fae5 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -45,12 +45,18 @@ DECLARE_FAULT_ATTR(gt_reset_failure); DECLARE_FAULT_ATTR(inject_csc_hw_error); DECLARE_FAULT_ATTR(wedge_cold_reset); +DECLARE_FAULT_ATTR(inject_mempage_offline); static bool csc_hw_error_available(struct xe_device *xe) { return !IS_SRIOV_VF(xe) && xe->info.platform == XE_BATTLEMAGE; } +static bool is_crescent_island_pf(struct xe_device *xe) +{ + return !IS_SRIOV_VF(xe) && xe->info.platform == XE_CRESCENTISLAND; +} + /* * Fault injection table. Each entry registers a debugfs attribute; add a * matching FAULT_ACTION() below for every entry added here. @@ -67,6 +73,9 @@ static struct { .is_visible = csc_hw_error_available }, { .name = "wedge_cold_reset", .attr = &wedge_cold_reset }, + { .name = "inject_mempage_offline", + .attr = &inject_mempage_offline, + .is_visible = is_crescent_island_pf }, }; /* @@ -82,6 +91,40 @@ bool xe_fault_##name(void) \ FAULT_ACTION(gt_reset, gt_reset_failure) FAULT_ACTION(csc_hw_error, inject_csc_hw_error) FAULT_ACTION(wedge_cold_reset, wedge_cold_reset) +FAULT_ACTION(mempage_offline, inject_mempage_offline) + +static ssize_t inject_mempage_offline_trigger(struct file *f, + const char __user *ubuf, + size_t size, loff_t *pos) +{ + struct xe_device *xe = file_inode(f)->i_private; + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_vram_region *vr = tile->mem.vram; + u64 pfn; + int ret; + + if (!vr) + return -ENODEV; + + ret = kstrtou64_from_user(ubuf, size, 0, &pfn); + if (ret) + return ret; + + if (!xe_fault_mempage_offline()) + return size; + + xe_warn(xe, "Page offlining test interface accessed. Notice: Offlined or reserved memory pages cannot be reclaimed dynamically. A driver rebind (unbind and bind loop) is required post-test to clean up.\n"); + if (pfn == 0) + return xe_ttm_vram_inject_fault(xe) ?: size; + + /* User provided PFN - convert to DPA and inject */ + return xe_ttm_vram_handle_addr_fault(xe, pfn << PAGE_SHIFT) ?: size; +} + +static const struct file_operations inject_mempage_offline_fops = { + .owner = THIS_MODULE, + .write = inject_mempage_offline_trigger, +}; static void xe_fault_inject_debugfs_register(struct xe_device *xe, struct dentry *root) @@ -96,6 +139,11 @@ static void xe_fault_inject_debugfs_register(struct xe_device *xe, fault_create_debugfs_attr(xe_fault_inject_entry[i].name, root, xe_fault_inject_entry[i].attr); } + + if (is_crescent_island_pf(xe)) { + debugfs_create_file("inject_mempage_offline_trigger", 0200, + root, xe, &inject_mempage_offline_fops); + } } static void read_residency_counter(struct xe_device *xe, struct xe_mmio *mmio, diff --git a/drivers/gpu/drm/xe/xe_debugfs.h b/drivers/gpu/drm/xe/xe_debugfs.h index 0dcd28fd7dc0..88d91c78036b 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.h +++ b/drivers/gpu/drm/xe/xe_debugfs.h @@ -14,11 +14,13 @@ struct xe_device; bool xe_fault_gt_reset(void); bool xe_fault_csc_hw_error(void); bool xe_fault_wedge_cold_reset(void); +bool xe_fault_mempage_offline(void); void xe_debugfs_register(struct xe_device *xe); #else static inline bool xe_fault_gt_reset(void) { return false; } static inline bool xe_fault_csc_hw_error(void) { return false; } static inline bool xe_fault_wedge_cold_reset(void) { return false; } +static inline bool xe_fault_mempage_offline(void) { return false; } static inline void xe_debugfs_register(struct xe_device *xe) { } #endif diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 6aa24eb064eb..9a514d983e90 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -944,6 +944,64 @@ int xe_ttm_vram_handle_addr_fault(struct xe_device *xe, u64 addr) } EXPORT_SYMBOL(xe_ttm_vram_handle_addr_fault); +/** + * xe_ttm_vram_inject_fault - Inject a VRAM page fault for testing + * @xe: xe device instance + * + * Picks the last unallocated VRAM page and reports it as faulted + * via xe_ttm_vram_handle_addr_fault(). Used by the fault-inject + * debugfs interface for testing page offlining. + * + * Note: Executing this test will permanently retire the allocated + * memory tracking pages. The driver must be rebinded (unbind and bind) + * post-test execution to reclaim the reserved space, as these pages + * cannot be freed or reclaimed dynamically while the current instance + * remains active. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_ttm_vram_inject_fault(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_vram_region *vr = tile->mem.vram; + struct xe_ttm_vram_mgr *vram_mgr = &vr->ttm; + struct gpu_buddy *mm = &vram_mgr->mm; + u64 addr; + + if (vr->actual_physical_size < PAGE_SIZE) + return -ENOSPC; + + addr = vr->actual_physical_size - PAGE_SIZE; + while (addr < vr->actual_physical_size) { + struct gpu_buddy_block *block; + bool found = false; + + scoped_guard(mutex, &vram_mgr->lock) { + block = gpu_buddy_allocated_addr_to_block(mm, addr); + if (!block) + found = true; + } + + /* + * Intentional race window: xe_ttm_vram_handle_addr_fault() + * re-acquires vram_mgr->lock internally, so we cannot hold + * it here. A concurrent allocation claiming this page between + * the two calls is an acceptable false negative for this + * test-only path. + */ + if (found) + return xe_ttm_vram_handle_addr_fault(xe, addr + vr->dpa_base); + + cond_resched(); + if (addr == 0) + break; + addr -= PAGE_SIZE; + } + + return -ENOSPC; +} +EXPORT_SYMBOL(xe_ttm_vram_inject_fault); + static int vram_bad_pages_show(struct seq_file *m, void *unused) { struct xe_device *xe = m->private; diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h index 48f39fd20e44..d77f067d197b 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h @@ -32,6 +32,7 @@ u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man); void xe_ttm_vram_get_used(struct ttm_resource_manager *man, u64 *used, u64 *used_visible); int xe_ttm_vram_handle_addr_fault(struct xe_device *xe, u64 addr); +int xe_ttm_vram_inject_fault(struct xe_device *xe); static inline struct xe_ttm_vram_mgr_resource * to_xe_ttm_vram_mgr_resource(struct ttm_resource *res) From fcc2431d2213dc4d04250c4f1ae87d9c3ae0d455 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Mon, 7 Sep 2026 10:30:12 +0530 Subject: [PATCH 23/76] drm/xe: Guard page-fault worker with runtime PM check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During VM teardown, the VM's runtime PM reference is dropped asynchronously, allowing the device to autosuspend while stale page faults belonging to the now-dead VM are still queued. When the page-fault worker later tries to ack one of these, it calls into guc_ct_send_locked() on an already-suspended device, tripping:   Assertion `!xe_pm_runtime_suspended(xe)` failed!   WARNING at xe_device.c:1267 xe_device_assert_mem_access+0x11c/0x140 [xe] A live VM/exec queue always holds a PM reference while it has outstanding work, so if the device is suspended at ack time, the owning context is already gone and the fault is stale. Take a runtime PM reference across the entire pagefault queue worker to safely deliver acks for torn-down VMs. v3: - Move PM ref to the generic xe_pagefault_queue_work using guard(xe_pm_runtime)(xe) instead of tracking it in the GuC backend(Matt Brost). v2: - Hold PM ref across the entire batch (begin/end) instead of per-ack. This prevents the device from autosuspending mid-batch, which would leave write_only acks written but the end flush skipped, and skip counter++, desyncing the cadence check.(Himal) - Add a comment explaining stale faults.(Himal) Fixes: f289f7807119 ("drm/xe: Add xe_guc_pagefault layer") Signed-off-by: Varun Gupta Reviewed-by: Matthew Brost Reviewed-by: Tejas Upadhyay Link: https://patch.msgid.link/20260907050011.497181-2-varun.gupta@intel.com Signed-off-by: Tejas Upadhyay --- drivers/gpu/drm/xe/xe_pagefault.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index e81ca24df37f..c82b8bc8bc70 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -17,6 +17,7 @@ #include "xe_log.h" #include "xe_pagefault.h" #include "xe_pagefault_types.h" +#include "xe_pm.h" #include "xe_svm.h" #include "xe_trace_bo.h" #include "xe_vm.h" @@ -610,6 +611,13 @@ static void xe_pagefault_queue_work(struct work_struct *w) u64 cache_start = XE_PAGEFAULT_CACHE_START_INVALID, cache_end = 0; u32 cache_asid = 0; + /* + * A live VM holds a PM reference, but a torn-down VM does not. + * Guard the entire worker loop to safely drain stale faults and + * prevent autosuspends from desyncing batched CT flushes. + */ + guard(xe_pm_runtime)(xe); + #define USM_QUEUE_MAX_RUNTIME_MS 20 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); From ca76137442edcafc089f19d189a55dc8d42ff7b8 Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Sat, 5 Sep 2026 21:31:37 +0800 Subject: [PATCH 24/76] drm/gpusvm: move dma_addr allocation before the notifier lock The dma_addr allocation was in a lazy allocation flow, it needs unlock and goto map_pages. The allocation only needs npages, so just do it before taking the lock. Drop the map_pages label and the relock flow, so the sequence becomes fault, allocate, then lock, validate, map and unlock. No functional change intended. Reviewed-by: Matthew Brost Signed-off-by: Honglei Huang Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260905133142.3628027-2-honghuan@amd.com --- drivers/gpu/drm/drm_gpusvm.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index a93eee7ddb9e..b507de539e67 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1516,10 +1516,18 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, if (err) goto err_free; + if (!svm_pages->dma_addr) { + svm_pages->dma_addr = + kvzalloc_objs(*svm_pages->dma_addr, npages); + if (!svm_pages->dma_addr) { + err = -ENOMEM; + goto err_free; + } + } + *state = (struct dma_iova_state){}; svm_pages->state_offset = 0; -map_pages: /* * Perform all dma mappings under the notifier lock to not * access freed pages. A notifier will either block on @@ -1540,18 +1548,6 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, goto retry; } - if (!svm_pages->dma_addr) { - /* Unlock and restart mapping to allocate memory. */ - drm_gpusvm_notifier_unlock(gpusvm); - svm_pages->dma_addr = - kvzalloc_objs(*svm_pages->dma_addr, npages); - if (!svm_pages->dma_addr) { - err = -ENOMEM; - goto err_free; - } - goto map_pages; - } - zdd = NULL; pagemap = NULL; num_dma_mapped = 0; From 1ae616edb68c22bb5c2378c4d23b923fd401e0c7 Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Sat, 5 Sep 2026 21:31:38 +0800 Subject: [PATCH 25/76] drm/gpusvm: extract drm_gpusvm_dma_map_pages() helper Move the per-device DMA mapping loop of drm_gpusvm_get_pages() into a helper drm_gpusvm_dma_map_pages(). The mapping logic is only moved, not changed, so there is no functional change. The helper maps the already-faulted pfns into one drm_gpusvm_pages instance under the notifier lock and unwinds its own partial mapping on error. The HMM fault and the notifier retry loop stay in get_pages() common code rather than being pushed down to drivers, so no driver has to reimplement the subtle fault and retry logic. With the mapping isolated per instance, get_pages() can later fault once and DMA map an array of drm_gpusvm_pages plus a count, one per owning drm_device. Suggested-by: Matthew Brost Reviewed-by: Matthew Brost Signed-off-by: Honglei Huang Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260905133142.3628027-3-honghuan@amd.com --- drivers/gpu/drm/drm_gpusvm.c | 225 ++++++++++++++++++++--------------- 1 file changed, 129 insertions(+), 96 deletions(-) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index b507de539e67..89c3061d8efb 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1442,115 +1442,41 @@ static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, } /** - * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct + * drm_gpusvm_dma_map_pages() - DMA map one drm_gpusvm_pages instance * @gpusvm: Pointer to the GPU SVM structure - * @svm_pages: The SVM pages to populate. This will contain the dma-addresses - * @mm: The mm corresponding to the CPU range - * @notifier: The corresponding notifier for the given CPU range - * @pages_start: Start CPU address for the pages - * @pages_end: End CPU address for the pages (exclusive) + * @svm_pages: The SVM pages instance to populate with dma-addresses + * @pfns: The already-faulted pfn array (size @npages) + * @npages: Number of pages in the CPU range * @ctx: GPU SVM context + * @dma_dir: DMA data direction for the mappings * - * This function gets and maps pages for CPU range and ensures they are - * mapped for DMA access. + * Map the faulted @pfns into @svm_pages for DMA access through its owning + * drm_device. Must be called under the notifier lock. On failure this unwinds + * the partial mapping of this instance before returning. * * Return: 0 on success, negative error code on failure. */ -int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, - struct drm_gpusvm_pages *svm_pages, - struct mm_struct *mm, - struct mmu_interval_notifier *notifier, - unsigned long pages_start, unsigned long pages_end, - const struct drm_gpusvm_ctx *ctx) +static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm, + struct drm_gpusvm_pages *svm_pages, + unsigned long *pfns, + unsigned long npages, + const struct drm_gpusvm_ctx *ctx, + enum dma_data_direction dma_dir) { - struct hmm_range hmm_range = { - .default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 : - HMM_PFN_REQ_WRITE), - .notifier = notifier, - .start = pages_start, - .end = pages_end, - .dev_private_owner = ctx->device_private_page_owner, - }; - void *zdd; - unsigned long timeout = - jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); - unsigned long remaining; + void *zdd = NULL; unsigned long i, j; - unsigned long npages = npages_in_range(pages_start, pages_end); - unsigned long num_dma_mapped; + unsigned long num_dma_mapped = 0; unsigned int order = 0; - unsigned long *pfns; int err = 0; - struct dev_pagemap *pagemap; + struct dev_pagemap *pagemap = NULL; struct drm_pagemap *dpagemap; struct drm_gpusvm_pages_flags flags; - enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE : - DMA_BIDIRECTIONAL; struct dma_iova_state *state = &svm_pages->state; - if (!svm_pages->drm) - return -EINVAL; - -retry: - remaining = timeout - jiffies; - - if (time_after_eq(jiffies, timeout)) - return -EBUSY; - - hmm_range.notifier_seq = mmu_interval_read_begin(notifier); - if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages)) - goto set_seqno; - - pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); - if (!pfns) - return -ENOMEM; - - if (!mmget_not_zero(mm)) { - err = -EFAULT; - goto err_free; - } - - hmm_range.hmm_pfns = pfns; - err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining); - mmput(mm); - if (err) - goto err_free; - - if (!svm_pages->dma_addr) { - svm_pages->dma_addr = - kvzalloc_objs(*svm_pages->dma_addr, npages); - if (!svm_pages->dma_addr) { - err = -ENOMEM; - goto err_free; - } - } - - *state = (struct dma_iova_state){}; - svm_pages->state_offset = 0; - - /* - * Perform all dma mappings under the notifier lock to not - * access freed pages. A notifier will either block on - * the notifier lock or unmap dma. - */ - drm_gpusvm_notifier_lock(gpusvm); + lockdep_assert_held(&gpusvm->notifier_lock); flags.__flags = svm_pages->flags.__flags; - if (flags.unmapped) { - drm_gpusvm_notifier_unlock(gpusvm); - err = -EFAULT; - goto err_free; - } - if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) { - drm_gpusvm_notifier_unlock(gpusvm); - kvfree(pfns); - goto retry; - } - - zdd = NULL; - pagemap = NULL; - num_dma_mapped = 0; for (i = 0, j = 0; i < npages; ++j) { struct page *page = hmm_pfn_to_page(pfns[i]); @@ -1666,17 +1592,124 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); + return 0; + +err_unmap: + svm_pages->flags.has_dma_mapping = true; + __drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped); + return err; +} + +/** + * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct + * @gpusvm: Pointer to the GPU SVM structure + * @svm_pages: The SVM pages to populate. This will contain the dma-addresses + * @mm: The mm corresponding to the CPU range + * @notifier: The corresponding notifier for the given CPU range + * @pages_start: Start CPU address for the pages + * @pages_end: End CPU address for the pages (exclusive) + * @ctx: GPU SVM context + * + * This function gets and maps pages for CPU range and ensures they are + * mapped for DMA access. + * + * Return: 0 on success, negative error code on failure. + */ +int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, + struct drm_gpusvm_pages *svm_pages, + struct mm_struct *mm, + struct mmu_interval_notifier *notifier, + unsigned long pages_start, unsigned long pages_end, + const struct drm_gpusvm_ctx *ctx) +{ + struct hmm_range hmm_range = { + .default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 : + HMM_PFN_REQ_WRITE), + .notifier = notifier, + .start = pages_start, + .end = pages_end, + .dev_private_owner = ctx->device_private_page_owner, + }; + unsigned long timeout = + jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + unsigned long remaining; + unsigned long npages = npages_in_range(pages_start, pages_end); + unsigned long *pfns; + int err = 0; + enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE : + DMA_BIDIRECTIONAL; + + if (!svm_pages->drm) + return -EINVAL; + +retry: + remaining = timeout - jiffies; + + if (time_after_eq(jiffies, timeout)) + return -EBUSY; + + hmm_range.notifier_seq = mmu_interval_read_begin(notifier); + if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages)) + goto set_seqno; + + pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); + if (!pfns) + return -ENOMEM; + + if (!mmget_not_zero(mm)) { + err = -EFAULT; + goto err_free; + } + + hmm_range.hmm_pfns = pfns; + err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining); + mmput(mm); + if (err) + goto err_free; + + if (!svm_pages->dma_addr) { + svm_pages->dma_addr = + kvzalloc_objs(*svm_pages->dma_addr, npages); + if (!svm_pages->dma_addr) { + err = -ENOMEM; + goto err_free; + } + } + + svm_pages->state = (struct dma_iova_state){}; + svm_pages->state_offset = 0; + + /* + * Perform all dma mappings under the notifier lock to not + * access freed pages. A notifier will either block on + * the notifier lock or unmap dma. + */ + drm_gpusvm_notifier_lock(gpusvm); + + if (svm_pages->flags.unmapped) { + drm_gpusvm_notifier_unlock(gpusvm); + err = -EFAULT; + goto err_free; + } + + if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) { + drm_gpusvm_notifier_unlock(gpusvm); + kvfree(pfns); + goto retry; + } + + err = drm_gpusvm_dma_map_pages(gpusvm, svm_pages, pfns, npages, ctx, + dma_dir); drm_gpusvm_notifier_unlock(gpusvm); + if (err) + goto err_free; + kvfree(pfns); set_seqno: svm_pages->notifier_seq = hmm_range.notifier_seq; return 0; -err_unmap: - svm_pages->flags.has_dma_mapping = true; - __drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped); - drm_gpusvm_notifier_unlock(gpusvm); err_free: kvfree(pfns); if (err == -EAGAIN) From cc2a6cbd6823678fe70ba76f88623827a6e06a87 Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Sat, 5 Sep 2026 21:31:39 +0800 Subject: [PATCH 26/76] drm/gpusvm: let drm_gpusvm_get_pages() map an array of pages With the N:1 drm_gpusvm_pages layout, one CPU range mirrored on several drm_devices, the caller had to invoke get_pages() once per device and repeat the HMM fault every time. Make get_pages() take a contiguous array of drm_gpusvm_pages plus a count: fault once, then DMA map each instance by drm_gpusvm_dma_map_pages() under a single read_retry gate. xe range and userptr callers are updated. Document the N:1 array usage in the Overview, showing how get_pages() and drm_gpusvm_range_set_unmapped() take the whole array and its count while the unmap and free paths stay per-instance. Suggested-by: Matthew Brost Reviewed-by: Matthew Brost Signed-off-by: Honglei Huang Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260905133142.3628027-4-honghuan@amd.com --- drivers/gpu/drm/drm_gpusvm.c | 140 ++++++++++++++++++++++++-------- drivers/gpu/drm/xe/xe_svm.c | 2 +- drivers/gpu/drm/xe/xe_userptr.c | 2 +- include/drm/drm_gpusvm.h | 1 + 4 files changed, 110 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 89c3061d8efb..d8ee0e5f2ae7 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -80,6 +80,13 @@ * }; * }; * + * static struct drm_gpusvm_pages * + * driver_pages(struct driver_range *drange) + * { + * return drange->num_pages == 1 ? &drange->inline_pages : + * drange->pages; + * } + * * In the N:1 case the driver allocates the pages array with a zeroing * allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with * drm_gpusvm_init_pages(), and frees each entry with @@ -89,6 +96,28 @@ * Each drm_gpusvm_pages must be zero-initialised and initialised with * drm_gpusvm_init_pages(), called once per entry. * + * The 1:1 examples below pass @num_pages == 1 and &drange->pages. In the + * N:1 case the driver instead passes the whole array and its count, so a + * single call faults the CPU range once and DMA maps it for every owning + * drm_device, e.g.: + * + * .. code-block:: c + * + * // GPU fault handler: one fault, one DMA mapping per device + * err = drm_gpusvm_get_pages(gpusvm, driver_pages(drange), + * drange->num_pages, gpusvm->mm, + * &range->notifier->notifier, + * drm_gpusvm_range_start(range), + * drm_gpusvm_range_end(range), &ctx); + * + * // Notifier callback: mark every instance unmapped in one call + * drm_gpusvm_range_set_unmapped(range, driver_pages(drange), + * drange->num_pages, mmu_range); + * + * The unmap and free paths stay per-instance: iterate @num_pages over + * driver_pages(drange) and call drm_gpusvm_unmap_pages() / + * drm_gpusvm_free_pages() for each entry. + * * - Operations: * Define the interface for driver-specific GPU SVM operations such as * range allocation, notifier allocation, and invalidations. @@ -232,7 +261,7 @@ * goto retry; * } * - * err = drm_gpusvm_get_pages(gpusvm, &drange->pages, + * err = drm_gpusvm_get_pages(gpusvm, &drange->pages, 1, * gpusvm->mm, &range->notifier->notifier, * drm_gpusvm_range_start(range), * drm_gpusvm_range_end(range), &ctx); @@ -1417,25 +1446,35 @@ EXPORT_SYMBOL_GPL(drm_gpusvm_pages_valid); /** * drm_gpusvm_pages_valid_unlocked() - GPU SVM pages valid unlocked * @gpusvm: Pointer to the GPU SVM structure - * @svm_pages: Pointer to the GPU SVM pages structure + * @svm_pages: Array of GPU SVM pages structures + * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages * - * This function determines if a GPU SVM pages are valid. Expected be called - * without holding gpusvm->notifier_lock. + * This function determines if every GPU SVM pages instance is valid, resetting + * every instance which is not so that get_pages() maps it afresh. It therefore + * has to walk them all. Expected be called without holding + * gpusvm->notifier_lock. * - * Return: True if GPU SVM pages are valid, False otherwise + * Return: True if all GPU SVM pages are valid, False otherwise */ static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, - struct drm_gpusvm_pages *svm_pages) + struct drm_gpusvm_pages *svm_pages, + unsigned int num_pages) { - bool pages_valid; + bool pages_valid = true; + unsigned int p; - if (!svm_pages->dma_addr) - return false; + for (p = 0; p < num_pages; ++p) { + if (!svm_pages[p].dma_addr) + return false; + } drm_gpusvm_notifier_lock(gpusvm); - pages_valid = drm_gpusvm_pages_valid(gpusvm, svm_pages); - if (!pages_valid) - __drm_gpusvm_free_pages(gpusvm, svm_pages); + for (p = 0; p < num_pages; ++p) { + if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) + continue; + __drm_gpusvm_free_pages(gpusvm, &svm_pages[p]); + pages_valid = false; + } drm_gpusvm_notifier_unlock(gpusvm); return pages_valid; @@ -1451,8 +1490,9 @@ static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, * @dma_dir: DMA data direction for the mappings * * Map the faulted @pfns into @svm_pages for DMA access through its owning - * drm_device. Must be called under the notifier lock. On failure this unwinds - * the partial mapping of this instance before returning. + * drm_device. Must be called under the notifier lock and only for an instance + * without a live mapping. On failure this unwinds the partial mapping of this + * instance before returning. * * Return: 0 on success, negative error code on failure. */ @@ -1475,6 +1515,9 @@ static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm, lockdep_assert_held(&gpusvm->notifier_lock); + *state = (struct dma_iova_state){}; + svm_pages->state_offset = 0; + flags.__flags = svm_pages->flags.__flags; for (i = 0, j = 0; i < npages; ++j) { @@ -1603,20 +1646,28 @@ static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm, /** * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct * @gpusvm: Pointer to the GPU SVM structure - * @svm_pages: The SVM pages to populate. This will contain the dma-addresses + * @svm_pages: Array of SVM pages instances to populate with dma addresses + * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages, must not be 0 * @mm: The mm corresponding to the CPU range * @notifier: The corresponding notifier for the given CPU range * @pages_start: Start CPU address for the pages * @pages_end: End CPU address for the pages (exclusive) * @ctx: GPU SVM context * - * This function gets and maps pages for CPU range and ensures they are - * mapped for DMA access. + * This function gets and maps pages for a CPU range and ensures they are + * mapped for DMA access. The HMM fault for the CPU range is performed once, + * the DMA mapping by drm_gpusvm_dma_map_pages() is then done per instance, + * one per owning drm_device. The retry against notifier races is kept here + * in common code so drivers never open code it. + * + * On error the instances mapped before the failing one stay mapped, so the + * caller must unmap and free every instance regardless of the return value. * * Return: 0 on success, negative error code on failure. */ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, struct drm_gpusvm_pages *svm_pages, + unsigned int num_pages, struct mm_struct *mm, struct mmu_interval_notifier *notifier, unsigned long pages_start, unsigned long pages_end, @@ -1638,10 +1689,15 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, int err = 0; enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + unsigned int p; - if (!svm_pages->drm) + if (!num_pages) return -EINVAL; + for (p = 0; p < num_pages; ++p) + if (!svm_pages[p].drm) + return -EINVAL; + retry: remaining = timeout - jiffies; @@ -1649,7 +1705,8 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, return -EBUSY; hmm_range.notifier_seq = mmu_interval_read_begin(notifier); - if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages)) + + if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages, num_pages)) goto set_seqno; pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); @@ -1667,18 +1724,17 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, if (err) goto err_free; - if (!svm_pages->dma_addr) { - svm_pages->dma_addr = - kvzalloc_objs(*svm_pages->dma_addr, npages); - if (!svm_pages->dma_addr) { + for (p = 0; p < num_pages; ++p) { + if (svm_pages[p].dma_addr) + continue; + svm_pages[p].dma_addr = + kvzalloc_objs(*svm_pages[p].dma_addr, npages); + if (!svm_pages[p].dma_addr) { err = -ENOMEM; goto err_free; } } - svm_pages->state = (struct dma_iova_state){}; - svm_pages->state_offset = 0; - /* * Perform all dma mappings under the notifier lock to not * access freed pages. A notifier will either block on @@ -1686,7 +1742,11 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, */ drm_gpusvm_notifier_lock(gpusvm); - if (svm_pages->flags.unmapped) { + /* + * drm_gpusvm_range_set_unmapped() flags the whole array in one go under + * the write lock, so any instance answers for all of them here. + */ + if (svm_pages[0].flags.unmapped) { drm_gpusvm_notifier_unlock(gpusvm); err = -EFAULT; goto err_free; @@ -1698,15 +1758,29 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, goto retry; } - err = drm_gpusvm_dma_map_pages(gpusvm, svm_pages, pfns, npages, ctx, - dma_dir); - drm_gpusvm_notifier_unlock(gpusvm); - if (err) - goto err_free; + for (p = 0; p < num_pages; ++p) { + if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) + continue; + err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns, + npages, ctx, dma_dir); + if (err) { + /* + * The failing instance was unwound by the helper. Keep + * the ones mapped earlier: the -EAGAIN retry reuses + * them, and the driver unmaps every instance with the + * range on the other error paths. + */ + drm_gpusvm_notifier_unlock(gpusvm); + goto err_free; + } + } + + drm_gpusvm_notifier_unlock(gpusvm); kvfree(pfns); set_seqno: - svm_pages->notifier_seq = hmm_range.notifier_seq; + for (p = 0; p < num_pages; ++p) + svm_pages[p].notifier_seq = hmm_range.notifier_seq; return 0; diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 627a741293d5..1c7793d8caa8 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -1598,7 +1598,7 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, lockdep_assert_held(&range->lock); - err = drm_gpusvm_get_pages(&vm->svm.gpusvm, &range->pages, + err = drm_gpusvm_get_pages(&vm->svm.gpusvm, &range->pages, 1, vm->svm.gpusvm.mm, &range->base.notifier->notifier, drm_gpusvm_range_start(&range->base), diff --git a/drivers/gpu/drm/xe/xe_userptr.c b/drivers/gpu/drm/xe/xe_userptr.c index 90ac141fc12d..9c1dac0fce6f 100644 --- a/drivers/gpu/drm/xe/xe_userptr.c +++ b/drivers/gpu/drm/xe/xe_userptr.c @@ -91,7 +91,7 @@ int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma) if (vma->gpuva.flags & XE_VMA_DESTROYED) return 0; - return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages, + return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 1, uvma->userptr.notifier.mm, &uvma->userptr.notifier, xe_vma_userptr(vma), diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h index b7d987bf76aa..d2b6f3d2b842 100644 --- a/include/drm/drm_gpusvm.h +++ b/include/drm/drm_gpusvm.h @@ -324,6 +324,7 @@ void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range, int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, struct drm_gpusvm_pages *svm_pages, + unsigned int num_pages, struct mm_struct *mm, struct mmu_interval_notifier *notifier, unsigned long pages_start, unsigned long pages_end, From 22a81666c2dc098e1ee9e294a13ccdeed5fc248d Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Sat, 5 Sep 2026 21:31:40 +0800 Subject: [PATCH 27/76] drm/gpusvm: make the DMA mapping step in get_pages() optional Some drivers (e.g. AMDXDNA) only need the CPU pages faulted in and tracked by the notifier, no need DMA mapping. Add a drm_gpusvm_ctx::no_dma_map flag. When set, get_pages() does the shared HMM fault and records notifier_seq, but skips svm_pages->drm validation, the dma_addr allocation and drm_gpusvm_dma_map_pages(). With no mapping state to check, the fault is redone on every call. The default (no_dma_map == 0) is unchanged. Suggested-by: Matthew Brost Reviewed-by: Matthew Brost Signed-off-by: Honglei Huang Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260905133142.3628027-5-honghuan@amd.com --- drivers/gpu/drm/drm_gpusvm.c | 44 ++++++++++++++++++++++++++---------- include/drm/drm_gpusvm.h | 9 ++++++++ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index d8ee0e5f2ae7..7efc35507f13 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1663,6 +1663,12 @@ static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm, * On error the instances mapped before the failing one stay mapped, so the * caller must unmap and free every instance regardless of the return value. * + * With &drm_gpusvm_ctx.no_dma_map no mapping state is recorded, so + * drm_gpusvm_pages_valid() never returns true and success is only a snapshot: + * the caller must recheck mmu_interval_read_retry() against the recorded + * &drm_gpusvm_pages.notifier_seq under the notifier lock, and hold it until + * its work is visible to invalidation. + * * Return: 0 on success, negative error code on failure. */ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, @@ -1689,14 +1695,21 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, int err = 0; enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + const bool map_dma = !ctx->no_dma_map; unsigned int p; if (!num_pages) return -EINVAL; - for (p = 0; p < num_pages; ++p) - if (!svm_pages[p].drm) - return -EINVAL; + if (ctx->no_dma_map && ctx->devmem_only) + return -EINVAL; + + if (map_dma) { + for (p = 0; p < num_pages; ++p) { + if (!svm_pages[p].drm) + return -EINVAL; + } + } retry: remaining = timeout - jiffies; @@ -1706,7 +1719,8 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, hmm_range.notifier_seq = mmu_interval_read_begin(notifier); - if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages, num_pages)) + if (map_dma && + drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages, num_pages)) goto set_seqno; pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); @@ -1724,14 +1738,16 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, if (err) goto err_free; - for (p = 0; p < num_pages; ++p) { - if (svm_pages[p].dma_addr) - continue; - svm_pages[p].dma_addr = - kvzalloc_objs(*svm_pages[p].dma_addr, npages); - if (!svm_pages[p].dma_addr) { - err = -ENOMEM; - goto err_free; + if (map_dma) { + for (p = 0; p < num_pages; ++p) { + if (svm_pages[p].dma_addr) + continue; + svm_pages[p].dma_addr = + kvzalloc_objs(*svm_pages[p].dma_addr, npages); + if (!svm_pages[p].dma_addr) { + err = -ENOMEM; + goto err_free; + } } } @@ -1758,6 +1774,9 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, goto retry; } + if (!map_dma) + goto done_mapping; + for (p = 0; p < num_pages; ++p) { if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) continue; @@ -1776,6 +1795,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, } } +done_mapping: drm_gpusvm_notifier_unlock(gpusvm); kvfree(pfns); set_seqno: diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h index d2b6f3d2b842..ec7b81957b1d 100644 --- a/include/drm/drm_gpusvm.h +++ b/include/drm/drm_gpusvm.h @@ -254,6 +254,14 @@ struct drm_gpusvm { * @allow_mixed: Allow mixed mappings in get pages. Mixing between system and * single dpagemap is supported, mixing between multiple dpagemap * is unsupported. + * @no_dma_map: Only fault the CPU pages for the range; skip the device DMA + * mapping step. Used by drivers that consume the faulted pages + * without needing a DMA mapping. In this mode @drm on the + * drm_gpusvm_pages is not required, no mapping state is recorded + * and drm_gpusvm_pages_valid() therefore never reports these + * pages as valid; the caller revalidates the snapshot itself, see + * drm_gpusvm_get_pages(). @devmem_only is rejected and no page + * type check is performed, so @allow_mixed has no effect. * * Context that is DRM GPUSVM is operating in (i.e. user arguments). */ @@ -266,6 +274,7 @@ struct drm_gpusvm_ctx { unsigned int devmem_possible :1; unsigned int devmem_only :1; unsigned int allow_mixed :1; + unsigned int no_dma_map :1; }; int drm_gpusvm_init(struct drm_gpusvm *gpusvm, From 8d242e452ed717055cae14f5d9c3cfb13b4a9c31 Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Sat, 5 Sep 2026 21:31:41 +0800 Subject: [PATCH 28/76] drm/gpusvm: keep a single DMA mapping inline for THP drm_gpusvm_get_pages() sizes the dma_addr array for one drm_pagemap_addr per page, but the mapping loop advances by page order, so a range backed by one huge page needs a single entry. For a 2 MiB THP that is an 8 KiB array holding 16 bytes of address. Union that entry with the array pointer, discriminated by a new inline_dma_mapping flag. When drm_gpusvm_dma_map_pages() ends up with one entry it stores it inline and frees the array, after the last error unwind, which still walks the array form. An unchecked dma_addr read is now type confusion rather than a compile error, so reads go through the new drm_gpusvm_pages_first_dma() accessor, including the two xe_pt_stage_bind() paths. Only get_pages() and the free path write the union, never the notifier, and both run under the driver lock that every address reader already holds. The unlocked short circuit in drm_gpusvm_pages_valid_unlocked() goes for the same reason: it cannot resolve the union, and every instance it rejects has to be reset before the allocation loop reuses it. Suggested-by: Matthew Brost Signed-off-by: Honglei Huang Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260905133142.3628027-6-honghuan@amd.com --- drivers/gpu/drm/drm_gpusvm.c | 48 ++++++++++++++++++++++++++++------ drivers/gpu/drm/xe/xe_pt.c | 7 ++--- drivers/gpu/drm/xe/xe_svm.h | 18 +++++++++++++ include/drm/drm_gpusvm.h | 50 +++++++++++++++++++++++++++++++++--- 4 files changed, 109 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 7efc35507f13..2c7c4c89dc4a 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1241,6 +1241,8 @@ static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, struct drm_gpusvm_pages_flags flags = { .__flags = svm_pages->flags.__flags, }; + const struct drm_pagemap_addr *addrs = + drm_gpusvm_pages_first_dma(svm_pages); bool use_iova = dma_use_iova(&svm_pages->state); /* @@ -1253,12 +1255,12 @@ static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, if (svm_pages->state_offset) dma_iova_unlink(dev, &svm_pages->state, 0, svm_pages->state_offset, - svm_pages->dma_addr[0].dir, 0); + addrs[0].dir, 0); dma_iova_free(dev, &svm_pages->state); } for (i = 0, j = 0; i < npages; j++) { - struct drm_pagemap_addr *addr = &svm_pages->dma_addr[j]; + const struct drm_pagemap_addr *addr = &addrs[j]; if (addr->proto == DRM_INTERCONNECT_SYSTEM) { /* @@ -1299,6 +1301,18 @@ static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm, { lockdep_assert_held(&gpusvm->notifier_lock); + if (svm_pages->flags.inline_dma_mapping) { + struct drm_gpusvm_pages_flags flags = { + .__flags = svm_pages->flags.__flags, + }; + + svm_pages->inline_addr = (struct drm_pagemap_addr){}; + flags.inline_dma_mapping = false; + /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ + WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); + return; + } + if (svm_pages->dma_addr) { kvfree(svm_pages->dma_addr); svm_pages->dma_addr = NULL; @@ -1463,11 +1477,6 @@ static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, bool pages_valid = true; unsigned int p; - for (p = 0; p < num_pages; ++p) { - if (!svm_pages[p].dma_addr) - return false; - } - drm_gpusvm_notifier_lock(gpusvm); for (p = 0; p < num_pages; ++p) { if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) @@ -1480,6 +1489,21 @@ static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, return pages_valid; } +/** + * drm_gpusvm_pages_inlinable() - Whether the dma address can be inlined + * @nentries: Number of entries the mapping loop produced + * + * A THP maps as one huge page, so the whole range needs a single device + * address: the dma_addr array can be freed and the address kept inline, + * which is where the memory saving comes from. + * + * Return: True if the mapping fits in a single drm_pagemap_addr. + */ +static bool drm_gpusvm_pages_inlinable(unsigned long nentries) +{ + return nentries == 1; +} + /** * drm_gpusvm_dma_map_pages() - DMA map one drm_gpusvm_pages instance * @gpusvm: Pointer to the GPU SVM structure @@ -1632,6 +1656,14 @@ static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm, if (pagemap) flags.has_devmem_pages = true; + if (drm_gpusvm_pages_inlinable(j)) { + struct drm_pagemap_addr addr = svm_pages->dma_addr[0]; + + kvfree(svm_pages->dma_addr); + svm_pages->inline_addr = addr; + flags.inline_dma_mapping = true; + } + /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); @@ -1740,7 +1772,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, if (map_dma) { for (p = 0; p < num_pages; ++p) { - if (svm_pages[p].dma_addr) + if (drm_gpusvm_pages_first_dma(&svm_pages[p])) continue; svm_pages[p].dma_addr = kvzalloc_objs(*svm_pages[p].dma_addr, npages); diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 5d990c1c3740..fa4b29da0b6c 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -831,7 +831,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, return -EAGAIN; } if (xe_svm_range_has_dma_mapping(range)) { - xe_res_first_dma(range->pages.dma_addr, 0, + xe_res_first_dma(xe_svm_range_first_dma(range), 0, xe_svm_range_size(range), &curs); xe_svm_range_debug(range, "BIND PREPARE - MIXED"); @@ -866,8 +866,9 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, if (!xe_vma_is_null(vma) && !range && !is_purged) { if (xe_vma_is_userptr(vma)) - xe_res_first_dma(to_userptr_vma(vma)->userptr.pages.dma_addr, 0, - xe_vma_size(vma), &curs); + xe_res_first_dma(drm_gpusvm_pages_first_dma + (&to_userptr_vma(vma)->userptr.pages), + 0, xe_vma_size(vma), &curs); else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), xe_vma_size(vma), &curs); diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index 2a0dc0d125c9..7eb80d4d6db4 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -220,6 +220,18 @@ static inline unsigned long xe_svm_range_size(struct xe_svm_range *range) return drm_gpusvm_range_size(&range->base); } +/** + * xe_svm_range_first_dma() - Resolve the device address array of a SVM range + * @range: SVM range + * + * Return: Pointer to the first device address, NULL if none is populated. + */ +static inline const struct drm_pagemap_addr * +xe_svm_range_first_dma(struct xe_svm_range *range) +{ + return drm_gpusvm_pages_first_dma(&range->pages); +} + void xe_svm_flush(struct xe_vm *vm); int xe_pagemap_shrinker_create(struct xe_device *xe); @@ -436,6 +448,12 @@ static inline bool xe_svm_range_is_removed(struct xe_svm_range *range) return false; } +static inline const struct drm_pagemap_addr * +xe_svm_range_first_dma(struct xe_svm_range *range) +{ + return NULL; +} + #define xe_svm_range_has_dma_mapping(...) false #endif /* CONFIG_DRM_XE_GPUSVM */ diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h index ec7b81957b1d..aaad5c9b510c 100644 --- a/include/drm/drm_gpusvm.h +++ b/include/drm/drm_gpusvm.h @@ -10,6 +10,7 @@ #include #include #include +#include struct dev_pagemap_ops; struct drm_device; @@ -18,7 +19,6 @@ struct drm_gpusvm_notifier; struct drm_gpusvm_ops; struct drm_gpusvm_range; struct drm_pagemap; -struct drm_pagemap_addr; /** * struct drm_gpusvm_ops - Operations structure for GPU SVM @@ -112,6 +112,7 @@ struct drm_gpusvm_notifier { * @unmapped: Flag indicating if the pages has been unmapped * @has_devmem_pages: Flag indicating if the pages has devmem pages * @has_dma_mapping: Flag indicating if the pages has a DMA mapping + * @inline_dma_mapping: Flag indicating if the pages have an inline DMA mapping * @__flags: Flags for pages in u16 form (used for READ_ONCE) */ struct drm_gpusvm_pages_flags { @@ -121,6 +122,7 @@ struct drm_gpusvm_pages_flags { u16 unmapped : 1; u16 has_devmem_pages : 1; u16 has_dma_mapping : 1; + u16 inline_dma_mapping : 1; }; u16 __flags; }; @@ -130,17 +132,27 @@ struct drm_gpusvm_pages_flags { * struct drm_gpusvm_pages - Structure representing a GPU SVM mapped pages * * @drm: The DRM device that owns the dma mappings - * @dma_addr: Device address array + * @dma_addr: Device address array, valid while @flags.inline_dma_mapping is + * not set + * @inline_addr: Device address inline address, valid while + * @flags.inline_dma_mapping is set * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping. * Note this is assuming only one drm_pagemap per range is allowed. * @state: DMA IOVA state for mapping. * @state_offset: DMA IOVA offset for mapping. * @notifier_seq: Notifier sequence number of the range's pages * @flags: Flags for the range; see &struct drm_gpusvm_pages_flags + * + * @dma_addr and @inline_addr share storage, discriminated by + * @flags.inline_dma_mapping. Driver should use drm_gpusvm_pages_first_dma() + * to access the correct DMA address. */ struct drm_gpusvm_pages { struct drm_device *drm; - struct drm_pagemap_addr *dma_addr; + union { + struct drm_pagemap_addr *dma_addr; + struct drm_pagemap_addr inline_addr; + }; struct drm_pagemap *dpagemap; struct dma_iova_state state; unsigned long state_offset; @@ -365,6 +377,38 @@ static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages, svm_pages->notifier_seq = LONG_MAX; } +/** + * drm_gpusvm_pages_first_dma() - Resolve the device address array + * @svm_pages: Pointer to the drm_gpusvm_pages. + * + * drm_gpusvm_pages use unions to optimize the storage of DMA addresses, + * this function abstracts the access to the first device address. The driver + * should use this helper instead of reading dma_addr directly to prevent + * array out of bounds access. + * + * Only get_pages() and the free path switch between the two union members. + * Both hold the notifier lock for read, so taking that lock does not stop + * them; callers need the driver lock that does, which every reader of the + * addresses holds anyway. The notifier never touches the union, so the + * pointer returned here stays good and can then be used under the notifier + * lock. + * + * Return: Pointer to the first device address, NULL if none is populated. + */ +static inline const struct drm_pagemap_addr * +drm_gpusvm_pages_first_dma(const struct drm_gpusvm_pages *svm_pages) +{ + struct drm_gpusvm_pages_flags flags = { + /* READ_ONCE pairs with the WRITE_ONCE of the flag writers */ + .__flags = READ_ONCE(svm_pages->flags.__flags), + }; + + if (flags.inline_dma_mapping) + return &svm_pages->inline_addr; + + return READ_ONCE(svm_pages->dma_addr); +} + /** * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function. * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible. From 18299151caddd61adc977a56ef54c3043f5ed85e Mon Sep 17 00:00:00 2001 From: Honglei Huang Date: Sat, 5 Sep 2026 21:31:42 +0800 Subject: [PATCH 29/76] drm/gpusvm: keep an IOVA mapped range dma address inline dma_iova_try_alloc() reserves one contiguous IOVA for the whole range and links each page at the next offset, so the device addresses run contiguously from entry 0 and one entry describes them all. A 2 MiB range of 4 KiB pages then drops the same 8 KiB array as a THP backed one. Fold only when state_offset covers the full range, which proves no device page was mapped in between, and only single page entries, so the order kept is 0 and stays true. Widening it instead would tell a consumer to use a huge page for npages separate CPU pages, which hangs Vega20 on amdgpu. The kept entry no longer bounds the segment, so skip the unmap walk when it has nothing to do, keyed off dpagemap rather than the flags, which are not published yet on the error unwind. Consumers need the same distinction, so drm_gpusvm_pages_first_dma() returns it alongside the array from one read of the flags; xe passes it to xe_res_first_dma(). Suggested-by: Matthew Brost Signed-off-by: Honglei Huang Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260905133142.3628027-7-honghuan@amd.com --- drivers/gpu/drm/drm_gpusvm.c | 41 +++++++++++++++++++++++------- drivers/gpu/drm/xe/xe_pt.c | 23 +++++++++++------ drivers/gpu/drm/xe/xe_res_cursor.h | 5 ++-- drivers/gpu/drm/xe/xe_svm.h | 8 +++--- include/drm/drm_gpusvm.h | 13 +++++++++- 5 files changed, 67 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 2c7c4c89dc4a..b6c9d3a07dc8 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1242,7 +1242,7 @@ static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, .__flags = svm_pages->flags.__flags, }; const struct drm_pagemap_addr *addrs = - drm_gpusvm_pages_first_dma(svm_pages); + drm_gpusvm_pages_first_dma(svm_pages, NULL); bool use_iova = dma_use_iova(&svm_pages->state); /* @@ -1259,7 +1259,15 @@ static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, dma_iova_free(dev, &svm_pages->state); } - for (i = 0, j = 0; i < npages; j++) { + /* + * With IOVA and no device page the unlink above tore every + * entry down, and that is also when the range may be folded + * to one entry, which must not be walked per entry. dpagemap + * is set before the first device_map(), so it is also right + * on the error path, where the flags are not published yet. + */ + for (i = 0, j = 0; + (!use_iova || dpagemap) && i < npages; j++) { const struct drm_pagemap_addr *addr = &addrs[j]; if (addr->proto == DRM_INTERCONNECT_SYSTEM) { @@ -1491,17 +1499,32 @@ static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, /** * drm_gpusvm_pages_inlinable() - Whether the dma address can be inlined + * @svm_pages: The SVM pages instance that was just mapped * @nentries: Number of entries the mapping loop produced + * @npages: Number of pages in the CPU range * - * A THP maps as one huge page, so the whole range needs a single device - * address: the dma_addr array can be freed and the address kept inline, - * which is where the memory saving comes from. + * A THP maps as one huge page, and an IOVA reservation links every page of + * the range at the next offset, so the device addresses run contiguously from + * entry 0. Either way one entry describes the whole range, so the dma_addr + * array can be freed and the address kept inline. + * + * state_offset advances only on the IOVA branch, so reaching the full range + * length proves no device page was mapped in between. Only single page + * entries fold, so the order kept is 0 and describes the range truthfully. + * Larger chunks, several huge pages among them, stay an array that is + * already short and that a consumer places with one PTE each. * * Return: True if the mapping fits in a single drm_pagemap_addr. */ -static bool drm_gpusvm_pages_inlinable(unsigned long nentries) +static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages, + unsigned long nentries, + unsigned long npages) { - return nentries == 1; + if (nentries == 1) + return true; + + return nentries == npages && dma_use_iova(&svm_pages->state) && + svm_pages->state_offset == npages * PAGE_SIZE; } /** @@ -1656,7 +1679,7 @@ static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm, if (pagemap) flags.has_devmem_pages = true; - if (drm_gpusvm_pages_inlinable(j)) { + if (drm_gpusvm_pages_inlinable(svm_pages, j, npages)) { struct drm_pagemap_addr addr = svm_pages->dma_addr[0]; kvfree(svm_pages->dma_addr); @@ -1772,7 +1795,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, if (map_dma) { for (p = 0; p < num_pages; ++p) { - if (drm_gpusvm_pages_first_dma(&svm_pages[p])) + if (drm_gpusvm_pages_first_dma(&svm_pages[p], NULL)) continue; svm_pages[p].dma_addr = kvzalloc_objs(*svm_pages[p].dma_addr, npages); diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index fa4b29da0b6c..7fb2fe1f826c 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -831,9 +831,12 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, return -EAGAIN; } if (xe_svm_range_has_dma_mapping(range)) { - xe_res_first_dma(xe_svm_range_first_dma(range), 0, - xe_svm_range_size(range), - &curs); + const struct drm_pagemap_addr *addr; + bool contiguous; + + addr = xe_svm_range_first_dma(range, &contiguous); + xe_res_first_dma(addr, 0, xe_svm_range_size(range), + contiguous, &curs); xe_svm_range_debug(range, "BIND PREPARE - MIXED"); } else { xe_assert(xe, false); @@ -865,11 +868,15 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, xe_bo_assert_held(bo); if (!xe_vma_is_null(vma) && !range && !is_purged) { - if (xe_vma_is_userptr(vma)) - xe_res_first_dma(drm_gpusvm_pages_first_dma - (&to_userptr_vma(vma)->userptr.pages), - 0, xe_vma_size(vma), &curs); - else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) + if (xe_vma_is_userptr(vma)) { + const struct drm_pagemap_addr *addr; + bool contiguous; + + addr = drm_gpusvm_pages_first_dma(&to_userptr_vma(vma)->userptr.pages, + &contiguous); + xe_res_first_dma(addr, 0, xe_vma_size(vma), contiguous, + &curs); + } else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma), xe_vma_size(vma), &curs); else diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h index 0522caafd89d..c3a037e5f34e 100644 --- a/drivers/gpu/drm/xe/xe_res_cursor.h +++ b/drivers/gpu/drm/xe/xe_res_cursor.h @@ -233,12 +233,13 @@ static inline void xe_res_first_sg(const struct sg_table *sg, * @dma_addr: struct drm_pagemap_addr array to walk * @start: Start of the range * @size: Size of the range + * @contiguous: Whether one entry describes the whole range * @cur: cursor object to initialize * * Start walking over the range of allocations between @start and @size. */ static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr, - u64 start, u64 size, + u64 start, u64 size, bool contiguous, struct xe_res_cursor *cur) { XE_WARN_ON(!dma_addr); @@ -248,7 +249,7 @@ static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr, cur->node = NULL; cur->start = start; cur->remaining = size; - cur->dma_seg_size = PAGE_SIZE << dma_addr->order; + cur->dma_seg_size = contiguous ? start + size : PAGE_SIZE << dma_addr->order; cur->dma_start = 0; cur->size = 0; cur->dma_addr = dma_addr; diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index 7eb80d4d6db4..2ef4ef026ccd 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -223,13 +223,14 @@ static inline unsigned long xe_svm_range_size(struct xe_svm_range *range) /** * xe_svm_range_first_dma() - Resolve the device address array of a SVM range * @range: SVM range + * @contiguous: Where to store whether one entry spans the whole range * * Return: Pointer to the first device address, NULL if none is populated. */ static inline const struct drm_pagemap_addr * -xe_svm_range_first_dma(struct xe_svm_range *range) +xe_svm_range_first_dma(struct xe_svm_range *range, bool *contiguous) { - return drm_gpusvm_pages_first_dma(&range->pages); + return drm_gpusvm_pages_first_dma(&range->pages, contiguous); } void xe_svm_flush(struct xe_vm *vm); @@ -449,8 +450,9 @@ static inline bool xe_svm_range_is_removed(struct xe_svm_range *range) } static inline const struct drm_pagemap_addr * -xe_svm_range_first_dma(struct xe_svm_range *range) +xe_svm_range_first_dma(struct xe_svm_range *range, bool *contiguous) { + *contiguous = false; return NULL; } diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h index aaad5c9b510c..9e35584812fd 100644 --- a/include/drm/drm_gpusvm.h +++ b/include/drm/drm_gpusvm.h @@ -380,12 +380,19 @@ static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages, /** * drm_gpusvm_pages_first_dma() - Resolve the device address array * @svm_pages: Pointer to the drm_gpusvm_pages. + * @contiguous: Where to store whether one entry spans the whole range, or NULL * * drm_gpusvm_pages use unions to optimize the storage of DMA addresses, * this function abstracts the access to the first device address. The driver * should use this helper instead of reading dma_addr directly to prevent * array out of bounds access. * + * @contiguous comes from the same read of the flags as the array itself, so a + * caller cannot see the two disagree and walk past that single entry into the + * fields behind it. When it is set the length comes from the range rather than + * from the order. The order still states what one PTE may cover: the range + * length for a huge page, PAGE_SIZE for an IOVA mapped range of single pages. + * * Only get_pages() and the free path switch between the two union members. * Both hold the notifier lock for read, so taking that lock does not stop * them; callers need the driver lock that does, which every reader of the @@ -396,13 +403,17 @@ static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages, * Return: Pointer to the first device address, NULL if none is populated. */ static inline const struct drm_pagemap_addr * -drm_gpusvm_pages_first_dma(const struct drm_gpusvm_pages *svm_pages) +drm_gpusvm_pages_first_dma(const struct drm_gpusvm_pages *svm_pages, + bool *contiguous) { struct drm_gpusvm_pages_flags flags = { /* READ_ONCE pairs with the WRITE_ONCE of the flag writers */ .__flags = READ_ONCE(svm_pages->flags.__flags), }; + if (contiguous) + *contiguous = flags.inline_dma_mapping; + if (flags.inline_dma_mapping) return &svm_pages->inline_addr; From 26793f00d66786f481c41ea6cc466383e229aaa3 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 3 Sep 2026 11:16:07 +0530 Subject: [PATCH 30/76] drm/xe/log: Add GUCSUBMIT component The proper SIGID for GuC submission failures is GT_TDR, but using the generic GT component may hide the real source of the report. Add GUCSUBMIT as a DRIVER component and map it to XE_SIGID_GT_TDR. Cc: Mallesh Koujalagi Cc: Badal Nilawar Cc: Matthew Brost Cc: Himal Prasad Ghimiray Cc: Rodrigo Vivi Reviewed-by: Michal Wajdeczko Suggested-by: Michal Wajdeczko Signed-off-by: Arvind Yadav Link: https://patchwork.freedesktop.org/patch/msgid/20260903054610.3640788-2-arvind.yadav@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/abi/xe_log_abi.h | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/xe/abi/xe_log_abi.h b/drivers/gpu/drm/xe/abi/xe_log_abi.h index d6105520173e..526aadf85fe9 100644 --- a/drivers/gpu/drm/xe/abi/xe_log_abi.h +++ b/drivers/gpu/drm/xe/abi/xe_log_abi.h @@ -144,6 +144,7 @@ enum xe_log_location_bits { define(DRIVER, 4, RTP, SW, "Register Table Processing") \ define(DRIVER, 5, WA, SW, "Workarounds") \ define(DRIVER, 6, PAGEFAULT, MEM_FAULT, "Page Fault") \ + define(DRIVER, 7, GUCSUBMIT, GT_TDR, "GuC Submission") \ /* */ \ define(DRIVER_HARDWARE, 1, REGS, IO_BUS, "Registers") \ define(DRIVER_HARDWARE, 2, GGTT, IO_BUS, "Global GTT") \ From 58980bf190cc9d142289acc6cc7376745ead8f37 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 3 Sep 2026 11:16:08 +0530 Subject: [PATCH 31/76] drm/xe/guc: Report reset failure using SIGID Route the GuC submission engine reset failure through the structured SIGID logging helper. Use the GUCSUBMIT component, which maps to XE_SIGID_GT_TDR, and pass -EIO as the errno value. Cc: Mallesh Koujalagi Cc: Badal Nilawar Cc: Matthew Brost Cc: Himal Prasad Ghimiray Cc: Rodrigo Vivi Reviewed-by: Michal Wajdeczko Suggested-by: Michal Wajdeczko Signed-off-by: Arvind Yadav Link: https://patchwork.freedesktop.org/patch/msgid/20260903054610.3640788-3-arvind.yadav@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_guc_submit.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 0a6e2b81b5a5..e44d06703020 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -35,6 +35,7 @@ #include "xe_guc_klv_helpers.h" #include "xe_guc_submit_types.h" #include "xe_hw_engine.h" +#include "xe_log.h" #include "xe_lrc.h" #include "xe_macros.h" #include "xe_map.h" @@ -3520,8 +3521,9 @@ int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 le reason = msg[2]; /* Unexpected failure of a hardware feature, log an actual error */ - xe_gt_err(gt, "GuC engine reset request failed on %d:%d because 0x%08X", - guc_class, instance, reason); + xe_log_err(gt, GUCSUBMIT, -EIO, + "engine reset failed on %u:%u, reason=%#x\n", + guc_class, instance, reason); xe_gt_reset_async(gt); From 36ba01e43061b58ab7318904c32740aeea3fcb4e Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 3 Sep 2026 11:16:09 +0530 Subject: [PATCH 32/76] drm/xe/svm: Report terminal page-fault failures using SIGID Use structured SIGID logging for SVM page-fault errors after retries are exhausted. Report these failures with the PAGEFAULT component, which maps to XE_SIGID_MEM_FAULT. Cc: Badal Nilawar Cc: Matthew Brost Cc: Himal Prasad Ghimiray Cc: Michal Wajdeczko Cc: Rodrigo Vivi Reviewed-by: Mallesh Koujalagi Suggested-by: Mallesh Koujalagi Signed-off-by: Arvind Yadav Link: https://patchwork.freedesktop.org/patch/msgid/20260903054610.3640788-4-arvind.yadav@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_svm.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 1c7793d8caa8..6c3033fc4db7 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -13,6 +13,7 @@ #include "xe_bo.h" #include "xe_exec_queue_types.h" #include "xe_gt_stats.h" +#include "xe_log.h" #include "xe_migrate.h" #include "xe_module.h" #include "xe_pagefault.h" @@ -1361,9 +1362,9 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, else goto retry; } else { - drm_err(&vm->xe->drm, - "VRAM allocation failed, retry count exceeded, asid=%u, errno=%pe\n", - vm->usm.asid, ERR_PTR(err)); + xe_log_err(gt, PAGEFAULT, err, + "VRAM allocation failed, retry count exceeded, ASID=%u\n", + vm->usm.asid); goto err_out; } } @@ -1384,9 +1385,9 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, range_debug(range, "PAGE FAULT - RETRY PAGES"); goto retry; } else { - drm_err(&vm->xe->drm, - "Get pages failed, retry count exceeded, asid=%u, gpusvm=%p, errno=%pe\n", - vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); + xe_log_err(gt, PAGEFAULT, err, + "Get pages failed, retry count exceeded, ASID=%u, GPUVM=%s\n", + vm->usm.asid, vm->svm.gpusvm.name); } } if (err) { From 39592150387d7e7382999ddef8714245a1b91c12 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 3 Sep 2026 11:16:10 +0530 Subject: [PATCH 33/76] drm/xe/gt: Report GT reset failure using SIGID Route the GT reset failure log through the structured SIGID logging helper. Failure to complete the full graphics reset means GT reset did not finish as expected. Use the GT component, which maps to XE_SIGID_GT_TDR, and pass the reset errno value to the helper. Also update the message to describe the actual failure. The driver does not clear GRDOM_FULL directly. The reset should complete and clear it within the timeout. Cc: Badal Nilawar Cc: Matthew Brost Cc: Himal Prasad Ghimiray Cc: Michal Wajdeczko Cc: Rodrigo Vivi Reviewed-by: Mallesh Koujalagi Suggested-by: Mallesh Koujalagi Signed-off-by: Arvind Yadav Link: https://patchwork.freedesktop.org/patch/msgid/20260903054610.3640788-5-arvind.yadav@intel.com Signed-off-by: Himal Prasad Ghimiray --- drivers/gpu/drm/xe/xe_gt.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 478e047031f4..775c826b68b4 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -70,6 +70,8 @@ #include "xe_wa.h" #include "xe_wopcm.h" +#define GRDOM_RESET_TIMEOUT_MS 5 + struct xe_gt *xe_gt_alloc(struct xe_tile *tile) { struct xe_device *xe = tile_to_xe(tile); @@ -828,10 +830,13 @@ static int do_gt_reset(struct xe_gt *gt) xe_gsc_wa_14015076503(gt, true); xe_mmio_write32(>->mmio, GDRST, GRDOM_FULL); - err = xe_mmio_wait32(>->mmio, GDRST, GRDOM_FULL, 0, 5000, NULL, false); + err = xe_mmio_wait32(>->mmio, GDRST, GRDOM_FULL, 0, + GRDOM_RESET_TIMEOUT_MS * USEC_PER_MSEC, + NULL, false); if (err) - xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n", - ERR_PTR(err)); + xe_log_err(gt, GT, err, + "full graphics reset not completed in %u ms\n", + GRDOM_RESET_TIMEOUT_MS); xe_gsc_wa_14015076503(gt, false); From 434514b6fe731e873808297c268fc52cdf4a1ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Thu, 3 Sep 2026 13:45:52 +0200 Subject: [PATCH 34/76] drm/xe: Flush LSC untyped L1 dataport cache after rcs/ccs batches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit emit_render_cache_flush() sets PIPE_CONTROL0_HDC_PIPELINE_FLUSH to flush the L2/HDC data cache before fence signalling, but it never requests a flush of the LSC untyped L1 data cache via the 'Untyped Data-Port Cache Flush Enable' bit in PIPE_CONTROL DWord0[11]. Per the Bspec, in 3D pipeline mode HDC Pipeline Flush is documented to also flush/invalidate the untyped L1 cache, but only depending on how HDC_CHICKEN0[13:11] is programmed. Starting with MTL, this coupling between HDC Pipeline Flush and the untyped L1 cache flush no longer holds in practice, regardless of how HDC_CHICKEN0 is programmed, so relying on it is not safe on newer platforms such as BMG. Mesa's Vulkan driver (anv) has been assuming the kernel flushes both caches between submissions, and hit user-visible corruption in apps such as Llama.cpp because of this gap; it now works around it by flushing both caches again from userspace at the end of every command buffer. Correctness between submissions on the same queue is userspace's responsibility and belongs in Mesa, not the kernel. However, for security we must ensure stale data can't leak through the untyped L1 dataport cache once memory is reclaimed or evicted, which requires the KMD to flush it before releasing memory for reuse. Prior to MTL, HDC_CHICKEN0 could be programmed (as already done for DG2 via Wa_22010960976/Wa_14013347512) to reliably keep HDC Pipeline Flush coupled to the untyped L1 cache flush, so those platforms are unaffected. Mesa's own anv driver found that on MTL the HW disconnected the two independently of how HDC_CHICKEN0 is programmed, and could not bring the old behavior back even by writing the register by hand; see Mesa commit 7c2ff46a4fc3 ("anv: don't prevent L1 untyped cache flush in 3D mode"). The kernel can't reliably request the flush from the CS on MTL either, so restrict the new PIPE_CONTROL bit to GRAPHICS_VERx100 >= 2000 (Xe2 and later), where it can be relied on. Explicitly set PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH together with PIPE_CONTROL0_HDC_PIPELINE_FLUSH in emit_render_cache_flush() on Xe2 and later, so the L1 data cache is known clean before memory is released for reuse, without depending on undocumented platform-specific HDC_CHICKEN0 behavior. Bspec: 56551 Link: https://gitlab.freedesktop.org/mesa/mesa/-/commit/7c2ff46a4fc3e537573ac9503057e0cd29b6fff3 Fixes: 9f8f93bee3ef ("drm/xe: Emit a render cache flush after each rcs/ccs batch") Reported-by: Lionel Landwerlin Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/8909 Cc: José Roberto de Souza Cc: intel-xe@lists.freedesktop.org Cc: # v6.8+ Assisted-by: GitHub_Copilot:claude-sonnet-5 Signed-off-by: Thomas Hellström Reviewed-by: Matthew Auld Link: https://patch.msgid.link/20260903114552.48634-1-thomas.hellstrom@linux.intel.com --- drivers/gpu/drm/xe/instructions/xe_gpu_commands.h | 1 + drivers/gpu/drm/xe/xe_ring_ops.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index 18d0fde8c98f..faf8d7e2c5c1 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -46,6 +46,7 @@ #define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2)) #define PIPE_CONTROL0_QUEUE_DRAIN_MODE BIT(12) +#define PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH BIT(11) /* gen12 */ #define PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE BIT(10) /* gen12 */ #define PIPE_CONTROL0_HDC_PIPELINE_FLUSH BIT(9) /* gen12 */ diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 39a670e91ba7..08b4a4283e96 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -212,6 +212,7 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) { struct xe_exec_queue *q = job->q; struct xe_gt *gt = q->gt; + struct xe_device *xe = gt_to_xe(gt); bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); u32 flags0, flags1; @@ -220,6 +221,16 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH; + /* + * Prior to MTL, HDC Pipeline Flush reliably also flushes the LSC + * untyped L1 dataport cache, provided HDC_CHICKEN0 is programmed + * correctly. Starting with MTL that coupling no longer holds + * regardless of how HDC_CHICKEN0 is programmed, but explicitly + * requesting the flush via PIPE_CONTROL is itself only reliable + * from Xe2 onward, so only gate it in on Xe2+. + */ + if (GRAPHICS_VERx100(xe) >= 2000) + flags0 |= PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH; flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH | PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | PIPE_CONTROL_DEPTH_CACHE_FLUSH | From 683eb75c150b4a08d1796e5e142ad26098407abe Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 8 Sep 2026 09:04:18 +0530 Subject: [PATCH 35/76] drm/xe: Add multi_queue_active_lrca debugfs Add a per-GT debugfs file, multi_queue_active_lrca, that prints, for every engine supporting multi-queue, the currently active queue ID (CSMQDEBUG) and the LRCA of the exec queue occupying that slot within the running multi-queue group. RING_CURRENT_LRCA only reports the primary queue's LRCA for the group and does not update to reflect the active queue in multi-queue mode, which makes it hard to tell which queue is actually running when debugging multi-queue CSB/context-switch issues. Resolve the active LRCA by matching the primary LRCA against each queue's group and picking the queue at the reported active_id position. v4: - Extracted multi_queue_active_lrca into a dedicated multi_queue_debugfs_list to prevent debugfs node registration on platforms lacking multi-queue support entirely, via xe_gt_has_multi_queue() gating. (Tejas) v3: - Use xe_exec_queue_get_lrc() instead of raw pointer dereference to safely handle concurrent multi-queue group creation and avoid race conditions (Sashiko) v2: - Maintain alphabetical order for includes and pf_only_debugfs_list (Tejas) - Export and reuse xe_lrc_get_multi_queue_active_queue_id() instead of duplicate MMIO read (Tejas) Bspec: 60321, 73976 Signed-off-by: Varun Gupta Link: https://patch.msgid.link/20260908033417.1019602-2-varun.gupta@intel.com Reviewed-by: Tejas Upadhyay Signed-off-by: Tejas Upadhyay --- drivers/gpu/drm/xe/xe_gt_debugfs.c | 54 ++++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.c | 73 ++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 5 ++ drivers/gpu/drm/xe/xe_lrc.c | 6 +-- drivers/gpu/drm/xe/xe_lrc.h | 1 + 5 files changed, 136 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index 361a70234d1f..bb09e70ee44c 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -13,6 +13,7 @@ #include #include +#include "regs/xe_engine_regs.h" #include "regs/xe_gt_regs.h" #include "xe_device.h" #include "xe_force_wake.h" @@ -25,6 +26,7 @@ #include "xe_gt_stats.h" #include "xe_gt_topology.h" #include "xe_guc_hwconfig.h" +#include "xe_guc_submit.h" #include "xe_hw_engine.h" #include "xe_lrc.h" #include "xe_mmio.h" @@ -130,6 +132,48 @@ static int hw_engines(struct xe_gt *gt, struct drm_printer *p) return 0; } +static int multi_queue_active_lrca(struct xe_gt *gt, struct drm_printer *p) +{ + struct xe_guc *guc = >->uc.guc; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) { + u32 cur_lrca, active_id, lrca; + unsigned int fw_ref; + + if (!xe_gt_supports_multi_queue(gt, hwe->class)) + continue; + + /* + * Forcewake is dropped before xe_guc_submit_active_multi_queue_lrca() + * below, which takes guc->submission_state.lock, to avoid holding a + * GT forcewake ref across a mutex acquired elsewhere in the opposite + * order. + */ + fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) { + drm_printf(p, "%s\tforcewake failed, skipping\n", hwe->name); + xe_force_wake_put(gt_to_fw(gt), fw_ref); + continue; + } + + cur_lrca = xe_mmio_read32(>->mmio, + RING_CURRENT_LRCA(hwe->mmio_base)); + active_id = xe_lrc_get_multi_queue_active_queue_id(hwe); + + xe_force_wake_put(gt_to_fw(gt), fw_ref); + + lrca = xe_guc_submit_active_multi_queue_lrca(guc, hwe, cur_lrca, + active_id); + + drm_printf(p, "%s\tactive_queue_id %u\tcurrent_lrca 0x%08x\tactive_lrca 0x%08x\n", + hwe->name, active_id, cur_lrca, lrca); + } + + return 0; +} + static int steering(struct xe_gt *gt, struct drm_printer *p) { xe_gt_mcr_steering_dump(gt, p); @@ -254,6 +298,11 @@ static const struct drm_info_list pf_only_debugfs_list[] = { { "steering", .show = xe_gt_debugfs_show_with_rpm, .data = steering }, }; +static const struct drm_info_list multi_queue_debugfs_list[] = { + { "multi_queue_active_lrca", + .show = xe_gt_debugfs_show_with_rpm, .data = multi_queue_active_lrca }, +}; + static ssize_t write_to_gt_call(const char __user *userbuf, size_t count, loff_t *ppos, void (*call)(struct xe_gt *), struct xe_gt *gt) { @@ -521,6 +570,11 @@ void xe_gt_debugfs_register(struct xe_gt *gt) ARRAY_SIZE(pf_only_debugfs_list), root, minor); + if (!IS_SRIOV_VF(xe) && xe_gt_has_multi_queue(gt)) + drm_debugfs_create_files(multi_queue_debugfs_list, + ARRAY_SIZE(multi_queue_debugfs_list), + root, minor); + if (xe_gt_is_main_type(gt) && !IS_DGFX(xe) && !IS_SRIOV_VF(xe)) debugfs_create_file("gt_ia_bias", 0600, root, gt, >_ia_bias_fops); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index e44d06703020..f3ba8abfc228 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -3881,6 +3881,79 @@ bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc) return false; } +/** + * xe_guc_submit_active_multi_queue_lrca() - Resolve the LRCA of the active + * queue in the multi-queue group currently running on an engine. + * @guc: the &xe_guc managing the exec queues + * @hwe: the &xe_hw_engine whose active queue is being resolved + * @cur_lrca: value read from RING_CURRENT_LRCA, identifies the running group + * @active_id: current Active Queue ID read from CSMQDEBUG (position in group) + * + * The running group is identified by matching @cur_lrca against the group's + * primary LRCA; @active_id then selects the active queue within that group. + * + * Return: the LRCA of the active queue, or 0 if no matching queue is found. + */ +u32 xe_guc_submit_active_multi_queue_lrca(struct xe_guc *guc, + struct xe_hw_engine *hwe, + u32 cur_lrca, u32 active_id) +{ + struct xe_exec_queue *q; + unsigned long index; + u32 lrca = 0; + + /* + * submission_state.lock also protects exec_queue teardown: an exec + * queue is removed from exec_queue_lookup before its group/primary + * are freed, so any q found in the xarray below has a live group + * and primary for as long as we hold the lock. + */ + guard(mutex)(&guc->submission_state.lock); + + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { + struct xe_exec_queue_group *group = q->multi_queue.group; + struct xe_lrc *active_lrc; + struct xe_lrc *primary_lrc; + + if (!q->multi_queue.valid || !group || !group->primary) + continue; + /* + * Multi-queue exec queues are bound to a hw engine class; + * GuC dynamically schedules them onto one of the class's + * physical instances, so there is no fixed queue-to-instance + * mapping to filter on here. + */ + if (q->class != hwe->class) + continue; + if (q->multi_queue.pos != active_id) + continue; + /* + * LRCAs are page-aligned (4K) addresses in GGTT; the low + * bits reported by RING_CURRENT_LRCA are not meaningful, so + * only compare bits [31:12]. + */ + primary_lrc = xe_exec_queue_get_lrc(group->primary, 0); + if (!primary_lrc) + continue; + + if ((xe_lrc_ggtt_addr(primary_lrc) ^ cur_lrca) & GENMASK(31, 12)) { + xe_lrc_put(primary_lrc); + continue; + } + + active_lrc = xe_exec_queue_get_lrc(q, 0); + xe_lrc_put(primary_lrc); + if (!active_lrc) + continue; + + lrca = xe_lrc_ggtt_addr(active_lrc); + xe_lrc_put(active_lrc); + break; + } + + return lrca; +} + /** * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all * exec queues registered to given GuC. diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index ccade320dc69..29abf07d8f04 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -11,6 +11,7 @@ struct drm_printer; struct xe_exec_queue; struct xe_guc; +struct xe_hw_engine; int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids); int xe_guc_submit_enable(struct xe_guc *guc); @@ -55,6 +56,10 @@ void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type); bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc); +u32 xe_guc_submit_active_multi_queue_lrca(struct xe_guc *guc, + struct xe_hw_engine *hwe, + u32 cur_lrca, u32 active_id); + int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch); #endif diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 25fe9dbc9141..f1cf1463f1b2 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -2705,7 +2705,7 @@ static u64 get_queue_timestamp(struct xe_hw_engine *hwe) RING_QUEUE_TIMESTAMP(hwe->mmio_base)); } -static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe) +u32 xe_lrc_get_multi_queue_active_queue_id(struct xe_hw_engine *hwe) { u32 val = xe_mmio_read32(&hwe->gt->mmio, RING_CSMQDEBUG(hwe->mmio_base)); @@ -2739,14 +2739,14 @@ static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc) if (!hwe) return xe_lrc_queue_timestamp(lrc); - if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos) + if (xe_lrc_get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos) return xe_lrc_queue_timestamp(lrc); /* queue is active, so store the queue timestamp register */ reg_queue_ts = get_queue_timestamp(hwe); /* double check queue and primary queue are both still active */ - if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos || + if (xe_lrc_get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos || !context_active(primary_lrc)) return xe_lrc_queue_timestamp(lrc); diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index 7be5e3da8bc8..a8ff4e59a1f4 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -158,6 +158,7 @@ int xe_lrc_lookup_default_reg_value(struct xe_gt *gt, u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs); void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority); +u32 xe_lrc_get_multi_queue_active_queue_id(struct xe_hw_engine *hwe); struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc); void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot); From f3391a0b12d7bf826a0b21600d2f294f3dce4c14 Mon Sep 17 00:00:00 2001 From: Ilia Levi Date: Tue, 8 Sep 2026 17:50:48 +0100 Subject: [PATCH 36/76] drm/xe/mmio_gem: forbid VMA split The fault handler assumes it always operates on a VMA spanning the entire GEM object. This does not hold when the VMA has been split, e.g. by a partial munmap or mprotect. In that case the handler may map wrong physical pages or cause SIGBUS. Handle this by forbidding VMA split, as partial unmaps are not deemed useful for MMIO GEMs. Suggested-by: Matthew Auld Signed-off-by: Ilia Levi Fixes: 1ffcf8b8ae8a ("drm/xe: Support for mmap-ing mmio regions") Reviewed-by: Matthew Auld Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20260908165046.1393557-11-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_mmio_gem.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 3741ae60f532..d54477b93b6e 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -39,10 +39,20 @@ struct xe_mmio_gem { phys_addr_t phys_addr; }; +static int xe_mmio_gem_vm_may_split(struct vm_area_struct *area, unsigned long addr) +{ + /* + * Forbid splitting. Together with VM_DONTEXPAND, this keeps the VMA + * matching the GEM object exactly. + */ + return -EINVAL; +} + static const struct vm_operations_struct vm_ops = { .open = drm_gem_vm_open, .close = drm_gem_vm_close, .fault = xe_mmio_gem_vm_fault, + .may_split = xe_mmio_gem_vm_may_split, }; static const struct drm_gem_object_funcs xe_mmio_gem_funcs = { From 1e8e28e35df0e77ae1b22fc091c1f422f62fa5e9 Mon Sep 17 00:00:00 2001 From: Ilia Levi Date: Tue, 8 Sep 2026 17:50:49 +0100 Subject: [PATCH 37/76] drm/xe/mmio_gem: use write-back mapping for dummy page Currently vmf_insert_pfn() maps the dummy page as UC, inheriting the VMA's page protection which was set for the real MMIO region. This conflicts with the direct map's WB mapping of the same page, creating a cache type alias which is architecturally undefined on some platforms. Use vmf_insert_pfn_prot() with a WB pgprot instead. Also simplify to fault in the requested page instead of the whole VMA. Fixes: 1ffcf8b8ae8a ("drm/xe: Support for mmap-ing mmio regions") Reported-by: Sashiko Closes: https://sashiko.dev/#/patchset/20260525125801.975038-6-ilia.levi%40intel.com Assisted-by: GitHub-Copilot:claude-opus-4.6 Signed-off-by: Ilia Levi Reviewed-by: Matthew Auld Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20260908165046.1393557-12-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_mmio_gem.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index d54477b93b6e..96f3bf46fd75 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -172,14 +172,13 @@ static void xe_mmio_gem_release_dummy_page(struct drm_device *dev, void *res) __free_page((struct page *)res); } -static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_area_struct *vma) +static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_fault *vmf) { + struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *base = vma->vm_private_data; struct drm_device *dev = base->dev; - vm_fault_t ret = VM_FAULT_NOPAGE; struct page *page; unsigned long pfn; - unsigned long i; page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) @@ -190,16 +189,8 @@ static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_area_struct *vma) pfn = page_to_pfn(page); - /* Map the entire VMA to the same dummy page */ - for (i = 0; i < base->size; i += PAGE_SIZE) { - unsigned long addr = vma->vm_start + i; - - ret = vmf_insert_pfn(vma, addr, pfn); - if (ret & VM_FAULT_ERROR) - break; - } - - return ret; + return vmf_insert_pfn_prot(vma, vmf->address, pfn, + vm_get_page_prot(vma->vm_flags)); } static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) @@ -219,7 +210,7 @@ static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) * It is assumed the userspace will receive the notification via some * other channel (e.g. drm uevent). */ - return xe_mmio_gem_vm_fault_dummy_page(vma); + return xe_mmio_gem_vm_fault_dummy_page(vmf); } for (i = 0; i < base->size; i += PAGE_SIZE) { From 6666ca9192f3bdf839aad33b9e1c9ebb7a222a29 Mon Sep 17 00:00:00 2001 From: Ilia Levi Date: Tue, 8 Sep 2026 17:50:50 +0100 Subject: [PATCH 38/76] drm/xe/mmio_gem: simplify fault handler loop Make the iteration over the addresses in the VMA more explicit. No functional change, as the VMA matches the GEM object exactly. Signed-off-by: Ilia Levi Reviewed-by: Matthew Auld Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20260908165046.1393557-13-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_mmio_gem.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 96f3bf46fd75..3b6d04efe81a 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -200,7 +200,7 @@ static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) struct xe_mmio_gem *obj = to_xe_mmio_gem(base); struct drm_device *dev = base->dev; vm_fault_t ret = VM_FAULT_NOPAGE; - unsigned long i; + unsigned long addr, pfn; int idx; if (!drm_dev_enter(dev, &idx)) { @@ -213,13 +213,13 @@ static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) return xe_mmio_gem_vm_fault_dummy_page(vmf); } - for (i = 0; i < base->size; i += PAGE_SIZE) { - unsigned long addr = vma->vm_start + i; - unsigned long phys_addr = obj->phys_addr + i; - - ret = vmf_insert_pfn(vma, addr, PHYS_PFN(phys_addr)); + pfn = PHYS_PFN(obj->phys_addr); + for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) { + ret = vmf_insert_pfn(vma, addr, pfn); if (ret & VM_FAULT_ERROR) break; + + pfn++; } drm_dev_exit(idx); From 32f0cb250598456d812fb7ca57a040282858323d Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Tue, 8 Sep 2026 17:50:51 +0100 Subject: [PATCH 39/76] drm/xe/mmio_gem: Revoke drm_vma_node on xe_mmio_gem destroy xe_mmio_gem_create() calls drm_vma_node_allow() but nothing ever calls drm_vma_node_revoke(). The drm_vma_offset_file rb-tree entry allocated by drm_vma_node_allow() is not freed by drm_gem_object_release(), so it is leaked on every create/destroy cycle. Add a struct drm_file * parameter to xe_mmio_gem_destroy() and call drm_vma_node_revoke() from there, mirroring the drm_vma_node_allow() call in xe_mmio_gem_create(). Fixes: 1ffcf8b8ae8a ("drm/xe: Support for mmap-ing mmio regions") Suggested-by: Ilia Levi Assisted-by: Claude:claude-opus-4.6 Signed-off-by: Shuicheng Lin Reviewed-by: Ilia Levi Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20260908165046.1393557-14-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_mmio_gem.c | 4 +++- drivers/gpu/drm/xe/xe_mmio_gem.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 3b6d04efe81a..3c42d8358c7d 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -138,14 +138,16 @@ static void xe_mmio_gem_free(struct drm_gem_object *base) /** * xe_mmio_gem_destroy - Destroy the GEM object that exposes an MMIO region * @gem: the GEM object to destroy + * @file: DRM file descriptor previously passed to xe_mmio_gem_create() * * This function releases resources associated with the GEM object created by * xe_mmio_gem_create(). * * See: "Exposing MMIO regions to userspace" */ -void xe_mmio_gem_destroy(struct xe_mmio_gem *gem) +void xe_mmio_gem_destroy(struct xe_mmio_gem *gem, struct drm_file *file) { + drm_vma_node_revoke(&gem->base.vma_node, file); xe_mmio_gem_free(&gem->base); } diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.h b/drivers/gpu/drm/xe/xe_mmio_gem.h index 4b76d5586ebb..80d7795f07c8 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.h +++ b/drivers/gpu/drm/xe/xe_mmio_gem.h @@ -15,6 +15,6 @@ struct xe_mmio_gem; struct xe_mmio_gem *xe_mmio_gem_create(struct xe_device *xe, struct drm_file *file, phys_addr_t phys_addr, size_t size); u64 xe_mmio_gem_mmap_offset(struct xe_mmio_gem *gem); -void xe_mmio_gem_destroy(struct xe_mmio_gem *gem); +void xe_mmio_gem_destroy(struct xe_mmio_gem *gem, struct drm_file *file); #endif /* _XE_MMIO_GEM_H_ */ From 8bf6213f9831e46313af4722a1ee6db1b7596378 Mon Sep 17 00:00:00 2001 From: Ilia Levi Date: Tue, 8 Sep 2026 17:50:52 +0100 Subject: [PATCH 40/76] drm/xe/mmio_gem: cache the dummy page per object Currently, when the fault handler provides a dummy page, it allocates a new one on every invocation and ties its lifetime to the drm_device via drmm_add_action_or_reset(). Concurrent faults after hot-unplug therefore accumulate pages that persist until device teardown. Cache a single dummy page in the xe_mmio_gem object and use dma_resv lock to protect its allocation. Free it with the object. v2: use dma_resv lock to protect the allocation (Matt Auld) Assisted-by: GitHub-Copilot:claude-opus-4.6 Signed-off-by: Ilia Levi Reviewed-by: Matthew Auld Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20260908165046.1393557-15-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_mmio_gem.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 3c42d8358c7d..970b1e2f4981 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -5,9 +5,9 @@ #include "xe_mmio_gem.h" +#include #include #include -#include #include "xe_device_types.h" @@ -37,6 +37,7 @@ static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *); struct xe_mmio_gem { struct drm_gem_object base; phys_addr_t phys_addr; + struct page *dummy_page; /* protected by the GEM's dma_resv */ }; static int xe_mmio_gem_vm_may_split(struct vm_area_struct *area, unsigned long addr) @@ -131,6 +132,8 @@ static void xe_mmio_gem_free(struct drm_gem_object *base) { struct xe_mmio_gem *obj = to_xe_mmio_gem(base); + if (obj->dummy_page) + __free_page(obj->dummy_page); drm_gem_object_release(base); kfree(obj); } @@ -169,27 +172,29 @@ static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct * return 0; } -static void xe_mmio_gem_release_dummy_page(struct drm_device *dev, void *res) +static int alloc_dummy_page_if_needed(struct drm_gem_object *base) { - __free_page((struct page *)res); + struct xe_mmio_gem *obj = to_xe_mmio_gem(base); + + dma_resv_lock(base->resv, NULL); + if (!obj->dummy_page) + obj->dummy_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + dma_resv_unlock(base->resv); + + return obj->dummy_page ? 0 : -ENOMEM; } static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *base = vma->vm_private_data; - struct drm_device *dev = base->dev; - struct page *page; + struct xe_mmio_gem *obj = to_xe_mmio_gem(base); unsigned long pfn; - page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) + if (alloc_dummy_page_if_needed(base)) return VM_FAULT_OOM; - if (drmm_add_action_or_reset(dev, xe_mmio_gem_release_dummy_page, page)) - return VM_FAULT_OOM; - - pfn = page_to_pfn(page); + pfn = page_to_pfn(obj->dummy_page); return vmf_insert_pfn_prot(vma, vmf->address, pfn, vm_get_page_prot(vma->vm_flags)); From fb2ee38bab8025ad6a7a9cbb4635c5a178e4a7bc Mon Sep 17 00:00:00 2001 From: Ilia Levi Date: Tue, 8 Sep 2026 17:50:53 +0100 Subject: [PATCH 41/76] drm/xe/mmio_gem: fix destroy flow xe_mmio_gem_destroy() currently frees the GEM object directly, bypassing reference counting. Since existing VMAs hold a reference and the fault handler accesses the object through vma->vm_private_data, this is use-after-free. Additionally, nothing prevents the fault handler from installing PTEs to the real MMIO after destroy. Fix this with proper synchronization and refcounting. Also, do not set vm_pgoff to zero. Many DRM drivers do this because helpers like dma_mmap_pages() interpret vm_pgoff as an intra-buffer page offset; leaving the DRM fake offset there would break these helpers. Those drivers can get away with zeroing it because they map eagerly - all PTEs are established before mmap returns, so vm_pgoff is never consulted again. Our driver does not use such helpers and the newly introduced call to drm_vma_node_unmap() relies on vm_pgoff being untouched. v2: (Matt Auld) - use dma_resv lock to serialize fault handler with destroy - SIGBUS on access after destroy Fixes: 1ffcf8b8ae8a ("drm/xe: Support for mmap-ing mmio regions") Assisted-by: GitHub-Copilot:claude-opus-4.6 Signed-off-by: Ilia Levi Reviewed-by: Matthew Auld Signed-off-by: Matthew Auld Link: https://patch.msgid.link/20260908165046.1393557-16-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_mmio_gem.c | 42 ++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 970b1e2f4981..5ffe03d36190 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -38,6 +38,7 @@ struct xe_mmio_gem { struct drm_gem_object base; phys_addr_t phys_addr; struct page *dummy_page; /* protected by the GEM's dma_resv */ + bool destroyed; /* protected by the GEM's dma_resv */ }; static int xe_mmio_gem_vm_may_split(struct vm_area_struct *area, unsigned long addr) @@ -150,8 +151,22 @@ static void xe_mmio_gem_free(struct drm_gem_object *base) */ void xe_mmio_gem_destroy(struct xe_mmio_gem *gem, struct drm_file *file) { - drm_vma_node_revoke(&gem->base.vma_node, file); - xe_mmio_gem_free(&gem->base); + struct drm_gem_object *base = &gem->base; + struct drm_device *dev = base->dev; + + drm_vma_node_revoke(&base->vma_node, file); + + dma_resv_lock(base->resv, NULL); + gem->destroyed = true; + dma_resv_unlock(base->resv); + /* + * Setting 'destroyed' under lock takes care of the subsequent faults. + * Zap the existing PTEs to cut off access to the real MMIO through + * currently mapped pages. + */ + drm_vma_node_unmap(&base->vma_node, dev->anon_inode->i_mapping); + + drm_gem_object_put(base); } static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct *vma) @@ -162,8 +177,6 @@ static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct * if ((vma->vm_flags & VM_SHARED) == 0) return -EINVAL; - /* Set vm_pgoff (used as a fake buffer offset by DRM) to 0 */ - vma->vm_pgoff = 0; vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma)); vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_NORESERVE); @@ -176,10 +189,9 @@ static int alloc_dummy_page_if_needed(struct drm_gem_object *base) { struct xe_mmio_gem *obj = to_xe_mmio_gem(base); - dma_resv_lock(base->resv, NULL); + dma_resv_assert_held(base->resv); if (!obj->dummy_page) obj->dummy_page = alloc_page(GFP_KERNEL | __GFP_ZERO); - dma_resv_unlock(base->resv); return obj->dummy_page ? 0 : -ENOMEM; } @@ -200,7 +212,7 @@ static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_fault *vmf) vm_get_page_prot(vma->vm_flags)); } -static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) +static vm_fault_t xe_mmio_gem_vm_fault_locked(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *base = vma->vm_private_data; @@ -210,6 +222,10 @@ static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) unsigned long addr, pfn; int idx; + dma_resv_assert_held(base->resv); + if (obj->destroyed) + return VM_FAULT_SIGBUS; + if (!drm_dev_enter(dev, &idx)) { /* * Provide a dummy page to avoid SIGBUS for events such as hot-unplug. @@ -232,3 +248,15 @@ static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) drm_dev_exit(idx); return ret; } + +static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct drm_gem_object *base = vma->vm_private_data; + vm_fault_t ret; + + dma_resv_lock(base->resv, NULL); + ret = xe_mmio_gem_vm_fault_locked(vmf); + dma_resv_unlock(base->resv); + return ret; +} From 7cf85c190814c209cb182eaba39e184c1655faa9 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Tue, 8 Sep 2026 17:50:54 +0100 Subject: [PATCH 42/76] drm/xe/mmio_gem: reject VM_EXEC and drop VM_DONTCOPY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hardware MMIO registers should never be executable; reject VM_EXEC at mmap time and clear VM_MAYEXEC to prevent later mprotect attempts. Also drop VM_DONTCOPY so that child processes across fork() can inherit the mapping and lazily fault in the PFNs, matching standard DRM GEM semantics and making the existing drm_gem_vm_open() callback functional. This aligns with existing PCI_BARRIER, which will use this in the next patch. We don't want any noticeable behaviour change there, since this will be user visible. This will also be the first user. Assisted-by: LLM Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Tejas Upadhyay Cc: Matthew Brost Cc: Ilia Levi Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20260908165046.1393557-17-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_mmio_gem.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c index 5ffe03d36190..3cdc9538957d 100644 --- a/drivers/gpu/drm/xe/xe_mmio_gem.c +++ b/drivers/gpu/drm/xe/xe_mmio_gem.c @@ -177,9 +177,12 @@ static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct * if ((vma->vm_flags & VM_SHARED) == 0) return -EINVAL; + if (vma->vm_flags & VM_EXEC) + return -EINVAL; + vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma)); - vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | - VM_DONTCOPY | VM_NORESERVE); + vm_flags_mod(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | + VM_NORESERVE, VM_MAYEXEC); /* Defer actual mapping to the fault handler. */ return 0; From 73ae70cf843f93904bb02f8f213b50acfbb84ce7 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Tue, 8 Sep 2026 17:50:55 +0100 Subject: [PATCH 43/76] drm/xe: convert PCI barrier mmap to use xe_mmio_gem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the PCI barrier mmap over to use xe_mmio_gem, which is a good match for this functionality. This has the following advantages: 1. Removes a bunch of code. 2. Replaces the fragile hard coded fake offset design. 3. Adds the first user for xe_mmio_gem, which is preferred over nuking it. There are also potentially other upcoming usecases wanting this type of functionality, so having standard component to do this would be good. There shouldn't be any big functional change here. From userspace pov, they still query the fake offset like before, just that now it is no longer hard coded in the KMD. v2 (Thomas): - Prefer scoped_guard(). Also, just annotate ALL locations, even if not strictly needed. Reflect that in the kernel-doc. This will also shut up static analysis tools. Assisted-by: LLM Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Tejas Upadhyay Cc: Matthew Brost Cc: Ilia Levi Reviewed-by: Ilia Levi Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20260908165046.1393557-18-matthew.auld@intel.com --- drivers/gpu/drm/xe/xe_bo.c | 51 +++++++++---- drivers/gpu/drm/xe/xe_bo.h | 1 - drivers/gpu/drm/xe/xe_device.c | 106 +++------------------------ drivers/gpu/drm/xe/xe_device_types.h | 13 ++++ 4 files changed, 61 insertions(+), 110 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index b162753cebb7..9f3f0cb95afa 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -28,6 +28,7 @@ #include "xe_ggtt.h" #include "xe_map.h" #include "xe_migrate.h" +#include "xe_mmio_gem.h" #include "xe_pat.h" #include "xe_pm.h" #include "xe_preempt_fence.h" @@ -3664,6 +3665,39 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, return err; } +static int xe_gem_pci_barrier_mmap_offset(struct xe_device *xe, struct drm_file *file, + struct drm_xe_gem_mmap_offset *args) +{ + struct xe_file *xef = file->driver_priv; + struct xe_mmio_gem **barrier = &xef->mmio_gem.pci_barrier; + + if (XE_IOCTL_DBG(xe, !IS_DGFX(xe))) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, args->handle)) + return -EINVAL; + + scoped_guard(mutex, &xef->mmio_gem.lock) { + if (!*barrier) { + phys_addr_t phys_addr; + +#define LAST_DB_PAGE_OFFSET 0x7ff000 + phys_addr = pci_resource_start(to_pci_dev(xe->drm.dev), 0) + + LAST_DB_PAGE_OFFSET; + *barrier = xe_mmio_gem_create(xe, file, phys_addr, SZ_4K); + if (IS_ERR(*barrier)) { + int err = PTR_ERR(*barrier); + + *barrier = NULL; + return err; + } + } + + args->offset = xe_mmio_gem_mmap_offset(*barrier); + } + return 0; +} + int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { @@ -3679,21 +3713,8 @@ int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, ~DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER)) return -EINVAL; - if (args->flags & DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER) { - if (XE_IOCTL_DBG(xe, !IS_DGFX(xe))) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, args->handle)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, PAGE_SIZE > SZ_4K)) - return -EINVAL; - - BUILD_BUG_ON(((XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT) + - SZ_4K) >= DRM_FILE_PAGE_OFFSET_START); - args->offset = XE_PCI_BARRIER_MMAP_OFFSET; - return 0; - } + if (args->flags & DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER) + return xe_gem_pci_barrier_mmap_offset(xe, file, args); gem_obj = drm_gem_object_lookup(file, args->handle); if (XE_IOCTL_DBG(xe, !gem_obj)) diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index eede678ad303..290ca624e2a7 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -87,7 +87,6 @@ #define XE_BO_PROPS_INVALID (-1) -#define XE_PCI_BARRIER_MMAP_OFFSET (0x50 << XE_PTE_SHIFT) /** * enum xe_madv_purgeable_state - Buffer object purgeable state enumeration diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 8583b2e9ecf4..205cb4e7f9e8 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -50,6 +50,7 @@ #include "xe_late_bind_fw.h" #include "xe_log.h" #include "xe_mmio.h" +#include "xe_mmio_gem.h" #include "xe_module.h" #include "xe_nvm.h" #include "xe_oa.h" @@ -111,6 +112,8 @@ static int xe_file_open(struct drm_device *dev, struct drm_file *file) mutex_init(&xef->exec_queue.lock); xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); + mutex_init(&xef->mmio_gem.lock); + file->driver_priv = xef; kref_init(&xef->refcount); @@ -133,6 +136,8 @@ static void xe_file_destroy(struct kref *ref) xa_destroy(&xef->vm.xa); mutex_destroy(&xef->vm.lock); + mutex_destroy(&xef->mmio_gem.lock); + xe_drm_client_put(xef->client); kfree(xef->process_name); kfree(xef); @@ -189,6 +194,13 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file) xa_for_each(&xef->vm.xa, idx, vm) xe_vm_close_and_put(vm); + scoped_guard(mutex, &xef->mmio_gem.lock) { + if (xef->mmio_gem.pci_barrier) { + xe_mmio_gem_destroy(xef->mmio_gem.pci_barrier, file); + xef->mmio_gem.pci_barrier = NULL; + } + } + xe_file_put(xef); } @@ -258,95 +270,6 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo #define xe_drm_compat_ioctl NULL #endif -static void barrier_open(struct vm_area_struct *vma) -{ - drm_dev_get(vma->vm_private_data); -} - -static void barrier_close(struct vm_area_struct *vma) -{ - drm_dev_put(vma->vm_private_data); -} - -static void barrier_release_dummy_page(struct drm_device *dev, void *res) -{ - struct page *dummy_page = (struct page *)res; - - __free_page(dummy_page); -} - -static vm_fault_t barrier_fault(struct vm_fault *vmf) -{ - struct drm_device *dev = vmf->vma->vm_private_data; - struct vm_area_struct *vma = vmf->vma; - vm_fault_t ret = VM_FAULT_NOPAGE; - pgprot_t prot; - int idx; - - prot = vma_get_page_prot(vma); - - if (drm_dev_enter(dev, &idx)) { - unsigned long pfn; - -#define LAST_DB_PAGE_OFFSET 0x7ff001 - pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) + - LAST_DB_PAGE_OFFSET); - ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn, - pgprot_noncached(prot)); - drm_dev_exit(idx); - } else { - struct page *page; - - /* Allocate new dummy page to map all the VA range in this VMA to it*/ - page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) - return VM_FAULT_OOM; - - /* Set the page to be freed using drmm release action */ - if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page)) - return VM_FAULT_OOM; - - ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page), - prot); - } - - return ret; -} - -static const struct vm_operations_struct vm_ops_barrier = { - .open = barrier_open, - .close = barrier_close, - .fault = barrier_fault, -}; - -static int xe_pci_barrier_mmap(struct file *filp, - struct vm_area_struct *vma) -{ - struct drm_file *priv = filp->private_data; - struct drm_device *dev = priv->minor->dev; - struct xe_device *xe = to_xe_device(dev); - - if (!IS_DGFX(xe)) - return -EINVAL; - - if (vma->vm_end - vma->vm_start > SZ_4K) - return -EINVAL; - - if (vma_is_cow_mapping(vma)) - return -EINVAL; - - if (vma->vm_flags & (VM_READ | VM_EXEC)) - return -EINVAL; - - vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC); - vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); - vma->vm_ops = &vm_ops_barrier; - vma->vm_private_data = dev; - drm_dev_get(vma->vm_private_data); - - return 0; -} - static int xe_mmap(struct file *filp, struct vm_area_struct *vma) { struct drm_file *priv = filp->private_data; @@ -355,11 +278,6 @@ static int xe_mmap(struct file *filp, struct vm_area_struct *vma) if (drm_dev_is_unplugged(dev)) return -ENODEV; - switch (vma->vm_pgoff) { - case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT: - return xe_pci_barrier_mmap(filp, vma); - } - return drm_gem_mmap(filp, vma); } diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 180d450a6deb..f88bacf63c83 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -40,6 +40,7 @@ struct intel_display; struct intel_dg_nvm_dev; struct xe_ggtt; struct xe_i2c; +struct xe_mmio_gem; struct xe_pat_ops; struct xe_pxp; struct xe_ttm_stolen_mgr; @@ -676,6 +677,18 @@ struct xe_file { /** @refcount: ref count of this xe file */ struct kref refcount; + + /** @mmio_gem: MMIO GEM objects for this xe file */ + struct { + /** + * @mmio_gem.lock: Protects allocation and attach of MMIO + * GEM objects on first use (singleton). All MMIO GEM access + * should be guarded by this lock. Prefer scoped_guard(). + */ + struct mutex lock; + /** @mmio_gem.pci_barrier: MMIO GEM object for PCI barrier mmap. */ + struct xe_mmio_gem *pci_barrier; + } mmio_gem; }; #endif From 0bae1d5438abecaf688b2df7ac30edf0482b8d90 Mon Sep 17 00:00:00 2001 From: Anoop Vijay Date: Fri, 4 Sep 2026 06:49:36 -0700 Subject: [PATCH 44/76] drm/xe/sysctrl: Add helper to query application status Add xe_sysctrl_check_app_status() to query the state of a System Controller application using get_app_status_by_id mailbox command. The helper maps xe_sysctrl_app_id values to firmware application IDs and returns the reported application state. Add a convenience wrapper to check diag firmware application readiness. Signed-off-by: Anoop Vijay Reviewed-by: Michael J. Ruhl Reviewed-by: Umesh Nerlige Ramappa Signed-off-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20260904134935.674507-5-anoop.c.vijay@intel.com --- drivers/gpu/drm/xe/xe_sysctrl.c | 73 +++++++++++++++++++ drivers/gpu/drm/xe/xe_sysctrl.h | 1 + drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h | 45 ++++++++++++ 3 files changed, 119 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c index 62ccc9be71b4..3b7fa2c9c333 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl.c +++ b/drivers/gpu/drm/xe/xe_sysctrl.c @@ -13,9 +13,11 @@ #include "xe_device.h" #include "xe_mmio.h" #include "xe_pm.h" +#include "xe_printk.h" #include "xe_soc_remapper.h" #include "xe_sysctrl.h" #include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" #include "xe_sysctrl_types.h" /** @@ -29,6 +31,20 @@ * This module provides initialization and support code for interacting * with System Controller through the mailbox interface. */ + +/* Application status flags reported in xe_sysctrl_app_status_resp.flags */ +#define XE_SYSCTRL_APP_RESP_VALID BIT(0) +#define XE_SYSCTRL_APP_RESP_BOOTED BIT(1) +#define XE_SYSCTRL_APP_RESP_INITIALIZED BIT(2) + +/* + * Known System Controller application identifiers, keyed by firmware + * application ID. + */ +enum xe_sysctrl_app_id { + XE_SYSCTRL_APP_DIAG = 0x0D, +}; + static void sysctrl_fini(void *arg) { struct xe_device *xe = arg; @@ -125,3 +141,60 @@ void xe_sysctrl_pm_resume(struct xe_device *xe) xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); } + +static enum xe_sysctrl_fw_status +xe_sysctrl_check_app_status(struct xe_device *xe, enum xe_sysctrl_app_id app_id) +{ + struct xe_sysctrl_app_status_req req = {}; + struct xe_sysctrl_app_status_resp resp = {}; + struct xe_sysctrl_mailbox_command cmd = {}; + size_t out_len = 0; + u32 flags; + int ret; + + req.app_id = (u8)app_id; + + xe_sysctrl_create_command(&cmd, XE_SYSCTRL_GROUP_CORE, XE_SYSCTRL_CMD_GET_APP_STATUS_BY_ID, + &req, sizeof(req), &resp, sizeof(resp)); + + ret = xe_sysctrl_send_command(&xe->sc, &cmd, &out_len); + if (ret) + return XE_SYSCTRL_FIRMWARE_COMM_FAILURE; + + if (out_len != sizeof(resp)) { + xe_err(xe, "sysctrl: unexpected get app status response length %zu (expected %zu)\n", + out_len, sizeof(resp)); + return XE_SYSCTRL_FIRMWARE_COMM_FAILURE; + } + + flags = resp.flags; + + if (!(flags & XE_SYSCTRL_APP_RESP_VALID)) + return XE_SYSCTRL_FIRMWARE_APP_INVALID; + + if (!(flags & XE_SYSCTRL_APP_RESP_BOOTED)) + return XE_SYSCTRL_FIRMWARE_APP_NOT_LOADED; + + if (!(flags & XE_SYSCTRL_APP_RESP_INITIALIZED)) + return XE_SYSCTRL_FIRMWARE_APP_BOOTED; + + return XE_SYSCTRL_FIRMWARE_APP_INITIALIZED; +} + +/** + * xe_sysctrl_is_diag_fw_ready() - Check if diag firmware is fully initialized + * @xe: xe device instance + * + * Returns true if diag firmware has reached the initialized state, indicating + * it is ready to handle requests. + * + * Callers must only invoke this on platforms where System Controller is + * present (xe->info.has_sysctrl). + * + * Return: true if diag firmware is initialized, false otherwise + */ +bool xe_sysctrl_is_diag_fw_ready(struct xe_device *xe) +{ + return xe_sysctrl_check_app_status(xe, XE_SYSCTRL_APP_DIAG) == + XE_SYSCTRL_FIRMWARE_APP_INITIALIZED; +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl.h b/drivers/gpu/drm/xe/xe_sysctrl.h index 090dffb6d55f..8dc576796890 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl.h +++ b/drivers/gpu/drm/xe/xe_sysctrl.h @@ -20,5 +20,6 @@ void xe_sysctrl_event(struct xe_sysctrl *sc); int xe_sysctrl_init(struct xe_device *xe); void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl); void xe_sysctrl_pm_resume(struct xe_device *xe); +bool xe_sysctrl_is_diag_fw_ready(struct xe_device *xe); #endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h index 66e7cbcc3f91..c236e5377f30 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h @@ -14,9 +14,11 @@ * enum xe_sysctrl_group - System Controller command groups * * @XE_SYSCTRL_GROUP_GFSP: GFSP group + * @XE_SYSCTRL_GROUP_CORE: Core group */ enum xe_sysctrl_group { XE_SYSCTRL_GROUP_GFSP = 0x01, + XE_SYSCTRL_GROUP_CORE = 0xFF, }; /** @@ -42,6 +44,49 @@ enum xe_sysctrl_gfsp_cmd { XE_SYSCTRL_CMD_SET_HEALTH = 0x0C, }; +/** + * enum xe_sysctrl_core_cmd - Commands supported by Core group + * + * @XE_SYSCTRL_CMD_GET_APP_STATUS_BY_ID: Retrieve application status by ID + */ +enum xe_sysctrl_core_cmd { + XE_SYSCTRL_CMD_GET_APP_STATUS_BY_ID = 0x05, +}; + +/** + * struct xe_sysctrl_app_status_req - Get application status request + * + * @app_id: Application ID for which to retrieve status + */ +struct xe_sysctrl_app_status_req { + u8 app_id; +} __packed; + +/** + * struct xe_sysctrl_app_status_resp - Get application status response + * @flags: Application status flags interpreted by xe_sysctrl_check_app_status() + */ +struct xe_sysctrl_app_status_resp { + u32 flags; +} __packed; + +/** + * enum xe_sysctrl_fw_status - System Controller firmware application lifecycle states + * + * @XE_SYSCTRL_FIRMWARE_APP_INVALID: app_id is not recognized by firmware + * @XE_SYSCTRL_FIRMWARE_APP_NOT_LOADED: application is known but has not yet booted + * @XE_SYSCTRL_FIRMWARE_APP_BOOTED: boot sequence completed, post-boot init pending + * @XE_SYSCTRL_FIRMWARE_APP_INITIALIZED: application fully operational + * @XE_SYSCTRL_FIRMWARE_COMM_FAILURE: communication with System Controller firmware failed + */ +enum xe_sysctrl_fw_status { + XE_SYSCTRL_FIRMWARE_APP_INVALID, + XE_SYSCTRL_FIRMWARE_APP_NOT_LOADED, + XE_SYSCTRL_FIRMWARE_APP_BOOTED, + XE_SYSCTRL_FIRMWARE_APP_INITIALIZED, + XE_SYSCTRL_FIRMWARE_COMM_FAILURE, +}; + /** * struct xe_sysctrl_mailbox_command - System Controller mailbox command */ From c81d8cdf29b6f13a3c50f961d997995b985f66c6 Mon Sep 17 00:00:00 2001 From: Anoop Vijay Date: Fri, 4 Sep 2026 06:49:37 -0700 Subject: [PATCH 45/76] drm/xe/sysctrl: Add helper to check oCode firmware readiness Add xe_sysctrl_is_oobmsm_fw_ready(), a convenience wrapper around xe_sysctrl_check_app_status() to check oCode application readiness. Signed-off-by: Anoop Vijay Reviewed-by: Michael J. Ruhl Reviewed-by: Umesh Nerlige Ramappa Signed-off-by: Umesh Nerlige Ramappa Link: https://patch.msgid.link/20260904134935.674507-6-anoop.c.vijay@intel.com --- drivers/gpu/drm/xe/xe_sysctrl.c | 19 +++++++++++++++++++ drivers/gpu/drm/xe/xe_sysctrl.h | 1 + 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c index 3b7fa2c9c333..4067e1dfdcd5 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl.c +++ b/drivers/gpu/drm/xe/xe_sysctrl.c @@ -42,6 +42,7 @@ * application ID. */ enum xe_sysctrl_app_id { + XE_SYSCTRL_APP_OCODE = 0x0C, XE_SYSCTRL_APP_DIAG = 0x0D, }; @@ -181,6 +182,24 @@ xe_sysctrl_check_app_status(struct xe_device *xe, enum xe_sysctrl_app_id app_id) return XE_SYSCTRL_FIRMWARE_APP_INITIALIZED; } +/** + * xe_sysctrl_is_oobmsm_fw_ready() - Check if oCode firmware is fully initialized + * @xe: xe device instance + * + * Returns true if oCode firmware has reached the initialized state, indicating + * it is ready to handle requests. + * + * Callers must only invoke this on platforms where System Controller is + * present (xe->info.has_sysctrl). + * + * Return: true if oCode firmware is initialized, false otherwise + */ +bool xe_sysctrl_is_oobmsm_fw_ready(struct xe_device *xe) +{ + return xe_sysctrl_check_app_status(xe, XE_SYSCTRL_APP_OCODE) == + XE_SYSCTRL_FIRMWARE_APP_INITIALIZED; +} + /** * xe_sysctrl_is_diag_fw_ready() - Check if diag firmware is fully initialized * @xe: xe device instance diff --git a/drivers/gpu/drm/xe/xe_sysctrl.h b/drivers/gpu/drm/xe/xe_sysctrl.h index 8dc576796890..b69a3f474236 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl.h +++ b/drivers/gpu/drm/xe/xe_sysctrl.h @@ -20,6 +20,7 @@ void xe_sysctrl_event(struct xe_sysctrl *sc); int xe_sysctrl_init(struct xe_device *xe); void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl); void xe_sysctrl_pm_resume(struct xe_device *xe); +bool xe_sysctrl_is_oobmsm_fw_ready(struct xe_device *xe); bool xe_sysctrl_is_diag_fw_ready(struct xe_device *xe); #endif From d7aac1a0235a6ce41e30cec385e2db8c33dad12d Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Wed, 9 Sep 2026 16:21:01 +0000 Subject: [PATCH 46/76] drm/xe/shrinker: Return the freed page count through a parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit __xe_shrinker_walk() and xe_shrinker_walk() return either the number of pages freed or a negative error, so the two cannot be reported at once. On error the pages already freed are dropped, and since xe_shrinker_scan() only accumulates non-negative returns while *scanned is updated by pointer, the shrinker tells mm that it scanned without freeing. Accumulate the count into a caller-provided counter and return only the status, so an error no longer discards what the walk had freed. Fixes: 00c8efc3180f ("drm/xe: Add a shrinker for xe bos") Assisted-by: Claude:claude-opus-5 Reviewed-by: Thomas Hellström Cc: Matthew Brost Link: https://patch.msgid.link/20260909162102.1097006-2-shuicheng.lin@intel.com Signed-off-by: Shuicheng Lin --- drivers/gpu/drm/xe/xe_shrinker.c | 62 ++++++++++++++------------------ 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_shrinker.c b/drivers/gpu/drm/xe/xe_shrinker.c index 83374cd57660..89445cd20238 100644 --- a/drivers/gpu/drm/xe/xe_shrinker.c +++ b/drivers/gpu/drm/xe/xe_shrinker.c @@ -54,13 +54,14 @@ xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgea write_unlock(&shrinker->lock); } -static s64 __xe_shrinker_walk(struct xe_device *xe, +static int __xe_shrinker_walk(struct xe_device *xe, struct ttm_operation_ctx *ctx, const struct xe_bo_shrink_flags flags, - unsigned long to_scan, unsigned long *scanned) + unsigned long to_scan, unsigned long *scanned, + unsigned long *freed) { unsigned int mem_type; - s64 freed = 0, lret; + s64 lret; for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) { struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type); @@ -82,7 +83,7 @@ static s64 __xe_shrinker_walk(struct xe_device *xe, if (lret < 0) return lret; - freed += lret; + *freed += lret; if (*scanned >= to_scan) break; } @@ -90,7 +91,7 @@ static s64 __xe_shrinker_walk(struct xe_device *xe, xe_assert(xe, !IS_ERR(ttm_bo)); } - return freed; + return 0; } /* @@ -99,40 +100,35 @@ static s64 __xe_shrinker_walk(struct xe_device *xe, * add writeback. This avoids stalls and explicit writebacks with light or * moderate memory pressure. */ -static s64 xe_shrinker_walk(struct xe_device *xe, +static int xe_shrinker_walk(struct xe_device *xe, struct ttm_operation_ctx *ctx, const struct xe_bo_shrink_flags flags, - unsigned long to_scan, unsigned long *scanned) + unsigned long to_scan, unsigned long *scanned, + unsigned long *freed) { bool no_wait_gpu = true; struct xe_bo_shrink_flags save_flags = flags; - s64 lret, freed; + int ret; swap(no_wait_gpu, ctx->no_wait_gpu); save_flags.writeback = false; - lret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned); + ret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned, freed); swap(no_wait_gpu, ctx->no_wait_gpu); - if (lret < 0 || *scanned >= to_scan) - return lret; + if (ret || *scanned >= to_scan) + return ret; - freed = lret; if (!ctx->no_wait_gpu) { - lret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned); - if (lret < 0) - return lret; - freed += lret; - if (*scanned >= to_scan) - return freed; + ret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned, + freed); + if (ret || *scanned >= to_scan) + return ret; } - if (flags.writeback) { - lret = __xe_shrinker_walk(xe, ctx, flags, to_scan, scanned); - if (lret < 0) - return lret; - freed += lret; - } + if (flags.writeback) + ret = __xe_shrinker_walk(xe, ctx, flags, to_scan, scanned, + freed); - return freed; + return ret; } static unsigned long @@ -214,7 +210,6 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con bool runtime_pm; bool purgeable; bool can_backup = !!(sc->gfp_mask & __GFP_FS); - s64 lret; nr_to_scan = sc->nr_to_scan; @@ -225,12 +220,9 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con /* Might need runtime PM. Try to wake early if it looks like it. */ runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup); - if (purgeable && nr_scanned < nr_to_scan) { - lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, - nr_to_scan, &nr_scanned); - if (lret >= 0) - freed += lret; - } + if (purgeable && nr_scanned < nr_to_scan) + xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, + nr_to_scan, &nr_scanned, &freed); sc->nr_scanned = nr_scanned; if (nr_scanned >= nr_to_scan || !can_backup) @@ -242,10 +234,8 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con shrink_flags.purge = false; - lret = xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, - nr_to_scan, &nr_scanned); - if (lret >= 0) - freed += lret; + xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, + nr_to_scan, &nr_scanned, &freed); sc->nr_scanned = nr_scanned; out: From 628f92b28bf4c371c10207daf6fc4caee0c0db2e Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Wed, 9 Sep 2026 16:21:02 +0000 Subject: [PATCH 47/76] drm/xe/shrinker: Take a runtime PM ref before shrinking non-system memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit __xe_shrinker_walk() walks the SYSTEM and TT LRUs without a runtime PM reference. Shrinking a bo outside system memory invalidates its GPU mappings, which needs the device resumed, so while it is runtime suspended the page table zap trips an assert and the TLB invalidation returns -ENODEV: WARNING: drivers/gpu/drm/xe/xe_bo.c:770 at xe_bo_move_notify+0x1fc/0x450 [xe] xe_bo_shrink+0x20f/0x2b0 [xe] __xe_shrinker_walk+0x174/0x410 [xe] xe_shrinker_scan+0x10c/0x1e0 [xe] do_shrink_slab+0x176/0x7e0 drop_caches_sysctl_handler+0x9c/0xf0 Take a reference before walking a memory type other than XE_PL_SYSTEM and stop there if it cannot be acquired. Reuse the shrinker's existing acquire path, which resumes the device directly where reclaim allows that and otherwise queues the PM worker for a later scan. Stop the walk once the scan target is met, so a satisfied scan does not wake the device. System memory is still reclaimed while the device is suspended. Gate this on xe_device_is_l2_flush_optimized(), the same condition under which xe_bo_trigger_rebind() issues the invalidation for a non-fault-mode vm, so reclaim is unaffected elsewhere. The System CCS copy already has its own reference in xe_bo_shrink(). Only a non-fault-mode vm can reach this, since a fault-mode vm requires LR mode and that holds a runtime PM reference for the vm's lifetime. Reproduced with igt@xe_madvise@dontneed-before-exec while the GPU is runtime suspended. v2: simplify needs_rpm check. (Matt) retarget Fixes tag since the issue occurs with the non-fault-mode path added by 4e7ebff69aed. v3: handle this in xe_shrinker.c instead of xe_bo.c (Thomas) v4: stop the walk once the scan target is met. (Sashiko) v5: rebase on the freed page accounting fix. (Sashiko) v6: reuse the shrinker acquire path so runtime pm can be resumed directly instead of always queueing a worker. (Thomas) v7: replace xe_pm_runtime_put() with xe_shrinker_runtime_pm_put(). (Thomas) Fixes: 4e7ebff69aed ("drm/xe/xe3p_lpg: flush shrinker bo cachelines manually") Assisted-by: Claude:claude-opus-5 Cc: Tejas Upadhyay Cc: Matthew Brost Reviewed-by: Thomas Hellström Link: https://patch.msgid.link/20260909162102.1097006-3-shuicheng.lin@intel.com Signed-off-by: Shuicheng Lin --- drivers/gpu/drm/xe/xe_shrinker.c | 78 +++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_shrinker.c b/drivers/gpu/drm/xe/xe_shrinker.c index 89445cd20238..deb4378c1ec1 100644 --- a/drivers/gpu/drm/xe/xe_shrinker.c +++ b/drivers/gpu/drm/xe/xe_shrinker.c @@ -54,13 +54,39 @@ xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgea write_unlock(&shrinker->lock); } -static int __xe_shrinker_walk(struct xe_device *xe, +static bool __xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker) +{ + struct xe_device *xe = shrinker->xe; + + if (xe_pm_runtime_get_if_active(xe)) + return true; + + if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) { + xe_pm_runtime_get(xe); + return true; + } + + queue_work(xe->unordered_wq, &shrinker->pm_worker); + + return false; +} + +static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm) +{ + if (runtime_pm) + xe_pm_runtime_put(shrinker->xe); +} + +static int __xe_shrinker_walk(struct xe_shrinker *shrinker, struct ttm_operation_ctx *ctx, const struct xe_bo_shrink_flags flags, unsigned long to_scan, unsigned long *scanned, unsigned long *freed) { + struct xe_device *xe = shrinker->xe; unsigned int mem_type; + bool rpm = false; + int ret = 0; s64 lret; for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) { @@ -75,23 +101,35 @@ static int __xe_shrinker_walk(struct xe_device *xe, if (!man || !man->use_tt) continue; + if (mem_type != XE_PL_SYSTEM && !rpm && + xe_device_is_l2_flush_optimized(xe)) { + if (!__xe_shrinker_runtime_pm_get(shrinker)) + break; + rpm = true; + } + ttm_bo_lru_for_each_reserved_guarded(&curs, man, &arg, ttm_bo) { if (!ttm_bo_shrink_suitable(ttm_bo, ctx)) continue; lret = xe_bo_shrink(ctx, ttm_bo, flags, scanned); - if (lret < 0) - return lret; + if (lret < 0) { + ret = lret; + goto out; + } *freed += lret; if (*scanned >= to_scan) - break; + goto out; } /* Trylocks should never error, just fail. */ xe_assert(xe, !IS_ERR(ttm_bo)); } - return 0; +out: + xe_shrinker_runtime_pm_put(shrinker, rpm); + + return ret; } /* @@ -100,7 +138,7 @@ static int __xe_shrinker_walk(struct xe_device *xe, * add writeback. This avoids stalls and explicit writebacks with light or * moderate memory pressure. */ -static int xe_shrinker_walk(struct xe_device *xe, +static int xe_shrinker_walk(struct xe_shrinker *shrinker, struct ttm_operation_ctx *ctx, const struct xe_bo_shrink_flags flags, unsigned long to_scan, unsigned long *scanned, @@ -112,20 +150,21 @@ static int xe_shrinker_walk(struct xe_device *xe, swap(no_wait_gpu, ctx->no_wait_gpu); save_flags.writeback = false; - ret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned, freed); + ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned, + freed); swap(no_wait_gpu, ctx->no_wait_gpu); if (ret || *scanned >= to_scan) return ret; if (!ctx->no_wait_gpu) { - ret = __xe_shrinker_walk(xe, ctx, save_flags, to_scan, scanned, + ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned, freed); if (ret || *scanned >= to_scan) return ret; } if (flags.writeback) - ret = __xe_shrinker_walk(xe, ctx, flags, to_scan, scanned, + ret = __xe_shrinker_walk(shrinker, ctx, flags, to_scan, scanned, freed); return ret; @@ -176,22 +215,7 @@ static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force, return false; } - if (!xe_pm_runtime_get_if_active(xe)) { - if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) { - xe_pm_runtime_get(xe); - return true; - } - queue_work(xe->unordered_wq, &shrinker->pm_worker); - return false; - } - - return true; -} - -static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm) -{ - if (runtime_pm) - xe_pm_runtime_put(shrinker->xe); + return __xe_shrinker_runtime_pm_get(shrinker); } static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) @@ -221,7 +245,7 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup); if (purgeable && nr_scanned < nr_to_scan) - xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, + xe_shrinker_walk(shrinker, &ctx, shrink_flags, nr_to_scan, &nr_scanned, &freed); sc->nr_scanned = nr_scanned; @@ -234,7 +258,7 @@ static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_con shrink_flags.purge = false; - xe_shrinker_walk(shrinker->xe, &ctx, shrink_flags, + xe_shrinker_walk(shrinker, &ctx, shrink_flags, nr_to_scan, &nr_scanned, &freed); sc->nr_scanned = nr_scanned; From 8721a51d44733b8b5cb41fa1bfe7fac2cf55107b Mon Sep 17 00:00:00 2001 From: Karthik Poosa Date: Mon, 7 Sep 2026 14:05:19 +0530 Subject: [PATCH 48/76] drm/xe/pcode: Increase default pcode mailbox timeout to 10ms The previous 1ms timeout may be insufficient for some mailbox commands, leading to spurious -ETIMEDOUT failures under load. Increase the default timeout to 10ms and introduce PCODE_DEFAULT_TIMEOUT_MS for a shared timeout definition, in line with pcode mailbox specification guidance. Signed-off-by: Karthik Poosa Link: https://patch.msgid.link/20260907083520.192418-2-karthik.poosa@intel.com Reviewed-by: Rodrigo Vivi Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_pcode.c | 2 +- drivers/gpu/drm/xe/xe_pcode.h | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pcode.c b/drivers/gpu/drm/xe/xe_pcode.c index d502205bb72a..266deecbb100 100644 --- a/drivers/gpu/drm/xe/xe_pcode.c +++ b/drivers/gpu/drm/xe/xe_pcode.c @@ -140,7 +140,7 @@ int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val0, u32 *val1) int err; mutex_lock(&tile->pcode.lock); - err = pcode_mailbox_rw(tile, mbox, val0, val1, 1, true, false); + err = pcode_mailbox_rw(tile, mbox, val0, val1, PCODE_DEFAULT_TIMEOUT_MS, true, false); mutex_unlock(&tile->pcode.lock); return err; diff --git a/drivers/gpu/drm/xe/xe_pcode.h b/drivers/gpu/drm/xe/xe_pcode.h index ba8a1d1b2152..7e43792b0037 100644 --- a/drivers/gpu/drm/xe/xe_pcode.h +++ b/drivers/gpu/drm/xe/xe_pcode.h @@ -18,6 +18,8 @@ struct xe_pcode_version { u32 engg; }; +#define PCODE_DEFAULT_TIMEOUT_MS 10 + int xe_pcode_init_early(struct xe_tile *tile); int xe_pcode_probe_early(struct xe_device *xe); int xe_pcode_ready(struct xe_device *xe, bool locked); @@ -31,7 +33,7 @@ int xe_pcode_write64_timeout(struct xe_tile *tile, u32 mbox, u32 data0, int xe_get_pcode_version(struct xe_device *xe, struct xe_pcode_version *version); #define xe_pcode_write(tile, mbox, val) \ - xe_pcode_write_timeout(tile, mbox, val, 1) + xe_pcode_write_timeout(tile, mbox, val, PCODE_DEFAULT_TIMEOUT_MS) int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request, u32 reply_mask, u32 reply, int timeout_ms); From 1a925ad51847db376fc0acd21fc5b76fb9c36cde Mon Sep 17 00:00:00 2001 From: Karthik Poosa Date: Mon, 7 Sep 2026 14:05:20 +0530 Subject: [PATCH 49/76] drm/xe/hwmon: Use shared pcode default timeout for power limit write Drop the local PL_WRITE_MBX_TIMEOUT_MS macro and use the common PCODE_DEFAULT_TIMEOUT_MS for the power limit write mailbox command. This removes a duplicate timeout definition and keeps hwmon in sync with the pcode default (now 10ms). Signed-off-by: Karthik Poosa Link: https://patch.msgid.link/20260907083520.192418-3-karthik.poosa@intel.com Reviewed-by: Rodrigo Vivi Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_hwmon.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index 5284cab6703d..5edeac961ec3 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -101,11 +101,6 @@ enum sensor_attr_power { #define PWR_ATTR_TO_STR(attr) (((attr) == hwmon_power_max) ? "PL1" : "PL2") -/* - * Timeout for power limit write mailbox command. - */ -#define PL_WRITE_MBX_TIMEOUT_MS (1) - /* Index of memory controller in READ_THERMAL_DATA output */ #define TEMP_INDEX_MCTRL 2 @@ -252,7 +247,7 @@ static int xe_hwmon_pcode_rmw_power_limit(const struct xe_hwmon *hwmon, u32 attr (channel == CHANNEL_CARD) ? WRITE_PSYSGPU_POWER_LIMIT : WRITE_PACKAGE_POWER_LIMIT, 0), - val0, val1, PL_WRITE_MBX_TIMEOUT_MS); + val0, val1, PCODE_DEFAULT_TIMEOUT_MS); if (ret) drm_dbg(&hwmon->xe->drm, "write failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n", channel, val0, val1, ret); From 9fd3b392bdf37e7135692c89303681e8d9e899be Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Wed, 9 Sep 2026 16:46:21 +0530 Subject: [PATCH 50/76] drm/xe: Skip clearing purged page-table BOs VRAM page offlining can clear a page-table BO's vmap before VM teardown calls xe_pt_clear(). Check for a purged BO under its dma-resv lock before writing to the mapping. v4(Sashiko): - dont leave svm notifier lock early v3(Sashiko): - Validate NULL vmap v2(Sashiko): - Fix lock ordering Reviewed-by: Matthew Brost Link: https://patch.msgid.link/20260909111620.1276402-2-tejas.upadhyay@intel.com Signed-off-by: Tejas Upadhyay --- drivers/gpu/drm/xe/xe_pt.c | 6 ++++-- drivers/gpu/drm/xe/xe_vm.c | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 7fb2fe1f826c..4b351dbf6572 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -236,9 +236,11 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) */ void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt) { - struct iosys_map *map = &pt->bo->vmap; + struct xe_bo *bo = pt->bo; - xe_map_memset(xe, map, 0, 0, SZ_4K); + xe_bo_assert_held(bo); + if (!iosys_map_is_null(&bo->vmap)) + xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K); } /** diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 753a5fc55baa..cc96e2b7f3a7 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1876,6 +1876,8 @@ static void xe_vm_close(struct xe_vm *vm) bound = drm_dev_enter(&xe->drm, &idx); down_write(&vm->lock); + xe_vm_lock(vm, false); + if (xe_vm_in_fault_mode(vm)) xe_svm_notifier_lock(vm); @@ -1903,6 +1905,8 @@ static void xe_vm_close(struct xe_vm *vm) if (xe_vm_in_fault_mode(vm)) xe_svm_notifier_unlock(vm); + + xe_vm_unlock(vm); up_write(&vm->lock); if (bound) From d9a31731de0c29e8e0feb71e2113d10cbd55df54 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Sat, 5 Sep 2026 22:01:56 +0200 Subject: [PATCH 51/76] drm/xe/tests: Fix default subplatform selection If a test does not specify details about the fake device it wants to create, our helper selects the device descriptor from the first entry from the pciidlist and uses NULL for the subplatform descriptor. While this works fine today, as the first device is a TGL which does not have any subplatforms, it will be broken if we reorder pciidlist. Signed-off-by: Michal Wajdeczko Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260905200157.3164-2-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/tests/xe_pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index bb0393475524..a82cab7a2b86 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -351,7 +351,7 @@ int xe_pci_fake_device_init(struct xe_device *xe) if (!data) { desc = (const void *)ent->driver_data; - subplatform_desc = NULL; + subplatform_desc = desc->subplatforms; goto done; } From 51c3ac1665265bf107d111927d242f0799ba11f8 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Sat, 5 Sep 2026 22:01:57 +0200 Subject: [PATCH 52/76] drm/xe/tests: Fix subplatform selection If a test does not specify subplatform details about the fake device it wants to create, our helper may select a sentinel of the subplatform descriptors list instead of the first available subplatform. Signed-off-by: Michal Wajdeczko Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260905200157.3164-3-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/tests/xe_pci.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index a82cab7a2b86..8ab4270cf5d8 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -364,13 +364,23 @@ int xe_pci_fake_device_init(struct xe_device *xe) if (!ent->device) return -ENODEV; + if (data->subplatform == XE_SUBPLATFORM_NONE) { + subplatform_desc = NULL; + goto done; + } + + if (data->subplatform == XE_SUBPLATFORM_UNINITIALIZED) { + subplatform_desc = desc->subplatforms; + goto done; + } + for (subplatform_desc = desc->subplatforms; subplatform_desc && subplatform_desc->subplatform; subplatform_desc++) if (subplatform_desc->subplatform == data->subplatform) break; - if (data->subplatform != XE_SUBPLATFORM_NONE && !subplatform_desc) + if (!subplatform_desc || !subplatform_desc->subplatform) return -ENODEV; done: From 424e8b5e61aadcfcc54c6ee27064471b46268116 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 4 Sep 2026 19:05:25 +0200 Subject: [PATCH 53/76] drm/xe: Drop unused parameter from xe_info_init We don't need anything from struct xe_device_desc in xe_info_init() Signed-off-by: Michal Wajdeczko Reviewed-by: Rodrigo Vivi Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260904170531.516-2-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/tests/xe_pci.c | 2 +- drivers/gpu/drm/xe/xe_pci.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index 8ab4270cf5d8..ab00d0e80a3a 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -392,7 +392,7 @@ int xe_pci_fake_device_init(struct xe_device *xe) return err; xe_info_init_early(xe, desc, subplatform_desc, &probed_info); - xe_info_init(xe, desc, &probed_info); + xe_info_init(xe, &probed_info); return 0; } diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 1e04e8ef2611..2d89977978d0 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -1010,7 +1010,6 @@ static int xe_probe_info(struct xe_device *xe, * present in device info. */ static int xe_info_init(struct xe_device *xe, - const struct xe_device_desc *desc, struct xe_probed_info *probed_info) { const struct xe_ip *graphics_ip; @@ -1243,7 +1242,7 @@ static int __xe_pci_probe(struct pci_dev *pdev, const struct xe_device_desc *des if (err) return err; - err = xe_info_init(xe, desc, &probed_info); + err = xe_info_init(xe, &probed_info); if (err) return err; From 3a7df8a43c783ed13bff8de9848fc179107727ea Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 4 Sep 2026 19:05:26 +0200 Subject: [PATCH 54/76] drm/xe: Keep reference to device descriptor While we already use struct intel_device_info, which has some fields copied or derived from the original struct xe_device_desc, we also want to keep the reference to the whole device descriptor, in case we would like to access more non-mutable device parameters. Similarly keep the reference to matched subplatform descriptor. Signed-off-by: Michal Wajdeczko Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260904170531.516-3-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/tests/xe_pci.c | 2 ++ drivers/gpu/drm/xe/xe_device_types.h | 6 ++++++ drivers/gpu/drm/xe/xe_pci.c | 2 ++ 3 files changed, 10 insertions(+) diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index ab00d0e80a3a..a0a08ee5f12b 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -384,6 +384,8 @@ int xe_pci_fake_device_init(struct xe_device *xe) return -ENODEV; done: + xe->desc = desc; + xe->subplatform_desc = subplatform_desc; xe->sriov.__mode = data && data->sriov_mode ? data->sriov_mode : XE_SRIOV_MODE_NONE; diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index f88bacf63c83..4661bfce2f4e 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -117,6 +117,12 @@ struct xe_device { /** @devcoredump: device coredump */ struct xe_devcoredump devcoredump; + /** @desc: device descriptor */ + const struct xe_device_desc *desc; + + /** @subplatform_desc: subplatform descriptor */ + const struct xe_subplatform_desc *subplatform_desc; + /** @info: device info */ struct intel_device_info { /** @info.platform_name: platform name */ diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 2d89977978d0..4fab47369c64 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -1207,6 +1207,8 @@ static int __xe_pci_probe(struct pci_dev *pdev, const struct xe_device_desc *des if (IS_ERR(xe)) return PTR_ERR(xe); + xe->desc = desc; + xe->subplatform_desc = subplatform_desc; xe->devres_group = group; pci_set_drvdata(pdev, &xe->drm); From e98e6b57dad5f7d072d85595d2138c84859bf5f0 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 4 Sep 2026 19:05:27 +0200 Subject: [PATCH 55/76] drm/xe: Drop redundant parameters from xe_info_init_early We can now obtain the const pointers to struct xe_device_desc and xe_subplatform_desc directly from the xe, no need to pass them as explicit parameters. Signed-off-by: Michal Wajdeczko Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260904170531.516-4-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/tests/xe_pci.c | 2 +- drivers/gpu/drm/xe/xe_pci.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index a0a08ee5f12b..ca833f1f3cce 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -393,7 +393,7 @@ int xe_pci_fake_device_init(struct xe_device *xe) if (err) return err; - xe_info_init_early(xe, desc, subplatform_desc, &probed_info); + xe_info_init_early(xe, &probed_info); xe_info_init(xe, &probed_info); return 0; diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 4fab47369c64..23edaf886d23 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -770,10 +770,10 @@ static int xe_probe_info_early(struct xe_device *xe, * passed to the driver at probe time from PCI ID table. */ static int xe_info_init_early(struct xe_device *xe, - const struct xe_device_desc *desc, - const struct xe_subplatform_desc *subplatform_desc, struct xe_probed_info *probed_info) { + const struct xe_subplatform_desc *subplatform_desc = xe->subplatform_desc; + const struct xe_device_desc *desc = xe->desc; int err; xe->info.devid = probed_info->devid; @@ -1221,7 +1221,7 @@ static int __xe_pci_probe(struct pci_dev *pdev, const struct xe_device_desc *des if (err) return err; - err = xe_info_init_early(xe, desc, subplatform_desc, &probed_info); + err = xe_info_init_early(xe, &probed_info); if (err) return err; From 31aa5fdb01c8e0ceb6a0ba84dde589b1db9b40b2 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 4 Sep 2026 19:05:28 +0200 Subject: [PATCH 56/76] drm/xe: Drop redundant parameter from xe_probe_info_early We can now obtain the const pointer to struct xe_device_desc directly from the xe, no need to pass it as explicit parameter. Signed-off-by: Michal Wajdeczko Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260904170531.516-5-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_pci.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 23edaf886d23..b429cd9c48a7 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -752,7 +752,6 @@ struct xe_probed_info { * Probe from the hardware the info required by xe_info_init_early(). */ static int xe_probe_info_early(struct xe_device *xe, - const struct xe_device_desc *desc, struct xe_probed_info *probed_info) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); @@ -760,7 +759,7 @@ static int xe_probe_info_early(struct xe_device *xe, probed_info->devid = pdev->device; probed_info->revid = pdev->revision; - xe_step_platform_get(desc->platform, probed_info->revid, &probed_info->step); + xe_step_platform_get(xe->desc->platform, probed_info->revid, &probed_info->step); return 0; } @@ -1217,7 +1216,7 @@ static int __xe_pci_probe(struct pci_dev *pdev, const struct xe_device_desc *des pci_set_master(pdev); - err = xe_probe_info_early(xe, desc, &probed_info); + err = xe_probe_info_early(xe, &probed_info); if (err) return err; From a158eba2fb0d82b7a11f316c6b4065d5861ba0dd Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 4 Sep 2026 19:05:29 +0200 Subject: [PATCH 57/76] drm/xe: Drop redundant parameter from xe_probe_info and friends We can now obtain the const pointer to struct xe_device_desc directly from the xe, no need to pass it as explicit parameter. Signed-off-by: Michal Wajdeczko Reviewed-by: Gustavo Sousa Link: https://patch.msgid.link/20260904170531.516-6-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/tests/xe_pci.c | 5 +++-- drivers/gpu/drm/xe/xe_pci.c | 13 ++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index ca833f1f3cce..5aefc03c00c7 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -311,10 +311,11 @@ const void *xe_pci_id_gen_param(struct kunit *test, const void *prev, char *desc EXPORT_SYMBOL_IF_KUNIT(xe_pci_id_gen_param); static int fake_probe_info(struct xe_device *xe, - const struct xe_device_desc *desc, struct xe_pci_fake_data *data, struct xe_probed_info *probed_info) { + const struct xe_device_desc *desc = xe->desc; + probed_info->tile_count = 1 + desc->max_remote_tiles; if (!data || desc->pre_gmdid_graphics_ip) { @@ -389,7 +390,7 @@ int xe_pci_fake_device_init(struct xe_device *xe) xe->sriov.__mode = data && data->sriov_mode ? data->sriov_mode : XE_SRIOV_MODE_NONE; - err = fake_probe_info(xe, desc, data, &probed_info); + err = fake_probe_info(xe, data, &probed_info); if (err) return err; diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index b429cd9c48a7..00b2b5dfb8f6 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -835,14 +835,13 @@ static int xe_info_init_early(struct xe_device *xe, } static void xe_probe_tile_count(struct xe_device *xe, - const struct xe_device_desc *desc, struct xe_probed_info *probed_info) { struct xe_mmio *mmio; u8 tile_count; u32 mtcfg; - probed_info->tile_count = 1 + desc->max_remote_tiles; + probed_info->tile_count = 1 + xe->desc->max_remote_tiles; /* * Probe for tile count only for platforms that support multiple @@ -945,9 +944,10 @@ static struct xe_gt *alloc_media_gt(struct xe_tile *tile, } static int xe_probe_ips(struct xe_device *xe, - const struct xe_device_desc *desc, struct xe_probed_info *probed_info) { + const struct xe_device_desc *desc = xe->desc; + /* * If this platform supports GMD_ID, we'll detect the proper IP * descriptor to use from hardware registers. @@ -988,14 +988,13 @@ static int xe_probe_ips(struct xe_device *xe, * Probe from the hardware the info required by xe_info_init(). */ static int xe_probe_info(struct xe_device *xe, - const struct xe_device_desc *desc, struct xe_probed_info *probed_info) { int err; - xe_probe_tile_count(xe, desc, probed_info); + xe_probe_tile_count(xe, probed_info); - err = xe_probe_ips(xe, desc, probed_info); + err = xe_probe_ips(xe, probed_info); if (err) return err; @@ -1239,7 +1238,7 @@ static int __xe_pci_probe(struct pci_dev *pdev, const struct xe_device_desc *des if (err) return err; - err = xe_probe_info(xe, desc, &probed_info); + err = xe_probe_info(xe, &probed_info); if (err) return err; From 7dbd24e9d5311b6afdbdd875dd247ab5ff54edca Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 4 Sep 2026 19:05:30 +0200 Subject: [PATCH 58/76] drm/xe/log: Relax location ID recognition It turned out that during early probe phase, VFs use detached from the xe_device, temporary xe_gt objects, which when used as location in xe_log() macros, will be treated by the dmesg decoration code as bogus, possibly triggering a WARN, and the output will look like: [drm] *ERROR* SIGID=104 (-ETIMEDOUT) LOC3.0? GUC: MMIO request ... instead of expected: [drm] *ERROR* SIGID=104 (-ETIMEDOUT) Tile0: GT0: GUC: MMIO request ... Relax the tile/GT id validation and instead of looking for the real objects, only check if encoded id is within the range of possible tiles or GTs on the current xe device, using data from the device descriptor rather then the object list. Fixes: 1151b9f6f465 ("drm/xe/log: Add component/location decorations to dmesg") Signed-off-by: Michal Wajdeczko Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20260904170531.516-7-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_log.c | 42 ++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_log.c b/drivers/gpu/drm/xe/xe_log.c index 5549ef6966fd..29eb16db3320 100644 --- a/drivers/gpu/drm/xe/xe_log.c +++ b/drivers/gpu/drm/xe/xe_log.c @@ -10,6 +10,7 @@ #include "xe_device.h" #include "xe_log.h" +#include "xe_pci_types.h" #include "xe_printk.h" static void log_emit_cper(struct pci_dev *pdev, int cper_sev, enum xe_sigid sigid, @@ -52,18 +53,24 @@ static const char *log_component_prefix(u32 component) return component ? log_unknown_component_prefix(component) : ""; } -static struct xe_gt *get_gt_safe(struct pci_dev *pdev, u8 id) +static bool allowed_tile_id(struct xe_device *xe, u8 tile_id) { - struct xe_device *xe = pdev_to_xe_device(pdev); - - return xe ? xe_device_get_gt(xe, id) : NULL; + return tile_id < 1 + xe->desc->max_remote_tiles; } -static struct xe_tile *get_tile_safe(struct pci_dev *pdev, u8 id) +static bool allowed_gt_id(struct xe_device *xe, u8 gt_id) { - struct xe_device *xe = pdev_to_xe_device(pdev); + return gt_id < (1 + xe->desc->max_remote_tiles) * xe->desc->max_gt_per_tile; +} - return xe && id < xe->info.tile_count ? &xe->tiles[id] : NULL; +static u8 gt_id_to_tile_id(struct xe_device *xe, u8 gt_id) +{ + return gt_id / xe->desc->max_gt_per_tile; +} + +static const char *location_suffix(bool valid) +{ + return valid ? ":" : "?"; } static const char *log_location_prefix(struct pci_dev *pdev, u32 location, char *buf, size_t size) @@ -76,17 +83,22 @@ static const char *log_location_prefix(struct pci_dev *pdev, u32 location, char goto unrecognized; strscpy(buf, "", size); } else if (type == XE_LOG_LOCATION_TYPE_TILE) { - struct xe_tile *tile = get_tile_safe(pdev, id); + struct xe_device *xe = xe_any_to_xe(pdev); + bool valid = xe ? allowed_tile_id(xe, id) : false; + const char *pad = location_suffix(valid); - if (!tile) - goto unrecognized; - snprintf(buf, size, "Tile%u: ", id); + pci_WARN(pdev, !valid && IS_ENABLED(CONFIG_DRM_XE_DEBUG), + "LOG: invalid tile identifier: %u\n", id); + snprintf(buf, size, "Tile%u%s ", id, pad); } else if (type == XE_LOG_LOCATION_TYPE_GT) { - struct xe_gt *gt = get_gt_safe(pdev, id); + struct xe_device *xe = xe_any_to_xe(pdev); + bool valid = xe ? allowed_gt_id(xe, id) : false; + const char *pad = location_suffix(valid); + u8 tile_id = xe ? gt_id_to_tile_id(xe, id) : 0; - if (!gt) - goto unrecognized; - snprintf(buf, size, "Tile%u: GT%u: ", gt->tile->id, id); + pci_WARN(pdev, !valid && IS_ENABLED(CONFIG_DRM_XE_DEBUG), + "LOG: invalid GT identifier: %u\n", id); + snprintf(buf, size, "Tile%u%s GT%u%s ", tile_id, pad, id, pad); } else { goto unrecognized; } From 8ba5c8b8ab3fd362267c11df2cd5a90ee46f6e24 Mon Sep 17 00:00:00 2001 From: Raag Jadav Date: Fri, 11 Sep 2026 17:45:47 +0530 Subject: [PATCH 59/76] drm/xe/i2c: Disable IRQ on unbind Currently, struct xe_i2c is freed before SGUnit IRQ is disabled in unbind path, leaving a potential UAF in case I2C IRQ is hit during this small window. Explicitly disable I2C IRQ in xe_i2c_remove() and fix this. Fixes: 0bb78ce09926 ("drm/xe/i2c: Wire up reset/postinstall for I2C IRQ") Signed-off-by: Raag Jadav Reviewed-by: Heikki Krogerus Link: https://patch.msgid.link/20260911121547.2407261-1-raag.jadav@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/xe_i2c.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index d8fa68206f41..51ca9fb73b70 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -324,8 +324,10 @@ void xe_i2c_pm_resume(struct xe_device *xe, bool d3cold) static void xe_i2c_remove(void *data) { struct xe_i2c *i2c = data; + struct xe_device *xe = tile_to_xe(i2c->mmio->tile); unsigned int i; + xe_i2c_irq_reset(xe); xe_amc_exit(i2c); for (i = 0; i < XE_I2C_MAX_CLIENTS; i++) { @@ -335,6 +337,7 @@ static void xe_i2c_remove(void *data) bus_unregister_notifier(&i2c_bus_type, &i2c->bus_notifier); xe_i2c_unregister_adapter(i2c); + xe->i2c = NULL; } /** From 10628c52a3732a10426499a3d462cc2e6bc371ae Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 11 Sep 2026 20:23:06 +0200 Subject: [PATCH 60/76] drm/xe/pf: Refactor VF LMEM BAR resize helper function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve and move diagnostics messages to the helper function to keep the caller function tidy. Signed-off-by: Michal Wajdeczko Reviewed-by: Michał Winiarski Link: https://patch.msgid.link/20260911182306.14973-1-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_pci_sriov.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index 35e6b53e18ce..c023a2a1fc17 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -82,16 +82,33 @@ static void pf_engine_activity_stats(struct xe_device *xe, unsigned int num_vfs, } } -static int resize_vf_vram_bar(struct xe_device *xe, int num_vfs) +static int pf_resize_vf_vram_bar(struct xe_device *xe, int num_vfs) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + char buf[10]; u32 sizes; + int size; + int err; + + if (!IS_DGFX(xe)) + return 0; sizes = pci_iov_vf_bar_get_sizes(pdev, VF_LMEM_BAR, num_vfs); if (!sizes) return 0; - return pci_iov_vf_bar_set_size(pdev, VF_LMEM_BAR, __fls(sizes)); + size = __fls(sizes); + string_get_size(pci_rebar_size_to_bytes(size), 1, STRING_UNITS_2, buf, sizeof(buf)); + + err = pci_iov_vf_bar_set_size(pdev, VF_LMEM_BAR, size); + if (err) { + xe_sriov_notice(xe, "Failed to resize VF BAR %u to %s (%pe)\n", + VF_LMEM_BAR, buf, ERR_PTR(err)); + return err; + } + + xe_sriov_dbg(xe, "VF LMEM BAR resized to %s\n", buf); + return 0; } static int pf_prepare_vfs_enabling(struct xe_device *xe) @@ -142,11 +159,7 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs) if (err < 0) goto failed; - if (IS_DGFX(xe)) { - err = resize_vf_vram_bar(xe, num_vfs); - if (err) - xe_sriov_info(xe, "Failed to set VF LMEM BAR size: %d\n", err); - } + pf_resize_vf_vram_bar(xe, num_vfs); err = pci_enable_sriov(pdev, num_vfs); if (err < 0) From 8968890fcccb6032b1f85afa73850f9b690ee350 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 11 Sep 2026 20:32:55 +0200 Subject: [PATCH 61/76] drm/xe/pf: Add _locked variant of the priority config function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will soon want to provision VFs in a single step, while already holding the master mutex. Split existing function into two. Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Link: https://patch.msgid.link/20260911183257.14999-2-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 34 +++++++++++++++++----- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h | 2 ++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index be0a413ee17c..8de8f42ed4d0 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -2622,6 +2622,30 @@ static u32 pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid) return config->sched_priority; } +/** + * xe_gt_sriov_pf_config_set_sched_priority_locked() - Configure scheduling priority. + * @gt: the &xe_gt + * @vfid: the VF identifier + * @priority: requested scheduling priority + * + * This function can only be called on PF with the master mutex hold. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_sched_priority_locked(struct xe_gt *gt, unsigned int vfid, + u32 priority) +{ + int err; + + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + err = pf_provision_sched_priority(gt, vfid, priority); + + return pf_config_set_u32_done(gt, vfid, priority, + pf_get_sched_priority(gt, vfid), + "scheduling priority", sched_priority_unit, err); +} + /** * xe_gt_sriov_pf_config_set_sched_priority() - Configure scheduling priority. * @gt: the &xe_gt @@ -2634,15 +2658,9 @@ static u32 pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid) */ int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority) { - int err; + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); - mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); - err = pf_provision_sched_priority(gt, vfid, priority); - mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); - - return pf_config_set_u32_done(gt, vfid, priority, - xe_gt_sriov_pf_config_get_sched_priority(gt, vfid), - "scheduling priority", sched_priority_unit, err); + return xe_gt_sriov_pf_config_set_sched_priority_locked(gt, vfid, priority); } /** diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h index a56e63f3660a..84246978d21e 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h @@ -77,6 +77,8 @@ int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid); int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority); +int xe_gt_sriov_pf_config_set_sched_priority_locked(struct xe_gt *gt, unsigned int vfid, + u32 priority); void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority); u32 xe_gt_sriov_pf_config_get_threshold(struct xe_gt *gt, unsigned int vfid, From d38c9c9437c802724f0dd1a37ed7fb8f0e34f2fd Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 11 Sep 2026 20:32:56 +0200 Subject: [PATCH 62/76] drm/xe/pf: Add _locked variant of the release config function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will soon want to unprovision all VFs in a single step, while already holding the master mutex. Split existing function into two. Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Link: https://patch.msgid.link/20260911183257.14999-3-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 51 ++++++++++++++-------- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h | 1 + 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 8de8f42ed4d0..146981aedf5f 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -2932,7 +2932,38 @@ static void pf_release_vf_config(struct xe_gt *gt, unsigned int vfid) } /** - * xe_gt_sriov_pf_config_release - Release and reset VF configuration. + * xe_gt_sriov_pf_config_release_locked() - Release and reset VF configuration. + * @gt: the &xe_gt + * @vfid: the VF identifier (can't be PF) + * @force: force configuration release + * + * Note: The caller must hold the master PF mutex. + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_release_locked(struct xe_gt *gt, unsigned int vfid, bool force) +{ + int err; + + xe_gt_assert(gt, vfid); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + err = pf_send_vf_cfg_reset(gt, vfid); + if (!err || force) + pf_release_vf_config(gt, vfid); + + if (unlikely(err)) { + xe_gt_sriov_notice(gt, "VF%u unprovisioning failed with error (%pe)%s\n", + vfid, ERR_PTR(err), + force ? " but all resources were released anyway!" : ""); + } + + return force ? 0 : err; +} + +/** + * xe_gt_sriov_pf_config_release() - Release and reset VF configuration. * @gt: the &xe_gt * @vfid: the VF identifier (can't be PF) * @force: force configuration release @@ -2943,23 +2974,9 @@ static void pf_release_vf_config(struct xe_gt *gt, unsigned int vfid) */ int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force) { - int err; + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); - xe_gt_assert(gt, vfid); - - mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); - err = pf_send_vf_cfg_reset(gt, vfid); - if (!err || force) - pf_release_vf_config(gt, vfid); - mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); - - if (unlikely(err)) { - xe_gt_sriov_notice(gt, "VF%u unprovisioning failed with error (%pe)%s\n", - vfid, ERR_PTR(err), - force ? " but all resources were released anyway!" : ""); - } - - return force ? 0 : err; + return xe_gt_sriov_pf_config_release_locked(gt, vfid, force); } static void pf_sanitize_ggtt(struct xe_ggtt_node *ggtt_region, unsigned int vfid) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h index 84246978d21e..d43da8599977 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h @@ -90,6 +90,7 @@ int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs) int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout); int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force); +int xe_gt_sriov_pf_config_release_locked(struct xe_gt *gt, unsigned int vfid, bool force); int xe_gt_sriov_pf_config_push(struct xe_gt *gt, unsigned int vfid, bool refresh); ssize_t xe_gt_sriov_pf_config_save(struct xe_gt *gt, unsigned int vfid, void *buf, size_t size); From 9c8ed392385177546fdd65e4463e292f0f647c6c Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Fri, 11 Sep 2026 20:32:57 +0200 Subject: [PATCH 63/76] drm/xe/pf: Add _locked variant of the fair sched config function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will soon want to auto provision all VFs in a single step, while already holding the master mutex. Split existing function into two. Signed-off-by: Michal Wajdeczko Reviewed-by: Piotr Piórkowski Link: https://patch.msgid.link/20260911183257.14999-4-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c | 23 +++++++++++++++++++--- drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h | 1 + 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 146981aedf5f..6947aeed458e 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -2784,15 +2784,16 @@ static bool pf_needs_provision_sched(struct xe_gt *gt, unsigned int num_vfs) #define XE_ADMIN_PF_SCHED_PRIORITY GUC_SCHED_PRIORITY_HIGH /** - * xe_gt_sriov_pf_config_set_fair_sched() - Provision PF and VFs with fair scheduling. + * xe_gt_sriov_pf_config_set_fair_sched_locked() - Provision PF and VFs with fair scheduling. * @gt: the &xe_gt * @num_vfs: number of VFs to provision (can't be 0) * + * The caller must hold the master PF mutex. * This function can only be called on PF. * * Return: 0 on success or a negative error code on failure. */ -int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs) +int xe_gt_sriov_pf_config_set_fair_sched_locked(struct xe_gt *gt, unsigned int num_vfs) { int result = 0; int err; @@ -2801,7 +2802,7 @@ int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs) xe_gt_assert(gt, XE_FAIR_EXEC_QUANTUM_MS); xe_gt_assert(gt, XE_FAIR_PREEMPT_TIMEOUT_US); - guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); if (!pf_needs_provision_sched(gt, num_vfs)) return 0; @@ -2822,6 +2823,22 @@ int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs) return result; } +/** + * xe_gt_sriov_pf_config_set_fair_sched() - Provision PF and VFs with fair scheduling. + * @gt: the &xe_gt + * @num_vfs: number of VFs to provision (can't be 0) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_gt_sriov_pf_config_set_fair_sched_locked(gt, num_vfs); +} + static int pf_provision_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index, u32 value) { diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h index d43da8599977..3e10697d2174 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h @@ -87,6 +87,7 @@ int xe_gt_sriov_pf_config_set_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index, u32 value); int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs); +int xe_gt_sriov_pf_config_set_fair_sched_locked(struct xe_gt *gt, unsigned int num_vfs); int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout); int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force); From baafc300cd079a5210c1e70e5ea3d93518e40b38 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Mon, 14 Sep 2026 16:18:02 +0300 Subject: [PATCH 64/76] drm/xe/nvm: add system controller region Add region 5 (System Controller) to the list of regions available on device. Signed-off-by: Alexander Usyskin Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20260914-nvm_add_sysctl_region-v1-1-75d710daf5d0@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_nvm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c index 1ea67eaeae24..51a1a431ad85 100644 --- a/drivers/gpu/drm/xe/xe_nvm.c +++ b/drivers/gpu/drm/xe/xe_nvm.c @@ -28,6 +28,7 @@ static const struct intel_dg_nvm_region regions[INTEL_DG_NVM_REGIONS] = { [0] = { .name = "DESCRIPTOR", }, [2] = { .name = "GSC", }, + [5] = { .name = "SYSCTL", }, [9] = { .name = "PADDING", }, [11] = { .name = "OptionROM", }, [12] = { .name = "DAM", }, From 88d8752341d7f1663494940fe70c5a574610966b Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Wed, 16 Sep 2026 23:58:25 +0200 Subject: [PATCH 65/76] drm/xe/guc: Add simple KLV decoder helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of retrieving data from the KLV dwords manually, add simple helpers that will do that together with sanity checks. Signed-off-by: Michal Wajdeczko Reviewed-by: Michał Winiarski Link: https://patch.msgid.link/20260916215827.13488-2-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_guc_klv_helpers.c | 78 +++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_guc_klv_helpers.h | 5 ++ 2 files changed, 83 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.c b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c index dc8612d761e3..dafb5b74e450 100644 --- a/drivers/gpu/drm/xe/xe_guc_klv_helpers.c +++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c @@ -372,6 +372,84 @@ int xe_guc_klv_parser(const u32 *klvs, u32 num_dwords, void *obj, return total; } +/** + * xe_guc_klv_decode_u16() - Decode u16 data from KLV. + * @value: array of dwords + * @len: number of dwords in @value + * @data: pointer where to store the data + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_klv_decode_u16(const u32 *value, u16 len, u16 *data) +{ + u32 tmp; + int err; + + err = xe_guc_klv_decode_u32(value, len, &tmp); + if (err) + return err; + if (upper_16_bits(tmp)) + return -EOVERFLOW; + *data = lower_16_bits(tmp); + return 0; +} + +/** + * xe_guc_klv_decode_u32() - Decode u32 data from KLV. + * @value: array of dwords + * @len: number of dwords in @value + * @data: pointer where to store the data + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_klv_decode_u32(const u32 *value, u16 len, u32 *data) +{ + if (len < to_num_dwords(sizeof(u32))) + return -ENODATA; + if (len > to_num_dwords(sizeof(u32))) + return -ENOMSG; + *data = *value; + return 0; +} + +/** + * xe_guc_klv_decode_u64() - Decode u64 data from KLV. + * @value: array of dwords + * @len: number of dwords in @value + * @data: pointer where to store the data + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_klv_decode_u64(const u32 *value, u16 len, u64 *data) +{ + if (len < to_num_dwords(sizeof(u64))) + return -ENODATA; + if (len != to_num_dwords(sizeof(u64))) + return -ENOMSG; + *data = make_u64(value[1], value[0]); + return 0; +} + +/** + * xe_guc_klv_decode_string() - Decode string data from KLV. + * @value: array of dwords + * @len: number of dwords in @value + * @data: pointer where to store the data + * + * Note: caller must use kfree() to release decoded string. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_klv_decode_string(const u32 *value, u16 len, char **data) +{ + const char *src = (const void *)value; + + if (!len) + return -ENOMSG; + *data = kstrndup(src, to_num_bytes(len), GFP_KERNEL); + return *data ? 0 : -ENOMEM; +} + #if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) #include "tests/xe_guc_klv_helpers_kunit.c" #endif diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.h b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h index cf3b29adc9d6..10af545a0f79 100644 --- a/drivers/gpu/drm/xe/xe_guc_klv_helpers.h +++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h @@ -26,6 +26,11 @@ u32 *xe_guc_klv_encode_object(u32 *klvs, u32 avail, u16 key, const void *obj, int xe_guc_klv_parser(const u32 *klvs, u32 num_dwords, void *obj, int (*decoder)(void *obj, u16 key, u16 len, const u32 *value)); +int xe_guc_klv_decode_u16(const u32 *value, u16 len, u16 *data); +int xe_guc_klv_decode_u32(const u32 *value, u16 len, u32 *data); +int xe_guc_klv_decode_u64(const u32 *value, u16 len, u64 *data); +int xe_guc_klv_decode_string(const u32 *value, u16 len, char **data); + /** * PREP_GUC_KLV - Prepare KLV header value based on provided key and len. * @key: KLV key From 29bc118ea12e62a9bc49ba01633cb4a2205d3399 Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Wed, 16 Sep 2026 23:58:26 +0200 Subject: [PATCH 66/76] drm/xe/tests: Add kunit tests for KLV decoders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that recently added KLV helpers work as expected. Signed-off-by: Michal Wajdeczko Reviewed-by: Michał Winiarski Link: https://patch.msgid.link/20260916215827.13488-3-michal.wajdeczko@intel.com --- .../drm/xe/tests/xe_guc_klv_helpers_kunit.c | 303 ++++++++++++++++++ 1 file changed, 303 insertions(+) diff --git a/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c b/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c index 82869363ab7e..4f7c81c9ebb8 100644 --- a/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c +++ b/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c @@ -373,6 +373,305 @@ static void test_encode_object_basic(struct kunit *test) obj_echo_encoder)); } +static const struct { + u32 klvs[4]; + int expected_err; + u16 expected_val; + const char *name; +} decode_u16_params[] = { + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 0), ~0 }, + .expected_err = -ENODATA, + .expected_val = 0xdead, + .name = "empty", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), 1, ~0 }, + .expected_err = 0, + .expected_val = 1, + .name = "valid", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), U16_MAX, ~0 }, + .expected_err = 0, + .expected_val = U16_MAX, + .name = "max", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), U16_MAX + 1, ~0 }, + .expected_err = -EOVERFLOW, + .expected_val = 0xdead, + .name = "big", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 2), 1, 2, ~0 }, + .expected_err = -ENOMSG, + .expected_val = 0xdead, + .name = "long", + }, +}; + +KUNIT_ARRAY_PARAM_DESC(decode_u16, decode_u16_params, name); + +static void test_decode_u16(struct kunit *test) +{ + typeof(decode_u16_params[0]) *param = test->param_value; + const u32 *klvs = param->klvs; + const u32 *value = klvs + GUC_KLV_LEN_MIN; + u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); + u16 data = 0xdead; + + KUNIT_ASSERT_EQ(test, param->expected_err, xe_guc_klv_decode_u16(value, len, &data)); + KUNIT_ASSERT_EQ(test, param->expected_val, data); +} + +static const struct { + u32 klvs[4]; + int expected_err; + u32 expected_val; + const char *name; +} decode_u32_params[] = { + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 0), ~0 }, + .expected_err = -ENODATA, + .expected_val = 0xdead, + .name = "empty", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), 1, ~0 }, + .expected_err = 0, + .expected_val = 1, + .name = "valid", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), U32_MAX, ~0 }, + .expected_err = 0, + .expected_val = U32_MAX, + .name = "max", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 2), 1, 2, ~0 }, + .expected_err = -ENOMSG, + .expected_val = 0xdead, + .name = "long", + }, +}; + +KUNIT_ARRAY_PARAM_DESC(decode_u32, decode_u32_params, name); + +static void test_decode_u32(struct kunit *test) +{ + typeof(decode_u32_params[0]) *param = test->param_value; + const u32 *klvs = param->klvs; + const u32 *value = klvs + GUC_KLV_LEN_MIN; + u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); + u32 data = 0xdead; + + KUNIT_ASSERT_EQ(test, param->expected_err, xe_guc_klv_decode_u32(value, len, &data)); + KUNIT_ASSERT_EQ(test, param->expected_val, data); +} + +static const struct { + u32 klvs[4]; + int expected_err; + u64 expected_val; + const char *name; +} decode_u64_params[] = { + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 0), ~0 }, + .expected_err = -ENODATA, + .expected_val = 0xdead, + .name = "empty", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), 1, ~0, }, + .expected_err = -ENODATA, + .expected_val = 0xdead, + .name = "short", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 2), 1, 2, ~0 }, + .expected_err = 0, + .expected_val = 0x200000001, + .name = "valid", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 2), U32_MAX, U32_MAX, 3 }, + .expected_err = 0, + .expected_val = U64_MAX, + .name = "max", + }, + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 3), 1, 2, 3, }, + .expected_err = -ENOMSG, + .expected_val = 0xdead, + .name = "long", + }, +}; + +KUNIT_ARRAY_PARAM_DESC(decode_u64, decode_u64_params, name); + +static void test_decode_u64(struct kunit *test) +{ + typeof(decode_u64_params[0]) *param = test->param_value; + const u32 *klvs = param->klvs; + const u32 *value = klvs + GUC_KLV_LEN_MIN; + u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); + u64 data = 0xdead; + + KUNIT_ASSERT_EQ(test, param->expected_err, xe_guc_klv_decode_u64(value, len, &data)); + KUNIT_ASSERT_EQ(test, param->expected_val, data); +} + +#define make_u32(a, b, c, d) \ + (FIELD_PREP_CONST(GENMASK_U32(31, 24), (a)) | \ + FIELD_PREP_CONST(GENMASK_U32(23, 16), (b)) | \ + FIELD_PREP_CONST(GENMASK_U32(15, 8), (c)) | \ + FIELD_PREP_CONST(GENMASK_U32(7, 0), (d))) + +static const struct { + u32 klvs[4]; + int expected_err; + const char *expected_str; + const char *name; +} decode_str_params[] = { + { + .klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 0), 'a', ~0 }, + .expected_err = -ENOMSG, + .expected_str = NULL, + .name = "empty", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 1), + make_u32(0, 0, 0, '1'), + }, + .expected_err = 0, + .expected_str = "1", + .name = "one", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 1), + make_u32('3', '2', 0, '1'), + '4', ~0 + }, + .expected_err = 0, + .expected_str = "1", + .name = "one_trash", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 1), + make_u32(0, 0, '2', '1'), + '3', ~0 + }, + .expected_err = 0, + .expected_str = "12", + .name = "two", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 1), + make_u32('3', 0, '2', '1'), + '4', ~0 + }, + .expected_err = 0, + .expected_str = "12", + .name = "two_trash", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 1), + make_u32(0, '3', '2', '1'), + '4', ~0 + }, + .expected_err = 0, + .expected_str = "123", + .name = "three", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 1), + make_u32('4', '3', '2', '1'), + '5', ~0 + }, + .expected_err = 0, + .expected_str = "1234", + .name = "dword", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 2), + make_u32('4', '3', '2', '1'), + 0, ~0 + }, + .expected_err = 0, + .expected_str = "1234", + .name = "four", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 2), + make_u32('4', '3', '2', '1'), + make_u32(0, 0, 0, '5'), + ~0, + }, + .expected_err = 0, + .expected_str = "12345", + .name = "five", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 2), + make_u32('4', '3', '2', '1'), + make_u32(0, 0, '6', '5'), + ~0, + }, + .expected_err = 0, + .expected_str = "123456", + .name = "six", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 2), + make_u32('4', '3', '2', '1'), + make_u32(0, '7', '6', '5'), + ~0, + }, + .expected_err = 0, + .expected_str = "1234567", + .name = "seven", + }, + { + .klvs = { + PREP_GUC_KLV_CONST(TEST_KEY, 2), + make_u32('4', '3', '2', '1'), + make_u32('8', '7', '6', '5'), + ~0, + }, + .expected_err = 0, + .expected_str = "12345678", + .name = "qword", + }, +}; + +KUNIT_ARRAY_PARAM_DESC(decode_str, decode_str_params, name); + +static void test_decode_str(struct kunit *test) +{ + typeof(decode_str_params[0]) *param = test->param_value; + const u32 *klvs = param->klvs; + const u32 *value = klvs + GUC_KLV_LEN_MIN; + u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); + char *data = NULL; + + KUNIT_EXPECT_EQ(test, param->expected_err, xe_guc_klv_decode_string(value, len, &data)); + if (!param->expected_err) + KUNIT_EXPECT_STREQ(test, param->expected_str, data); + kfree(data); +} + static void __drm_printfn_kunit(struct drm_printer *p, struct va_format *vaf) { struct kunit *test = p->arg; @@ -417,6 +716,10 @@ static struct kunit_case guc_klv_helpers_test_cases[] = { KUNIT_CASE(test_encode_object_klv), KUNIT_CASE(test_encode_object_nested), KUNIT_CASE(test_encode_object_basic), + KUNIT_CASE_PARAM(test_decode_u16, decode_u16_gen_params), + KUNIT_CASE_PARAM(test_decode_u32, decode_u32_gen_params), + KUNIT_CASE_PARAM(test_decode_u64, decode_u64_gen_params), + KUNIT_CASE_PARAM(test_decode_str, decode_str_gen_params), KUNIT_CASE(test_print), {} }; From b361f6452e4a6f5cf2c6b045db99d4b116abff2b Mon Sep 17 00:00:00 2001 From: Michal Wajdeczko Date: Wed, 16 Sep 2026 23:58:27 +0200 Subject: [PATCH 67/76] drm/xe/pf: Use KLV decode helpers to parse migration packet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Start using recently added KLV helpers for parsing migration packet to properly detect any malformed KLV formats. Signed-off-by: Michal Wajdeczko Reviewed-by: Michał Winiarski Link: https://patch.msgid.link/20260916215827.13488-4-michal.wajdeczko@intel.com --- drivers/gpu/drm/xe/xe_sriov_packet.c | 31 ++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.c b/drivers/gpu/drm/xe/xe_sriov_packet.c index e9ae9c9744ea..f50dc4e3a611 100644 --- a/drivers/gpu/drm/xe/xe_sriov_packet.c +++ b/drivers/gpu/drm/xe/xe_sriov_packet.c @@ -394,24 +394,43 @@ static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid) static int descriptor_decoder(void *arg, u16 key, u16 len, const u32 *value) { struct xe_device *xe = arg; + int err; xe_sriov_dbg_verbose(xe, "found KLV %#x %s\n", key, xe_guc_klv_key_to_string(key)); switch (key) { - case MIGRATION_KLV_DEVICE_DEVID_KEY: - if (*value != xe->info.devid) { + case MIGRATION_KLV_DEVICE_DEVID_KEY: { + u16 devid; + + err = xe_guc_klv_decode_u16(value, len, &devid); + if (err) { + xe_sriov_warn(xe, "Aborting migration, corrupted key %#x (%pe)\n", + key, ERR_PTR(err)); + return err; + } + if (devid != xe->info.devid) { xe_sriov_warn(xe, "Aborting migration, devid mismatch %#06x!=%#06x\n", - *value, xe->info.devid); + devid, xe->info.devid); return -ENODEV; } break; - case MIGRATION_KLV_DEVICE_REVID_KEY: - if (*value != xe->info.revid) { + } + case MIGRATION_KLV_DEVICE_REVID_KEY: { + u16 revid; + + err = xe_guc_klv_decode_u16(value, len, &revid); + if (err) { + xe_sriov_warn(xe, "Aborting migration, corrupted key %#x (%pe)\n", + key, ERR_PTR(err)); + return err; + } + if (revid != xe->info.revid) { xe_sriov_warn(xe, "Aborting migration, revid mismatch %#06x!=%#06x\n", - *value, xe->info.revid); + revid, xe->info.revid); return -ENODEV; } break; + } default: if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { struct drm_printer p = xe_dbg_printer(xe); From 62abb2341b1e37fd849b531ffb3136b29898fab3 Mon Sep 17 00:00:00 2001 From: Sk Anirban Date: Thu, 10 Sep 2026 22:28:56 +0530 Subject: [PATCH 68/76] drm/xe/debugfs: Expose per-engine idle status Extend the existing "hw_engines" debugfs to iterate over all hardware engines and dump their idle state: whether the engine pipe is idle or busy, whether C6 entry is allowed, and (where available) the forcewake request status. v2: Use existing debugfs (Matt Brost) v3: Rename macro (Vinay) Signed-off-by: Sk Anirban Reviewed-by: Vinay Belgaumkar Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260910165855.1124574-2-sk.anirban@intel.com --- drivers/gpu/drm/xe/regs/xe_gt_regs.h | 27 +++++++++++++++ drivers/gpu/drm/xe/xe_devcoredump.c | 4 ++- drivers/gpu/drm/xe/xe_gt_debugfs.c | 52 +++++++++++++++++++++++++++- drivers/gpu/drm/xe/xe_guc_capture.c | 2 -- 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 48c515d91882..c255632c34d0 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -196,6 +196,33 @@ #define MSC_MSAA_REODER_BUF_BYPASS_DISABLE REG_BIT(14) #define FAST_CLEAR_VALIGN_FIX REG_BIT(13) +#define MSG_IDLE_CS XE_REG(0x8000) +#define MSG_IDLE_VCS0 XE_REG(0x8004) +#define MSG_IDLE_VCS1 XE_REG(0x8008) +#define MSG_IDLE_VCS2 XE_REG(0x80C0) +#define MSG_IDLE_VCS3 XE_REG(0x80C4) +#define MSG_IDLE_VCS4 XE_REG(0x80C8) +#define MSG_IDLE_VCS5 XE_REG(0x80CC) +#define MSG_IDLE_VCS6 XE_REG(0x80D0) +#define MSG_IDLE_VCS7 XE_REG(0x80D4) +#define MSG_IDLE_VECS0 XE_REG(0x8010) +#define MSG_IDLE_VECS1 XE_REG(0x80D8) +#define MSG_IDLE_VECS2 XE_REG(0x80DC) +#define MSG_IDLE_VECS3 XE_REG(0x80E0) +#define MSG_IDLE_BCS0 XE_REG(0x800C) +#define MSG_IDLE_BCS1 XE_REG(0x8680) +#define MSG_IDLE_BCS2 XE_REG(0x8684) +#define MSG_IDLE_BCS3 XE_REG(0x8688) +#define MSG_IDLE_BCS4 XE_REG(0x868C) +#define MSG_IDLE_BCS5 XE_REG(0x8690) +#define MSG_IDLE_BCS6 XE_REG(0x8694) +#define MSG_IDLE_BCS7 XE_REG(0x8698) +#define MSG_IDLE_BCS8 XE_REG(0x869C) +#define MSG_IDLE_GSCCS0 XE_REG(0xA62C) +#define MSG_IDLE_INDICATION REG_BIT(0) +#define MSG_IDLE_C6_ALLOWED REG_BIT(1) +#define MSG_IDLE_FW_REQ REG_GENMASK(13, 9) + #define XE2LPM_CCCHKNREG1 XE_REG(0x82a8) #define VF_PREEMPTION XE_REG(0x83a4, XE_REG_OPTION_MASKED) diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c index 5f2b90b18f97..b918f046ae15 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.c +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -127,8 +127,10 @@ static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count, drm_puts(&p, "\n**** HW Engines ****\n"); for (i = 0; i < XE_NUM_HW_ENGINES; i++) - if (ss->hwe[i]) + if (ss->hwe[i]) { xe_engine_snapshot_print(ss->hwe[i], &p); + drm_puts(&p, "\n"); + } drm_puts(&p, "\n**** VM state ****\n"); xe_vm_snapshot_print(ss->vm, &p); diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index bb09e70ee44c..405cc076d843 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -119,6 +119,38 @@ int xe_gt_debugfs_show_with_rpm(struct seq_file *m, void *data) static int hw_engines(struct xe_gt *gt, struct drm_printer *p) { + static const struct { + struct xe_reg reg; + bool has_fw; + } msg_idle_reg[XE_NUM_HW_ENGINES] = { + [XE_HW_ENGINE_RCS0] = { MSG_IDLE_CS, true }, + [XE_HW_ENGINE_BCS0] = { MSG_IDLE_BCS0, true }, + [XE_HW_ENGINE_BCS1] = { MSG_IDLE_BCS1, false }, + [XE_HW_ENGINE_BCS2] = { MSG_IDLE_BCS2, false }, + [XE_HW_ENGINE_BCS3] = { MSG_IDLE_BCS3, false }, + [XE_HW_ENGINE_BCS4] = { MSG_IDLE_BCS4, false }, + [XE_HW_ENGINE_BCS5] = { MSG_IDLE_BCS5, false }, + [XE_HW_ENGINE_BCS6] = { MSG_IDLE_BCS6, false }, + [XE_HW_ENGINE_BCS7] = { MSG_IDLE_BCS7, false }, + [XE_HW_ENGINE_BCS8] = { MSG_IDLE_BCS8, false }, + [XE_HW_ENGINE_VCS0] = { MSG_IDLE_VCS0, true }, + [XE_HW_ENGINE_VCS1] = { MSG_IDLE_VCS1, true }, + [XE_HW_ENGINE_VCS2] = { MSG_IDLE_VCS2, true }, + [XE_HW_ENGINE_VCS3] = { MSG_IDLE_VCS3, true }, + [XE_HW_ENGINE_VCS4] = { MSG_IDLE_VCS4, true }, + [XE_HW_ENGINE_VCS5] = { MSG_IDLE_VCS5, true }, + [XE_HW_ENGINE_VCS6] = { MSG_IDLE_VCS6, true }, + [XE_HW_ENGINE_VCS7] = { MSG_IDLE_VCS7, true }, + [XE_HW_ENGINE_VECS0] = { MSG_IDLE_VECS0, true }, + [XE_HW_ENGINE_VECS1] = { MSG_IDLE_VECS1, true }, + [XE_HW_ENGINE_VECS2] = { MSG_IDLE_VECS2, true }, + [XE_HW_ENGINE_VECS3] = { MSG_IDLE_VECS3, true }, + [XE_HW_ENGINE_CCS0] = { MSG_IDLE_CS, true }, + [XE_HW_ENGINE_CCS1] = { MSG_IDLE_CS, true }, + [XE_HW_ENGINE_CCS2] = { MSG_IDLE_CS, true }, + [XE_HW_ENGINE_CCS3] = { MSG_IDLE_CS, true }, + [XE_HW_ENGINE_GSCCS0] = { MSG_IDLE_GSCCS0, false }, + }; struct xe_hw_engine *hwe; enum xe_hw_engine_id id; @@ -126,9 +158,27 @@ static int hw_engines(struct xe_gt *gt, struct drm_printer *p) if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) return -ETIMEDOUT; - for_each_hw_engine(hwe, gt, id) + for_each_hw_engine(hwe, gt, id) { + struct xe_reg reg = msg_idle_reg[id].reg; + u32 val; + xe_hw_engine_print(hwe, p); + val = xe_mmio_read32(>->mmio, reg); + + if (msg_idle_reg[id].has_fw) + drm_printf(p, "\tpipe %s, C6 %s, fw_req 0x%02x\n", + val & MSG_IDLE_INDICATION ? "idle" : "busy", + val & MSG_IDLE_C6_ALLOWED ? "allowed" : "not allowed", + REG_FIELD_GET(MSG_IDLE_FW_REQ, val)); + else + drm_printf(p, "\tpipe %s, C6 %s\n", + val & MSG_IDLE_INDICATION ? "idle" : "busy", + val & MSG_IDLE_C6_ALLOWED ? "allowed" : "not allowed"); + + drm_puts(p, "\n"); + } + return 0; } diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index 82df19b304e1..5a4d085660a2 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -1868,8 +1868,6 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm snapshot_print_by_list_order(snapshot, p, GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, list); } - - drm_puts(p, "\n"); } /** From e199c851c0ab608a0ca89e7be1756a1461b41a2c Mon Sep 17 00:00:00 2001 From: Sk Anirban Date: Wed, 9 Sep 2026 17:19:32 +0530 Subject: [PATCH 69/76] drm/xe/gt_throttle: Report power brake as a throttle reason on CRI CRI defines bit 5 of the perf limit reasons register as a power brake (PWRBRK) indicator. Add PWRBRK_MASK and a reason_pwrbrk sysfs attribute for CRI in place of reason_ratl. Signed-off-by: Sk Anirban Fixes: 8578e6d0546c ("drm/xe/gt_throttle: Drop individual show functions") Reviewed-by: Raag Jadav Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260909114931.1039331-2-sk.anirban@intel.com --- drivers/gpu/drm/xe/regs/xe_gt_regs.h | 1 + drivers/gpu/drm/xe/xe_gt_throttle.c | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index c255632c34d0..2e059cb36c82 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -684,6 +684,7 @@ #define MEM_THERMAL_MASK REG_BIT(2) #define VR_THERMAL_MASK REG_BIT(3) #define ICCMAX_MASK REG_BIT(4) +#define PWRBRK_MASK REG_BIT(5) #define SOC_AVG_THERMAL_MASK REG_BIT(6) #define FASTVMODE_MASK REG_BIT(7) #define PSYS_PL1_MASK REG_BIT(12) diff --git a/drivers/gpu/drm/xe/xe_gt_throttle.c b/drivers/gpu/drm/xe/xe_gt_throttle.c index 1e7e3a31aa69..c0af5484611d 100644 --- a/drivers/gpu/drm/xe/xe_gt_throttle.c +++ b/drivers/gpu/drm/xe/xe_gt_throttle.c @@ -39,7 +39,7 @@ * - ``reason_mem_thermal``: Memory thermal * - ``reason_vr_thermal``: VR thermal * - ``reason_iccmax``: ICCMAX - * - ``reason_ratl``: RATL thermal algorithm + * - ``reason_pwrbrk``: Power brake * - ``reason_soc_avg_thermal``: SoC average temp * - ``reason_fastvmode``: VR is hitting FastVMode * - ``reason_psys_pl1``: PSYS PL1 @@ -200,6 +200,7 @@ static THROTTLE_ATTR_RO(reason_psys_pl1, PSYS_PL1_MASK); static THROTTLE_ATTR_RO(reason_psys_pl2, PSYS_PL2_MASK); static THROTTLE_ATTR_RO(reason_p0_freq, P0_FREQ_MASK); static THROTTLE_ATTR_RO(reason_psys_crit, PSYS_CRIT_MASK); +static THROTTLE_ATTR_RO(reason_pwrbrk, PWRBRK_MASK); static struct attribute *cri_throttle_attrs[] = { /* Common */ @@ -209,12 +210,12 @@ static struct attribute *cri_throttle_attrs[] = { &attr_reason_pl2.attr.attr, &attr_reason_pl4.attr.attr, &attr_reason_prochot.attr.attr, - &attr_reason_ratl.attr.attr, /* CRI */ &attr_reason_vr_thermal.attr.attr, &attr_reason_soc_thermal.attr.attr, &attr_reason_mem_thermal.attr.attr, &attr_reason_iccmax.attr.attr, + &attr_reason_pwrbrk.attr.attr, &attr_reason_soc_avg_thermal.attr.attr, &attr_reason_fastvmode.attr.attr, &attr_reason_psys_pl1.attr.attr, From af14e3705345cb57c53b237169873233052a5c16 Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Mon, 14 Sep 2026 21:53:18 +0000 Subject: [PATCH 70/76] drm/xe/tlb_inval: Treat wedged-device invalidations as complete A TLB invalidation issued on a wedged device fails with -ENOTRECOVERABLE. xe_tlb_inval_issue() squashes only -ECANCELED, so the error reaches callers that treat it as unexpected and WARN, tainting the kernel on a wedge that was deliberately caused: ggtt_invalidate_gt_tlb() drivers/gpu/drm/xe/xe_ggtt.c xe_svm_invalidate() drivers/gpu/drm/xe/xe_svm.c xe_bo_trigger_rebind() drivers/gpu/drm/xe/xe_bo.c xe_vma_userptr_do_inval() drivers/gpu/drm/xe/xe_userptr.c igt@xe_exec_reset@gt-reset-fault-injection hits the GGTT one, turning an otherwise passing run into an abort: *ERROR* SIGID=102 FATAL (-EIO) WEDGED: Device declared wedged! Tile0: GT1: Failed to invalidate GGTT (-ENOTRECOVERABLE) WARNING: drivers/gpu/drm/xe/xe_ggtt.c:588 at ggtt_invalidate_gt_tlb Workqueue: xe-guc-destroy-wq __guc_exec_queue_destroy_async [xe] ggtt_node_remove+0xe3/0x100 [xe] xe_ggtt_remove_bo+0x89/0x2c0 [xe] xe_ttm_bo_destroy+0xcb/0x330 [xe] ... xe_lrc_destroy+0x74/0x90 [xe] xe_exec_queue_fini+0x2d/0x60 [xe] -ECANCELED and -ENOTRECOVERABLE mean the same thing at this layer: the message was dropped rather than delivered, and the fence has already been signalled before the error is returned, so there is nothing left to wait for. Squash both. A wedged device is only recovered by a fresh initialisation, so the error return in xe_bo_trigger_rebind() becomes unreachable by design. v2: fix all invalidation paths. (Sashiko) Fixes: 50fa9acac26f ("drm/xe/guc: distinguish wedged from recoverable cancellation") Assisted-by: Claude:claude-opus-5 Cc: Sk Anirban Cc: Matthew Brost Signed-off-by: Shuicheng Lin Reviewed-by: Matthew Brost Signed-off-by: Matthew Brost Link: https://patch.msgid.link/20260914215318.200603-1-shuicheng.lin@intel.com --- drivers/gpu/drm/xe/xe_tlb_inval.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index 7c3bf32d9258..dd1030de9ab6 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -280,7 +280,8 @@ static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence) xe_tlb_inval_fence_signal_unlocked((__fence)); \ mutex_unlock(&(__tlb_inval)->seqno_lock); \ \ - __ret == -ECANCELED ? 0 : __ret; \ + /* Undelivered: fence already signalled, report done */ \ + (__ret == -ECANCELED || __ret == -ENOTRECOVERABLE) ? 0 : __ret; \ }) /** From fd831e3af1e0858cdf12b49635c4ffb4600442fa Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 17 Sep 2026 13:38:50 -0700 Subject: [PATCH 71/76] drm/xe: Add mnemonic error reason to page fault diagnostics xe_pagefault_print() was firing without indicating why servicing of a fault failed, making it hard to distinguish benign races (e.g. a VM's file descriptor closing mid-fault) from real bugs. Add enum xe_pagefault_error, uniquely identifying the high-level point at which xe_pagefault_service(), xe_pagefault_handle_vma(), and xe_svm_handle_pagefault() fail. Since consumer.page_addr is always page (4K) aligned, steal the reserved low byte to carry this code without growing struct xe_pagefault. Add xe_pagefault_set_error() / xe_pagefault_get_error() to encode/decode it, and xe_pagefault_addr() as the single accessor for the real (masked) faulted address; update all real address consumers (xe_pagefault_match(), cache alignment, xe_vm_add_fault_entry_pf()) to go through it instead of reading consumer.page_addr directly. xe_pagefault_set_start_addr() now preserves any already-recorded error bits rather than clobbering them. xe_pagefault_print() decodes the error via a new xe_pagefault_error_to_str() and reports it in an added "Error:" line. xe_pagefault_asid_to_vm() now distinguishes an ASID with no VM at all (XE_PAGEFAULT_ERROR_VM_NOT_FOUND, e.g. the owning file descriptor was already closed) from a VM found but not in fault mode (XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE). Signed-off-by: Matthew Brost Assisted-by: GitHub_Copilot:claude-sonnet-5 Reviewed-by: Himal Prasad Ghimiray Link: https://patch.msgid.link/20260917203850.321385-1-matthew.brost@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_pagefault.c | 102 +++++++++++++++++++----- drivers/gpu/drm/xe/xe_pagefault.h | 67 +++++++++++++++- drivers/gpu/drm/xe/xe_pagefault_types.h | 82 +++++++++++++++++++ drivers/gpu/drm/xe/xe_svm.c | 21 ++++- drivers/gpu/drm/xe/xe_vm.c | 3 +- 5 files changed, 250 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index c82b8bc8bc70..1286be7fdcf2 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -158,8 +158,14 @@ static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma, lockdep_assert_held(&vm->lock); needs_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); - if (needs_vram < 0 || (needs_vram && xe_vma_is_userptr(vma))) - return needs_vram < 0 ? needs_vram : -EACCES; + if (needs_vram < 0) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK); + return needs_vram; + } + if (needs_vram && xe_vma_is_userptr(vma)) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR); + return -EACCES; + } xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, 1); xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_KB, @@ -178,13 +184,17 @@ static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma, } do { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_NONE); + if (xe_vma_is_userptr(vma) && xe_vma_userptr_check_repin(to_userptr_vma(vma))) { struct xe_userptr_vma *uvma = to_userptr_vma(vma); err = xe_vma_userptr_pin_pages(uvma); - if (err) + if (err) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN); return err; + } } /* Lock VM and BOs dma-resv */ @@ -195,8 +205,10 @@ static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma, needs_vram == 1); drm_exec_retry_on_contention(&exec); xe_validation_retry_on_oom(&ctx, &err); - if (err) + if (err) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_VALIDATE); break; + } /* Bind VMA only to the GT that has faulted */ trace_xe_vma_pf_bind(vma); @@ -206,6 +218,7 @@ static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma, if (IS_ERR(fence)) { err = PTR_ERR(fence); xe_validation_retry_on_oom(&ctx, &err); + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_REBIND); break; } } @@ -230,16 +243,22 @@ xe_pagefault_access_is_atomic(enum xe_pagefault_access_type access_type) return (access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK) == XE_PAGEFAULT_ACCESS_TYPE_ATOMIC; } -static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid) +static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_pagefault *pf, u32 asid) { + struct xe_device *xe = gt_to_xe(pf->gt); struct xe_vm *vm; down_read(&xe->usm.lock); vm = xa_load(&xe->usm.asid_to_vm, asid); - if (vm && xe_vm_in_fault_mode(vm)) - xe_vm_get(vm); - else + if (!vm) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_NOT_FOUND); vm = ERR_PTR(-EINVAL); + } else if (!xe_vm_in_fault_mode(vm)) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE); + vm = ERR_PTR(-EINVAL); + } else { + xe_vm_get(vm); + } up_read(&xe->usm.lock); return vm; @@ -248,7 +267,6 @@ static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid) static int xe_pagefault_service(struct xe_pagefault *pf) { struct xe_gt *gt = pf->gt; - struct xe_device *xe = gt_to_xe(gt); struct xe_vm *vm; struct xe_vma *vma = NULL; int err; @@ -259,7 +277,7 @@ static int xe_pagefault_service(struct xe_pagefault *pf) if (pf->consumer.fault_type_level == XE_PAGEFAULT_TYPE_LEVEL_NACK) return -EFAULT; - vm = xe_pagefault_asid_to_vm(xe, asid); + vm = xe_pagefault_asid_to_vm(pf, asid); if (IS_ERR(vm)) return PTR_ERR(vm); @@ -267,18 +285,21 @@ static int xe_pagefault_service(struct xe_pagefault *pf) if (xe_vm_is_closed(vm)) { err = -ENOENT; + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_CLOSED); goto unlock_vm; } - vma = xe_vm_find_vma_by_addr(vm, pf->consumer.page_addr); + vma = xe_vm_find_vma_by_addr(vm, xe_pagefault_addr(pf)); if (!vma) { err = -EINVAL; + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_NOT_FOUND); goto unlock_vm; } if (xe_vma_read_only(vma) && pf->consumer.access_type != XE_PAGEFAULT_ACCESS_TYPE_READ) { err = -EPERM; + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION); goto unlock_vm; } @@ -286,7 +307,7 @@ static int xe_pagefault_service(struct xe_pagefault *pf) if (xe_vma_is_cpu_addr_mirror(vma)) err = xe_svm_handle_pagefault(vm, vma, pf, gt, - pf->consumer.page_addr, atomic); + xe_pagefault_addr(pf), atomic); else err = xe_pagefault_handle_vma(gt, vma, pf, atomic); @@ -375,7 +396,7 @@ static bool xe_pagefault_match(struct xe_pagefault *pf, u64 start, u64 end, u64 cache_asid) { struct xe_device *xe = gt_to_xe(pf->gt); - u64 page_addr = pf->consumer.page_addr; + u64 page_addr = xe_pagefault_addr(pf); u32 pf_asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id); xe_assert(xe, pf->consumer.alloc_state != @@ -499,7 +520,7 @@ static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue, if (FIELD_GET(XE_PAGEFAULT_REQUEUE_MASK, lpf->consumer.fault_type_level)) align = SZ_4K; - pf_work->cache.start = ALIGN_DOWN(lpf->consumer.page_addr, align); + pf_work->cache.start = ALIGN_DOWN(xe_pagefault_addr(lpf), align); pf_work->cache.end = pf_work->cache.start + align; pf_work->cache.asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, lpf->consumer.id); pf_work->cache.pf = lpf; @@ -542,10 +563,53 @@ static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue, return true; } +static const char *xe_pagefault_error_to_str(enum xe_pagefault_error error) +{ + switch (error) { + case XE_PAGEFAULT_ERROR_NONE: + return "NONE"; + case XE_PAGEFAULT_ERROR_VM_NOT_FOUND: + return "VM_NOT_FOUND"; + case XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE: + return "VM_NOT_IN_FAULT_MODE"; + case XE_PAGEFAULT_ERROR_VM_CLOSED: + return "VM_CLOSED"; + case XE_PAGEFAULT_ERROR_VMA_NOT_FOUND: + return "VMA_NOT_FOUND"; + case XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION: + return "READ_ONLY_VIOLATION"; + case XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK: + return "VMA_NEEDS_VRAM_CHECK"; + case XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR: + return "VMA_ATOMIC_USERPTR"; + case XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN: + return "VMA_USERPTR_PIN"; + case XE_PAGEFAULT_ERROR_VMA_VALIDATE: + return "VMA_VALIDATE"; + case XE_PAGEFAULT_ERROR_VMA_REBIND: + return "VMA_REBIND"; + case XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR: + return "SVM_GARBAGE_COLLECTOR"; + case XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND: + return "SVM_RANGE_NOT_FOUND"; + case XE_PAGEFAULT_ERROR_SVM_REBIND: + return "SVM_REBIND"; + case XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK: + return "SVM_NEEDS_VRAM_CHECK"; + case XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND: + return "SVM_VMA_NOT_FOUND"; + case XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED: + return "SVM_SERVICE_FAILED"; + default: + return "UNKNOWN"; + } +} + static void xe_pagefault_print(struct xe_pagefault *pf) { u8 engine_class = FIELD_GET(XE_PAGEFAULT_ENGINE_CLASS_MASK, pf->consumer.engine_class_instance); + u64 addr = xe_pagefault_addr(pf); xe_gt_info(pf->gt, "\n\tASID: %lu\n" "\tFaulted Address: 0x%08x%08x\n" @@ -554,11 +618,12 @@ static void xe_pagefault_print(struct xe_pagefault *pf) "\tFaultLevel: %lu\n" "\tEngineClass: %d %s\n" "\tEngineInstance: %lu\n" - "\tSRCID: 0x%02lx\n", + "\tSRCID: 0x%02lx\n" + "\tError: %s\n", FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id), - upper_32_bits(pf->consumer.page_addr), - lower_32_bits(pf->consumer.page_addr), + upper_32_bits(addr), + lower_32_bits(addr), FIELD_GET(XE_PAGEFAULT_TYPE_MASK, pf->consumer.fault_type_level), FIELD_GET(XE_PAGEFAULT_ACCESS_TYPE_MASK, @@ -570,7 +635,8 @@ static void xe_pagefault_print(struct xe_pagefault *pf) FIELD_GET(XE_PAGEFAULT_ENGINE_INSTANCE_MASK, pf->consumer.engine_class_instance), FIELD_GET(XE_PAGEFAULT_SRCID_MASK, - pf->consumer.id)); + pf->consumer.id), + xe_pagefault_error_to_str(xe_pagefault_get_error(pf))); } static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *pf) diff --git a/drivers/gpu/drm/xe/xe_pagefault.h b/drivers/gpu/drm/xe/xe_pagefault.h index e9c5d1f03760..799c984dee84 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.h +++ b/drivers/gpu/drm/xe/xe_pagefault.h @@ -6,6 +6,8 @@ #ifndef _XE_PAGEFAULT_H_ #define _XE_PAGEFAULT_H_ +#include + #include "xe_pagefault_types.h" struct drm_printer; @@ -21,6 +23,61 @@ int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf); void xe_pagefault_print_info(struct xe_device *xe, struct drm_printer *p); +/* + * consumer.page_addr is always page (4K) aligned, so the low bits are + * reserved and unused by the real address. Steal a byte of those bits to + * record an &enum xe_pagefault_error describing the high-level point at + * which servicing of the fault failed, so it can be reported by + * xe_pagefault_print(). All real address consumers of page_addr must go + * through xe_pagefault_addr() to mask off these reserved bits. + */ +#define XE_PAGEFAULT_ERROR_MASK GENMASK_ULL(7, 0) + +/** + * xe_pagefault_set_error() - record the failure reason for a pagefault + * @pf: Pagefault entry + * @error: Failure reason + * + * Encodes @error into the reserved low bits of consumer.page_addr. Should be + * called at the high-level point a pagefault fails to service so + * xe_pagefault_print() can later report a mnemonic failure reason. + */ +static inline void +xe_pagefault_set_error(struct xe_pagefault *pf, enum xe_pagefault_error error) +{ + pf->consumer.page_addr &= ~XE_PAGEFAULT_ERROR_MASK; + pf->consumer.page_addr |= FIELD_PREP(XE_PAGEFAULT_ERROR_MASK, error); +} + +/** + * xe_pagefault_get_error() - read the failure reason for a pagefault + * @pf: Pagefault entry + * + * Return: The &enum xe_pagefault_error previously recorded via + * xe_pagefault_set_error(), or %XE_PAGEFAULT_ERROR_NONE if none was recorded. + */ +static inline enum xe_pagefault_error +xe_pagefault_get_error(struct xe_pagefault *pf) +{ + return FIELD_GET(XE_PAGEFAULT_ERROR_MASK, pf->consumer.page_addr); +} + +/** + * xe_pagefault_addr() - read the real faulted address for a pagefault + * @pf: Pagefault entry + * + * consumer.page_addr may have failure reason bits encoded into its reserved + * low bits by xe_pagefault_set_error(). This masks those bits off, returning + * the real page address. All accesses to the faulted address must go through + * this helper rather than reading consumer.page_addr directly. + * + * Return: The real (page aligned) faulted address. + */ +static inline u64 xe_pagefault_addr(struct xe_pagefault *pf) +{ + return pf->consumer.page_addr & ~XE_PAGEFAULT_ERROR_MASK; +} + #define XE_PAGEFAULT_END_ADDR_MASK (~0xfffull) /** @@ -69,11 +126,17 @@ static inline u64 xe_pagefault_end_addr(struct xe_pagefault *pf) * The pagefault consumer stores the resolved fault range so subsequent faults * hitting the same range can be immediately acknowledged without re-running * the full fault handling path. + * + * The start address shares storage with the failure reason recorded by + * xe_pagefault_set_error() and therefore must be masked with + * %XE_PAGEFAULT_ERROR_MASK before storing so any previously recorded error is + * preserved. */ static inline void xe_pagefault_set_start_addr(struct xe_pagefault *pf, u64 start_addr) { - pf->consumer.page_addr = start_addr; + pf->consumer.page_addr &= XE_PAGEFAULT_ERROR_MASK; + pf->consumer.page_addr |= (start_addr & ~XE_PAGEFAULT_ERROR_MASK); } /** @@ -87,7 +150,7 @@ xe_pagefault_set_start_addr(struct xe_pagefault *pf, u64 start_addr) */ static inline u64 xe_pagefault_start_addr(struct xe_pagefault *pf) { - return pf->consumer.page_addr; + return xe_pagefault_addr(pf); } #endif diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h index f16bab29bfc3..6e1c0360648d 100644 --- a/drivers/gpu/drm/xe/xe_pagefault_types.h +++ b/drivers/gpu/drm/xe/xe_pagefault_types.h @@ -31,6 +31,88 @@ enum xe_pagefault_type { XE_PAGEFAULT_TYPE_ATOMIC_ACCESS_VIOLATION = 2, }; +/** + * enum xe_pagefault_error - Xe page fault servicing error + * + * Uniquely identifies the high-level point at which servicing of a page + * fault failed. Encoded into the reserved low bits of + * &xe_pagefault.consumer.page_addr (which is always page aligned) so the + * failure reason can be threaded back up to xe_pagefault_print() without + * growing the size of struct xe_pagefault. See xe_pagefault_set_error() and + * xe_pagefault_error_to_str(). + */ +enum xe_pagefault_error { + /** @XE_PAGEFAULT_ERROR_NONE: No error recorded */ + XE_PAGEFAULT_ERROR_NONE = 0, + /** + * @XE_PAGEFAULT_ERROR_VM_NOT_FOUND: VM lookup by ASID failed, e.g. + * the VM's file descriptor was already closed and the ASID has been + * torn down + */ + XE_PAGEFAULT_ERROR_VM_NOT_FOUND, + /** + * @XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE: VM found by ASID lookup + * but is not in fault mode + */ + XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE, + /** @XE_PAGEFAULT_ERROR_VM_CLOSED: VM found but already closed */ + XE_PAGEFAULT_ERROR_VM_CLOSED, + /** @XE_PAGEFAULT_ERROR_VMA_NOT_FOUND: No VMA covers the faulted address */ + XE_PAGEFAULT_ERROR_VMA_NOT_FOUND, + /** + * @XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION: Write/atomic fault on a + * read-only VMA + */ + XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION, + /** + * @XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK: Failed determining if VMA + * requires VRAM for an atomic access + */ + XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK, + /** + * @XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR: Atomic access requires VRAM + * but VMA is a userptr, which is unsupported + */ + XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR, + /** @XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN: Userptr page pin/repin failed */ + XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN, + /** + * @XE_PAGEFAULT_ERROR_VMA_VALIDATE: Failed to lock/validate VMA's BO + * or migrate it to VRAM + */ + XE_PAGEFAULT_ERROR_VMA_VALIDATE, + /** @XE_PAGEFAULT_ERROR_VMA_REBIND: Failed to rebind VMA into page tables */ + XE_PAGEFAULT_ERROR_VMA_REBIND, + /** + * @XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR: Failed processing + * pending SVM garbage collection (unmaps) prior to servicing the + * fault + */ + XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR, + /** + * @XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND: Failed to find or insert + * an SVM range covering the faulted address + */ + XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND, + /** @XE_PAGEFAULT_ERROR_SVM_REBIND: Failed to rebind an SVM range into page tables */ + XE_PAGEFAULT_ERROR_SVM_REBIND, + /** + * @XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK: Failed determining if SVM + * VMA requires VRAM for an atomic access + */ + XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK, + /** + * @XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND: SVM VMA re-lookup after a + * range split failed to find a covering VMA + */ + XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND, + /** + * @XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED: SVM range population, + * migration, or bind failed + */ + XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED, +}; + /** struct xe_pagefault_ops - Xe pagefault ops (producer) */ struct xe_pagefault_ops { /** diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 6c3033fc4db7..f39e647512ad 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -1304,16 +1304,20 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, /* Always process UNMAPs first so view SVM ranges is current */ err = xe_svm_garbage_collector(vm); - if (err) + if (err) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR); return err; + } dpagemap = ctx.devmem_only ? xe_tile_local_pagemap(tile) : xe_vma_resolve_pagemap(vma, tile); ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap); range = xe_svm_range_find_or_insert(vm, fault_addr, vma, &ctx); - if (IS_ERR(range)) + if (IS_ERR(range)) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND); return PTR_ERR(range); + } xe_svm_range_fault_count_stats_incr(gt, range); @@ -1415,6 +1419,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, err = PTR_ERR(fence); xe_validation_retry_on_oom(&vctx, &err); xe_svm_range_bind_us_stats_incr(gt, range, bind_start); + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_REBIND); break; } } @@ -1437,6 +1442,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, err_out: if (err == -EAGAIN) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_NONE); ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ range_debug(range, "PAGE FAULT - RETRY BIND"); goto retry; @@ -1469,8 +1475,10 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, int need_vram, ret; retry: need_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); - if (need_vram < 0) + if (need_vram < 0) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK); return need_vram; + } ret = __xe_svm_handle_pagefault(vm, vma, pf, gt, fault_addr, need_vram ? true : false); @@ -1480,11 +1488,16 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, * may have been split by xe_svm_range_set_default_attr. */ vma = xe_vm_find_vma_by_addr(vm, fault_addr); - if (!vma) + if (!vma) { + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND); return -EINVAL; + } + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_NONE); goto retry; } + if (ret && xe_pagefault_get_error(pf) == XE_PAGEFAULT_ERROR_NONE) + xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED); return ret; } diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index cc96e2b7f3a7..6b793ef31537 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -29,6 +29,7 @@ #include "xe_exec_queue.h" #include "xe_gt.h" #include "xe_migrate.h" +#include "xe_pagefault.h" #include "xe_pat.h" #include "xe_pm.h" #include "xe_preempt_fence.h" @@ -643,7 +644,7 @@ void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf) return; } - e->address = pf->consumer.page_addr; + e->address = xe_pagefault_addr(pf); /* * TODO: * Address precision is currently always SZ_4K, but this may change From 36a86c23588b8f57c9d20feb4cf5a2ab27e3baba Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 17 Sep 2026 13:31:58 -0700 Subject: [PATCH 72/76] drm/xe: Keep walking on SVM eviction failure The desired behavior for SVM eviction failures, which can occur due to various uncontrollable races, is for TTM to continue walking the LRU list and look for another eviction candidate. This is expressed by returning -ENOSPC from the ->move() callback. Adjust the SVM eviction failure path because of races in ->move() to return -ENOSPC so that TTM continues searching for another buffer to evict. Fixes: 3ca608dc7561 ("drm/xe: Basic SVM BO eviction") Cc: stable@vger.kernel.org Signed-off-by: Matthew Brost Reviewed-by: Himal Prasad Ghimiray Link: https://patch.msgid.link/20260917203158.292823-1-matthew.brost@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_bo.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 9f3f0cb95afa..f2ab9bf43a86 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1044,6 +1044,13 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, } else { drm_dbg(&xe->drm, "Evict system allocator BO failed=%pe\n", ERR_PTR(ret)); + /* + * The semantic we want upon SVM eviction failure + * because of racing access is keep walking for + * eviction, which is -ENOSPC. + */ + if (ret == -EBUSY) + ret = -ENOSPC; } goto out; From b66b036c959f49decd2202db4026d3ae21e01388 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Fri, 18 Sep 2026 13:43:21 +0530 Subject: [PATCH 73/76] drm/pagemap: Prevent CPU stalls during unbounded pagemap teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit drm_pagemap_dev_unhold_work() processes an unbatched llist of pagemaps on the system workqueue. This creates potential latency traps during heavy teardown cycles due to two compounding factors: 1. The loop contains a per-item drm_dbg() trace. If dynamic debug is enabled on a slow serial console, synchronous print latency scales linearly with the batch size and can block worker execution. 2. The llist can accumulate large batches during intensive unmap operations, monopolizing worker resources without yielding. This causes below log warning: "workqueue: drm_pagemap_dev_unhold_work [drm_gpusvm_helper] hogged CPU for >10000us 19 times, consider switching to WQ_UNBOUND" Fix this by implementing a latency mitigation strategy: - Move the teardown work to system_unbound_wq to avoid tying up per-CPU bound worker pools. - Add cond_resched() inside the teardown loop to yield the CPU during large batch teardowns, ensuring system responsiveness. - Convert the drm_dbg() trace to drm_dbg_ratelimited() to mitigate serial console bottlenecks while preserving debuggability. v2: - Changed workqueue from deprecated system_unbound_wq to system_dfl_wq. (Arvind Yadav) Fixes: a26084328ac4 ("drm/pagemap, drm/xe: Manage drm_pagemap provider lifetimes") Reviewed-by: Arvind Yadav Reviewed-by: Thomas Hellström Signed-off-by: Varun Gupta Link: https://patch.msgid.link/20260918081320.30389-2-varun.gupta@intel.com Signed-off-by: Himal Prasad Ghimiray Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/drm_pagemap.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 892b325fa99b..0867bfbde5c5 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -986,7 +986,7 @@ static void drm_pagemap_release(struct kref *ref) dpagemap->dev_hold = NULL; drm_pagemap_shrinker_add(dpagemap); llist_add(&dev_hold->link, &drm_pagemap_unhold_list); - schedule_work(&drm_pagemap_work); + queue_work(system_dfl_wq, &drm_pagemap_work); /* * Here, either the provider device is still alive, since if called from * page_free(), the caller is holding a reference on the dev_pagemap, @@ -1009,10 +1009,11 @@ static void drm_pagemap_dev_unhold_work(struct work_struct *work) struct drm_device *drm = dev_hold->drm; struct module *module = drm->driver->fops->owner; - drm_dbg(drm, "Releasing reference on provider device and module.\n"); + drm_dbg_ratelimited(drm, "Releasing reference on provider device and module.\n"); drm_dev_put(drm); module_put(module); kfree(dev_hold); + cond_resched(); } } From 3bc497a40178a9e0ec91456fd9a99ba7434094b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= Date: Mon, 14 Sep 2026 13:46:17 +0200 Subject: [PATCH 74/76] drm/xe: Make p2pdma distance check verbose via dynamic debug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pci_p2pdma_distance() can optionally print details about how it computed the ACS/PCIe topology distance between the two devices, but xe_dma_buf_attach() always called it with verbose set to false, silently hiding useful diagnostics whenever a dma-buf attach unset attach->peer2peer. Tie the verbose argument to a dedicated dynamic-debug callsite tagged "xe dma-buf p2p debug" (in addition to keeping the existing DRM_UT_DRIVER category as a default), so the extra p2pdma distance logging can be switched on without a recompile in any of the following ways: - Enable the whole DRIVER category for xe via drm.debug, which is already used elsewhere in the driver: # echo 0x10 > /sys/module/drm/parameters/debug - Enable dynamic debug for the whole file: # echo 'file xe_dma_buf.c +p' > \ /sys/kernel/debug/dynamic_debug/control - Enable dynamic debug for just this callsite by function name: # echo 'func xe_dma_buf_attach +p' > \ /sys/kernel/debug/dynamic_debug/control - Enable dynamic debug by matching the format string used to register the callsite: # echo 'format "xe dma-buf p2p debug" +p' > \ /sys/kernel/debug/dynamic_debug/control - Enable it at boot via the dyndbg= kernel command line parameter: dyndbg="func xe_dma_buf_attach +p" Any of the above flips verbose_p2p_dist to true for that attach call, causing pci_p2pdma_distance() to log its distance calculation without requiring the whole DRM_UT_DRIVER category to be enabled. v2: - Guard the _dynamic_func_call() invocation and xe_ddebug_flip_bool() with a CONFIG_DYNAMIC_DEBUG check so the build does not break when dynamic debug is disabled. (Sashiko-bot) Cc: Matthew Brost Cc: Matthew Auld Assisted-by: LLM Signed-off-by: Thomas Hellström Reviewed-by: Matthew Auld Reviewed-by: Michael J. Ruhl Link: https://patch.msgid.link/20260914114617.112012-1-thomas.hellstrom@linux.intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_dma_buf.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index 5d9f1cd24b7f..6a85b292dee7 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -22,13 +23,29 @@ MODULE_IMPORT_NS("DMA_BUF"); +#if defined(CONFIG_DYNAMIC_DEBUG) || \ + (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) +static void xe_ddebug_flip_bool(const struct _ddebug *desc, bool *flag) +{ + *flag = true; +} + +#define xe_dma_buf_p2p_debug_flip(flag) \ + _dynamic_func_call("xe dma-buf p2p debug", xe_ddebug_flip_bool, flag) +#else +#define xe_dma_buf_p2p_debug_flip(flag) do { } while (0) +#endif + static int xe_dma_buf_attach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) { struct drm_gem_object *obj = attach->dmabuf->priv; + bool verbose_p2p_dist = drm_debug_enabled(DRM_UT_DRIVER); + + xe_dma_buf_p2p_debug_flip(&verbose_p2p_dist); if (attach->peer2peer && - pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0) + pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, verbose_p2p_dist) < 0) attach->peer2peer = false; if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT)) From 25a7255afb70b608672a50c921edf20d83ce745e Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Thu, 17 Sep 2026 15:33:39 -0700 Subject: [PATCH 75/76] drm/xe: Consolidate semaphore instruction definitions Drop the definition of MI_SEMAPHORE_WAIT_TOKEN and replace its uses with MI_SEMAPHORE_WAIT. Both #define's were representing the same opcode 0x1C and this GPU instruction has always been named "MI_SEMAPHORE_WAIT" in the official specs. The "*_TOKEN" definition appears to have been copied from i915's code. i915 needed two separate definitions to reflect the slightly different instruction lengths gen8-gen11 platforms (4 dwords) vs gen12 and later platforms (5 dwords, with the extra dword representing a token), but Xe only supports newer platforms with the 5-dword form, so only a single definition is needed and should match the official name given in the specs. Bspec: 14054, 45749, 60244 Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20260917-semaphore_inst-v1-1-61e49405f71d@intel.com Signed-off-by: Matt Roper --- drivers/gpu/drm/xe/instructions/xe_mi_commands.h | 7 +------ drivers/gpu/drm/xe/xe_ring_ops.c | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/xe/instructions/xe_mi_commands.h b/drivers/gpu/drm/xe/instructions/xe_mi_commands.h index ad7d98f2dbba..835ef75bbeb1 100644 --- a/drivers/gpu/drm/xe/instructions/xe_mi_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_mi_commands.h @@ -36,6 +36,7 @@ #define MI_SEMAPHORE_WAIT (__MI_INSTR(0x1c) | XE_INSTR_NUM_DW(5)) #define MI_SEMW_GGTT REG_BIT(22) +#define MI_SEMW_REGISTER_POLL REG_BIT(16) #define MI_SEMW_POLL REG_BIT(15) #define MI_SEMW_COMPARE_OP_MASK REG_GENMASK(14, 12) #define COMPARE_OP_SAD_GT_SDD 0 @@ -94,10 +95,4 @@ #define MI_SET_APPID_SESSION_ID_MASK REG_GENMASK(6, 0) #define MI_SET_APPID_SESSION_ID(x) REG_FIELD_PREP(MI_SET_APPID_SESSION_ID_MASK, x) -#define MI_SEMAPHORE_WAIT_TOKEN (__MI_INSTR(0x1c) | XE_INSTR_NUM_DW(5)) /* XeLP+ */ -#define MI_SEMAPHORE_REGISTER_POLL REG_BIT(16) -#define MI_SEMAPHORE_POLL REG_BIT(15) -#define MI_SEMAPHORE_CMP_OP_MASK REG_GENMASK(14, 12) -#define MI_SEMAPHORE_SAD_EQ_SDD REG_FIELD_PREP(MI_SEMAPHORE_CMP_OP_MASK, 4) - #endif diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 08b4a4283e96..46ab1f0f3564 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -55,8 +55,8 @@ __emit_aux_table_inv(u32 *cmd, const struct xe_reg reg, u32 adj_offset) MI_LRI_MMIO_REMAP_EN; *cmd++ = reg.addr + adj_offset; *cmd++ = AUX_INV; - *cmd++ = MI_SEMAPHORE_WAIT_TOKEN | MI_SEMAPHORE_REGISTER_POLL | - MI_SEMAPHORE_POLL | MI_SEMAPHORE_SAD_EQ_SDD; + *cmd++ = MI_SEMAPHORE_WAIT | MI_SEMW_REGISTER_POLL | + MI_SEMW_POLL | MI_SEMW_COMPARE(SAD_EQ_SDD); *cmd++ = 0; *cmd++ = reg.addr + adj_offset; *cmd++ = 0; From 320662381ca243a3364663e917a7447afaf618fa Mon Sep 17 00:00:00 2001 From: Vinay Belgaumkar Date: Wed, 16 Sep 2026 12:06:14 -0700 Subject: [PATCH 76/76] drm/xe: Poll GT for C6 before D3 Ensure GT C6 state before D3 transition in runtime suspend flow. This avoids the scenario where GT is being accessed after D3 state is programmed. If the check for C6 fails, resume the device so it is not stuck in an intermediate state. v2: Force resume when we cancel suspend due to C6 check (Sashiko) v3: Abort the suspend properly when idle check fails (Sashiko) v4: Only check for C6 during suspend (Rodrigo). Have check in xe_pci_runtime_suspend() instead of xe_pci_suspend(). v5: Don't checkfor PVC (Sashiko), rename function from_device_c6 to _device_idle v6: Rename to _wait_all_c6 and move PVC check to gt_idle (Rodrigo) Cc: Badal Nilawar Cc: Rodrigo Vivi Signed-off-by: Vinay Belgaumkar Reviewed-by: Rodrigo Vivi Link: https://patch.msgid.link/20260916190614.3047807-1-vinay.belgaumkar@intel.com Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_gt_idle.c | 39 +++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_gt_idle.h | 1 + drivers/gpu/drm/xe/xe_pci.c | 13 ++++++++++- drivers/gpu/drm/xe/xe_pm.c | 18 +++++++++++++++ drivers/gpu/drm/xe/xe_pm.h | 1 + 5 files changed, 71 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index c208d7563e36..2eefd62f8d59 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -8,10 +8,12 @@ #include #include +#include #include "xe_force_wake.h" #include "xe_device.h" #include "xe_gt.h" #include "xe_gt_idle.h" +#include "xe_gt_printk.h" #include "xe_gt_sysfs.h" #include "xe_guc_pc.h" #include "regs/xe_gt_regs.h" @@ -451,3 +453,40 @@ int xe_gt_idle_disable_c6(struct xe_gt *gt) return 0; } + +static int wait_for_gt_c6_state(struct xe_gt *gt, u32 timeout_ms) +{ + struct xe_guc_pc *pc = >->uc.guc.pc; + enum xe_gt_idle_state state; + + return poll_timeout_us(state = gt->gtidle.idle_status(pc), + state == GT_IDLE_C6, + 20, + timeout_ms * USEC_PER_MSEC, + false); +} + +/** + * xe_gt_idle_wait_for_c6 - Poll for GT C6 + * @gt: GT object + * @timeout_ms: wait time in ms + * + * This function waits for GT to enter C6. + * + * Return: 0 on success, -EAGAIN otherwise + */ +int xe_gt_idle_wait_for_c6(struct xe_gt *gt, u32 timeout_ms) +{ + if (IS_SRIOV_VF(gt_to_xe(gt))) + return 0; + + if (gt_to_xe(gt)->info.platform == XE_PVC) + return 0; + + if (wait_for_gt_c6_state(gt, timeout_ms)) { + xe_gt_dbg(gt, "GT is not in C6\n"); + return -EAGAIN; + } + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_idle.h b/drivers/gpu/drm/xe/xe_gt_idle.h index 9c34a155e102..2c8e9adf4a68 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.h +++ b/drivers/gpu/drm/xe/xe_gt_idle.h @@ -18,5 +18,6 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt); void xe_gt_idle_disable_pg(struct xe_gt *gt); int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p); u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle); +int xe_gt_idle_wait_for_c6(struct xe_gt *gt, u32 timeout_ms); #endif /* _XE_GT_IDLE_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 00b2b5dfb8f6..d66d5a0339ae 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -1379,7 +1379,7 @@ static int xe_pci_runtime_suspend(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); struct xe_device *xe = pdev_to_xe_device(pdev); - int err; + int err, ret; /* * We hold an additional reference to the runtime PM to keep PF in D0 @@ -1394,6 +1394,17 @@ static int xe_pci_runtime_suspend(struct device *dev) if (err) return err; + err = xe_pm_wait_all_c6(xe); + if (err) { + drm_dbg(&xe->drm, "Resuming - GT C6 check failed!"); + ret = xe_pm_runtime_resume(xe); + if (ret) { + drm_err(&xe->drm, "Resume failed after suspend was canceled"); + return ret; + } + return err; + } + pci_save_state(pdev); if (xe->d3cold.allowed) { diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index f517bf453b54..4a745c3a933b 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -1030,6 +1030,24 @@ void xe_pm_d3cold_allowed_toggle(struct xe_device *xe) mutex_unlock(&xe->d3cold.lock); } +/** + * xe_pm_wait_all_c6() - Wait for all GTs to enter C6. + * @xe: xe device instance + * + * Return: 0 on success, -EAGAIN on failure + */ +int xe_pm_wait_all_c6(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + + for_each_gt(gt, xe, id) + if (xe_gt_idle_wait_for_c6(gt, 200)) + return -EAGAIN; + + return 0; +} + /** * xe_pm_module_init() - Perform xe_pm specific module initialization. * diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index 6d5ab09cb769..fd8b2eab5de9 100644 --- a/drivers/gpu/drm/xe/xe_pm.h +++ b/drivers/gpu/drm/xe/xe_pm.h @@ -36,6 +36,7 @@ void xe_pm_d3cold_allowed_toggle(struct xe_device *xe); bool xe_rpm_reclaim_safe(const struct xe_device *xe); struct task_struct *xe_pm_read_callback_task(struct xe_device *xe); int xe_pm_block_on_suspend(struct xe_device *xe); +int xe_pm_wait_all_c6(struct xe_device *xe); void xe_pm_might_block_on_suspend(void); int xe_pm_module_init(void);