github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

UAPI Changes:

Browse Source 
- New sysfs to expose device UID (Badal)
 
 Driver Changes:
  - Nuke PTs only after unlinking contested VMAs (Auld)
  - Limit sg segment size to PAGE_SIZE on Xen PV (Szymon)
  - Move locking to xe_sriov_pf_reprovision_default (Michal)
  - New workarounds (Tilak, Tales, Daniel)
  - Keep VF LMEM BAR size low if no VFs enabled (Marcin)
  - Drop ratelimiting from dev_unhold_work debug trace (Varun)
  - System Controller's Maibox response improvement (Anoop)
  - General clean-ups (Roper, Lin)
  - Revert "drm/xe: Set mask bits for CCS_MODE register" (Roper)
  - CPU binds and ULLS on migration queue (Brost)
  - Re-apply an i2c patch silently removed by a merge (Bartosz)
  - New PTL PCI ID (Atwood)
  - Enable PXP for NVL-S (Alan)
  - Let xe_guc_ct_send_locked() callers reserve G2H credit (Sujaritha)
  - Implement print vfunc for page fault info logging (Brian)
  - Do not dma_fence_put zero fences in err_out path (Jonathan)
  - Add focused VM teardown and page fault tracing (Daniel)
  - Fix ULLS chained job loss and GT reset replay (Varun)
  - Refresh TTM LRU on SVM prefetch (Himal)
  - Fix xe_mmio_wait32() to honor delay/sleep maximums (Alan)
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCgAdFiEEbSBwaO7dZQkcLOKj+mJfZA7rE8oFAmq+WHAACgkQ+mJfZA7r
 E8rjYAgAnOyPBR8WJmBFX5GYfE5X5JEK0Ce+bsgbhkDmMsBNe8DJFwML3PHrPC0p
 yalGfrpjgQVIRhOAc8wKzXVBM0bBDJftNwfOTEEdjaUOtXYa+uipvJRs9XqvNpvw
 BO/rNs0lLnPFgHZGU4ieB/cmGwjmIcrv6IekfUkUyvBmq3Kv278mlwR4SawOXkaI
 wQTs8QE+WbpNMwz1Ch5dijAQCFeLovgNE0WVTArzQ6UNWLPsv7vnOtqBHc1jlvLr
 IT9SgsPyGVpKCcxF3P4etgWaytJOU16NABSokI1BT2ShULcPfhFqlbCYNMei3VsP
 h5DQzH36x4aD4qNy9iTDMz2r150oGw==
 =sqoe
 -----END PGP SIGNATURE-----

Merge tag 'drm-xe-next-2026-10-01' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-next

UAPI Changes:
 - New sysfs to expose device UID (Badal)

Driver Changes:
 - Nuke PTs only after unlinking contested VMAs (Auld)
 - Limit sg segment size to PAGE_SIZE on Xen PV (Szymon)
 - Move locking to xe_sriov_pf_reprovision_default (Michal)
 - New workarounds (Tilak, Tales, Daniel)
 - Keep VF LMEM BAR size low if no VFs enabled (Marcin)
 - Drop ratelimiting from dev_unhold_work debug trace (Varun)
 - System Controller's Maibox response improvement (Anoop)
 - General clean-ups (Roper, Lin)
 - Revert "drm/xe: Set mask bits for CCS_MODE register" (Roper)
 - CPU binds and ULLS on migration queue (Brost)
 - Re-apply an i2c patch silently removed by a merge (Bartosz)
 - New PTL PCI ID (Atwood)
 - Enable PXP for NVL-S (Alan)
 - Let xe_guc_ct_send_locked() callers reserve G2H credit (Sujaritha)
 - Implement print vfunc for page fault info logging (Brian)
 - Do not dma_fence_put zero fences in err_out path (Jonathan)
 - Add focused VM teardown and page fault tracing (Daniel)
 - Fix ULLS chained job loss and GT reset replay (Varun)
 - Refresh TTM LRU on SVM prefetch (Himal)
 - Fix xe_mmio_wait32() to honor delay/sleep maximums (Alan)

Signed-off-by: Dave Airlie <airlied@redhat.com>

From: Rodrigo Vivi <rodrigo.vivi@intel.com>
Link: https://patch.msgid.link/ar5YsqAK3bS4Ta6Z@intel.com
This commit is contained in:
Dave Airlie 2026-10-02 07:43:30 +10:00
commit 845c5cc369
81 changed files with 2717 additions and 1370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Documentation/ABI/testing/sysfs-driver-intel-xe-gpu Normal file
View File

@ -0,0 +1,10 @@
What: /sys/bus/pci/drivers/xe/.../device_uid
Date: October 2026
KernelVersion: 7.4
Contact: intel-xe@lists.freedesktop.org
Description:
RO. Unique 64-bit identifier of the GPU SoC device, exposed as
hexadecimal value.
This sysfs file is present only on supported Intel Xe platforms.
Accessible only to users with administrative privileges.

3
Documentation/gpu/xe/xe_migrate.rst
View File

@ -6,3 +6,6 @@ Migrate Layer
.. kernel-doc:: drivers/gpu/drm/xe/xe_migrate_doc.h
:doc: Migrate Layer
.. kernel-doc:: drivers/gpu/drm/xe/xe_migrate.c
:doc: ULLS (Ultra Low Latency Submission) for migration jobs

105
drivers/gpu/drm/drm_gpusvm.c
View File

@ -1528,7 +1528,49 @@ static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages,
}
/**
* drm_gpusvm_dma_map_pages() - DMA map one drm_gpusvm_pages instance
* drm_gpusvm_walk_devmem() - Invoke the devmem callback across faulted pages
* @gpusvm: Pointer to the GPU SVM structure
* @pfns: The already-faulted pfn array (size @npages)
* @npages: Number of pages in the CPU range
* @ctx: GPU SVM context, with a non-NULL &drm_gpusvm_ctx.devmem_fn
*
* Invoke &drm_gpusvm_ctx.devmem_fn once per contiguous run of @pfns backed by
* the same device-memory allocation. Must be called under the notifier lock.
*/
static void drm_gpusvm_walk_devmem(struct drm_gpusvm *gpusvm,
unsigned long *pfns,
unsigned long npages,
const struct drm_gpusvm_ctx *ctx)
{
struct drm_pagemap_devmem *last = NULL;
unsigned int order = 0;
unsigned long i;
lockdep_assert_held(&gpusvm->notifier_lock);
for (i = 0; i < npages; i += 1 << order) {
struct page *page = hmm_pfn_to_page(pfns[i]);
struct drm_pagemap_devmem *devmem;
order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
if (!is_device_private_page(page) &&
!is_device_coherent_page(page)) {
last = NULL;
continue;
}
devmem = drm_pagemap_page_to_devmem(page);
if (devmem == last)
continue;
last = devmem;
ctx->devmem_fn(devmem);
}
}
/**
* drm_gpusvm_dma_map_pages() - Walk and DMA map one drm_gpusvm_pages instance
* @gpusvm: Pointer to the GPU SVM structure
* @svm_pages: The SVM pages instance to populate with dma-addresses
* @pfns: The already-faulted pfn array (size @npages)
@ -1536,10 +1578,10 @@ static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages,
* @ctx: GPU SVM context
* @dma_dir: DMA data direction for the mappings
*
* Map the faulted @pfns into @svm_pages for DMA access through its owning
* drm_device. Must be called under the notifier lock and only for an instance
* without a live mapping. On failure this unwinds the partial mapping of this
* instance before returning.
* Walk the faulted @pfns and map them into @svm_pages for DMA access through
* its owning drm_device. Must be called under the notifier lock and only for
* an instance without a live mapping. On failure this unwinds the partial
* mapping of this instance before returning.
*
* Return: 0 on success, negative error code on failure.
*/
@ -1774,9 +1816,20 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
if (map_dma &&
drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages, num_pages))
goto set_seqno;
/*
* drm_gpusvm_pages_valid_unlocked() also resets any instance whose
* mapping was invalidated, so it must run on the map_dma path even when
* devmem_fn suppresses the fast path; otherwise a stale inline mapping
* is later mistaken for a dma_addr array by drm_gpusvm_dma_map_pages().
*/
if (map_dma) {
bool pages_valid = drm_gpusvm_pages_valid_unlocked(gpusvm,
svm_pages,
num_pages);
if (!ctx->devmem_fn && pages_valid)
goto set_seqno;
}
pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
if (!pfns)
@ -1829,28 +1882,30 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
goto retry;
}
if (!map_dma)
goto done_mapping;
if (ctx->devmem_fn)
drm_gpusvm_walk_devmem(gpusvm, pfns, npages, ctx);
for (p = 0; p < num_pages; ++p) {
if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p]))
continue;
if (map_dma) {
for (p = 0; p < num_pages; ++p) {
if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p]))
continue;
err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns,
npages, ctx, dma_dir);
if (err) {
/*
* The failing instance was unwound by the helper. Keep
* the ones mapped earlier: the -EAGAIN retry reuses
* them, and the driver unmaps every instance with the
* range on the other error paths.
*/
drm_gpusvm_notifier_unlock(gpusvm);
goto err_free;
err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns,
npages, ctx, dma_dir);
if (err) {
/*
* The failing instance was unwound by the
* helper. Keep the ones mapped earlier: the
* -EAGAIN retry reuses them, and the driver
* unmaps every instance with the range on the
* other error paths.
*/
drm_gpusvm_notifier_unlock(gpusvm);
goto err_free;
}
}
}
done_mapping:
drm_gpusvm_notifier_unlock(gpusvm);
kvfree(pfns);
set_seqno:

17
drivers/gpu/drm/drm_pagemap.c
View File

@ -1102,7 +1102,7 @@ static void drm_pagemap_dev_unhold_work(struct work_struct *work)
struct drm_device *drm = dev_hold->drm;
struct module *module = drm->driver->fops->owner;
drm_dbg_ratelimited(drm, "Releasing reference on provider device and module.\n");
drm_dbg(drm, "Releasing reference on provider device and module.\n");
drm_dev_put(drm);
module_put(module);
kfree(dev_hold);
@ -1629,6 +1629,21 @@ struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page)
}
EXPORT_SYMBOL_GPL(drm_pagemap_page_to_dpagemap);
/**
* drm_pagemap_page_to_devmem() - Return the devmem allocation backing a page
* @page: The struct page.
*
* Return: The &drm_pagemap_devmem backing @page. Undefined if @page was not
* populated from a &drm_pagemap.
*/
struct drm_pagemap_devmem *drm_pagemap_page_to_devmem(struct page *page)
{
struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page);
return zdd->devmem_allocation;
}
EXPORT_SYMBOL_GPL(drm_pagemap_page_to_devmem);
/**
* drm_pagemap_populate_mm() - Populate a virtual range with device memory pages
* @dpagemap: Pointer to the drm_pagemap managing the device memory

1
drivers/gpu/drm/xe/Makefile
View File

@ -35,6 +35,7 @@ $(obj)/generated/%_device_wa_oob.c $(obj)/generated/%_device_wa_oob.h: $(obj)/xe
xe-y += xe_bb.o \
xe_bo.o \
xe_bo_evict.o \
xe_cpu_bind.o \
xe_dep_scheduler.o \
xe_devcoredump.o \
xe_device.o \

1
drivers/gpu/drm/xe/abi/guc_klvs_abi.h
View File

@ -525,6 +525,7 @@ enum xe_guc_klv_ids {
GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG = 0x900e,
GUC_WA_KLV_REMAP_RANGED_TLB_INV = 0x900f,
GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64 = 0x9010,
GUC_WA_KLV_IGNORE_MULTIQ_CTX_COMPLETE_WITH_PREEMPT_IN_CSB = 0x9011,
};
/**

3
drivers/gpu/drm/xe/regs/xe_gt_regs.h
View File

@ -468,6 +468,7 @@
#define L3_SQ_DISABLE_COAMA_2WAY_COH REG_BIT(30)
#define L3_SQ_DISABLE_COAMA REG_BIT(22)
#define COMPMEMRD256BOVRFETCHEN REG_BIT(20)
#define NONCOMPMEMRD256BOVRFETCHEN REG_BIT(14)
#define L3SQCREG3 XE_REG_MCR(0xb108)
#define COMPPWOVERFETCHEN REG_BIT(28)
@ -636,7 +637,7 @@
* [4-6] RSVD
* [7] Disabled
*/
#define CCS_MODE XE_REG(0x14804, XE_REG_OPTION_MASKED)
#define CCS_MODE XE_REG(0x14804)
#define CCS_MODE_CSLICE_0_3_MASK REG_GENMASK(11, 0) /* 3 bits per cslice */
#define CCS_MODE_CSLICE_MASK 0x7 /* CCS0-3 + rsvd */
#define CCS_MODE_CSLICE_WIDTH ilog2(CCS_MODE_CSLICE_MASK + 1)

2
drivers/gpu/drm/xe/regs/xe_regs.h
View File

@ -30,6 +30,8 @@
#define XEHP_MTCFG_ADDR XE_REG(0x101800)
#define TILE_COUNT REG_GENMASK(15, 8)
#define CRI_DEVICE_UID XE_REG(0x102008)
#define GGC XE_REG(0x108040)
#define GMS_MASK REG_GENMASK(15, 8)
#define GGMS_MASK REG_GENMASK(7, 6)

4
drivers/gpu/drm/xe/xe_bo.c
View File

@ -1879,7 +1879,7 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
xe_drm_client_remove_bo(bo);
#endif
if (bo->vm && xe_bo_is_user(bo))
if (bo->vm && (xe_bo_is_user(bo) || bo->flags & XE_BO_FLAG_PAGETABLE))
xe_vm_put(bo->vm);
if (bo->parent_obj)
@ -2575,7 +2575,7 @@ __xe_bo_create_locked(struct xe_device *xe,
* by having all the vm's bo refereferences released at vm close
* time.
*/
if (vm && xe_bo_is_user(bo))
if (vm && (xe_bo_is_user(bo) || bo->flags & XE_BO_FLAG_PAGETABLE))
xe_vm_get(vm);
bo->vm = vm;

29
drivers/gpu/drm/xe/xe_bo.h
View File

@ -9,6 +9,8 @@
#include <drm/drm_prime.h>
#include <drm/ttm/ttm_tt.h>
#include <xen/xen.h>
#include "xe_bo_types.h"
#include "xe_ggtt.h"
#include "xe_macros.h"
@ -494,6 +496,8 @@ void __xe_bo_release_dummy(struct kref *kref);
* @bo: The bo to put.
* @deferred: List to which to add the buffer object if we cannot put, or
* NULL if the function is to put unconditionally.
* @added: BO was added to deferred list, written back to caller, can be NULL if
* writeback is not needed. Only set to true when added, never set to false.
*
* Since the final freeing of an object includes both sleeping and (!)
* memory allocation in the dma_resv individualization, it's not ok
@ -513,7 +517,8 @@ void __xe_bo_release_dummy(struct kref *kref);
* false otherwise.
*/
static inline bool
xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)
xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred,
bool *added)
{
if (!deferred) {
xe_bo_put(bo);
@ -523,6 +528,9 @@ xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)
if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy))
return false;
if (added)
*added = true;
return llist_add(&bo->freed, deferred);
}
@ -539,7 +547,7 @@ xe_bo_put_async(struct xe_bo *bo)
{
struct xe_bo_dev *bo_device = &xe_bo_device(bo)->bo_device;
if (xe_bo_put_deferred(bo, &bo_device->async_list))
if (xe_bo_put_deferred(bo, &bo_device->async_list, NULL))
schedule_work(&bo_device->async_free);
}
@ -574,6 +582,23 @@ static inline unsigned int xe_sg_segment_size(struct device *dev)
struct scatterlist __maybe_unused sg;
size_t max = BIT_ULL(sizeof(sg.length) * 8) - 1;
/*
* For Xen PV guests pages aren't contiguous in DMA (machine) address
* space. The DMA API takes care of that both in dma_alloc_* (by
* calling into the hypervisor to make the pages contiguous) and in
* dma_map_* (by bounce buffering). But xe (like i915, see commit
* 78a07fe777c4) ignores the coherency aspects of the DMA API and thus
* can't cope with bounce buffering actually happening, so add a hack
* here to force small allocations and mappings when running in PV
* mode on Xen.
*
* Note this will still break if bounce buffering is required for other
* reasons, like confidential computing hypervisors or PCIe root ports
* with addressing limitations.
*/
if (xen_pv_domain())
return PAGE_SIZE;
max = min_t(size_t, max, dma_max_mapping_size(dev));
/*

2
drivers/gpu/drm/xe/xe_bo_types.h
View File

@ -90,8 +90,6 @@ struct xe_bo {
/** @freed: List node for delayed put. */
struct llist_node freed;
/** @update_index: Update index if PT BO */
int update_index;
/** @created: Whether the bo has passed initial creation */
bool created;

67
drivers/gpu/drm/xe/xe_configfs.c
View File

@ -61,6 +61,7 @@
* ├── survivability_mode
* ├── gt_types_allowed
* ├── engines_allowed
* ├── migrate_ulls_period_ms
* ├── enable_psmi
* └── disable_vram_page_offline
*
@ -262,6 +263,20 @@
*
* This attribute can only be set before binding to the device.
*
* Migrate ULLS Period (ms)
* ------------------------
*
* Migrate ULLS period, in milliseconds. This is the delay between entering
* migrate ULLS (a continuously running batch) and exiting it. Migrate ULLS is
* currently entered during page faults and SVM prefetch operations. Default 5,
* zero indicates ULLS is disabled.
*
* How to disable migration ULLS:
*
* # echo 0 > /sys/kernel/config/xe/0000:03:00.0/migrate_ulls_period_ms
*
* This attribute can only be set before binding to the device.
*
* Remove devices
* ==============
*
@ -283,6 +298,7 @@ struct xe_config_group_device {
struct xe_config_device {
u64 gt_types_allowed;
u64 engines_allowed;
u32 migrate_ulls_period_ms;
struct wa_bb ctx_restore_post_bb[XE_ENGINE_CLASS_MAX];
struct wa_bb ctx_restore_mid_bb[XE_ENGINE_CLASS_MAX];
bool survivability_mode;
@ -306,6 +322,7 @@ struct xe_config_group_device {
static const struct xe_config_device device_defaults = {
.gt_types_allowed = U64_MAX,
.engines_allowed = U64_MAX,
.migrate_ulls_period_ms = 5,
.survivability_mode = false,
.enable_psmi = false,
.enable_multi_queue = true,
@ -631,6 +648,33 @@ static ssize_t enable_multi_queue_store(struct config_item *item, const char *pa
return len;
}
static ssize_t migrate_ulls_period_ms_show(struct config_item *item, char *page)
{
struct xe_config_device *dev = to_xe_config_device(item);
return sprintf(page, "%u\n", dev->migrate_ulls_period_ms);
}
static ssize_t migrate_ulls_period_ms_store(struct config_item *item,
const char *page, size_t len)
{
struct xe_config_group_device *dev = to_xe_config_group_device(item);
u32 val;
int ret;
ret = kstrtou32(page, 0, &val);
if (ret)
return ret;
guard(mutex)(&dev->lock);
if (is_bound(dev))
return -EBUSY;
dev->config.migrate_ulls_period_ms = val;
return len;
}
static ssize_t disable_vram_page_offline_show(struct config_item *item, char *page)
{
struct xe_config_device *dev = to_xe_config_device(item);
@ -896,6 +940,7 @@ static ssize_t ctx_restore_post_bb_store(struct config_item *item,
CONFIGFS_ATTR(, ctx_restore_mid_bb);
CONFIGFS_ATTR(, ctx_restore_post_bb);
CONFIGFS_ATTR(, enable_multi_queue);
CONFIGFS_ATTR(, migrate_ulls_period_ms);
CONFIGFS_ATTR(, enable_psmi);
CONFIGFS_ATTR(, disable_vram_page_offline);
CONFIGFS_ATTR(, engines_allowed);
@ -906,6 +951,7 @@ static struct configfs_attribute *xe_config_device_attrs[] = {
&attr_ctx_restore_mid_bb,
&attr_ctx_restore_post_bb,
&attr_enable_multi_queue,
&attr_migrate_ulls_period_ms,
&attr_enable_psmi,
&attr_disable_vram_page_offline,
&attr_engines_allowed,
@ -1189,6 +1235,7 @@ static void dump_custom_dev_config(struct pci_dev *pdev,
PRI_CUSTOM_ATTR("%llx", gt_types_allowed);
PRI_CUSTOM_ATTR("%llx", engines_allowed);
PRI_CUSTOM_ATTR("%u", migrate_ulls_period_ms);
PRI_CUSTOM_ATTR("%d", enable_multi_queue);
PRI_CUSTOM_ATTR("%d", enable_psmi);
PRI_CUSTOM_ATTR("%d", disable_vram_page_offline);
@ -1340,6 +1387,26 @@ bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev)
return ret;
}
/**
* xe_configfs_get_migrate_ulls_period_ms - get configfs migrate_ulls_period_ms setting
* @pdev: pci device
*
* Return: Migrate ULLS period in milliseconds (zero is disabled).
*/
u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev)
{
struct xe_config_group_device *dev = find_xe_config_group_device(pdev);
u32 ret;
if (!dev)
return device_defaults.migrate_ulls_period_ms;
ret = dev->config.migrate_ulls_period_ms;
config_group_put(&dev->group);
return ret;
}
/**
* xe_configfs_get_disable_vram_page_offline - get configfs disable_vram_page_offline setting
* @pdev: pci device

2
drivers/gpu/drm/xe/xe_configfs.h
View File

@ -24,6 +24,7 @@ bool xe_configfs_media_gt_allowed(struct pci_dev *pdev);
u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev);
bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev);
bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev);
u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev);
bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev);
u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
enum xe_engine_class class,
@ -45,6 +46,7 @@ static inline bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) { return t
static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; }
static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; }
static inline bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) { return true; }
static inline u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev) { return 5; }
static inline bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev) { return false; }
static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
enum xe_engine_class class,

296
drivers/gpu/drm/xe/xe_cpu_bind.c Normal file
View File

@ -0,0 +1,296 @@
// SPDX-License-Identifier: MIT
/*
* Copyright © 2026 Intel Corporation
*/
#include <drm/drm_managed.h>
#include <linux/mutex.h>
#include "xe_cpu_bind.h"
#include "xe_device_types.h"
#include "xe_exec_queue.h"
#include "xe_pt.h"
#include "xe_sched_job.h"
#include "xe_trace_bo.h"
#include "xe_vm.h"
/**
* struct xe_cpu_bind - cpu_bind context.
*/
struct xe_cpu_bind {
/** @xe: Xe device */
struct xe_device *xe;
/** @q: Default exec queue used for kernel binds */
struct xe_exec_queue *q;
/** @job_mutex: Timeline mutex for @q. */
struct mutex job_mutex;
};
static bool is_cpu_bind_queue(struct xe_cpu_bind *cpu_bind,
struct xe_exec_queue *q)
{
return cpu_bind->q == q;
}
static void xe_cpu_bind_fini(void *arg)
{
struct xe_cpu_bind *cpu_bind = arg;
mutex_destroy(&cpu_bind->job_mutex);
xe_exec_queue_put(cpu_bind->q);
}
/**
* xe_cpu_bind_init() - Initialize a cpu_bind context
* @xe: &struct xe_device
*
* Return: 0 if successful, negative error code on failure
*/
int xe_cpu_bind_init(struct xe_device *xe)
{
struct xe_cpu_bind *cpu_bind =
drmm_kzalloc(&xe->drm, sizeof(*cpu_bind), GFP_KERNEL);
struct xe_exec_queue *q;
if (!cpu_bind)
return -ENOMEM;
q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe), NULL,
EXEC_QUEUE_FLAG_KERNEL |
EXEC_QUEUE_FLAG_MIGRATE, 0);
if (IS_ERR(q))
return PTR_ERR(q);
cpu_bind->xe = xe;
cpu_bind->q = q;
xe->cpu_bind = cpu_bind;
mutex_init(&cpu_bind->job_mutex);
fs_reclaim_acquire(GFP_KERNEL);
might_lock(&cpu_bind->job_mutex);
fs_reclaim_release(GFP_KERNEL);
return devm_add_action_or_reset(cpu_bind->xe->drm.dev, xe_cpu_bind_fini,
cpu_bind);
}
/**
* xe_cpu_bind_queue() - Get the bind queue from cpu_bind context.
* @cpu_bind: The cpu bind context.
*
* Return: Pointer to bind queue.
*/
struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind)
{
return cpu_bind->q;
}
/**
* xe_cpu_bind_update_pgtables_execute() - Update a VM's PTEs via the CPU
* @vm: The VM being updated
* @tile: The tile being updated
* @ops: The CPU bind PT update ops
* @pt_op: The VM PT update op
* @num_ops: The number of The VM PT update ops
* @force_clear: Force clear operation
*
* Execute the VM PT update ops array which results in a VM's PTEs being updated
* via the CPU.
*/
void
xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile,
const struct xe_cpu_bind_pt_update_ops *ops,
struct xe_vm_pgtable_update_op *pt_op,
u32 num_ops, bool force_clear)
{
u32 j, i;
for (j = 0; j < num_ops; ++j, ++pt_op) {
for (i = 0; i < pt_op->num_entries; i++) {
const struct xe_vm_pgtable_update *update =
&pt_op->entries[i];
xe_assert(vm->xe, !iosys_map_is_null(&update->pt_bo->vmap));
if (pt_op->bind && !force_clear)
ops->populate(tile, &update->pt_bo->vmap,
update);
else
ops->clear(vm, tile, &update->pt_bo->vmap,
update);
}
}
trace_xe_vm_cpu_bind(vm);
xe_device_wmb(vm->xe);
}
static struct dma_fence *
xe_cpu_bind_update_pgtables_no_job(struct xe_cpu_bind *cpu_bind,
struct xe_cpu_bind_pt_update *pt_update)
{
const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops;
struct xe_vm *vm = pt_update->vops->vm;
struct xe_tile *tile;
int err, id;
if (ops->pre_commit) {
pt_update->job = NULL;
err = ops->pre_commit(pt_update);
if (err)
return ERR_PTR(err);
}
for_each_tile(tile, vm->xe, id) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&pt_update->vops->pt_update_ops[tile->id];
if (!pt_update_ops->pt_job_ops)
continue;
xe_cpu_bind_update_pgtables_execute(vm, tile, ops,
pt_update_ops->pt_job_ops->ops,
pt_update_ops->pt_job_ops->current_op,
false);
}
return dma_fence_get_stub();
}
static struct dma_fence *
xe_cpu_bind_update_pgtables_job(struct xe_cpu_bind *cpu_bind,
struct xe_cpu_bind_pt_update *pt_update)
{
const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops;
struct xe_exec_queue *q = pt_update->vops->q;
struct xe_device *xe = cpu_bind->xe;
struct xe_sched_job *job;
struct dma_fence *fence;
struct xe_tile *tile;
int err, id;
bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
job = xe_sched_job_create(q, NULL);
if (IS_ERR(job))
return ERR_CAST(job);
xe_assert(xe, job->is_pt_job);
if (ops->pre_commit) {
pt_update->job = job;
err = ops->pre_commit(pt_update);
if (err)
goto err_job;
}
if (is_cpu_bind)
mutex_lock(&cpu_bind->job_mutex);
job->pt_update[0].vm = pt_update->vops->vm;
job->pt_update[0].ops = ops;
for_each_tile(tile, xe, id) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&pt_update->vops->pt_update_ops[tile->id];
job->pt_update[0].pt_job_ops[tile->id] =
xe_pt_job_ops_get(pt_update_ops->pt_job_ops);
}
xe_sched_job_arm(job);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
if (is_cpu_bind)
mutex_unlock(&cpu_bind->job_mutex);
return fence;
err_job:
xe_sched_job_put(job);
return ERR_PTR(err);
}
/**
* xe_cpu_bind_update_pgtables() - Pipelined page-table update
* @cpu_bind: The cpu bind context.
* @pt_update: PT update arguments
*
* Perform a pipelined page-table update. The update descriptors are typically
* built under the same lock critical section as a call to this function. If
* using the default engine for the updates, they will be performed in the
* order they grab the job_mutex. If different engines are used, external
* synchronization is needed for overlapping updates to maintain page-table
* consistency. Note that the meaning of "overlapping" is that the updates
* touch the same page-table, which might be a higher-level page-directory.
* If no pipelining is needed, then updates may be performed by the cpu.
*
* Return: A dma_fence that, when signaled, indicates the update completion.
*/
struct dma_fence *
xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind,
struct xe_cpu_bind_pt_update *pt_update)
{
struct dma_fence *fence;
fence = xe_cpu_bind_update_pgtables_no_job(cpu_bind, pt_update);
/* -ETIME indicates a job is needed, anything else is legit error */
if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)
return fence;
return xe_cpu_bind_update_pgtables_job(cpu_bind, pt_update);
}
/**
* xe_cpu_bind_job_lock() - Lock cpu_bind job lock
* @cpu_bind: The cpu bind context.
* @q: Queue associated with the operation which requires a lock
*
* Lock the cpu_bind job lock if the queue is a cpu bind queue, otherwise
* assert the VM's dma-resv is held (user queue's have own locking).
*/
void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind,
struct xe_exec_queue *q)
{
bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
if (is_cpu_bind)
mutex_lock(&cpu_bind->job_mutex);
else
xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */
}
/**
* xe_cpu_bind_job_unlock() - Unlock cpu_bind job lock
* @cpu_bind: The cpu bind context.
* @q: Queue associated with the operation which requires a lock
*
* Unlock the cpu_bind job lock if the queue is a cpu bind queue, otherwise
* assert the VM's dma-resv is held (user queue's have own locking).
*/
void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind,
struct xe_exec_queue *q)
{
bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
if (is_cpu_bind)
mutex_unlock(&cpu_bind->job_mutex);
else
xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */
}
#if IS_ENABLED(CONFIG_PROVE_LOCKING)
/**
* xe_cpu_bind_job_lock_assert() - Assert cpu_bind job lock held of queue
* @q: cpu bind queue
*/
void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q)
{
struct xe_device *xe = gt_to_xe(q->gt);
struct xe_cpu_bind *cpu_bind = xe->cpu_bind;
xe_assert(xe, q == cpu_bind->q);
lockdep_assert_held(&cpu_bind->job_mutex);
}
#endif

111
drivers/gpu/drm/xe/xe_cpu_bind.h Normal file
View File

@ -0,0 +1,111 @@
/* SPDX-License-Identifier: MIT */
/*
* Copyright © 2026 Intel Corporation
*/
#ifndef _XE_CPU_BIND_H_
#define _XE_CPU_BIND_H_
#include <linux/types.h>
struct dma_fence;
struct iosys_map;
struct xe_cpu_bind;
struct xe_cpu_bind_pt_update;
struct xe_device;
struct xe_tlb_inval_job;
struct xe_tile;
struct xe_vm;
struct xe_vm_pgtable_update;
struct xe_vm_pgtable_update_op;
struct xe_vma_ops;
/**
* struct xe_cpu_bind_pt_update_ops - Callbacks for the
* xe_cpu_bind_update_pgtables() function.
*/
struct xe_cpu_bind_pt_update_ops {
/**
* @populate: Populate a page-table with ptes.
* @tile: The tile for the current operation.
* @map: struct iosys_map into the memory to be populated.
* @update: Information about the PTEs to be inserted.
*
* This interface is intended to be used as a callback into the
* page-table system to populate shared page-tables with PTEs.
*/
void (*populate)(struct xe_tile *tile, struct iosys_map *map,
const struct xe_vm_pgtable_update *update);
/**
* @clear: Clear a page-table's ptes.
* @vm: VM being updated
* @tile: The tile for the current operation.
* @map: struct iosys_map into the page-table to be cleared.
* @update: Information about the PTEs to be cleared.
*
* This interface is intended to be used as a callback into the
* page-table system to clear PTEs from shared page-tables.
*/
void (*clear)(struct xe_vm *vm, struct xe_tile *tile,
struct iosys_map *map,
const struct xe_vm_pgtable_update *update);
/**
* @pre_commit: Callback to be called just before arming the
* sched_job.
* @pt_update: Pointer to embeddable callback argument.
*
* Return: 0 on success, negative error code on error.
*/
int (*pre_commit)(struct xe_cpu_bind_pt_update *pt_update);
};
/**
* struct xe_cpu_bind_pt_update - Argument to the struct
* xe_cpu_bind_pt_update_ops callbacks.
*
* Intended to be subclassed to support additional arguments if necessary.
*/
struct xe_cpu_bind_pt_update {
/** @ops: Pointer to the struct xe_cpu_bind_pt_update_ops callbacks */
const struct xe_cpu_bind_pt_update_ops *ops;
/** @vops: VMA operations */
struct xe_vma_ops *vops;
/** @job: The bind job, or NULL if the update is issued immediatel */
struct xe_sched_job *job;
/**
* @ijobs: The TLB invalidation jobs, individual instances can be NULL
*/
#define XE_CPU_BIND_INVAL_JOB_COUNT 4
struct xe_tlb_inval_job *ijobs[XE_CPU_BIND_INVAL_JOB_COUNT];
};
int xe_cpu_bind_init(struct xe_device *xe);
struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind);
void
xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile,
const struct xe_cpu_bind_pt_update_ops *ops,
struct xe_vm_pgtable_update_op *pt_op,
u32 num_ops, bool force_clear);
struct dma_fence *
xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind,
struct xe_cpu_bind_pt_update *pt_update);
void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind,
struct xe_exec_queue *q);
void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind,
struct xe_exec_queue *q);
#if IS_ENABLED(CONFIG_PROVE_LOCKING)
void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q);
#else
static inline void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q)
{
}
#endif
#endif

46
drivers/gpu/drm/xe/xe_devcoredump.c
View File

@ -74,11 +74,6 @@ static struct xe_device *coredump_to_xe(const struct xe_devcoredump *coredump)
return container_of(coredump, struct xe_device, devcoredump);
}
static struct xe_guc *exec_queue_to_guc(struct xe_exec_queue *q)
{
return &q->gt->uc.guc;
}
static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count,
ssize_t start,
struct xe_devcoredump *coredump)
@ -325,40 +320,44 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work)
}
static void devcoredump_snapshot(struct xe_devcoredump *coredump,
struct xe_gt *gt,
struct xe_exec_queue *q,
struct xe_sched_job *job)
{
struct xe_devcoredump_snapshot *ss = &coredump->snapshot;
struct xe_guc *guc = exec_queue_to_guc(q);
struct xe_guc *guc = &gt->uc.guc;
const char *process_name = "no process";
bool cookie;
ss->snapshot_time = ktime_get_real();
ss->boot_time = ktime_get_boottime();
if (q->vm && q->vm->xef) {
if (q && q->vm && q->vm->xef) {
process_name = q->vm->xef->process_name;
ss->pid = q->vm->xef->pid;
}
strscpy(ss->process_name, process_name);
ss->gt = q->gt;
ss->gt = gt;
INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work);
/* keep going if fw fails as we still want to save the memory and SW data */
CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
cookie = dma_fence_begin_signalling();
ss->guc.log = xe_guc_log_snapshot_capture(&guc->log, true);
ss->guc.ct = xe_guc_ct_snapshot_capture(&guc->ct);
ss->ge = xe_guc_exec_queue_snapshot_capture(q);
if (job)
ss->job = xe_sched_job_snapshot_capture(job);
ss->vm = xe_vm_snapshot_capture(q->vm);
xe_engine_snapshot_capture_for_queue(q);
if (q) {
ss->ge = xe_guc_exec_queue_snapshot_capture(q);
if (job)
ss->job = xe_sched_job_snapshot_capture(job);
ss->vm = xe_vm_snapshot_capture(q->vm);
xe_engine_snapshot_capture_for_queue(q);
}
queue_work(system_dfl_wq, &ss->work);
@ -366,19 +365,24 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
}
/**
* xe_devcoredump - Take the required snapshots and initialize coredump device.
* @q: The faulty xe_exec_queue, where the issue was detected.
* @job: The faulty xe_sched_job, where the issue was detected.
* __xe_devcoredump - Take the required snapshots and initialize coredump device.
* @gt: The GT where the issue was detected.
* @q: The faulty xe_exec_queue, where the issue was detected, may be NULL for
* hangs that are not tied to an exec queue (e.g. TLB invalidation
* timeouts); in that case only the GT-level state (GuC log and CT state)
* is captured.
* @job: The faulty xe_sched_job, where the issue was detected, may be NULL.
* @fmt: Printf format + args to describe the reason for the core dump
*
* This function should be called at the crash time within the serialized
* gt_reset. It is skipped if we still have the core dump device available
* with the information of the 'first' snapshot.
*/
__printf(3, 4)
void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const char *fmt, ...)
__printf(4, 5)
void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q,
struct xe_sched_job *job, const char *fmt, ...)
{
struct xe_device *xe = gt_to_xe(q->gt);
struct xe_device *xe = gt_to_xe(gt);
struct xe_devcoredump *coredump = &xe->devcoredump;
va_list varg;
@ -396,7 +400,7 @@ void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const cha
coredump->snapshot.reason = kvasprintf(GFP_ATOMIC, fmt, varg);
va_end(varg);
devcoredump_snapshot(coredump, q, job);
devcoredump_snapshot(coredump, gt, q, job);
drm_info(&xe->drm, "Xe device coredump has been created\n");
drm_info(&xe->drm, "Check your /sys/class/drm/card%d/device/devcoredump/data\n",

15
drivers/gpu/drm/xe/xe_devcoredump.h
View File

@ -11,15 +11,17 @@
struct drm_printer;
struct xe_device;
struct xe_exec_queue;
struct xe_gt;
struct xe_sched_job;
#ifdef CONFIG_DEV_COREDUMP
void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const char *fmt, ...);
void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q,
struct xe_sched_job *job, const char *fmt, ...);
int xe_devcoredump_init(struct xe_device *xe);
#else
static inline void xe_devcoredump(struct xe_exec_queue *q,
struct xe_sched_job *job,
const char *fmt, ...)
static inline void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q,
struct xe_sched_job *job,
const char *fmt, ...)
{
}
@ -29,6 +31,11 @@ static inline int xe_devcoredump_init(struct xe_device *xe)
}
#endif
#define xe_devcoredump(_q, _job, _fmt, ...) \
__xe_devcoredump((_q)->gt, _q, _job, _fmt, ##__VA_ARGS__)
#define xe_devcoredump_gt(_gt, _fmt, ...) \
__xe_devcoredump(_gt, NULL, NULL, _fmt, ##__VA_ARGS__)
void xe_print_blob_ascii85(struct drm_printer *p, const char *prefix, char suffix,
const void *blob, size_t offset, size_t size);

14
drivers/gpu/drm/xe/xe_device.c
View File

@ -27,6 +27,7 @@
#include "xe_bo_evict.h"
#include "xe_configfs.h"
#include "xe_debugfs.h"
#include "xe_cpu_bind.h"
#include "xe_defaults.h"
#include "xe_devcoredump.h"
#include "xe_device_sysfs.h"
@ -671,6 +672,7 @@ static void vf_update_device_info(struct xe_device *xe)
xe->info.skip_guc_pc = 1;
xe->info.skip_pcode = 1;
xe->info.has_drm_ras = false;
xe->info.has_device_uid = false;
}
static int xe_device_vram_alloc(struct xe_device *xe)
@ -871,6 +873,12 @@ static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe)
}
#endif
static void xe_uid_probe(struct xe_device *xe)
{
if (xe->info.has_device_uid)
xe->device_uid = xe_mmio_read64_2x32(xe_root_tile_mmio(xe), CRI_DEVICE_UID);
}
int xe_device_probe(struct xe_device *xe)
{
struct xe_tile *tile;
@ -878,6 +886,8 @@ int xe_device_probe(struct xe_device *xe)
int err;
u8 id;
xe_uid_probe(xe);
xe_pat_init_early(xe);
err = xe_sriov_init(xe);
@ -993,6 +1003,10 @@ int xe_device_probe(struct xe_device *xe)
if (err)
return err;
err = xe_cpu_bind_init(xe);
if (err)
return err;
err = xe_pagefault_init(xe);
if (err)
return err;

36
drivers/gpu/drm/xe/xe_device_sysfs.c
View File

@ -8,6 +8,7 @@
#include <linux/pci.h>
#include <linux/sysfs.h>
#include "regs/xe_regs.h"
#include "xe_device.h"
#include "xe_device_sysfs.h"
#include "xe_mmio.h"
@ -264,6 +265,35 @@ static const struct attribute_group auto_link_downgrade_attr_group = {
.attrs = auto_link_downgrade_attrs,
};
/**
* DOC: Device Unique ID
*
* On supported platforms, Xe driver exposes a unique 64-bit GPU SOC
* device identifier through the 'device_uid' sysfs entry.
*
* See Documentation/ABI/testing/sysfs-driver-intel-xe-gpu for the ABI
* specification.
*/
static ssize_t
device_uid_show(struct device *dev, struct device_attribute *attr, char *buf)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct xe_device *xe = pdev_to_xe_device(pdev);
return sysfs_emit(buf, "0x%016llx\n", xe->device_uid);
}
static DEVICE_ATTR_ADMIN_RO(device_uid);
static struct attribute *device_uid_attrs[] = {
&dev_attr_device_uid.attr,
NULL
};
static const struct attribute_group device_uid_attr_group = {
.attrs = device_uid_attrs,
};
int xe_device_sysfs_init(struct xe_device *xe)
{
struct device *dev = xe->drm.dev;
@ -285,5 +315,11 @@ int xe_device_sysfs_init(struct xe_device *xe)
return ret;
}
if (xe->info.has_device_uid) {
ret = devm_device_add_group(dev, &device_uid_attr_group);
if (ret)
return ret;
}
return 0;
}

15
drivers/gpu/drm/xe/xe_device_types.h
View File

@ -38,6 +38,7 @@
struct drm_pagemap_shrinker;
struct intel_display;
struct intel_dg_nvm_dev;
struct xe_cpu_bind;
struct xe_ggtt;
struct xe_i2c;
struct xe_mmio_gem;
@ -89,9 +90,7 @@ enum xe_page_size_alloc_ctrl_mode {
#define XE_VRAM_FLAGS_NEED64K BIT(0)
#define XE_GT0 0
#define XE_GT1 1
#define XE_MAX_TILES_PER_DEVICE (XE_GT1 + 1)
#define XE_MAX_TILES_PER_DEVICE 2
/*
* Highest GT/tile count for any platform. Used only for memory allocation
@ -180,6 +179,8 @@ struct xe_device {
u8 has_cached_pt:1;
/** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */
u8 has_device_atomics_on_smem:1;
/** @info.has_device_uid: Device supports unique 64-bit GPU SOC ID */
u8 has_device_uid:1;
/** @info.has_drm_ras: Device supports drm_ras (Reliability, Availability, Serviceability) */
u8 has_drm_ras:1;
/** @info.has_fan_control: Device supports fan control */
@ -230,6 +231,8 @@ struct xe_device {
u8 has_usm:1;
/** @info.has_64bit_timestamp: Device supports 64-bit timestamps */
u8 has_64bit_timestamp:1;
/** @info.has_pt_mirror: Device has PT mirroring across tiles */
u8 has_pt_mirror:1;
/** @info.is_dgfx: is discrete device */
u8 is_dgfx:1;
/** @info.needs_scratch: needs scratch page for oob prefetch to work */
@ -264,6 +267,9 @@ struct xe_device {
bool oob_initialized;
} wa_active;
/** @device_uid: unique 64-bit GPU SOC identifier */
u64 device_uid;
/** @survivability: survivability information for device */
struct xe_survivability survivability;
@ -566,6 +572,9 @@ struct xe_device {
/** @sc: System Controller */
struct xe_sysctrl sc;
/** @cpu_bind: CPU bind object */
struct xe_cpu_bind *cpu_bind;
/** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */
u32 atomic_svm_timeslice_ms;

2
drivers/gpu/drm/xe/xe_drm_client.c
View File

@ -256,7 +256,7 @@ static void show_meminfo(struct drm_printer *p, struct drm_file *file)
xe_assert(xef->xe, !list_empty(&bo->client_link));
}
xe_bo_put_deferred(bo, &deferred);
xe_bo_put_deferred(bo, &deferred, NULL);
}
spin_unlock(&client->bos_lock);

161
drivers/gpu/drm/xe/xe_exec_queue.c
View File

@ -14,6 +14,7 @@
#include <uapi/drm/xe_drm.h>
#include "xe_bo.h"
#include "xe_cpu_bind.h"
#include "xe_dep_scheduler.h"
#include "xe_device.h"
#include "xe_gt.h"
@ -142,9 +143,8 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q)
{
int i;
for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i)
if (q->tlb_inval[i].dep_scheduler)
xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler);
for_each_tlb_inval(q, i)
xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler);
if (xe_exec_queue_uses_pxp(q))
xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
@ -166,31 +166,34 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q)
static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q)
{
struct xe_tile *tile = gt_to_tile(q->gt);
int i;
struct xe_tile *tile;
int i = 0, j;
u8 id;
for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) {
struct xe_dep_scheduler *dep_scheduler;
struct xe_gt *gt;
struct workqueue_struct *wq;
for_each_tile(tile, xe, id) {
for (j = 0; j < (XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1); ++j, ++i) {
struct xe_dep_scheduler *dep_scheduler;
struct xe_gt *gt;
struct workqueue_struct *wq;
if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT)
gt = tile->primary_gt;
else
gt = tile->media_gt;
if (j == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT)
gt = tile->primary_gt;
else
gt = tile->media_gt;
if (!gt)
continue;
if (!gt)
continue;
wq = gt->tlb_inval.job_wq;
wq = gt->tlb_inval.job_wq;
#define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */
dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name,
MAX_TLB_INVAL_JOBS);
if (IS_ERR(dep_scheduler))
return PTR_ERR(dep_scheduler);
dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name,
MAX_TLB_INVAL_JOBS);
if (IS_ERR(dep_scheduler))
return PTR_ERR(dep_scheduler);
q->tlb_inval[i].dep_scheduler = dep_scheduler;
q->tlb_inval[i].dep_scheduler = dep_scheduler;
}
}
#undef MAX_TLB_INVAL_JOBS
@ -224,7 +227,6 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
q->ops = gt->exec_queue_ops;
INIT_LIST_HEAD(&q->lr.link);
INIT_LIST_HEAD(&q->vm_exec_queue_link);
INIT_LIST_HEAD(&q->multi_gt_link);
INIT_LIST_HEAD(&q->hw_engine_group_link);
INIT_LIST_HEAD(&q->pxp.link);
spin_lock_init(&q->multi_queue.lock);
@ -636,7 +638,6 @@ ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
void xe_exec_queue_destroy(struct kref *ref)
{
struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
struct xe_exec_queue *eq, *next;
int i;
xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0);
@ -648,15 +649,9 @@ void xe_exec_queue_destroy(struct kref *ref)
xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
xe_exec_queue_last_fence_put_unlocked(q);
for_each_tlb_inval(i)
for_each_tlb_inval(q, i)
xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i);
if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {
list_for_each_entry_safe(eq, next, &q->multi_gt_list,
multi_gt_link)
xe_exec_queue_put(eq);
}
if (q->user_vm) {
xe_vm_put(q->user_vm);
q->user_vm = NULL;
@ -1343,7 +1338,6 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
u64_to_user_ptr(args->instances);
struct xe_hw_engine *hwe;
struct xe_vm *vm;
struct xe_tile *tile;
struct xe_exec_queue *q = NULL;
u32 logical_mask;
u32 flags = 0;
@ -1392,31 +1386,16 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
return -ENOENT;
}
for_each_tile(tile, xe, id) {
struct xe_exec_queue *new;
flags |= EXEC_QUEUE_FLAG_VM;
flags |= EXEC_QUEUE_FLAG_VM;
if (id)
flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD;
new = xe_exec_queue_create_bind(xe, tile, vm, flags,
args->extensions);
if (IS_ERR(new)) {
up_read(&vm->lock);
xe_vm_put(vm);
err = PTR_ERR(new);
if (q)
goto put_exec_queue;
return err;
}
if (id == 0)
q = new;
else
list_add_tail(&new->multi_gt_list,
&q->multi_gt_link);
}
q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe),
vm, flags, args->extensions);
up_read(&vm->lock);
xe_vm_put(vm);
if (IS_ERR(q)) {
err = PTR_ERR(q);
return err;
}
} else {
logical_mask = calc_validate_logical_mask(xe, eci,
args->width,
@ -1567,6 +1546,7 @@ bool xe_exec_queue_is_lr(struct xe_exec_queue *q)
/**
* xe_exec_queue_is_idle() - Whether an exec_queue is idle.
* @q: The exec_queue
* @extra_jobs: Extra jobs on the queue
*
* FIXME: Need to determine what to use as the short-lived
* timeline lock for the exec_queues, so that the return value
@ -1578,22 +1558,9 @@ bool xe_exec_queue_is_lr(struct xe_exec_queue *q)
*
* Return: True if the exec_queue is idle, false otherwise.
*/
bool xe_exec_queue_is_idle(struct xe_exec_queue *q)
bool xe_exec_queue_is_idle(struct xe_exec_queue *q, int extra_jobs)
{
if (xe_exec_queue_is_parallel(q)) {
int i;
for (i = 0; i < q->width; ++i) {
if (xe_lrc_seqno(q->lrc[i]) !=
q->lrc[i]->fence_ctx.next_seqno - 1)
return false;
}
return true;
}
return xe_lrc_seqno(q->lrc[0]) ==
q->lrc[0]->fence_ctx.next_seqno - 1;
return !(atomic_read(&q->job_cnt) - extra_jobs);
}
/**
@ -1651,14 +1618,6 @@ void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q)
*/
void xe_exec_queue_kill(struct xe_exec_queue *q)
{
struct xe_exec_queue *eq = q, *next;
list_for_each_entry_safe(eq, next, &eq->multi_gt_list,
multi_gt_link) {
q->ops->kill(eq);
xe_vm_remove_compute_exec_queue(q->vm, eq);
}
q->ops->kill(q);
xe_vm_remove_compute_exec_queue(q->vm, q);
}
@ -1709,7 +1668,7 @@ static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q,
struct xe_vm *vm)
{
if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) {
xe_migrate_job_lock_assert(q);
xe_cpu_bind_job_lock_assert(q);
} else if (q->flags & EXEC_QUEUE_FLAG_VM) {
lockdep_assert_held(&vm->lock);
} else {
@ -1819,42 +1778,40 @@ void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm,
* xe_exec_queue_tlb_inval_last_fence_put() - Drop ref to last TLB invalidation fence
* @q: The exec queue
* @vm: The VM the engine does a bind for
* @type: Either primary or media GT
* @idx: Index of tlb invalidation
*/
void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q,
struct xe_vm *vm,
unsigned int type)
unsigned int idx)
{
xe_exec_queue_last_fence_lockdep_assert(q, vm);
xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, type);
xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, idx);
}
/**
* xe_exec_queue_tlb_inval_last_fence_put_unlocked() - Drop ref to last TLB
* invalidation fence unlocked
* @q: The exec queue
* @type: Either primary or media GT
* @idx: Index of tlb invalidation
*
* Only safe to be called from xe_exec_queue_destroy().
*/
void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
unsigned int type)
unsigned int idx)
{
xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
xe_assert(gt_to_xe(q->gt), idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
dma_fence_put(q->tlb_inval[type].last_fence);
q->tlb_inval[type].last_fence = NULL;
dma_fence_put(q->tlb_inval[idx].last_fence);
q->tlb_inval[idx].last_fence = NULL;
}
/**
* xe_exec_queue_tlb_inval_last_fence_get() - Get last fence for TLB invalidation
* @q: The exec queue
* @vm: The VM the engine does a bind for
* @type: Either primary or media GT
* @idx: Index of tlb invalidation
*
* Get last fence, takes a ref
*
@ -1862,22 +1819,21 @@ void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
*/
struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q,
struct xe_vm *vm,
unsigned int type)
unsigned int idx)
{
struct dma_fence *fence;
xe_exec_queue_last_fence_lockdep_assert(q, vm);
xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
EXEC_QUEUE_FLAG_MIGRATE));
if (q->tlb_inval[type].last_fence &&
if (q->tlb_inval[idx].last_fence &&
test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
&q->tlb_inval[type].last_fence->flags))
xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
&q->tlb_inval[idx].last_fence->flags))
xe_exec_queue_tlb_inval_last_fence_put(q, vm, idx);
fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub();
fence = q->tlb_inval[idx].last_fence ?: dma_fence_get_stub();
dma_fence_get(fence);
return fence;
}
@ -1887,26 +1843,25 @@ struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q
* @q: The exec queue
* @vm: The VM the engine does a bind for
* @fence: The fence
* @type: Either primary or media GT
* @idx: Index of tlb invalidation
*
* Set the last fence for the tlb invalidation type on the queue. Increases
* Set the last fence for the tlb invalidation client on the queue. Increases
* reference count for fence, when closing queue
* xe_exec_queue_tlb_inval_last_fence_put should be called.
*/
void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q,
struct xe_vm *vm,
struct dma_fence *fence,
unsigned int type)
unsigned int idx)
{
xe_exec_queue_last_fence_lockdep_assert(q, vm);
xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
EXEC_QUEUE_FLAG_MIGRATE));
xe_assert(vm->xe, !dma_fence_is_container(fence));
xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
q->tlb_inval[type].last_fence = dma_fence_get(fence);
xe_exec_queue_tlb_inval_last_fence_put(q, vm, idx);
q->tlb_inval[idx].last_fence = dma_fence_get(fence);
}
/**

16
drivers/gpu/drm/xe/xe_exec_queue.h
View File

@ -14,9 +14,9 @@ struct drm_file;
struct xe_device;
struct xe_file;
#define for_each_tlb_inval(__i) \
for (__i = XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT; \
__i <= XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT; ++__i)
#define for_each_tlb_inval(__q, __i) \
for (__i = 0; __i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++__i) \
for_each_if((__q)->tlb_inval[__i].dep_scheduler)
struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
u32 logical_mask, u16 width,
@ -116,7 +116,7 @@ static inline struct xe_exec_queue *xe_exec_queue_multi_queue_primary(struct xe_
bool xe_exec_queue_is_lr(struct xe_exec_queue *q);
bool xe_exec_queue_is_idle(struct xe_exec_queue *q);
bool xe_exec_queue_is_idle(struct xe_exec_queue *q, int extra_jobs);
void xe_exec_queue_kill(struct xe_exec_queue *q);
@ -141,19 +141,19 @@ void xe_exec_queue_last_fence_set(struct xe_exec_queue *e, struct xe_vm *vm,
void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q,
struct xe_vm *vm,
unsigned int type);
unsigned int idx);
void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
unsigned int type);
unsigned int idx);
struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q,
struct xe_vm *vm,
unsigned int type);
unsigned int idx);
void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q,
struct xe_vm *vm,
struct dma_fence *fence,
unsigned int type);
unsigned int idx);
void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q);

20
drivers/gpu/drm/xe/xe_exec_queue_types.h
View File

@ -137,16 +137,14 @@ struct xe_exec_queue {
#define EXEC_QUEUE_FLAG_KERNEL BIT(0)
/* for VM jobs. Caller needs to hold rpm ref when creating queue with this flag */
#define EXEC_QUEUE_FLAG_VM BIT(1)
/* child of VM queue for multi-tile VM jobs */
#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(2)
/* kernel exec_queue only, set priority to highest level */
#define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(3)
#define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(2)
/* flag to indicate low latency hint to guc */
#define EXEC_QUEUE_FLAG_LOW_LATENCY BIT(4)
#define EXEC_QUEUE_FLAG_LOW_LATENCY BIT(3)
/* for migration (kernel copy, clear, bind) jobs */
#define EXEC_QUEUE_FLAG_MIGRATE BIT(5)
#define EXEC_QUEUE_FLAG_MIGRATE BIT(4)
/* for programming COMMON_SLICE_CHICKEN3 on first submission */
#define EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX BIT(6)
#define EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX BIT(5)
/**
* @flags: flags for this exec queue, should statically setup aside from ban
@ -157,13 +155,6 @@ struct xe_exec_queue {
/** @ban_reason: Bitmask of ban reasons (DRM_XE_EXEC_QUEUE_BAN_REASON_*) */
atomic_t ban_reason;
union {
/** @multi_gt_list: list head for VM bind engines if multi-GT */
struct list_head multi_gt_list;
/** @multi_gt_link: link for VM bind engines if multi-GT */
struct list_head multi_gt_link;
};
union {
/** @execlist: execlist backend specific state for exec queue */
struct xe_execlist_exec_queue *execlist;
@ -230,7 +221,8 @@ struct xe_exec_queue {
#define XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT 0
#define XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT 1
#define XE_EXEC_QUEUE_TLB_INVAL_COUNT (XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1)
#define XE_EXEC_QUEUE_TLB_INVAL_COUNT \
((XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1) * 2)
/** @tlb_inval: TLB invalidations exec queue state */
struct {

6
drivers/gpu/drm/xe/xe_gt_ccs_mode.c
View File

@ -70,12 +70,6 @@ static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines)
}
}
/*
* Mask bits need to be set for the register. Though only Xe2+
* platforms require setting of mask bits, it won't harm for older
* platforms as these bits are unused there.
*/
mode |= CCS_MODE_CSLICE_0_3_MASK << 16;
xe_mmio_write32(&gt->mmio, CCS_MODE, mode);
xe_gt_dbg(gt, "CCS_MODE=%x config:%08x, num_engines:%d, num_slices:%d\n",

16
drivers/gpu/drm/xe/xe_guc.c
View File

@ -1469,6 +1469,22 @@ int xe_guc_suspend(struct xe_guc *guc)
return 0;
}
/**
* xe_guc_bo_wa_flags - Extra BO flags for memory shared with the GuC
* @gt: the &xe_gt whose GuC the buffer will be shared with
*
* Wa_22016122933: on the standalone media GT, memory shared between the
* CPU and the GuC must not be mapped cached on the CPU side, otherwise
* the CPU can read stale data written by the GuC (e.g. G2H CTB writes)
* for multiple seconds.
*
* Return: additional XE_BO_FLAG_* to use when allocating GuC-shared memory
*/
u32 xe_guc_bo_wa_flags(struct xe_gt *gt)
{
return XE_GT_WA(gt, 22016122933) ? XE_BO_FLAG_NEEDS_UC : 0;
}
void xe_guc_notify(struct xe_guc *guc)
{
struct xe_gt *gt = guc_to_gt(guc);

2
drivers/gpu/drm/xe/xe_guc.h
View File

@ -30,6 +30,7 @@
xe_guc_fw_version_at_least((guc), MAKE_GUC_VER_ARGS(ver))
struct drm_printer;
struct xe_gt;
void xe_guc_comm_init_early(struct xe_guc *guc);
int xe_guc_init_noalloc(struct xe_guc *guc);
@ -45,6 +46,7 @@ void xe_guc_runtime_suspend(struct xe_guc *guc);
void xe_guc_runtime_resume(struct xe_guc *guc);
int xe_guc_suspend(struct xe_guc *guc);
int xe_guc_softreset(struct xe_guc *guc);
u32 xe_guc_bo_wa_flags(struct xe_gt *gt);
void xe_guc_notify(struct xe_guc *guc);
int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);
int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len);

11
drivers/gpu/drm/xe/xe_guc_ads.c
View File

@ -383,6 +383,12 @@ static void guc_waklv_init(struct xe_guc_ads *ads)
guc_waklv_enable(ads, NULL, 0, &offset, &remain,
GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64);
/* GuC only applies this WA for scheduled MultiQ contexts; no KMD-side gating needed. */
if (XE_GT_WA(gt, 16030224309) &&
GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 73))
guc_waklv_enable(ads, NULL, 0, &offset, &remain,
GUC_WA_KLV_IGNORE_MULTIQ_CTX_COMPLETE_WITH_PREEMPT_IN_CSB);
/*
* On GuC firmware 70.66 and above, use the Feature KLV (shared with the
* WA KLV buffer); older firmware uses GUC_CTL_DISABLE_MULTI_QUEUE in
@ -435,7 +441,8 @@ int xe_guc_ads_init(struct xe_guc_ads *ads)
XE_BO_FLAG_SYSTEM |
XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE |
XE_BO_FLAG_PINNED_NORESTORE);
XE_BO_FLAG_PINNED_NORESTORE |
xe_guc_bo_wa_flags(gt));
if (IS_ERR(bo))
return PTR_ERR(bo);
@ -864,7 +871,7 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads,
}
}
if (XE_GT_WA(hwe->gt, 16023105232))
if (XE_GT_WA(hwe->gt, 16023105232) || XE_GT_WA(hwe->gt, 14025941587))
guc_mmio_regset_write_one(ads, regset_map,
RING_IDLEDLY(hwe->mmio_base),
count++);

23
drivers/gpu/drm/xe/xe_guc_ct.c
View File

@ -376,7 +376,8 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
XE_BO_FLAG_SYSTEM |
XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE |
XE_BO_FLAG_PINNED_NORESTORE);
XE_BO_FLAG_PINNED_NORESTORE |
xe_guc_bo_wa_flags(gt));
if (IS_ERR(bo))
return PTR_ERR(bo);
@ -386,7 +387,8 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
XE_BO_FLAG_SYSTEM |
XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE |
XE_BO_FLAG_PINNED_NORESTORE);
XE_BO_FLAG_PINNED_NORESTORE |
xe_guc_bo_wa_flags(gt));
if (IS_ERR(bo))
return PTR_ERR(bo);
@ -1288,11 +1290,23 @@ int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
* @ct: GuC CT object
* @action: payload dwords (HxG header dword is expected at @action[-1])
* @len: number of payload dwords in @action
* @g2h_len: G2H response space to reserve in dwords, or 0
* @num_g2h: number of G2H messages expected, or 0
* @defer_flush: defer publishing/doorbell for batching
*
* Sends a single H2G message to the GuC CT buffer while the caller already
* holds @ct->lock.
*
* Callers that expect the GuC to reply with a G2H message must reserve the
* matching credit via @g2h_len / @num_g2h. The G2H receive path releases that
* credit unconditionally when the reply arrives, so failing to reserve it here
* corrupts the G2H accounting and kills the CT channel.
*
* Reserving G2H credit may wait for, and process, outstanding G2H messages, so
* this must not be called with a non-zero @g2h_len from a G2H handler. Handlers
* should use xe_guc_ct_send_g2h_handler(), with any reply credit reserved up
* front by the H2G that triggered the handler.
*
* If @defer_flush is false, the function completes the submission immediately:
* it makes the payload visible to the device, updates the H2G descriptor and
* rings the GuC doorbell.
@ -1310,11 +1324,12 @@ int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
* Must be called with @ct->lock held.
*/
int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
bool defer_flush)
u32 g2h_len, u32 num_g2h, bool defer_flush)
{
int ret;
ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL, defer_flush);
ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL,
defer_flush);
if (ret == -EDEADLK)
kick_reset(ct);

2
drivers/gpu/drm/xe/xe_guc_ct.h
View File

@ -56,7 +56,7 @@ static inline void xe_guc_ct_irq_handler(struct xe_guc_ct *ct)
int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
u32 g2h_len, u32 num_g2h);
int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
bool defer_flush);
u32 g2h_len, u32 num_g2h, bool defer_flush);
int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
u32 *response_buffer);
static inline int

6
drivers/gpu/drm/xe/xe_guc_engine_activity.c
View File

@ -97,7 +97,8 @@ static int allocate_engine_activity_buffers(struct xe_guc *guc,
metadata_bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(metadata_size),
ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM |
XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE,
XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE |
xe_guc_bo_wa_flags(gt),
false);
if (IS_ERR(metadata_bo))
@ -105,7 +106,8 @@ static int allocate_engine_activity_buffers(struct xe_guc *guc,
bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(size),
ttm_bo_type_kernel, XE_BO_FLAG_VRAM_IF_DGFX(tile) |
XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE, false);
XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE |
xe_guc_bo_wa_flags(gt), false);
if (IS_ERR(bo)) {
xe_bo_unpin_map_no_vm(metadata_bo);

2
drivers/gpu/drm/xe/xe_guc_hwconfig.c
View File

@ -65,7 +65,7 @@ int xe_guc_hwconfig_init(struct xe_guc *guc)
/*
* All hwconfig the same across GTs so only GT0 needs to be configured
*/
if (gt->info.id != XE_GT0)
if (gt != xe_root_mmio_gt(xe))
return 0;
/* ADL_P, DG2+ supports hwconfig table */

7
drivers/gpu/drm/xe/xe_guc_log.c
View File

@ -17,6 +17,7 @@
#include "xe_force_wake.h"
#include "xe_gt_printk.h"
#include "xe_gt_types.h"
#include "xe_guc.h"
#include "xe_map.h"
#include "xe_mmio.h"
#include "xe_module.h"
@ -624,14 +625,16 @@ void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p)
int xe_guc_log_init(struct xe_guc_log *log)
{
struct xe_device *xe = log_to_xe(log);
struct xe_tile *tile = gt_to_tile(log_to_gt(log));
struct xe_gt *gt = log_to_gt(log);
struct xe_tile *tile = gt_to_tile(gt);
struct xe_bo *bo;
bo = xe_managed_bo_create_pin_map(xe, tile, GUC_LOG_SIZE,
XE_BO_FLAG_SYSTEM |
XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE |
XE_BO_FLAG_PINNED_NORESTORE);
XE_BO_FLAG_PINNED_NORESTORE |
xe_guc_bo_wa_flags(gt));
if (IS_ERR(bo))
return PTR_ERR(bo);

35
drivers/gpu/drm/xe/xe_guc_pagefault.c
View File

@ -4,9 +4,11 @@
*/
#include "abi/guc_actions_abi.h"
#include "xe_gt_printk.h"
#include "xe_guc.h"
#include "xe_guc_ct.h"
#include "xe_guc_pagefault.h"
#include "xe_hw_engine.h"
#include "xe_pagefault.h"
#include "xe_pagefault_types.h"
@ -55,7 +57,8 @@ static void guc_ack_fault(struct xe_pagefault *pf, int err)
bool write_only = guc->pagefault_ack_counter++ &
(XE_GUC_PAGEFAULT_FLUSH_PERIOD - 1);
xe_guc_ct_send_locked(&guc->ct, action, ARRAY_SIZE(action),
/* Pagefault acks are fire-and-forget, no G2H reply expected. */
xe_guc_ct_send_locked(&guc->ct, action, ARRAY_SIZE(action), 0, 0,
write_only);
}
@ -68,10 +71,40 @@ static void guc_ack_fault_end(void *private)
xe_guc_ct_unlock(&guc->ct);
}
static void xe_guc_pagefault_print(struct xe_pagefault *pf,
const char *err_str)
{
const u32 *msg = pf->producer.msg;
u32 engine_class = FIELD_GET(PFD_ENG_CLASS, msg[0]);
xe_gt_info(pf->gt, "\n\tASID: %lu\n"
"\tFaulted Address: 0x%08lx%08lx\n"
"\tFaultType: %lu\n"
"\tAccessType: %lu\n"
"\tFaultLevel: %lu\n"
"\tEngineClass: %u %s\n"
"\tEngineInstance: %lu\n"
"\tSRCID: 0x%02lx\n"
"\tError: %s\n",
FIELD_GET(PFD_ASID, msg[1]),
FIELD_GET(PFD_VIRTUAL_ADDR_HI, msg[3]),
FIELD_GET(PFD_VIRTUAL_ADDR_LO, msg[2]) <<
PFD_VIRTUAL_ADDR_LO_SHIFT,
FIELD_GET(PFD_FAULT_TYPE, msg[2]),
FIELD_GET(PFD_ACCESS_TYPE, msg[2]),
FIELD_GET(PFD_FAULT_LEVEL, msg[0]),
engine_class,
xe_hw_engine_class_to_str(engine_class),
FIELD_GET(PFD_ENG_INSTANCE, msg[0]),
FIELD_GET(PFD_SRC_ID, msg[0]),
err_str);
}
static const struct xe_pagefault_ops guc_pagefault_ops = {
.ack_fault_begin = guc_ack_fault_begin,
.ack_fault = guc_ack_fault,
.ack_fault_end = guc_ack_fault_end,
.print = xe_guc_pagefault_print,
};
/**

3
drivers/gpu/drm/xe/xe_guc_pc.c
View File

@ -1391,7 +1391,8 @@ int xe_guc_pc_init(struct xe_guc_pc *pc)
XE_BO_FLAG_VRAM_IF_DGFX(tile) |
XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE |
XE_BO_FLAG_PINNED_NORESTORE);
XE_BO_FLAG_PINNED_NORESTORE |
xe_guc_bo_wa_flags(gt));
if (IS_ERR(bo))
return PTR_ERR(bo);

90
drivers/gpu/drm/xe/xe_guc_submit.c
View File

@ -19,6 +19,7 @@
#include "abi/guc_klvs_abi.h"
#include "xe_assert.h"
#include "xe_bo.h"
#include "xe_cpu_bind.h"
#include "xe_devcoredump.h"
#include "xe_device.h"
#include "xe_exec_queue.h"
@ -42,6 +43,7 @@
#include "xe_mocs.h"
#include "xe_module.h"
#include "xe_pm.h"
#include "xe_pt.h"
#include "xe_ring_ops_types.h"
#include "xe_sched_job.h"
#include "xe_sleep.h"
@ -1188,9 +1190,15 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
if (!job->restore_replay || job->last_replay) {
/*
* A chained ULLS job relies on its predecessor's postamble to
* publish its ring tail. After a reset the context comes back
* disabled with that postamble gone, so publish it here.
*/
if (xe_exec_queue_is_parallel(q))
wq_item_append(q);
else
else if (!xe_sched_job_ulls_is_chained(job) ||
!exec_queue_enabled(q))
xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
job->last_replay = false;
}
@ -1207,6 +1215,9 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
if (exec_queue_suspended(q))
return;
if (xe_sched_job_ulls_is_chained(job))
xe_lrc_set_ulls_semaphore(lrc, xe_sched_job_lrc_seqno(job));
if (!exec_queue_enabled(q)) {
action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
action[len++] = q->guc->id;
@ -1220,13 +1231,14 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
set_exec_queue_pending_enable(q);
set_exec_queue_enabled(q);
trace_xe_exec_queue_scheduling_enable(q);
} else {
} else if (!xe_sched_job_ulls_is_chained(job)) {
action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;
action[len++] = q->guc->id;
trace_xe_exec_queue_submit(q);
}
xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);
if (!xe_sched_job_ulls_is_chained(job) || num_g2h)
xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);
if (extra_submit) {
len = 0;
@ -1238,21 +1250,70 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
}
}
static bool is_pt_job(struct xe_sched_job *job)
{
return job->is_pt_job;
}
static void run_pt_job(struct xe_device *xe, struct xe_sched_job *job,
bool force_clear)
{
struct xe_tile *tile;
int id;
for_each_tile(tile, xe, id) {
struct xe_pt_job_ops *pt_job_ops =
job->pt_update[0].pt_job_ops[id];
if (!pt_job_ops || !pt_job_ops->current_op)
continue;
xe_cpu_bind_update_pgtables_execute(job->pt_update[0].vm, tile,
job->pt_update[0].ops,
pt_job_ops->ops,
pt_job_ops->current_op,
force_clear);
}
}
static void put_pt_job(struct xe_device *xe, struct xe_sched_job *job)
{
struct xe_tile *tile;
int id;
for_each_tile(tile, xe, id) {
struct xe_pt_job_ops *pt_job_ops =
job->pt_update[0].pt_job_ops[id];
xe_pt_job_ops_put(pt_job_ops);
}
}
static struct dma_fence *
guc_exec_queue_run_job(struct drm_sched_job *drm_job)
{
struct xe_sched_job *job = to_xe_sched_job(drm_job);
struct xe_exec_queue *q = job->q;
struct xe_guc *guc = exec_queue_to_guc(q);
bool killed_or_banned_or_wedged =
exec_queue_killed_or_banned_or_wedged(q);
bool killed_or_banned_or_wedged_or_error =
exec_queue_killed_or_banned_or_wedged(q) ||
xe_sched_job_is_error(job);
xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) ||
exec_queue_banned(q) || exec_queue_suspended(q));
trace_xe_sched_job_run(job);
if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) {
if (is_pt_job(job)) {
xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q));
run_pt_job(guc_to_xe(guc), job,
killed_or_banned_or_wedged_or_error);
put_pt_job(guc_to_xe(guc), job);
dma_fence_put(job->fence); /* Drop ref from xe_sched_job_arm */
return NULL;
} else if (!killed_or_banned_or_wedged_or_error) {
if (xe_exec_queue_is_multi_queue_secondary(q)) {
struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
@ -2049,6 +2110,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
struct xe_guc_exec_queue *ge;
long timeout;
int err, i;
int max_jobs = (xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES);
xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc)));
@ -2088,8 +2150,11 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
submit_wq = primary->guc->sched.base.submit_wq;
}
if (q->vm && q->vm->flags & XE_VM_FLAG_MIGRATION)
max_jobs = min(max_jobs, LRC_MIGRATION_ULLS_SEMAPHORE_COUNT - 1);
err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
submit_wq, max_jobs, 64,
timeout, guc_to_gt(guc)->ordered_wq, NULL,
ge->name, gt_to_xe(q->gt)->drm.dev);
if (err)
@ -2993,6 +3058,15 @@ static void guc_exec_queue_start(struct xe_exec_queue *q)
* increasing as jobs are written out.
*/
q->lrc[i]->ring.tail = job->ptrs[i].head;
/*
* A ULLS job's slot can have been consumed by
* the CS without the job running, leaving the
* saved head past it. Rewind so the re-emitted
* job is ahead of the head, not behind it.
*/
if (xe_sched_job_is_ulls(job))
xe_lrc_set_ring_head(q->lrc[i],
job->ptrs[i].head);
xe_lrc_set_ring_tail(q->lrc[i],
xe_lrc_ring_head(q->lrc[i]));
}
@ -3686,6 +3760,7 @@ xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q)
snapshot->logical_mask = q->logical_mask;
snapshot->width = q->width;
snapshot->refcount = kref_read(&q->refcount);
snapshot->jobcount = atomic_read(&q->job_cnt);
snapshot->sched_timeout = sched->base.timeout;
snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us;
snapshot->sched_props.preempt_timeout_us =
@ -3758,6 +3833,7 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps
drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask);
drm_printf(p, "\tWidth: %d\n", snapshot->width);
drm_printf(p, "\tRef: %d\n", snapshot->refcount);
drm_printf(p, "\tJob count: %u\n", snapshot->jobcount);
drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout);
drm_printf(p, "\tTimeslice: %u (us)\n",
snapshot->sched_props.timeslice_us);

2
drivers/gpu/drm/xe/xe_guc_submit_types.h
View File

@ -77,6 +77,8 @@ struct xe_guc_submit_exec_queue_snapshot {
u16 width;
/** @refcount: ref count of this exec queue */
u32 refcount;
/** @jobcount: job count of this exec queue */
u32 jobcount;
/**
* @sched_timeout: the time after which a job is removed from the
* scheduler.

96
drivers/gpu/drm/xe/xe_hw_engine.c
View File

@ -585,28 +585,102 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe,
xe_reg_whitelist_process_engine(hwe);
}
static u32 idledly_floor_ticks(u32 idledly_ns, u32 idledly_units_ps)
{
return DIV_ROUND_DOWN_ULL((u64)idledly_ns * 1000, idledly_units_ps);
}
static void adjust_idledly(struct xe_hw_engine *hwe)
{
struct xe_gt *gt = hwe->gt;
u32 idledly, maxcnt;
u32 idledly, idledly_hw, idledly_reg_val, maxcnt;
u32 idledly_units_ps = 8 * gt->info.timestamp_base;
u32 maxcnt_units_ns = 640;
bool inhibit_switch = 0;
bool inhibit_switch = false;
bool wa_applied = false;
bool clamped_below_maxcnt = false;
if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_GT_WA(gt, 16023105232)) {
idledly = xe_mmio_read32(&gt->mmio, RING_IDLEDLY(hwe->mmio_base));
if ((!IS_SRIOV_VF(gt_to_xe(gt)) && XE_GT_WA(gt, 16023105232)) ||
XE_GT_WA(gt, 14025941587)) {
u32 mincnt_idledly_ns = 5000;
/* xe_gt_clock_init() warns and zeroes timestamp_base on unknown crystal clock. */
if (!idledly_units_ps)
return;
idledly_reg_val = xe_mmio_read32(&gt->mmio, RING_IDLEDLY(hwe->mmio_base));
maxcnt = xe_mmio_read32(&gt->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base));
inhibit_switch = idledly & INHIBIT_SWITCH_UNTIL_PREEMPTED;
idledly = REG_FIELD_GET(IDLE_DELAY, idledly);
idledly = DIV_ROUND_CLOSEST(idledly * idledly_units_ps, 1000);
inhibit_switch = idledly_reg_val & INHIBIT_SWITCH_UNTIL_PREEMPTED;
idledly = REG_FIELD_GET(IDLE_DELAY, idledly_reg_val);
idledly = DIV_ROUND_CLOSEST_ULL((u64)idledly * idledly_units_ps, 1000);
idledly_hw = idledly;
maxcnt = REG_FIELD_GET(IDLE_WAIT_TIME, maxcnt);
maxcnt *= maxcnt_units_ns;
if (xe_gt_WARN_ON(gt, idledly >= maxcnt || inhibit_switch)) {
idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * 1000),
idledly_units_ps);
xe_mmio_write32(&gt->mmio, RING_IDLEDLY(hwe->mmio_base), idledly);
/*
* Wa_14025941587 is applied before Wa_16023105232, which takes
* priority if the two ever conflict (not expected in practice).
*/
if (XE_GT_WA(gt, 14025941587) &&
idledly < mincnt_idledly_ns) {
idledly = mincnt_idledly_ns;
wa_applied = true;
}
if (XE_GT_WA(gt, 16023105232)) {
/* Clear the inhibit switch without disturbing a valid delay. */
if (inhibit_switch) {
idledly_reg_val &= ~INHIBIT_SWITCH_UNTIL_PREEMPTED;
wa_applied = true;
}
/* Warn only on the value read from hardware. */
xe_gt_WARN_ON(gt, idledly_hw >= maxcnt);
if (idledly >= maxcnt) {
/* maxcnt may be 0 if IDLE_WAIT_TIME is unprogrammed. */
idledly = maxcnt ? maxcnt - 1 : 0;
clamped_below_maxcnt = true;
wa_applied = true;
}
}
if (wa_applied) {
u32 idledly_ticks;
/*
* Wa_16023105232 requires idledly < maxcnt, so floor
* that clamp; otherwise round up to guarantee the
* Wa_14025941587 minimum survives tick quantization.
*/
if (clamped_below_maxcnt)
idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps);
else
idledly_ticks = DIV_ROUND_UP_ULL((u64)idledly * 1000,
idledly_units_ps);
/*
* Tick quantization can still push the rounded-up value
* to/above maxcnt; re-floor here so Wa_16023105232 keeps
* priority even in that case.
*/
if (!clamped_below_maxcnt && XE_GT_WA(gt, 16023105232) &&
(u64)idledly_ticks * idledly_units_ps >= (u64)maxcnt * 1000) {
xe_gt_dbg(gt, "idledly %s: %u ticks would exceed maxcnt=%u, so flooring\n",
hwe->name, idledly_ticks, maxcnt);
idledly = maxcnt ? maxcnt - 1 : 0;
idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps);
}
idledly_reg_val &= ~IDLE_DELAY;
idledly_reg_val |= REG_FIELD_PREP(IDLE_DELAY, idledly_ticks);
xe_gt_dbg(gt, "idledly %s: set %u max=%u inh=%u ts=%u\n",
hwe->name, idledly, maxcnt,
!!inhibit_switch, gt->info.timestamp_base);
xe_mmio_write32(&gt->mmio,
RING_IDLEDLY(hwe->mmio_base),
idledly_reg_val);
}
}
}

22
drivers/gpu/drm/xe/xe_i2c.c
View File

@ -111,14 +111,9 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c)
{
struct pci_dev *pci = to_pci_dev(i2c->drm_dev);
struct platform_device *pdev;
struct fwnode_handle *fwnode;
int ret;
u32 id;
fwnode = fwnode_create_software_node(xe_i2c_adapter_properties, NULL);
if (IS_ERR(fwnode))
return PTR_ERR(fwnode);
id = (pci_domain_nr(pci->bus) << 16) | pci_dev_id(pci);
/*
@ -128,14 +123,16 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c)
* platform_device_register_full() is done.
*/
pdev = platform_device_alloc(adapter_name, id);
if (!pdev) {
ret = -ENOMEM;
goto err_fwnode_remove;
}
if (!pdev)
return -ENOMEM;
ret = device_create_managed_software_node(&pdev->dev,
xe_i2c_adapter_properties,
NULL);
if (ret)
goto err_pdev_put;
pdev->dev.parent = i2c->drm_dev;
pdev->dev.fwnode = fwnode;
i2c->adapter_node = fwnode;
i2c->pdev = pdev;
ret = platform_device_add(pdev);
@ -146,8 +143,6 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c)
err_pdev_put:
platform_device_put(pdev);
err_fwnode_remove:
fwnode_remove_software_node(fwnode);
return ret;
}
@ -155,7 +150,6 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c)
static void xe_i2c_unregister_adapter(struct xe_i2c *i2c)
{
platform_device_unregister(i2c->pdev);
fwnode_remove_software_node(i2c->adapter_node);
}
/**

1
drivers/gpu/drm/xe/xe_i2c.h
View File

@ -33,7 +33,6 @@ struct xe_i2c_endpoint {
};
struct xe_i2c {
struct fwnode_handle *adapter_node;
struct platform_device *pdev;
struct i2c_adapter *adapter;
struct i2c_client *client[XE_I2C_MAX_CLIENTS];

73
drivers/gpu/drm/xe/xe_lrc.c
View File

@ -706,6 +706,7 @@ u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc)
#define LRC_CTX_JOB_TIMESTAMP_OFFSET 512
#define LRC_ENGINE_ID_PPHWSP_OFFSET 1024
#define LRC_PARALLEL_PPHWSP_OFFSET 2048
#define LRC_ULLS_PPHWSP_OFFSET 2048 /* Mutually exclusive with parallel */
#define LRC_SEQNO_OFFSET 0
#define LRC_START_SEQNO_OFFSET (LRC_SEQNO_OFFSET + 8)
@ -768,6 +769,12 @@ static inline u32 __xe_lrc_engine_id_offset(struct xe_lrc *lrc)
return xe_lrc_pphwsp_offset(lrc) + LRC_ENGINE_ID_PPHWSP_OFFSET;
}
static u32 __xe_lrc_ulls_offset(struct xe_lrc *lrc)
{
/* The ulls is stored in the driver-defined portion of PPHWSP */
return xe_lrc_pphwsp_offset(lrc) + LRC_ULLS_PPHWSP_OFFSET;
}
static u32 __xe_lrc_ctx_timestamp_offset(struct xe_lrc *lrc)
{
return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP * sizeof(u32);
@ -835,6 +842,7 @@ DECL_MAP_ADDR_HELPERS(ctx_job_timestamp, lrc->bo)
DECL_MAP_ADDR_HELPERS(ctx_timestamp, lrc->bo)
DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo)
DECL_MAP_ADDR_HELPERS(parallel, lrc->bo)
DECL_MAP_ADDR_HELPERS(ulls, lrc->bo)
DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo)
DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo)
DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo)
@ -1805,6 +1813,26 @@ void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail)
xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, tail);
}
/**
* xe_lrc_ring_tail_ggtt_addr() - Saved ring tail GGTT address
* @lrc: Pointer to the lrc.
*
* GGTT address of the ring tail as saved for this context - in the indirect
* ring state on platforms which have it, otherwise in the context image.
* This is what a context restore loads the tail register from, so a ring
* which advances the tail register itself must keep this in sync.
*
* Returns: saved ring tail GGTT address
*/
u32 xe_lrc_ring_tail_ggtt_addr(struct xe_lrc *lrc)
{
if (xe_lrc_has_indirect_ring_state(lrc))
return __xe_lrc_indirect_ring_ggtt_addr(lrc) +
INDIRECT_CTX_RING_TAIL * sizeof(u32);
return __xe_lrc_regs_ggtt_addr(lrc) + CTX_RING_TAIL * sizeof(u32);
}
u32 xe_lrc_ring_tail(struct xe_lrc *lrc)
{
if (xe_lrc_has_indirect_ring_state(lrc))
@ -1984,6 +2012,51 @@ static u32 xe_lrc_engine_id(struct xe_lrc *lrc)
return xe_map_read32(xe, &map);
}
#define semaphore_offset(seqno) \
(sizeof(u32) * ((seqno) % LRC_MIGRATION_ULLS_SEMAPHORE_COUNT))
/**
* xe_lrc_ulls_semaphore_ggtt_addr() - ULLS semaphore GGTT address
* @lrc: Pointer to the lrc.
* @seqno: seqno of current job.
*
* Calculate ULLS semaphore GGTT address based on input seqno
*
* Returns: ULLS semaphore GGTT address
*/
u32 xe_lrc_ulls_semaphore_ggtt_addr(struct xe_lrc *lrc, u32 seqno)
{
xe_assert(lrc_to_xe(lrc), semaphore_offset(seqno) <
LRC_PPHWSP_SIZE - LRC_ULLS_PPHWSP_OFFSET);
return __xe_lrc_ulls_ggtt_addr(lrc) + semaphore_offset(seqno);
}
/**
* xe_lrc_set_ulls_semaphore() - Set ULLS semaphore
* @lrc: Pointer to the lrc.
* @seqno: seqno of current job.
*
* Set ULLS semaphore based on input seqno
*/
void xe_lrc_set_ulls_semaphore(struct xe_lrc *lrc, u32 seqno)
{
struct xe_device *xe = lrc_to_xe(lrc);
struct iosys_map map = __xe_lrc_ulls_map(lrc);
xe_assert(xe, semaphore_offset(seqno) <
LRC_PPHWSP_SIZE - LRC_ULLS_PPHWSP_OFFSET);
/*
* The ring contents this semaphore releases are ordered by the
* xe_device_wmb() at the end of xe_lrc_write_ring().
*/
iosys_map_incr(&map, semaphore_offset(seqno));
xe_map_write32(xe, &map, LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL);
xe_device_wmb(xe); /* Flush write to hardware */
}
static int instr_dw(u32 cmd_header)
{
/* GFXPIPE "SINGLE_DW" opcodes are a single dword */

4
drivers/gpu/drm/xe/xe_lrc.h
View File

@ -112,6 +112,7 @@ u32 xe_lrc_regs_offset(struct xe_lrc *lrc);
void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail);
u32 xe_lrc_ring_tail(struct xe_lrc *lrc);
u32 xe_lrc_ring_tail_ggtt_addr(struct xe_lrc *lrc);
void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head);
u32 xe_lrc_ring_head(struct xe_lrc *lrc);
u32 xe_lrc_ring_space(struct xe_lrc *lrc);
@ -127,6 +128,9 @@ void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe);
void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
u32 *regs);
u32 xe_lrc_ulls_semaphore_ggtt_addr(struct xe_lrc *lrc, u32 seqno);
void xe_lrc_set_ulls_semaphore(struct xe_lrc *lrc, u32 seqno);
u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr);
void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val);

4
drivers/gpu/drm/xe/xe_lrc_types.h
View File

@ -12,6 +12,10 @@
struct xe_bo;
#define LRC_MIGRATION_ULLS_SEMAPHORE_COUNT 64 /* Must be pow2 */
#define LRC_MIGRATION_ULLS_SEMAPHORE_CLEAR 0
#define LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL 1
/**
* struct xe_lrc - Logical ring context (LRC) and submission ring object
*/

771
drivers/gpu/drm/xe/xe_migrate.c
View File

@ -8,6 +8,7 @@
#include <linux/bitfield.h>
#include <linux/sizes.h>
#include <drm/drm_drv.h>
#include <drm/drm_managed.h>
#include <drm/drm_pagemap.h>
#include <drm/ttm/ttm_tt.h>
@ -22,6 +23,7 @@
#include "xe_assert.h"
#include "xe_bb.h"
#include "xe_bo.h"
#include "xe_configfs.h"
#include "xe_exec_queue.h"
#include "xe_ggtt.h"
#include "xe_gt.h"
@ -32,6 +34,7 @@
#include "xe_mem_pool.h"
#include "xe_mocs.h"
#include "xe_pat.h"
#include "xe_pm.h"
#include "xe_printk.h"
#include "xe_pt.h"
#include "xe_res_cursor.h"
@ -45,6 +48,157 @@
#include "xe_vm.h"
#include "xe_vram.h"
/**
* DOC: ULLS (Ultra Low Latency Submission) for migration jobs
*
* Migration jobs issued on behalf of GPU page faults and SVM prefetches sit
* directly in the critical path of a stalled GPU workload. The dominant cost
* of such a job is not the copy or clear itself but the submission latency:
* the H2G round trip to GuC, the GuC scheduling decision, and the hardware
* context switch required to place the migration LRC on an engine.
*
* ULLS removes that cost by keeping the migration context resident and
* *running* on the hardware engine across jobs. Instead of the ring going
* empty and the context being switched out between jobs, the tail of every
* ULLS job parks the engine on a semaphore wait for the *next* job's
* semaphore, and then advances the ring tail itself. Submitting the next job
* therefore costs the CPU a single write to signal that semaphore - no H2G,
* no GuC round trip, no context switch, no MMIO.
*
* Requirements
* ------------
*
* ULLS is only used on dGFX platforms with USM support, where a hardware
* engine is reserved exclusively for migration jobs. Because the engine
* spins on a semaphore while ULLS is active, it cannot be shared with
* user submissions.
*
* ULLS can be disabled by setting the ``migrate_ulls_period_ms`` configfs
* attribute to 0. Otherwise, the same attribute controls how long ULLS
* remains active before exiting, in milliseconds.
*
* Fixed size jobs
* ---------------
*
* A job updates the ring tail to cover its successor, but it is emitted long
* before that successor exists, so it can not know how much ring the
* successor will occupy. Every ULLS job is therefore padded out to exactly
* ULLS_JOB_SIZE_BYTES, which lets the next tail be computed arithmetically
* from where the current job started.
*
* This is why the shorter jobs still have to reach the same size: the "last"
* job skips the batch buffers and the postamble, and pads the difference with
* MI_NOOP. The "first" job is not covered by any predecessor's tail update
* and so is unconstrained, but is padded anyway to keep the arithmetic
* uniform.
*
* Leaving ULLS mode always goes through a "last" job, which emits no tail
* update, so an ordinary variable length migration job never follows a
* prediction.
*
* Semaphores
* ----------
*
* The semaphores live in the driver-defined portion of the migration LRC's
* PPHWSP (see LRC_ULLS_PPHWSP_OFFSET, mutually exclusive with the parallel
* submission area). There are LRC_MIGRATION_ULLS_SEMAPHORE_COUNT of them and
* a job's semaphore is selected by ``seqno % COUNT``, so the semaphore ring
* wraps with the job seqnos. To guarantee a job can never overwrite the
* semaphore of a job still in flight, the GuC backend caps the migration
* queue's scheduler job count at LRC_MIGRATION_ULLS_SEMAPHORE_COUNT - 1.
*
* Ring layout of a ULLS job
* -------------------------
*
* Emitted by emit_migration_job_gen12() in xe_ring_ops.c::
*
* preamble: clear semaphore[seqno] (reuse for a later wrap)
* <copy timestamp, start seqno store>
* <batch buffer start(s)> (skipped on first/last job)
* <seqno write + user interrupt>
* postamble: wait on semaphore[seqno + 1]
* SDI saved ring tail = end of next job
* LRI RING_TAIL = end of next job
* (skipped on the last job)
* pad: MI_NOOP up to ULLS_JOB_SIZE_DW
*
* The preamble clears the current job's semaphore so it can be reused once
* the seqno space wraps. The postamble is what keeps the engine busy: it
* blocks on the next job's semaphore, which is only signaled when that job is
* actually submitted, and then advances the ring tail over it. It advances
* the saved tail as well as the tail register, keeping the two in step
* without any help from the CPU, so a context save and restore can not rewind
* the tail behind work which has already been published.
*
* The tail register write must be non-posted, i.e. it must not carry
* MI_LRI_FORCE_POSTED. Posted, the new tail is free to land after the command
* streamer has already drained the rest of the job, at which point the command
* streamer sees head == the old tail and parks as though the ring were empty.
* A parked context can be switched off the hardware, and the fast path below
* has no H2G with which to ask GuC to bring it back.
*
* The tail must be published after the semaphore wait, not before it. The
* command streamer fetches ring contents up to the tail while it is parked,
* so a tail published ahead of the wait lets it fetch the next job's slot
* before the CPU has written the job there. Once released it then executes
* the MI_NOOPs it fetched instead of the job, that job's fence never signals,
* and the engine drains and idles with nothing left to wake it.
*
* Submission fast path
* --------------------
*
* In submit_exec_queue() (xe_guc_submit.c), a ULLS job that is not the first
* one reduces to::
*
* xe_lrc_set_ulls_semaphore(lrc, seqno); release previous job
*
* The XE_GUC_ACTION_SCHED_CONTEXT H2G is suppressed, and so is the write of
* the saved ring tail: the previous job's postamble is parked on this job's
* semaphore and publishes this job's tail, both in the tail register and in
* the context image, as soon as it is released. The semaphore signal is all
* that is left, and the engine walks straight into this job.
*
* This does assume the context stays resident for as long as ULLS mode is
* active. Nothing else is scheduled on the reserved engine, so the only ways
* off the hardware are the "last" job below, or a reset - and a migration job
* failing already wedges the device.
*
* Enter / exit
* ------------
*
* xe_migrate_ulls_enter() is called from the page fault handler and from the
* SVM prefetch path, i.e. exactly where low latency migration matters. It
* takes a PM runtime reference (the device must not suspend while the engine
* spins), then submits a "first" ULLS job. That first job carries no batch
* buffer; it exists only to get the context onto the hardware through the
* normal GuC path and to leave the engine waiting on the next semaphore,
* pipelining the GuC/HW context switch out of the critical path.
*
* No forcewake reference is required. Nothing in the fast path touches MMIO,
* and the engine keeps itself awake for as long as it is executing the ring.
* Not needing host MMIO access is also what lets ULLS run on SRIOV VFs.
*
* Keeping an engine spinning costs power, so ULLS is not left enabled
* indefinitely. Every enter and every ULLS job submission re-arms
* @xe_migrate.ulls.exit_work with a ULLS_EXIT_JIFFIES delay. When it fires
* with the queue idle, it submits a "last" ULLS job - again with no batch
* buffer and, crucially, with no postamble semaphore wait or tail update -
* which lets the ring drain so the context can be switched off the hardware.
* The PM reference is then dropped. If the queue was not idle, the worker
* simply re-arms itself.
*
* Job state
* ---------
*
* The state above is communicated to the ring ops and GuC backend via
* @xe_sched_job.ulls, set under @xe_migrate.job_mutex:
*
* - %ULLS_NONE: job submitted outside of ULLS mode
* - %ULLS_ENTER: job that enters ULLS mode
* - %ULLS_ACTIVE: job submitted while in ULLS mode
* - %ULLS_EXIT: job that exits ULLS mode
*/
/**
* struct xe_migrate - migrate context.
*/
@ -75,18 +229,23 @@ struct xe_migrate {
* Protected by @job_mutex.
*/
struct dma_fence *fence;
/**
* @vm_update_sa: For integrated, used to suballocate page-tables
* out of the pt_bo.
*/
struct drm_suballoc_manager vm_update_sa;
/** @min_chunk_size: For dgfx, Minimum chunk size */
u64 min_chunk_size;
/** @ulls: ULLS support */
struct {
/** @ulls.exit_ms: ULLS exit period milliseconds */
u32 exit_ms;
/** @ulls.enabled: ULLS is enabled, protected by job_mutex */
bool enabled;
/** @ulls.exit_work: ULLS exit worker */
struct delayed_work exit_work;
} ulls;
};
#define ULLS_EXIT_JIFFIES(_m) msecs_to_jiffies((_m)->ulls.exit_ms)
#define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */
#define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */
#define NUM_KERNEL_PDE 15
#define NUM_PT_SLOTS 48
#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M
#define MAX_NUM_PTE 512
@ -101,9 +260,30 @@ struct xe_migrate {
*/
#define MAX_PTE_PER_SDI 0x1FEU
static bool xe_migrate_ulls_enabled(struct xe_migrate *m)
{
lockdep_assert_held(&m->job_mutex);
return m->ulls.enabled;
}
static void xe_migrate_ulls_toggle_enable(struct xe_migrate *m, bool enabled)
{
lockdep_assert_held(&m->job_mutex);
m->ulls.enabled = enabled;
}
static void xe_migrate_fini(void *arg)
{
struct xe_migrate *m = arg;
struct xe_device *xe = tile_to_xe(m->tile);
disable_delayed_work_sync(&m->ulls.exit_work);
scoped_guard(mutex, &m->job_mutex) {
if (xe_migrate_ulls_enabled(m)) {
xe_pm_runtime_put(xe);
xe_migrate_ulls_toggle_enable(m, false);
}
}
xe_vm_lock(m->q->vm, false);
xe_bo_unpin(m->pt_bo);
@ -111,7 +291,6 @@ static void xe_migrate_fini(void *arg)
dma_fence_put(m->fence);
xe_bo_put(m->pt_bo);
drm_suballoc_manager_fini(&m->vm_update_sa);
mutex_destroy(&m->job_mutex);
xe_vm_close_and_put(m->q->vm);
xe_exec_queue_put(m->q);
@ -231,8 +410,6 @@ static int xe_migrate_pt_bo_alloc(struct xe_tile *tile, struct xe_migrate *m,
BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);
/* Must be a multiple of 64K to support all platforms */
BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K);
/* And one slot reserved for the 4KiB page table updates */
BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1));
/* Need to be sure everything fits in the first PT, or create more */
xe_tile_assert(tile, m->batch_base_ofs + xe_bo_size(batch) < SZ_2M);
@ -255,7 +432,8 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
struct xe_device *xe = tile_to_xe(tile);
u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
u8 id = tile->id;
u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;
u32 num_entries = NUM_PT_SLOTS, num_level =
xe_vm_pt_root(vm, id)->level;
#define VRAM_IDENTITY_MAP_PT_COUNT 4
u32 num_setup = num_level + VRAM_IDENTITY_MAP_PT_COUNT;
#undef VRAM_IDENTITY_MAP_PT_COUNT
@ -267,7 +445,7 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
u64 l1_pt_ofs = xe_bo_size(bo) - 5 * XE_PAGE_SIZE;
entry = vm->pt_ops->pde_encode_bo(bo, l1_pt_ofs);
xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry);
xe_pt_write(xe, &xe_vm_pt_root(vm, id)->bo->vmap, 0, entry);
map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE;
@ -388,17 +566,9 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
}
}
if (ofs)
*ofs = map_ofs;
}
static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs)
{
/*
* Example layout created above, with root level = 3:
* [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's
* [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's
* [PT9...PT40]: Userspace PT's for VM_BIND, 4 KiB PTE's
* [PT41 = PDE 0] [PT44...PT47 = 4K and 2M vram identity maps]
*
* This makes the lowest part of the VM point to the pagetables.
@ -406,19 +576,13 @@ static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs)
* and flushes, other parts of the VM can be used either for copying and
* clearing.
*
* For performance, the kernel reserves PDE's, so about 20 are left
* for async VM updates.
*
* To make it easier to work, each scratch PT is put in slot (1 + PT #)
* everywhere, this allows lockless updates to scratch pages by using
* the different addresses in VM.
*/
#define NUM_VMUSA_UNIT_PER_PAGE 32
#define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE)
#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64))
drm_suballoc_manager_init(&m->vm_update_sa,
(size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) *
NUM_VMUSA_UNIT_PER_PAGE, 0);
if (ofs)
*ofs = map_ofs;
}
static bool xe_migrate_needs_ccs_emit(struct xe_device *xe)
@ -463,7 +627,6 @@ static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m
return err;
xe_migrate_prepare_vm(tile, m, vm, &map_ofs);
xe_migrate_suballoc_manager_init(m, map_ofs);
drm_exec_retry_on_contention(&exec);
xe_validation_retry_on_oom(&ctx, &err);
}
@ -471,6 +634,150 @@ static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m
return err;
}
static struct dma_fence *__xe_migrate_job_push(struct xe_migrate *m,
struct xe_sched_job *job,
enum xe_ulls_state ulls)
{
struct dma_fence *fence;
lockdep_assert_held(&m->job_mutex);
xe_tile_assert(m->tile, m->q == job->q);
job->ulls = ulls;
xe_sched_job_arm(job);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
return fence;
}
/*
* Arm and push a migration job, tagging it as a ULLS job and deferring the
* ULLS exit while ULLS mode is active.
*
* Returns a reference to the job's finished fence.
*/
static struct dma_fence *xe_migrate_job_push(struct xe_migrate *m,
struct xe_sched_job *job)
{
enum xe_ulls_state ulls = ULLS_NONE;
lockdep_assert_held(&m->job_mutex);
if (xe_migrate_ulls_enabled(m)) {
ulls = ULLS_ACTIVE;
mod_delayed_work(system_percpu_wq, &m->ulls.exit_work,
ULLS_EXIT_JIFFIES(m));
}
return __xe_migrate_job_push(m, job, ulls);
}
/**
* xe_migrate_ulls_enter() - Enter ULLS mode
* @m: The migration context.
*
* If DGFX, enter ULLS mode bypassing GuC / HW context switches by utilizing
* semaphore and continuously running batches.
*/
void xe_migrate_ulls_enter(struct xe_migrate *m)
{
struct xe_device *xe = tile_to_xe(m->tile);
struct xe_sched_job *job = NULL;
u64 batch_addr[2] = { 0, 0 };
bool alloc = false;
xe_assert(xe, xe->info.has_usm);
if (!IS_DGFX(xe) || !m->ulls.exit_ms)
return;
job_alloc:
if (alloc) {
/*
* Must be done outside job_mutex as that lock is tainted with
* reclaim.
*/
job = xe_sched_job_create(m->q, batch_addr);
if (WARN_ON_ONCE(IS_ERR(job)))
return; /* Not fatal */
}
mutex_lock(&m->job_mutex);
if (!xe_migrate_ulls_enabled(m)) {
struct dma_fence *fence;
if (!job) {
alloc = true;
mutex_unlock(&m->job_mutex);
goto job_alloc;
}
/* Pairs with PM put on ULLS exit */
xe_pm_runtime_get_noresume(xe);
xe_sched_job_get(job);
fence = __xe_migrate_job_push(m, job, ULLS_ENTER);
dma_fence_put(fence);
xe_dbg(xe, "Migrate ULLS mode enter");
xe_migrate_ulls_toggle_enable(m, true);
}
if (job)
xe_sched_job_put(job);
if (xe_migrate_ulls_enabled(m))
mod_delayed_work(system_percpu_wq, &m->ulls.exit_work,
ULLS_EXIT_JIFFIES(m));
mutex_unlock(&m->job_mutex);
}
static void xe_migrate_ulls_exit(struct work_struct *work)
{
struct xe_migrate *m = container_of(work, struct xe_migrate,
ulls.exit_work.work);
struct xe_device *xe = tile_to_xe(m->tile);
struct xe_sched_job *job = NULL;
struct dma_fence *fence = NULL;
u64 batch_addr[2] = { 0, 0 };
int idx;
xe_assert(xe, m->ulls.enabled);
if (!drm_dev_enter(&xe->drm, &idx))
return;
/*
* Must be done outside job_mutex as that lock is tainted with
* reclaim and must be done holding a pm ref.
*/
job = xe_sched_job_create(m->q, batch_addr);
if (WARN_ON_ONCE(IS_ERR(job))) {
drm_dev_exit(idx);
mod_delayed_work(system_percpu_wq, &m->ulls.exit_work,
ULLS_EXIT_JIFFIES(m));
return; /* Not fatal */
}
scoped_guard(mutex, &m->job_mutex) {
if (xe_exec_queue_is_idle(m->q, 1)) {
fence = __xe_migrate_job_push(m, job, ULLS_EXIT);
dma_fence_put(fence);
xe_pm_runtime_put(xe); /* Pairs with PM get in enter */
xe_migrate_ulls_toggle_enable(m, false);
cancel_delayed_work(&m->ulls.exit_work);
xe_dbg(xe, "Migrate ULLS mode exit");
} else {
xe_sched_job_put(job);
mod_delayed_work(system_percpu_wq, &m->ulls.exit_work,
ULLS_EXIT_JIFFIES(m));
}
}
drm_dev_exit(idx);
}
/**
* xe_migrate_init() - Initialize a migrate context
* @m: The migration context
@ -529,6 +836,8 @@ int xe_migrate_init(struct xe_migrate *m)
might_lock(&m->job_mutex);
fs_reclaim_release(GFP_KERNEL);
INIT_DELAYED_WORK(&m->ulls.exit_work, xe_migrate_ulls_exit);
err = devm_add_action_or_reset(xe->drm.dev, xe_migrate_fini, m);
if (err)
return err;
@ -546,6 +855,9 @@ int xe_migrate_init(struct xe_migrate *m)
(unsigned long long)m->min_chunk_size);
}
m->ulls.exit_ms =
xe_configfs_get_migrate_ulls_period_ms(to_pci_dev(xe->drm.dev));
return err;
err_out:
@ -1056,10 +1368,8 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
}
mutex_lock(&m->job_mutex);
xe_sched_job_arm(job);
dma_fence_put(fence);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
fence = xe_migrate_job_push(m, job);
dma_fence_put(m->fence);
m->fence = dma_fence_get(fence);
@ -1146,6 +1456,9 @@ struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate)
return migrate->q->lrc[0];
}
/* XXX: With CPU binds this can be removed in a follow up */
#define NUM_KERNEL_PDE 15
static u64 migrate_vm_ppgtt_addr_tlb_inval(void)
{
/*
@ -1484,10 +1797,8 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off
DMA_RESV_USAGE_BOOKKEEP));
scoped_guard(mutex, &m->job_mutex) {
xe_sched_job_arm(job);
dma_fence_put(fence);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
fence = xe_migrate_job_push(m, job);
dma_fence_put(m->fence);
m->fence = dma_fence_get(fence);
@ -1721,10 +2032,8 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
}
mutex_lock(&m->job_mutex);
xe_sched_job_arm(job);
dma_fence_put(fence);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
fence = xe_migrate_job_push(m, job);
dma_fence_put(m->fence);
m->fence = dma_fence_get(fence);
@ -1754,55 +2063,6 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
return fence;
}
static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
const struct xe_vm_pgtable_update_op *pt_op,
const struct xe_vm_pgtable_update *update,
struct xe_migrate_pt_update *pt_update)
{
const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
u32 chunk;
u32 ofs = update->ofs, size = update->qwords;
/*
* If we have 512 entries (max), we would populate it ourselves,
* and update the PDE above it to the new pointer.
* The only time this can only happen if we have to update the top
* PDE. This requires a BO that is almost vm->size big.
*
* This shouldn't be possible in practice.. might change when 16K
* pages are used. Hence the assert.
*/
xe_tile_assert(tile, update->qwords < MAX_NUM_PTE);
if (!ppgtt_ofs)
ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),
xe_bo_addr(update->pt_bo, 0,
XE_PAGE_SIZE), false);
do {
u64 addr = ppgtt_ofs + ofs * 8;
chunk = min(size, MAX_PTE_PER_SDI);
/* Ensure populatefn can do memset64 by aligning bb->cs */
if (!(bb->len & 1))
bb->cs[bb->len++] = MI_NOOP;
bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
bb->cs[bb->len++] = lower_32_bits(addr);
bb->cs[bb->len++] = upper_32_bits(addr);
if (pt_op->bind)
ops->populate(pt_update, tile, NULL, bb->cs + bb->len,
ofs, chunk, update);
else
ops->clear(pt_update, tile, NULL, bb->cs + bb->len,
ofs, chunk, update);
bb->len += chunk * 2;
ofs += chunk;
size -= chunk;
} while (size);
}
struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m)
{
return xe_vm_get(m->q->vm);
@ -1818,284 +2078,6 @@ struct migrate_test_params {
container_of(_priv, struct migrate_test_params, base)
#endif
static struct dma_fence *
xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
struct xe_migrate_pt_update *pt_update)
{
XE_TEST_DECLARE(struct migrate_test_params *test =
to_migrate_test_params
(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)
const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
struct xe_vm *vm = pt_update->vops->vm;
struct xe_vm_pgtable_update_ops *pt_update_ops =
&pt_update->vops->pt_update_ops[pt_update->tile_id];
int err;
u32 i, j;
if (XE_TEST_ONLY(test && test->force_gpu))
return ERR_PTR(-ETIME);
if (ops->pre_commit) {
pt_update->job = NULL;
err = ops->pre_commit(pt_update);
if (err)
return ERR_PTR(err);
}
for (i = 0; i < pt_update_ops->num_ops; ++i) {
const struct xe_vm_pgtable_update_op *pt_op =
&pt_update_ops->ops[i];
for (j = 0; j < pt_op->num_entries; j++) {
const struct xe_vm_pgtable_update *update =
&pt_op->entries[j];
if (pt_op->bind)
ops->populate(pt_update, m->tile,
&update->pt_bo->vmap, NULL,
update->ofs, update->qwords,
update);
else
ops->clear(pt_update, m->tile,
&update->pt_bo->vmap, NULL,
update->ofs, update->qwords, update);
}
}
trace_xe_vm_cpu_bind(vm);
xe_device_wmb(vm->xe);
return dma_fence_get_stub();
}
static struct dma_fence *
__xe_migrate_update_pgtables(struct xe_migrate *m,
struct xe_migrate_pt_update *pt_update,
struct xe_vm_pgtable_update_ops *pt_update_ops)
{
const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
struct xe_tile *tile = m->tile;
struct xe_gt *gt = tile->primary_gt;
struct xe_device *xe = tile_to_xe(tile);
struct xe_sched_job *job;
struct dma_fence *fence;
struct drm_suballoc *sa_bo = NULL;
struct xe_bb *bb;
u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0;
u32 num_updates = 0, current_update = 0;
u64 addr;
int err = 0;
bool is_migrate = pt_update_ops->q == m->q;
bool usm = is_migrate && xe->info.has_usm;
for (i = 0; i < pt_update_ops->num_ops; ++i) {
struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
struct xe_vm_pgtable_update *updates = pt_op->entries;
num_updates += pt_op->num_entries;
for (j = 0; j < pt_op->num_entries; ++j) {
u32 num_cmds = DIV_ROUND_UP(updates[j].qwords,
MAX_PTE_PER_SDI);
/* align noop + MI_STORE_DATA_IMM cmd prefix */
batch_size += 4 * num_cmds + updates[j].qwords * 2;
}
}
/* fixed + PTE entries */
if (IS_DGFX(xe))
batch_size += 2;
else
batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) +
num_updates * 2;
bb = xe_bb_new(gt, batch_size, usm);
if (IS_ERR(bb))
return ERR_CAST(bb);
/* For sysmem PTE's, need to map them in our hole.. */
if (!IS_DGFX(xe)) {
u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
u32 ptes, ofs;
ppgtt_ofs = NUM_KERNEL_PDE - 1;
if (!is_migrate) {
u32 num_units = DIV_ROUND_UP(num_updates,
NUM_VMUSA_WRITES_PER_UNIT);
if (num_units > m->vm_update_sa.size) {
err = -ENOBUFS;
goto err_bb;
}
sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units,
GFP_KERNEL, true, 0);
if (IS_ERR(sa_bo)) {
err = PTR_ERR(sa_bo);
goto err_bb;
}
ppgtt_ofs = NUM_KERNEL_PDE +
(drm_suballoc_soffset(sa_bo) /
NUM_VMUSA_UNIT_PER_PAGE);
page_ofs = (drm_suballoc_soffset(sa_bo) %
NUM_VMUSA_UNIT_PER_PAGE) *
VM_SA_UPDATE_UNIT_SIZE;
}
/* Map our PT's to gtt */
i = 0;
j = 0;
ptes = num_updates;
ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;
while (ptes) {
u32 chunk = min(MAX_PTE_PER_SDI, ptes);
u32 idx = 0;
bb->cs[bb->len++] = MI_STORE_DATA_IMM |
MI_SDI_NUM_QW(chunk);
bb->cs[bb->len++] = ofs;
bb->cs[bb->len++] = 0; /* upper_32_bits */
for (; i < pt_update_ops->num_ops; ++i) {
struct xe_vm_pgtable_update_op *pt_op =
&pt_update_ops->ops[i];
struct xe_vm_pgtable_update *updates = pt_op->entries;
for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) {
struct xe_vm *vm = pt_update->vops->vm;
struct xe_bo *pt_bo = updates[j].pt_bo;
if (idx == chunk)
goto next_cmd;
xe_tile_assert(tile, xe_bo_size(pt_bo) == SZ_4K);
/* Map a PT at most once */
if (pt_bo->update_index < 0)
pt_bo->update_index = current_update;
addr = vm->pt_ops->pte_encode_bo(pt_bo, 0,
pat_index, 0);
bb->cs[bb->len++] = lower_32_bits(addr);
bb->cs[bb->len++] = upper_32_bits(addr);
}
j = 0;
}
next_cmd:
ptes -= chunk;
ofs += chunk * sizeof(u64);
}
bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
update_idx = bb->len;
addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
(page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
for (i = 0; i < pt_update_ops->num_ops; ++i) {
struct xe_vm_pgtable_update_op *pt_op =
&pt_update_ops->ops[i];
struct xe_vm_pgtable_update *updates = pt_op->entries;
for (j = 0; j < pt_op->num_entries; ++j) {
struct xe_bo *pt_bo = updates[j].pt_bo;
write_pgtable(tile, bb, addr +
pt_bo->update_index * XE_PAGE_SIZE,
pt_op, &updates[j], pt_update);
}
}
} else {
/* phys pages, no preamble required */
bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
update_idx = bb->len;
for (i = 0; i < pt_update_ops->num_ops; ++i) {
struct xe_vm_pgtable_update_op *pt_op =
&pt_update_ops->ops[i];
struct xe_vm_pgtable_update *updates = pt_op->entries;
for (j = 0; j < pt_op->num_entries; ++j)
write_pgtable(tile, bb, 0, pt_op, &updates[j],
pt_update);
}
}
job = xe_bb_create_migration_job(pt_update_ops->q, bb,
xe_migrate_batch_base(m, usm),
update_idx);
if (IS_ERR(job)) {
err = PTR_ERR(job);
goto err_sa;
}
xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB);
if (ops->pre_commit) {
pt_update->job = job;
err = ops->pre_commit(pt_update);
if (err)
goto err_job;
}
if (is_migrate)
mutex_lock(&m->job_mutex);
xe_sched_job_arm(job);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
if (is_migrate)
mutex_unlock(&m->job_mutex);
xe_bb_free(bb, fence);
drm_suballoc_free(sa_bo, fence);
return fence;
err_job:
xe_sched_job_put(job);
err_sa:
drm_suballoc_free(sa_bo, NULL);
err_bb:
xe_bb_free(bb, NULL);
return ERR_PTR(err);
}
/**
* xe_migrate_update_pgtables() - Pipelined page-table update
* @m: The migrate context.
* @pt_update: PT update arguments
*
* Perform a pipelined page-table update. The update descriptors are typically
* built under the same lock critical section as a call to this function. If
* using the default engine for the updates, they will be performed in the
* order they grab the job_mutex. If different engines are used, external
* synchronization is needed for overlapping updates to maintain page-table
* consistency. Note that the meaning of "overlapping" is that the updates
* touch the same page-table, which might be a higher-level page-directory.
* If no pipelining is needed, then updates may be performed by the cpu.
*
* Return: A dma_fence that, when signaled, indicates the update completion.
*/
struct dma_fence *
xe_migrate_update_pgtables(struct xe_migrate *m,
struct xe_migrate_pt_update *pt_update)
{
struct xe_vm_pgtable_update_ops *pt_update_ops =
&pt_update->vops->pt_update_ops[pt_update->tile_id];
struct dma_fence *fence;
fence = xe_migrate_update_pgtables_cpu(m, pt_update);
/* -ETIME indicates a job is needed, anything else is legit error */
if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)
return fence;
return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops);
}
/**
* xe_migrate_wait() - Complete all operations using the xe_migrate context
* @m: Migrate context to wait for.
@ -2327,9 +2309,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m,
}
mutex_lock(&m->job_mutex);
xe_sched_job_arm(job);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
fence = xe_migrate_job_push(m, job);
dma_fence_put(m->fence);
m->fence = dma_fence_get(fence);
@ -2597,56 +2577,6 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo,
return IS_ERR(fence) ? PTR_ERR(fence) : 0;
}
/**
* xe_migrate_job_lock() - Lock migrate job lock
* @m: The migration context.
* @q: Queue associated with the operation which requires a lock
*
* Lock the migrate job lock if the queue is a migration queue, otherwise
* assert the VM's dma-resv is held (user queue's have own locking).
*/
void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q)
{
bool is_migrate = q == m->q;
if (is_migrate)
mutex_lock(&m->job_mutex);
else
xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */
}
/**
* xe_migrate_job_unlock() - Unlock migrate job lock
* @m: The migration context.
* @q: Queue associated with the operation which requires a lock
*
* Unlock the migrate job lock if the queue is a migration queue, otherwise
* assert the VM's dma-resv is held (user queue's have own locking).
*/
void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q)
{
bool is_migrate = q == m->q;
if (is_migrate)
mutex_unlock(&m->job_mutex);
else
xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */
}
#if IS_ENABLED(CONFIG_PROVE_LOCKING)
/**
* xe_migrate_job_lock_assert() - Assert migrate job lock held of queue
* @q: Migrate queue
*/
void xe_migrate_job_lock_assert(struct xe_exec_queue *q)
{
struct xe_migrate *m = gt_to_tile(q->gt)->migrate;
xe_gt_assert(q->gt, q == m->q);
lockdep_assert_held(&m->job_mutex);
}
#endif
#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
#include "tests/xe_migrate.c"
#endif
@ -2696,10 +2626,7 @@ int xe_migrate_debug_ccs_overlap(struct xe_migrate *m,
xe_sched_job_add_migrate_flush(job, MI_FLUSH_DW_CCS);
mutex_lock(&m->job_mutex);
xe_sched_job_arm(job);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
fence = xe_migrate_job_push(m, job);
mutex_unlock(&m->job_mutex);
dma_fence_wait(fence, false);

95
drivers/gpu/drm/xe/xe_migrate.h
View File

@ -24,6 +24,7 @@ struct xe_pt;
struct xe_tile;
struct xe_vm;
struct xe_vm_pgtable_update;
struct xe_vm_pgtable_update_op;
struct xe_vma;
enum xe_sriov_vf_ccs_rw_ctxs;
@ -33,83 +34,6 @@ enum xe_migrate_copy_dir {
XE_MIGRATE_COPY_TO_SRAM,
};
/**
* struct xe_migrate_pt_update_ops - Callbacks for the
* xe_migrate_update_pgtables() function.
*/
struct xe_migrate_pt_update_ops {
/**
* @populate: Populate a command buffer or page-table with ptes.
* @pt_update: Embeddable callback argument.
* @tile: The tile for the current operation.
* @map: struct iosys_map into the memory to be populated.
* @pos: If @map is NULL, map into the memory to be populated.
* @ofs: qword offset into @map, unused if @map is NULL.
* @num_qwords: Number of qwords to write.
* @update: Information about the PTEs to be inserted.
*
* This interface is intended to be used as a callback into the
* page-table system to populate command buffers or shared
* page-tables with PTEs.
*/
void (*populate)(struct xe_migrate_pt_update *pt_update,
struct xe_tile *tile, struct iosys_map *map,
void *pos, u32 ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update);
/**
* @clear: Clear a command buffer or page-table with ptes.
* @pt_update: Embeddable callback argument.
* @tile: The tile for the current operation.
* @map: struct iosys_map into the memory to be populated.
* @pos: If @map is NULL, map into the memory to be populated.
* @ofs: qword offset into @map, unused if @map is NULL.
* @num_qwords: Number of qwords to write.
* @update: Information about the PTEs to be inserted.
*
* This interface is intended to be used as a callback into the
* page-table system to populate command buffers or shared
* page-tables with PTEs.
*/
void (*clear)(struct xe_migrate_pt_update *pt_update,
struct xe_tile *tile, struct iosys_map *map,
void *pos, u32 ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update);
/**
* @pre_commit: Callback to be called just before arming the
* sched_job.
* @pt_update: Pointer to embeddable callback argument.
*
* Return: 0 on success, negative error code on error.
*/
int (*pre_commit)(struct xe_migrate_pt_update *pt_update);
};
/**
* struct xe_migrate_pt_update - Argument to the
* struct xe_migrate_pt_update_ops callbacks.
*
* Intended to be subclassed to support additional arguments if necessary.
*/
struct xe_migrate_pt_update {
/** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */
const struct xe_migrate_pt_update_ops *ops;
/** @vops: VMA operations */
struct xe_vma_ops *vops;
/** @job: The job if a GPU page-table update. NULL otherwise */
struct xe_sched_job *job;
/**
* @ijob: The TLB invalidation job for primary GT. NULL otherwise
*/
struct xe_tlb_inval_job *ijob;
/**
* @mjob: The TLB invalidation job for media GT. NULL otherwise
*/
struct xe_tlb_inval_job *mjob;
/** @tile_id: Tile ID of the update */
u8 tile_id;
};
struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile);
int xe_migrate_init(struct xe_migrate *m);
@ -165,27 +89,14 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);
struct dma_fence *
xe_migrate_update_pgtables(struct xe_migrate *m,
struct xe_migrate_pt_update *pt_update);
void xe_migrate_wait(struct xe_migrate *m);
#if IS_ENABLED(CONFIG_PROVE_LOCKING)
void xe_migrate_job_lock_assert(struct xe_exec_queue *q);
#else
static inline void xe_migrate_job_lock_assert(struct xe_exec_queue *q)
{
}
#endif
void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q);
void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q);
#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
int xe_migrate_debug_ccs_overlap(struct xe_migrate *m,
struct xe_bo *scratch_bo,
bool write_to_ccs);
#endif
void xe_migrate_ulls_enter(struct xe_migrate *m);
#endif

5
drivers/gpu/drm/xe/xe_mmio.c
View File

@ -320,6 +320,7 @@ u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg)
return (u64)udw << 32 | ldw;
}
#define __XE_MMIO_WAIT_MAX_BACKOFF_100MS (100 * USEC_PER_MSEC)
static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val,
u32 timeout_us, u32 *out_val, bool atomic, bool expect_match)
{
@ -350,10 +351,10 @@ static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u
wait = ktime_us_delta(end, cur);
if (atomic)
udelay(wait);
udelay(min_t(s64, wait, (MAX_UDELAY_MS * 1000)));
else
usleep_range(wait, wait << 1);
wait <<= 1;
wait = min_t(s64, wait << 1, __XE_MMIO_WAIT_MAX_BACKOFF_100MS);
}
if (ret != 0) {

53
drivers/gpu/drm/xe/xe_pagefault.c
View File

@ -13,8 +13,8 @@
#include "xe_gt_printk.h"
#include "xe_gt_types.h"
#include "xe_gt_stats.h"
#include "xe_hw_engine.h"
#include "xe_log.h"
#include "xe_migrate.h"
#include "xe_pagefault.h"
#include "xe_pagefault_types.h"
#include "xe_pm.h"
@ -267,6 +267,7 @@ static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_pagefault *pf, u32 asid)
static int xe_pagefault_service(struct xe_pagefault *pf)
{
struct xe_gt *gt = pf->gt;
struct xe_device *xe = gt_to_xe(gt);
struct xe_vm *vm;
struct xe_vma *vma = NULL;
int err;
@ -278,8 +279,15 @@ static int xe_pagefault_service(struct xe_pagefault *pf)
return -EFAULT;
vm = xe_pagefault_asid_to_vm(pf, asid);
if (IS_ERR(vm))
return PTR_ERR(vm);
if (IS_ERR(vm)) {
err = PTR_ERR(vm);
trace_xe_pagefault_fail(xe, NULL, NULL, asid,
xe_pagefault_addr(pf),
xe_pagefault_get_error(pf), err);
return err;
}
xe_migrate_ulls_enter(gt_to_tile(gt)->migrate);
down_read(&vm->lock);
@ -312,6 +320,11 @@ static int xe_pagefault_service(struct xe_pagefault *pf)
err = xe_pagefault_handle_vma(gt, vma, pf, atomic);
unlock_vm:
if (err)
trace_xe_pagefault_fail(xe, vm, vma, asid,
xe_pagefault_addr(pf),
xe_pagefault_get_error(pf), err);
up_read(&vm->lock);
xe_vm_put(vm);
@ -607,36 +620,12 @@ static const char *xe_pagefault_error_to_str(enum xe_pagefault_error error)
static void xe_pagefault_print(struct xe_pagefault *pf)
{
u8 engine_class = FIELD_GET(XE_PAGEFAULT_ENGINE_CLASS_MASK,
pf->consumer.engine_class_instance);
u64 addr = xe_pagefault_addr(pf);
enum xe_pagefault_error err = xe_pagefault_get_error(pf);
xe_gt_info(pf->gt, "\n\tASID: %lu\n"
"\tFaulted Address: 0x%08x%08x\n"
"\tFaultType: %lu\n"
"\tAccessType: %lu\n"
"\tFaultLevel: %lu\n"
"\tEngineClass: %d %s\n"
"\tEngineInstance: %lu\n"
"\tSRCID: 0x%02lx\n"
"\tError: %s\n",
FIELD_GET(XE_PAGEFAULT_ASID_MASK,
pf->consumer.id),
upper_32_bits(addr),
lower_32_bits(addr),
FIELD_GET(XE_PAGEFAULT_TYPE_MASK,
pf->consumer.fault_type_level),
FIELD_GET(XE_PAGEFAULT_ACCESS_TYPE_MASK,
pf->consumer.access_type),
FIELD_GET(XE_PAGEFAULT_LEVEL_MASK,
pf->consumer.fault_type_level),
engine_class,
xe_hw_engine_class_to_str(engine_class),
FIELD_GET(XE_PAGEFAULT_ENGINE_INSTANCE_MASK,
pf->consumer.engine_class_instance),
FIELD_GET(XE_PAGEFAULT_SRCID_MASK,
pf->consumer.id),
xe_pagefault_error_to_str(xe_pagefault_get_error(pf)));
if (WARN_ON_ONCE(!pf->producer.ops->print))
return;
pf->producer.ops->print(pf, xe_pagefault_error_to_str(err));
}
static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *pf)

10
drivers/gpu/drm/xe/xe_pagefault_types.h
View File

@ -139,6 +139,16 @@ struct xe_pagefault_ops {
* Page fault producer ends acknowledgment from the consumer.
*/
void (*ack_fault_end)(void *private);
/**
* @print: Print pagefault info
* @pf: Page fault
* @err_str: Consumer servicing error string
*
* Print pagefault identification fields decoded from producer,
* along with consumer-specific @err_str prints.
* Must be implemented by all producers.
*/
void (*print)(struct xe_pagefault *pf, const char *err_str);
};
/**

5
drivers/gpu/drm/xe/xe_pci.c
View File

@ -371,6 +371,7 @@ static const __maybe_unused struct xe_device_desc pvc_desc = {
.has_display = false,
.has_drm_ras = true,
.has_gsc_nvm = 1,
.has_pt_mirror = 1,
.has_heci_gscfi = 1,
.max_gt_per_tile = 1,
.max_remote_tiles = 1,
@ -460,6 +461,7 @@ static const struct xe_device_desc nvls_desc = {
.has_flat_ccs = 1,
.has_pre_prod_wa = 1,
.has_sriov = true,
.has_pxp = true,
.max_gt_per_tile = 2,
MULTI_LRC_MASK,
.va_bits = 48,
@ -471,6 +473,7 @@ static const struct xe_device_desc cri_desc = {
PLATFORM(CRESCENTISLAND),
.dma_mask_size = 52,
.has_display = false,
.has_device_uid = true,
.has_drm_ras = true,
.has_flat_ccs = false,
.has_gsc_nvm = 1,
@ -791,6 +794,7 @@ static int xe_info_init_early(struct xe_device *xe,
xe->info.is_dgfx = desc->is_dgfx;
xe->info.has_cached_pt = desc->has_cached_pt;
xe->info.has_device_uid = desc->has_device_uid;
xe->info.has_drm_ras = desc->has_drm_ras;
xe->info.has_fan_control = desc->has_fan_control;
/* runtime fusing may force flat_ccs to disabled later */
@ -806,6 +810,7 @@ static int xe_info_init_early(struct xe_device *xe,
xe->info.has_mert = desc->has_mert;
xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist;
xe->info.has_pre_prod_wa = desc->has_pre_prod_wa;
xe->info.has_pt_mirror = desc->has_pt_mirror;
xe->info.has_pxp = desc->has_pxp;
xe->info.has_soc_remapper_sysctrl = desc->has_soc_remapper_sysctrl;
xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem;

3
drivers/gpu/drm/xe/xe_pci_sriov.c
View File

@ -177,6 +177,7 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs)
return num_vfs;
failed:
pf_resize_vf_vram_bar(xe, xe->sriov.pf.device_total_vfs);
xe_sriov_pf_unprovision_vfs(xe, num_vfs);
xe_pm_runtime_put(xe);
pf_finish_vfs_enabling(xe);
@ -204,6 +205,8 @@ static int pf_disable_vfs(struct xe_device *xe)
pci_disable_sriov(pdev);
pf_resize_vf_vram_bar(xe, xe->sriov.pf.device_total_vfs);
xe_sriov_pf_reprovision_default(xe);
pf_reset_vfs(xe, num_vfs);

4
drivers/gpu/drm/xe/xe_pci_types.h
View File

@ -40,6 +40,7 @@ struct xe_device_desc {
u8 has_cached_pt:1;
u8 has_display:1;
u8 has_device_uid:1;
u8 has_drm_ras:1;
u8 has_fan_control:1;
u8 has_flat_ccs:1;
@ -52,8 +53,9 @@ struct xe_device_desc {
u8 has_mbx_power_limits:1;
u8 has_mbx_thermal_info:1;
u8 has_mert:1;
u8 has_pre_prod_wa:1;
u8 has_page_reclaim_hw_assist:1;
u8 has_pre_prod_wa:1;
u8 has_pt_mirror:1;
u8 has_pxp:1;
u8 has_soc_remapper_sysctrl:1;
u8 has_soc_remapper_telem:1;

779
drivers/gpu/drm/xe/xe_pt.c
View File

File diff suppressed because it is too large Load Diff

12
drivers/gpu/drm/xe/xe_pt.h
View File

@ -39,14 +39,18 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred);
void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt);
int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops);
struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
int xe_pt_update_ops_prepare(struct xe_device *xe, struct xe_vma_ops *vops);
struct dma_fence *xe_pt_update_ops_run(struct xe_device *xe,
struct xe_vma_ops *vops);
void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops);
void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
void xe_pt_update_ops_fini(struct xe_device *xe, struct xe_vma_ops *vops);
void xe_pt_update_ops_abort(struct xe_device *xe, struct xe_vma_ops *vops);
bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma);
bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,
struct xe_svm_range *range);
struct xe_pt_job_ops *xe_pt_job_ops_alloc(u32 num_ops);
struct xe_pt_job_ops *xe_pt_job_ops_get(struct xe_pt_job_ops *pt_job_ops);
void xe_pt_job_ops_put(struct xe_pt_job_ops *pt_job_ops);
#endif

49
drivers/gpu/drm/xe/xe_pt_types.h
View File

@ -65,12 +65,18 @@ struct xe_vm_pgtable_update {
/** @qwords: number of PTE's to write */
u32 qwords;
/** @pt: opaque pointer useful for the caller of xe_migrate_update_pgtables */
/**
* @pt: opaque pointer useful for PT building in the bind IOCTL. Only
* safe to touch during the bind IOCTL (i.e., not in bind jobs).
*/
struct xe_pt *pt;
/** @pt_entries: Newly added pagetable entries */
struct xe_pt_entry *pt_entries;
/** @level: level of update */
unsigned int level;
/** @flags: Target flags */
u32 flags;
};
@ -91,14 +97,29 @@ struct xe_vm_pgtable_update_op {
bool rebind;
};
/**
* struct xe_pt_job_ops - Page-table update operations (dynamically allocated)
*
* This is the portion of &struct xe_vma_ops and
* &struct xe_vm_pgtable_update_ops that is dynamically allocated, as it
* must remain valid until the associated bind job completes. A reference
* count controls its lifetime.
*/
struct xe_pt_job_ops {
/** @current_op: current page-table update operation */
u32 current_op;
/** @refcount: reference count */
struct kref refcount;
/** @deferred: list of deferred PT entries to destroy */
struct llist_head deferred;
/** @ops: page-table update operations */
struct xe_vm_pgtable_update_op *ops;
};
/** struct xe_vm_pgtable_update_ops: page table update operations */
struct xe_vm_pgtable_update_ops {
/** @ops: operations */
struct xe_vm_pgtable_update_op *ops;
/** @deferred: deferred list to destroy PT entries */
struct llist_head deferred;
/** @q: exec queue for PT operations */
struct xe_exec_queue *q;
/** @pt_job_ops: PT update operations dynamic allocation*/
struct xe_pt_job_ops *pt_job_ops;
/** @prl: embedded page reclaim list */
struct xe_page_reclaim_list prl;
/** @start: start address of ops */
@ -107,24 +128,10 @@ struct xe_vm_pgtable_update_ops {
u64 last;
/** @num_ops: number of operations */
u32 num_ops;
/** @current_op: current operations */
u32 current_op;
/** @needs_svm_lock: Needs SVM lock */
bool needs_svm_lock;
/** @needs_invalidation: Needs invalidation */
bool needs_invalidation;
/**
* @wait_vm_bookkeep: PT operations need to wait until VM is idle
* (bookkeep dma-resv slots are idle) and stage all future VM activity
* behind these operations (install PT operations into VM kernel
* dma-resv slot).
*/
bool wait_vm_bookkeep;
/**
* @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv
* slots are idle.
*/
bool wait_vm_kernel;
};
#endif

86
drivers/gpu/drm/xe/xe_ring_ops.c
View File

@ -143,9 +143,8 @@ static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i)
{
dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW |
MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0;
dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
(flush_flags & MI_INVALIDATE_TLB);
dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
dw[i++] = 0;
dw[i++] = val;
@ -505,6 +504,74 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
}
static int emit_ulls_preamble(struct xe_lrc *lrc, u32 *dw, int i, u32 seqno)
{
u32 addr = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno);
return emit_store_imm_ggtt(addr, LRC_MIGRATION_ULLS_SEMAPHORE_CLEAR,
dw, i);
}
/*
* Advance the ring tail from within the ring, so submitting the next ULLS job
* needs nothing from the CPU beyond signalling the semaphore. All ULLS jobs
* occupy exactly ULLS_JOB_SIZE_BYTES, so the tail the next job ends at is two
* job slots on from where this job started, even though that job has not been
* emitted yet. Both LRC and MMIO ring tail advanced in step.
*/
static int emit_ulls_ring_tail(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw,
int i, u32 head)
{
u32 next_tail = (head + 2 * ULLS_JOB_SIZE_BYTES) & (lrc->ring.size - 1);
xe_gt_assert(gt, IS_ALIGNED(next_tail, 8));
i = emit_store_imm_ggtt(xe_lrc_ring_tail_ggtt_addr(lrc), next_tail,
dw, i);
dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) |
MI_LRI_LRM_CS_MMIO;
dw[i++] = RING_TAIL(0).addr;
dw[i++] = next_tail;
return i;
}
/*
* Park the engine on the next job's semaphore, then publish its tail.
*
* The tail must not be published before the wait. The command streamer
* fetches ring contents up to the tail while parked, so it would fetch the
* next job's slot while it still holds MI_NOOPs and execute those once
* released, rather than the job the CPU writes there later. Publishing after
* the wait keeps the slot beyond the tail until the job is in place.
*/
static int emit_ulls_postamble(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw,
int i, u32 seqno, u32 head)
{
dw[i++] = MI_SEMAPHORE_WAIT |
MI_SEMW_GGTT |
MI_SEMW_POLL |
MI_SEMW_COMPARE(SAD_EQ_SDD);
dw[i++] = LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL;
dw[i++] = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno + 1);
dw[i++] = 0;
dw[i++] = 0;
return emit_ulls_ring_tail(gt, lrc, dw, i, head);
}
/* Pad out to the fixed ULLS job size */
static int emit_ulls_pad(struct xe_gt *gt, u32 *dw, int i)
{
xe_gt_assert(gt, i <= ULLS_JOB_SIZE_DW);
while (i < ULLS_JOB_SIZE_DW)
dw[i++] = MI_NOOP;
return i;
}
static void emit_migration_job_gen12(struct xe_sched_job *job,
struct xe_lrc *lrc, u32 *head,
u32 seqno)
@ -518,10 +585,16 @@ static void emit_migration_job_gen12(struct xe_sched_job *job,
xe_gt_assert(gt, !job->ring_ops_force_reset);
if (xe_sched_job_is_ulls(job))
i = emit_ulls_preamble(lrc, dw, i, seqno);
i = emit_copy_timestamp(xe, lrc, dw, i);
i = emit_store_imm_ggtt(saddr, seqno, dw, i);
if (!xe_sched_job_ulls_has_batch(job))
goto seqno_write;
dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
@ -532,12 +605,19 @@ static void emit_migration_job_gen12(struct xe_sched_job *job,
i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
seqno_write:
i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno,
job->migrate_flush_flags,
dw, i);
i = emit_user_interrupt(dw, i);
if (xe_sched_job_ulls_parks(job))
i = emit_ulls_postamble(gt, lrc, dw, i, seqno, *head);
if (xe_sched_job_is_ulls(job))
i = emit_ulls_pad(gt, dw, i);
xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));

24
drivers/gpu/drm/xe/xe_ring_ops_types.h
View File

@ -6,6 +6,7 @@
#ifndef _XE_RING_OPS_TYPES_H_
#define _XE_RING_OPS_TYPES_H_
#include <linux/build_bug.h>
#include <linux/types.h>
struct xe_gt;
@ -14,6 +15,29 @@ struct xe_sched_job;
#define MAX_JOB_SIZE_DW 74
#define MAX_JOB_SIZE_BYTES (MAX_JOB_SIZE_DW * 4)
/*
* ULLS migration jobs advance the ring tail from within the ring itself, so a
* job has to know where its successor will end before that successor has been
* emitted. Every ULLS job is therefore padded to a fixed size, letting the
* next tail be derived arithmetically.
*
* Sized for the largest such job, emitted by emit_migration_job_gen12():
* preamble (4), copy timestamp (8, its size on an SRIOV VF), start seqno
* store (4), arbitration off (1), batch buffer starts (2 * 3), pre-parser
* bracketed flush invalidate (6), seqno flush (4), user interrupt (3) and
* postamble (7 + 5).
*/
#define ULLS_JOB_SIZE_DW 48
#define ULLS_JOB_SIZE_BYTES (ULLS_JOB_SIZE_DW * 4)
/*
* RING_TAIL only encodes a qword aligned offset, and xe_lrc_write_ring()
* appends a NOP to anything shorter, either of which would desynchronise the
* ring from the tail a job predicts for its successor.
*/
static_assert(ULLS_JOB_SIZE_BYTES % 8 == 0);
static_assert(ULLS_JOB_SIZE_DW <= MAX_JOB_SIZE_DW);
/**
* struct xe_ring_ops - Ring operations
*/

101
drivers/gpu/drm/xe/xe_sched_job.c
View File

@ -26,19 +26,22 @@ static struct kmem_cache *xe_sched_job_parallel_slab;
int __init xe_sched_job_module_init(void)
{
struct xe_sched_job *job;
size_t size;
size = struct_size(job, ptrs, 1);
xe_sched_job_slab =
kmem_cache_create("xe_sched_job",
sizeof(struct xe_sched_job) +
sizeof(struct xe_job_ptrs), 0,
kmem_cache_create("xe_sched_job", size, 0,
SLAB_HWCACHE_ALIGN, NULL);
if (!xe_sched_job_slab)
return -ENOMEM;
size = max_t(size_t,
struct_size(job, ptrs,
XE_HW_ENGINE_MAX_INSTANCE),
struct_size(job, pt_update, 1));
xe_sched_job_parallel_slab =
kmem_cache_create("xe_sched_job_parallel",
sizeof(struct xe_sched_job) +
sizeof(struct xe_job_ptrs) *
XE_HW_ENGINE_MAX_INSTANCE, 0,
kmem_cache_create("xe_sched_job_parallel", size, 0,
SLAB_HWCACHE_ALIGN, NULL);
if (!xe_sched_job_parallel_slab) {
kmem_cache_destroy(xe_sched_job_slab);
@ -70,8 +73,9 @@ static void job_free(struct xe_sched_job *job)
struct xe_exec_queue *q = job->q;
bool is_migration = xe_sched_job_is_migration(q);
kmem_cache_free(xe_exec_queue_is_parallel(job->q) || is_migration ?
xe_sched_job_parallel_slab : xe_sched_job_slab, job);
kmem_cache_free(job->is_pt_job || xe_exec_queue_is_parallel(job->q) ||
is_migration ? xe_sched_job_parallel_slab :
xe_sched_job_slab, job);
}
static struct xe_device *job_to_xe(struct xe_sched_job *job)
@ -84,6 +88,9 @@ static void xe_sched_job_free_fences(struct xe_sched_job *job)
{
int i;
if (job->is_pt_job)
return;
for (i = 0; i < job->q->width; ++i) {
struct xe_job_ptrs *ptrs = &job->ptrs[i];
@ -93,10 +100,23 @@ static void xe_sched_job_free_fences(struct xe_sched_job *job)
}
}
/**
* xe_sched_job_create() - Create a scheduler job
* @q: exec queue to create the scheduler job for
* @batch_addr: array of batch addresses for the job; must match the width of
* @q, or NULL to indicate a PT job that does not require a batch address
*
* Create a scheduler job for submission.
*
* Context: Reclaim
*
* Return: a &xe_sched_job object on success, or an ERR_PTR on failure.
*/
struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
u64 *batch_addr)
{
bool is_migration = xe_sched_job_is_migration(q);
struct xe_device *xe = gt_to_xe(q->gt);
struct xe_sched_job *job;
int err;
int i;
@ -105,10 +125,15 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
/* only a kernel context can submit a vm-less job */
XE_WARN_ON(!q->vm && !(q->flags & EXEC_QUEUE_FLAG_KERNEL));
job = job_alloc(xe_exec_queue_is_parallel(q) || is_migration);
xe_assert(xe, batch_addr ||
q->flags & (EXEC_QUEUE_FLAG_VM | EXEC_QUEUE_FLAG_MIGRATE));
job = job_alloc(!batch_addr || xe_exec_queue_is_parallel(q) ||
is_migration);
if (!job)
return ERR_PTR(-ENOMEM);
job->is_pt_job = !batch_addr;
job->q = q;
job->sample_timestamp = U64_MAX;
kref_init(&job->refcount);
@ -119,34 +144,38 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
if (err)
goto err_free;
for (i = 0; i < q->width; ++i) {
struct dma_fence *fence = xe_lrc_alloc_seqno_fence();
struct dma_fence_chain *chain;
if (!batch_addr) {
job->fence = dma_fence_get_stub();
} else {
for (i = 0; i < q->width; ++i) {
struct dma_fence *fence = xe_lrc_alloc_seqno_fence();
struct dma_fence_chain *chain;
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
goto err_sched_job;
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
goto err_sched_job;
}
job->ptrs[i].lrc_fence = fence;
if (i + 1 == q->width)
continue;
chain = dma_fence_chain_alloc();
if (!chain) {
err = -ENOMEM;
goto err_sched_job;
}
job->ptrs[i].chain_fence = chain;
}
job->ptrs[i].lrc_fence = fence;
if (i + 1 == q->width)
continue;
width = q->width;
if (is_migration)
width = 2;
chain = dma_fence_chain_alloc();
if (!chain) {
err = -ENOMEM;
goto err_sched_job;
}
job->ptrs[i].chain_fence = chain;
for (i = 0; i < width; ++i)
job->ptrs[i].batch_addr = batch_addr[i];
}
width = q->width;
if (is_migration)
width = 2;
for (i = 0; i < width; ++i)
job->ptrs[i].batch_addr = batch_addr[i];
atomic_inc(&q->job_cnt);
xe_pm_runtime_get_noresume(job_to_xe(job));
trace_xe_sched_job_create(job);
@ -246,7 +275,7 @@ bool xe_sched_job_completed(struct xe_sched_job *job)
void xe_sched_job_arm(struct xe_sched_job *job)
{
struct xe_exec_queue *q = job->q;
struct dma_fence *fence, *prev;
struct dma_fence *fence = job->fence, *prev;
struct xe_vm *vm = q->vm;
u64 seqno = 0;
int i;
@ -259,6 +288,9 @@ void xe_sched_job_arm(struct xe_sched_job *job)
xe_vm_assert_held(q->vm);
}
if (job->is_pt_job)
goto arm;
if (vm && !xe_sched_job_is_migration(q) && !xe_vm_in_lr_mode(vm) &&
(vm->batch_invalidate_tlb || vm->tlb_flush_seqno != q->tlb_flush_seqno)) {
xe_vm_assert_held(vm);
@ -286,6 +318,7 @@ void xe_sched_job_arm(struct xe_sched_job *job)
fence = &chain->base;
}
arm:
job->fence = dma_fence_get(fence); /* Pairs with put in scheduler */
drm_sched_job_arm(&job->drm);
}
@ -329,7 +362,7 @@ xe_sched_job_snapshot_capture(struct xe_sched_job *job)
snapshot->batch_addr_len = q->width;
for (i = 0; i < q->width; i++)
snapshot->batch_addr[i] =
snapshot->batch_addr[i] = job->is_pt_job ? 0 :
xe_device_uncanonicalize_addr(xe, job->ptrs[i].batch_addr);
return snapshot;

56
drivers/gpu/drm/xe/xe_sched_job.h
View File

@ -83,6 +83,62 @@ xe_sched_job_add_migrate_flush(struct xe_sched_job *job, u32 flags)
job->migrate_flush_flags = flags;
}
/**
* xe_sched_job_is_ulls() - Is a ULLS job
* @job: Xe schedule job object
*
* Return: True if @job is submitted as part of a ULLS sequence, False
* otherwise.
*/
static inline bool xe_sched_job_is_ulls(struct xe_sched_job *job)
{
return job->ulls != ULLS_NONE;
}
/**
* xe_sched_job_ulls_has_batch() - Does a job carry batch buffers
* @job: Xe schedule job object
*
* The ULLS jobs which enter and exit ULLS mode exist only to move the
* migration context on and off the hardware, and carry no batch buffers.
*
* Return: True if @job carries batch buffers, False otherwise.
*/
static inline bool xe_sched_job_ulls_has_batch(struct xe_sched_job *job)
{
return job->ulls == ULLS_NONE || job->ulls == ULLS_ACTIVE;
}
/**
* xe_sched_job_ulls_parks() - Does a job park the engine for its successor
* @job: Xe schedule job object
*
* A ULLS job which is not the last one emits a postamble, parking the engine
* on its successor's semaphore and publishing that successor's ring tail.
*
* Return: True if @job emits a ULLS postamble, False otherwise.
*/
static inline bool xe_sched_job_ulls_parks(struct xe_sched_job *job)
{
return job->ulls == ULLS_ENTER || job->ulls == ULLS_ACTIVE;
}
/**
* xe_sched_job_ulls_is_chained() - Has a job's predecessor already published it
* @job: Xe schedule job object
*
* A ULLS job which is not the first one has had its ring tail published by its
* predecessor's postamble, which also left the engine parked on this job's
* semaphore. Submitting it is a semaphore write alone - no H2G and no ring
* tail write.
*
* Return: True if @job was published by its predecessor, False otherwise.
*/
static inline bool xe_sched_job_ulls_is_chained(struct xe_sched_job *job)
{
return job->ulls == ULLS_ACTIVE || job->ulls == ULLS_EXIT;
}
bool xe_sched_job_is_migration(struct xe_exec_queue *q);
struct xe_sched_job_snapshot *xe_sched_job_snapshot_capture(struct xe_sched_job *job);

49
drivers/gpu/drm/xe/xe_sched_job_types.h
View File

@ -10,10 +10,28 @@
#include <drm/gpu_scheduler.h>
struct xe_exec_queue;
struct dma_fence;
struct dma_fence_chain;
struct xe_exec_queue;
struct xe_cpu_bind_pt_update_ops;
struct xe_pt_job_ops;
struct xe_tile;
struct xe_vm;
/**
* struct xe_pt_update_args - PT update arguments
*/
struct xe_pt_update_args {
/** @vm: VM which is being bound */
struct xe_vm *vm;
/** @ops: CPU bind PT update ops */
const struct xe_cpu_bind_pt_update_ops *ops;
#define XE_PT_UPDATE_JOB_OPS_COUNT 2
/** @pt_job_ops: PT job ops state */
struct xe_pt_job_ops *pt_job_ops[XE_PT_UPDATE_JOB_OPS_COUNT];
};
/**
* struct xe_job_ptrs - Per hw engine instance data
*/
@ -31,6 +49,23 @@ struct xe_job_ptrs {
u32 head;
};
/**
* enum xe_ulls_state - ULLS state of a migration job
*
* Describes where a job sits in a ULLS (Ultra Low Latency Submission)
* sequence. See the ULLS documentation in xe_migrate.c.
*/
enum xe_ulls_state {
/** @ULLS_NONE: Not a ULLS job */
ULLS_NONE = 0,
/** @ULLS_ENTER: Job which enters ULLS mode */
ULLS_ENTER,
/** @ULLS_ACTIVE: Job submitted while in ULLS mode */
ULLS_ACTIVE,
/** @ULLS_EXIT: Job which exits ULLS mode */
ULLS_EXIT,
};
/**
* struct xe_sched_job - Xe schedule job (batch buffer tracking)
*/
@ -61,6 +96,8 @@ struct xe_sched_job {
u32 migrate_flush_flags;
/** @sample_timestamp: Sampling of job timestamp in TDR */
u64 sample_timestamp;
/** @ulls: ULLS state of this job */
enum xe_ulls_state ulls;
/** @ring_ops_flush_tlb: The ring ops need to flush TLB before payload. */
bool ring_ops_flush_tlb;
/** @ring_ops_force_reset: The ring ops need to trigger a reset before payload. */
@ -71,8 +108,14 @@ struct xe_sched_job {
bool restore_replay;
/** @last_replay: last job being replayed */
bool last_replay;
/** @ptrs: per instance pointers. */
struct xe_job_ptrs ptrs[];
/** @is_pt_job: is a PT job */
bool is_pt_job;
union {
/** @ptrs: per instance pointers. */
DECLARE_FLEX_ARRAY(struct xe_job_ptrs, ptrs);
/** @pt_update: PT update arguments */
DECLARE_FLEX_ARRAY(struct xe_pt_update_args, pt_update);
};
};
struct xe_sched_job_snapshot {

4
drivers/gpu/drm/xe/xe_sriov_pf_provision.c
View File

@ -112,7 +112,7 @@ static int pf_reprovision_default(struct xe_device *xe)
int result = 0;
int err;
guard(mutex)(xe_sriov_pf_master_mutex(xe));
lockdep_assert_held(xe_sriov_pf_master_mutex(xe));
for_each_gt(gt, xe, id) {
err = xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, false);
@ -138,6 +138,8 @@ int xe_sriov_pf_reprovision_default(struct xe_device *xe)
{
xe_assert(xe, IS_SRIOV_PF(xe));
guard(mutex)(xe_sriov_pf_master_mutex(xe));
if (!pf_auto_provisioning_mode(xe))
return 0;

68
drivers/gpu/drm/xe/xe_svm.c
View File

@ -9,6 +9,7 @@
#include <drm/drm_managed.h>
#include <drm/drm_pagemap.h>
#include <drm/drm_pagemap_util.h>
#include <drm/ttm/ttm_bo.h>
#include "xe_bo.h"
#include "xe_exec_queue_types.h"
@ -1625,6 +1626,73 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range,
return err;
}
/**
* xe_svm_devmem_lru_bump() - Move a range's backing BO to the TTM LRU tail
* @devmem_allocation: The device-memory allocation backing the range's pages
*
* Intended as a &drm_gpusvm_ctx.devmem_fn, so drm_gpusvm_get_pages() calls it
* under the GPUSVM notifier lock. That lock sits in the reclaim path, so
* dma-resv cannot be acquired blocking beneath it; dma-resv is also the outer
* lock of the notifier when memory is allocated and migrated, so a blocking
* acquire here would invert that order. Hence the trylock. The BO is private
* (dedicated dma-resv), so the trylock succeeds unless the BO is being moved
* elsewhere (e.g. eviction); a contended BO is simply skipped.
*
* A shared device_private_page_owner can hand us allocations backed by a
* foreign driver; skip anything that is not an Xe BO before casting.
*/
void xe_svm_devmem_lru_bump(struct drm_pagemap_devmem *devmem_allocation)
{
#if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP)
struct xe_bo *bo;
if (devmem_allocation->ops != &dpagemap_devmem_ops)
return;
bo = to_xe_bo(devmem_allocation);
if (!dma_resv_trylock(bo->ttm.base.resv))
return;
ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
dma_resv_unlock(bo->ttm.base.resv);
#endif
}
/**
* xe_svm_range_prefetch_lru_bump() - Bump the TTM LRU for an already-valid range
* @vm: Pointer to the struct xe_vm
* @vma: Pointer to the VMA covering the range
* @range: Pointer to the xe SVM range structure
* @dpagemap: Target pagemap of the prefetch, or %NULL for system memory
*
* A valid range skips migration and binding, so nothing else refreshes its
* backing BOs on the LRU. Re-fault the CPU pages without touching the DMA
* mappings (@no_dma_map) and move each backing BO to the LRU tail.
*
* Prefetching is a best-effort optimization, so any failure is ignored; a
* later fault will refault the range.
*/
void xe_svm_range_prefetch_lru_bump(struct xe_vm *vm, struct xe_vma *vma,
struct xe_svm_range *range,
struct drm_pagemap *dpagemap)
{
struct drm_gpusvm_ctx ctx = {
.read_only = xe_vma_read_only(vma),
.device_private_page_owner =
xe_svm_private_page_owner(vm, !dpagemap),
.no_dma_map = 1,
.devmem_fn = xe_svm_devmem_lru_bump,
};
guard(mutex)(&range->lock);
if (xe_svm_range_is_removed(range))
return;
xe_svm_range_get_pages(vm, range, &ctx);
}
/**
* xe_svm_ranges_zap_ptes_in_range - clear ptes of svm ranges in input range
* @vm: Pointer to the xe_vm structure

18
drivers/gpu/drm/xe/xe_svm.h
View File

@ -128,6 +128,12 @@ struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr,
int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range,
struct drm_gpusvm_ctx *ctx);
void xe_svm_devmem_lru_bump(struct drm_pagemap_devmem *devmem_allocation);
void xe_svm_range_prefetch_lru_bump(struct xe_vm *vm, struct xe_vma *vma,
struct xe_svm_range *range,
struct drm_pagemap *dpagemap);
bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma,
const struct drm_pagemap *dpagemap);
@ -356,6 +362,18 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range,
return -EINVAL;
}
static inline
void xe_svm_devmem_lru_bump(struct drm_pagemap_devmem *devmem_allocation)
{
}
static inline
void xe_svm_range_prefetch_lru_bump(struct xe_vm *vm, struct xe_vma *vma,
struct xe_svm_range *range,
struct drm_pagemap *dpagemap)
{
}
static inline struct xe_svm_range *to_xe_range(struct drm_gpusvm_range *r)
{
return NULL;

20
drivers/gpu/drm/xe/xe_sync.c
View File

@ -345,15 +345,9 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
return ERR_PTR(-EOPNOTSUPP);
if (q->flags & EXEC_QUEUE_FLAG_VM) {
struct xe_exec_queue *__q;
struct xe_tile *tile;
u8 id;
for_each_tile(tile, vm->xe, id) {
num_fence++;
for_each_tlb_inval(q, i)
num_fence++;
for_each_tlb_inval(i)
num_fence++;
}
fences = kmalloc_objs(*fences, num_fence);
if (!fences)
@ -361,17 +355,9 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
fences[current_fence++] =
xe_exec_queue_last_fence_get(q, vm);
for_each_tlb_inval(i)
for_each_tlb_inval(q, i)
fences[current_fence++] =
xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
list_for_each_entry(__q, &q->multi_gt_list,
multi_gt_link) {
fences[current_fence++] =
xe_exec_queue_last_fence_get(__q, vm);
for_each_tlb_inval(i)
fences[current_fence++] =
xe_exec_queue_tlb_inval_last_fence_get(__q, vm, i);
}
xe_assert(vm->xe, current_fence == num_fence);
cf = dma_fence_array_create(num_fence, fences,

7
drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
View File

@ -334,6 +334,9 @@ void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 gr
* If a response is expected, @cmd->data_out must point to a buffer of
* size @cmd->data_out_len supplied by caller.
*
* @cmd->timeout_ms specifies the response timeout in milliseconds.
* A value of 0 uses %XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS.
*
* On success, @rdata_len is updated with number of valid response bytes
* returned by firmware, bounded by @cmd->data_out_len.
*
@ -347,6 +350,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc,
u8 group_id, command_code;
u8 *mbox_cmd = NULL;
size_t cmd_size = 0;
unsigned int timeout_ms;
int ret;
guard(xe_pm_runtime_noresume)(xe);
@ -360,6 +364,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc,
group_id = XE_SYSCTRL_APP_HDR_GROUP_ID(&cmd->header);
command_code = XE_SYSCTRL_APP_HDR_COMMAND(&cmd->header);
timeout_ms = cmd->timeout_ms ?: XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS;
might_sleep();
@ -375,7 +380,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc,
ret = sysctrl_send_command(sc, mbox_cmd, cmd_size,
cmd->data_out, cmd->data_out_len, rdata_len,
XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS);
timeout_ms);
if (ret)
xe_err(xe, "sysctrl: Mailbox command failed: %pe\n", ERR_PTR(ret));

3
drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
View File

@ -105,6 +105,9 @@ struct xe_sysctrl_mailbox_command {
/** @data_out_len: Size of output buffer in bytes (0 if no response expected) */
size_t data_out_len;
/** @timeout_ms: Response timeout in ms, or 0 for %XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS */
unsigned int timeout_ms;
};
/* Modify as needed */

54
drivers/gpu/drm/xe/xe_tlb_inval.c
View File

@ -5,6 +5,7 @@
#include <drm/drm_managed.h>
#include "xe_devcoredump.h"
#include "xe_device_types.h"
#include "xe_force_wake.h"
#include "xe_gt_stats.h"
@ -13,6 +14,7 @@
#include "xe_guc_tlb_inval.h"
#include "xe_mmio.h"
#include "xe_pm.h"
#include "xe_printk.h"
#include "xe_tlb_inval.h"
#include "xe_trace.h"
@ -29,6 +31,12 @@
#define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS
/* The frontend is only ever embedded in a GT */
static struct xe_gt *tlb_inval_to_gt(struct xe_tlb_inval *tlb_inval)
{
return container_of(tlb_inval, struct xe_gt, tlb_inval);
}
static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence)
{
if (WARN_ON_ONCE(!fence->tlb_inval))
@ -73,6 +81,7 @@ static void xe_tlb_inval_fence_timeout(struct work_struct *work)
struct xe_device *xe = tlb_inval->xe;
struct xe_tlb_inval_fence *fence, *next;
long timeout_delay = tlb_inval->ops->timeout_delay(tlb_inval);
int timedout_seqno = 0, seqno_recv = 0;
tlb_inval->ops->flush(tlb_inval);
@ -90,13 +99,45 @@ static void xe_tlb_inval_fence_timeout(struct work_struct *work)
"TLB invalidation fence timeout, seqno=%d recv=%d",
fence->seqno, tlb_inval->seqno_recv);
if (!timedout_seqno) {
/*
* Hold a PM reference across the capture below. Every
* pending fence holds one, so the device is awake
* here, but signalling them may drop the last
* reference and let it autosuspend before the
* snapshot touches the hardware.
*/
xe_pm_runtime_get_noresume(xe);
}
timedout_seqno = fence->seqno;
if (!tlb_inval->timedout_seqno) {
tlb_inval->timedout_seqno = fence->seqno;
tlb_inval->timedout_inval_time = fence->inval_time;
tlb_inval->timedout_time = ktime_get();
}
fence->base.error = -ETIME;
xe_tlb_inval_fence_signal(fence);
}
if (!list_empty(&tlb_inval->pending_fences))
queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr,
timeout_delay);
seqno_recv = tlb_inval->seqno_recv;
spin_unlock_irq(&tlb_inval->pending_lock);
/*
* Capture the GuC log and CT state so the firmware side of the hang
* can be inspected; there is no queue or job to blame here. Must be
* outside pending_lock as the capture takes sleeping locks, hence
* @seqno_recv is sampled above while the lock is still held.
*/
if (timedout_seqno) {
xe_devcoredump_gt(tlb_inval_to_gt(tlb_inval),
"TLB invalidation fence timeout, seqno=%d recv=%d",
timedout_seqno, seqno_recv);
xe_pm_runtime_put(xe);
}
}
/**
@ -207,6 +248,7 @@ void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval)
else
pending_seqno = tlb_inval->seqno - 1;
WRITE_ONCE(tlb_inval->seqno_recv, pending_seqno);
tlb_inval->timedout_seqno = 0;
list_for_each_entry_safe(fence, next,
&tlb_inval->pending_fences, link)
@ -435,6 +477,18 @@ void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno)
WRITE_ONCE(tlb_inval->seqno_recv, seqno);
if (tlb_inval->timedout_seqno &&
xe_tlb_inval_seqno_past(tlb_inval, tlb_inval->timedout_seqno)) {
ktime_t now = ktime_get();
xe_warn(xe,
"TLB invalidation late ack: seqno=%d recv=%d, request-to-ack=%lldms, timeout-to-ack=%lldms",
tlb_inval->timedout_seqno, seqno,
ktime_ms_delta(now, tlb_inval->timedout_inval_time),
ktime_ms_delta(now, tlb_inval->timedout_time));
tlb_inval->timedout_seqno = 0;
}
list_for_each_entry_safe(fence, next,
&tlb_inval->pending_fences, link) {
trace_xe_tlb_inval_fence_recv(xe, fence);

28
drivers/gpu/drm/xe/xe_tlb_inval_job.c
View File

@ -4,6 +4,7 @@
*/
#include "xe_assert.h"
#include "xe_cpu_bind.h"
#include "xe_dep_job_types.h"
#include "xe_dep_scheduler.h"
#include "xe_exec_queue.h"
@ -12,7 +13,6 @@
#include "xe_page_reclaim.h"
#include "xe_tlb_inval.h"
#include "xe_tlb_inval_job.h"
#include "xe_migrate.h"
#include "xe_pm.h"
#include "xe_vm.h"
@ -39,8 +39,8 @@ struct xe_tlb_inval_job {
u64 start;
/** @end: End address to invalidate */
u64 end;
/** @type: GT type */
int type;
/** @idx: Index of tlb invalidation */
int idx;
/** @fence_armed: Fence has been armed */
bool fence_armed;
};
@ -87,7 +87,7 @@ static const struct xe_dep_job_ops dep_job_ops = {
* @vm: VM which TLB invalidation is being issued for
* @start: Start address to invalidate
* @end: End address to invalidate
* @type: GT type
* @idx: Index of tlb invalidation
*
* Create a TLB invalidation job and initialize internal fields. The caller is
* responsible for releasing the creation reference.
@ -97,7 +97,7 @@ static const struct xe_dep_job_ops dep_job_ops = {
struct xe_tlb_inval_job *
xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
struct xe_dep_scheduler *dep_scheduler,
struct xe_vm *vm, u64 start, u64 end, int type)
struct xe_vm *vm, u64 start, u64 end, int idx)
{
struct xe_tlb_inval_job *job;
struct drm_sched_entity *entity =
@ -105,8 +105,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
struct xe_tlb_inval_fence *ifence;
int err;
xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
job = kmalloc_obj(*job);
if (!job)
@ -120,7 +119,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
job->fence_armed = false;
xe_page_reclaim_list_init(&job->prl);
job->dep.ops = &dep_job_ops;
job->type = type;
job->idx = idx;
kref_init(&job->refcount);
xe_exec_queue_get(q); /* Pairs with put in xe_tlb_inval_job_destroy */
xe_vm_get(vm); /* Pairs with put in xe_tlb_inval_job_destroy */
@ -219,7 +218,6 @@ int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job)
/**
* xe_tlb_inval_job_push() - TLB invalidation job push
* @job: TLB invalidation job to push
* @m: The migration object being used
* @fence: Dependency for TLB invalidation job
*
* Pushes a TLB invalidation job for execution, using @fence as a dependency.
@ -231,11 +229,11 @@ int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job)
* Return: Job's finished fence on success, cannot fail
*/
struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
struct xe_migrate *m,
struct dma_fence *fence)
{
struct xe_tlb_inval_fence *ifence =
container_of(job->fence, typeof(*ifence), base);
struct xe_cpu_bind *cpu_bind = gt_to_xe(job->q->gt)->cpu_bind;
if (!dma_fence_is_signaled(fence)) {
void *ptr;
@ -259,11 +257,11 @@ struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
job->fence_armed = true;
/*
* We need the migration lock to protect the job's seqno and the spsc
* queue, only taken on migration queue, user queues protected dma-resv
* We need the cpu_bind lock to protect the job's seqno and the spsc
* queue, only taken on cpu_bind queue, user queues protected dma-resv
* VM lock.
*/
xe_migrate_job_lock(m, job->q);
xe_cpu_bind_job_lock(cpu_bind, job->q);
/* Creation ref pairs with put in xe_tlb_inval_job_destroy */
xe_tlb_inval_fence_init(job->tlb_inval, ifence, false);
@ -280,9 +278,9 @@ struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
/* Let the upper layers fish this out */
xe_exec_queue_tlb_inval_last_fence_set(job->q, job->vm,
&job->dep.drm.s_fence->finished,
job->type);
job->idx);
xe_migrate_job_unlock(m, job->q);
xe_cpu_bind_job_unlock(cpu_bind, job->q);
/*
* Not using job->fence, as it has its own dma-fence context, which does

4
drivers/gpu/drm/xe/xe_tlb_inval_job.h
View File

@ -11,7 +11,6 @@
struct dma_fence;
struct xe_dep_scheduler;
struct xe_exec_queue;
struct xe_migrate;
struct xe_page_reclaim_list;
struct xe_tlb_inval;
struct xe_tlb_inval_job;
@ -20,7 +19,7 @@ struct xe_vm;
struct xe_tlb_inval_job *
xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
struct xe_dep_scheduler *dep_scheduler,
struct xe_vm *vm, u64 start, u64 end, int type);
struct xe_vm *vm, u64 start, u64 end, int idx);
void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
struct xe_page_reclaim_list *prl);
@ -28,7 +27,6 @@ void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job);
struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
struct xe_migrate *m,
struct dma_fence *fence);
void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job);

17
drivers/gpu/drm/xe/xe_tlb_inval_types.h
View File

@ -112,6 +112,23 @@ struct xe_tlb_inval {
* @pending_lock: protects @pending_fences and updating @seqno_recv.
*/
spinlock_t pending_lock;
/**
* @timedout_seqno: seqno of the most recent timed out TLB
* invalidation, 0 if none. Used to measure how late the ack for a
* timed out invalidation actually arrives. Protected by
* @pending_lock.
*/
int timedout_seqno;
/**
* @timedout_inval_time: request time of @timedout_seqno. Protected by
* @pending_lock.
*/
ktime_t timedout_inval_time;
/**
* @timedout_time: time @timedout_seqno was signaled with -ETIME.
* Protected by @pending_lock.
*/
ktime_t timedout_time;
/**
* @fence_tdr: schedules a delayed call to xe_tlb_fence_timeout after
* the timeout interval is over.

2
drivers/gpu/drm/xe/xe_trace.h
View File

@ -261,7 +261,7 @@ DECLARE_EVENT_CLASS(xe_sched_job,
__entry->flags = job->q->flags;
__entry->error = job->fence ? job->fence->error : 0;
__entry->fence = job->fence;
__entry->batch_addr = (u64)job->ptrs[0].batch_addr;
__entry->batch_addr = job->is_pt_job ? 0 : (u64)job->ptrs[0].batch_addr;
),
TP_printk("dev=%s, fence=%p, seqno=%u, lrc_seqno=%u, gt=%u, guc_id=%d, batch_addr=0x%012llx, guc_state=0x%x, flags=0x%x, error=%d",

58
drivers/gpu/drm/xe/xe_trace_bo.h
View File

@ -16,10 +16,15 @@
#include "xe_bo_types.h"
#include "xe_vm.h"
#define __dev_name_xe(xe) dev_name((xe)->drm.dev)
#define __dev_name_bo(bo) dev_name(xe_bo_device(bo)->drm.dev)
#define __dev_name_vm(vm) dev_name((vm)->xe->drm.dev)
#define __dev_name_vma(vma) __dev_name_vm(xe_vma_vm(vma))
#define XE_VM_TRACE_MODE_DMA_FENCE 0
#define XE_VM_TRACE_MODE_PREEMPT_FENCE 1
#define XE_VM_TRACE_MODE_FAULT 2
DECLARE_EVENT_CLASS(xe_bo,
TP_PROTO(struct xe_bo *bo),
TP_ARGS(bo),
@ -194,6 +199,7 @@ DECLARE_EVENT_CLASS(xe_vm,
__field(struct xe_vm *, vm)
__field(u32, asid)
__field(u32, flags)
__field(u8, mode)
),
TP_fast_assign(
@ -201,11 +207,20 @@ DECLARE_EVENT_CLASS(xe_vm,
__entry->vm = vm;
__entry->asid = vm->usm.asid;
__entry->flags = vm->flags;
__entry->mode = xe_vm_in_fault_mode(vm) ?
XE_VM_TRACE_MODE_FAULT :
xe_vm_in_preempt_fence_mode(vm) ?
XE_VM_TRACE_MODE_PREEMPT_FENCE :
XE_VM_TRACE_MODE_DMA_FENCE;
),
TP_printk("dev=%s, vm=%p, asid=0x%05x, vm flags=0x%05x",
TP_printk("dev=%s, vm=%p, asid=0x%05x, vm flags=0x%05x, mode=%s",
__get_str(dev), __entry->vm, __entry->asid,
__entry->flags)
__entry->flags,
__print_symbolic(__entry->mode,
{ XE_VM_TRACE_MODE_DMA_FENCE, "dma-fence" },
{ XE_VM_TRACE_MODE_PREEMPT_FENCE, "preempt-fence" },
{ XE_VM_TRACE_MODE_FAULT, "fault" }))
);
DEFINE_EVENT(xe_vm, xe_vm_kill,
@ -223,6 +238,16 @@ DEFINE_EVENT(xe_vm, xe_vm_free,
TP_ARGS(vm)
);
DEFINE_EVENT(xe_vm, xe_vm_close_and_put,
TP_PROTO(struct xe_vm *vm),
TP_ARGS(vm)
);
DEFINE_EVENT(xe_vm, xe_vm_asid_release,
TP_PROTO(struct xe_vm *vm),
TP_ARGS(vm)
);
DEFINE_EVENT(xe_vm, xe_vm_cpu_bind,
TP_PROTO(struct xe_vm *vm),
TP_ARGS(vm)
@ -253,6 +278,35 @@ DEFINE_EVENT(xe_vm, xe_vm_ops_fail,
TP_ARGS(vm)
);
TRACE_EVENT(xe_pagefault_fail,
TP_PROTO(struct xe_device *xe, struct xe_vm *vm, struct xe_vma *vma,
u32 asid, u64 page_addr, u8 reason, int err),
TP_ARGS(xe, vm, vma, asid, page_addr, reason, err),
TP_STRUCT__entry(__string(dev, __dev_name_xe(xe))
__field(struct xe_vm *, vm)
__field(struct xe_vma *, vma)
__field(u32, asid)
__field(u64, page_addr)
__field(u8, reason)
__field(int, err)
),
TP_fast_assign(__assign_str(dev);
__entry->vm = vm;
__entry->vma = vma;
__entry->asid = asid;
__entry->page_addr = page_addr;
__entry->reason = reason;
__entry->err = err;
),
TP_printk("dev=%s, vm=%p, vma=%p, asid=0x%05x, page_addr=0x%012llx, reason=%u, err=%d",
__get_str(dev), __entry->vm, __entry->vma,
__entry->asid, __entry->page_addr, __entry->reason,
__entry->err)
);
#endif
/* This part must be outside protection */

4
drivers/gpu/drm/xe/xe_tuning.c
View File

@ -100,6 +100,10 @@ VISIBLE_IF_KUNIT const struct xe_rtp_table_sr gt_tunings = XE_RTP_TABLE_SR(
XE_RTP_ACTIONS(FIELD_SET(GAMSTLB_CTRL, BANK_HASH_MODE,
BANK_HASH_4KB_MODE))
},
{ XE_RTP_NAME("Tuning: Force All Non-Compressible Memory Reads to be 256B Overfetches"),
XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED)),
XE_RTP_ACTIONS(SET(L3SQCREG2, NONCOMPMEMRD256BOVRFETCHEN))
},
);
EXPORT_SYMBOL_IF_KUNIT(gt_tunings);

1
drivers/gpu/drm/xe/xe_uc_fw.c
View File

@ -141,6 +141,7 @@ struct fw_blobs_by_type {
/* for the GSC FW we match the compatibility version and not the release one */
#define XE_GSC_FIRMWARE_DEFS(fw_def, major_ver) \
fw_def(NOVALAKE_S, GT_TYPE_ANY, major_ver(xe, gsc, nvl, 108, 0, 0)) \
fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, gsc, ptl, 105, 1, 0)) \
fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, gsc, lnl, 104, 1, 0)) \
fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, gsc, mtl, 102, 1, 0))

275
drivers/gpu/drm/xe/xe_vm.c
View File

@ -24,6 +24,7 @@
#include "regs/xe_gtt_defs.h"
#include "xe_assert.h"
#include "xe_bo.h"
#include "xe_cpu_bind.h"
#include "xe_device.h"
#include "xe_drm_client.h"
#include "xe_exec_queue.h"
@ -148,7 +149,7 @@ static bool xe_vm_is_idle(struct xe_vm *vm)
xe_vm_assert_held(vm);
list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
if (!xe_exec_queue_is_idle(q))
if (!xe_exec_queue_is_idle(q, 0))
return false;
}
@ -682,11 +683,9 @@ static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds)
if (!vops->pt_update_ops[i].num_ops)
continue;
vops->pt_update_ops[i].ops =
kmalloc_objs(*vops->pt_update_ops[i].ops,
vops->pt_update_ops[i].num_ops,
GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
if (!vops->pt_update_ops[i].ops)
vops->pt_update_ops[i].pt_job_ops =
xe_pt_job_ops_alloc(vops->pt_update_ops[i].num_ops);
if (!vops->pt_update_ops[i].pt_job_ops)
return array_of_binds ? -ENOBUFS : -ENOMEM;
}
@ -733,7 +732,7 @@ static void xe_vma_ops_fini(struct xe_vma_ops *vops)
xe_vma_svm_prefetch_ops_fini(vops);
for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
kfree(vops->pt_update_ops[i].ops);
xe_pt_job_ops_put(vops->pt_update_ops[i].pt_job_ops);
}
static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask, int inc_val)
@ -798,16 +797,16 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
struct xe_vma *vma, *next;
struct xe_vma_ops vops;
struct xe_vma_op *op, *next_op;
int err, i;
int err;
lockdep_assert_held(&vm->lock);
if ((xe_vm_in_lr_mode(vm) && !rebind_worker) ||
list_empty(&vm->rebind_list))
return 0;
xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
vops.pt_update_ops[i].wait_vm_bookkeep = true;
xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind),
NULL, 0);
vops.flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
xe_vm_assert_held(vm);
list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
@ -847,28 +846,32 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
return err;
}
static u8 adjust_rebind_tile_mask(struct xe_vm *vm, u8 tile_mask)
{
if (vm->xe->info.has_pt_mirror)
return tile_mask;
return (0x1 << vm->xe->info.tile_count) - 1;
}
struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_mask)
{
struct dma_fence *fence = NULL;
struct xe_vma_ops vops;
struct xe_vma_op *op, *next_op;
struct xe_tile *tile;
u8 id;
int err;
lockdep_assert_held(&vm->lock);
xe_vm_assert_held(vm);
xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT;
for_each_tile(tile, vm->xe, id) {
vops.pt_update_ops[id].wait_vm_bookkeep = true;
vops.pt_update_ops[tile->id].q =
xe_migrate_exec_queue(tile->migrate);
}
xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind),
NULL, 0);
vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT |
XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
err = xe_vm_ops_add_rebind(&vops, vma, tile_mask);
err = xe_vm_ops_add_rebind(&vops, vma,
adjust_rebind_tile_mask(vm, tile_mask));
if (err)
return ERR_PTR(err);
@ -942,8 +945,6 @@ struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm,
struct dma_fence *fence = NULL;
struct xe_vma_ops vops;
struct xe_vma_op *op, *next_op;
struct xe_tile *tile;
u8 id;
int err;
lockdep_assert_held(&range->lock);
@ -952,15 +953,13 @@ struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm,
xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma));
xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT;
for_each_tile(tile, vm->xe, id) {
vops.pt_update_ops[id].wait_vm_bookkeep = true;
vops.pt_update_ops[tile->id].q =
xe_migrate_exec_queue(tile->migrate);
}
xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind),
NULL, 0);
vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT |
XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
err = xe_vm_ops_add_range_rebind(&vops, vma, range, tile_mask);
err = xe_vm_ops_add_range_rebind(&vops, vma, range,
adjust_rebind_tile_mask(vm, tile_mask));
if (err)
return ERR_PTR(err);
@ -1025,8 +1024,6 @@ struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm,
struct dma_fence *fence = NULL;
struct xe_vma_ops vops;
struct xe_vma_op *op, *next_op;
struct xe_tile *tile;
u8 id;
int err;
lockdep_assert_held(&range->lock);
@ -1037,12 +1034,9 @@ struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm,
if (!range->tile_present)
return dma_fence_get_stub();
xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
for_each_tile(tile, vm->xe, id) {
vops.pt_update_ops[id].wait_vm_bookkeep = true;
vops.pt_update_ops[tile->id].q =
xe_migrate_exec_queue(tile->migrate);
}
xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind),
NULL, 0);
vops.flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
err = xe_vm_ops_add_range_unbind(&vops, range);
if (err)
@ -1713,9 +1707,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
init_rwsem(&vm->exec_queues.lock);
xe_vm_init_prove_locking(xe, vm);
for_each_tile(tile, xe, id)
xe_range_fence_tree_init(&vm->rftree[id]);
xe_range_fence_tree_init(&vm->rftree);
vm->pt_ops = &xelp_pt_ops;
@ -1757,7 +1749,8 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
for_each_tile(tile, xe, id) {
if (flags & XE_VM_FLAG_MIGRATION &&
tile->id != XE_VM_FLAG_TILE_ID(flags))
tile->id != XE_VM_FLAG_TILE_ID(flags) &&
(vm->xe->info.has_pt_mirror || id))
continue;
vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level,
@ -1813,7 +1806,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
struct xe_exec_queue *q;
u32 create_flags = EXEC_QUEUE_FLAG_VM;
if (!vm->pt_root[id])
if (!vm->pt_root[id] || vm->q)
continue;
if (!xef) /* Not from userspace */
@ -1824,7 +1817,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
err = PTR_ERR(q);
goto err_close;
}
vm->q[id] = q;
vm->q = q;
}
}
@ -1857,8 +1850,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
xe_svm_fini(vm);
err_no_resv:
mutex_destroy(&vm->snap_mutex);
for_each_tile(tile, xe, id)
xe_range_fence_tree_fini(&vm->rftree[id]);
xe_range_fence_tree_fini(&vm->rftree);
ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move);
if (vm->xef)
xe_file_put(vm->xef);
@ -1918,13 +1910,13 @@ void xe_vm_close_and_put(struct xe_vm *vm)
{
LIST_HEAD(contested);
struct xe_device *xe = vm->xe;
struct xe_tile *tile;
struct xe_vma *vma, *next_vma;
struct drm_gpuva *gpuva, *next;
u8 id;
xe_assert(xe, !vm->preempt.num_exec_queues);
trace_xe_vm_close_and_put(vm);
xe_vm_close(vm);
if (xe_vm_in_preempt_fence_mode(vm)) {
mutex_lock(&xe->rebind_resume_lock);
@ -1935,24 +1927,18 @@ void xe_vm_close_and_put(struct xe_vm *vm)
if (xe_vm_in_fault_mode(vm))
xe_svm_close(vm);
down_write(&vm->lock);
for_each_tile(tile, xe, id) {
if (vm->q[id]) {
int i;
if (vm->q) {
int i;
xe_exec_queue_last_fence_put(vm->q[id], vm);
for_each_tlb_inval(i)
xe_exec_queue_tlb_inval_last_fence_put(vm->q[id], vm, i);
}
}
up_write(&vm->lock);
down_write(&vm->lock);
xe_exec_queue_last_fence_put(vm->q, vm);
for_each_tlb_inval(vm->q, i)
xe_exec_queue_tlb_inval_last_fence_put(vm->q, vm, i);
up_write(&vm->lock);
for_each_tile(tile, xe, id) {
if (vm->q[id]) {
xe_exec_queue_kill(vm->q[id]);
xe_exec_queue_put(vm->q[id]);
vm->q[id] = NULL;
}
xe_exec_queue_kill(vm->q);
xe_exec_queue_put(vm->q);
vm->q = NULL;
}
down_write(&vm->lock);
@ -1979,21 +1965,13 @@ void xe_vm_close_and_put(struct xe_vm *vm)
vma->gpuva.flags |= XE_VMA_DESTROYED;
}
/*
* All vm operations will add shared fences to resv.
* The only exception is eviction for a shared object,
* but even so, the unbind when evicted would still
* install a fence to resv. Hence it's safe to
* destroy the pagetables immediately.
*/
xe_vm_free_scratch(vm);
xe_vm_pt_destroy(vm);
xe_vm_unlock(vm);
/*
* VM is now dead, cannot re-add nodes to vm->vmas if it's NULL
* Since we hold a refcount to the bo, we can remove and free
* the members safely without locking.
* Unlink and destroy all contested external-BO VMAs before destroying
* the page tables. Otherwise, concurrent eviction holding only bo->resv
* can walk the BO's VMAs and attempt to invalidate/zap page tables that
* have already been freed.
*/
list_for_each_entry_safe(vma, next_vma, &contested,
combined_links.destroy) {
@ -2001,6 +1979,11 @@ void xe_vm_close_and_put(struct xe_vm *vm)
xe_vma_destroy_unlocked(vma);
}
xe_vm_lock(vm, false);
xe_vm_free_scratch(vm);
xe_vm_pt_destroy(vm);
xe_vm_unlock(vm);
xe_svm_fini(vm);
up_write(&vm->lock);
@ -2012,6 +1995,8 @@ void xe_vm_close_and_put(struct xe_vm *vm)
xe_assert(xe, xe->info.has_asid);
xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION));
trace_xe_vm_asid_release(vm);
lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid);
xe_assert(xe, lookup == vm);
}
@ -2019,8 +2004,7 @@ void xe_vm_close_and_put(struct xe_vm *vm)
xe_vm_clear_fault_entries(vm);
for_each_tile(tile, xe, id)
xe_range_fence_tree_fini(&vm->rftree[id]);
xe_range_fence_tree_fini(&vm->rftree);
xe_vm_put(vm);
}
@ -2080,13 +2064,13 @@ struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id)
u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile)
{
return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0);
return vm->pt_ops->pde_encode_bo(xe_vm_pt_root(vm, tile->id)->bo, 0);
}
static struct xe_exec_queue *
to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
{
return q ? q : vm->q[0];
return q ? q : vm->q;
}
static struct xe_user_fence *
@ -2548,8 +2532,10 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
ctx.devmem_possible = IS_DGFX(vm->xe) &&
IS_ENABLED(CONFIG_DRM_XE_PAGEMAP);
for_each_tile(tile, vm->xe, id)
for_each_tile(tile, vm->xe, id) {
xe_migrate_ulls_enter(tile->migrate);
tile_mask |= 0x1 << id;
}
if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) {
dpagemap = xe_vma_resolve_pagemap(vma,
@ -2584,6 +2570,11 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
dpagemap, &valid_pages)) {
xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID");
xe_assert(vm->xe, valid_pages);
if (dpagemap)
xe_svm_range_prefetch_lru_bump(vm, vma, svm_range,
dpagemap);
need_put = true;
goto check_next_range;
}
@ -3260,6 +3251,7 @@ static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops *vops,
ctx.devmem_possible = devmem_possible;
ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0;
ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap);
ctx.devmem_fn = xe_svm_devmem_lru_bump;
skip_threads = op->prefetch_range.ranges_count == 1 ||
(!dpagemap && !(vops->flags &
@ -3515,28 +3507,18 @@ static void trace_xe_vm_ops_execute(struct xe_vma_ops *vops)
op_trace(op);
}
static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops)
static int vm_ops_setup(struct xe_vm *vm, struct xe_vma_ops *vops)
{
struct xe_exec_queue *q = vops->q;
struct xe_tile *tile;
int number_tiles = 0;
u8 id;
for_each_tile(tile, vm->xe, id) {
for_each_tile(tile, vm->xe, id)
if (vops->pt_update_ops[id].num_ops)
++number_tiles;
if (vops->pt_update_ops[id].q)
continue;
if (q) {
vops->pt_update_ops[id].q = q;
if (vm->pt_root[id] && !list_empty(&q->multi_gt_list))
q = list_next_entry(q, multi_gt_list);
} else {
vops->pt_update_ops[id].q = vm->q[id];
}
}
if (!vops->q)
vops->q = vm->q;
return number_tiles;
}
@ -3544,26 +3526,21 @@ static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops)
static struct dma_fence *ops_execute(struct xe_vm *vm,
struct xe_vma_ops *vops)
{
struct xe_tile *tile;
struct xe_device *xe = vm->xe;
struct dma_fence *fence = NULL;
struct dma_fence **fences = NULL;
struct dma_fence_array *cf = NULL;
int number_tiles = 0, current_fence = 0, n_fence = 0, err, i;
u8 id;
int current_fence = 0, n_fence = 1, err, i;
number_tiles = vm_ops_setup_tile_args(vm, vops);
if (number_tiles == 0)
if (!vm_ops_setup(vm, vops))
return ERR_PTR(-ENODATA);
for_each_tile(tile, vm->xe, id) {
++n_fence;
if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT))
for_each_tlb_inval(i)
++n_fence;
if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) {
for_each_tlb_inval(vops->q, i)
++n_fence;
}
fences = kmalloc_objs(*fences, n_fence);
fences = kcalloc(n_fence, sizeof(*fences), GFP_KERNEL);
if (!fences) {
fence = ERR_PTR(-ENOMEM);
goto err_trace;
@ -3575,65 +3552,39 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
goto err_out;
}
for_each_tile(tile, vm->xe, id) {
if (!vops->pt_update_ops[id].num_ops)
continue;
err = xe_pt_update_ops_prepare(tile, vops);
if (err) {
fence = ERR_PTR(err);
goto err_out;
}
err = xe_pt_update_ops_prepare(xe, vops);
if (err) {
fence = ERR_PTR(err);
goto err_out;
}
trace_xe_vm_ops_execute(vops);
for_each_tile(tile, vm->xe, id) {
struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q;
fence = xe_pt_update_ops_run(xe, vops);
if (IS_ERR(fence))
goto err_out;
fences[current_fence++] = fence;
fence = NULL;
if (!vops->pt_update_ops[id].num_ops)
goto collect_fences;
fence = xe_pt_update_ops_run(tile, vops);
if (IS_ERR(fence))
goto err_out;
collect_fences:
fences[current_fence++] = fence ?: dma_fence_get_stub();
if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)
continue;
xe_migrate_job_lock(tile->migrate, q);
for_each_tlb_inval(i)
if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) {
xe_cpu_bind_job_lock(xe->cpu_bind, vops->q);
for_each_tlb_inval(vops->q, i)
fences[current_fence++] =
xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
xe_migrate_job_unlock(tile->migrate, q);
xe_exec_queue_tlb_inval_last_fence_get(vops->q,
vm, i);
xe_cpu_bind_job_unlock(xe->cpu_bind, vops->q);
}
xe_assert(vm->xe, current_fence == n_fence);
xe_assert(xe, current_fence == n_fence);
dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1),
1);
fence = &cf->base;
for_each_tile(tile, vm->xe, id) {
if (!vops->pt_update_ops[id].num_ops)
continue;
xe_pt_update_ops_fini(tile, vops);
}
xe_pt_update_ops_fini(xe, vops);
return fence;
err_out:
for_each_tile(tile, vm->xe, id) {
if (!vops->pt_update_ops[id].num_ops)
continue;
xe_pt_update_ops_abort(tile, vops);
}
while (current_fence)
dma_fence_put(fences[--current_fence]);
xe_pt_update_ops_abort(xe, vops);
kfree(fences);
kfree(cf);
@ -3943,6 +3894,8 @@ static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
vops->syncs = syncs;
vops->num_syncs = num_syncs;
vops->flags = 0;
vops->start = ~0x0ull;
vops->last = 0x0ull;
}
static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo,
@ -4137,7 +4090,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
syncs_user = u64_to_user_ptr(args->syncs);
for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
struct xe_exec_queue *__q = q ?: vm->q[0];
struct xe_exec_queue *__q = q ?: vm->q;
err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
&syncs_user[num_syncs],
@ -5142,3 +5095,21 @@ void xe_vm_remove_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
}
up_write(&vm->exec_queues.lock);
}
/**
* xe_vm_pt_root() - Retrieve VM page-table root
* @vm: The VM.
* @tile_id: Tile ID
*
* Retrieve VM page-table root for a tile ID, used to abstract if PT mirroring is
* enabled across tiles.
*
* Return: VM page-table root for a tile ID
*/
struct xe_pt *xe_vm_pt_root(struct xe_vm *vm, u8 tile_id)
{
if (vm->xe->info.has_pt_mirror)
return vm->pt_root[tile_id];
return vm->pt_root[0];
}

3
drivers/gpu/drm/xe/xe_vm.h
View File

@ -436,4 +436,7 @@ static inline struct drm_exec *xe_vm_validation_exec(struct xe_vm *vm)
((READ_ONCE(tile_present) & ~READ_ONCE(tile_invalidated)) & BIT((tile)->id))
void xe_vma_mem_attr_copy(struct xe_vma_mem_attr *to, struct xe_vma_mem_attr *from);
struct xe_pt *xe_vm_pt_root(struct xe_vm *vm, u8 tile_id);
#endif

6
drivers/gpu/drm/xe/xe_vm_madvise.c
View File

@ -713,12 +713,6 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil
attr_type = array_index_nospec(args->type, ARRAY_SIZE(madvise_funcs));
/* Ensure the madvise function exists for this type */
if (!madvise_funcs[attr_type]) {
err = -EINVAL;
goto err_fini;
}
madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args,
&details);

12
drivers/gpu/drm/xe/xe_vm_types.h
View File

@ -262,7 +262,7 @@ struct xe_vm {
struct xe_device *xe;
/* exec queue used for (un)binding vma's */
struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE];
struct xe_exec_queue *q;
/** @lru_bulk_move: Bulk LRU move list for this VM's BOs */
struct ttm_lru_bulk_move lru_bulk_move;
@ -317,7 +317,7 @@ struct xe_vm {
* @rftree: range fence tree to track updates to page table structure.
* Used to implement conflict tracking between independent bind engines.
*/
struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE];
struct xe_range_fence_tree rftree;
const struct xe_pt_ops *pt_ops;
@ -557,6 +557,10 @@ struct xe_vma_ops {
u32 num_syncs;
/** @pt_update_ops: page table update operations */
struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
/** @start: start address of ops */
u64 start;
/** @last: last address of ops */
u64 last;
/** @flag: signify the properties within xe_vma_ops*/
#define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0)
#define XE_VMA_OPS_FLAG_MADVISE BIT(1)
@ -566,6 +570,10 @@ struct xe_vma_ops {
#define XE_VMA_OPS_FLAG_MODIFIES_GPUVA BIT(5)
#define XE_VMA_OPS_FLAG_DOWNGRADE_LOCK BIT(6)
#define XE_VMA_OPS_FLAG_HAS_SVM_VALID_RANGE BIT(7)
#define XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP BIT(8)
#define XE_VMA_OPS_FLAG_WAIT_VM_KERNEL BIT(9)
#define XE_VMA_OPS_FLAG_NEEDS_INVALIDATION BIT(10)
#define XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK BIT(11)
u32 flags;
#ifdef TEST_VM_OPS_ERROR
/** @inject_error: inject error to test error handling */

4
drivers/gpu/drm/xe/xe_wa_oob.rules
View File

@ -14,6 +14,7 @@
16017236439 PLATFORM(PVC)
14019821291 MEDIA_VERSION_RANGE(1300, 2000)
14015076503 MEDIA_VERSION(1300)
22016122933 MEDIA_VERSION(1300)
14018913170 GRAPHICS_VERSION_RANGE(1270, 1274)
MEDIA_VERSION(1300)
PLATFORM(DG2)
@ -72,3 +73,6 @@
16029897822 MEDIA_VERSION(3500)
GRAPHICS_VERSION(3510)
14027054324 GRAPHICS_VERSION(3511)
14025941587 GRAPHICS_VERSION_RANGE(2001, 3511), FUNC(xe_rtp_match_not_sriov_vf)
MEDIA_VERSION_RANGE(1301, 3503), FUNC(xe_rtp_match_not_sriov_vf)
16030224309 GRAPHICS_VERSION(3510)

6
include/drm/drm_gpusvm.h
View File

@ -274,6 +274,11 @@ struct drm_gpusvm {
* pages as valid; the caller revalidates the snapshot itself, see
* drm_gpusvm_get_pages(). @devmem_only is rejected and no page
* type check is performed, so @allow_mixed has no effect.
* @devmem_fn: Optional callback invoked under the notifier lock, once per
* contiguous run of pages backed by the same &drm_pagemap_devmem.
* Must not sleep and must be idempotent, as a non-contiguous
* allocation is reported more than once. Fires even with
* @no_dma_map. May be NULL.
*
* Context that is DRM GPUSVM is operating in (i.e. user arguments).
*/
@ -281,6 +286,7 @@ struct drm_gpusvm_ctx {
void *device_private_page_owner;
unsigned long check_pages_threshold;
unsigned long timeslice_ms;
void (*devmem_fn)(struct drm_pagemap_devmem *devmem);
unsigned int in_notifier :1;
unsigned int read_only :1;
unsigned int devmem_possible :1;

7
include/drm/drm_pagemap.h
View File

@ -258,6 +258,8 @@ struct drm_pagemap *drm_pagemap_create(struct drm_device *drm,
struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page);
struct drm_pagemap_devmem *drm_pagemap_page_to_devmem(struct page *page);
void drm_pagemap_put(struct drm_pagemap *dpagemap);
#else
@ -267,6 +269,11 @@ static inline struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page
return NULL;
}
static inline struct drm_pagemap_devmem *drm_pagemap_page_to_devmem(struct page *page)
{
return NULL;
}
static inline void drm_pagemap_put(struct drm_pagemap *dpagemap)
{
}

1
include/drm/intel/pciids.h
View File

@ -880,6 +880,7 @@
MACRO__(0xB08F, ## __VA_ARGS__), \
MACRO__(0xB090, ## __VA_ARGS__), \
MACRO__(0xB0A0, ## __VA_ARGS__), \
MACRO__(0xB0A1, ## __VA_ARGS__), \
MACRO__(0xB0B0, ## __VA_ARGS__)
/* WCL */