github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-28 09:04:39 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

- Remove double page flip on initial plane (Maarten)

Browse Source 
- Properly setup userptr pfn_flags_mask (Auld)
 - Fix GT "for each engine" workarounds (Tvrtko)
 - Fix userptr races and missed validations (Thomas, Brost)
 - Userptr invalid page access fixes (Thomas)
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEEbSBwaO7dZQkcLOKj+mJfZA7rE8oFAmfJ4sIACgkQ+mJfZA7r
 E8rQTQf9FAfEO06akhCbW5hxLmOSy8yns1WGES5PcVAQ1DLxtu5W8X3jDIbXG0VR
 RjMVidd/jYlwpJwbOo2boXdnFGyuKM+3G6pfbo7BMKvbwalfbppIKdWajVv2wkQr
 Ye98jJUVZAnvcJJUEk1WHh0Oxr8sxidKpKg8xM0L3TUcDZx7bubyvYipv7/dF8qg
 uSbbfqua5heSrWeJV897VqbpM9f9Xzsv/Sw8/Tt4ldOOMrnlu0SU6tQHeAEHqvgM
 Vq1y8ub/JpyeQHCRdkoGJV8V8tcFVOQp488D0uJ/Z3eGUlnhWUdDOj7lj6EphSjX
 Xs0sUK0iS1MqPFuuSKu3/FD54W8kzw==
 =fJ2d
 -----END PGP SIGNATURE-----

Merge tag 'drm-xe-fixes-2025-03-06' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-fixes

- Remove double page flip on initial plane (Maarten)
- Properly setup userptr pfn_flags_mask (Auld)
- Fix GT "for each engine" workarounds (Tvrtko)
- Fix userptr races and missed validations (Thomas, Brost)
- Userptr invalid page access fixes (Thomas)

Signed-off-by: Dave Airlie <airlied@redhat.com>

From: Rodrigo Vivi <rodrigo.vivi@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/Z8ni6w3tskCFL11O@intel.com
This commit is contained in:
Dave Airlie 2025-03-07 09:03:14 +10:00
commit eea6520c22
10 changed files with 292 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
drivers/gpu/drm/xe/display/xe_plane_initial.c
View File

@ -194,8 +194,6 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc,
to_intel_plane(crtc->base.primary);
struct intel_plane_state *plane_state =
to_intel_plane_state(plane->base.state);
struct intel_crtc_state *crtc_state =
to_intel_crtc_state(crtc->base.state);
struct drm_framebuffer *fb;
struct i915_vma *vma;
@ -241,14 +239,6 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc,
atomic_or(plane->frontbuffer_bit, &to_intel_frontbuffer(fb)->bits);
plane_config->vma = vma;
/*
* Flip to the newly created mapping ASAP, so we can re-use the
* first part of GGTT for WOPCM, prevent flickering, and prevent
* the lookup of sysmem scratch pages.
*/
plane->check_plane(crtc_state, plane_state);
plane->async_flip(NULL, plane, crtc_state, plane_state, true);
return;
nofb:

4
drivers/gpu/drm/xe/xe_gt.c
View File

@ -380,9 +380,7 @@ int xe_gt_init_early(struct xe_gt *gt)
if (err)
return err;
xe_wa_process_gt(gt);
xe_wa_process_oob(gt);
xe_tuning_process_gt(gt);
xe_force_wake_init_gt(gt, gt_to_fw(gt));
spin_lock_init(&gt->global_invl_lock);
@ -474,6 +472,8 @@ static int all_fw_domain_init(struct xe_gt *gt)
}
xe_gt_mcr_set_implicit_defaults(gt);
xe_wa_process_gt(gt);
xe_tuning_process_gt(gt);
xe_reg_sr_apply_mmio(&gt->reg_sr, gt);
err = xe_gt_clock_init(gt);

194
drivers/gpu/drm/xe/xe_hmm.c
View File

@ -19,11 +19,10 @@ static u64 xe_npages_in_range(unsigned long start, unsigned long end)
return (end - start) >> PAGE_SHIFT;
}
/*
/**
* xe_mark_range_accessed() - mark a range is accessed, so core mm
* have such information for memory eviction or write back to
* hard disk
*
* @range: the range to mark
* @write: if write to this range, we mark pages in this range
* as dirty
@ -43,15 +42,51 @@ static void xe_mark_range_accessed(struct hmm_range *range, bool write)
}
}
/*
static int xe_alloc_sg(struct xe_device *xe, struct sg_table *st,
struct hmm_range *range, struct rw_semaphore *notifier_sem)
{
unsigned long i, npages, hmm_pfn;
unsigned long num_chunks = 0;
int ret;
/* HMM docs says this is needed. */
ret = down_read_interruptible(notifier_sem);
if (ret)
return ret;
if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) {
up_read(notifier_sem);
return -EAGAIN;
}
npages = xe_npages_in_range(range->start, range->end);
for (i = 0; i < npages;) {
unsigned long len;
hmm_pfn = range->hmm_pfns[i];
xe_assert(xe, hmm_pfn & HMM_PFN_VALID);
len = 1UL << hmm_pfn_to_map_order(hmm_pfn);
/* If order > 0 the page may extend beyond range->start */
len -= (hmm_pfn & ~HMM_PFN_FLAGS) & (len - 1);
i += len;
num_chunks++;
}
up_read(notifier_sem);
return sg_alloc_table(st, num_chunks, GFP_KERNEL);
}
/**
* xe_build_sg() - build a scatter gather table for all the physical pages/pfn
* in a hmm_range. dma-map pages if necessary. dma-address is save in sg table
* and will be used to program GPU page table later.
*
* @xe: the xe device who will access the dma-address in sg table
* @range: the hmm range that we build the sg table from. range->hmm_pfns[]
* has the pfn numbers of pages that back up this hmm address range.
* @st: pointer to the sg table.
* @notifier_sem: The xe notifier lock.
* @write: whether we write to this range. This decides dma map direction
* for system pages. If write we map it bi-diretional; otherwise
* DMA_TO_DEVICE
@ -78,43 +113,84 @@ static void xe_mark_range_accessed(struct hmm_range *range, bool write)
* Returns 0 if successful; -ENOMEM if fails to allocate memory
*/
static int xe_build_sg(struct xe_device *xe, struct hmm_range *range,
struct sg_table *st, bool write)
struct sg_table *st,
struct rw_semaphore *notifier_sem,
bool write)
{
unsigned long npages = xe_npages_in_range(range->start, range->end);
struct device *dev = xe->drm.dev;
struct page **pages;
u64 i, npages;
int ret;
struct scatterlist *sgl;
struct page *page;
unsigned long i, j;
npages = xe_npages_in_range(range->start, range->end);
pages = kvmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
if (!pages)
return -ENOMEM;
lockdep_assert_held(notifier_sem);
for (i = 0; i < npages; i++) {
pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]);
xe_assert(xe, !is_device_private_page(pages[i]));
i = 0;
for_each_sg(st->sgl, sgl, st->nents, j) {
unsigned long hmm_pfn, size;
hmm_pfn = range->hmm_pfns[i];
page = hmm_pfn_to_page(hmm_pfn);
xe_assert(xe, !is_device_private_page(page));
size = 1UL << hmm_pfn_to_map_order(hmm_pfn);
size -= page_to_pfn(page) & (size - 1);
i += size;
if (unlikely(j == st->nents - 1)) {
if (i > npages)
size -= (i - npages);
sg_mark_end(sgl);
}
sg_set_page(sgl, page, size << PAGE_SHIFT, 0);
}
xe_assert(xe, i == npages);
ret = sg_alloc_table_from_pages_segment(st, pages, npages, 0, npages << PAGE_SHIFT,
xe_sg_segment_size(dev), GFP_KERNEL);
if (ret)
goto free_pages;
ret = dma_map_sgtable(dev, st, write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_NO_KERNEL_MAPPING);
if (ret) {
sg_free_table(st);
st = NULL;
}
free_pages:
kvfree(pages);
return ret;
return dma_map_sgtable(dev, st, write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_NO_KERNEL_MAPPING);
}
/*
static void xe_hmm_userptr_set_mapped(struct xe_userptr_vma *uvma)
{
struct xe_userptr *userptr = &uvma->userptr;
struct xe_vm *vm = xe_vma_vm(&uvma->vma);
lockdep_assert_held_write(&vm->lock);
lockdep_assert_held(&vm->userptr.notifier_lock);
mutex_lock(&userptr->unmap_mutex);
xe_assert(vm->xe, !userptr->mapped);
userptr->mapped = true;
mutex_unlock(&userptr->unmap_mutex);
}
void xe_hmm_userptr_unmap(struct xe_userptr_vma *uvma)
{
struct xe_userptr *userptr = &uvma->userptr;
struct xe_vma *vma = &uvma->vma;
bool write = !xe_vma_read_only(vma);
struct xe_vm *vm = xe_vma_vm(vma);
struct xe_device *xe = vm->xe;
if (!lockdep_is_held_type(&vm->userptr.notifier_lock, 0) &&
!lockdep_is_held_type(&vm->lock, 0) &&
!(vma->gpuva.flags & XE_VMA_DESTROYED)) {
/* Don't unmap in exec critical section. */
xe_vm_assert_held(vm);
/* Don't unmap while mapping the sg. */
lockdep_assert_held(&vm->lock);
}
mutex_lock(&userptr->unmap_mutex);
if (userptr->sg && userptr->mapped)
dma_unmap_sgtable(xe->drm.dev, userptr->sg,
write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);
userptr->mapped = false;
mutex_unlock(&userptr->unmap_mutex);
}
/**
* xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr
*
* @uvma: the userptr vma which hold the scatter gather table
*
* With function xe_userptr_populate_range, we allocate storage of
@ -124,16 +200,9 @@ static int xe_build_sg(struct xe_device *xe, struct hmm_range *range,
void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)
{
struct xe_userptr *userptr = &uvma->userptr;
struct xe_vma *vma = &uvma->vma;
bool write = !xe_vma_read_only(vma);
struct xe_vm *vm = xe_vma_vm(vma);
struct xe_device *xe = vm->xe;
struct device *dev = xe->drm.dev;
xe_assert(xe, userptr->sg);
dma_unmap_sgtable(dev, userptr->sg,
write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);
xe_assert(xe_vma_vm(&uvma->vma)->xe, userptr->sg);
xe_hmm_userptr_unmap(uvma);
sg_free_table(userptr->sg);
userptr->sg = NULL;
}
@ -166,13 +235,20 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma,
{
unsigned long timeout =
jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
unsigned long *pfns, flags = HMM_PFN_REQ_FAULT;
unsigned long *pfns;
struct xe_userptr *userptr;
struct xe_vma *vma = &uvma->vma;
u64 userptr_start = xe_vma_userptr(vma);
u64 userptr_end = userptr_start + xe_vma_size(vma);
struct xe_vm *vm = xe_vma_vm(vma);
struct hmm_range hmm_range;
struct hmm_range hmm_range = {
.pfn_flags_mask = 0, /* ignore pfns */
.default_flags = HMM_PFN_REQ_FAULT,
.start = userptr_start,
.end = userptr_end,
.notifier = &uvma->userptr.notifier,
.dev_private_owner = vm->xe,
};
bool write = !xe_vma_read_only(vma);
unsigned long notifier_seq;
u64 npages;
@ -199,19 +275,14 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma,
return -ENOMEM;
if (write)
flags |= HMM_PFN_REQ_WRITE;
hmm_range.default_flags |= HMM_PFN_REQ_WRITE;
if (!mmget_not_zero(userptr->notifier.mm)) {
ret = -EFAULT;
goto free_pfns;
}
hmm_range.default_flags = flags;
hmm_range.hmm_pfns = pfns;
hmm_range.notifier = &userptr->notifier;
hmm_range.start = userptr_start;
hmm_range.end = userptr_end;
hmm_range.dev_private_owner = vm->xe;
while (true) {
hmm_range.notifier_seq = mmu_interval_read_begin(&userptr->notifier);
@ -238,16 +309,37 @@ int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma,
if (ret)
goto free_pfns;
ret = xe_build_sg(vm->xe, &hmm_range, &userptr->sgt, write);
ret = xe_alloc_sg(vm->xe, &userptr->sgt, &hmm_range, &vm->userptr.notifier_lock);
if (ret)
goto free_pfns;
ret = down_read_interruptible(&vm->userptr.notifier_lock);
if (ret)
goto free_st;
if (mmu_interval_read_retry(hmm_range.notifier, hmm_range.notifier_seq)) {
ret = -EAGAIN;
goto out_unlock;
}
ret = xe_build_sg(vm->xe, &hmm_range, &userptr->sgt,
&vm->userptr.notifier_lock, write);
if (ret)
goto out_unlock;
xe_mark_range_accessed(&hmm_range, write);
userptr->sg = &userptr->sgt;
xe_hmm_userptr_set_mapped(uvma);
userptr->notifier_seq = hmm_range.notifier_seq;
up_read(&vm->userptr.notifier_lock);
kvfree(pfns);
return 0;
out_unlock:
up_read(&vm->userptr.notifier_lock);
free_st:
sg_free_table(&userptr->sgt);
free_pfns:
kvfree(pfns);
return ret;
}

7
drivers/gpu/drm/xe/xe_hmm.h
View File

@ -3,9 +3,16 @@
* Copyright © 2024 Intel Corporation
*/
#ifndef _XE_HMM_H_
#define _XE_HMM_H_
#include <linux/types.h>
struct xe_userptr_vma;
int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, bool is_mm_mmap_locked);
void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma);
void xe_hmm_userptr_unmap(struct xe_userptr_vma *uvma);
#endif

96
drivers/gpu/drm/xe/xe_pt.c
View File

@ -28,6 +28,8 @@ struct xe_pt_dir {
struct xe_pt pt;
/** @children: Array of page-table child nodes */
struct xe_ptw *children[XE_PDES];
/** @staging: Array of page-table staging nodes */
struct xe_ptw *staging[XE_PDES];
};
#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
@ -48,9 +50,10 @@ static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt)
return container_of(pt, struct xe_pt_dir, pt);
}
static struct xe_pt *xe_pt_entry(struct xe_pt_dir *pt_dir, unsigned int index)
static struct xe_pt *
xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index)
{
return container_of(pt_dir->children[index], struct xe_pt, base);
return container_of(pt_dir->staging[index], struct xe_pt, base);
}
static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
@ -125,6 +128,7 @@ struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
}
pt->bo = bo;
pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL;
pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL;
if (vm->xef)
xe_drm_client_add_bo(vm->xef->client, pt->bo);
@ -206,8 +210,8 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
for (i = 0; i < XE_PDES; i++) {
if (xe_pt_entry(pt_dir, i))
xe_pt_destroy(xe_pt_entry(pt_dir, i), flags,
if (xe_pt_entry_staging(pt_dir, i))
xe_pt_destroy(xe_pt_entry_staging(pt_dir, i), flags,
deferred);
}
}
@ -376,8 +380,10 @@ xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent,
/* Continue building a non-connected subtree. */
struct iosys_map *map = &parent->bo->vmap;
if (unlikely(xe_child))
if (unlikely(xe_child)) {
parent->base.children[offset] = &xe_child->base;
parent->base.staging[offset] = &xe_child->base;
}
xe_pt_write(xe_walk->vm->xe, map, offset, pte);
parent->num_live++;
@ -614,6 +620,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
.ops = &xe_pt_stage_bind_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
.staging = true,
},
.vm = xe_vma_vm(vma),
.tile = tile,
@ -873,7 +880,7 @@ static void xe_pt_cancel_bind(struct xe_vma *vma,
}
}
static void xe_pt_commit_locks_assert(struct xe_vma *vma)
static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma)
{
struct xe_vm *vm = xe_vma_vm(vma);
@ -885,6 +892,16 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma)
xe_vm_assert_held(vm);
}
static void xe_pt_commit_locks_assert(struct xe_vma *vma)
{
struct xe_vm *vm = xe_vma_vm(vma);
xe_pt_commit_prepare_locks_assert(vma);
if (xe_vma_is_userptr(vma))
lockdep_assert_held_read(&vm->userptr.notifier_lock);
}
static void xe_pt_commit(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries, struct llist_head *deferred)
@ -895,13 +912,17 @@ static void xe_pt_commit(struct xe_vma *vma,
for (i = 0; i < num_entries; i++) {
struct xe_pt *pt = entries[i].pt;
struct xe_pt_dir *pt_dir;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
int j_ = j + entries[i].ofs;
pt_dir->children[j_] = pt_dir->staging[j_];
xe_pt_destroy(oldpte, xe_vma_vm(vma)->flags, deferred);
}
}
@ -913,7 +934,7 @@ static void xe_pt_abort_bind(struct xe_vma *vma,
{
int i, j;
xe_pt_commit_locks_assert(vma);
xe_pt_commit_prepare_locks_assert(vma);
for (i = num_entries - 1; i >= 0; --i) {
struct xe_pt *pt = entries[i].pt;
@ -928,10 +949,10 @@ static void xe_pt_abort_bind(struct xe_vma *vma,
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
u32 j_ = j + entries[i].ofs;
struct xe_pt *newpte = xe_pt_entry(pt_dir, j_);
struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, j_);
struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
pt_dir->children[j_] = oldpte ? &oldpte->base : 0;
pt_dir->staging[j_] = oldpte ? &oldpte->base : 0;
xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);
}
}
@ -943,7 +964,7 @@ static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
{
u32 i, j;
xe_pt_commit_locks_assert(vma);
xe_pt_commit_prepare_locks_assert(vma);
for (i = 0; i < num_entries; i++) {
struct xe_pt *pt = entries[i].pt;
@ -961,10 +982,10 @@ static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
struct xe_pt *newpte = entries[i].pt_entries[j].pt;
struct xe_pt *oldpte = NULL;
if (xe_pt_entry(pt_dir, j_))
oldpte = xe_pt_entry(pt_dir, j_);
if (xe_pt_entry_staging(pt_dir, j_))
oldpte = xe_pt_entry_staging(pt_dir, j_);
pt_dir->children[j_] = &newpte->base;
pt_dir->staging[j_] = &newpte->base;
entries[i].pt_entries[j].pt = oldpte;
}
}
@ -1213,42 +1234,22 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
return 0;
uvma = to_userptr_vma(vma);
if (xe_pt_userptr_inject_eagain(uvma))
xe_vma_userptr_force_invalidate(uvma);
notifier_seq = uvma->userptr.notifier_seq;
if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode(vm))
return 0;
if (!mmu_interval_read_retry(&uvma->userptr.notifier,
notifier_seq) &&
!xe_pt_userptr_inject_eagain(uvma))
notifier_seq))
return 0;
if (xe_vm_in_fault_mode(vm)) {
if (xe_vm_in_fault_mode(vm))
return -EAGAIN;
} else {
spin_lock(&vm->userptr.invalidated_lock);
list_move_tail(&uvma->userptr.invalidate_link,
&vm->userptr.invalidated);
spin_unlock(&vm->userptr.invalidated_lock);
if (xe_vm_in_preempt_fence_mode(vm)) {
struct dma_resv_iter cursor;
struct dma_fence *fence;
long err;
dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
DMA_RESV_USAGE_BOOKKEEP);
dma_resv_for_each_fence_unlocked(&cursor, fence)
dma_fence_enable_sw_signaling(fence);
dma_resv_iter_end(&cursor);
err = dma_resv_wait_timeout(xe_vm_resv(vm),
DMA_RESV_USAGE_BOOKKEEP,
false, MAX_SCHEDULE_TIMEOUT);
XE_WARN_ON(err <= 0);
}
}
/*
* Just continue the operation since exec or rebind worker
* will take care of rebinding.
*/
return 0;
}
@ -1514,6 +1515,7 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma,
.ops = &xe_pt_stage_unbind_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
.staging = true,
},
.tile = tile,
.modified_start = xe_vma_start(vma),
@ -1555,7 +1557,7 @@ static void xe_pt_abort_unbind(struct xe_vma *vma,
{
int i, j;
xe_pt_commit_locks_assert(vma);
xe_pt_commit_prepare_locks_assert(vma);
for (i = num_entries - 1; i >= 0; --i) {
struct xe_vm_pgtable_update *entry = &entries[i];
@ -1568,7 +1570,7 @@ static void xe_pt_abort_unbind(struct xe_vma *vma,
continue;
for (j = entry->ofs; j < entry->ofs + entry->qwords; j++)
pt_dir->children[j] =
pt_dir->staging[j] =
entries[i].pt_entries[j - entry->ofs].pt ?
&entries[i].pt_entries[j - entry->ofs].pt->base : NULL;
}
@ -1581,7 +1583,7 @@ xe_pt_commit_prepare_unbind(struct xe_vma *vma,
{
int i, j;
xe_pt_commit_locks_assert(vma);
xe_pt_commit_prepare_locks_assert(vma);
for (i = 0; i < num_entries; ++i) {
struct xe_vm_pgtable_update *entry = &entries[i];
@ -1595,8 +1597,8 @@ xe_pt_commit_prepare_unbind(struct xe_vma *vma,
pt_dir = as_xe_pt_dir(pt);
for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) {
entry->pt_entries[j - entry->ofs].pt =
xe_pt_entry(pt_dir, j);
pt_dir->children[j] = NULL;
xe_pt_entry_staging(pt_dir, j);
pt_dir->staging[j] = NULL;
}
}
}

3
drivers/gpu/drm/xe/xe_pt_walk.c
View File

@ -74,7 +74,8 @@ int xe_pt_walk_range(struct xe_ptw *parent, unsigned int level,
u64 addr, u64 end, struct xe_pt_walk *walk)
{
pgoff_t offset = xe_pt_offset(addr, level, walk);
struct xe_ptw **entries = parent->children ? parent->children : NULL;
struct xe_ptw **entries = walk->staging ? (parent->staging ?: NULL) :
(parent->children ?: NULL);
const struct xe_pt_walk_ops *ops = walk->ops;
enum page_walk_action action;
struct xe_ptw *child;

4
drivers/gpu/drm/xe/xe_pt_walk.h
View File

@ -11,12 +11,14 @@
/**
* struct xe_ptw - base class for driver pagetable subclassing.
* @children: Pointer to an array of children if any.
* @staging: Pointer to an array of staging if any.
*
* Drivers could subclass this, and if it's a page-directory, typically
* embed an array of xe_ptw pointers.
*/
struct xe_ptw {
struct xe_ptw **children;
struct xe_ptw **staging;
};
/**
@ -41,6 +43,8 @@ struct xe_pt_walk {
* as shared pagetables.
*/
bool shared_pt_mode;
/** @staging: Walk staging PT structure */
bool staging;
};
/**

100
drivers/gpu/drm/xe/xe_vm.c
View File

@ -579,51 +579,26 @@ static void preempt_rebind_work_func(struct work_struct *w)
trace_xe_vm_rebind_worker_exit(vm);
}
static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
const struct mmu_notifier_range *range,
unsigned long cur_seq)
static void __vma_userptr_invalidate(struct xe_vm *vm, struct xe_userptr_vma *uvma)
{
struct xe_userptr *userptr = container_of(mni, typeof(*userptr), notifier);
struct xe_userptr_vma *uvma = container_of(userptr, typeof(*uvma), userptr);
struct xe_userptr *userptr = &uvma->userptr;
struct xe_vma *vma = &uvma->vma;
struct xe_vm *vm = xe_vma_vm(vma);
struct dma_resv_iter cursor;
struct dma_fence *fence;
long err;
xe_assert(vm->xe, xe_vma_is_userptr(vma));
trace_xe_vma_userptr_invalidate(vma);
if (!mmu_notifier_range_blockable(range))
return false;
vm_dbg(&xe_vma_vm(vma)->xe->drm,
"NOTIFIER: addr=0x%016llx, range=0x%016llx",
xe_vma_start(vma), xe_vma_size(vma));
down_write(&vm->userptr.notifier_lock);
mmu_interval_set_seq(mni, cur_seq);
/* No need to stop gpu access if the userptr is not yet bound. */
if (!userptr->initial_bind) {
up_write(&vm->userptr.notifier_lock);
return true;
}
/*
* Tell exec and rebind worker they need to repin and rebind this
* userptr.
*/
if (!xe_vm_in_fault_mode(vm) &&
!(vma->gpuva.flags & XE_VMA_DESTROYED) && vma->tile_present) {
!(vma->gpuva.flags & XE_VMA_DESTROYED)) {
spin_lock(&vm->userptr.invalidated_lock);
list_move_tail(&userptr->invalidate_link,
&vm->userptr.invalidated);
spin_unlock(&vm->userptr.invalidated_lock);
}
up_write(&vm->userptr.notifier_lock);
/*
* Preempt fences turn into schedule disables, pipeline these.
* Note that even in fault mode, we need to wait for binds and
@ -641,11 +616,37 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
false, MAX_SCHEDULE_TIMEOUT);
XE_WARN_ON(err <= 0);
if (xe_vm_in_fault_mode(vm)) {
if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {
err = xe_vm_invalidate_vma(vma);
XE_WARN_ON(err);
}
xe_hmm_userptr_unmap(uvma);
}
static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
const struct mmu_notifier_range *range,
unsigned long cur_seq)
{
struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier);
struct xe_vma *vma = &uvma->vma;
struct xe_vm *vm = xe_vma_vm(vma);
xe_assert(vm->xe, xe_vma_is_userptr(vma));
trace_xe_vma_userptr_invalidate(vma);
if (!mmu_notifier_range_blockable(range))
return false;
vm_dbg(&xe_vma_vm(vma)->xe->drm,
"NOTIFIER: addr=0x%016llx, range=0x%016llx",
xe_vma_start(vma), xe_vma_size(vma));
down_write(&vm->userptr.notifier_lock);
mmu_interval_set_seq(mni, cur_seq);
__vma_userptr_invalidate(vm, uvma);
up_write(&vm->userptr.notifier_lock);
trace_xe_vma_userptr_invalidate_complete(vma);
return true;
@ -655,6 +656,34 @@ static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = {
.invalidate = vma_userptr_invalidate,
};
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
/**
* xe_vma_userptr_force_invalidate() - force invalidate a userptr
* @uvma: The userptr vma to invalidate
*
* Perform a forced userptr invalidation for testing purposes.
*/
void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
{
struct xe_vm *vm = xe_vma_vm(&uvma->vma);
/* Protect against concurrent userptr pinning */
lockdep_assert_held(&vm->lock);
/* Protect against concurrent notifiers */
lockdep_assert_held(&vm->userptr.notifier_lock);
/*
* Protect against concurrent instances of this function and
* the critical exec sections
*/
xe_vm_assert_held(vm);
if (!mmu_interval_read_retry(&uvma->userptr.notifier,
uvma->userptr.notifier_seq))
uvma->userptr.notifier_seq -= 2;
__vma_userptr_invalidate(vm, uvma);
}
#endif
int xe_vm_userptr_pin(struct xe_vm *vm)
{
struct xe_userptr_vma *uvma, *next;
@ -1012,6 +1041,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
INIT_LIST_HEAD(&userptr->invalidate_link);
INIT_LIST_HEAD(&userptr->repin_link);
vma->gpuva.gem.offset = bo_offset_or_userptr;
mutex_init(&userptr->unmap_mutex);
err = mmu_interval_notifier_insert(&userptr->notifier,
current->mm,
@ -1053,6 +1083,7 @@ static void xe_vma_destroy_late(struct xe_vma *vma)
* them anymore
*/
mmu_interval_notifier_remove(&userptr->notifier);
mutex_destroy(&userptr->unmap_mutex);
xe_vm_put(vm);
} else if (xe_vma_is_null(vma)) {
xe_vm_put(vm);
@ -2286,8 +2317,17 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops,
break;
}
case DRM_GPUVA_OP_UNMAP:
xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
break;
case DRM_GPUVA_OP_PREFETCH:
/* FIXME: Need to skip some prefetch ops */
vma = gpuva_to_vma(op->base.prefetch.va);
if (xe_vma_is_userptr(vma)) {
err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
if (err)
return err;
}
xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
break;
default:

10
drivers/gpu/drm/xe/xe_vm.h
View File

@ -274,9 +274,17 @@ static inline void vm_dbg(const struct drm_device *dev,
const char *format, ...)
{ /* noop */ }
#endif
#endif
struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm);
void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap);
void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p);
void xe_vm_snapshot_free(struct xe_vm_snapshot *snap);
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma);
#else
static inline void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
{
}
#endif
#endif

8
drivers/gpu/drm/xe/xe_vm_types.h
View File

@ -59,12 +59,16 @@ struct xe_userptr {
struct sg_table *sg;
/** @notifier_seq: notifier sequence number */
unsigned long notifier_seq;
/** @unmap_mutex: Mutex protecting dma-unmapping */
struct mutex unmap_mutex;
/**
* @initial_bind: user pointer has been bound at least once.
* write: vm->userptr.notifier_lock in read mode and vm->resv held.
* read: vm->userptr.notifier_lock in write mode or vm->resv held.
*/
bool initial_bind;
/** @mapped: Whether the @sgt sg-table is dma-mapped. Protected by @unmap_mutex. */
bool mapped;
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
u32 divisor;
#endif
@ -227,8 +231,8 @@ struct xe_vm {
* up for revalidation. Protected from access with the
* @invalidated_lock. Removing items from the list
* additionally requires @lock in write mode, and adding
* items to the list requires the @userptr.notifer_lock in
* write mode.
* items to the list requires either the @userptr.notifer_lock in
* write mode, OR @lock in write mode.
*/
struct list_head invalidated;
} userptr;