github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-26 01:52:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

drm/amdkfd: implement restore_mqd callbacks for GFX12/12.1

Browse Source 
kfd_mqd_manager_v12.c (GFX 12.0) and kfd_mqd_manager_v12_1.c (GFX 12.1)
do not implement restore_mqd callbacks, leaving the function pointers
NULL and causing CRIU restore to return -EOPNOTSUPP on GFX12.

Implement restore_mqd for both compute and SDMA queues in
kfd_mqd_manager_v12.c and kfd_mqd_manager_v12_1.c, modeled after the
GFX 11 implementation with the following improvements:
- update cp_mqd_base_addr_lo/hi to the newly allocated MQD address,
  fixing a pre-existing gap shared with v11 where the in-MQD copy
  still pointed at the old checkpoint-time address after restore
- memset the full allocation before memcpy for compute queues to avoid
  stale data in the GTT sub-allocator tail; SDMA MQDs use sizeof(*m)
  since they are packed at mqd_size stride in a shared BO

checkpoint_mqd registration is deferred to a follow-up patch that also
implements get_checkpoint_info, so that checkpoint and restore are
enabled together as a complete and testable unit.

Note: GFX12.1 restore handles XCC0 only. Multi-XCC CRIU restore is
currently unreachable due to a separate validation issue in
kfd_criu_restore_queue(). A pr_warn_once() is emitted if a multi-XCC
device is encountered.

Signed-off-by: Vladimir Marioukhine <Vladimir.Marioukhine@amd.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
(cherry picked from commit b1f9601237d050f5df478464cf51bf1fff29a256)
Cc: stable@vger.kernel.org
This commit is contained in:
Vladimir Marioukhine 2026-08-12 13:19:46 -04:00 committed by Alex Deucher
parent 63e19ef3dd
commit 5f28bb1c2c
3 changed files with 132 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
View File

@ -770,10 +770,12 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
q->properties.type)];
if (qd && !mqd_mgr->restore_mqd) {
pr_debug("restore_mqd not implemented for this GPU\n");
pr_debug("restore_mqd not implemented for queue type %d\n",
q->properties.type);
retval = -EOPNOTSUPP;
goto deallocate_vmid;
}
if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
retval = allocate_hqd(dqm, q);
if (retval)
@ -2250,7 +2252,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
q->properties.type)];
if (qd && !mqd_mgr->restore_mqd) {
pr_debug("restore_mqd not implemented for this GPU\n");
pr_debug("restore_mqd not implemented for queue type %d\n",
q->properties.type);
retval = -EOPNOTSUPP;
goto out_deallocate_doorbell;
}

59
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
View File

@ -380,6 +380,63 @@ static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
#endif
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp, const void *mqd_src,
const void *ctl_stack_src, const u32 ctl_stack_size)
{
u64 addr;
struct v12_compute_mqd *m;
m = (struct v12_compute_mqd *)mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size));
memcpy(m, mqd_src, sizeof(*m));
/* Update MQD base address to the newly allocated location */
m->cp_mqd_base_addr_lo = lower_32_bits(addr);
m->cp_mqd_base_addr_hi = upper_32_bits(addr);
m->cp_hqd_pq_doorbell_control &=
~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK;
m->cp_hqd_pq_doorbell_control |=
qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control);
*mqd = m;
if (gart_addr)
*gart_addr = addr;
qp->is_active = 0;
}
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src,
const void *ctl_stack_src,
const u32 ctl_stack_size)
{
u64 addr;
struct v12_sdma_mqd *m;
m = (struct v12_sdma_mqd *)mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size));
memcpy(m, mqd_src, sizeof(*m));
m->sdmax_rlcx_doorbell_offset =
qp->doorbell_off << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
*mqd = m;
if (gart_addr)
*gart_addr = addr;
qp->is_active = 0;
}
struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
struct kfd_node *dev)
{
@ -407,6 +464,7 @@ struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
mqd->mqd_size = sizeof(struct v12_compute_mqd);
mqd->get_wave_state = get_wave_state;
mqd->mqd_stride = kfd_mqd_stride;
mqd->restore_mqd = restore_mqd;
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif
@ -453,6 +511,7 @@ struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
mqd->is_occupied = kfd_is_occupied_sdma;
mqd->mqd_size = sizeof(struct v12_sdma_mqd);
mqd->mqd_stride = kfd_mqd_stride;
mqd->restore_mqd = restore_mqd_sdma;
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
#endif

68
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
View File

@ -641,6 +641,72 @@ static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
#endif
static void restore_mqd_v12_1(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp, const void *mqd_src,
const void *ctl_stack_src, const u32 ctl_stack_size)
{
u64 addr;
struct v12_1_compute_mqd *m;
/*
* GFX12.1 is multi-XCC capable but this restore handles XCC0 only.
* Multi-XCC CRIU restore is currently unreachable because
* kfd_criu_restore_queue() validates against unscaled mqd_size.
*/
if (NUM_XCC(mm->dev->xcc_mask) > 1)
pr_warn_once("GFX12.1 multi-XCC CRIU restore not fully supported\n");
m = (struct v12_1_compute_mqd *)mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size) *
NUM_XCC(mm->dev->xcc_mask));
memcpy(m, mqd_src, sizeof(*m));
/* Update MQD base address to the newly allocated location */
m->cp_mqd_base_addr_lo = lower_32_bits(addr);
m->cp_mqd_base_addr_hi = upper_32_bits(addr);
m->cp_hqd_pq_doorbell_control &=
~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK;
m->cp_hqd_pq_doorbell_control |=
qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control);
*mqd = m;
if (gart_addr)
*gart_addr = addr;
qp->is_active = 0;
}
static void restore_mqd_sdma_v12_1(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
const void *mqd_src,
const void *ctl_stack_src,
const u32 ctl_stack_size)
{
u64 addr;
struct v12_sdma_mqd *m;
m = (struct v12_sdma_mqd *)mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size));
memcpy(m, mqd_src, sizeof(*m));
m->sdmax_rlcx_doorbell_offset =
qp->doorbell_off << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
*mqd = m;
if (gart_addr)
*gart_addr = addr;
qp->is_active = 0;
}
struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
struct kfd_node *dev)
{
@ -668,6 +734,7 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
mqd->mqd_size = sizeof(struct v12_1_compute_mqd);
mqd->get_wave_state = get_wave_state_v12_1;
mqd->mqd_stride = kfd_mqd_stride;
mqd->restore_mqd = restore_mqd_v12_1;
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif
@ -714,6 +781,7 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
mqd->is_occupied = kfd_is_occupied_sdma;
mqd->mqd_size = sizeof(struct v12_sdma_mqd);
mqd->mqd_stride = kfd_mqd_stride;
mqd->restore_mqd = restore_mqd_sdma_v12_1;
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
#endif