diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index ea9d87450eae..a23384571193 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -1257,6 +1257,99 @@ static int resume_single_queue(struct device_queue_manager *dqm, return 0; } +/* Unpin the MQD BO at S4 suspend so it is evicted into the hibernation image; + * dqm_repin_mqd_bo() pins it back on resume. Gated on adev->in_s4 so runtime + * eviction is untouched. + */ +static void dqm_evict_mqd_bo(struct device_queue_manager *dqm, struct queue *q) +{ + struct mqd_manager *mqd_mgr; + struct amdgpu_bo *bo; + + if (!dqm->dev->adev->in_s4) + return; + if (!mqd_on_vram(dqm->dev->adev)) + return; + if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE) + return; + if (!q->mqd_mem_obj || !q->mqd_mem_obj->mem) + return; + + /* Without update_mqd_gpu_addr() the MQD self-address cannot be fixed up + * after a repin, so skip eviction (with a warning) instead of faulting. + */ + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)]; + if (!mqd_mgr->update_mqd_gpu_addr) { + dev_warn_once(dqm->dev->adev->dev, + "MQD is in VRAM but update_mqd_gpu_addr is not implemented; skipping hibernation eviction\n"); + return; + } + + bo = q->mqd_mem_obj->mem; + if (amdgpu_bo_reserve(bo, false)) + return; + + amdgpu_bo_unpin(bo); + amdgpu_bo_unreserve(bo); + q->mqd = NULL; + q->needs_mqd_repin = true; +} + +/* Repin the MQD BO to VRAM and refresh the cached mapping and GPU addresses. + * Used both on resume and when a queue is destroyed before resume has repinned + * it. A no-op unless a repin is owed (needs_mqd_repin set). + */ +static int dqm_repin_mqd_bo(struct device_queue_manager *dqm, struct queue *q) +{ + struct mqd_manager *mqd_mgr; + struct amdgpu_bo *bo; + void *cpu_ptr; + int r; + + if (!q->needs_mqd_repin) + return 0; + if (!q->mqd_mem_obj || !q->mqd_mem_obj->mem) + return 0; + + bo = q->mqd_mem_obj->mem; + r = amdgpu_bo_reserve(bo, false); + if (r) + return r; + r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM); + if (r) { + amdgpu_bo_unreserve(bo); + dev_err(dqm->dev->adev->dev, + "Failed to repin MQD of queue %d to VRAM: %d\n", + q->properties.queue_id, r); + return r; + } + /* The BO may have moved; refresh the kernel mapping and gpu address. */ + amdgpu_bo_kunmap(bo); + r = amdgpu_bo_kmap(bo, &cpu_ptr); + amdgpu_bo_unreserve(bo); + if (r) { + dev_err(dqm->dev->adev->dev, + "Failed to remap MQD of queue %d: %d\n", + q->properties.queue_id, r); + return r; + } + + q->mqd_mem_obj->cpu_ptr = cpu_ptr; + q->mqd_mem_obj->gpu_addr = amdgpu_bo_gpu_offset(bo); + q->gart_mqd_addr = q->mqd_mem_obj->gpu_addr; + q->mqd = cpu_ptr; + + mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( + q->properties.type)]; + if (mqd_mgr->update_mqd_gpu_addr) + mqd_mgr->update_mqd_gpu_addr(mqd_mgr, q->mqd, + q->mqd_mem_obj, + &q->properties); + + q->needs_mqd_repin = false; + return 0; +} + static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, struct qcm_process_device *qpd) { @@ -1353,6 +1446,8 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, goto out; } } + + dqm_evict_mqd_bo(dqm, q); } if (!dqm->dev->kfd->shared_resources.enable_mes) { @@ -1492,6 +1587,13 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm, q->properties.is_active = true; increment_queue_count(dqm, &pdd->qpd, q); + retval = dqm_repin_mqd_bo(dqm, q); + if (retval) { + dev_err(dev, "Failed to repin MQD for queue %d\n", + q->properties.queue_id); + goto out; + } + if (dqm->dev->kfd->shared_resources.enable_mes) { retval = add_queue_mes(dqm, q, qpd); if (retval) { @@ -2763,6 +2865,8 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, qpd->pqm->process, q->device, -1, false, NULL, 0); + /* Repin the MQD BO if still evicted for hibernation, before it is freed. */ + dqm_repin_mqd_bo(dqm, q); mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); return retval; @@ -3020,6 +3124,8 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, list_del(&q->list); qpd->queue_count--; dqm_unlock(dqm); + /* Repin the MQD BO if still evicted for hibernation, before free. */ + dqm_repin_mqd_bo(dqm, q); mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj); dqm_lock(dqm); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h index 59eff3389d39..38b46b696243 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h @@ -117,6 +117,14 @@ struct mqd_manager { const void *ctl_stack_src, const u32 ctl_stack_size); + /* Patch the MQD's cached self GPU address after the MQD BO has moved + * (e.g. repinned to a new VRAM location on hibernation resume). The MQD + * contents are otherwise preserved. + */ + void (*update_mqd_gpu_addr)(struct mqd_manager *mm, void *mqd, + struct kfd_mem_obj *mqd_mem_obj, + struct queue_properties *p); + #if defined(CONFIG_DEBUG_FS) int (*debugfs_show_mqd)(struct seq_file *m, void *data); #endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index 75e5a9f67d50..b95720198e28 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -476,6 +476,20 @@ static void restore_mqd(struct mqd_manager *mm, void **mqd, qp->is_active = 0; } +static void update_mqd_gpu_addr(struct mqd_manager *mm, void *mqd, + struct kfd_mem_obj *mqd_mem_obj, + struct queue_properties *qp) +{ + struct v9_mqd *m = get_mqd(mqd); + uint64_t addr = mqd_mem_obj->gpu_addr; + + m->cp_mqd_base_addr_lo = lower_32_bits(addr); + m->cp_mqd_base_addr_hi = upper_32_bits(addr); + + if (mqd_on_vram(mm->dev->adev)) + amdgpu_device_flush_hdp(mm->dev->adev, NULL); +} + static void init_mqd_hiq(struct mqd_manager *mm, void **mqd, struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, struct queue_properties *q) @@ -860,6 +874,30 @@ static void restore_mqd_v9_4_3(struct mqd_manager *mm, void **mqd, if (mqd_on_vram(mm->dev->adev)) amdgpu_device_flush_hdp(mm->dev->adev, NULL); } + +static void update_mqd_gpu_addr_v9_4_3(struct mqd_manager *mm, void *mqd, + struct kfd_mem_obj *mqd_mem_obj, + struct queue_properties *qp) +{ + struct kfd_mem_obj xcc_mqd_mem_obj; + uint64_t offset = mm->mqd_stride(mm, qp); + u32 num_xcc = NUM_XCC(mm->dev->xcc_mask); + struct v9_mqd *m; + int xcc; + + memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj)); + + for (xcc = 0; xcc < num_xcc; xcc++) { + get_xcc_mqd(mqd_mem_obj, &xcc_mqd_mem_obj, offset * xcc); + m = get_mqd(mqd + offset * xcc); + m->cp_mqd_base_addr_lo = lower_32_bits(xcc_mqd_mem_obj.gpu_addr); + m->cp_mqd_base_addr_hi = upper_32_bits(xcc_mqd_mem_obj.gpu_addr); + } + + if (mqd_on_vram(mm->dev->adev)) + amdgpu_device_flush_hdp(mm->dev->adev, NULL); +} + static int destroy_mqd_v9_4_3(struct mqd_manager *mm, void *mqd, enum kfd_preempt_type type, unsigned int timeout, uint32_t pipe_id, uint32_t queue_id) @@ -1017,6 +1055,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, mqd->get_wave_state = get_wave_state_v9_4_3; mqd->checkpoint_mqd = checkpoint_mqd_v9_4_3; mqd->restore_mqd = restore_mqd_v9_4_3; + mqd->update_mqd_gpu_addr = update_mqd_gpu_addr_v9_4_3; } else { mqd->init_mqd = init_mqd; mqd->load_mqd = load_mqd; @@ -1025,6 +1064,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, mqd->get_wave_state = get_wave_state; mqd->checkpoint_mqd = checkpoint_mqd; mqd->restore_mqd = restore_mqd; + mqd->update_mqd_gpu_addr = update_mqd_gpu_addr; } break; case KFD_MQD_TYPE_HIQ: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 2ea1cfd330a9..d8631847f0eb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -638,6 +638,12 @@ struct queue { uint32_t gang_ctx_array_index; struct amdgpu_bo *wptr_bo_gart; + + /* The VRAM-resident MQD BO (mqd_on_vram()) is unpinned at S4 suspend so + * TTM evicts it into the hibernation image, and repinned on resume. Set + * while the BO is unpinned so the resume path knows to repin it. + */ + bool needs_mqd_repin; }; enum KFD_MQD_TYPE {