github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 01:32:21 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

amd-drm-next-7.2-2026-05-06:

Browse Source 
amdgpu:
 - GFX9 fixes
 - Hawaii SMU fixes
 - SDMA4 fix
 - GART fixes
 - Userq fixes
 - Finish support for using multiple SDMA queues for TTM operations
 - SWSMU updates
 - Misc cleanups and fixes
 - GC 12.1 updates
 - RAS updates
 - SMU 15.0.8 updates
 - DCN 4.2 updates
 - DC type conversion fixes
 - Enable DC power module
 - Replay/PSR updates
 - SMU 13.x updates
 - Compute queue quantum MQD updates
 - ASPM fix
 - GPUVM fixes
 - DCE 6 fixes
 - Align VKMS with common implementation
 - RDNA 4 fix
 - DC analog support fixes
 - UVD 3 fixes
 - TCC harvesting fixes for SI
 - GC 11 APU module reload fix
 - NBIO 6.3.2 support
 - IH 7.1 updates
 - DC cursor fixes
 - VCN user fence fixes
 - JPEG user fence fixes
 - DC support for connectors without DDC
 - Prefer ROM BAR for default VGA device
 - DC bandwidth fixes
 
 amdkfd:
 - GPUVM TLB flush fix
 - Hotplug fix
 - Boundary check fixes
 - Misc cleanups and fixes
 - SVM fixes
 - CRIU fixes
 
 radeon:
 - Hawaii SMU fixes
 - Misc cleanups and fixes
 -----BEGIN PGP SIGNATURE-----
 
 iHUEABYKAB0WIQQgO5Idg2tXNTSZAr293/aFa7yZ2AUCaftq6AAKCRC93/aFa7yZ
 2LFhAP9p75Di8sVXIcrANLztwSH8mgV+7Q+gNnnugYusJ+BUawEAs09LSjG/nefa
 iWd+iUzJY40FuIWo+whXn7cZlU2w/g0=
 =St8u
 -----END PGP SIGNATURE-----

Merge tag 'amd-drm-next-7.2-2026-05-06' of https://gitlab.freedesktop.org/agd5f/linux into drm-next

amd-drm-next-7.2-2026-05-06:

amdgpu:
- GFX9 fixes
- Hawaii SMU fixes
- SDMA4 fix
- GART fixes
- Userq fixes
- Finish support for using multiple SDMA queues for TTM operations
- SWSMU updates
- Misc cleanups and fixes
- GC 12.1 updates
- RAS updates
- SMU 15.0.8 updates
- DCN 4.2 updates
- DC type conversion fixes
- Enable DC power module
- Replay/PSR updates
- SMU 13.x updates
- Compute queue quantum MQD updates
- ASPM fix
- GPUVM fixes
- DCE 6 fixes
- Align VKMS with common implementation
- RDNA 4 fix
- DC analog support fixes
- UVD 3 fixes
- TCC harvesting fixes for SI
- GC 11 APU module reload fix
- NBIO 6.3.2 support
- IH 7.1 updates
- DC cursor fixes
- VCN user fence fixes
- JPEG user fence fixes
- DC support for connectors without DDC
- Prefer ROM BAR for default VGA device
- DC bandwidth fixes

amdkfd:
- GPUVM TLB flush fix
- Hotplug fix
- Boundary check fixes
- Misc cleanups and fixes
- SVM fixes
- CRIU fixes

radeon:
- Hawaii SMU fixes
- Misc cleanups and fixes

From: Alex Deucher <alexander.deucher@amd.com>
Link: https://patch.msgid.link/20260506164726.1733646-1-alexander.deucher@amd.com
Signed-off-by: Dave Airlie <airlied@redhat.com>
This commit is contained in:
Dave Airlie 2026-05-07 09:50:50 +10:00
commit bc0400e6f4
342 changed files with 78605 additions and 7914 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Documentation/gpu/amdgpu/amdgpu-glossary.rst
View File

@ -233,8 +233,15 @@ we have a dedicated glossary for Display Core at
TC
Texture Cache
TCC
Texture Cache per Channel - L2 cache attached to the memory channels.
May be used when shader cores are accessing memory.
Despite "Texture" in the name, this is used by any kind of memory access.
TCCs may be mapped to TCPs, depending on the architecture.
TCP (AMDGPU)
Texture Cache per Pipe. Even though the name "Texture" is part of this
Texture Cache per Pipe - L1 cache attached to each CU.
Even though the name "Texture" is part of this
acronym, the TCP represents the path to memory shaders; i.e., it is not
related to texture. The name is a leftover from older designs where shader
stages had different cache designs; it refers to the L1 cache in older

2
drivers/gpu/drm/amd/amdgpu/Makefile
View File

@ -90,7 +90,7 @@ amdgpu-y += \
nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o soc24.o \
sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o \
nbio_v7_9.o aqua_vanjaram.o nbio_v7_11.o lsdma_v7_0.o hdp_v7_0.o nbif_v6_3_1.o \
cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o
cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o nbio_v6_3_2.o
# add DF block
amdgpu-y += \

44
drivers/gpu/drm/amd/amdgpu/amdgpu.h
View File

@ -464,7 +464,7 @@ int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
#define AMDGPU_MAX_WB 1024 /* Reserve at most 1024 WB slots for amdgpu-owned rings. */
/**
* amdgpu_wb - This struct is used for small GPU memory allocation.
* struct amdgpu_wb - This struct is used for small GPU memory allocation.
*
* This struct is used to allocate a small amount of GPU memory that can be
* used to shadow certain states into the memory. This is especially useful for
@ -538,44 +538,6 @@ struct amdgpu_allowed_register_entry {
bool grbm_indexed;
};
/**
* enum amd_reset_method - Methods for resetting AMD GPU devices
*
* @AMD_RESET_METHOD_NONE: The device will not be reset.
* @AMD_RESET_LEGACY: Method reserved for SI, CIK and VI ASICs.
* @AMD_RESET_MODE0: Reset the entire ASIC. Not currently available for the
* any device.
* @AMD_RESET_MODE1: Resets all IP blocks on the ASIC (SDMA, GFX, VCN, etc.)
* individually. Suitable only for some discrete GPU, not
* available for all ASICs.
* @AMD_RESET_MODE2: Resets a lesser level of IPs compared to MODE1. Which IPs
* are reset depends on the ASIC. Notably doesn't reset IPs
* shared with the CPU on APUs or the memory controllers (so
* VRAM is not lost). Not available on all ASICs.
* @AMD_RESET_LINK: Triggers SW-UP link reset on other GPUs
* @AMD_RESET_BACO: BACO (Bus Alive, Chip Off) method powers off and on the card
* but without powering off the PCI bus. Suitable only for
* discrete GPUs.
* @AMD_RESET_PCI: Does a full bus reset using core Linux subsystem PCI reset
* and does a secondary bus reset or FLR, depending on what the
* underlying hardware supports.
*
* Methods available for AMD GPU driver for resetting the device. Not all
* methods are suitable for every device. User can override the method using
* module parameter `reset_method`.
*/
enum amd_reset_method {
AMD_RESET_METHOD_NONE = -1,
AMD_RESET_METHOD_LEGACY = 0,
AMD_RESET_METHOD_MODE0,
AMD_RESET_METHOD_MODE1,
AMD_RESET_METHOD_MODE2,
AMD_RESET_METHOD_LINK,
AMD_RESET_METHOD_BACO,
AMD_RESET_METHOD_PCI,
AMD_RESET_METHOD_ON_INIT,
};
struct amdgpu_video_codec_info {
u32 codec_type;
u32 max_width;
@ -1373,6 +1335,8 @@ int emu_soc_asic_init(struct amdgpu_device *adev);
#define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
#define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
#include "amdgpu_reset.h"
/*
* ASICs macro.
*/
@ -1462,6 +1426,8 @@ ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring);
ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset);
void amdgpu_sdma_set_vm_pte_scheds(struct amdgpu_device *adev,
const struct amdgpu_vm_pte_funcs *vm_pte_funcs);
void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev,
const struct amdgpu_buffer_funcs *buffer_funcs);
/* atpx handler */
#if defined(CONFIG_VGA_SWITCHEROO)

4
drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
View File

@ -33,6 +33,7 @@
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/acpi.h>
#include <linux/vgaarb.h>
/*
* BIOS.
*/
@ -467,7 +468,8 @@ static bool amdgpu_prefer_rom_resource(struct amdgpu_device *adev)
{
struct resource *res = &adev->pdev->resource[PCI_ROM_RESOURCE];
return (res->flags & IORESOURCE_ROM_SHADOW);
return (res->flags & IORESOURCE_ROM_SHADOW) ||
adev->pdev == vga_default_device();
}
static bool amdgpu_get_bios_dgpu(struct amdgpu_device *adev)

12
drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
View File

@ -153,7 +153,7 @@ int amdgpu_cper_entry_fill_fatal_section(struct amdgpu_device *adev,
FATAL_SEC_OFFSET(hdr->sec_cnt, idx));
amdgpu_cper_entry_fill_section_desc(adev, section_desc, false, false,
CPER_SEV_FATAL, CRASHDUMP, FATAL_SEC_LEN,
CPER_SEV_FATAL_UNCORRECTED, CRASHDUMP, FATAL_SEC_LEN,
FATAL_SEC_OFFSET(hdr->sec_cnt, idx));
section->body.reg_ctx_type = CPER_CTX_TYPE_CRASH;
@ -215,7 +215,7 @@ int amdgpu_cper_entry_fill_bad_page_threshold_section(struct amdgpu_device *adev
NONSTD_SEC_OFFSET(hdr->sec_cnt, idx));
amdgpu_cper_entry_fill_section_desc(adev, section_desc, true, false,
CPER_SEV_FATAL, RUNTIME, NONSTD_SEC_LEN,
CPER_SEV_FATAL_UNCORRECTED, RUNTIME, NONSTD_SEC_LEN,
NONSTD_SEC_OFFSET(hdr->sec_cnt, idx));
section->hdr.valid_bits.err_info_cnt = 1;
@ -312,7 +312,7 @@ int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev,
reg_data.synd_lo = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]);
reg_data.synd_hi = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]);
amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL);
amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL_UNCORRECTED);
ret = amdgpu_cper_entry_fill_fatal_section(adev, fatal, 0, reg_data);
if (ret)
return ret;
@ -337,7 +337,7 @@ int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev)
amdgpu_cper_entry_fill_hdr(adev, bp_threshold,
AMDGPU_CPER_TYPE_BP_THRESHOLD,
CPER_SEV_FATAL);
CPER_SEV_FATAL_UNCORRECTED);
ret = amdgpu_cper_entry_fill_bad_page_threshold_section(adev, bp_threshold, 0);
if (ret)
return ret;
@ -353,14 +353,14 @@ static enum cper_error_severity amdgpu_aca_err_type_to_cper_sev(struct amdgpu_de
{
switch (aca_err_type) {
case ACA_ERROR_TYPE_UE:
return CPER_SEV_FATAL;
return CPER_SEV_FATAL_UNCORRECTED;
case ACA_ERROR_TYPE_CE:
return CPER_SEV_NON_FATAL_CORRECTED;
case ACA_ERROR_TYPE_DEFERRED:
return CPER_SEV_NON_FATAL_UNCORRECTED;
default:
dev_err(adev->dev, "Unknown ACA error type!\n");
return CPER_SEV_FATAL;
return CPER_SEV_FATAL_UNCORRECTED;
}
}

21
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
View File

@ -2459,7 +2459,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
if (r)
goto init_failed;
amdgpu_ttm_set_buffer_funcs_status(adev, true);
amdgpu_ttm_enable_buffer_funcs(adev);
/* Don't init kfd if whole hive need to be reset during init */
if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
@ -3147,7 +3147,7 @@ static int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
amdgpu_virt_request_full_gpu(adev, false);
}
amdgpu_ttm_set_buffer_funcs_status(adev, false);
amdgpu_ttm_disable_buffer_funcs(adev);
r = amdgpu_device_ip_suspend_phase1(adev);
if (r)
@ -3362,7 +3362,7 @@ static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
r = amdgpu_device_ip_resume_phase2(adev);
amdgpu_ttm_set_buffer_funcs_status(adev, true);
amdgpu_ttm_enable_buffer_funcs(adev);
if (r)
return r;
@ -3700,7 +3700,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
adev->num_rings = 0;
RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
adev->mman.buffer_funcs = NULL;
adev->mman.buffer_funcs_ring = NULL;
adev->mman.num_buffer_funcs_scheds = 0;
adev->vm_manager.vm_pte_funcs = NULL;
adev->vm_manager.vm_pte_num_scheds = 0;
adev->gmc.gmc_funcs = NULL;
@ -3857,6 +3857,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
* completed before the need for a different level is detected.
*/
amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT);
amdgpu_device_check_iommu_direct_map(adev);
/* early init functions */
r = amdgpu_device_ip_early_init(adev);
if (r)
@ -4114,8 +4117,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
if (px)
vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
amdgpu_device_check_iommu_direct_map(adev);
adev->pm_nb.notifier_call = amdgpu_device_pm_notifier;
r = register_pm_notifier(&adev->pm_nb);
if (r)
@ -4213,7 +4214,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
/* disable ras feature must before hw fini */
amdgpu_ras_pre_fini(adev);
amdgpu_ttm_set_buffer_funcs_status(adev, false);
amdgpu_ttm_disable_buffer_funcs(adev);
/*
* device went through surprise hotplug; we need to destroy topology
@ -4480,7 +4481,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
if (r)
goto unwind_userq;
amdgpu_ttm_set_buffer_funcs_status(adev, false);
amdgpu_ttm_disable_buffer_funcs(adev);
amdgpu_fence_driver_hw_fini(adev);
@ -4494,7 +4495,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
return 0;
unwind_evict:
amdgpu_ttm_set_buffer_funcs_status(adev, true);
amdgpu_ttm_enable_buffer_funcs(adev);
amdgpu_fence_driver_hw_init(adev);
unwind_userq:
@ -5228,7 +5229,7 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
if (r)
goto out;
amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
amdgpu_ttm_enable_buffer_funcs(tmp_adev);
r = amdgpu_device_ip_resume_phase3(tmp_adev);
if (r)

29
drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
View File

@ -64,6 +64,7 @@
#include "nbio_v7_2.h"
#include "nbio_v7_7.h"
#include "nbif_v6_3_1.h"
#include "nbio_v6_3_2.h"
#include "hdp_v5_0.h"
#include "hdp_v5_2.h"
#include "hdp_v6_0.h"
@ -3193,6 +3194,9 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
adev->nbio.funcs = &nbif_v6_3_1_funcs;
adev->nbio.hdp_flush_reg = &nbif_v6_3_1_hdp_flush_reg;
break;
case IP_VERSION(6, 3, 2):
adev->nbio.funcs = &nbio_v6_3_2_funcs;
break;
default:
break;
}
@ -3416,3 +3420,28 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
return 0;
}
int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev,
uint16_t *major, uint16_t *minor)
{
uint8_t *discovery_bin = adev->discovery.bin;
struct table_info *info;
union gc_info *gc_info;
u16 offset;
if (!discovery_bin)
return -EINVAL;
if (amdgpu_discovery_get_table_info(adev, &info, GC))
return -EINVAL;
offset = le16_to_cpu(info->offset);
if (!offset)
return -EINVAL;
gc_info = (union gc_info *)(discovery_bin + offset);
if (major)
*major = le16_to_cpu(gc_info->v1.header.version_major);
if (minor)
*minor = le16_to_cpu(gc_info->v1.header.version_minor);
return 0;
}

2
drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
View File

@ -48,6 +48,8 @@ int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
uint32_t *nps_type,
struct amdgpu_gmc_memrange *ranges,
int *range_cnt, bool refresh);
int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev,
uint16_t *major, uint16_t *minor);
void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p);

13
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
View File

@ -3149,11 +3149,7 @@ static int __init amdgpu_init(void)
r = amdgpu_sync_init();
if (r)
goto error_sync;
r = amdgpu_userq_fence_slab_init();
if (r)
goto error_fence;
return r;
amdgpu_register_atpx_handler();
amdgpu_acpi_detect();
@ -3161,7 +3157,7 @@ static int __init amdgpu_init(void)
/* Ignore KFD init failures when CONFIG_HSA_AMD is not set. */
r = amdgpu_amdkfd_init();
if (r && r != -ENOENT)
goto error_fence;
goto error_fini_sync;
if (amdgpu_pp_feature_mask & PP_OVERDRIVE_MASK) {
add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
@ -3172,10 +3168,8 @@ static int __init amdgpu_init(void)
/* let modprobe override vga console setting */
return pci_register_driver(&amdgpu_kms_pci_driver);
error_fence:
error_fini_sync:
amdgpu_sync_fini();
error_sync:
return r;
}
@ -3186,7 +3180,6 @@ static void __exit amdgpu_exit(void)
amdgpu_unregister_atpx_handler();
amdgpu_acpi_release();
amdgpu_sync_fini();
amdgpu_userq_fence_slab_fini();
mmu_notifier_synchronize();
amdgpu_xcp_drv_release();
}

13
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
View File

@ -262,12 +262,19 @@ void amdgpu_gart_table_ram_free(struct amdgpu_device *adev)
*/
int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
{
int r;
if (adev->gart.bo != NULL)
return 0;
return amdgpu_bo_create_kernel(adev, adev->gart.table_size, PAGE_SIZE,
AMDGPU_GEM_DOMAIN_VRAM, &adev->gart.bo,
NULL, (void *)&adev->gart.ptr);
r = amdgpu_bo_create_kernel(adev, adev->gart.table_size, PAGE_SIZE,
AMDGPU_GEM_DOMAIN_VRAM, &adev->gart.bo,
NULL, (void *)&adev->gart.ptr);
if (r)
return r;
memset_io(adev->gart.ptr, adev->gart.gart_pte_flags, adev->gart.table_size);
return 0;
}
/**

25
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
View File

@ -27,6 +27,7 @@
*/
#include <linux/ktime.h>
#include <linux/module.h>
#include <linux/overflow.h>
#include <linux/pagemap.h>
#include <linux/pci.h>
#include <linux/dma-buf.h>
@ -1217,13 +1218,14 @@ int amdgpu_gem_list_handles_ioctl(struct drm_device *dev, void *data,
return ret;
}
static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
int width,
int cpp,
bool tiled)
static unsigned int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
unsigned int width,
unsigned int cpp,
bool tiled)
{
int aligned = width;
int pitch_mask = 0;
unsigned int aligned = width;
unsigned int pitch_mask = 0;
unsigned int pitch;
switch (cpp) {
case 1:
@ -1238,9 +1240,12 @@ static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
break;
}
aligned += pitch_mask;
if (check_add_overflow(aligned, pitch_mask, &aligned))
return 0;
aligned &= ~pitch_mask;
return aligned * cpp;
if (check_mul_overflow(aligned, cpp, &pitch))
return 0;
return pitch;
}
int amdgpu_mode_dumb_create(struct drm_file *file_priv,
@ -1267,8 +1272,12 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
args->pitch = amdgpu_gem_align_pitch(adev, args->width,
DIV_ROUND_UP(args->bpp, 8), 0);
if (!args->pitch)
return -EINVAL;
args->size = (u64)args->pitch * args->height;
args->size = ALIGN(args->size, PAGE_SIZE);
if (!args->size)
return -EINVAL;
domain = amdgpu_bo_get_preferred_domain(adev,
amdgpu_display_supported_domains(adev, flags));
r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,

51
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
View File

@ -2686,3 +2686,54 @@ void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev)
#endif
}
int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
struct amdgpu_ring *kiq_ring = &kiq->ring;
unsigned long flags;
int i;
if (adev->enable_mes)
return -EINVAL;
if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
return -EINVAL;
spin_lock_irqsave(&kiq->ring_lock, flags);
if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
spin_unlock_irqrestore(&kiq->ring_lock, flags);
return -ENOMEM;
}
/* assert preemption condition */
amdgpu_ring_set_preempt_cond_exec(ring, false);
/* assert IB preemption, emit the trailing fence */
kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
ring->trail_fence_gpu_addr,
++ring->trail_seq);
amdgpu_ring_commit(kiq_ring);
spin_unlock_irqrestore(&kiq->ring_lock, flags);
/* poll the trailing fence */
for (i = 0; i < adev->usec_timeout; i++) {
if (ring->trail_seq ==
le32_to_cpu(*(ring->trail_fence_cpu_addr)))
break;
udelay(1);
}
/* deassert preemption condition */
amdgpu_ring_set_preempt_cond_exec(ring, true);
if (i >= adev->usec_timeout) {
DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
return -EINVAL;
}
return 0;
}

2
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
View File

@ -664,6 +664,8 @@ void amdgpu_gfx_csb_preamble_end(u32 *buffer, u32 count);
void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev);
void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev);
int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring);
static inline const char *amdgpu_gfx_compute_mode_desc(int mode)
{
switch (mode) {

16
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
View File

@ -280,6 +280,15 @@ void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc
mc->gart_size >> 20, mc->gart_start, mc->gart_end);
}
void amdgpu_gmc_set_gart_size(struct amdgpu_device *adev, u64 default_size)
{
if (amdgpu_gart_size == -1)
adev->gmc.gart_size =
default_size + adev->pm.smu_prv_buffer_size;
else
adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
}
/**
* amdgpu_gmc_gart_location - try to find GART location
*
@ -711,12 +720,14 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
uint32_t vmhub, uint32_t flush_type)
{
struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
struct amdgpu_ring *ring;
struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
struct dma_fence *fence;
struct amdgpu_job *job;
int r;
ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
if (!hub->sdma_invalidation_workaround || vmid ||
!adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready ||
!ring->sched.ready) {
@ -1005,6 +1016,9 @@ void amdgpu_gmc_noretry_set(struct amdgpu_device *adev)
gc_ver == IP_VERSION(9, 5, 0) ||
gc_ver >= IP_VERSION(10, 3, 0));
/* For GFX12.1 B0, set xnack (retry) on as default */
if (gc_ver == IP_VERSION(12, 1, 0) && (adev->rev_id & 0xf) == 0x1)
noretry_default = false;
if (!amdgpu_sriov_xnack_support(adev))
gmc->noretry = 1;
else

4
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
View File

@ -365,6 +365,8 @@ struct amdgpu_gmc {
bool flush_tlb_needs_extra_type_0;
bool flush_tlb_needs_extra_type_2;
bool flush_pasid_uses_kiq;
bool override_pte;
};
#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
@ -484,4 +486,6 @@ void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev,
struct amdgpu_mem_partition_info *mem_ranges);
int amdgpu_gmc_get_vram_info(struct amdgpu_device *adev,
int *vram_width, int *vram_type, int *vram_vendor);
void amdgpu_gmc_set_gart_size(struct amdgpu_device *adev, u64 default_size);
#endif

6
drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
View File

@ -91,6 +91,12 @@ struct amdgpu_ih_funcs {
uint64_t (*decode_iv_ts)(struct amdgpu_ih_ring *ih, u32 rptr,
signed int offset);
void (*set_rptr)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih);
/* Decode IH cookie node_id into a human-readable die name string.
* Returns buf, or NULL if this IH version does not support node_id decoding.
*/
const char *(*node_id_to_die_name)(struct amdgpu_device *adev,
unsigned int node_id,
char *buf, size_t size);
};
#define amdgpu_ih_get_wptr(adev, ih) (adev)->irq.ih_funcs->get_wptr((adev), (ih))

1
drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
View File

@ -39,7 +39,6 @@ struct amdgpu_imu_funcs {
int (*switch_compute_partition)(struct amdgpu_device *adev,
int num_xccs_per_xcp,
int compute_partition_mode);
void (*init_mcm_addr_lut)(struct amdgpu_device *adev);
};
struct imu_rlc_ram_golden {

84
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
View File

@ -103,7 +103,7 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
int amdgpu_mes_init(struct amdgpu_device *adev)
{
int i, r, num_pipes;
int i, r, num_pipes, num_queues = 0;
u32 total_vmid_mask, reserved_vmid_mask;
int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
u32 gfx_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.me.num_pipe_per_me,
@ -159,7 +159,8 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
adev->mes.compute_hqd_mask[i] = compute_hqd_mask;
}
num_pipes = adev->sdma.num_instances;
num_pipes = adev->sdma.num_inst_per_xcc ?
adev->sdma.num_inst_per_xcc : adev->sdma.num_instances;
if (num_pipes > AMDGPU_MES_MAX_SDMA_PIPES)
dev_warn(adev->dev, "more SDMA pipes than supported by MES! (%d vs %d)\n",
num_pipes, AMDGPU_MES_MAX_SDMA_PIPES);
@ -216,8 +217,27 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
if (r)
goto error_doorbell;
if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)) {
/* When queue/pipe reset is done in MES instead of in the
* driver, MES passes hung queues information to the driver in
* hung_queue_hqd_info. Calculate required space to store this
* information.
*/
for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
num_queues += hweight32(adev->mes.compute_hqd_mask[i]);
for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc;
adev->mes.hung_queue_hqd_info_offset = num_queues;
adev->mes.hung_queue_db_array_size = num_queues * 2;
}
if (adev->mes.hung_queue_db_array_size) {
for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
r = amdgpu_bo_create_kernel(adev,
adev->mes.hung_queue_db_array_size * sizeof(u32),
PAGE_SIZE,
@ -264,10 +284,10 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
&adev->mes.event_log_cpu_addr);
for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i],
&adev->mes.hung_queue_db_array_gpu_addr[i],
&adev->mes.hung_queue_db_array_cpu_addr[i]);
if (adev->mes.hung_queue_db_array_gpu_obj[i])
amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i],
&adev->mes.hung_queue_db_array_gpu_addr[i],
&adev->mes.hung_queue_db_array_cpu_addr[i]);
if (adev->mes.sch_ctx_ptr[i])
amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);
if (adev->mes.query_status_fence_ptr[i])
@ -281,7 +301,7 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
mutex_destroy(&adev->mes.mutex_hidden);
}
int amdgpu_mes_suspend(struct amdgpu_device *adev)
int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id)
{
struct mes_suspend_gang_input input;
int r;
@ -291,6 +311,7 @@ int amdgpu_mes_suspend(struct amdgpu_device *adev)
memset(&input, 0x0, sizeof(struct mes_suspend_gang_input));
input.suspend_all_gangs = 1;
input.xcc_id = xcc_id;
/*
* Avoid taking any other locks under MES lock to avoid circular
@ -305,7 +326,7 @@ int amdgpu_mes_suspend(struct amdgpu_device *adev)
return r;
}
int amdgpu_mes_resume(struct amdgpu_device *adev)
int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id)
{
struct mes_resume_gang_input input;
int r;
@ -315,6 +336,7 @@ int amdgpu_mes_resume(struct amdgpu_device *adev)
memset(&input, 0x0, sizeof(struct mes_resume_gang_input));
input.resume_all_gangs = 1;
input.xcc_id = xcc_id;
/*
* Avoid taking any other locks under MES lock to avoid circular
@ -428,7 +450,7 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
{
struct mes_detect_and_reset_queue_input input;
u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id];
int r, i;
int hqd_info_offset = adev->mes.hung_queue_hqd_info_offset, r, i;
if (!hung_db_num || !hung_db_array)
return -EINVAL;
@ -443,26 +465,34 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
adev->mes.hung_queue_db_array_size * sizeof(u32));
input.queue_type = queue_type;
input.detect_only = detect_only;
input.xcc_id = xcc_id;
r = adev->mes.funcs->detect_and_reset_hung_queues(&adev->mes,
&input);
if (r) {
dev_err(adev->dev, "failed to detect and reset\n");
} else {
*hung_db_num = 0;
for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {
if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
hung_db_array[i] = db_array[i];
*hung_db_num += 1;
}
}
/*
* TODO: return HQD info for MES scheduled user compute queue reset cases
* stored in hung_db_array hqd info offset to full array size
*/
if (r && detect_only) {
dev_err(adev->dev, "Failed to detect hung queues\n");
return r;
}
*hung_db_num = 0;
/* MES passes hung queues' doorbell to driver */
for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {
/* Finding hung queues where db_array[i] is a valid doorbell */
if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
hung_db_array[i] = db_array[i];
*hung_db_num += 1;
}
}
if (r && !(*hung_db_num)) {
dev_err(adev->dev, "Failed to detect and reset hung queues\n");
return r;
}
for (i = hqd_info_offset; i < hqd_info_offset + *hung_db_num; i++)
hung_db_array[i] = db_array[i];
return r;
}
@ -758,6 +788,12 @@ bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev)
amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 0, 0));
}
bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev)
{
return (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) &&
(adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x73);
}
/* Fix me -- node_id is used to identify the correct MES instances in the future */
static int amdgpu_mes_set_enforce_isolation(struct amdgpu_device *adev,
uint32_t node_id, bool enable)

23
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
View File

@ -170,6 +170,19 @@ struct amdgpu_mes {
uint64_t shared_cmd_buf_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
};
struct amdgpu_mes_hung_queue_hqd_info {
union {
struct {
u32 queue_type: 3; // queue type
u32 pipe_index: 4; // pipe index
u32 queue_index: 8; // queue index
u32 reserved: 17;
};
u32 bit0_31;
};
};
struct amdgpu_mes_gang {
int gang_id;
int priority;
@ -312,8 +325,9 @@ struct mes_reset_queue_input {
};
struct mes_detect_and_reset_queue_input {
uint32_t queue_type;
bool detect_only;
u32 queue_type;
bool detect_only;
u32 xcc_id;
};
struct mes_inv_tlbs_pasid_input {
@ -429,8 +443,8 @@ int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe);
int amdgpu_mes_init(struct amdgpu_device *adev);
void amdgpu_mes_fini(struct amdgpu_device *adev);
int amdgpu_mes_suspend(struct amdgpu_device *adev);
int amdgpu_mes_resume(struct amdgpu_device *adev);
int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id);
int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id);
int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
struct amdgpu_ring *ring, uint32_t xcc_id);
@ -534,6 +548,7 @@ static inline void amdgpu_mes_unlock(struct amdgpu_mes *mes)
}
bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev);
bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev);
int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev);

4
drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
View File

@ -38,7 +38,6 @@
#include <drm/drm_probe_helper.h>
#include <linux/i2c.h>
#include <linux/i2c-algo-bit.h>
#include <linux/hrtimer.h>
#include "amdgpu_irq.h"
#include <drm/display/drm_dp_mst_helper.h>
@ -505,9 +504,6 @@ struct amdgpu_crtc {
u32 line_time;
u32 lb_vblank_lead_lines;
struct drm_display_mode hw_mode;
/* for virtual dce */
struct hrtimer vblank_timer;
enum amdgpu_interrupt_state vsync_timer_enabled;
int otg_inst;
struct drm_pending_vblank_event *event;

10
drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
View File

@ -84,3 +84,13 @@ int amdgpu_nbio_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *
amdgpu_ras_block_late_fini(adev, ras_block);
return r;
}
void amdgpu_nbio_program_aspm(struct amdgpu_device *adev)
{
if (!amdgpu_device_should_use_aspm(adev))
return;
if (adev->nbio.funcs->program_aspm)
adev->nbio.funcs->program_aspm(adev);
}

2
drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
View File

@ -121,4 +121,6 @@ u64 amdgpu_nbio_get_pcie_replay_count(struct amdgpu_device *adev);
bool amdgpu_nbio_is_replay_cnt_supported(struct amdgpu_device *adev);
void amdgpu_nbio_program_aspm(struct amdgpu_device *adev);
#endif

18
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
View File

@ -717,13 +717,17 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
bo->tbo.resource->mem_type == TTM_PL_VRAM) {
struct dma_fence *fence;
r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence);
r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
bo, bo->tbo.base.resv, &fence,
true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
if (unlikely(r))
goto fail_unreserve;
dma_resv_add_fence(bo->tbo.base.resv, fence,
DMA_RESV_USAGE_KERNEL);
dma_fence_put(fence);
if (fence) {
dma_resv_add_fence(bo->tbo.base.resv, fence,
DMA_RESV_USAGE_KERNEL);
dma_fence_put(fence);
}
}
if (!bp->resv)
amdgpu_bo_unreserve(bo);
@ -1325,9 +1329,9 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
if (r)
goto out;
r = amdgpu_fill_buffer(amdgpu_ttm_next_clear_entity(adev),
abo, 0, &bo->base._resv,
&fence, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
abo, &bo->base._resv, &fence,
false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
if (WARN_ON(r))
goto out;

9
drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
View File

@ -518,7 +518,7 @@ static int psp_sw_init(struct amdgpu_ip_block *ip_block)
}
ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
(amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf) ?
(amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf || adev->gmc.xgmi.connected_to_cpu) ?
AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
&psp->fw_pri_bo,
&psp->fw_pri_mc_addr,
@ -3527,7 +3527,12 @@ int psp_init_toc_microcode(struct psp_context *psp, const char *chip_name)
const struct psp_firmware_header_v1_0 *toc_hdr;
int err = 0;
err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED,
if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(15, 0, 8) &&
adev->rev_id == 0)
err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED,
"amdgpu/%s_toc_1.bin", chip_name);
else
err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED,
"amdgpu/%s_toc.bin", chip_name);
if (err)
goto out;

18
drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c
View File

@ -956,3 +956,21 @@ uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst,
}
return ret;
}
uint32_t amdgpu_read_indexed_register(struct amdgpu_device *adev,
u32 se_num, u32 sh_num, u32 reg_offset)
{
uint32_t val;
mutex_lock(&adev->grbm_idx_mutex);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
val = RREG32(reg_offset);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
mutex_unlock(&adev->grbm_idx_mutex);
return val;
}

3
drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
View File

@ -160,4 +160,7 @@ uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst,
uint32_t reg_addr, char reg_name[],
uint32_t expected_value, uint32_t mask);
uint32_t amdgpu_read_indexed_register(struct amdgpu_device *adev,
u32 se_num, u32 sh_num, u32 reg_offset);
#endif /* __AMDGPU_REG_ACCESS_H__ */

69
drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
View File

@ -46,6 +46,47 @@ enum AMDGPU_RESET_SRCS {
AMDGPU_RESET_SRC_USERQ,
};
/**
* enum amd_reset_method - Methods for resetting AMD GPU devices
*
* @AMD_RESET_METHOD_NONE: The device will not be reset.
* @AMD_RESET_METHOD_LEGACY: Method reserved for SI, CIK and VI ASICs.
* @AMD_RESET_METHOD_MODE0: Reset the entire ASIC. Not currently available for
* the any device.
* @AMD_RESET_METHOD_MODE1: Resets all IP blocks on the ASIC (SDMA, GFX, VCN,
* etc.) individually. Suitable only for some discrete GPU,
* not available for all ASICs.
* @AMD_RESET_METHOD_MODE2: Resets a lesser level of IPs compared to MODE1.
* Which IPs are reset depends on the ASIC. Notably doesn't
* reset IPs shared with the CPU on APUs or the memory
* controllers (so VRAM is not lost). Not available on all
* ASICs.
* @AMD_RESET_METHOD_LINK: Triggers SW-UP link reset on other GPUs
* @AMD_RESET_METHOD_BACO: BACO (Bus Alive, Chip Off) method powers off and on
* the card but without powering off the PCI bus. Suitable
* only for discrete GPUs.
* @AMD_RESET_METHOD_PCI: Does a full bus reset using core Linux subsystem
* PCI reset and does a secondary bus reset or FLR,
* depending on what the underlying hardware supports.
* @AMD_RESET_METHOD_ON_INIT: Does a device reset during the driver init
* sequence.
*
* Methods available for AMD GPU driver for resetting the device. Not all
* methods are suitable for every device. User can override the method using
* module parameter `reset_method`.
*/
enum amd_reset_method {
AMD_RESET_METHOD_NONE = -1,
AMD_RESET_METHOD_LEGACY = 0,
AMD_RESET_METHOD_MODE0,
AMD_RESET_METHOD_MODE1,
AMD_RESET_METHOD_MODE2,
AMD_RESET_METHOD_LINK,
AMD_RESET_METHOD_BACO,
AMD_RESET_METHOD_PCI,
AMD_RESET_METHOD_ON_INIT,
};
struct amdgpu_reset_context {
enum amd_reset_method method;
struct amdgpu_device *reset_req_dev;
@ -56,6 +97,20 @@ struct amdgpu_reset_context {
enum AMDGPU_RESET_SRCS src;
};
struct amdgpu_reset_control {
void *handle;
struct work_struct reset_work;
struct mutex reset_lock;
struct amdgpu_reset_handler *(
*reset_handlers)[AMDGPU_RESET_MAX_HANDLERS];
atomic_t in_reset;
enum amd_reset_method active_reset;
struct amdgpu_reset_handler *(*get_reset_handler)(
struct amdgpu_reset_control *reset_ctl,
struct amdgpu_reset_context *context);
void (*async_reset)(struct work_struct *work);
};
struct amdgpu_reset_handler {
enum amd_reset_method reset_method;
int (*prepare_env)(struct amdgpu_reset_control *reset_ctl,
@ -72,20 +127,6 @@ struct amdgpu_reset_handler {
int (*do_reset)(struct amdgpu_device *adev);
};
struct amdgpu_reset_control {
void *handle;
struct work_struct reset_work;
struct mutex reset_lock;
struct amdgpu_reset_handler *(
*reset_handlers)[AMDGPU_RESET_MAX_HANDLERS];
atomic_t in_reset;
enum amd_reset_method active_reset;
struct amdgpu_reset_handler *(*get_reset_handler)(
struct amdgpu_reset_control *reset_ctl,
struct amdgpu_reset_context *context);
void (*async_reset)(struct work_struct *work);
};
enum amdgpu_reset_domain_type {
SINGLE_DEVICE,

328
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
View File

@ -171,7 +171,7 @@ amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entit
{
struct amdgpu_ring *ring;
ring = adev->mman.buffer_funcs_ring;
ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
amdgpu_ring_pad_ib(ring, &job->ibs[0]);
WARN_ON(job->ibs[0].length_dw > num_dw);
@ -420,8 +420,8 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
if (old_mem->mem_type == TTM_PL_VRAM &&
(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
struct dma_fence *wipe_fence = NULL;
r = amdgpu_fill_buffer(entity, abo, 0, NULL, &wipe_fence,
AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
r = amdgpu_ttm_clear_buffer(entity, abo, NULL, &wipe_fence,
false, AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
if (r) {
goto error;
} else if (wipe_fence) {
@ -1563,7 +1563,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
if (!adev->mman.sdma_access_ptr)
return -EACCES;
if (!drm_dev_enter(adev_to_drm(adev), &idx))
if (!adev->mman.buffer_funcs_enabled || !drm_dev_enter(adev_to_drm(adev), &idx))
return -ENODEV;
if (write)
@ -1715,10 +1715,7 @@ static void amdgpu_ttm_init_fw_resv_region(struct amdgpu_device *adev)
reserve_size = max(reserve_size, (uint32_t)280 << 20);
else if (!adev->bios &&
amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
if (hweight32(adev->aid_mask) == 1)
reserve_size = max(reserve_size, (uint32_t)128 << 20);
else
reserve_size = max(reserve_size, (uint32_t)144 << 20);
reserve_size = max(reserve_size, (uint32_t)150 << 20);
} else if (!reserve_size)
reserve_size = DISCOVERY_TMR_OFFSET;
@ -2101,7 +2098,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
}
/* Change the size here instead of the init above so only lpfn is affected */
amdgpu_ttm_set_buffer_funcs_status(adev, false);
amdgpu_ttm_disable_buffer_funcs(adev);
#ifdef CONFIG_64BIT
#ifdef CONFIG_X86
if (adev->gmc.xgmi.connected_to_cpu)
@ -2281,115 +2278,92 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
}
/**
* amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
* amdgpu_ttm_enable_buffer_funcs - enable use of buffer functions
*
* @adev: amdgpu_device pointer
* @enable: true when we can use buffer functions.
*
* Enable/disable use of buffer functions during suspend/resume. This should
* Enable use of buffer functions during suspend/resume. This should
* only be called at bootup or when userspace isn't running.
*/
void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev)
{
struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
u32 num_clear_entities, num_move_entities;
uint64_t size;
int r, i, j;
if (!adev->mman.initialized || amdgpu_in_reset(adev) ||
adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu)
adev->mman.buffer_funcs_enabled || adev->gmc.is_app_apu)
return;
if (enable) {
struct amdgpu_ring *ring;
struct drm_gpu_scheduler *sched;
if (!adev->mman.num_buffer_funcs_scheds) {
dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use");
return;
}
if (!adev->mman.buffer_funcs_ring || !adev->mman.buffer_funcs_ring->sched.ready) {
dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use");
return;
}
/* default_entity doesn't need multiple schedulers so pass only 1. */
r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
&adev->mman.default_entity,
DRM_SCHED_PRIORITY_KERNEL,
adev->mman.buffer_funcs_scheds, 1, 0);
if (r < 0) {
dev_err(adev->dev,
"Failed setting up TTM entity (%d)\n", r);
return;
}
num_clear_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES);
num_move_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES);
adev->mman.clear_entities = kcalloc(num_clear_entities,
sizeof(struct amdgpu_ttm_buffer_entity),
GFP_KERNEL);
atomic_set(&adev->mman.next_clear_entity, 0);
if (!adev->mman.clear_entities)
goto error_free_default_entity;
adev->mman.num_clear_entities = num_clear_entities;
for (i = 0; i < num_clear_entities; i++) {
r = amdgpu_ttm_buffer_entity_init(
&adev->mman.gtt_mgr,
&adev->mman.clear_entities[i],
DRM_SCHED_PRIORITY_KERNEL,
adev->mman.buffer_funcs_scheds,
adev->mman.num_buffer_funcs_scheds, 1);
num_clear_entities = 1;
num_move_entities = 1;
ring = adev->mman.buffer_funcs_ring;
sched = &ring->sched;
r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
&adev->mman.default_entity,
DRM_SCHED_PRIORITY_KERNEL,
&sched, 1, 0);
if (r < 0) {
dev_err(adev->dev,
"Failed setting up TTM entity (%d)\n", r);
return;
}
adev->mman.clear_entities = kcalloc(num_clear_entities,
sizeof(struct amdgpu_ttm_buffer_entity),
GFP_KERNEL);
atomic_set(&adev->mman.next_clear_entity, 0);
if (!adev->mman.clear_entities)
for (j = 0; j < i; j++)
amdgpu_ttm_buffer_entity_fini(
&adev->mman.gtt_mgr, &adev->mman.clear_entities[j]);
adev->mman.num_clear_entities = 0;
kfree(adev->mman.clear_entities);
goto error_free_default_entity;
adev->mman.num_clear_entities = num_clear_entities;
for (i = 0; i < num_clear_entities; i++) {
r = amdgpu_ttm_buffer_entity_init(
&adev->mman.gtt_mgr, &adev->mman.clear_entities[i],
DRM_SCHED_PRIORITY_NORMAL, &sched, 1, 1);
if (r < 0) {
for (j = 0; j < i; j++)
amdgpu_ttm_buffer_entity_fini(
&adev->mman.gtt_mgr, &adev->mman.clear_entities[j]);
kfree(adev->mman.clear_entities);
adev->mman.num_clear_entities = 0;
adev->mman.clear_entities = NULL;
goto error_free_default_entity;
}
}
}
adev->mman.num_move_entities = num_move_entities;
atomic_set(&adev->mman.next_move_entity, 0);
for (i = 0; i < num_move_entities; i++) {
r = amdgpu_ttm_buffer_entity_init(
&adev->mman.gtt_mgr,
&adev->mman.move_entities[i],
DRM_SCHED_PRIORITY_NORMAL, &sched, 1, 2);
adev->mman.num_move_entities = num_move_entities;
atomic_set(&adev->mman.next_move_entity, 0);
for (i = 0; i < num_move_entities; i++) {
r = amdgpu_ttm_buffer_entity_init(
&adev->mman.gtt_mgr,
&adev->mman.move_entities[i],
DRM_SCHED_PRIORITY_KERNEL,
adev->mman.buffer_funcs_scheds,
adev->mman.num_buffer_funcs_scheds, 2);
if (r < 0) {
for (j = 0; j < i; j++)
amdgpu_ttm_buffer_entity_fini(
&adev->mman.gtt_mgr, &adev->mman.move_entities[j]);
adev->mman.num_move_entities = 0;
goto error_free_clear_entities;
}
if (r < 0) {
for (j = 0; j < i; j++)
amdgpu_ttm_buffer_entity_fini(
&adev->mman.gtt_mgr,
&adev->mman.move_entities[j]);
adev->mman.num_move_entities = 0;
goto error_free_clear_entities;
}
} else {
amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
&adev->mman.default_entity);
for (i = 0; i < adev->mman.num_clear_entities; i++)
amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
&adev->mman.clear_entities[i]);
for (i = 0; i < adev->mman.num_move_entities; i++)
amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
&adev->mman.move_entities[i]);
/* Drop all the old fences since re-creating the scheduler entities
* will allocate new contexts.
*/
ttm_resource_manager_cleanup(man);
kfree(adev->mman.clear_entities);
adev->mman.clear_entities = NULL;
adev->mman.num_clear_entities = 0;
adev->mman.num_move_entities = 0;
}
/* this just adjusts TTM size idea, which sets lpfn to the correct value */
if (enable)
size = adev->gmc.real_vram_size;
else
size = adev->gmc.visible_vram_size;
man->size = size;
adev->mman.buffer_funcs_enabled = enable;
man->size = adev->gmc.real_vram_size;
adev->mman.buffer_funcs_enabled = true;
return;
@ -2405,6 +2379,42 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
&adev->mman.default_entity);
}
/**
* amdgpu_ttm_disable_buffer_funcs - disable use of buffer functions
*
* @adev: amdgpu_device pointer
*/
void amdgpu_ttm_disable_buffer_funcs(struct amdgpu_device *adev)
{
struct ttm_resource_manager *man =
ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
int i;
if (!adev->mman.buffer_funcs_enabled || amdgpu_in_reset(adev))
return;
amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
&adev->mman.default_entity);
for (i = 0; i < adev->mman.num_move_entities; i++)
amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
&adev->mman.move_entities[i]);
for (i = 0; i < adev->mman.num_clear_entities; i++)
amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
&adev->mman.clear_entities[i]);
/* Drop all the old fences since re-creating the scheduler entities
* will allocate new contexts.
*/
ttm_resource_manager_cleanup(man);
kfree(adev->mman.clear_entities);
adev->mman.clear_entities = NULL;
adev->mman.num_clear_entities = 0;
adev->mman.num_move_entities = 0;
man->size = adev->gmc.visible_vram_size;
adev->mman.buffer_funcs_enabled = false;
}
static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
struct amdgpu_ttm_buffer_entity *entity,
unsigned int num_dw,
@ -2449,7 +2459,7 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,
unsigned int i;
int r;
ring = adev->mman.buffer_funcs_ring;
ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
if (!ring->sched.ready) {
dev_err(adev->dev,
@ -2524,77 +2534,23 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev,
}
/**
* amdgpu_ttm_clear_buffer - clear memory buffers
* @bo: amdgpu buffer object
* @resv: reservation object
* @fence: dma_fence associated with the operation
* amdgpu_ttm_clear_buffer - fill a buffer with 0
* @entity: entity to use
* @bo: the bo to fill
* @resv: fences contained in this reservation will be used as dependencies.
* @out_fence: the fence from the last clear will be stored here. It might be
* NULL if no job was run.
* @consider_clear_status: true if region reported as cleared by amdgpu_res_cleared()
* are skipped.
* @k_job_id: trace id
*
* Clear the memory buffer resource.
*
* Returns:
* 0 for success or a negative error code on failure.
*/
int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity,
struct amdgpu_bo *bo,
struct dma_resv *resv,
struct dma_fence **fence)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
struct amdgpu_ttm_buffer_entity *entity;
struct amdgpu_res_cursor cursor;
u64 addr;
int r = 0;
if (!adev->mman.buffer_funcs_enabled)
return -EINVAL;
if (!fence)
return -EINVAL;
entity = &adev->mman.clear_entities[0];
*fence = dma_fence_get_stub();
amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor);
mutex_lock(&entity->lock);
while (cursor.remaining) {
struct dma_fence *next = NULL;
u64 size;
if (amdgpu_res_cleared(&cursor)) {
amdgpu_res_next(&cursor, cursor.size);
continue;
}
/* Never clear more than 256MiB at once to avoid timeouts */
size = min(cursor.size, 256ULL << 20);
r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &cursor,
0, false, &size, &addr);
if (r)
goto err;
r = amdgpu_ttm_fill_mem(adev, entity, 0, addr, size, resv,
&next, true,
AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
if (r)
goto err;
dma_fence_put(*fence);
*fence = next;
amdgpu_res_next(&cursor, size);
}
err:
mutex_unlock(&entity->lock);
return r;
}
int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
struct amdgpu_bo *bo,
uint32_t src_data,
struct dma_resv *resv,
struct dma_fence **f,
u64 k_job_id)
struct dma_fence **out_fence,
bool consider_clear_status,
u64 k_job_id)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
struct dma_fence *fence = NULL;
@ -2611,6 +2567,11 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
struct dma_fence *next;
uint64_t cur_size, to;
if (consider_clear_status && amdgpu_res_cleared(&dst)) {
amdgpu_res_next(&dst, dst.size);
continue;
}
/* Never fill more than 256MiB at once to avoid timeouts */
cur_size = min(dst.size, 256ULL << 20);
@ -2620,7 +2581,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
goto error;
r = amdgpu_ttm_fill_mem(adev, entity,
src_data, to, cur_size, resv,
0, to, cur_size, resv,
&next, true, k_job_id);
if (r)
goto error;
@ -2632,9 +2593,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
}
error:
mutex_unlock(&entity->lock);
if (f)
*f = dma_fence_get(fence);
dma_fence_put(fence);
*out_fence = fence;
return r;
}
@ -2682,6 +2641,41 @@ int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type)
return ttm_resource_manager_evict_all(&adev->mman.bdev, man);
}
void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev,
const struct amdgpu_buffer_funcs *buffer_funcs)
{
struct drm_gpu_scheduler *sched;
struct amdgpu_vmhub *hub;
int i, n;
adev->mman.buffer_funcs = buffer_funcs;
for (i = 0, n = 0; i < adev->sdma.num_instances; i++) {
if (adev->sdma.has_page_queue)
sched = &adev->sdma.instance[i].page.sched;
else
sched = &adev->sdma.instance[i].ring.sched;
if (!sched->ready)
continue;
adev->mman.buffer_funcs_scheds[n++] = sched;
}
if (n == 0) {
adev->mman.num_buffer_funcs_scheds = 0;
drm_warn(&adev->ddev, "No working sdma ring available\n");
return;
}
/* Navi1x's workaround requires us to limit to a single SDMA sched
* for ttm.
*/
hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
adev->mman.num_buffer_funcs_scheds = hub->sdma_invalidation_workaround ?
1 : n;
}
#if defined(CONFIG_DEBUG_FS)
static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused)

20
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
View File

@ -87,7 +87,8 @@ struct amdgpu_mman {
/* buffer handling */
const struct amdgpu_buffer_funcs *buffer_funcs;
struct amdgpu_ring *buffer_funcs_ring;
struct drm_gpu_scheduler *buffer_funcs_scheds[AMDGPU_MAX_RINGS];
u32 num_buffer_funcs_scheds;
bool buffer_funcs_enabled;
/* @default_entity: for workarounds, has no gart windows */
@ -184,8 +185,8 @@ void amdgpu_ttm_unmark_vram_reserved(struct amdgpu_device *adev,
int amdgpu_ttm_init(struct amdgpu_device *adev);
void amdgpu_ttm_fini(struct amdgpu_device *adev);
void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev,
bool enable);
void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev);
void amdgpu_ttm_disable_buffer_funcs(struct amdgpu_device *adev);
int amdgpu_copy_buffer(struct amdgpu_device *adev,
struct amdgpu_ttm_buffer_entity *entity,
uint64_t src_offset,
@ -193,15 +194,12 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,
struct dma_resv *resv,
struct dma_fence **fence,
bool vm_needs_flush, uint32_t copy_flags);
int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity,
struct amdgpu_bo *bo,
struct dma_resv *resv,
struct dma_fence **fence);
int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
struct amdgpu_bo *bo,
uint32_t src_data,
struct dma_resv *resv,
struct dma_fence **f,
u64 k_job_id);
struct dma_fence **out_fence,
bool consider_clear_status,
u64 k_job_id);
struct amdgpu_ttm_buffer_entity *amdgpu_ttm_next_clear_entity(struct amdgpu_device *adev);
int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);

28
drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
View File

@ -32,29 +32,9 @@
#include "amdgpu.h"
#include "amdgpu_userq_fence.h"
static const struct dma_fence_ops amdgpu_userq_fence_ops;
static struct kmem_cache *amdgpu_userq_fence_slab;
#define AMDGPU_USERQ_MAX_HANDLES (1U << 16)
int amdgpu_userq_fence_slab_init(void)
{
amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
sizeof(struct amdgpu_userq_fence),
0,
SLAB_HWCACHE_ALIGN,
NULL);
if (!amdgpu_userq_fence_slab)
return -ENOMEM;
return 0;
}
void amdgpu_userq_fence_slab_fini(void)
{
rcu_barrier();
kmem_cache_destroy(amdgpu_userq_fence_slab);
}
static const struct dma_fence_ops amdgpu_userq_fence_ops;
static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
{
@ -231,7 +211,7 @@ void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
{
*userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
*userq_fence = kmalloc(sizeof(**userq_fence), GFP_KERNEL);
return *userq_fence ? 0 : -ENOMEM;
}
@ -342,7 +322,7 @@ static void amdgpu_userq_fence_free(struct rcu_head *rcu)
amdgpu_userq_fence_driver_put(fence_drv);
kvfree(userq_fence->fence_drv_array);
kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
kfree(userq_fence);
}
static void amdgpu_userq_fence_release(struct dma_fence *f)
@ -545,7 +525,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
if (r) {
mutex_unlock(&userq_mgr->userq_mutex);
kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
kfree(userq_fence);
goto put_gobj_write;
}

3
drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h
View File

@ -58,9 +58,6 @@ struct amdgpu_userq_fence_driver {
char timeline_name[TASK_COMM_LEN];
};
int amdgpu_userq_fence_slab_init(void);
void amdgpu_userq_fence_slab_fini(void);
void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv);
void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv);
int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,

123
drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
View File

@ -5,6 +5,7 @@
#include <drm/drm_simple_kms_helper.h>
#include <drm/drm_gem_framebuffer_helper.h>
#include <drm/drm_vblank.h>
#include <drm/drm_vblank_helper.h>
#include "amdgpu.h"
#ifdef CONFIG_DRM_AMDGPU_SI
@ -42,81 +43,6 @@ static const u32 amdgpu_vkms_formats[] = {
DRM_FORMAT_XRGB8888,
};
static enum hrtimer_restart amdgpu_vkms_vblank_simulate(struct hrtimer *timer)
{
struct amdgpu_crtc *amdgpu_crtc = container_of(timer, struct amdgpu_crtc, vblank_timer);
struct drm_crtc *crtc = &amdgpu_crtc->base;
struct amdgpu_vkms_output *output = drm_crtc_to_amdgpu_vkms_output(crtc);
u64 ret_overrun;
bool ret;
ret_overrun = hrtimer_forward_now(&amdgpu_crtc->vblank_timer,
output->period_ns);
if (ret_overrun != 1)
drm_warn(amdgpu_crtc->base.dev,
"%s: vblank timer overrun count: %llu\n",
__func__, ret_overrun);
ret = drm_crtc_handle_vblank(crtc);
/* Don't queue timer again when vblank is disabled. */
if (!ret)
return HRTIMER_NORESTART;
return HRTIMER_RESTART;
}
static int amdgpu_vkms_enable_vblank(struct drm_crtc *crtc)
{
struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
struct amdgpu_vkms_output *out = drm_crtc_to_amdgpu_vkms_output(crtc);
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
drm_calc_timestamping_constants(crtc, &crtc->mode);
out->period_ns = ktime_set(0, vblank->framedur_ns);
hrtimer_start(&amdgpu_crtc->vblank_timer, out->period_ns, HRTIMER_MODE_REL);
return 0;
}
static void amdgpu_vkms_disable_vblank(struct drm_crtc *crtc)
{
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
hrtimer_try_to_cancel(&amdgpu_crtc->vblank_timer);
}
static bool amdgpu_vkms_get_vblank_timestamp(struct drm_crtc *crtc,
int *max_error,
ktime_t *vblank_time,
bool in_vblank_irq)
{
struct amdgpu_vkms_output *output = drm_crtc_to_amdgpu_vkms_output(crtc);
struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
if (!READ_ONCE(vblank->enabled)) {
*vblank_time = ktime_get();
return true;
}
*vblank_time = READ_ONCE(amdgpu_crtc->vblank_timer.node.expires);
if (WARN_ON(*vblank_time == vblank->time))
return true;
/*
* To prevent races we roll the hrtimer forward before we do any
* interrupt processing - this is how real hw works (the interrupt is
* only generated after all the vblank registers are updated) and what
* the vblank core expects. Therefore we need to always correct the
* timestampe by one frame.
*/
*vblank_time -= output->period_ns;
return true;
}
static const struct drm_crtc_funcs amdgpu_vkms_crtc_funcs = {
.set_config = drm_atomic_helper_set_config,
.destroy = drm_crtc_cleanup,
@ -124,45 +50,11 @@ static const struct drm_crtc_funcs amdgpu_vkms_crtc_funcs = {
.reset = drm_atomic_helper_crtc_reset,
.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
.enable_vblank = amdgpu_vkms_enable_vblank,
.disable_vblank = amdgpu_vkms_disable_vblank,
.get_vblank_timestamp = amdgpu_vkms_get_vblank_timestamp,
DRM_CRTC_VBLANK_TIMER_FUNCS,
};
static void amdgpu_vkms_crtc_atomic_enable(struct drm_crtc *crtc,
struct drm_atomic_commit *state)
{
drm_crtc_vblank_on(crtc);
}
static void amdgpu_vkms_crtc_atomic_disable(struct drm_crtc *crtc,
struct drm_atomic_commit *state)
{
drm_crtc_vblank_off(crtc);
}
static void amdgpu_vkms_crtc_atomic_flush(struct drm_crtc *crtc,
struct drm_atomic_commit *state)
{
unsigned long flags;
if (crtc->state->event) {
spin_lock_irqsave(&crtc->dev->event_lock, flags);
if (drm_crtc_vblank_get(crtc) != 0)
drm_crtc_send_vblank_event(crtc, crtc->state->event);
else
drm_crtc_arm_vblank_event(crtc, crtc->state->event);
spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
crtc->state->event = NULL;
}
}
static const struct drm_crtc_helper_funcs amdgpu_vkms_crtc_helper_funcs = {
.atomic_flush = amdgpu_vkms_crtc_atomic_flush,
.atomic_enable = amdgpu_vkms_crtc_atomic_enable,
.atomic_disable = amdgpu_vkms_crtc_atomic_disable,
DRM_CRTC_HELPER_VBLANK_FUNCS,
};
static int amdgpu_vkms_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
@ -187,10 +79,6 @@ static int amdgpu_vkms_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
amdgpu_crtc->encoder = NULL;
amdgpu_crtc->connector = NULL;
amdgpu_crtc->vsync_timer_enabled = AMDGPU_IRQ_STATE_DISABLE;
hrtimer_setup(&amdgpu_crtc->vblank_timer, &amdgpu_vkms_vblank_simulate, CLOCK_MONOTONIC,
HRTIMER_MODE_REL);
return ret;
}
@ -538,11 +426,6 @@ static int amdgpu_vkms_sw_init(struct amdgpu_ip_block *ip_block)
static int amdgpu_vkms_sw_fini(struct amdgpu_ip_block *ip_block)
{
struct amdgpu_device *adev = ip_block->adev;
int i = 0;
for (i = 0; i < adev->mode_info.num_crtc; i++)
if (adev->mode_info.crtcs[i])
hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer);
drm_kms_helper_poll_fini(adev_to_drm(adev));
drm_mode_config_cleanup(adev_to_drm(adev));

2
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
View File

@ -1163,7 +1163,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
params.pages_addr = pages_addr;
params.unlocked = unlocked;
params.needs_flush = flush_tlb;
params.allow_override = allow_override;
params.override_pte = allow_override && adev->gmc.override_pte;
INIT_LIST_HEAD(&params.tlb_flush_waitlist);
amdgpu_vm_eviction_lock(vm);

6
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
View File

@ -296,10 +296,10 @@ struct amdgpu_vm_update_params {
bool needs_flush;
/**
* @allow_override: true for memory that is not uncached: allows MTYPE
* to be overridden for NUMA local memory.
* @override_pte: true for memory that is not uncached and gmc override function is
* implemented to allow MTYPE to be overridden for NUMA local memory.
*/
bool allow_override;
bool override_pte;
/**
* @tlb_flush_waitlist: temporary storage for BOs until tlb_flush

11
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
View File

@ -88,12 +88,21 @@ static int amdgpu_vm_cpu_update(struct amdgpu_vm_update_params *p,
trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags, p->immediate);
if (!p->pages_addr && p->override_pte)
amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, addr, &flags);
for (i = 0; i < count; i++) {
u64 oflags = flags;
value = p->pages_addr ?
amdgpu_vm_map_gart(p->pages_addr, addr) :
addr;
if (p->pages_addr && p->override_pte)
amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, value, &oflags);
amdgpu_gmc_set_pte_pde(p->adev, (void *)(uintptr_t)pe,
i, value, flags);
i, value, oflags);
addr += incr;
}
return 0;

9
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
View File

@ -710,15 +710,6 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
if (level == AMDGPU_VM_PTB)
amdgpu_vm_pte_update_noretry_flags(adev, &flags);
/* APUs mapping system memory may need different MTYPEs on different
* NUMA nodes. Only do this for contiguous ranges that can be assumed
* to be on the same NUMA node.
*/
if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
adev->gmc.gmc_funcs->override_vm_pte_flags &&
num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
flags);
}

11
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
View File

@ -257,6 +257,9 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
}
if (!p->pages_addr) {
if (p->override_pte)
amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, addr, &flags);
/* set page commands needed */
amdgpu_vm_sdma_set_ptes(p, bo, pe, addr, count,
incr, flags);
@ -275,8 +278,14 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
p->num_dw_left -= nptes * 2;
pte = (uint64_t *)&(p->job->ibs->ptr[p->num_dw_left]);
for (i = 0; i < nptes; ++i, addr += incr) {
u64 oflags = flags;
pte[i] = amdgpu_vm_map_gart(p->pages_addr, addr);
pte[i] |= flags;
if (p->override_pte)
amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, pte[i], &oflags);
pte[i] |= oflags;
}
amdgpu_vm_sdma_copy_ptes(p, bo, pe, nptes);

13
drivers/gpu/drm/amd/amdgpu/cik_sdma.c
View File

@ -939,7 +939,6 @@ static int cik_sdma_early_init(struct amdgpu_ip_block *ip_block)
cik_sdma_set_ring_funcs(adev);
cik_sdma_set_irq_funcs(adev);
cik_sdma_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &cik_sdma_vm_pte_funcs);
return 0;
@ -1000,8 +999,15 @@ static int cik_sdma_sw_fini(struct amdgpu_ip_block *ip_block)
static int cik_sdma_hw_init(struct amdgpu_ip_block *ip_block)
{
struct amdgpu_device *adev = ip_block->adev;
int r;
return cik_sdma_start(adev);
r = cik_sdma_start(adev);
if (r)
return r;
cik_sdma_set_buffer_funcs(adev);
return 0;
}
static int cik_sdma_hw_fini(struct amdgpu_ip_block *ip_block)
@ -1340,8 +1346,7 @@ static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = {
static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)
{
adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &cik_sdma_buffer_funcs);
}
const struct amdgpu_ip_block_version cik_sdma_ip_block =

5
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
View File

@ -7011,6 +7011,11 @@ static int gfx_v10_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
mqd->cp_hqd_ib_control = tmp;
tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
mqd->cp_hqd_quantum = tmp;
/* set static priority for a compute queue/ring */
mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;

59
drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
View File

@ -4417,6 +4417,11 @@ static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
mqd->cp_hqd_quantum = tmp;
mqd->cp_hqd_active = prop->hqd_active;
/* set UQ fenceaddress */
@ -5219,7 +5224,7 @@ static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
/**
* GFX soft reset will impact MES, need resume MES when do GFX soft reset
*/
return amdgpu_mes_resume(adev);
return amdgpu_mes_resume(adev, 0);
}
static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
@ -6227,56 +6232,6 @@ static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
ring->set_q_mode_offs = offs;
}
static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
{
int i, r = 0;
struct amdgpu_device *adev = ring->adev;
struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
struct amdgpu_ring *kiq_ring = &kiq->ring;
unsigned long flags;
if (adev->enable_mes)
return -EINVAL;
if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
return -EINVAL;
spin_lock_irqsave(&kiq->ring_lock, flags);
if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
spin_unlock_irqrestore(&kiq->ring_lock, flags);
return -ENOMEM;
}
/* assert preemption condition */
amdgpu_ring_set_preempt_cond_exec(ring, false);
/* assert IB preemption, emit the trailing fence */
kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
ring->trail_fence_gpu_addr,
++ring->trail_seq);
amdgpu_ring_commit(kiq_ring);
spin_unlock_irqrestore(&kiq->ring_lock, flags);
/* poll the trailing fence */
for (i = 0; i < adev->usec_timeout; i++) {
if (ring->trail_seq ==
le32_to_cpu(*(ring->trail_fence_cpu_addr)))
break;
udelay(1);
}
if (i >= adev->usec_timeout) {
r = -EINVAL;
DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
}
/* deassert preemption condition */
amdgpu_ring_set_preempt_cond_exec(ring, true);
return r;
}
static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
{
struct amdgpu_device *adev = ring->adev;
@ -7308,7 +7263,7 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
.emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
.emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
.init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
.preempt_ib = gfx_v11_0_ring_preempt_ib,
.preempt_ib = amdgpu_gfx_ring_preempt_ib,
.emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
.emit_wreg = gfx_v11_0_ring_emit_wreg,
.emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,

57
drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
View File

@ -3268,6 +3268,11 @@ static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
mqd->cp_hqd_quantum = tmp;
mqd->cp_hqd_active = prop->hqd_active;
/* set UQ fenceaddress */
@ -4611,56 +4616,6 @@ static unsigned gfx_v12_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
return ret;
}
static int gfx_v12_0_ring_preempt_ib(struct amdgpu_ring *ring)
{
int i, r = 0;
struct amdgpu_device *adev = ring->adev;
struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
struct amdgpu_ring *kiq_ring = &kiq->ring;
unsigned long flags;
if (adev->enable_mes)
return -EINVAL;
if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
return -EINVAL;
spin_lock_irqsave(&kiq->ring_lock, flags);
if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
spin_unlock_irqrestore(&kiq->ring_lock, flags);
return -ENOMEM;
}
/* assert preemption condition */
amdgpu_ring_set_preempt_cond_exec(ring, false);
/* assert IB preemption, emit the trailing fence */
kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
ring->trail_fence_gpu_addr,
++ring->trail_seq);
amdgpu_ring_commit(kiq_ring);
spin_unlock_irqrestore(&kiq->ring_lock, flags);
/* poll the trailing fence */
for (i = 0; i < adev->usec_timeout; i++) {
if (ring->trail_seq ==
le32_to_cpu(*(ring->trail_fence_cpu_addr)))
break;
udelay(1);
}
if (i >= adev->usec_timeout) {
r = -EINVAL;
DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
}
/* deassert preemption condition */
amdgpu_ring_set_preempt_cond_exec(ring, true);
return r;
}
static void gfx_v12_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
uint32_t reg_val_offs)
{
@ -5531,7 +5486,7 @@ static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_gfx = {
.pad_ib = amdgpu_ring_generic_pad_ib,
.emit_cntxcntl = gfx_v12_0_ring_emit_cntxcntl,
.init_cond_exec = gfx_v12_0_ring_emit_init_cond_exec,
.preempt_ib = gfx_v12_0_ring_preempt_ib,
.preempt_ib = amdgpu_gfx_ring_preempt_ib,
.emit_wreg = gfx_v12_0_ring_emit_wreg,
.emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
.emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,

99
drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
View File

@ -61,7 +61,7 @@
#define regCP_HQD_IB_CONTROL_DEFAULT 0x00100000
MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec.bin");
MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc.bin");
MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc_1.bin");
#define SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0 0x00000001
#define DEFAULT_SH_MEM_CONFIG \
@ -243,9 +243,9 @@ static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
amdgpu_ring_write(ring,
/* memory (1) or register (0) */
(WAIT_REG_MEM_MEM_SPACE(mem_space) |
WAIT_REG_MEM_OPERATION(opt) | /* wait */
WAIT_REG_MEM_FUNCTION(3))); /* equal */
(PACKET3_WAIT_REG_MEM__MEM_SPACE(mem_space) |
PACKET3_WAIT_REG_MEM__OPERATION(opt) | /* wait */
PACKET3_WAIT_REG_MEM__FUNCTION(3))); /* equal */
if (mem_space)
BUG_ON(addr0 & 0x3); /* Dword align */
@ -335,7 +335,7 @@ static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
}
ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
ib.ptr[1] = PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1);
ib.ptr[2] = lower_32_bits(gpu_addr);
ib.ptr[3] = upper_32_bits(gpu_addr);
ib.ptr[4] = 0xDEADBEEF;
@ -409,7 +409,13 @@ static int gfx_v12_1_init_microcode(struct amdgpu_device *adev)
amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
if (!amdgpu_sriov_vf(adev)) {
err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) &&
adev->rev_id == 0)
err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
AMDGPU_UCODE_REQUIRED,
"amdgpu/%s_rlc_1.bin", ucode_prefix);
else
err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
AMDGPU_UCODE_REQUIRED,
"amdgpu/%s_rlc.bin", ucode_prefix);
if (err)
@ -712,10 +718,19 @@ static void gfx_v12_1_select_me_pipe_q(struct amdgpu_device *adev,
soc_v1_0_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));
}
#define regGFX_IMU_PARTITION_SWITCH 0x5f8c
#define regGFX_IMU_PARTITION_SWITCH_BASE_IDX 1
#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP__SHIFT 0x2
#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP_MASK 0x0000003CL
static int gfx_v12_1_get_xccs_per_xcp(struct amdgpu_device *adev)
{
/* Fill this in when the interface is ready */
return 1;
u32 reg_data;
/* the register data is expected to be the same on all instances */
reg_data = RREG32_SOC15(GC, GET_INST(GC, 0), regGFX_IMU_PARTITION_SWITCH);
return REG_GET_FIELD(reg_data, GFX_IMU_PARTITION_SWITCH, TOTAL_XCCS_IN_XCP);
}
static int gfx_v12_1_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
@ -1136,6 +1151,7 @@ static int gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
{
uint16_t major_ver, minor_ver;
int i, j, k, r, ring_id = 0;
unsigned num_compute_rings;
int xcc_id, num_xcc;
@ -1146,6 +1162,15 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
adev->gfx.mec.num_mec = 1;
adev->gfx.mec.num_pipe_per_mec = 4;
adev->gfx.mec.num_queue_per_pipe = 8;
if (!amdgpu_discovery_get_gc_major_minor_version(
adev, &major_ver, &minor_ver)) {
if (major_ver == 1 && minor_ver == 3) {
adev->gfx.config.max_cu_per_sh /= 2;
dev_dbg(adev->dev, "Halving max_cu_per_sh for GC Discovery table v1:3 %d\n",
adev->gfx.config.max_cu_per_sh);
}
}
break;
default:
adev->gfx.mec.num_mec = 2;
@ -1450,6 +1475,19 @@ static void gfx_v12_1_get_tcc_info(struct amdgpu_device *adev)
{
}
static void gfx_v12_1_xcc_xnack_set_chicken_bits(struct amdgpu_device *adev, int xcc_id)
{
/* NOTE: COMPRESSION_ENABLE is used a chicken bit to enable/disable xcc xnack */
mutex_lock(&adev->srbm_mutex);
if (!adev->gmc.noretry) {
WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x1);
} else
WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x0);
mutex_unlock(&adev->srbm_mutex);
}
static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev,
int xcc_id)
{
@ -1477,6 +1515,8 @@ static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev,
mutex_unlock(&adev->srbm_mutex);
gfx_v12_1_xcc_init_compute_vmid(adev, xcc_id);
gfx_v12_1_xcc_xnack_set_chicken_bits(adev, xcc_id);
}
static void gfx_v12_1_constants_init(struct amdgpu_device *adev)
@ -2246,6 +2286,11 @@ static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m,
mqd->cp_mqd_stride_size = prop->mqd_stride_size ? prop->mqd_stride_size :
AMDGPU_MQD_SIZE_ALIGN(adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size);
tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
mqd->cp_hqd_quantum = tmp;
mqd->cp_hqd_active = prop->hqd_active;
return 0;
@ -3355,7 +3400,7 @@ static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring,
uint32_t flags)
{
unsigned vmid = AMDGPU_JOB_GET_VMID(job);
u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
u32 control = PACKET3_INDIRECT_BUFFER__VALID(1) | ib->length_dw | (vmid << 24);
/* Currently, there is a high possibility to get wave ID mismatch
* between ME and GDS, leading to a hw deadlock, because ME generates
@ -3391,15 +3436,15 @@ static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
/* RELEASE_MEM - flush caches, send int */
amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ(1) |
PACKET3_RELEASE_MEM_GCR_GLV_WB |
PACKET3_RELEASE_MEM_GCR_GL2_WB |
PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(2) |
PACKET3_RELEASE_MEM_TEMPORAL(3) |
PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__GCR_SEQ(1) |
PACKET3_RELEASE_MEM__GCR_GLV_WB |
PACKET3_RELEASE_MEM__GCR_GL2_WB |
PACKET3_RELEASE_MEM__GCR_GL2_SCOPE(2) |
PACKET3_RELEASE_MEM__TEMPORAL(3) |
PACKET3_RELEASE_MEM__EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
PACKET3_RELEASE_MEM__EVENT_INDEX(5)));
amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__DATA_SEL(write64bit ? 2 : 1) |
PACKET3_RELEASE_MEM__INT_SEL(int_sel ? 2 : 0)));
/*
* the address should be Qword aligned if 64bit write, Dword
@ -3460,7 +3505,7 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
/* write fence seq to the "addr" */
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
amdgpu_ring_write(ring, (WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1)));
amdgpu_ring_write(ring, lower_32_bits(addr));
amdgpu_ring_write(ring, upper_32_bits(addr));
amdgpu_ring_write(ring, lower_32_bits(seq));
@ -3468,7 +3513,7 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
if (flags & AMDGPU_FENCE_FLAG_INT) {
/* set register to trigger INT */
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
amdgpu_ring_write(ring, (WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(0) | PACKET3_WRITE_DATA__WR_CONFIRM(1)));
amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
amdgpu_ring_write(ring, 0);
amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
@ -3507,7 +3552,7 @@ static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring,
cmd = (1 << 16); /* no inc addr */
break;
default:
cmd = WR_CONFIRM;
cmd = PACKET3_WRITE_DATA__WR_CONFIRM(1);
break;
}
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
@ -3810,12 +3855,12 @@ static int gfx_v12_1_rlc_poison_irq(struct amdgpu_device *adev,
static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring)
{
const unsigned int gcr_cntl =
PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1) |
PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(2);
PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_INV(1) |
PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_WB(1) |
PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_INV(1) |
PACKET3_ACQUIRE_MEM__GCR_CNTL__GLK_INV(1) |
PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV(1) |
PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE(2);
/* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));

415
drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
View File

@ -58,11 +58,36 @@
#define PACKET3_DISPATCH_DIRECT 0x15
#define PACKET3_DISPATCH_INDIRECT 0x16
#define PACKET3_ATOMIC_MEM 0x1E
#define PACKET3_ATOMIC_MEM__ATOMIC(x) ((((unsigned)(x)) & 0x7F) << 0)
#define PACKET3_ATOMIC_MEM__COMMAND(x) ((((unsigned)(x)) & 0xF) << 8)
#define PACKET3_ATOMIC_MEM__SCOPE(x) ((((unsigned)(x)) & 0x3) << 23)
#define PACKET3_ATOMIC_MEM__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25)
#define PACKET3_ATOMIC_MEM__ADDR_LO(x) (((unsigned)(x)))
#define PACKET3_ATOMIC_MEM__ADDR_HI(x) (((unsigned)(x)))
#define PACKET3_ATOMIC_MEM__SRC_DATA_LO(x) (((unsigned)(x)))
#define PACKET3_ATOMIC_MEM__SRC_DATA_HI(x) (((unsigned)(x)))
#define PACKET3_ATOMIC_MEM__CMP_DATA_LO(x) (((unsigned)(x)))
#define PACKET3_ATOMIC_MEM__CMP_DATA_HI(x) (((unsigned)(x)))
#define PACKET3_ATOMIC_MEM__LOOP_INTERVAL(x) ((((unsigned)(x)) & 0x1FFF) << 0)
#define PACKET3_ATOMIC_MEM__COMMAND__SINGLE_PASS_ATOMIC 0
#define PACKET3_ATOMIC_MEM__COMMAND__LOOP_UNTIL_COMPARE_SATISFIED 1
#define PACKET3_ATOMIC_MEM__COMMAND__WAIT_FOR_WRITE_CONFIRMATION 2
#define PACKET3_ATOMIC_MEM__COMMAND__SEND_AND_CONTINUE 3
#define PACKET3_ATOMIC_MEM__SCOPE__CU 0
#define PACKET3_ATOMIC_MEM__SCOPE__SE 1
#define PACKET3_ATOMIC_MEM__SCOPE__DEVICE 2
#define PACKET3_ATOMIC_MEM__SCOPE__SYSTEM 3
#define PACKET3_ATOMIC_MEM__TEMPORAL__RT 0
#define PACKET3_ATOMIC_MEM__TEMPORAL__NT 1
#define PACKET3_ATOMIC_MEM__TEMPORAL__FW 2
#define PACKET3_ATOMIC_MEM__TEMPORAL__UC 3
#define PACKET3_OCCLUSION_QUERY 0x1F
#define PACKET3_SET_PREDICATION 0x20
#define PACKET3_REG_RMW 0x21
#define PACKET3_COND_EXEC 0x22
#define PACKET3_PRED_EXEC 0x23
#define PACKET3_PRED_EXEC__EXEC_COUNT(x) ((((unsigned)(x)) & 0x3FFF) << 0)
#define PACKET3_PRED_EXEC__VIRTUALXCCID_SELECT(x) ((((unsigned)(x)) & 0xFF) << 24)
#define PACKET3_DRAW_INDIRECT 0x24
#define PACKET3_DRAW_INDEX_INDIRECT 0x25
#define PACKET3_INDEX_BASE 0x26
@ -74,7 +99,11 @@
#define PACKET3_DRAW_INDEX_MULTI_AUTO 0x30
#define PACKET3_DRAW_INDEX_OFFSET_2 0x35
#define PACKET3_WRITE_DATA 0x37
#define WRITE_DATA_DST_SEL(x) (((x) & 0xf) << 8)
#define PACKET3_WRITE_DATA__DST_SEL(x) ((((unsigned)(x)) & 0xF) << 8)
#define PACKET3_WRITE_DATA__DST_SEL__MEM_MAPPED_REGISTER 0
#define PACKET3_WRITE_DATA__DST_SEL__TC_L2 2
#define PACKET3_WRITE_DATA__DST_SEL__MEMORY 5
#define PACKET3_WRITE_DATA__DST_SEL__MEMORY_MAPPED_ADC_PERSISTENT_STATE 6
/* 0 - register
* 1 - reserved
* 2 - tc_l2
@ -83,27 +112,52 @@
* 5 - memory (same as tc_l2)
* 6 - memory_mapped_adc_persistent_state
*/
#define WRITE_DATA_SCOPE(x) (((x) & 0x3) << 12)
#define WRITE_DATA_MODE(x) (((x) & 0x3) << 14)
#define PACKET3_WRITE_DATA__SCOPE(x) ((((unsigned)(x)) & 0x3) << 12)
#define PACKET3_WRITE_DATA__SCOPE__CU 0
#define PACKET3_WRITE_DATA__SCOPE__SE 1
#define PACKET3_WRITE_DATA__SCOPE__DEVICE 2
#define PACKET3_WRITE_DATA__SCOPE__SYSTEM 3
#define PACKET3_WRITE_DATA__MODE(x) ((((unsigned)(x)) & 0x3) << 14)
#define PACKET3_WRITE_DATA__MODE__LOCAL_XCD 0
#define PACKET3_WRITE_DATA__MODE__REMOTE_OR_LOCAL_AID 1
#define PACKET3_WRITE_DATA__MODE__REMOTE_XCD 2
#define PACKET3_WRITE_DATA__MODE__REMOTE_MID 3
/* 0 - local xcd
* 1 - remote/local aid
* 2 - remote xcd
* 3 - remote mid
*/
#define WRITE_DATA_ADDR_INCR (1 << 16)
#define WRITE_DATA_MID_DIE_ID(x) (((x) & 0x3) << 18)
#define WR_CONFIRM (1 << 20)
#define WRITE_DATA_XCD_DIE_ID(x) (((x) & 0xf) << 21)
#define WRITE_DATA_TEMPORAL(x) (((x) & 0x3) << 25)
/* 0 - rt
* 1 - nt
* 2 - ht
* 3 - lu
*/
#define WRITE_DATA_COOP_DISABLE (1 << 27)
#define PACKET3_WRITE_DATA__ADDR_INCR(x) ((((unsigned)(x)) & 0x1) << 16)
#define PACKET3_WRITE_DATA__ADDR_INCR__INCREMENT_ADDRESS 0
#define PACKET3_WRITE_DATA__ADDR_INCR__DO_NOT_INCREMENT_ADDRESS 1
#define PACKET3_WRITE_DATA__MID_DIE_ID(x) ((((unsigned)(x)) & 0x3) << 18)
#define PACKET3_WRITE_DATA__WR_CONFIRM(x) ((((unsigned)(x)) & 0x1) << 20)
#define PACKET3_WRITE_DATA__WR_CONFIRM__DO_NOT_WAIT_FOR_WRITE_CONFIRMATION 0
#define PACKET3_WRITE_DATA__WR_CONFIRM__WAIT_FOR_WRITE_CONFIRMATION 1
#define PACKET3_WRITE_DATA__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xF) << 21)
#define PACKET3_WRITE_DATA__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25)
#define PACKET3_WRITE_DATA__TEMPORAL__RT 0
#define PACKET3_WRITE_DATA__TEMPORAL__NT 1
#define PACKET3_WRITE_DATA__TEMPORAL__HT 2
#define PACKET3_WRITE_DATA__TEMPORAL__LU 3
#define PACKET3_WRITE_DATA__COOP_DISABLE(x) ((((unsigned)(x)) & 0x1) << 27)
#define PACKET3_WRITE_DATA__COOP_DISABLE__MASTER_AND_SLAVE_COOP 0
#define PACKET3_WRITE_DATA__COOP_DISABLE__MASTER_ONLY 1
#define PACKET3_WRITE_DATA__DST_MMREG_ADDR_LO(x) ((unsigned)(x))
#define PACKET3_WRITE_DATA__DST_MEM_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
#define PACKET3_WRITE_DATA__DST_MMREG_ADDR_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
#define PACKET3_WRITE_DATA__DST_MEM_ADDR_HI(x) ((unsigned)(x))
#define PACKET3_WRITE_DATA__DATA(x) ((unsigned)(x))
#define PACKET3_DRAW_INDEX_INDIRECT_MULTI 0x38
#define PACKET3_WAIT_REG_MEM 0x3C
#define WAIT_REG_MEM_FUNCTION(x) (((x) & 0x7) << 0)
#define PACKET3_WAIT_REG_MEM__FUNCTION(x) ((((unsigned)(x)) & 0x7) << 0)
#define PACKET3_WAIT_REG_MEM__FUNCTION__ALWAYS_PASS 0
#define PACKET3_WAIT_REG_MEM__FUNCTION__LESS_THAN_REF_VALUE 1
#define PACKET3_WAIT_REG_MEM__FUNCTION__LESS_THAN_EQUAL_TO_THE_REF_VALUE 2
#define PACKET3_WAIT_REG_MEM__FUNCTION__EQUAL_TO_THE_REFERENCE_VALUE 3
#define PACKET3_WAIT_REG_MEM__FUNCTION__NOT_EQUAL_REFERENCE_VALUE 4
#define PACKET3_WAIT_REG_MEM__FUNCTION__GREATER_THAN_OR_EQUAL_REFERENCE_VALUE 5
#define PACKET3_WAIT_REG_MEM__FUNCTION__GREATER_THAN_REFERENCE_VALUE 6
/* 0 - always
* 1 - <
* 2 - <=
@ -112,117 +166,222 @@
* 5 - >=
* 6 - >
*/
#define WAIT_REG_MEM_MEM_SPACE(x) (((x) & 0x3) << 4)
#define PACKET3_WAIT_REG_MEM__MEM_SPACE(x) ((((unsigned)(x)) & 0x3) << 4)
#define PACKET3_WAIT_REG_MEM__MEM_SPACE__REGISTER_SPACE 0
#define PACKET3_WAIT_REG_MEM__MEM_SPACE__MEMORY_SPACE 1
/* 0 - reg
* 1 - mem
*/
#define WAIT_REG_MEM_OPERATION(x) (((x) & 0x3) << 6)
#define PACKET3_WAIT_REG_MEM__OPERATION(x) ((((unsigned)(x)) & 0x3) << 6)
#define PACKET3_WAIT_REG_MEM__OPERATION__WAIT_REG_MEM 0
#define PACKET3_WAIT_REG_MEM__OPERATION__WR_WAIT_WR_REG 1
#define PACKET3_WAIT_REG_MEM__OPERATION__WAIT_MEM_PREEMPTABLE 3
/* 0 - wait_reg_mem
* 1 - wr_wait_wr_reg
* 2 - reserved
* 3 - wait_mem_preemptable
*/
#define WAIT_REG_MEM_MODE(x) (((x) & 0x3) << 10)
#define PACKET3_WAIT_REG_MEM__MODE(x) ((((unsigned)(x)) & 0x3) << 10)
#define PACKET3_WAIT_REG_MEM__MODE__LOCAL_XCD 0
#define PACKET3_WAIT_REG_MEM__MODE__REMOTE_OR_LOCAL_AID 1
#define PACKET3_WAIT_REG_MEM__MODE__REMOTE_XCD 2
#define PACKET3_WAIT_REG_MEM__MODE__REMOTE_MID 3
/* 0 - local xcd
* 1 - remote/local aid
* 2 - remote xcd
* 3 - remote mid
*/
#define WAIT_REG_MEM_MID_DIE_ID(x) (((x) & 0x3) << 12)
#define WAIT_REG_MEM_XCD_DIE_ID(x) (((x) & 0xf) << 14)
#define WAIT_REG_MEM_MES_INTR_PIPE(x) (((x) & 0x3) << 22)
#define WAIT_REG_MEM_MES_ACTION(x) (((x) & 0x1) << 24)
#define WAIT_REG_MEM_TEMPORAL(x) (((x) & 0x3) << 25)
#define PACKET3_WAIT_REG_MEM__MID_DIE_ID(x) ((((unsigned)(x)) & 0x3) << 12)
#define PACKET3_WAIT_REG_MEM__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xf) << 14)
#define PACKET3_WAIT_REG_MEM__MES_INTR_PIPE(x) ((((unsigned)(x)) & 0x3) << 22)
#define PACKET3_WAIT_REG_MEM__MES_ACTION(x) ((((unsigned)(x)) & 0x1) << 24)
#define PACKET3_WAIT_REG_MEM__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25)
#define PACKET3_WAIT_REG_MEM__TEMPORAL__RT 0
#define PACKET3_WAIT_REG_MEM__TEMPORAL__NT 1
#define PACKET3_WAIT_REG_MEM__TEMPORAL__HT 2
#define PACKET3_WAIT_REG_MEM__TEMPORAL__LU 3
/* 0 - rt
* 1 - nt
* 2 - ht
* 3 - lu
*/
#define PACKET3_WAIT_REG_MEM__MEM_POLL_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
#define PACKET3_WAIT_REG_MEM__REG_POLL_ADDR(x) ((unsigned)(x))
#define PACKET3_WAIT_REG_MEM__REG_WRITE_ADDR1(x) ((((unsigned)(x)) & 0x3FFFF) << 0)
#define PACKET3_WAIT_REG_MEM__MEM_POLL_ADDR_HI(x) ((unsigned)(x))
#define PACKET3_WAIT_REG_MEM__REG_POLL_ADDR_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
#define PACKET3_WAIT_REG_MEM__REG_WRITE_ADDR2(x) ((((unsigned)(x)) & 0x3FFFF) << 0)
#define PACKET3_WAIT_REG_MEM__REFERENCE(x) ((unsigned)(x))
#define PACKET3_WAIT_REG_MEM__MASK(x) ((unsigned)(x))
#define PACKET3_WAIT_REG_MEM__POLL_INTERVAL(x) ((((unsigned)(x)) & 0xFFFF) << 0)
#define PACKET3_WAIT_REG_MEM__OPTIMIZE_ACE_OFFLOAD_MODE(x) ((((unsigned)(x)) & 0x1) << 31)
#define PACKET3_INDIRECT_BUFFER 0x3F
#define INDIRECT_BUFFER_VALID (1 << 23)
#define INDIRECT_BUFFER_TEMPORAL(x) (x) << 28)
/* 0 - rt
* 1 - nt
* 2 - ht
* 3 - lu
*/
#define PACKET3_INDIRECT_BUFFER__IB_BASE_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
#define PACKET3_INDIRECT_BUFFER__IB_BASE_HI(x) ((unsigned)(x))
#define PACKET3_INDIRECT_BUFFER__IB_SIZE(x) ((((unsigned)(x)) & 0xFFFFF) << 0)
#define PACKET3_INDIRECT_BUFFER__CHAIN(x) ((((unsigned)(x)) & 0x1) << 20)
#define PACKET3_INDIRECT_BUFFER__OFFLOAD_POLLING(x) ((((unsigned)(x)) & 0x1) << 21)
#define PACKET3_INDIRECT_BUFFER__VALID(x) ((((unsigned)(x)) & 0x1) << 23)
#define PACKET3_INDIRECT_BUFFER__VMID(x) ((((unsigned)(x)) & 0xF) << 24)
#define PACKET3_INDIRECT_BUFFER__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 28)
#define PACKET3_INDIRECT_BUFFER__INHERIT_VMID(x) ((((unsigned)(x)) & 0x1) << 30)
#define PACKET3_INDIRECT_BUFFER__PRIV(x) ((((unsigned)(x)) & 0x1) << 31)
#define PACKET3_INDIRECT_BUFFER__TEMPORAL__RT 0
#define PACKET3_INDIRECT_BUFFER__TEMPORAL__NT 1
#define PACKET3_INDIRECT_BUFFER__TEMPORAL__HT 2
#define PACKET3_INDIRECT_BUFFER__TEMPORAL__LU 3
#define PACKET3_COND_INDIRECT_BUFFER 0x3F
#define PACKET3_COPY_DATA 0x40
#define COPY_DATA_SRC_SEL(x) (((x) & 0xf) << 0)
#define COPY_DATA_DST_SEL(x) (((x) & 0xf) << 8)
#define COPY_DATA_SRC_SCOPE(x) (((x) & 0x3) << 4)
#define COPY_DATA_DST_SCOPE(x) (((x) & 0x3) << 27)
#define COPY_DATA_MODE(x) (((x) & 0x3) << 6)
/* 0 - local xcd
* 1 - remote/local aid
* 2 - remote xcd
* 3 - remote mid
*/
#define COPY_DATA_SRC_TEMPORAL(x) (((x) & 0x3) << 13)
#define COPY_DATA_DST_TEMPORAL(x) (((x) & 0x3) << 25)
/* 0 - rt
* 1 - nt
* 2 - ht
* 3 - lu
*/
#define COPY_DATA_COUNT_SEL (1 << 16)
#define COPY_DATA_SRC_DST_REMOTE_MODE(x) (((x)) & 0x1 << 16)
/* 0 - src remote
* 1 - dst remote
*/
#define COPY_DATA_MID_DIE_ID(x) (((x) & 0x3) << 18)
#define COPY_DATA_XCD_DIE_ID(x) (((x) & 0xf) << 21)
#define COPY_DATA_PQ_EXE_STATUS (1 << 27)
#define PACKET3_COPY_DATA__SRC_SEL(x) ((((unsigned)(x)) & 0xF) << 0)
#define PACKET3_COPY_DATA__SRC_SCOPE(x) ((((unsigned)(x)) & 0x3) << 4)
#define PACKET3_COPY_DATA__MODE(x) ((((unsigned)(x)) & 0x3) << 6)
#define PACKET3_COPY_DATA__DST_SEL(x) ((((unsigned)(x)) & 0xF) << 8)
#define PACKET3_COPY_DATA__SRC_TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 13)
#define PACKET3_COPY_DATA__COUNT_SEL(x) ((((unsigned)(x)) & 0x1) << 16)
#define PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE(x) ((((unsigned)(x)) & 0x1) << 17)
#define PACKET3_COPY_DATA__MID_DIE_ID(x) ((((unsigned)(x)) & 0x3) << 18)
#define PACKET3_COPY_DATA__WR_CONFIRM(x) ((((unsigned)(x)) & 0x1) << 20)
#define PACKET3_COPY_DATA__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xF) << 21)
#define PACKET3_COPY_DATA__DST_TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25)
#define PACKET3_COPY_DATA__DST_SCOPE(x) ((((unsigned)(x)) & 0x3) << 27)
#define PACKET3_COPY_DATA__PQ_EXE_STATUS(x) ((((unsigned)(x)) & 0x1) << 29)
#define PACKET3_COPY_DATA__SRC_REG_OFFSET_LO(x) ((unsigned)(x))
#define PACKET3_COPY_DATA__SRC_32B_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
#define PACKET3_COPY_DATA__SRC_64B_ADDR_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3)
#define PACKET3_COPY_DATA__IMM_DATA(x) ((unsigned)(x))
#define PACKET3_COPY_DATA__SRC_REG_OFFSET_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
#define PACKET3_COPY_DATA__SRC_MEMTC_ADDR_HI(x) ((unsigned)(x))
#define PACKET3_COPY_DATA__SRC_IMM_DATA(x) ((unsigned)(x))
#define PACKET3_COPY_DATA__DST_REG_OFFSET_LO(x) ((unsigned)(x))
#define PACKET3_COPY_DATA__DST_32B_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
#define PACKET3_COPY_DATA__DST_64B_ADDR_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3)
#define PACKET3_COPY_DATA__DST_REG_OFFSET_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
#define PACKET3_COPY_DATA__DST_ADDR_HI(x) ((unsigned)(x))
#define PACKET3_COPY_DATA__SRC_SEL__MEM_MAPPED_REGISTER 0
#define PACKET3_COPY_DATA__SRC_SEL__TC_L2_OBSOLETE 1
#define PACKET3_COPY_DATA__SRC_SEL__TC_L2 2
#define PACKET3_COPY_DATA__SRC_SEL__PERFCOUNTERS 4
#define PACKET3_COPY_DATA__SRC_SEL__IMMEDIATE_DATA 5
#define PACKET3_COPY_DATA__SRC_SEL__ATOMIC_RETURN_DATA 6
#define PACKET3_COPY_DATA__SRC_SEL__GPU_CLOCK_COUNT 9
#define PACKET3_COPY_DATA__SRC_SEL__SYSTEM_CLOCK_COUNT 10
#define PACKET3_COPY_DATA__SRC_SCOPE__CU 0
#define PACKET3_COPY_DATA__SRC_SCOPE__SE 1
#define PACKET3_COPY_DATA__SRC_SCOPE__DEVICE 2
#define PACKET3_COPY_DATA__SRC_SCOPE__SYSTEM 3
#define PACKET3_COPY_DATA__MODE__LOCAL_XCD 0
#define PACKET3_COPY_DATA__MODE__REMOTE_OR_LOCAL_AID 1
#define PACKET3_COPY_DATA__MODE__REMOTE_XCD 2
#define PACKET3_COPY_DATA__MODE__REMOTE_MID 3
#define PACKET3_COPY_DATA__DST_SEL__MEM_MAPPED_REGISTER 0
#define PACKET3_COPY_DATA__DST_SEL__TC_L2 2
#define PACKET3_COPY_DATA__DST_SEL__PERFCOUNTERS 4
#define PACKET3_COPY_DATA__DST_SEL__TC_L2_OBSOLETE 5
#define PACKET3_COPY_DATA__DST_SEL__MEM_MAPPED_REG_DC 6
#define PACKET3_COPY_DATA__SRC_TEMPORAL__RT 0
#define PACKET3_COPY_DATA__SRC_TEMPORAL__NT 1
#define PACKET3_COPY_DATA__SRC_TEMPORAL__HT 2
#define PACKET3_COPY_DATA__SRC_TEMPORAL__LU 3
#define PACKET3_COPY_DATA__COUNT_SEL__32_BITS_OF_DATA 0
#define PACKET3_COPY_DATA__COUNT_SEL__64_BITS_OF_DATA 1
#define PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE__SRC_IS_REMOTE 0
#define PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE__DST_IS_REMOTE 1
#define PACKET3_COPY_DATA__WR_CONFIRM__DO_NOT_WAIT_FOR_CONFIRMATION 0
#define PACKET3_COPY_DATA__WR_CONFIRM__WAIT_FOR_CONFIRMATION 1
#define PACKET3_COPY_DATA__DST_TEMPORAL__RT 0
#define PACKET3_COPY_DATA__DST_TEMPORAL__NT 1
#define PACKET3_COPY_DATA__DST_TEMPORAL__HT 2
#define PACKET3_COPY_DATA__DST_TEMPORAL__LU 3
#define PACKET3_COPY_DATA__DST_SCOPE__CU 0
#define PACKET3_COPY_DATA__DST_SCOPE__SE 1
#define PACKET3_COPY_DATA__DST_SCOPE__DEVICE 2
#define PACKET3_COPY_DATA__DST_SCOPE__SYSTEM 3
#define PACKET3_COPY_DATA__PQ_EXE_STATUS__DEFAULT 0
#define PACKET3_COPY_DATA__PQ_EXE_STATUS__PHASE_UPDATE 1
#define PACKET3_PFP_SYNC_ME 0x42
#define PACKET3_COND_WRITE 0x45
#define PACKET3_EVENT_WRITE 0x46
#define EVENT_TYPE(x) ((x) << 0)
#define EVENT_INDEX(x) ((x) << 8)
/* 0 - any non-TS event
* 1 - ZPASS_DONE, PIXEL_PIPE_STAT_*
* 2 - SAMPLE_PIPELINESTAT
* 3 - SAMPLE_STREAMOUTSTAT*
* 4 - *S_PARTIAL_FLUSH
*/
#define PACKET3_EVENT_WRITE__EVENT_TYPE(x) ((((unsigned)(x)) & 0x3F) << 0)
#define PACKET3_EVENT_WRITE__EVENT_INDEX(x) ((((unsigned)(x)) & 0xF) << 8)
#define PACKET3_EVENT_WRITE__OFFLOAD_ENABLE(x) ((((unsigned)(x)) & 0x1) << 31)
#define PACKET3_EVENT_WRITE__ADDRESS_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3)
#define PACKET3_EVENT_WRITE__ADDRESS_HI(x) ((unsigned)(x))
#define PACKET3_EVENT_WRITE__EVENT_INDEX__OTHER 0
#define PACKET3_EVENT_WRITE__EVENT_INDEX__SAMPLE_PIPELINESTAT 2
#define PACKET3_EVENT_WRITE__EVENT_INDEX__CS_PARTIAL_FLUSH 4
#define PACKET3_EVENT_WRITE_EOP 0x47
#define PACKET3_EVENT_WRITE_EOS 0x48
#define PACKET3_RELEASE_MEM 0x49
#define PACKET3_RELEASE_MEM_EVENT_TYPE(x) ((x) << 0)
#define PACKET3_RELEASE_MEM_EVENT_INDEX(x) ((x) << 8)
#define PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(x) ((x) << 12)
#define PACKET3_RELEASE_MEM_GCR_GLV_INV (1 << 14)
#define PACKET3_RELEASE_MEM_GCR_GL2_US (1 << 16)
#define PACKET3_RELEASE_MEM_GCR_GL2_RANGE(x) ((x) << 17)
#define PACKET3_RELEASE_MEM_GCR_GL2_DISCARD (1 << 19)
#define PACKET3_RELEASE_MEM_GCR_GL2_INV (1 << 20)
#define PACKET3_RELEASE_MEM_GCR_GL2_WB (1 << 21)
#define PACKET3_RELEASE_MEM_GCR_SEQ(x) ((x) << 22)
#define PACKET3_RELEASE_MEM_GCR_GLV_WB (1 << 24)
#define PACKET3_RELEASE_MEM_TEMPORAL(x) ((x) << 25)
#define PACKET3_RELEASE_MEM__EVENT_TYPE(x) ((((unsigned)(x)) & 0x3F) << 0)
#define PACKET3_RELEASE_MEM__WAIT_SYNC(x) ((((unsigned)(x)) & 0x1) << 7)
#define PACKET3_RELEASE_MEM__EVENT_INDEX(x) ((((unsigned)(x)) & 0xF) << 8)
#define PACKET3_RELEASE_MEM__EVENT_INDEX__END_OF_PIPE 5
#define PACKET3_RELEASE_MEM__EVENT_INDEX__SHADER_DONE 6
#define PACKET3_RELEASE_MEM__GCR_CNTL(x) ((((unsigned)(x)) & 0x1FFF) << 12)
#define PACKET3_RELEASE_MEM__GCR_GL2_SCOPE(x) ((x) << 12)
#define PACKET3_RELEASE_MEM__GCR_GLV_INV (1 << 14)
#define PACKET3_RELEASE_MEM__GCR_GL2_US (1 << 16)
#define PACKET3_RELEASE_MEM__GCR_GL2_RANGE(x) ((x) << 17)
#define PACKET3_RELEASE_MEM__GCR_GL2_DISCARD (1 << 19)
#define PACKET3_RELEASE_MEM__GCR_GL2_INV (1 << 20)
#define PACKET3_RELEASE_MEM__GCR_GL2_WB (1 << 21)
#define PACKET3_RELEASE_MEM__GCR_SEQ(x) ((x) << 22)
#define PACKET3_RELEASE_MEM__GCR_GLV_WB (1 << 24)
#define PACKET3_RELEASE_MEM__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25)
#define PACKET3_RELEASE_MEM__TEMPORAL__RT 0
#define PACKET3_RELEASE_MEM__TEMPORAL__NT 1
#define PACKET3_RELEASE_MEM__TEMPORAL__HT 2
#define PACKET3_RELEASE_MEM__TEMPORAL__LU 3
/* 0 - temporal__release_mem__rt
* 1 - temporal__release_mem__nt
* 2 - temporal__release_mem__ht
* 3 - temporal__release_mem__lu
*/
#define PACKET3_RELEASE_MEM_PQ_EXE_STATUS (1 << 28)
#define PACKET3_RELEASE_MEM_GCR_GLK_INV (1 << 30)
#define PACKET3_RELEASE_MEM__PQ_EXE_STATUS(x) ((((unsigned)(x)) & 0x1) << 28)
#define PACKET3_RELEASE_MEM__PQ_EXE_STATUS__DEFAULT 0
#define PACKET3_RELEASE_MEM__PQ_EXE_STATUS__PHASE_UPDATE 1
#define PACKET3_RELEASE_MEM__GCR_GLK_INV (1 << 30)
#define PACKET3_RELEASE_MEM_DST_SEL(x) ((x) << 16)
/* 0 - memory controller
* 1 - TC/L2
* 2 - register
*/
#define PACKET3_RELEASE_MEM_MES_INTR_PIPE(x) ((x) << 20)
#define PACKET3_RELEASE_MEM_MES_ACTION_ID(x) ((x) << 22)
#define PACKET3_RELEASE_MEM_INT_SEL(x) ((x) << 24)
/* 0 - none
* 1 - interrupt only (DATA_SEL = 0)
* 2 - interrupt when data write is confirmed
*/
#define PACKET3_RELEASE_MEM_ADD_DOOREBLL_OFFSET(x) (1 << 28)
#define PACKET3_RELEASE_MEM_DATA_SEL(x) ((x) << 29)
#define PACKET3_RELEASE_MEM__DST_SEL(x) ((((unsigned)(x)) & 0x3) << 16)
#define PACKET3_RELEASE_MEM__DST_SEL__MEMORY_CONTROLLER 0
#define PACKET3_RELEASE_MEM__DST_SEL__TC_L2 1
#define PACKET3_RELEASE_MEM__DST_SEL__QUQUE_WRITE_POINTER_REGISTER 2
#define PACKET3_RELEASE_MEM__DST_SEL__QUQUE_WRITE_POINTER_POLL_MASK_BIT 3
#define PACKET3_RELEASE_MEM__MES_INTR_PIPE(x) ((((unsigned)(x)) & 0x3) << 20)
#define PACKET3_RELEASE_MEM__MES_ACTION_ID(x) ((((unsigned)(x)) & 0x3) << 22)
#define PACKET3_RELEASE_MEM__MES_ACTION_ID__NO_MES_NOTIFICATION 0
#define PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_AND_FENCE 1
#define PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_NO_FENCE_THEN_ADDRESS_PAYLOAD 2
#define PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_AND_ADDRESS_PAYLOAD 3
#define PACKET3_RELEASE_MEM__INT_SEL(x) ((((unsigned)(x)) & 0x7) << 24)
#define PACKET3_RELEASE_MEM__INT_SEL__NONE 0
#define PACKET3_RELEASE_MEM__INT_SEL__SEND_INTERRUPT_ONLY 1
#define PACKET3_RELEASE_MEM__INT_SEL__SEND_INTERRUPT_AFTER_WRITE_CONFIRM 2
#define PACKET3_RELEASE_MEM__INT_SEL__SEND_DATA_AND_WRITE_CONFIRM 3
#define PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID 4
#define PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID_BASED_ON_32_BIT_COMPARE 5
#define PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID_BASED_ON_64_BIT_COMPARE 6
#define PACKET3_RELEASE_MEM__ADD_DOOREBLL_OFFSET(x) ((((unsigned)(x)) & 0x1) << 28)
#define PACKET3_RELEASE_MEM__DATA_SEL(x) ((((unsigned)(x)) & 0x7) << 29)
#define PACKET3_RELEASE_MEM__DATA_SEL__NONE 0
#define PACKET3_RELEASE_MEM__DATA_SEL__SEND_32_BIT_LOW 1
#define PACKET3_RELEASE_MEM__DATA_SEL__SEND_64_BIT_DATA 2
#define PACKET3_RELEASE_MEM__DATA_SEL__SEND_GPU_CLOCK_COUNTER 3
#define PACKET3_RELEASE_MEM__DATA_SEL__SEND_SYSTEM_CLOCK_COUNTER 4
/* 0 - discard
* 1 - send low 32bit data
* 2 - send 64bit data
* 3 - send 64bit GPU counter value
* 4 - send 64bit sys counter value
*/
#define PACKET3_RELEASE_MEM__ADDRESS_LO_32B(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
#define PACKET3_RELEASE_MEM__ADDRESS_LO_64B(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3)
#define PACKET3_RELEASE_MEM__ADDRESS_HI(x) ((unsigned)(x))
#define PACKET3_RELEASE_MEM__DATA_LO(x) ((unsigned)(x))
#define PACKET3_RELEASE_MEM__CMP_DATA_LO(x) ((unsigned)(x))
#define PACKET3_RELEASE_MEM__DATA_HI(x) ((unsigned)(x))
#define PACKET3_RELEASE_MEM__CMP_DATA_HI(x) ((unsigned)(x))
#define PACKET3_RELEASE_MEM__INT_CTXID(x) ((unsigned)(x))
#define PACKET3_PREAMBLE_CNTL 0x4A
# define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE (2 << 28)
@ -279,58 +438,78 @@
# define PACKET3_DMA_DATA_CMD_DIS_WC (1 << 30)
#define PACKET3_CONTEXT_REG_RMW 0x51
#define PACKET3_ACQUIRE_MEM 0x58
/* 1. HEADER
* 2. COHER_CNTL [30:0]
* 2.1 ENGINE_SEL [31:31]
* 2. COHER_SIZE [31:0]
* 3. COHER_SIZE_HI [7:0]
* 4. COHER_BASE_LO [31:0]
* 5. COHER_BASE_HI [23:0]
* 7. POLL_INTERVAL [15:0]
* 8. GCR_CNTL [18:0]
*/
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(x) ((x) << 0)
/* 1. HEADER */
#define PACKET3_ACQUIRE_MEM__COHER_SIZE(x) ((unsigned)(x))
/* 3. COHER_SIZE [31:0] */
#define PACKET3_ACQUIRE_MEM__COHER_SIZE_HI(x) ((((unsigned)(x)) & 0xFF) << 0)
/* 4. COHER_SIZE_HI [7:0] */
#define PACKET3_ACQUIRE_MEM__COHER_BASE_LO(x) ((unsigned)(x))
/* 5. COHER_BASE_LO [31:0] */
#define PACKET3_ACQUIRE_MEM__COHER_BASE_HI(x) ((((unsigned)(x)) & 0xFFFFFF) << 0)
/* 6. COHER_BASE_HI [23:0] */
#define PACKET3_ACQUIRE_MEM__POLL_INTERVAL(x) ((((unsigned)(x)) & 0xFFFF) << 0)
/* 7. POLL_INTERVAL [15:0] */
#define PACKET3_ACQUIRE_MEM__GCR_CNTL(x) ((((unsigned)(x)) & 0x7FFFF) << 0)
/* 8. GCR_CNTL [18:0] */
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV(x) ((x) << 0)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__NOP 0
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__ALL 1
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__RANGE 2
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__FIRST_LAST 3
/*
* 0:NOP
* 1:ALL
* 2:RANGE
* 3:FIRST_LAST
*/
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_RANGE(x) ((x) << 2)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE(x) ((x) << 2)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__ALL 0
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__RANGE 2
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__FIRST_LAST 3
/*
* 0:ALL
* 1:reserved
* 2:RANGE
* 3:FIRST_LAST
*/
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(x) ((x) << 4)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE(x) ((x) << 4)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__DEVICE 0
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__SYSTEM 1
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__FORCE_ALL 2
/*
* 0:Device scope
* 1:System scope
* 2:Force INV/WB all
* 3:Reserved
*/
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_WB(x) ((x) << 6)
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(x) ((x) << 7)
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(x) ((x) << 8)
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_US(x) ((x) << 10)
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_RANGE(x) ((x) << 11)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_WB(x) ((x) << 6)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLK_INV(x) ((x) << 7)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_INV(x) ((x) << 8)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_US(x) ((x) << 10)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE(x) ((x) << 11)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__ALL 0
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__VOL 1
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__RANGE 2
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__FIRST_LAST 3
/*
* 0:ALL
* 1:VOL
* 2:RANGE
* 3:FIRST_LAST
*/
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_DISCARD(x) ((x) << 13)
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(x) ((x) << 14)
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(x) ((x) << 15)
#define PACKET3_ACQUIRE_MEM_GCR_CNTL_SEQ(x) ((x) << 16)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_DISCARD(x) ((x) << 13)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_INV(x) ((x) << 14)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_WB(x) ((x) << 15)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ(x) ((x) << 16)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__PARALLET 0
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__FORWARD 1
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__REVERSE 2
/*
* 0: PARALLEL
* 1: FORWARD
* 2: REVERSE
*/
#define PACKET3_ACQUIRE_MEM_GCR_RANGE_IS_PA (1 << 18)
#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GCR_RANGE_IS_PA (1 << 18)
#define PACKET3_GEN_PDEPTE 0x5B
#define PACKET3_PRIME_UTCL2 0x5D
#define PACKET3_LOAD_UCONFIG_REG 0x5E
@ -348,11 +527,19 @@
#define PACKET3_SET_SH_REG 0x76
#define PACKET3_SET_SH_REG_START 0x00002c00
#define PACKET3_SET_SH_REG_END 0x00003000
#define PACKET3_SET_SH_REG__REG_OFFSET(x) ((((unsigned)(x)) & 0xFFFF) << 0)
#define PACKET3_SET_SH_REG__VMID_SHIFT(x) ((((unsigned)(x)) & 0x1F) << 23)
#define PACKET3_SET_SH_REG__INDEX(x) ((((unsigned)(x)) & 0xF) << 28)
#define PACKET3_SET_SH_REG__REG_DATA(x) (((unsigned)(x)))
#define PACKET3_SET_SH_REG__INDEX__DEFAULT 0
#define PACKET3_SET_SH_REG__INDEX__INSERT_VMID 1
#define PACKET3_SET_SH_REG_OFFSET 0x77
#define PACKET3_SET_QUEUE_REG 0x78
#define PACKET3_SET_UCONFIG_REG 0x79
#define PACKET3_SET_UCONFIG_REG_START 0x0000c000
#define PACKET3_SET_UCONFIG_REG_END 0x0000c400
#define PACKET3_SET_UCONFIG_REG__REG_OFFSET(x) ((((unsigned)(x)) & 0xFFFF) << 0)
#define PACKET3_SET_UCONFIG_REG__REG_DATA(x) (((unsigned)(x)))
#define PACKET3_SET_UCONFIG_REG_INDEX 0x7A
#define PACKET3_DISPATCH_DRAW_PREAMBLE 0x8C
#define PACKET3_DISPATCH_DRAW 0x8D

7
drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
View File

@ -2819,6 +2819,7 @@ static void gfx_v7_0_mqd_init(struct amdgpu_device *adev,
{
u64 hqd_gpu_addr;
u64 wb_gpu_addr;
u32 tmp;
/* init the mqd struct */
memset(mqd, 0, sizeof(struct cik_mqd));
@ -2923,7 +2924,11 @@ static void gfx_v7_0_mqd_init(struct amdgpu_device *adev,
mqd->cp_hqd_atomic1_preop_lo = RREG32(mmCP_HQD_ATOMIC1_PREOP_LO);
mqd->cp_hqd_atomic1_preop_hi = RREG32(mmCP_HQD_ATOMIC1_PREOP_HI);
mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
tmp = RREG32(mmCP_HQD_QUANTUM);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10);
mqd->cp_hqd_quantum = tmp;
mqd->cp_hqd_pipe_priority = RREG32(mmCP_HQD_PIPE_PRIORITY);
mqd->cp_hqd_queue_priority = RREG32(mmCP_HQD_QUEUE_PRIORITY);
mqd->cp_hqd_iq_rptr = RREG32(mmCP_HQD_IQ_RPTR);

6
drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
View File

@ -4532,7 +4532,11 @@ static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
/* set static priority for a queue/ring */
gfx_v8_0_mqd_set_priority(ring, mqd);
mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
tmp = RREG32(mmCP_HQD_QUANTUM);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10);
mqd->cp_hqd_quantum = tmp;
/* map_queues packet doesn't need activate the queue,
* so only kiq need set this field.

9
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
View File

@ -3667,7 +3667,11 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
/* set static priority for a queue/ring */
gfx_v9_0_mqd_set_priority(ring, mqd);
mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
tmp = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
mqd->cp_hqd_quantum = tmp;
/* map_queues packet doesn't need activate the queue,
* so only kiq need set this field.
@ -5660,9 +5664,6 @@ static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
{
struct amdgpu_device *adev = ring->adev;
/* we only allocate 32bit for each seq wb address */
BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
/* write fence seq to the "addr" */
amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |

6
drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
View File

@ -1943,7 +1943,11 @@ static int gfx_v9_4_3_xcc_mqd_init(struct amdgpu_ring *ring, int xcc_id)
/* set static priority for a queue/ring */
gfx_v9_4_3_mqd_set_priority(ring, mqd);
mqd->cp_hqd_quantum = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_QUANTUM);
tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_QUANTUM);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
mqd->cp_hqd_quantum = tmp;
/* map_queues packet doesn't need activate the queue,
* so only kiq need set this field.

24
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
View File

@ -707,20 +707,16 @@ static int gmc_v10_0_mc_init(struct amdgpu_device *adev)
adev->gmc.visible_vram_size = adev->gmc.aper_size;
/* set the gart size */
if (amdgpu_gart_size == -1) {
switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
default:
adev->gmc.gart_size = 512ULL << 20;
break;
case IP_VERSION(10, 3, 1): /* DCE SG support */
case IP_VERSION(10, 3, 3): /* DCE SG support */
case IP_VERSION(10, 3, 6): /* DCE SG support */
case IP_VERSION(10, 3, 7): /* DCE SG support */
adev->gmc.gart_size = 1024ULL << 20;
break;
}
} else {
adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
case IP_VERSION(10, 3, 1): /* DCE SG support */
case IP_VERSION(10, 3, 3): /* DCE SG support */
case IP_VERSION(10, 3, 6): /* DCE SG support */
case IP_VERSION(10, 3, 7): /* DCE SG support */
amdgpu_gmc_set_gart_size(adev, SZ_1G);
break;
default:
amdgpu_gmc_set_gart_size(adev, SZ_512M);
break;
}
gmc_v10_0_vram_gtt_location(adev, &adev->gmc);

5
drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
View File

@ -709,10 +709,7 @@ static int gmc_v11_0_mc_init(struct amdgpu_device *adev)
adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
/* set the gart size */
if (amdgpu_gart_size == -1)
adev->gmc.gart_size = 512ULL << 20;
else
adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
amdgpu_gmc_set_gart_size(adev, SZ_512M);
gmc_v11_0_vram_gtt_location(adev, &adev->gmc);

11
drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
View File

@ -729,8 +729,10 @@ static int gmc_v12_0_mc_init(struct amdgpu_device *adev)
int r;
if (adev->gmc.xgmi.connected_to_cpu)
adev->gmc.mc_vram_size =
adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;
/* On A+A, manage driver allocation range to the local
* node segment and prevent allocations on remote HBM.
*/
adev->gmc.mc_vram_size = adev->gmc.xgmi.node_segment_size;
else
adev->gmc.mc_vram_size =
adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
@ -763,10 +765,7 @@ static int gmc_v12_0_mc_init(struct amdgpu_device *adev)
adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
/* set the gart size */
if (amdgpu_gart_size == -1) {
adev->gmc.gart_size = 512ULL << 20;
} else
adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
amdgpu_gmc_set_gart_size(adev, SZ_512M);
gmc_v12_0_vram_gtt_location(adev, &adev->gmc);

13
drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
View File

@ -112,6 +112,8 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
const char *hub_name;
int ret, xcc_id = 0;
uint32_t status = 0;
const char *die_name;
char die_name_buf[32];
u64 addr;
node_id = entry->node_id;
@ -201,6 +203,17 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
dev_err(adev->dev, " in page starting at address 0x%016llx from IH client %d (%s)\n",
addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]);
if (adev->irq.ih_funcs &&
adev->irq.ih_funcs->node_id_to_die_name) {
die_name = adev->irq.ih_funcs->node_id_to_die_name(adev, node_id,
die_name_buf,
sizeof(die_name_buf));
if (die_name)
dev_err(adev->dev,
" cookie node_id %d fault from die %s\n",
node_id, die_name);
}
if (amdgpu_sriov_vf(adev))
return 0;

28
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
View File

@ -328,24 +328,18 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
adev->gmc.visible_vram_size = adev->gmc.aper_size;
/* set the gart size */
if (amdgpu_gart_size == -1) {
switch (adev->asic_type) {
case CHIP_HAINAN: /* no MM engines */
default:
adev->gmc.gart_size = 256ULL << 20;
break;
case CHIP_VERDE: /* UVD, VCE do not support GPUVM */
case CHIP_TAHITI: /* UVD, VCE do not support GPUVM */
case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */
case CHIP_OLAND: /* UVD, VCE do not support GPUVM */
adev->gmc.gart_size = 1024ULL << 20;
break;
}
} else {
adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
switch (adev->asic_type) {
case CHIP_VERDE: /* UVD, VCE do not support GPUVM */
case CHIP_TAHITI: /* UVD, VCE do not support GPUVM */
case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */
case CHIP_OLAND: /* UVD, VCE do not support GPUVM */
amdgpu_gmc_set_gart_size(adev, SZ_1G);
break;
case CHIP_HAINAN: /* no MM engines */
default:
amdgpu_gmc_set_gart_size(adev, SZ_256M);
break;
}
adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
gmc_v6_0_vram_gtt_location(adev, &adev->gmc);
return 0;

30
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
View File

@ -394,27 +394,21 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
adev->gmc.visible_vram_size = adev->gmc.aper_size;
/* set the gart size */
if (amdgpu_gart_size == -1) {
switch (adev->asic_type) {
case CHIP_TOPAZ: /* no MM engines */
default:
adev->gmc.gart_size = 256ULL << 20;
break;
switch (adev->asic_type) {
#ifdef CONFIG_DRM_AMDGPU_CIK
case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
case CHIP_HAWAII: /* UVD, VCE do not support GPUVM */
case CHIP_KAVERI: /* UVD, VCE do not support GPUVM */
case CHIP_KABINI: /* UVD, VCE do not support GPUVM */
case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
adev->gmc.gart_size = 1024ULL << 20;
break;
case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
case CHIP_HAWAII: /* UVD, VCE do not support GPUVM */
case CHIP_KAVERI: /* UVD, VCE do not support GPUVM */
case CHIP_KABINI: /* UVD, VCE do not support GPUVM */
case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
amdgpu_gmc_set_gart_size(adev, SZ_1G);
break;
#endif
}
} else {
adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
case CHIP_TOPAZ: /* no MM engines */
default:
amdgpu_gmc_set_gart_size(adev, SZ_256M);
break;
}
adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
gmc_v7_0_vram_gtt_location(adev, &adev->gmc);
return 0;

34
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
View File

@ -585,27 +585,21 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
adev->gmc.visible_vram_size = adev->gmc.aper_size;
/* set the gart size */
if (amdgpu_gart_size == -1) {
switch (adev->asic_type) {
case CHIP_POLARIS10: /* all engines support GPUVM */
case CHIP_POLARIS11: /* all engines support GPUVM */
case CHIP_POLARIS12: /* all engines support GPUVM */
case CHIP_VEGAM: /* all engines support GPUVM */
default:
adev->gmc.gart_size = 256ULL << 20;
break;
case CHIP_TONGA: /* UVD, VCE do not support GPUVM */
case CHIP_FIJI: /* UVD, VCE do not support GPUVM */
case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */
case CHIP_STONEY: /* UVD does not support GPUVM, DCE SG support */
adev->gmc.gart_size = 1024ULL << 20;
break;
}
} else {
adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
switch (adev->asic_type) {
case CHIP_TONGA: /* UVD, VCE do not support GPUVM */
case CHIP_FIJI: /* UVD, VCE do not support GPUVM */
case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */
case CHIP_STONEY: /* UVD does not support GPUVM, DCE SG support */
amdgpu_gmc_set_gart_size(adev, SZ_1G);
break;
case CHIP_POLARIS10: /* all engines support GPUVM */
case CHIP_POLARIS11: /* all engines support GPUVM */
case CHIP_POLARIS12: /* all engines support GPUVM */
case CHIP_VEGAM: /* all engines support GPUVM */
default:
amdgpu_gmc_set_gart_size(adev, SZ_256M);
break;
}
adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
gmc_v8_0_vram_gtt_location(adev, &adev->gmc);
return 0;

75
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
View File

@ -1204,21 +1204,6 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
{
int local_node, nid;
/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system
* memory can use more efficient MTYPEs.
*/
if (!(adev->flags & AMD_IS_APU) ||
amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 3))
return;
/* Only direct-mapped memory allows us to determine the NUMA node from
* the DMA address.
*/
if (!adev->ram_is_direct_mapped) {
dev_dbg_ratelimited(adev->dev, "RAM is not direct mapped\n");
return;
}
/* MTYPE_NC is the same default and can be overridden.
* MTYPE_UC will be present if the memory is extended-coherent
* and can also be overridden.
@ -1231,11 +1216,7 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
return;
}
/* FIXME: Only supported on native mode for now. For carve-out, the
* NUMA affinity of the GPU/VM needs to come from the PCI info because
* memory partitions are not associated with different NUMA nodes.
*/
if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
if (vm->mem_id >= 0) {
local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
} else {
dev_dbg_ratelimited(adev->dev, "Only native mode APU is supported.\n");
@ -1344,6 +1325,20 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
{
adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes, local system
* memory can use more efficient MTYPEs.
*
* APUs mapping system memory may need different MTYPEs on different
* NUMA nodes.
*
* Only direct-mapped memory allows us to determine the NUMA node from
* the DMA address.
*/
adev->gmc.override_pte = adev->gmc.is_app_apu &&
num_possible_nodes() > 1 &&
amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&
adev->ram_is_direct_mapped;
}
static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
@ -1736,31 +1731,25 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
adev->gmc.visible_vram_size = adev->gmc.aper_size;
/* set the gart size */
if (amdgpu_gart_size == -1) {
switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
case IP_VERSION(9, 0, 1): /* all engines support GPUVM */
case IP_VERSION(9, 2, 1): /* all engines support GPUVM */
case IP_VERSION(9, 4, 0):
case IP_VERSION(9, 4, 1):
case IP_VERSION(9, 4, 2):
case IP_VERSION(9, 4, 3):
case IP_VERSION(9, 4, 4):
case IP_VERSION(9, 5, 0):
default:
adev->gmc.gart_size = 512ULL << 20;
break;
case IP_VERSION(9, 1, 0): /* DCE SG support */
case IP_VERSION(9, 2, 2): /* DCE SG support */
case IP_VERSION(9, 3, 0):
adev->gmc.gart_size = 1024ULL << 20;
break;
}
} else {
adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
case IP_VERSION(9, 1, 0): /* DCE SG support */
case IP_VERSION(9, 2, 2): /* DCE SG support */
case IP_VERSION(9, 3, 0):
amdgpu_gmc_set_gart_size(adev, SZ_1G);
break;
case IP_VERSION(9, 0, 1): /* all engines support GPUVM */
case IP_VERSION(9, 2, 1): /* all engines support GPUVM */
case IP_VERSION(9, 4, 0):
case IP_VERSION(9, 4, 1):
case IP_VERSION(9, 4, 2):
case IP_VERSION(9, 4, 3):
case IP_VERSION(9, 4, 4):
case IP_VERSION(9, 5, 0):
default:
amdgpu_gmc_set_gart_size(adev, SZ_512M);
break;
}
adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
return 0;

40
drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
View File

@ -798,6 +798,43 @@ static void ih_v7_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64
return;
}
/*
* ih_v7_0_node_id_to_die_name - Decode IH cookie node_id to a die name string
*
* Currently, only applies to IH v7_1. For other IH versions returns NULL.
*
* IH v7_1 node_id encoding:
* node_id[N:3] = MID index
* node_id[2:0] = sub-slot: 0=MID, 1=AID, 2-5=AID.XCD, 6-7=RSV
*/
static const char *ih_v7_0_node_id_to_die_name(struct amdgpu_device *adev,
unsigned int node_id,
char *buf, size_t size)
{
int mid_id, sub_slot;
/* Node ID to die name decoding is only defined for IH v7_1 currenlty. */
if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) != IP_VERSION(7, 1, 0))
return NULL;
mid_id = node_id >> 3;
sub_slot = node_id & 0x7;
if (mid_id > 1)
return "UNKNOWN";
if (sub_slot == 0)
snprintf(buf, size, "MID%d", mid_id);
else if (sub_slot == 1)
snprintf(buf, size, "AID%d", mid_id);
else if (sub_slot <= 5)
snprintf(buf, size, "AID%d.XCD%d", mid_id, sub_slot - 2);
else
snprintf(buf, size, "RSV");
return buf;
}
static const struct amd_ip_funcs ih_v7_0_ip_funcs = {
.name = "ih_v7_0",
.early_init = ih_v7_0_early_init,
@ -819,7 +856,8 @@ static const struct amdgpu_ih_funcs ih_v7_0_funcs = {
.get_wptr = ih_v7_0_get_wptr,
.decode_iv = amdgpu_ih_decode_iv_helper,
.decode_iv_ts = amdgpu_ih_decode_iv_ts_helper,
.set_rptr = ih_v7_0_set_rptr
.set_rptr = ih_v7_0_set_rptr,
.node_id_to_die_name = ih_v7_0_node_id_to_die_name,
};
static void ih_v7_0_set_interrupt_funcs(struct amdgpu_device *adev)

12
drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
View File

@ -140,27 +140,17 @@ static int imu_v12_1_switch_compute_partition(struct amdgpu_device *adev,
int ret;
if (adev->psp.funcs) {
/*TODO: revisit asp interface once it's avaialble */
ret = psp_spatial_partition(&adev->psp,
NUM_XCC(adev->gfx.xcc_mask) /
num_xccs_per_xcp);
ret = psp_spatial_partition(&adev->psp, compute_partition_mode);
if (ret)
return ret;
}
adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp;
return 0;
}
static void imu_v12_1_init_mcm_addr_lut(struct amdgpu_device *adev)
{
/* todo: fill in when interface is ready */
}
const struct amdgpu_imu_funcs gfx_v12_1_imu_funcs = {
.init_microcode = imu_v12_1_init_microcode,
.load_microcode = imu_v12_1_load_microcode,
.switch_compute_partition = imu_v12_1_switch_compute_partition,
.init_mcm_addr_lut = imu_v12_1_init_mcm_addr_lut,
};

2
drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
View File

@ -149,7 +149,7 @@ static int jpeg_v2_5_sw_init(struct amdgpu_ip_block *ip_block)
else
ring->vm_hub = AMDGPU_MMHUB0(0);
ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + 8 * i;
sprintf(ring->name, "jpeg_dec_%d", i);
snprintf(ring->name, sizeof(ring->name), "jpeg_dec_%d", i);
r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst[i].irq,
0, AMDGPU_RING_PRIO_DEFAULT, NULL);
if (r)

99
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
View File

@ -30,34 +30,6 @@
#define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
#define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
static int
mes_userq_map_gtt_bo_to_gart(struct amdgpu_bo *bo)
{
int ret;
ret = amdgpu_bo_reserve(bo, true);
if (ret) {
DRM_ERROR("Failed to reserve bo. ret %d\n", ret);
goto err_reserve_bo_failed;
}
ret = amdgpu_ttm_alloc_gart(&bo->tbo);
if (ret) {
DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret);
goto err_map_bo_gart_failed;
}
amdgpu_bo_unreserve(bo);
bo = amdgpu_bo_ref(bo);
return 0;
err_map_bo_gart_failed:
amdgpu_bo_unreserve(bo);
err_reserve_bo_failed:
return ret;
}
static int
mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
struct amdgpu_userq_mgr *uq_mgr,
@ -65,55 +37,62 @@ mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
uint64_t wptr)
{
struct amdgpu_bo_va_mapping *wptr_mapping;
struct amdgpu_vm *wptr_vm;
struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj;
struct amdgpu_bo *obj;
struct amdgpu_vm *vm = queue->vm;
struct drm_exec exec;
int ret;
wptr_vm = queue->vm;
ret = amdgpu_bo_reserve(wptr_vm->root.bo, false);
if (ret)
return ret;
wptr &= AMDGPU_GMC_HOLE_MASK;
wptr_mapping = amdgpu_vm_bo_lookup_mapping(wptr_vm, wptr >> PAGE_SHIFT);
amdgpu_bo_unreserve(wptr_vm->root.bo);
if (!wptr_mapping) {
DRM_ERROR("Failed to lookup wptr bo\n");
return -EINVAL;
drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
drm_exec_until_all_locked(&exec) {
ret = amdgpu_vm_lock_pd(vm, &exec, 1);
drm_exec_retry_on_contention(&exec);
if (unlikely(ret))
goto fail_lock;
wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT);
if (!wptr_mapping) {
ret = -EINVAL;
goto fail_lock;
}
obj = wptr_mapping->bo_va->base.bo;
ret = drm_exec_lock_obj(&exec, &obj->tbo.base);
drm_exec_retry_on_contention(&exec);
if (unlikely(ret))
goto fail_lock;
}
wptr_obj->obj = wptr_mapping->bo_va->base.bo;
wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo);
if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) {
DRM_ERROR("Requested GART mapping for wptr bo larger than one page\n");
return -EINVAL;
}
ret = mes_userq_map_gtt_bo_to_gart(wptr_obj->obj);
if (ret) {
DRM_ERROR("Failed to map wptr bo to GART\n");
return ret;
}
ret = amdgpu_bo_reserve(wptr_obj->obj, true);
if (ret) {
DRM_ERROR("Failed to reserve wptr bo\n");
return ret;
ret = -EINVAL;
goto fail_map;
}
/* TODO use eviction fence instead of pinning. */
ret = amdgpu_bo_pin(wptr_obj->obj, AMDGPU_GEM_DOMAIN_GTT);
if (ret) {
drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin wptr bo\n");
goto unresv_bo;
DRM_ERROR("Failed to pin wptr bo. ret %d\n", ret);
goto fail_map;
}
ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo);
if (ret) {
DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret);
goto fail_map;
}
queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);
amdgpu_bo_unreserve(wptr_obj->obj);
drm_exec_fini(&exec);
return 0;
unresv_bo:
amdgpu_bo_unreserve(wptr_obj->obj);
fail_map:
amdgpu_bo_unref(&wptr_obj->obj);
fail_lock:
drm_exec_fini(&exec);
return ret;
}
@ -266,7 +245,7 @@ static int mes_userq_detect_and_reset(struct amdgpu_device *adev,
if (found_hung_queue) {
/* Resume scheduling after hang recovery */
r = amdgpu_mes_resume(adev);
r = amdgpu_mes_resume(adev, input.xcc_id);
}
return r;

147
drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
View File

@ -44,8 +44,11 @@ static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block);
static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id);
static int mes_v12_1_setup_coop_mode(struct amdgpu_device *adev, int xcc_id);
#define MES_EOP_SIZE 2048
#define MES12_HUNG_DB_OFFSET_ARRAY_SIZE 8 /* [0:3] = db offset [4:7] hqd info */
#define MES12_HUNG_HQD_INFO_OFFSET 4
#define regCP_HQD_IB_CONTROL_MES_12_1_DEFAULT 0x100000
#define XCC_MID_MASK 0x41000000
@ -229,7 +232,7 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
xcc_id, pipe, x_pkt->header.opcode);
r = amdgpu_fence_wait_polling(ring, seq, timeout);
if (r < 1 || !*status_ptr) {
if (r < 1 || !lower_32_bits(*status_ptr)) {
if (misc_op_str)
dev_err(adev->dev,
"MES(%d, %d) failed to respond to msg=%s (%s)\n",
@ -468,13 +471,43 @@ static int mes_v12_1_unmap_legacy_queue(struct amdgpu_mes *mes,
static int mes_v12_1_suspend_gang(struct amdgpu_mes *mes,
struct mes_suspend_gang_input *input)
{
return 0;
union MESAPI__SUSPEND mes_suspend_gang_pkt;
memset(&mes_suspend_gang_pkt, 0, sizeof(mes_suspend_gang_pkt));
mes_suspend_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
mes_suspend_gang_pkt.header.opcode = MES_SCH_API_SUSPEND;
mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs;
mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr;
mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr;
mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value;
/* Suspend gang is handled by master MES */
return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE,
&mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt),
offsetof(union MESAPI__SUSPEND, api_status));
}
static int mes_v12_1_resume_gang(struct amdgpu_mes *mes,
struct mes_resume_gang_input *input)
{
return 0;
union MESAPI__RESUME mes_resume_gang_pkt;
memset(&mes_resume_gang_pkt, 0, sizeof(mes_resume_gang_pkt));
mes_resume_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
mes_resume_gang_pkt.header.opcode = MES_SCH_API_RESUME;
mes_resume_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs;
mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr;
/* Resume gang is handled by master MES */
return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE,
&mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt),
offsetof(union MESAPI__RESUME, api_status));
}
static int mes_v12_1_query_sched_status(struct amdgpu_mes *mes,
@ -621,11 +654,15 @@ static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes,
mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 100;
if (mes->enable_coop_mode && pipe == AMDGPU_MES_SCHED_PIPE) {
/* From version 0x74 above, pipe1 support use shared command buffer
to distribute some tasks on individual XCCs*/
if (mes->enable_coop_mode &&
((pipe == AMDGPU_MES_SCHED_PIPE) ||
((mes->kiq_version & AMDGPU_MES_VERSION_MASK) >= 0x74))) {
master_xcc_id = mes->master_xcc_ids[inst];
mes_set_hw_res_1_pkt.mes_coop_mode = 1;
mes_set_hw_res_1_pkt.coop_sch_shared_mc_addr =
mes->shared_cmd_buf_gpu_addr[master_xcc_id];
mes->shared_cmd_buf_gpu_addr[master_xcc_id + pipe];
}
return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
@ -633,21 +670,10 @@ static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes,
offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
}
static void mes_v12_1_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
{
/*
* GFX V12 has only one GFX pipe, but 8 queues in it.
* GFX pipe 0 queue 0 is being used by Kernel queue.
* Set GFX pipe 0 queue 1-7 for MES scheduling
* mask = 1111 1110b
*/
pkt->gfx_hqd_mask[0] = 0xFE;
}
static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
int pipe, int xcc_id)
{
int i;
int i, status;
struct amdgpu_device *adev = mes->adev;
union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
@ -667,7 +693,9 @@ static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
mes_set_hw_res_pkt.compute_hqd_mask[i] =
mes->compute_hqd_mask[i];
mes_v12_1_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
for (i = 0; i < MAX_GFX_PIPES; i++)
mes_set_hw_res_pkt.gfx_hqd_mask[i] =
mes->gfx_hqd_mask[i];
for (i = 0; i < MAX_SDMA_PIPES; i++)
mes_set_hw_res_pkt.sdma_hqd_mask[i] =
@ -715,9 +743,23 @@ static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
if (adev->enforce_isolation[0] == AMDGPU_ENFORCE_ISOLATION_ENABLE)
mes_set_hw_res_pkt.limit_single_process = 1;
return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
status = mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
&mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
/* get MES scheduler versions */
mutex_lock(&adev->srbm_mutex);
soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
if (pipe == AMDGPU_MES_SCHED_PIPE)
adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
mutex_unlock(&adev->srbm_mutex);
return status;
}
static void mes_v12_1_init_aggregated_doorbell(struct amdgpu_mes *mes,
@ -837,6 +879,33 @@ static int mes_v12_1_reset_legacy_queue(struct amdgpu_mes *mes,
}
#endif
static int mes_v12_1_detect_and_reset_hung_queues(struct amdgpu_mes *mes,
struct mes_detect_and_reset_queue_input *input)
{
union MESAPI__RESET mes_reset_queue_pkt;
memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
mes_reset_queue_pkt.queue_type =
convert_to_mes_queue_type(input->queue_type);
mes_reset_queue_pkt.doorbell_offset_addr =
mes->hung_queue_db_array_gpu_addr[0];
if (input->detect_only)
mes_reset_queue_pkt.hang_detect_only = 1;
else
mes_reset_queue_pkt.hang_detect_then_reset = 1;
return mes_v12_1_submit_pkt_and_poll_completion(mes,
input->xcc_id, AMDGPU_MES_SCHED_PIPE,
&mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
offsetof(union MESAPI__RESET, api_status));
}
static int mes_v12_inv_tlb_convert_hub_id(uint8_t id)
{
/*
@ -894,6 +963,7 @@ static const struct amdgpu_mes_funcs mes_v12_1_funcs = {
.resume_gang = mes_v12_1_resume_gang,
.misc_op = mes_v12_1_misc_op,
.reset_hw_queue = mes_v12_1_reset_hw_queue,
.detect_and_reset_hung_queues = mes_v12_1_detect_and_reset_hung_queues,
.invalidate_tlbs_pasid = mes_v12_1_inv_tlbs_pasid,
};
@ -1157,9 +1227,6 @@ static int mes_v12_1_allocate_shared_cmd_buf(struct amdgpu_device *adev,
{
int r, inst = MES_PIPE_INST(xcc_id, pipe);
if (pipe == AMDGPU_MES_KIQ_PIPE)
return 0;
r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
AMDGPU_GEM_DOMAIN_VRAM,
&adev->mes.shared_cmd_buf_obj[inst],
@ -1405,18 +1472,6 @@ static int mes_v12_1_queue_init(struct amdgpu_device *adev,
mes_v12_1_queue_init_register(ring, xcc_id);
}
/* get MES scheduler/KIQ versions */
mutex_lock(&adev->srbm_mutex);
soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
if (pipe == AMDGPU_MES_SCHED_PIPE)
adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
mutex_unlock(&adev->srbm_mutex);
return 0;
}
@ -1723,6 +1778,10 @@ static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
goto failure;
if (adev->enable_uni_mes) {
r = mes_v12_1_setup_coop_mode(adev, xcc_id);
if (r)
goto failure;
r = mes_v12_1_set_hw_resources(&adev->mes,
AMDGPU_MES_KIQ_PIPE, xcc_id);
if (r)
@ -1845,9 +1904,6 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id)
goto failure;
if (adev->enable_uni_mes) {
r = mes_v12_1_setup_coop_mode(adev, xcc_id);
if (r)
goto failure;
mes_v12_1_set_hw_resources_1(&adev->mes,
AMDGPU_MES_SCHED_PIPE, xcc_id);
}
@ -1897,24 +1953,12 @@ static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
static int mes_v12_1_suspend(struct amdgpu_ip_block *ip_block)
{
int r;
r = amdgpu_mes_suspend(ip_block->adev);
if (r)
return r;
return mes_v12_1_hw_fini(ip_block);
}
static int mes_v12_1_resume(struct amdgpu_ip_block *ip_block)
{
int r;
r = mes_v12_1_hw_init(ip_block);
if (r)
return r;
return amdgpu_mes_resume(ip_block->adev);
return mes_v12_1_hw_init(ip_block);
}
static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block)
@ -1922,6 +1966,9 @@ static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block)
struct amdgpu_device *adev = ip_block->adev;
int pipe, r;
adev->mes.hung_queue_db_array_size = MES12_HUNG_DB_OFFSET_ARRAY_SIZE;
adev->mes.hung_queue_hqd_info_offset = MES12_HUNG_HQD_INFO_OFFSET;
for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
r = amdgpu_mes_init_microcode(adev, pipe);
if (r)

369
drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c Normal file
View File

@ -0,0 +1,369 @@
/*
* Copyright 2025 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
#include "nbio/nbio_6_3_2_offset.h"
#include "nbio/nbio_6_3_2_sh_mask.h"
#include "amdgpu.h"
#include "nbio_v6_3_2.h"
static u32 nbio_v6_3_2_get_pcie_index_offset(struct amdgpu_device *adev)
{
return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2);
}
static u32 nbio_v6_3_2_get_pcie_data_offset(struct amdgpu_device *adev)
{
return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA2);
}
static u32 nbio_v6_3_2_get_pcie_index_hi_offset(struct amdgpu_device *adev)
{
return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2_HI);
}
static u32 nbio_v6_3_2_get_rev_id(struct amdgpu_device *adev)
{
u32 tmp;
/* TODO: RCC_STRAP0_RCC_DEV0_EPF0_STRAP0 is not accessible from
* guest side. It requires bootloader to update specific fields
* in ip discovery table to identify soc revision id.
* Return 0 when the function is called from guest side until
* bootloader change is available.
*/
if (amdgpu_sriov_vf(adev))
return 0;
tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0);
tmp = REG_GET_FIELD(tmp, RCC_STRAP0_RCC_DEV0_EPF0_STRAP0,
STRAP_ATI_REV_ID_DEV0_F0);
return tmp;
}
static void nbio_v6_3_2_mc_access_enable(struct amdgpu_device *adev,
bool enable)
{
if (enable)
WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN,
BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK | BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK);
else
WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0);
}
static void nbio_v6_3_2_init_registers(struct amdgpu_device *adev)
{
WREG32_SOC15(NBIO, 0, regXCD_DOORBELL_FENCE_1,
(0xff & ~(adev->gfx.xcc_mask)) <<
XCD_DOORBELL_FENCE_1__XCD_0_DOORBELL_DISABLE__SHIFT);
}
static u32 nbio_v6_3_2_get_memsize(struct amdgpu_device *adev)
{
return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE);
}
static void nbio_v6_3_2_enable_doorbell_aperture(struct amdgpu_device *adev,
bool enable)
{
/* Enable to allow doorbell pass thru on pre-silicon bare-metal */
WREG32_SOC15(NBIO, 0, regGDC0_DOORBELL_ACCESS_EN_PF, 0xfffff);
WREG32_FIELD15_PREREG(NBIO, 0, RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN,
BIF_DOORBELL_APER_EN, enable ? 1 : 0);
}
static void nbio_v6_3_2_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
bool enable)
{
u32 tmp = 0;
if (enable) {
tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
DOORBELL_SELFRING_GPA_APER_EN, 1) |
REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
DOORBELL_SELFRING_GPA_APER_MODE, 1) |
REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
DOORBELL_SELFRING_GPA_APER_SIZE, 0);
WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW,
lower_32_bits(adev->doorbell.base));
WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH,
upper_32_bits(adev->doorbell.base));
}
WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, tmp);
}
static void nbio_v6_3_2_enable_doorbell_interrupt(struct amdgpu_device *adev,
bool enable)
{
WREG32_FIELD15_PREREG(NBIO, 0, BIF_BX0_BIF_DOORBELL_INT_CNTL,
DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1);
}
static void nbio_v6_3_2_ih_control(struct amdgpu_device *adev)
{
u32 interrupt_cntl;
/* setup interrupt control */
WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL);
/*
* BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
* BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
*/
interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL,
IH_DUMMY_RD_OVERRIDE, 0);
/* BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL,
IH_REQ_NONSNOOP_EN, 0);
WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl);
}
static void nbio_v6_3_2_ih_doorbell_range(struct amdgpu_device *adev,
bool use_doorbell, int doorbell_index)
{
u32 ih_doorbell_range = 0;
u32 ih_doorbell_range1 = 0;
if (use_doorbell) {
ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
S2A_DOORBELL_PORT1_ENABLE,
0x1);
ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
S2A_DOORBELL_PORT1_AWID,
0x0);
ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
S2A_DOORBELL_PORT1_RANGE_OFFSET,
doorbell_index);
ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
S2A_DOORBELL_PORT1_RANGE_SIZE,
8);
ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
0x0);
ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
S2A_DOORBELL_PORT1_TARGET_PORT_TYPE,
0x3);
ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
S2A_DOORBELL_PORT1_TARGET_DIEID,
0x0);
ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
S2A_DOORBELL_PORT1_TARGET_PORT_ID,
0x0);
}
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range);
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1, ih_doorbell_range1);
}
static void nbio_v6_3_2_gc_doorbell_init(struct amdgpu_device *adev)
{
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL, 0x30000007);
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1, 0x3);
}
static void nbio_v6_3_2_sdma_doorbell_range(struct amdgpu_device *adev,
int instance, bool use_doorbell,
int doorbell_index,
int doorbell_size)
{
if (instance == 0) {
u32 doorbell_range = 0;
u32 doorbell_range1 = 0;
if (use_doorbell) {
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
S2A_DOORBELL_PORT6_ENABLE,
0x1);
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
S2A_DOORBELL_PORT6_AWID,
0xe);
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
S2A_DOORBELL_PORT6_RANGE_OFFSET,
doorbell_index);
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
S2A_DOORBELL_PORT6_RANGE_SIZE,
doorbell_size);
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE,
0xe);
doorbell_range1 = REG_SET_FIELD(doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
S2A_DOORBELL_PORT6_TARGET_PORT_TYPE,
0x3);
doorbell_range1 = REG_SET_FIELD(doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
S2A_DOORBELL_PORT6_TARGET_DIEID,
0x0);
doorbell_range1 = REG_SET_FIELD(doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
S2A_DOORBELL_PORT6_TARGET_PORT_ID,
0x0);
}
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, doorbell_range);
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1, doorbell_range1);
}
}
static void nbio_v6_3_2_vcn_doorbell_range(struct amdgpu_device *adev,
bool use_doorbell, int doorbell_index,
int instance)
{
u32 doorbell_range = 0;
u32 doorbell_range1 = 0;
if (use_doorbell) {
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
S2A_DOORBELL_PORT2_ENABLE,
0x1);
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
S2A_DOORBELL_PORT2_AWID,
(instance % adev->vcn.num_inst_per_aid) ? 0x7 : 0x4);
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
S2A_DOORBELL_PORT2_RANGE_OFFSET,
doorbell_index);
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
S2A_DOORBELL_PORT2_RANGE_SIZE,
8);
doorbell_range = REG_SET_FIELD(doorbell_range,
GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE,
(instance % adev->vcn.num_inst_per_aid) ? 0x7 : 0x4);
doorbell_range1 = REG_SET_FIELD(doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
S2A_DOORBELL_PORT2_TARGET_PORT_TYPE,
0x3);
doorbell_range1 = REG_SET_FIELD(doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
S2A_DOORBELL_PORT2_TARGET_DIEID,
(instance / adev->vcn.num_inst_per_aid) ? 0x3 : 0x0);
doorbell_range1 = REG_SET_FIELD(doorbell_range1,
GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
S2A_DOORBELL_PORT2_TARGET_PORT_ID,
0x0);
}
switch (instance) {
case 0:
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, doorbell_range);
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1, doorbell_range1);
break;
case 1:
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL, doorbell_range);
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1, doorbell_range1);
break;
case 2:
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL, doorbell_range);
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1, doorbell_range1);
break;
case 3:
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1, doorbell_range1);
break;
default:
dev_err(adev->dev,
"amdgpu: invalid vcn instance set when program doorbell range\n");
break;
}
}
static int nbio_v6_3_2_get_compute_partition_mode(struct amdgpu_device *adev)
{
u32 tmp, px;
tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_COMPUTE_STATUS);
px = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_COMPUTE_STATUS,
PARTITION_MODE);
return px;
}
static bool nbio_v6_3_2_is_nps_switch_requested(struct amdgpu_device *adev)
{
u32 tmp;
tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS);
tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS,
CHANGE_STATUS);
/* 0x8 - NPS switch requested */
return (tmp == 0x8);
}
static u32 nbio_v6_3_2_get_memory_partition_mode(struct amdgpu_device *adev,
u32 *supp_modes)
{
u32 tmp;
tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS);
tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS, NPS_MODE);
if (supp_modes) {
*supp_modes =
RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_CAP);
}
return ffs(tmp);
}
const struct amdgpu_nbio_funcs nbio_v6_3_2_funcs = {
.get_pcie_index_offset = nbio_v6_3_2_get_pcie_index_offset,
.get_pcie_data_offset = nbio_v6_3_2_get_pcie_data_offset,
.get_pcie_index_hi_offset = nbio_v6_3_2_get_pcie_index_hi_offset,
.get_rev_id = nbio_v6_3_2_get_rev_id,
.mc_access_enable = nbio_v6_3_2_mc_access_enable,
.get_memsize = nbio_v6_3_2_get_memsize,
.enable_doorbell_aperture = nbio_v6_3_2_enable_doorbell_aperture,
.enable_doorbell_selfring_aperture = nbio_v6_3_2_enable_doorbell_selfring_aperture,
.enable_doorbell_interrupt = nbio_v6_3_2_enable_doorbell_interrupt,
.get_compute_partition_mode = nbio_v6_3_2_get_compute_partition_mode,
.get_memory_partition_mode = nbio_v6_3_2_get_memory_partition_mode,
.is_nps_switch_requested = nbio_v6_3_2_is_nps_switch_requested,
.ih_control = nbio_v6_3_2_ih_control,
.ih_doorbell_range = nbio_v6_3_2_ih_doorbell_range,
.gc_doorbell_init = nbio_v6_3_2_gc_doorbell_init,
.sdma_doorbell_range = nbio_v6_3_2_sdma_doorbell_range,
.vcn_doorbell_range = nbio_v6_3_2_vcn_doorbell_range,
.init_registers = nbio_v6_3_2_init_registers,
};

31
drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright 2025 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
#ifndef __NBIO_V6_3_2_H__
#define __NBIO_V6_3_2_H__
#include "soc15_common.h"
extern const struct amdgpu_nbio_funcs nbio_v6_3_2_funcs;
#endif

31
drivers/gpu/drm/amd/amdgpu/nv.c
View File

@ -354,29 +354,12 @@ static struct soc15_allowed_register_entry nv_allowed_read_registers[] = {
{ SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)},
};
static uint32_t nv_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
u32 sh_num, u32 reg_offset)
{
uint32_t val;
mutex_lock(&adev->grbm_idx_mutex);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
val = RREG32(reg_offset);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
mutex_unlock(&adev->grbm_idx_mutex);
return val;
}
static uint32_t nv_get_register_value(struct amdgpu_device *adev,
bool indexed, u32 se_num,
u32 sh_num, u32 reg_offset)
{
if (indexed) {
return nv_read_indexed_register(adev, se_num, sh_num, reg_offset);
return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
} else {
if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
return adev->gfx.config.gb_addr_config;
@ -511,16 +494,6 @@ static int nv_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
return 0;
}
static void nv_program_aspm(struct amdgpu_device *adev)
{
if (!amdgpu_device_should_use_aspm(adev))
return;
if (adev->nbio.funcs->program_aspm)
adev->nbio.funcs->program_aspm(adev);
}
const struct amdgpu_ip_block_version nv_common_ip_block = {
.type = AMD_IP_BLOCK_TYPE_COMMON,
.major = 1,
@ -984,7 +957,7 @@ static int nv_common_hw_init(struct amdgpu_ip_block *ip_block)
adev->nbio.funcs->apply_l1_link_width_reconfig_wa(adev);
/* enable aspm */
nv_program_aspm(adev);
amdgpu_nbio_program_aspm(adev);
/* setup nbio registers */
adev->nbio.funcs->init_registers(adev);
/* remap HDP registers to a hole in mmio space,

1
drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
View File

@ -32,6 +32,7 @@
#include "mp/mp_15_0_8_sh_mask.h"
MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc.bin");
MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc_1.bin");
static int psp_v15_0_8_init_microcode(struct psp_context *psp)
{

8
drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
View File

@ -828,7 +828,6 @@ static int sdma_v2_4_early_init(struct amdgpu_ip_block *ip_block)
return r;
sdma_v2_4_set_ring_funcs(adev);
sdma_v2_4_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v2_4_vm_pte_funcs);
sdma_v2_4_set_irq_funcs(adev);
@ -898,7 +897,9 @@ static int sdma_v2_4_hw_init(struct amdgpu_ip_block *ip_block)
if (r)
return r;
return r;
sdma_v2_4_set_buffer_funcs(adev);
return 0;
}
static int sdma_v2_4_hw_fini(struct amdgpu_ip_block *ip_block)
@ -1235,8 +1236,7 @@ static const struct amdgpu_buffer_funcs sdma_v2_4_buffer_funcs = {
static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev)
{
adev->mman.buffer_funcs = &sdma_v2_4_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v2_4_buffer_funcs);
}
const struct amdgpu_ip_block_version sdma_v2_4_ip_block = {

8
drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
View File

@ -1108,7 +1108,6 @@ static int sdma_v3_0_early_init(struct amdgpu_ip_block *ip_block)
return r;
sdma_v3_0_set_ring_funcs(adev);
sdma_v3_0_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v3_0_vm_pte_funcs);
sdma_v3_0_set_irq_funcs(adev);
@ -1184,7 +1183,9 @@ static int sdma_v3_0_hw_init(struct amdgpu_ip_block *ip_block)
if (r)
return r;
return r;
sdma_v3_0_set_buffer_funcs(adev);
return 0;
}
static int sdma_v3_0_hw_fini(struct amdgpu_ip_block *ip_block)
@ -1677,8 +1678,7 @@ static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = {
static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev)
{
adev->mman.buffer_funcs = &sdma_v3_0_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v3_0_buffer_funcs);
}
const struct amdgpu_ip_block_version sdma_v3_0_ip_block =

21
drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
View File

@ -889,7 +889,7 @@ static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
/* write the fence */
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
/* zero in first two bits */
BUG_ON(addr & 0x3);
WARN_ON(addr & 0x3);
amdgpu_ring_write(ring, lower_32_bits(addr));
amdgpu_ring_write(ring, upper_32_bits(addr));
amdgpu_ring_write(ring, lower_32_bits(seq));
@ -899,7 +899,7 @@ static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
addr += 4;
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
/* zero in first two bits */
BUG_ON(addr & 0x3);
WARN_ON(addr & 0x3);
amdgpu_ring_write(ring, lower_32_bits(addr));
amdgpu_ring_write(ring, upper_32_bits(addr));
amdgpu_ring_write(ring, upper_32_bits(seq));
@ -1775,7 +1775,6 @@ static int sdma_v4_0_early_init(struct amdgpu_ip_block *ip_block)
adev->sdma.has_page_queue = true;
sdma_v4_0_set_ring_funcs(adev);
sdma_v4_0_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v4_0_vm_pte_funcs);
sdma_v4_0_set_irq_funcs(adev);
sdma_v4_0_set_ras_funcs(adev);
@ -1961,6 +1960,7 @@ static int sdma_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
static int sdma_v4_0_hw_init(struct amdgpu_ip_block *ip_block)
{
struct amdgpu_device *adev = ip_block->adev;
int r;
if (adev->flags & AMD_IS_APU)
amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_SDMA, false, 0);
@ -1968,7 +1968,12 @@ static int sdma_v4_0_hw_init(struct amdgpu_ip_block *ip_block)
if (!amdgpu_sriov_vf(adev))
sdma_v4_0_init_golden_registers(adev);
return sdma_v4_0_start(adev);
r = sdma_v4_0_start(adev);
if (r)
return r;
sdma_v4_0_set_buffer_funcs(adev);
return 0;
}
static int sdma_v4_0_hw_fini(struct amdgpu_ip_block *ip_block)
@ -2626,13 +2631,9 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_buffer_funcs = {
static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)
{
if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= IP_VERSION(4, 4, 0))
adev->mman.buffer_funcs = &sdma_v4_4_buffer_funcs;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_buffer_funcs);
else
adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs;
if (adev->sdma.has_page_queue)
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
else
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_0_buffer_funcs);
}
static void sdma_v4_0_get_ras_error_count(uint32_t value,

12
drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
View File

@ -1368,7 +1368,6 @@ static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block)
adev->sdma.has_page_queue = true;
sdma_v4_4_2_set_ring_funcs(adev);
sdma_v4_4_2_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v4_4_2_vm_pte_funcs);
sdma_v4_4_2_set_irq_funcs(adev);
sdma_v4_4_2_set_ras_funcs(adev);
@ -1568,8 +1567,11 @@ static int sdma_v4_4_2_hw_init(struct amdgpu_ip_block *ip_block)
sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
r = sdma_v4_4_2_inst_start(adev, inst_mask, false);
if (r)
return r;
sdma_v4_4_2_set_buffer_funcs(adev);
return r;
return 0;
}
static int sdma_v4_4_2_hw_fini(struct amdgpu_ip_block *ip_block)
@ -2316,11 +2318,7 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
{
adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
if (adev->sdma.has_page_queue)
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
else
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_2_buffer_funcs);
}
/**

11
drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
View File

@ -1373,7 +1373,6 @@ static int sdma_v5_0_early_init(struct amdgpu_ip_block *ip_block)
return r;
sdma_v5_0_set_ring_funcs(adev);
sdma_v5_0_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v5_0_vm_pte_funcs);
sdma_v5_0_set_irq_funcs(adev);
sdma_v5_0_set_mqd_funcs(adev);
@ -1472,8 +1471,11 @@ static int sdma_v5_0_hw_init(struct amdgpu_ip_block *ip_block)
sdma_v5_0_init_golden_registers(adev);
r = sdma_v5_0_start(adev);
if (r)
return r;
sdma_v5_0_set_buffer_funcs(adev);
return r;
return 0;
}
static int sdma_v5_0_hw_fini(struct amdgpu_ip_block *ip_block)
@ -2052,10 +2054,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_0_buffer_funcs = {
static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev)
{
if (adev->mman.buffer_funcs == NULL) {
adev->mman.buffer_funcs = &sdma_v5_0_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
}
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_0_buffer_funcs);
}
const struct amdgpu_ip_block_version sdma_v5_0_ip_block = {

14
drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
View File

@ -1264,7 +1264,6 @@ static int sdma_v5_2_early_init(struct amdgpu_ip_block *ip_block)
return r;
sdma_v5_2_set_ring_funcs(adev);
sdma_v5_2_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v5_2_vm_pte_funcs);
sdma_v5_2_set_irq_funcs(adev);
sdma_v5_2_set_mqd_funcs(adev);
@ -1385,8 +1384,14 @@ static int sdma_v5_2_sw_fini(struct amdgpu_ip_block *ip_block)
static int sdma_v5_2_hw_init(struct amdgpu_ip_block *ip_block)
{
struct amdgpu_device *adev = ip_block->adev;
int r;
return sdma_v5_2_start(adev);
r = sdma_v5_2_start(adev);
if (r)
return r;
sdma_v5_2_set_buffer_funcs(adev);
return 0;
}
static int sdma_v5_2_hw_fini(struct amdgpu_ip_block *ip_block)
@ -2056,10 +2061,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = {
static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev)
{
if (adev->mman.buffer_funcs == NULL) {
adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
}
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_2_buffer_funcs);
}
const struct amdgpu_ip_block_version sdma_v5_2_ip_block = {

5
drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
View File

@ -1313,7 +1313,6 @@ static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
return r;
sdma_v6_0_set_ring_funcs(adev);
sdma_v6_0_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v6_0_vm_pte_funcs);
sdma_v6_0_set_irq_funcs(adev);
sdma_v6_0_set_mqd_funcs(adev);
@ -1477,6 +1476,7 @@ static int sdma_v6_0_hw_init(struct amdgpu_ip_block *ip_block)
r = sdma_v6_0_start(adev);
if (r)
return r;
sdma_v6_0_set_buffer_funcs(adev);
return sdma_v6_0_set_userq_trap_interrupts(adev, true);
}
@ -1886,8 +1886,7 @@ static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = {
static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev)
{
adev->mman.buffer_funcs = &sdma_v6_0_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v6_0_buffer_funcs);
}
const struct amdgpu_ip_block_version sdma_v6_0_ip_block = {

5
drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
View File

@ -1299,7 +1299,6 @@ static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
}
sdma_v7_0_set_ring_funcs(adev);
sdma_v7_0_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_0_vm_pte_funcs);
sdma_v7_0_set_irq_funcs(adev);
sdma_v7_0_set_mqd_funcs(adev);
@ -1432,6 +1431,7 @@ static int sdma_v7_0_hw_init(struct amdgpu_ip_block *ip_block)
r = sdma_v7_0_start(adev);
if (r)
return r;
sdma_v7_0_set_buffer_funcs(adev);
return sdma_v7_0_set_userq_trap_interrupts(adev, true);
}
@ -1836,8 +1836,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = {
static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev)
{
adev->mman.buffer_funcs = &sdma_v7_0_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_0_buffer_funcs);
}
const struct amdgpu_ip_block_version sdma_v7_0_ip_block = {

12
drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
View File

@ -1287,7 +1287,6 @@ static int sdma_v7_1_early_init(struct amdgpu_ip_block *ip_block)
}
sdma_v7_1_set_ring_funcs(adev);
sdma_v7_1_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_1_vm_pte_funcs);
sdma_v7_1_set_irq_funcs(adev);
sdma_v7_1_set_mqd_funcs(adev);
@ -1386,10 +1385,16 @@ static int sdma_v7_1_hw_init(struct amdgpu_ip_block *ip_block)
{
struct amdgpu_device *adev = ip_block->adev;
uint32_t inst_mask;
int r;
inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
return sdma_v7_1_inst_start(adev, inst_mask);
r = sdma_v7_1_inst_start(adev, inst_mask);
if (r)
return r;
sdma_v7_1_set_buffer_funcs(adev);
return 0;
}
static int sdma_v7_1_hw_fini(struct amdgpu_ip_block *ip_block)
@ -1776,8 +1781,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_1_buffer_funcs = {
static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev)
{
adev->mman.buffer_funcs = &sdma_v7_1_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_1_buffer_funcs);
}
const struct amdgpu_ip_block_version sdma_v7_1_ip_block = {

12
drivers/gpu/drm/amd/amdgpu/si_dma.c
View File

@ -487,7 +487,6 @@ static int si_dma_early_init(struct amdgpu_ip_block *ip_block)
adev->sdma.num_instances = SDMA_MAX_INSTANCE;
si_dma_set_ring_funcs(adev);
si_dma_set_buffer_funcs(adev);
amdgpu_sdma_set_vm_pte_scheds(adev, &si_dma_vm_pte_funcs);
si_dma_set_irq_funcs(adev);
@ -543,8 +542,14 @@ static int si_dma_sw_fini(struct amdgpu_ip_block *ip_block)
static int si_dma_hw_init(struct amdgpu_ip_block *ip_block)
{
struct amdgpu_device *adev = ip_block->adev;
int r;
return si_dma_start(adev);
r = si_dma_start(adev);
if (r)
return r;
si_dma_set_buffer_funcs(adev);
return 0;
}
static int si_dma_hw_fini(struct amdgpu_ip_block *ip_block)
@ -833,8 +838,7 @@ static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = {
static void si_dma_set_buffer_funcs(struct amdgpu_device *adev)
{
adev->mman.buffer_funcs = &si_dma_buffer_funcs;
adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
amdgpu_sdma_set_buffer_funcs_scheds(adev, &si_dma_buffer_funcs);
}
const struct amdgpu_ip_block_version si_dma_ip_block =

30
drivers/gpu/drm/amd/amdgpu/soc15.c
View File

@ -401,29 +401,12 @@ static struct soc15_allowed_register_entry soc15_allowed_read_registers[] = {
{ SOC15_REG_ENTRY(GC, 0, mmDB_DEBUG2)},
};
static uint32_t soc15_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
u32 sh_num, u32 reg_offset)
{
uint32_t val;
mutex_lock(&adev->grbm_idx_mutex);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
val = RREG32(reg_offset);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
mutex_unlock(&adev->grbm_idx_mutex);
return val;
}
static uint32_t soc15_get_register_value(struct amdgpu_device *adev,
bool indexed, u32 se_num,
u32 sh_num, u32 reg_offset)
{
if (indexed) {
return soc15_read_indexed_register(adev, se_num, sh_num, reg_offset);
return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
} else {
if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
return adev->gfx.config.gb_addr_config;
@ -695,15 +678,6 @@ static int soc15_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk
return 0;
}
static void soc15_program_aspm(struct amdgpu_device *adev)
{
if (!amdgpu_device_should_use_aspm(adev))
return;
if (adev->nbio.funcs->program_aspm)
adev->nbio.funcs->program_aspm(adev);
}
const struct amdgpu_ip_block_version vega10_common_ip_block =
{
.type = AMD_IP_BLOCK_TYPE_COMMON,
@ -1284,7 +1258,7 @@ static int soc15_common_hw_init(struct amdgpu_ip_block *ip_block)
struct amdgpu_device *adev = ip_block->adev;
/* enable aspm */
soc15_program_aspm(adev);
amdgpu_nbio_program_aspm(adev);
/* setup nbio registers */
adev->nbio.funcs->init_registers(adev);
/* remap HDP registers to a hole in mmio space,

30
drivers/gpu/drm/amd/amdgpu/soc21.c
View File

@ -306,29 +306,12 @@ static struct soc15_allowed_register_entry soc21_allowed_read_registers[] = {
{ SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG)},
};
static uint32_t soc21_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
u32 sh_num, u32 reg_offset)
{
uint32_t val;
mutex_lock(&adev->grbm_idx_mutex);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
val = RREG32(reg_offset);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
mutex_unlock(&adev->grbm_idx_mutex);
return val;
}
static uint32_t soc21_get_register_value(struct amdgpu_device *adev,
bool indexed, u32 se_num,
u32 sh_num, u32 reg_offset)
{
if (indexed) {
return soc21_read_indexed_register(adev, se_num, sh_num, reg_offset);
return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
} else {
if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG) && adev->gfx.config.gb_addr_config)
return adev->gfx.config.gb_addr_config;
@ -470,15 +453,6 @@ static int soc21_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk
return 0;
}
static void soc21_program_aspm(struct amdgpu_device *adev)
{
if (!amdgpu_device_should_use_aspm(adev))
return;
if (adev->nbio.funcs->program_aspm)
adev->nbio.funcs->program_aspm(adev);
}
const struct amdgpu_ip_block_version soc21_common_ip_block = {
.type = AMD_IP_BLOCK_TYPE_COMMON,
.major = 1,
@ -925,7 +899,7 @@ static int soc21_common_hw_init(struct amdgpu_ip_block *ip_block)
struct amdgpu_device *adev = ip_block->adev;
/* enable aspm */
soc21_program_aspm(adev);
amdgpu_nbio_program_aspm(adev);
/* setup nbio registers */
adev->nbio.funcs->init_registers(adev);
/* remap HDP registers to a hole in mmio space,

21
drivers/gpu/drm/amd/amdgpu/soc24.c
View File

@ -132,31 +132,12 @@ static struct soc15_allowed_register_entry soc24_allowed_read_registers[] = {
{ SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG)},
};
static uint32_t soc24_read_indexed_register(struct amdgpu_device *adev,
u32 se_num,
u32 sh_num,
u32 reg_offset)
{
uint32_t val;
mutex_lock(&adev->grbm_idx_mutex);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
val = RREG32(reg_offset);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
mutex_unlock(&adev->grbm_idx_mutex);
return val;
}
static uint32_t soc24_get_register_value(struct amdgpu_device *adev,
bool indexed, u32 se_num,
u32 sh_num, u32 reg_offset)
{
if (indexed) {
return soc24_read_indexed_register(adev, se_num, sh_num, reg_offset);
return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
} else {
if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG) &&
adev->gfx.config.gb_addr_config)

33
drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
View File

@ -184,31 +184,13 @@ static struct soc15_allowed_register_entry soc_v1_0_allowed_read_registers[] = {
{ SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG_1) },
};
static uint32_t soc_v1_0_read_indexed_register(struct amdgpu_device *adev,
u32 se_num,
u32 sh_num,
u32 reg_offset)
{
uint32_t val;
mutex_lock(&adev->grbm_idx_mutex);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
val = RREG32(reg_offset);
if (se_num != 0xffffffff || sh_num != 0xffffffff)
amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
mutex_unlock(&adev->grbm_idx_mutex);
return val;
}
static uint32_t soc_v1_0_get_register_value(struct amdgpu_device *adev,
bool indexed, u32 se_num,
u32 sh_num, u32 reg_offset)
{
if (indexed) {
return soc_v1_0_read_indexed_register(adev, se_num, sh_num, reg_offset);
return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
} else {
if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG_1) &&
adev->gfx.config.gb_addr_config)
@ -330,6 +312,8 @@ static int soc_v1_0_common_early_init(struct amdgpu_ip_block *ip_block)
return -EINVAL;
}
adev->nbio.funcs->init_registers(adev);
return 0;
}
@ -737,15 +721,8 @@ static int soc_v1_0_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
num_xcc_per_xcp = __soc_v1_0_get_xcc_per_xcp(xcp_mgr, mode);
if (adev->gfx.imu.funcs &&
adev->gfx.imu.funcs->switch_compute_partition) {
ret = adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode);
if (ret)
goto out;
}
if (adev->gfx.imu.funcs &&
adev->gfx.imu.funcs->init_mcm_addr_lut &&
amdgpu_emu_mode)
adev->gfx.imu.funcs->init_mcm_addr_lut(adev);
adev->gfx.imu.funcs->switch_compute_partition)
adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode);
/* Init info about new xcps */
*num_xcps = num_xcc / num_xcc_per_xcp;

17
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
View File

@ -147,11 +147,6 @@ static int kfd_open(struct inode *inode, struct file *filep)
if (IS_ERR(process))
return PTR_ERR(process);
if (kfd_process_init_cwsr_apu(process, filep)) {
kfd_unref_process(process);
return -EFAULT;
}
/* filep now owns the reference returned by kfd_create_process */
filep->private_data = process;
@ -1360,7 +1355,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
if (WARN_ON_ONCE(!peer_pdd))
continue;
kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
kfd_flush_tlb(peer_pdd);
}
kfree(devices_arr);
@ -1455,7 +1450,7 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
if (WARN_ON_ONCE(!peer_pdd))
continue;
if (flush_tlb)
kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
kfd_flush_tlb(peer_pdd);
/* Remove dma mapping after tlb flush to avoid IO_PAGE_FAULT */
err = amdgpu_amdkfd_gpuvm_dmaunmap_mem(mem, peer_pdd->drm_priv);
@ -2340,6 +2335,9 @@ static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
const bool criu_resume = true;
u64 offset;
if (bo_priv->idr_handle > INT_MAX)
return -EINVAL;
if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
if (bo_bucket->size !=
kfd_doorbell_process_slice(pdd->dev->kfd))
@ -3537,9 +3535,8 @@ static int kfd_mmap(struct file *filep, struct vm_area_struct *vma)
return kfd_event_mmap(process, vma);
case KFD_MMAP_TYPE_RESERVED_MEM:
if (!dev)
return -ENODEV;
return kfd_reserved_mem_mmap(dev, process, vma);
pr_warn("KFD_MMAP_TYPE_RESERVED_MEM is no longer supported\n");
return -EINVAL;
case KFD_MMAP_TYPE_MMIO:
if (!dev)
return -ENODEV;

33
drivers/gpu/drm/amd/amdkfd/kfd_device.c
View File

@ -1737,37 +1737,6 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr
return false;
}
/* check if there is kfd process still uses adev */
static bool kgd2kfd_check_device_idle(struct amdgpu_device *adev)
{
struct kfd_process *p;
struct hlist_node *p_temp;
unsigned int temp;
struct kfd_node *dev;
mutex_lock(&kfd_processes_mutex);
if (hash_empty(kfd_processes_table)) {
mutex_unlock(&kfd_processes_mutex);
return true;
}
/* check if there is device still use adev */
hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
for (int i = 0; i < p->n_pdds; i++) {
dev = p->pdds[i]->dev;
if (dev->adev == adev) {
mutex_unlock(&kfd_processes_mutex);
return false;
}
}
}
mutex_unlock(&kfd_processes_mutex);
return true;
}
/** kgd2kfd_teardown_processes - gracefully tear down existing
* kfd processes that use adev
*
@ -1800,7 +1769,7 @@ void kgd2kfd_teardown_processes(struct amdgpu_device *adev)
mutex_unlock(&kfd_processes_mutex);
/* wait all kfd processes use adev terminate */
while (!kgd2kfd_check_device_idle(adev))
while (!!atomic_read(&adev->kfd.dev->kfd_processes_count))
cond_resched();
}

157
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
View File

@ -71,6 +71,12 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
struct queue *q, const uint32_t *restore_sdma_id);
static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
static int resume_all_queues_mes(struct device_queue_manager *dqm);
static int suspend_all_queues_mes(struct device_queue_manager *dqm);
static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm,
u32 doorbell_offset);
static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd);
static inline
enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
@ -273,13 +279,19 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
return r;
}
static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd)
static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd,
bool is_for_reset,
bool flush_mes_queue)
{
struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
int r;
struct mes_remove_queue_input queue_input;
/* queue was already removed during reset */
if (q->properties.is_reset)
return 0;
if (!dqm->sched_running || dqm->sched_halt)
return 0;
if (!down_read_trylock(&adev->reset_domain->sem))
@ -288,6 +300,7 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
queue_input.doorbell_offset = q->properties.doorbell_off;
queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
queue_input.remove_queue_after_reset = flush_mes_queue;
queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
amdgpu_mes_lock(&adev->mes);
@ -295,7 +308,13 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
amdgpu_mes_unlock(&adev->mes);
up_read(&adev->reset_domain->sem);
if (is_for_reset)
return r;
if (r) {
if (!suspend_all_queues_mes(dqm))
return resume_all_queues_mes(dqm);
dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",
q->properties.doorbell_off);
dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
@ -305,6 +324,12 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
return r;
}
static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd)
{
return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
}
static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
{
struct device_process_node *cur;
@ -359,6 +384,92 @@ static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
return retval;
}
static int reset_queues_mes(struct device_queue_manager *dqm)
{
struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
int hqd_info_size = adev->mes.hung_queue_hqd_info_offset;
int num_hung = 0, r = 0, i, pipe, queue, queue_type;
u32 *hung_array = dqm->hung_db_array;
struct amdgpu_mes_hung_queue_hqd_info *hqd_info = dqm->hqd_info;
struct kfd_process_device *pdd;
struct queue *q;
if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) {
r = -ENOTRECOVERABLE;
goto fail;
}
/* reset should be used only in dqm locked queue reset */
if (WARN_ON(dqm->detect_hang_count > 0))
return 0;
if (!amdgpu_gpu_recovery) {
r = -ENOTRECOVERABLE;
goto fail;
}
if (!hung_array || !hqd_info) {
r = -ENOMEM;
goto fail;
}
memset(hqd_info, 0, hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info));
/*
* AMDGPU_RING_TYPE_COMPUTE parameter does not matter if called
* post suspend_all as reset & detect will return all hung queue types.
*
* Passed parameter is for targeting queues not scheduled by MES add_queue.
*/
r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
false, &num_hung, hung_array, ffs(dqm->dev->xcc_mask) - 1);
if (!num_hung || r) {
r = -ENOTRECOVERABLE;
goto fail;
}
/* MES resets queue/pipe and cleans up internally */
for (i = 0; i < num_hung; i++) {
hqd_info[i].bit0_31 = hung_array[i + hqd_info_size];
pipe = hqd_info[i].pipe_index;
queue = hqd_info[i].queue_index;
queue_type = hqd_info[i].queue_type;
if (queue_type != MES_QUEUE_TYPE_COMPUTE &&
queue_type != MES_QUEUE_TYPE_SDMA) {
pr_warn("Unsupported hung queue reset type: %d\n", queue_type);
hung_array[i] = AMDGPU_MES_INVALID_DB_OFFSET;
continue;
}
q = find_queue_by_doorbell_offset(dqm, hung_array[i]);
if (!q) {
r = -ENOTRECOVERABLE;
goto fail;
}
pdd = kfd_get_process_device_data(q->device, q->process);
if (!pdd) {
r = -ENODEV;
goto fail;
}
pr_warn("Hang detected doorbell %x pipe %d queue %d type %d\n",
hung_array[i], pipe, queue, queue_type);
/* Proceed remove_queue with reset=true */
remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false);
set_queue_as_reset(dqm, q, &pdd->qpd);
}
dqm->detect_hang_count = num_hung;
kfd_signal_reset_event(dqm->dev);
fail:
dqm->detect_hang_count = 0;
return r;
}
static int suspend_all_queues_mes(struct device_queue_manager *dqm)
{
struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
@ -367,10 +478,13 @@ static int suspend_all_queues_mes(struct device_queue_manager *dqm)
if (!down_read_trylock(&adev->reset_domain->sem))
return -EIO;
r = amdgpu_mes_suspend(adev);
r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1);
up_read(&adev->reset_domain->sem);
if (r) {
if (!reset_queues_mes(dqm))
return 0;
dev_err(adev->dev, "failed to suspend gangs from MES\n");
dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
kfd_hws_hang(dqm);
@ -387,7 +501,7 @@ static int resume_all_queues_mes(struct device_queue_manager *dqm)
if (!down_read_trylock(&adev->reset_domain->sem))
return -EIO;
r = amdgpu_mes_resume(adev);
r = amdgpu_mes_resume(adev, ffs(dqm->dev->xcc_mask) - 1);
up_read(&adev->reset_domain->sem);
if (r) {
@ -572,7 +686,7 @@ static int allocate_vmid(struct device_queue_manager *dqm,
qpd->vmid,
qpd->page_table_base);
/* invalidate the VM context after pasid and vmid mapping is set up */
kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
kfd_flush_tlb(qpd_to_pdd(qpd));
if (dqm->dev->kfd2kgd->set_scratch_backing_va)
dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
@ -610,7 +724,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
if (flush_texture_cache_nocpsch(q->device, qpd))
dev_err(dev, "Failed to flush TC\n");
kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
kfd_flush_tlb(qpd_to_pdd(qpd));
/* Release the vmid mapping */
set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
@ -1284,7 +1398,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
dqm->dev->adev,
qpd->vmid,
qpd->page_table_base);
kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
kfd_flush_tlb(pdd);
}
/* Take a safe reference to the mm_struct, which may otherwise
@ -1821,6 +1935,9 @@ static int start_cpsch(struct device_queue_manager *dqm)
{
struct device *dev = dqm->dev->adev->dev;
int retval, num_hw_queue_slots;
struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
int hung_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev);
int hqd_info_size = adev->mes.hung_queue_hqd_info_offset;
dqm_lock(dqm);
@ -1870,6 +1987,11 @@ static int start_cpsch(struct device_queue_manager *dqm)
goto fail_detect_hang_buffer;
}
dqm->hung_db_array = kzalloc(hung_array_size * sizeof(u32), GFP_KERNEL);
dqm->hqd_info = kzalloc(
hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info),
GFP_KERNEL);
dqm_unlock(dqm);
return 0;
@ -1910,6 +2032,9 @@ static int stop_cpsch(struct device_queue_manager *dqm)
pm_uninit(&dqm->packet_mgr);
kfree(dqm->detect_hang_info);
dqm->detect_hang_info = NULL;
kfree(dqm->hung_db_array);
kfree(dqm->hqd_info);
dqm_unlock(dqm);
return ret;
@ -2137,6 +2262,7 @@ static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q
q->properties.queue_id, pdd->process->lead_thread->pid);
pdd->has_reset_queue = true;
q->properties.is_reset = true;
if (q->properties.is_active) {
q->properties.is_active = false;
decrement_queue_count(dqm, qpd, q);
@ -2203,6 +2329,23 @@ static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uin
return NULL;
}
static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset)
{
struct device_process_node *cur;
struct qcm_process_device *qpd;
struct queue *q;
list_for_each_entry(cur, &dqm->queues, list) {
qpd = cur->qpd;
list_for_each_entry(q, &qpd->queues_list, list) {
if (doorbell_offset == q->properties.doorbell_off)
return q;
}
}
return NULL;
}
static int reset_hung_queues(struct device_queue_manager *dqm)
{
int r = 0, reset_count = 0, i;

4
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
View File

@ -32,7 +32,6 @@
#include "kfd_priv.h"
#include "kfd_mqd_manager.h"
#define VMID_NUM 16
#define KFD_MES_PROCESS_QUANTUM 100000
@ -285,6 +284,9 @@ struct device_queue_manager {
struct dqm_detect_hang_info *detect_hang_info;
size_t detect_hang_info_size;
int detect_hang_count;
/* for per-queue reset with mes */
u32 *hung_db_array;
struct amdgpu_mes_hung_queue_hqd_info *hqd_info;
};
void device_queue_manager_init_cik(

19
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
View File

@ -54,6 +54,7 @@ static int update_qpd_v12_1(struct device_queue_manager *dqm,
struct kfd_process_device *pdd;
struct amdgpu_device *adev = dqm->dev->adev;
struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
bool xnack_enabled;
pdd = qpd_to_pdd(qpd);
qpd->vm_cntx_cntl = hub->vm_cntx_cntl;
@ -71,16 +72,18 @@ static int update_qpd_v12_1(struct device_queue_manager *dqm,
qpd->sh_mem_ape1_base = 0;
}
if (KFD_SUPPORT_XNACK_PER_PROCESS(dqm->dev)) {
if (!pdd->process->xnack_enabled) {
qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
qpd->vm_cntx_cntl &=
xnack_enabled = KFD_SUPPORT_XNACK_PER_PROCESS(dqm->dev) ?
pdd->process->xnack_enabled :
!pdd->dev->kfd->noretry;
if (!xnack_enabled) {
qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
qpd->vm_cntx_cntl &=
~(1 << GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT);
} else {
qpd->sh_mem_config &= ~(1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT);
qpd->vm_cntx_cntl |=
} else {
qpd->sh_mem_config &= ~(1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT);
qpd->vm_cntx_cntl |=
(1 << GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT);
}
}
qpd->sh_mem_bases = compute_sh_mem_bases_64bit(pdd);

5
drivers/gpu/drm/amd/amdkfd/kfd_events.c
View File

@ -483,6 +483,11 @@ int kfd_criu_restore_event(struct file *devkfd,
}
*priv_data_offset += sizeof(*ev_priv);
if (ev_priv->event_id > INT_MAX) {
ret = -EINVAL;
goto exit;
}
if (ev_priv->user_handle) {
ret = kfd_kmap_event_page(p, ev_priv->user_handle);
if (ret)

3
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
View File

@ -129,13 +129,14 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
struct dma_fence **mfence)
{
const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;
struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
struct amdgpu_ring *ring;
struct amdgpu_ttm_buffer_entity *entity;
u64 gart_s, gart_d;
struct dma_fence *next;
u64 size;
int r;
ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
entity = &adev->mman.move_entities[0];
mutex_lock(&entity->lock);

6
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
View File

@ -32,6 +32,10 @@
#include "amdgpu_amdkfd.h"
#include "kfd_device_queue_manager.h"
static void update_mqd(struct mqd_manager *mm, void *mqd,
struct queue_properties *q,
struct mqd_update_info *minfo);
static inline struct v12_1_compute_mqd *get_mqd(void *mqd)
{
return (struct v12_1_compute_mqd *)mqd;
@ -215,7 +219,7 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
*mqd = m;
if (gart_addr)
*gart_addr = addr;
mm->update_mqd(mm, m, q, NULL);
update_mqd(mm, m, q, NULL);
}
static int load_mqd(struct mqd_manager *mm, void *mqd,

18
drivers/gpu/drm/amd/amdkfd/kfd_priv.h
View File

@ -215,8 +215,7 @@ enum cache_policy {
((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) || \
(KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)) || \
(KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4)) || \
(KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)) || \
(KFD_GC_VERSION(dev) == IP_VERSION(12, 1, 0)))
(KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)))
struct kfd_node;
@ -523,6 +522,7 @@ struct queue_properties {
uint32_t pm4_target_xcc;
bool is_dbg_wa;
bool is_user_cu_masked;
bool is_reset;
/* Not relevant for user mode queues in cp scheduling */
unsigned int vmid;
/* Relevant only for sdma queues*/
@ -986,9 +986,6 @@ struct kfd_process {
struct kobject *kobj_queues;
struct attribute attr_pasid;
/* Keep track cwsr init */
bool has_cwsr;
/* Exception code enable mask and status */
uint64_t exception_enable_mask;
uint64_t exception_status;
@ -1104,8 +1101,6 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
struct vm_area_struct *vma);
void kfd_process_notifier_release_internal(struct kfd_process *p);
/* KFD process API for creating and translating handles */
@ -1222,9 +1217,6 @@ void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
bool enabled);
/* CWSR initialization */
int kfd_process_init_cwsr_apu(struct kfd_process *process, struct file *filep);
/* CRIU */
/*
* Need to increment KFD_CRIU_PRIV_VERSION each time a change is made to any of the CRIU private
@ -1554,13 +1546,13 @@ void kfd_signal_reset_event(struct kfd_node *dev);
void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid);
void kfd_signal_process_terminate_event(struct kfd_process *p);
static inline void kfd_flush_tlb(struct kfd_process_device *pdd,
enum TLB_FLUSH_TYPE type)
static inline void kfd_flush_tlb(struct kfd_process_device *pdd)
{
struct amdgpu_device *adev = pdd->dev->adev;
struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
amdgpu_vm_flush_compute_tlb(adev, vm, type, pdd->dev->xcc_mask);
amdgpu_vm_flush_compute_tlb(adev, vm, TLB_FLUSH_HEAVYWEIGHT,
pdd->dev->xcc_mask);
}
static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev)

76
drivers/gpu/drm/amd/amdkfd/kfd_process.c
View File

@ -1409,50 +1409,6 @@ void kfd_cleanup_processes(void)
mmu_notifier_synchronize();
}
int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
{
unsigned long offset;
int i;
if (p->has_cwsr)
return 0;
for (i = 0; i < p->n_pdds; i++) {
struct kfd_node *dev = p->pdds[i]->dev;
struct qcm_process_device *qpd = &p->pdds[i]->qpd;
if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
continue;
offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
MAP_SHARED, offset);
if (IS_ERR_VALUE(qpd->tba_addr)) {
int err = qpd->tba_addr;
dev_err(dev->adev->dev,
"Failure to set tba address. error %d.\n", err);
qpd->tba_addr = 0;
qpd->cwsr_kaddr = NULL;
return err;
}
memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
}
p->has_cwsr = true;
return 0;
}
static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
{
struct kfd_node *dev = pdd->dev;
@ -2228,38 +2184,6 @@ int kfd_resume_all_processes(void)
return ret;
}
int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
struct vm_area_struct *vma)
{
struct kfd_process_device *pdd;
struct qcm_process_device *qpd;
if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");
return -EINVAL;
}
pdd = kfd_get_process_device_data(dev, process);
if (!pdd)
return -EINVAL;
qpd = &pdd->qpd;
qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(KFD_CWSR_TBA_TMA_SIZE));
if (!qpd->cwsr_kaddr) {
dev_err(dev->adev->dev,
"Error allocating per process CWSR buffer.\n");
return -ENOMEM;
}
vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
/* Mapping pages to user process */
return remap_pfn_range(vma, vma->vm_start,
PFN_DOWN(__pa(qpd->cwsr_kaddr)),
KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
}
/* assumes caller holds process lock. */
int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
{

4
drivers/gpu/drm/amd/amdkfd/kfd_svm.c
View File

@ -1424,7 +1424,7 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
if (r)
break;
}
kfd_flush_tlb(pdd, TLB_FLUSH_HEAVYWEIGHT);
kfd_flush_tlb(pdd);
}
return r;
@ -1571,7 +1571,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
}
}
kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
kfd_flush_tlb(pdd);
}
return r;

1
drivers/gpu/drm/amd/amdkfd/kfd_topology.c
View File

@ -2027,6 +2027,7 @@ static void kfd_topology_set_capabilities(struct kfd_topology_device *dev)
if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 1, 0)) {
dev->node_props.capability |=
HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED;
dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED;
dev->node_props.capability2 |=
HSA_CAP2_TRAP_DEBUG_LDS_OUT_OF_ADDR_RANGE_SUPPORTED;
}

346
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
View File

@ -104,6 +104,7 @@
#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
#include "modules/inc/mod_freesync.h"
#include "modules/inc/mod_power.h"
#include "modules/power/power_helpers.h"
static_assert(AMDGPU_DMUB_NOTIFICATION_MAX == DMUB_NOTIFICATION_MAX, "AMDGPU_DMUB_NOTIFICATION_MAX mismatch");
@ -1881,6 +1882,70 @@ static enum dmub_ips_disable_type dm_get_default_ips_mode(
return ret;
}
static int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm)
{
struct mod_power_init_params init_data[MAX_NUM_EDP];
if (dm->num_of_edps == 0) {
drm_dbg_driver(
dm->ddev,
"amdgpu: No eDP detected, skip initializing power module\n");
return 0;
}
/* Initialize all the power module parameters */
for (int i = 0; i < dm->num_of_edps; i++) {
init_data[i].allow_psr_smu_optimizations =
!!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT);
init_data[i].allow_psr_multi_disp_optimizations =
!!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT);
/* See dm_late_init */
init_data[i].backlight_ramping_override = false;
init_data[i].backlight_ramping_start = 0xCCCC;
init_data[i].backlight_ramping_reduction = 0xCCCCCCCC;
init_data[i].def_varibright_level = 0;
init_data[i].abm_config_setting = 0;
init_data[i].num_backlight_levels = 101;
init_data[i].use_nits_based_brightness = false;
init_data[i].panel_max_millinits = 0;
init_data[i].panel_min_millinits = 0;
init_data[i].disable_fractional_pwm =
!(amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK);
init_data[i].use_custom_backlight_caps = false;
init_data[i].custom_backlight_caps_config_no = 0;
init_data[i].use_linear_backlight_curve = false;
init_data[i].def_varibright_enable = 0;
init_data[i].varibright_level = 0;
/*
* Power module uses 16-bit backlight levels (0xFFFF max) rather
* than 8-bit(0XFF max)
*/
init_data[i].min_backlight_pwm =
dm->backlight_caps[i].min_input_signal * 0x101;
init_data[i].max_backlight_pwm =
dm->backlight_caps[i].max_input_signal * 0x101;
init_data[i].min_abm_backlight =
dm->backlight_caps[i].min_input_signal * 0x101;
/* Min backlight level after ABM reduction, Don't allow below 1%
* 0xFFFF x 0.01 = 0x28F
*/
init_data[i].min_abm_backlight = (init_data[i].min_abm_backlight < 0x28F) ?
0x28F : init_data[i].min_abm_backlight;
}
dm->power_module = mod_power_create(dm->dc, init_data, dm->num_of_edps);
if (!dm->power_module) {
drm_err(dm->ddev, "amdgpu: Error allocating memory for power module\n");
return -ENOMEM;
}
mod_power_hw_init(dm->power_module);
drm_dbg_driver(dm->ddev, "amdgpu: Power module init done\n");
return 0;
}
static int amdgpu_dm_init(struct amdgpu_device *adev)
{
struct dc_init_data init_data;
@ -1898,6 +1963,8 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
mutex_init(&adev->dm.dc_lock);
mutex_init(&adev->dm.audio_lock);
spin_lock_init(&adev->dm.dmub_lock);
if (amdgpu_dm_irq_init(adev)) {
drm_err(adev_to_drm(adev), "failed to initialize DM IRQ support.\n");
goto error;
@ -2198,6 +2265,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
goto error;
}
if (amdgpu_dm_init_power_module(&adev->dm))
goto error;
/* create fake encoders for MST */
dm_dp_create_fake_mst_encoders(adev);
@ -2334,6 +2404,10 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev)
adev->dm.freesync_module = NULL;
}
if (adev->dm.power_module) {
mod_power_destroy(adev->dm.power_module);
adev->dm.power_module = NULL;
}
mutex_destroy(&adev->dm.audio_lock);
mutex_destroy(&adev->dm.dc_lock);
mutex_destroy(&adev->dm.dpia_aux_lock);
@ -5054,8 +5128,8 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
#define AMDGPU_DM_MIN_SPREAD ((AMDGPU_DM_DEFAULT_MAX_BACKLIGHT - AMDGPU_DM_DEFAULT_MIN_BACKLIGHT) / 2)
#define AUX_BL_DEFAULT_TRANSITION_TIME_MS 50
static void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
int bl_idx)
void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
int bl_idx)
{
struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[bl_idx];
@ -5217,15 +5291,34 @@ static u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *cap
max - min);
}
static struct dc_stream_state *dm_find_stream_with_link(
struct amdgpu_display_manager *dm,
struct dc_link *link)
{
struct dc_state *cur_dc_state = dm->dc->current_state;
struct dc_stream_state *stream = NULL;
int i;
for (i = 0; i < cur_dc_state->stream_count; i++) {
stream = cur_dc_state->streams[i];
if (stream->link == link)
return stream;
}
return NULL;
}
static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
int bl_idx,
u32 user_brightness)
{
struct amdgpu_dm_backlight_caps *caps;
struct dc_link *link;
u32 brightness;
bool rc, reallow_idle = false;
u32 brightness = 0;
bool rc = false, reallow_idle = false;
struct drm_connector *connector;
struct dc_stream_state *stream;
unsigned int min, max;
list_for_each_entry(connector, &dm->ddev->mode_config.connector_list, head) {
struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
@ -5255,13 +5348,6 @@ static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
if (caps->brightness_mask)
brightness |= caps->brightness_mask;
/* Change brightness based on AUX property */
mutex_lock(&dm->dc_lock);
if (dm->dc->caps.ips_support && dm->dc->ctx->dmub_srv->idle_allowed) {
dc_allow_idle_optimizations(dm->dc, false);
reallow_idle = true;
}
if (trace_amdgpu_dm_brightness_enabled()) {
trace_amdgpu_dm_brightness(__builtin_return_address(0),
user_brightness,
@ -5270,22 +5356,45 @@ static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
power_supply_is_system_supplied() > 0);
}
if (caps->aux_support) {
rc = dc_link_set_backlight_level_nits(link, true, brightness,
AUX_BL_DEFAULT_TRANSITION_TIME_MS);
if (!rc)
DRM_DEBUG("DM: Failed to update backlight via AUX on eDP[%d]\n", bl_idx);
} else {
struct set_backlight_level_params backlight_level_params = { 0 };
stream = dm_find_stream_with_link(dm, link);
if (!stream)
return;
backlight_level_params.backlight_pwm_u16_16 = brightness;
backlight_level_params.transition_time_in_ms = 0;
rc = dc_link_set_backlight_level(link, &backlight_level_params);
if (!rc)
DRM_DEBUG("DM: Failed to update backlight on eDP[%d]\n", bl_idx);
mutex_lock(&dm->dc_lock);
if (dm->dc->caps.ips_support && dm->dc->ctx->dmub_srv->idle_allowed) {
dc_allow_idle_optimizations(dm->dc, false);
reallow_idle = true;
}
if (caps->aux_support) {
rc = mod_power_set_backlight_nits(dm->power_module, stream, brightness,
AUX_BL_DEFAULT_TRANSITION_TIME_MS, false, true);
} else {
/* power module uses millipercent */
get_brightness_range(caps, &min, &max);
brightness = DIV_ROUND_CLOSEST(brightness * 100, (max - min)) * 1000;
rc = mod_power_set_backlight_percent(dm->power_module, stream,
brightness, 0, false);
}
/*
* Some kms clients create a ramped backlight transition effect
* by rapidly changing the backlight. Yet we must wait on dmcub
* fw to exit psr/replay before programming backlight. To
* prevent lag, keep disable psr/replay and let the next atomic
* flip clear the event.
*
* ToDo: use ISM to handle rapidly backlight change
*
* Rapidly backlight change is similar to rapidly cursor events,
* which is now handled by ISM. ISM can delay the event until system
* is really idle, so we may use ISM to handle backlight change as well.
*/
amdgpu_dm_psr_set_event(dm, stream, true,
psr_event_hw_programming, true);
amdgpu_dm_replay_set_event(dm, stream, true,
replay_event_hw_programming, true);
if (dm->dc->caps.ips_support && reallow_idle)
dc_allow_idle_optimizations(dm->dc, true);
@ -5503,6 +5612,8 @@ static void setup_backlight_device(struct amdgpu_display_manager *dm,
static void amdgpu_set_panel_orientation(struct drm_connector *connector);
/*
* In this architecture, the association
* connector -> encoder -> crtc
@ -5744,7 +5855,7 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
psr_feature_enabled = false;
if (psr_feature_enabled) {
amdgpu_dm_set_psr_caps(link);
amdgpu_dm_set_psr_caps(link, aconnector);
drm_info(adev_to_drm(adev), "%s: PSR support %d, DC PSR ver %d, sink PSR ver %d DPCD caps 0x%x su_y_granularity %d\n",
aconnector->base.name,
link->psr_settings.psr_feature_enabled,
@ -9808,7 +9919,8 @@ static void update_stream_irq_parameters(
spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
}
static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
static void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm,
struct dm_crtc_state *old_state,
struct dm_crtc_state *new_state)
{
bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state);
@ -9827,6 +9939,13 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0);
drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR off->on: Get vblank ref\n",
__func__, new_state->base.crtc->base.id);
scoped_guard(mutex, &dm->dc_lock) {
amdgpu_dm_psr_set_event(dm, new_state->stream, true,
psr_event_vrr_transition, true);
amdgpu_dm_replay_set_event(dm, new_state->stream, true,
replay_event_vrr, true);
}
} else if (old_vrr_active && !new_vrr_active) {
/* Transition VRR active -> inactive:
* Allow vblank irq disable again for fixed refresh rate.
@ -9835,6 +9954,13 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
drm_crtc_vblank_put(new_state->base.crtc);
drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR on->off: Drop vblank ref\n",
__func__, new_state->base.crtc->base.id);
scoped_guard(mutex, &dm->dc_lock) {
amdgpu_dm_psr_set_event(dm, new_state->stream, false,
psr_event_vrr_transition, false);
amdgpu_dm_replay_set_event(dm, new_state->stream, false,
replay_event_vrr, false);
}
}
}
@ -9932,7 +10058,8 @@ static void amdgpu_dm_update_cursor(struct drm_plane *plane,
}
}
static void amdgpu_dm_enable_self_refresh(struct amdgpu_crtc *acrtc_attach,
static void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm,
struct amdgpu_crtc *acrtc_attach,
const struct dm_crtc_state *acrtc_state,
const u64 current_ts)
{
@ -9940,20 +10067,10 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_crtc *acrtc_attach,
struct replay_settings *pr = &acrtc_state->stream->link->replay_settings;
struct amdgpu_dm_connector *aconn =
(struct amdgpu_dm_connector *)acrtc_state->stream->dm_stream_context;
bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state);
if (acrtc_state->update_type > UPDATE_TYPE_FAST) {
if (pr->config.replay_supported && !pr->replay_feature_enabled)
amdgpu_dm_link_setup_replay(acrtc_state->stream->link, aconn);
else if (psr->psr_version != DC_PSR_VERSION_UNSUPPORTED &&
!psr->psr_feature_enabled)
if (!aconn->disallow_edp_enter_psr)
amdgpu_dm_link_setup_psr(acrtc_state->stream);
}
/* Decrement skip count when SR is enabled and we're doing fast updates. */
if (acrtc_state->update_type == UPDATE_TYPE_FAST &&
(psr->psr_feature_enabled || pr->config.replay_supported)) {
(psr->psr_feature_enabled || pr->replay_feature_enabled)) {
if (aconn->sr_skip_count > 0)
aconn->sr_skip_count--;
@ -9968,17 +10085,15 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_crtc *acrtc_attach,
* of update events.
* See `amdgpu_dm_crtc_vblank_control_worker()`.
*/
if (!vrr_active &&
acrtc_attach->dm_irq_params.allow_sr_entry &&
#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY
!amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) &&
#endif
(current_ts - psr->psr_dirty_rects_change_timestamp_ns) > 500000000) {
if (pr->replay_feature_enabled && !pr->replay_allow_active)
amdgpu_dm_replay_enable(acrtc_state->stream, true);
if (psr->psr_version == DC_PSR_VERSION_SU_1 &&
!psr->psr_allow_active && !aconn->disallow_edp_enter_psr)
amdgpu_dm_psr_enable(acrtc_state->stream);
if (acrtc_attach->dm_irq_params.allow_sr_entry &&
(current_ts - psr->psr_dirty_rects_change_timestamp_ns) > 500000000) {
amdgpu_dm_psr_set_event(dm, acrtc_state->stream, false,
psr_event_hw_programming, false);
amdgpu_dm_replay_set_event(dm, acrtc_state->stream, true,
replay_event_general_ui, true);
amdgpu_dm_replay_set_event(dm, acrtc_state->stream, false,
replay_event_hw_programming, false);
}
} else {
acrtc_attach->dm_irq_params.allow_sr_entry = false;
@ -10140,15 +10255,12 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
*/
if (acrtc_state->stream->link->psr_settings.psr_version >= DC_PSR_VERSION_SU_1 &&
acrtc_attach->dm_irq_params.allow_sr_entry &&
#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY
!amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) &&
#endif
dirty_rects_changed) {
mutex_lock(&dm->dc_lock);
acrtc_state->stream->link->psr_settings.psr_dirty_rects_change_timestamp_ns =
timestamp_ns;
if (acrtc_state->stream->link->psr_settings.psr_allow_active)
amdgpu_dm_psr_disable(acrtc_state->stream, true);
amdgpu_dm_psr_set_event(dm, acrtc_state->stream, true,
psr_event_hw_programming, true);
mutex_unlock(&dm->dc_lock);
}
}
@ -10313,15 +10425,6 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
if (acrtc_state->abm_level != dm_old_crtc_state->abm_level)
bundle->stream_update.abm_level = &acrtc_state->abm_level;
mutex_lock(&dm->dc_lock);
if ((acrtc_state->update_type > UPDATE_TYPE_FAST) || vrr_active) {
if (acrtc_state->stream->link->replay_settings.replay_allow_active)
amdgpu_dm_replay_disable(acrtc_state->stream);
if (acrtc_state->stream->link->psr_settings.psr_allow_active)
amdgpu_dm_psr_disable(acrtc_state->stream, true);
}
mutex_unlock(&dm->dc_lock);
/*
* If FreeSync state on the stream has changed then we need to
* re-adjust the min/max bounds now that DC doesn't handle this
@ -10359,8 +10462,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
if (dm_old_crtc_state->active_planes != acrtc_state->active_planes)
dm_update_pflip_irq_state(drm_to_adev(dev),
acrtc_attach);
amdgpu_dm_enable_self_refresh(acrtc_attach, acrtc_state, timestamp_ns);
amdgpu_dm_enable_self_refresh(dm, acrtc_attach, acrtc_state,
timestamp_ns);
mutex_unlock(&dm->dc_lock);
}
@ -10479,6 +10582,102 @@ static void dm_clear_writeback(struct amdgpu_display_manager *dm,
dc_stream_remove_writeback(dm->dc, crtc_state->stream, 0);
}
/**
* amdgpu_dm_mod_power_update_streams - update mod_power stream state on modeset
* @state: the drm atomic state
* @dm: the display manager to update mod_power on
*
* Notify mod_power of stream changes on modeset events, and disable PSR/Replay
* in preparation for hardware programming. See also
* amdgpu_dm_mod_power_setup_streams() for post-modeset mod_power setup.
*/
static void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commit *state,
struct amdgpu_display_manager *dm)
{
struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state;
struct drm_crtc_state *old_crtc_state, *new_crtc_state;
struct amdgpu_dm_connector *aconnector;
struct drm_crtc *crtc;
int i = 0;
for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
continue;
/*
* Update mod_power on modeset event in preparation for hw
* programming. Always use the old stream, since it would have
* been previously added to mod_power. If old stream is null (on
* crtc enable, for example), mod_power will no-op, which is the
* desried behavior.
*/
if (old_crtc_state->active) {
scoped_guard(mutex, &dm->dc_lock) {
amdgpu_dm_psr_set_event(dm, dm_old_crtc_state->stream, true,
psr_event_hw_programming, true);
amdgpu_dm_replay_set_event(dm, dm_old_crtc_state->stream, true,
replay_event_hw_programming, true);
}
}
if (new_crtc_state->active) {
aconnector = (struct amdgpu_dm_connector *)
dm_new_crtc_state->stream->dm_stream_context;
if (old_crtc_state->active) {
mod_power_replace_stream(dm->power_module,
dm_old_crtc_state->stream,
dm_new_crtc_state->stream,
&aconnector->psr_caps);
} else {
mod_power_add_stream(dm->power_module,
dm_new_crtc_state->stream,
&aconnector->psr_caps);
}
} else if (old_crtc_state->active) {
mod_power_remove_stream(dm->power_module,
dm_old_crtc_state->stream);
}
}
}
/**
* amdgpu_dm_mod_power_setup_streams - setup mod_power stream state post modeset
* @state: the drm atomic state
* @dm: the display manager to update mod_power on
*
* Notify mod_power of mode_change. This needs to be done after dc_stream
* updates have been committed, and VRR parameters have been updated.
*/
static void amdgpu_dm_mod_power_setup_streams(struct drm_atomic_commit *state,
struct amdgpu_display_manager *dm)
{
struct dm_crtc_state *dm_new_crtc_state;
struct drm_crtc_state *new_crtc_state;
struct amdgpu_crtc *acrtc;
struct drm_crtc *crtc;
int i = 0;
for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
acrtc = to_amdgpu_crtc(crtc);
if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
continue;
if (new_crtc_state->active) {
amdgpu_dm_link_setup_replay(dm_new_crtc_state->stream,
&acrtc->dm_irq_params.vrr_params);
mod_power_notify_mode_change(dm->power_module,
dm_new_crtc_state->stream,
false);
}
}
}
static void amdgpu_dm_commit_streams(struct drm_atomic_commit *state,
struct dc_state *dc_state)
{
@ -10522,6 +10721,8 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_commit *state,
acrtc->wb_enabled = false;
}
amdgpu_dm_mod_power_update_streams(state, dm);
for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
new_crtc_state, i) {
struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
@ -10626,13 +10827,10 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_commit *state,
}
} /* for_each_crtc_in_state() */
/* if there mode set or reset, disable eDP PSR, Replay */
/* if there mode set or reset, flush vblank work queue */
if (mode_set_reset_required) {
if (dm->vblank_control_workqueue)
flush_workqueue(dm->vblank_control_workqueue);
amdgpu_dm_replay_disable_all(dm);
amdgpu_dm_psr_disable_all(dm);
}
dm_enable_per_frame_crtc_master_sync(dc_state);
@ -11105,7 +11303,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
manage_dm_interrupts(adev, acrtc, dm_new_crtc_state);
}
/* Handle vrr on->off / off->on transitions */
amdgpu_dm_handle_vrr_transition(dm_old_crtc_state, dm_new_crtc_state);
amdgpu_dm_handle_vrr_transition(dm, dm_old_crtc_state, dm_new_crtc_state);
#ifdef CONFIG_DEBUG_FS
if (new_crtc_state->active &&
@ -11143,6 +11341,8 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
#endif
}
amdgpu_dm_mod_power_setup_streams(state, dm);
for_each_new_crtc_in_state(state, crtc, new_crtc_state, j)
if (new_crtc_state->async_flip)
wait_for_vblank = false;
@ -13701,11 +13901,17 @@ int amdgpu_dm_process_dmub_set_config_sync(
bool dm_execute_dmub_cmd(const struct dc_context *ctx, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type)
{
struct amdgpu_device *adev = ctx->driver_context;
guard(spinlock_irqsave)(&adev->dm.dmub_lock);
return dc_dmub_srv_cmd_run(ctx->dmub_srv, cmd, wait_type);
}
bool dm_execute_dmub_cmd_list(const struct dc_context *ctx, unsigned int count, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type)
{
struct amdgpu_device *adev = ctx->driver_context;
guard(spinlock_irqsave)(&adev->dm.dmub_lock);
return dc_dmub_srv_cmd_run_list(ctx->dmub_srv, count, cmd, wait_type);
}

10
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
View File

@ -463,6 +463,13 @@ struct amdgpu_display_manager {
*/
struct mutex dc_lock;
/**
* @dmub_lock:
*
* Guards access to DMUB command submission.
*/
spinlock_t dmub_lock;
/**
* @audio_lock:
*
@ -568,6 +575,7 @@ struct amdgpu_display_manager {
struct amdgpu_dm_backlight_caps backlight_caps[AMDGPU_DM_MAX_NUM_EDP];
struct mod_freesync *freesync_module;
struct mod_power *power_module;
struct hdcp_workqueue *hdcp_workqueue;
/**
@ -835,6 +843,7 @@ struct amdgpu_dm_connector {
bool force_yuv420_output;
bool force_yuv422_output;
struct dsc_preferred_settings dsc_settings;
struct psr_caps psr_caps;
union dp_downstream_port_present mst_downstream_port_present;
/* Cached display modes */
struct drm_display_mode freesync_vid_base;
@ -1149,4 +1158,5 @@ int amdgpu_dm_initialize_hdmi_connector(struct amdgpu_dm_connector *aconnector);
void retrieve_dmi_info(struct amdgpu_display_manager *dm);
void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm, int bl_idx);
#endif /* __AMDGPU_DM_H__ */

36
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
View File

@ -503,7 +503,6 @@ int amdgpu_dm_crtc_configure_crc_source(struct drm_crtc *crtc,
{
struct amdgpu_device *adev = drm_to_adev(crtc->dev);
struct dc_stream_state *stream_state = dm_crtc_state->stream;
struct amdgpu_dm_connector *aconnector = NULL;
bool enable = amdgpu_dm_is_valid_crc_source(source);
int ret = 0;
enum crc_poly_mode crc_poly_mode = CRC_POLY_MODE_16;
@ -512,21 +511,17 @@ int amdgpu_dm_crtc_configure_crc_source(struct drm_crtc *crtc,
if (!stream_state)
return -EINVAL;
/* Get connector from stream */
aconnector = (struct amdgpu_dm_connector *)stream_state->dm_stream_context;
mutex_lock(&adev->dm.dc_lock);
/* Notify power module about CRC window active to disable PSR/Replay
* Power module will check caps internally and skip if not supported
*/
if (enable) {
/* For PSR1, check that the panel has exited PSR */
if (stream_state->link->psr_settings.psr_version < DC_PSR_VERSION_SU_1)
amdgpu_dm_psr_wait_disable(stream_state);
amdgpu_dm_psr_set_event(&adev->dm, stream_state, true,
psr_event_crc_window_active, true);
/* Set flag to disallow enter replay when CRC source is enabled */
if (aconnector)
aconnector->disallow_edp_enter_replay = true;
amdgpu_dm_replay_disable(stream_state);
amdgpu_dm_replay_set_event(&adev->dm, stream_state, true,
replay_event_crc_window_active, true);
}
/* CRC polynomial selection only support for DCN3.6+ except DCN4.0.1 */
@ -559,11 +554,15 @@ int amdgpu_dm_crtc_configure_crc_source(struct drm_crtc *crtc,
}
if (!enable) {
/* Clear flag to allow enter replay when CRC source is disabled */
if (aconnector)
aconnector->disallow_edp_enter_replay = false;
}
/* Notify power module about CRC window inactive to re-enable PSR/Replay
* Power module will check caps internally and skip if not supported
*/
amdgpu_dm_psr_set_event(&adev->dm, stream_state, false,
psr_event_crc_window_active, false);
amdgpu_dm_replay_set_event(&adev->dm, stream_state, false,
replay_event_crc_window_active, false);
}
unlock:
mutex_unlock(&adev->dm.dc_lock);
@ -760,10 +759,13 @@ void amdgpu_dm_crtc_handle_crc_irq(struct drm_crtc *crtc)
uint32_t crcs[3];
unsigned long flags;
if (crtc == NULL)
if (!crtc || !crtc->state || !crtc->dev)
return;
crtc_state = to_dm_crtc_state(crtc->state);
if (!crtc_state->stream)
return;
stream_state = crtc_state->stream;
acrtc = to_amdgpu_crtc(crtc);
drm_dev = crtc->dev;

70
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
View File

@ -34,6 +34,7 @@
#include "amdgpu_dm_plane.h"
#include "amdgpu_dm_trace.h"
#include "amdgpu_dm_debugfs.h"
#include "modules/inc/mod_power.h"
#define HPD_DETECTION_PERIOD_uS 2000000
#define HPD_DETECTION_TIME_uS 100000
@ -100,68 +101,33 @@ bool amdgpu_dm_crtc_vrr_active(const struct dm_crtc_state *dm_state)
}
/**
* amdgpu_dm_crtc_set_panel_sr_feature() - Manage panel self-refresh features.
* @dm: amdgpu display manager instance.
* @acrtc: CRTC whose panel self-refresh state is being updated.
* @stream: DC stream associated with @acrtc.
* @vblank_enabled: Whether the DRM vblank counter is currently enabled.
* @allow_sr_entry: Whether entry into self-refresh mode is allowed.
* amdgpu_dm_crtc_set_static_screen_optimze() - Toggle static screen optimizations.
*
* The DRM vblank counter enable/disable action is used as the trigger to enable
* or disable various panel self-refresh features:
* @dm: display manager
* @stream: DC stream state
* @sso_enable: desired static screen optimization state
* @allow_sr_entry: whether entry into self-refresh mode is allowed
*
* Panel Replay and PSR SU
* - Enable when:
* - VRR is disabled
* - vblank counter is disabled
* - entry is allowed: usermode demonstrates an adequate number of fast
* commits
* - CRC capture window isn't active
* - Keep enabled even when vblank counter gets enabled
*
* PSR1
* - Enable condition same as above
* - Disable when vblank counter is enabled
* This function uses the static-screen optimization state as the trigger to
* set/clear the Replay and PSR vsync-related events.
*/
void amdgpu_dm_crtc_set_panel_sr_feature(
void amdgpu_dm_crtc_set_static_screen_optimze(
struct amdgpu_display_manager *dm,
struct amdgpu_crtc *acrtc,
struct dc_stream_state *stream,
bool vblank_enabled, bool allow_sr_entry)
bool sso_enable, bool allow_sr_entry)
{
struct dc_link *link = stream->link;
bool is_sr_active = (link->replay_settings.replay_allow_active ||
link->psr_settings.psr_allow_active);
bool is_crc_window_active = false;
bool vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
bool set_vsync_event = !sso_enable;
#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY
is_crc_window_active =
amdgpu_dm_crc_window_is_activated(&acrtc->base);
#endif
if (!allow_sr_entry)
return;
if (link->replay_settings.replay_feature_enabled && !vrr_active &&
allow_sr_entry && !is_sr_active && !is_crc_window_active) {
amdgpu_dm_replay_enable(stream, true);
} else if (vblank_enabled) {
if (link->psr_settings.psr_version < DC_PSR_VERSION_SU_1 && is_sr_active)
amdgpu_dm_psr_disable(stream, false);
} else if (link->psr_settings.psr_feature_enabled && !vrr_active &&
allow_sr_entry && !is_sr_active && !is_crc_window_active) {
amdgpu_dm_replay_set_event(dm, stream,
set_vsync_event, replay_event_vsync, set_vsync_event);
struct amdgpu_dm_connector *aconn =
(struct amdgpu_dm_connector *) stream->dm_stream_context;
if (!aconn->disallow_edp_enter_psr) {
amdgpu_dm_psr_enable(stream);
if (dm->idle_workqueue &&
(dm->dc->config.disable_ips == DMUB_IPS_ENABLE) &&
dm->dc->idle_optimizations_allowed &&
dm->idle_workqueue->enable &&
!dm->idle_workqueue->running)
schedule_work(&dm->idle_workqueue->work);
}
}
if (link->psr_settings.psr_version < DC_PSR_VERSION_SU_1)
amdgpu_dm_psr_set_event(dm, stream,
set_vsync_event, psr_event_vsync, set_vsync_event);
}
bool amdgpu_dm_is_headless(struct amdgpu_device *adev)

5
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
View File

@ -27,11 +27,10 @@
#ifndef __AMDGPU_DM_CRTC_H__
#define __AMDGPU_DM_CRTC_H__
void amdgpu_dm_crtc_set_panel_sr_feature(
void amdgpu_dm_crtc_set_static_screen_optimze(
struct amdgpu_display_manager *dm,
struct amdgpu_crtc *acrtc,
struct dc_stream_state *stream,
bool vblank_enabled, bool allow_sr_entry);
bool sso_enable, bool allow_sr_entry);
void amdgpu_dm_crtc_handle_vblank(struct amdgpu_crtc *acrtc);

60
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
View File

@ -33,6 +33,7 @@
#include "amdgpu_dm.h"
#include "amdgpu_dm_debugfs.h"
#include "amdgpu_dm_replay.h"
#include "amdgpu_dm_psr.h"
#include "dm_helpers.h"
#include "dmub/dmub_srv.h"
#include "resource.h"
@ -3300,11 +3301,26 @@ static int disallow_edp_enter_psr_get(void *data, u64 *val)
static int disallow_edp_enter_psr_set(void *data, u64 val)
{
struct amdgpu_dm_connector *aconnector = data;
struct dc_link *link = aconnector->dc_link;
aconnector->disallow_edp_enter_psr = val ? true : false;
aconnector->disallow_edp_enter_psr = (val != 0);
/* eDP PSR enable / disable is happened during mode change in power module.
* Only psr_settings.psr_version is used to decide whether PSR is enabled or not.
* So here we only update psr_version based on debugfs setting.
* If disallow_edp_enter_psr is true, set psr_version to unsupported;
* if disallow_edp_enter_psr is false, set psr_version based on sink capability.
*/
if (aconnector->disallow_edp_enter_psr)
link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
else if (aconnector->psr_caps.psr_version == 1)
link->psr_settings.psr_version = DC_PSR_VERSION_1;
else if (aconnector->psr_caps.psr_version == 2)
link->psr_settings.psr_version = DC_PSR_VERSION_SU_1;
return 0;
}
/* check if kernel disallow eDP enter replay state
* cat /sys/kernel/debug/dri/0/eDP-X/disallow_edp_enter_replay
* 0: allow edp enter replay; 1: disallow
@ -3346,11 +3362,27 @@ static int disallow_edp_enter_replay_get(void *data, u64 *val)
static int disallow_edp_enter_replay_set(void *data, u64 val)
{
struct amdgpu_dm_connector *aconnector = data;
struct dc_link *link = aconnector->dc_link;
aconnector->disallow_edp_enter_replay = val ? true : false;
aconnector->disallow_edp_enter_replay = (val != 0);
/* eDP replay enable / disable is happened during mode change in power module.
* Only replay_settings.config.replay_supported is used to decide whether
* replay is enabled or not. So here we only update replay_supported based on
* debugfs setting.
* If disallow_edp_enter_replay is true, set replay_supported to false.
* if disallow_edp_enter_replay is false, set replay_supported back based on
* sink replay capability.
*/
if (aconnector->disallow_edp_enter_replay)
link->replay_settings.config.replay_supported = false;
else
link->replay_settings.config.replay_supported =
link->replay_settings.config.replay_cap_support;
return 0;
}
static int dmub_trace_mask_set(void *data, u64 val)
{
struct amdgpu_device *adev = data;
@ -3485,6 +3517,7 @@ DEFINE_DEBUGFS_ATTRIBUTE(disallow_edp_enter_replay_fops,
DEFINE_DEBUGFS_ATTRIBUTE(ips_residency_cntl_fops, ips_residency_cntl_get,
ips_residency_cntl_set, "%llu\n");
DEFINE_SHOW_ATTRIBUTE(current_backlight);
DEFINE_SHOW_ATTRIBUTE(target_backlight);
DEFINE_SHOW_ATTRIBUTE(ips_status);
@ -3855,28 +3888,35 @@ DEFINE_DEBUGFS_ATTRIBUTE(crc_win_y_end_fops, crc_win_y_end_get,
static int crc_win_update_set(void *data, u64 val)
{
struct drm_crtc *crtc = data;
struct amdgpu_crtc *acrtc;
struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
struct amdgpu_device *adev = drm_to_adev(crtc->dev);
if (val) {
acrtc = to_amdgpu_crtc(crtc);
mutex_lock(&adev->dm.dc_lock);
/* PSR may write to OTG CRC window control register,
* so close it before starting secure_display.
/* PSR Replay may write to OTG CRC window control register,
* so inactive it before starting secure_display by sending disable event.
*/
amdgpu_dm_psr_disable(acrtc->dm_irq_params.stream, true);
amdgpu_dm_psr_set_event(&adev->dm, acrtc->dm_irq_params.stream, true,
psr_event_crc_window_active, true);
amdgpu_dm_replay_set_event(&adev->dm, acrtc->dm_irq_params.stream, true,
replay_event_crc_window_active, true);
spin_lock_irq(&adev_to_drm(adev)->event_lock);
acrtc->dm_irq_params.window_param[0].enable = true;
acrtc->dm_irq_params.window_param[0].update_win = true;
acrtc->dm_irq_params.window_param[0].skip_frame_cnt = 0;
acrtc->dm_irq_params.crc_window_activated = true;
spin_unlock_irq(&adev_to_drm(adev)->event_lock);
mutex_unlock(&adev->dm.dc_lock);
} else {
/* Clear disable events to allow PSR/Replay to active */
mutex_lock(&adev->dm.dc_lock);
amdgpu_dm_psr_set_event(&adev->dm, acrtc->dm_irq_params.stream, false,
psr_event_crc_window_active, false);
amdgpu_dm_replay_set_event(&adev->dm, acrtc->dm_irq_params.stream, false,
replay_event_crc_window_active, false);
mutex_unlock(&adev->dm.dc_lock);
}
return 0;
}

26
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
View File

@ -291,24 +291,16 @@ static void dm_ism_commit_idle_optimization_state(struct amdgpu_dm_ism *ism,
*/
if (stream && stream->link) {
/*
* If allow_panel_sso is true when disabling vblank, allow
* deeper panel sleep states such as PSR1 and Replay static
* screen optimization.
*/
if (!vblank_enabled && allow_panel_sso) {
amdgpu_dm_crtc_set_panel_sr_feature(
dm, acrtc, stream, false,
acrtc->dm_irq_params.allow_sr_entry);
} else if (vblank_enabled) {
/* Make sure to exit SSO on vblank enable */
amdgpu_dm_crtc_set_panel_sr_feature(
dm, acrtc, stream, true,
acrtc->dm_irq_params.allow_sr_entry);
}
/*
* Else, vblank_enabled == false and allow_panel_sso == false;
* do nothing here.
* If the OS requires vblank events (or vblank is otherwise enabled),
* do not allow static screen optimizations.
*
* Keep ism->allow_static_screen_optimizations unchanged so the
* hysteresis-based decision can be reused once vblank is disabled.
*/
allow_panel_sso = allow_panel_sso && !vblank_enabled;
amdgpu_dm_crtc_set_static_screen_optimze(
dm, stream, allow_panel_sso,
acrtc->dm_irq_params.allow_sr_entry);
}
/*

244
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
View File

@ -58,171 +58,76 @@ static bool link_supports_psrsu(struct dc_link *link)
return false;
}
static void amdgpu_dm_psr_fill_caps(struct dc_link *link, struct psr_caps *caps)
{
struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
unsigned int power_opts = 0;
if (amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT)
power_opts |= psr_power_opt_smu_opt_static_screen;
power_opts |= psr_power_opt_z10_static_screen;
if (link->psr_settings.psr_version == DC_PSR_VERSION_1)
caps->psr_version = 1;
else if (link->psr_settings.psr_version == DC_PSR_VERSION_SU_1)
caps->psr_version = 2;
caps->psr_rfb_setup_time = (6 - dpcd_caps->psr_info.psr_dpcd_caps.bits.PSR_SETUP_TIME) * 55;
caps->psr_exit_link_training_required =
!dpcd_caps->psr_info.psr_dpcd_caps.bits.LINK_TRAINING_ON_EXIT_NOT_REQUIRED;
caps->edp_revision = dpcd_caps->edp_rev;
caps->support_ver = dpcd_caps->psr_info.psr_version;
caps->su_granularity_required =
dpcd_caps->psr_info.psr_dpcd_caps.bits.SU_GRANULARITY_REQUIRED;
caps->y_coordinate_required = dpcd_caps->psr_info.psr_dpcd_caps.bits.Y_COORDINATE_REQUIRED;
caps->su_y_granularity = dpcd_caps->psr_info.psr2_su_y_granularity_cap;
caps->alpm_cap = dpcd_caps->alpm_caps.bits.AUX_WAKE_ALPM_CAP;
caps->standby_support = dpcd_caps->alpm_caps.bits.PM_STATE_2A_SUPPORT;
caps->rate_control_caps = 0; /* TODO: read in rc caps from aux */
caps->psr_power_opt_flag = power_opts;
}
/*
* amdgpu_dm_set_psr_caps() - set link psr capabilities
* @link: link
*
* @aconnector: amdgpu_dm_connector
*/
void amdgpu_dm_set_psr_caps(struct dc_link *link)
bool amdgpu_dm_set_psr_caps(struct dc_link *link, struct amdgpu_dm_connector *aconnector)
{
if (!(link->connector_signal & SIGNAL_TYPE_EDP)) {
link->psr_settings.psr_feature_enabled = false;
return;
}
struct dc *dc;
unsigned int panel_inst = 0;
if (link->type == dc_connection_none) {
link->psr_settings.psr_feature_enabled = false;
return;
}
if (link->dpcd_caps.psr_info.psr_version == 0) {
link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
link->psr_settings.psr_feature_enabled = false;
} else {
unsigned int panel_inst = 0;
if (link_supports_psrsu(link))
link->psr_settings.psr_version = DC_PSR_VERSION_SU_1;
else
link->psr_settings.psr_version = DC_PSR_VERSION_1;
link->psr_settings.psr_feature_enabled = true;
/*disable allow psr/psrsu/replay on eDP1*/
if (dc_get_edp_link_panel_inst(link->ctx->dc, link, &panel_inst) && panel_inst == 1) {
link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
link->psr_settings.psr_feature_enabled = false;
}
}
}
/*
* amdgpu_dm_link_setup_psr() - configure psr link
* @stream: stream state
*
* Return: true if success
*/
bool amdgpu_dm_link_setup_psr(struct dc_stream_state *stream)
{
struct dc_link *link = NULL;
struct psr_config psr_config = {0};
struct psr_context psr_context = {0};
struct dc *dc = NULL;
bool ret = false;
if (stream == NULL)
if (!link || !aconnector)
return false;
link = stream->link;
dc = link->ctx->dc;
if (link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED) {
mod_power_calc_psr_configs(&psr_config, link, stream);
/* Reset psr version first */
link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
/* linux DM specific updating for psr config fields */
psr_config.allow_smu_optimizations =
(amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT) &&
mod_power_only_edp(dc->current_state, stream);
psr_config.allow_multi_disp_optimizations =
(amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT);
if (!dc->caps.dmub_caps.psr)
return false;
if (link->psr_settings.psr_version == DC_PSR_VERSION_SU_1) {
if (!psr_su_set_dsc_slice_height(dc, link, stream, &psr_config))
return false;
}
if (!(link->connector_signal & SIGNAL_TYPE_EDP))
return false;
ret = dc_link_setup_psr(link, stream, &psr_config, &psr_context);
if (link->type == dc_connection_none)
return false;
}
DRM_DEBUG_DRIVER("PSR link: %d\n", link->psr_settings.psr_feature_enabled);
if (link->dpcd_caps.psr_info.psr_version == 0)
return false;
return ret;
}
/*disable allow psr/psrsu/replay on eDP1*/
if (dc_get_edp_link_panel_inst(link->ctx->dc, link, &panel_inst) && panel_inst == 1)
return false;
/*
* amdgpu_dm_psr_enable() - enable psr f/w
* @stream: stream state
*
*/
void amdgpu_dm_psr_enable(struct dc_stream_state *stream)
{
struct dc_link *link = stream->link;
unsigned int vsync_rate_hz = 0;
struct dc_static_screen_params params = {0};
/* Calculate number of static frames before generating interrupt to
* enter PSR.
*/
// Init fail safe of 2 frames static
unsigned int num_frames_static = 2;
unsigned int power_opt = 0;
bool psr_enable = true;
if (link_supports_psrsu(link))
link->psr_settings.psr_version = DC_PSR_VERSION_SU_1;
else
link->psr_settings.psr_version = DC_PSR_VERSION_1;
DRM_DEBUG_DRIVER("Enabling psr...\n");
vsync_rate_hz = div64_u64(div64_u64((
stream->timing.pix_clk_100hz * (uint64_t)100),
stream->timing.v_total),
stream->timing.h_total);
/* Round up
* Calculate number of frames such that at least 30 ms of time has
* passed.
*/
if (vsync_rate_hz != 0) {
unsigned int frame_time_microsec = 1000000 / vsync_rate_hz;
num_frames_static = (30000 / frame_time_microsec) + 1;
}
params.triggers.cursor_update = true;
params.triggers.overlay_update = true;
params.triggers.surface_update = true;
params.num_frames = num_frames_static;
dc_stream_set_static_screen_params(link->ctx->dc,
&stream, 1,
&params);
/*
* Only enable static-screen optimizations for PSR1. For PSR SU, this
* causes vstartup interrupt issues, used by amdgpu_dm to send vblank
* events.
*/
if (link->psr_settings.psr_version < DC_PSR_VERSION_SU_1)
power_opt |= psr_power_opt_z10_static_screen;
dc_link_set_psr_allow_active(link, &psr_enable, false, false, &power_opt);
if (link->ctx->dc->caps.ips_support)
dc_allow_idle_optimizations(link->ctx->dc, true);
}
/*
* amdgpu_dm_psr_disable() - disable psr f/w
* @stream: stream state
*
* Return: true if success
*/
bool amdgpu_dm_psr_disable(struct dc_stream_state *stream, bool wait)
{
bool psr_enable = false;
DRM_DEBUG_DRIVER("Disabling psr...\n");
return dc_link_set_psr_allow_active(stream->link, &psr_enable, wait, false, NULL);
}
/*
* amdgpu_dm_psr_disable_all() - disable psr f/w for all streams
* if psr is enabled on any stream
*
* Return: true if success
*/
bool amdgpu_dm_psr_disable_all(struct amdgpu_display_manager *dm)
{
DRM_DEBUG_DRIVER("Disabling psr if psr is enabled on any stream\n");
return dc_set_psr_allow_active(dm->dc, false);
amdgpu_dm_psr_fill_caps(link, &aconnector->psr_caps);
return true;
}
/*
@ -250,36 +155,37 @@ bool amdgpu_dm_psr_is_active_allowed(struct amdgpu_display_manager *dm)
break;
}
}
return allow_active;
}
/**
* amdgpu_dm_psr_wait_disable() - Wait for eDP panel to exit PSR
* @stream: stream state attached to the eDP link
/*
* amdgpu_dm_psr_set_event() - set or clear PSR event for stream
* @dm: pointer to amdgpu_display_manager
* @stream: pointer to dc_stream_state
* @set_event: true to set event, false to clear event
* @event: PSR event type
* @wait_for_disable: whether to wait for PSR to be disabled
*
* Waits for a max of 500ms for the eDP panel to exit PSR.
*
* Return: true if panel exited PSR, false otherwise.
* Return: true if successful, false otherwise
*/
bool amdgpu_dm_psr_wait_disable(struct dc_stream_state *stream)
bool amdgpu_dm_psr_set_event(struct amdgpu_display_manager *dm, struct dc_stream_state *stream,
bool set_event, enum psr_event event, bool wait_for_disable)
{
enum dc_psr_state psr_state = PSR_STATE0;
struct dc_link *link = stream->link;
int retry_count;
unsigned int psr_events;
if (link == NULL)
/* Validate all required parameters */
if (!stream || !stream->link ||
!stream->link->psr_settings.psr_feature_enabled)
return false;
for (retry_count = 0; retry_count <= 1000; retry_count++) {
dc_link_get_psr_state(link, &psr_state);
if (psr_state == PSR_STATE0)
break;
udelay(500);
}
if (retry_count == 1000)
/* Get current psr events */
if (!mod_power_get_psr_event(dm->power_module, stream, &psr_events))
return false;
return true;
/* If all events already in desired state, return true. */
if ((psr_events & event) == (set_event ? event : 0))
return true;
return mod_power_set_psr_event(dm->power_module, stream,
set_event, event, wait_for_disable);
}

13
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h
View File

@ -28,16 +28,15 @@
#define AMDGPU_DM_AMDGPU_DM_PSR_H_
#include "amdgpu.h"
#include "dc.h"
#include "modules/inc/mod_power.h"
/* the number of pageflips before enabling psr */
#define AMDGPU_DM_PSR_ENTRY_DELAY 5
void amdgpu_dm_set_psr_caps(struct dc_link *link);
void amdgpu_dm_psr_enable(struct dc_stream_state *stream);
bool amdgpu_dm_link_setup_psr(struct dc_stream_state *stream);
bool amdgpu_dm_psr_disable(struct dc_stream_state *stream, bool wait);
bool amdgpu_dm_psr_disable_all(struct amdgpu_display_manager *dm);
bool amdgpu_dm_set_psr_caps(struct dc_link *link, struct amdgpu_dm_connector *aconnector);
bool amdgpu_dm_psr_is_active_allowed(struct amdgpu_display_manager *dm);
bool amdgpu_dm_psr_wait_disable(struct dc_stream_state *stream);
bool amdgpu_dm_psr_set_event(struct amdgpu_display_manager *dm,
struct dc_stream_state *stream, bool set_event, enum psr_event event,
bool wait_for_disable);
#endif /* AMDGPU_DM_AMDGPU_DM_PSR_H_ */

143
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c
View File

@ -27,7 +27,6 @@
#include "amdgpu_dm_replay.h"
#include "dc_dmub_srv.h"
#include "dc.h"
#include "dm_helpers.h"
#include "amdgpu_dm.h"
#include "modules/power/power_helpers.h"
#include "dmub/inc/dmub_cmd.h"
@ -99,13 +98,29 @@ bool amdgpu_dm_set_replay_caps(struct dc_link *link, struct amdgpu_dm_connector
!dc->ctx->dmub_srv->dmub->feature_caps.replay_supported)
return false;
/* Mark Replay is supported in link and update related attributes
* This flag presents DPCD caps & amd_vsdb caps satisfy replay requirement.
*/
pr_config.replay_cap_support = true;
// Mark Replay is supported in pr_config
pr_config.replay_supported = true;
pr_config.replay_enable_option = pr_enable_option_general_ui |
pr_enable_option_static_screen |
pr_enable_option_static_screen_coasting;
pr_config.replay_power_opt_supported = replay_power_opt_smu_opt_static_screen |
replay_power_opt_z10_static_screen;
pr_config.replay_smu_opt_supported = false;
pr_config.replay_support_fast_resync_in_ultra_sleep_mode =
aconnector->max_vfreq >= 2 * aconnector->min_vfreq;
pr_config.force_disable_desync_error_check = false;
debug_flags = (union replay_debug_flags *)&pr_config.debug_flags;
debug_flags->u32All = 0;
debug_flags->bitfields.visual_confirm =
link->ctx->dc->debug.visual_confirm == VISUAL_CONFIRM_REPLAY;
debug_flags->bitfields.skip_crtc_disabled = dc->debug.replay_skip_crtc_disabled;
init_replay_config(link, &pr_config);
@ -113,104 +128,80 @@ bool amdgpu_dm_set_replay_caps(struct dc_link *link, struct amdgpu_dm_connector
}
/*
* amdgpu_dm_link_setup_replay() - configure replay link
* @link: link
* @aconnector: aconnector
* amdgpu_dm_link_setup_replay() - config replay settings
* @stream: pointer to dc_stream_state structure
* @vrr_params: pointer to mod_vrr_params structure containing VRR parameters
*
* config replay link settings including coasting vtotal calculations.
*
* Return: true if successful, false if any parameter is invalid or replay not supported
*/
bool amdgpu_dm_link_setup_replay(struct dc_link *link, struct amdgpu_dm_connector *aconnector)
bool amdgpu_dm_link_setup_replay(struct dc_stream_state *stream,
struct mod_vrr_params *vrr_params)
{
struct replay_config *pr_config;
struct dc_link *link;
unsigned int static_coasting_vtotal;
unsigned int nom_coasting_vtotal;
if (link == NULL || aconnector == NULL)
if (!stream || !stream->link || !vrr_params)
return false;
pr_config = &link->replay_settings.config;
if (!pr_config->replay_supported)
link = stream->link;
if (!link->replay_settings.config.replay_supported)
return false;
pr_config->replay_power_opt_supported = 0x11;
pr_config->replay_smu_opt_supported = false;
pr_config->replay_enable_option |= pr_enable_option_static_screen;
pr_config->replay_support_fast_resync_in_ultra_sleep_mode = aconnector->max_vfreq >= 2 * aconnector->min_vfreq;
pr_config->replay_timing_sync_supported = false;
if (link->replay_settings.replay_feature_enabled)
return true;
if (!pr_config->replay_timing_sync_supported)
pr_config->replay_enable_option &= ~pr_enable_option_general_ui;
calculate_replay_link_off_frame_count(link, stream->timing.v_total,
stream->timing.h_total);
link->replay_settings.replay_feature_enabled = true;
nom_coasting_vtotal = stream->timing.v_total;
static_coasting_vtotal = mod_freesync_calc_v_total_from_refresh(stream,
vrr_params->min_refresh_in_uhz);
set_replay_coasting_vtotal(link, PR_COASTING_TYPE_NOM,
nom_coasting_vtotal);
set_replay_coasting_vtotal(link, PR_COASTING_TYPE_STATIC,
static_coasting_vtotal);
return true;
}
/*
* amdgpu_dm_replay_enable() - enable replay f/w
* @stream: stream state
* amdgpu_dm_replay_set_event() - set or clear replay event for a stream
* @dm: pointer to amdgpu_display_manager
* @stream: pointer to dc_stream_state
* @set_event: true to set event, false to clear event
* @event: replay event type to set or clear
* @wait_for_disable: whether to wait for replay to be disabled before returning
*
* Return: true if success
* This function sets or clears a specific replay event for the given stream.
* It temporarily disables idle optimizations during the operation to ensure
* hardware access is available.
*
* Return: true if successful, false if any parameter is invalid or operation fails
*/
bool amdgpu_dm_replay_enable(struct dc_stream_state *stream, bool wait)
bool amdgpu_dm_replay_set_event(struct amdgpu_display_manager *dm,
struct dc_stream_state *stream,
bool set_event,
enum replay_event event,
bool wait_for_disable)
{
bool replay_active = true;
struct dc_link *link = NULL;
struct amdgpu_dm_connector *aconnector = NULL;
unsigned int replay_events;
if (stream == NULL)
/* Validate all required parameters */
if (!stream || !stream->link ||
!stream->link->replay_settings.replay_feature_enabled)
return false;
/* Check if replay is disabled by connector flag */
aconnector = (struct amdgpu_dm_connector *)stream->dm_stream_context;
if (!aconnector || aconnector->disallow_edp_enter_replay) {
/* Get current replay events */
if (!mod_power_get_replay_event(dm->power_module, stream, &replay_events))
return false;
}
link = stream->link;
if (link) {
link->dc->link_srv->dp_setup_replay(link, stream);
link->dc->link_srv->edp_set_coasting_vtotal(link, stream->timing.v_total, 0);
DRM_DEBUG_DRIVER("Enabling replay...\n");
link->dc->link_srv->edp_set_replay_allow_active(link, &replay_active, wait, false, NULL);
/* If all events already in desired state, return true. */
if ((replay_events & event) == (set_event ? event : 0))
return true;
}
return false;
}
/*
* amdgpu_dm_replay_disable() - disable replay f/w
* @stream: stream state
*
* Return: true if success
*/
bool amdgpu_dm_replay_disable(struct dc_stream_state *stream)
{
bool replay_active = false;
struct dc_link *link = NULL;
if (stream == NULL)
return false;
link = stream->link;
if (link) {
DRM_DEBUG_DRIVER("Disabling replay...\n");
link->dc->link_srv->edp_set_replay_allow_active(stream->link, &replay_active, true, false, NULL);
return true;
}
return false;
}
/*
* amdgpu_dm_replay_disable_all() - disable replay f/w
* if replay is enabled on any stream
*
* Return: true if success
*/
bool amdgpu_dm_replay_disable_all(struct amdgpu_display_manager *dm)
{
DRM_DEBUG_DRIVER("Disabling replay if replay is enabled on any stream\n");
return dc_set_replay_allow_active(dm->dc, false);
return mod_power_set_replay_event(dm->power_module, stream,
set_event, event, wait_for_disable);
}

Some files were not shown because too many files have changed in this diff Show More